MediaFileManagers now use JSON instead of the TLMessage objects.

src/main/kotlin/de/fabianonline/telegram_backup/Database.kt

@@ -43,6 +43,8 @@ import java.nio.file.Files
 import java.nio.file.StandardCopyOption
 import java.nio.file.FileAlreadyExistsException
 import java.text.SimpleDateFormat
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
 
 import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager
 import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory
@@ -110,14 +112,16 @@ class Database constructor(val file_base: String, val user_manager: UserManager)
 
 	fun getMessagesWithMediaCount() = queryInt("SELECT COUNT(*) FROM messages WHERE has_media=1")
 
-	fun getMessagesWithMedia(limit: Int = 0, offset: Int = 0): LinkedList<TLMessage?> {
+	fun getMessagesWithMedia(limit: Int = 0, offset: Int = 0): LinkedList<Pair<Int, JsonObject>> {
 		try {
-			val list = LinkedList<TLMessage?>()
+			val list = LinkedList<Pair<Int, JsonObject>>()
-			var query = "SELECT data FROM messages WHERE has_media=1 ORDER BY id"
+			var query = "SELECT id, json FROM messages WHERE has_media=1 AND json IS NOT NULL ORDER BY id"
 			if (limit > 0) query += " LIMIT ${limit} OFFSET ${offset}"
 			val rs = executeQuery(query)
+			val parser = JsonParser()
 			while (rs.next()) {
-				list.add(bytesToTLMessage(rs.getBytes(1)))
+				val obj = parser.parse(rs.getString(2)).obj
+				list.add(Pair<Int, JsonObject>(rs.getInt(1), obj))
 			}
 			rs.close()
 			return list
@@ -325,7 +329,7 @@ class Database constructor(val file_base: String, val user_manager: UserManager)
 				}
 				ps.setString(7, text)
 				ps.setString(8, "" + msg.getDate())
-				val f = FileManagerFactory.getFileManager(msg, user_manager, file_base, settings)
+				val f = FileManagerFactory.getFileManager(msg, file_base, settings)
 				if (f == null) {
 					ps.setNull(9, Types.BOOLEAN)
 					ps.setNull(10, Types.VARCHAR)

src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt

@@ -316,7 +316,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn
 			} else {
 				ps.setInt(1, msg.getFwdFrom().getFromId())
 			}
-			val f = FileManagerFactory.getFileManager(msg, db.user_manager, db.file_base, settings = null)
+			val f = FileManagerFactory.getFileManager(msg, db.file_base, settings = null)
 			if (f == null) {
 				ps.setNull(2, Types.VARCHAR)
 				ps.setNull(3, Types.VARCHAR)

src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt

@@ -36,6 +36,7 @@ import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
 import org.slf4j.LoggerFactory
 import org.slf4j.Logger
 import com.google.gson.Gson
+import com.github.salomonbrys.kotson.*
 import com.github.kittinunf.fuel.Fuel
 import com.github.kittinunf.result.Result
 import com.github.kittinunf.result.getAs
@@ -243,20 +244,21 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte
 			offset += limit
 			logger.debug("Database returned {} messages with media", messages.size)
 			prog.onMediaDownloadStart(messages.size)
-			for (msg in messages) {
+			for (pair in messages) {
-				if (msg == null) continue
+				val id = pair.first
-				val m = FileManagerFactory.getFileManager(msg, user_manager, file_base, settings=settings)
+				val json = pair.second
+				try {
+				val m = FileManagerFactory.getFileManager(json, file_base, settings=settings)!!
 				logger.trace("message {}, {}, {}, {}, {}",
-					msg.getId(),
+					id,
-					msg.getMedia().javaClass.getSimpleName().replace("TLMessageMedia", "…"),
+					m.javaClass.getSimpleName(),
-					m!!.javaClass.getSimpleName(),
 					if (m.isEmpty) "empty" else "non-empty",
 					if (m.downloaded) "downloaded" else "not downloaded")
 				if (m.isEmpty) {
 					prog.onMediaDownloadedEmpty()
 				} else if (m.downloaded) {
 					prog.onMediaAlreadyPresent(m)
-				} else if (settings.max_file_age>0 && (System.currentTimeMillis() / 1000) - msg.date > settings.max_file_age * 24 * 60 * 60) {
+				} else if (settings.max_file_age>0 && (System.currentTimeMillis() / 1000) - json["date"].int > settings.max_file_age * 24 * 60 * 60) {
 					prog.onMediaTooOld()
 				} else {
 					try {
@@ -269,8 +271,13 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte
 					} catch (e: TimeoutException) {
 						// do nothing - skip this file
 						prog.onMediaSkipped()
+					
 					}
 				}
+				} catch (e: IllegalStateException) {
+					println(json.toPrettyJson())
+					throw e
+				}
 			}
 		}
 		prog.onMediaDownloadFinished()

src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt

@@ -129,7 +129,7 @@ internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Data
 			db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings)
 			System.out.print('.')
 			if (abs_msg is TLMessage && settings.download_media==true) {
-				val fm = FileManagerFactory.getFileManager(abs_msg, user_manager, file_base, settings)
+				val fm = FileManagerFactory.getFileManager(abs_msg, file_base, settings)
 				if (fm != null && !fm.isEmpty && !fm.downloaded) {
 					try {
 						fm.download()

src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt

@@ -224,6 +224,9 @@ fun String.anonymize(): String {
 fun Any.toJson(): String = Gson().toJson(this)
 fun Any.toPrettyJson(): String = GsonBuilder().setPrettyPrinting().create().toJson(this)
 
+fun JsonObject.isA(name: String): Boolean = this.contains("_constructor") && this["_constructor"].string.startsWith(name + "#")
+fun JsonElement.isA(name: String): Boolean = this.obj.isA(name)
+
 class MaxTriesExceededException(): RuntimeException("Max tries exceeded") {}
 
 fun TLAbsMessage.toJson(): String {

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt

@@ -26,14 +26,17 @@ import de.fabianonline.telegram_backup.Settings
 import java.io.IOException
 import java.io.File
 import java.util.concurrent.TimeoutException
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.*
 
-abstract class AbstractMediaFileManager(protected var message: TLMessage, protected var user: UserManager, val file_base: String) {
+abstract class AbstractMediaFileManager(private var json: JsonObject, val file_base: String) {
 	open var isEmpty = false
 	abstract val size: Int
 	abstract val extension: String
 
 	open val downloaded: Boolean
-		get() = File(targetPathAndFilename).isFile()
+		get() = !isEmpty && File(targetPathAndFilename).isFile()
 
 	val downloading: Boolean
 		get() = File("${targetPathAndFilename}.downloading").isFile()
@@ -47,10 +50,10 @@ abstract class AbstractMediaFileManager(protected var message: TLMessage, protec
 
 	open val targetFilename: String
 		get() {
-			val message_id = message.getId()
+			val message_id = json["id"].int
-			var to = message.getToId()
+			var to = json["toId"].obj
-			if (to is TLPeerChannel) {
+			if (to.isA("peerChannel")) {
-				val channel_id = to.getChannelId()
+				val channel_id = to["channelId"].int
 				return "channel_${channel_id}_${message_id}.$extension"
 			} else return "${message_id}.$extension"
 		}
@@ -77,8 +80,8 @@ abstract class AbstractMediaFileManager(protected var message: TLMessage, protec
 	}
 
 	companion object {
-		fun throwUnexpectedObjectError(o: Any) {
+		fun throwUnexpectedObjectError(constructor: String) {
-			throw RuntimeException("Unexpected " + o.javaClass.getName())
+			throw RuntimeException("Unexpected ${constructor}")
 		}
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt

@@ -24,44 +24,38 @@ import com.github.badoualy.telegram.tl.exception.RpcErrorException
 
 import java.io.IOException
 import java.util.concurrent.TimeoutException
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.*
 
-open class DocumentFileManager(msg: TLMessage, user: UserManager, file_base: String) : AbstractMediaFileManager(msg, user, file_base) {
+open class DocumentFileManager(message: JsonObject, file_base: String) : AbstractMediaFileManager(message, file_base) {
-	protected var doc: TLDocument? = null
+	//protected var doc: TLDocument? = null
 	override lateinit var extension: String
 
 	open val isSticker: Boolean
-		get() {
+		get() = json.get("attributes")?.array?.any{it.obj.isA("documentAttributeSticker")} ?: false
-			if (this.isEmpty || doc == null) return false
-			return doc!!.getAttributes()?.filter { it is TLDocumentAttributeSticker }?.isNotEmpty() ?: false
-		}
 
 	override val size: Int
-		get() = if (doc != null) doc!!.getSize() else 0
+		get() = json["size"].int
 
 	open override val letter: String = "d"
 	open override val name: String = "document"
 	open override val description: String = "Document"
 	
+	private val json = message["media"]["document"].obj
+
 	init {
-		val d = (msg.getMedia() as TLMessageMediaDocument).getDocument()
-		if (d is TLDocument) {
-			this.doc = d
-		} else if (d is TLDocumentEmpty) {
-			this.isEmpty = true
-		} else {
-			throwUnexpectedObjectError(d)
-		}
 		extension = processExtension()
 	}
 
 	private fun processExtension(): String {
-		if (doc == null) return "empty"
+		//if (doc == null) return "empty"
 		var ext: String? = null
 		var original_filename: String? = null
-		if (doc!!.getAttributes() != null)
+		if (json.contains("attributes"))
-			for (attr in doc!!.getAttributes()) {
+			for (attr in json["attributes"].array) {
-				if (attr is TLDocumentAttributeFilename) {
+				if (attr.obj["_constructor"].string.startsWith("documentAttributeFilename")) {
-					original_filename = attr.getFileName()
+					original_filename = attr.obj["fileName"].string
 				}
 			}
 		if (original_filename != null) {
@@ -70,7 +64,7 @@ open class DocumentFileManager(msg: TLMessage, user: UserManager, file_base: Str
 
 		}
 		if (ext == null) {
-			ext = extensionFromMimetype(doc!!.getMimeType())
+			ext = extensionFromMimetype(json["mimeType"].string)
 		}
 
 		// Sometimes, extensions contain a trailing double quote. Remove this. Fixes #12.
@@ -81,9 +75,7 @@ open class DocumentFileManager(msg: TLMessage, user: UserManager, file_base: Str
 
 	@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
 	override fun download(): Boolean {
-		if (doc != null) {
+		DownloadManager.downloadFile(targetPathAndFilename, size, json["dcId"].int, json["id"].long, json["accessHash"].long)
-			DownloadManager.downloadFile(targetPathAndFilename, size, doc!!.getDcId(), doc!!.getId(), doc!!.getAccessHash())
-		}
 		return true
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt

@@ -36,50 +36,55 @@ import java.io.File
 import java.io.FileOutputStream
 import java.util.ArrayList
 import java.util.LinkedList
+import java.util.NoSuchElementException
 import java.net.URL
 import java.util.concurrent.TimeoutException
 import com.google.gson.*
 import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.*
 
 import org.apache.commons.io.FileUtils
 
 object FileManagerFactory {
-	fun getFileManager(m: TLMessage?, u: UserManager, file_base: String, settings: Settings?): AbstractMediaFileManager? {
+	fun getFileManager(m: TLMessage?, file_base: String, settings: Settings?): AbstractMediaFileManager? {
 		if (m == null) return null
 		val json = Gson().toJsonTree(m).obj
-		return getFileManager(json, u, file_base, settings)
+		return getFileManager(json, file_base, settings)
 	}
 	            
-	fun getFileManager(m: JsonObject?, u: UserManager, file_base: String, settings: Settings?): AbstractMediaFileManager? {
+	fun getFileManager(message: JsonObject?, file_base: String, settings: Settings?): AbstractMediaFileManager? {
-		if (m == null) return null
+		if (message == null) return null
-		val media = m.get("media")?.obj ?: return null
+		try {
+			val media = message.get("media")?.obj ?: return null
 			
-		if (media.contains("photo")) {
+			if (media.isA("messageMediaPhoto")) {
-			return PhotoFileManager(media["photo"].obj, u, file_base)
+				return PhotoFileManager(message, file_base)
+			} else if (media.isA("messageMediaDocument")) {
+				val d = DocumentFileManager(message, file_base)
+				return if (d.isSticker) StickerFileManager(message, file_base) else d
+			} else if (media.isA("messageMediaGeo")) {
+				return GeoFileManager(message, file_base, settings)
+			} else if (media.isA("messageMediaEmpty")) {
+				return UnsupportedFileManager(message, file_base, "empty")
+			} else if (media.isA("messageMediaUnsupported")) {
+				return UnsupportedFileManager(message, file_base, "unsupported")
+			} else if (media.isA("messageMediaWebPage")) {
+				return UnsupportedFileManager(message, file_base, "webpage")
+			} else if (media.isA("messageMediaContact")) {
+				return UnsupportedFileManager(message, file_base, "contact")
+			} else if (media.isA("messageMediaVenue")) {
+				return UnsupportedFileManager(message, file_base, "venue")
+			} else {
+				AbstractMediaFileManager.throwUnexpectedObjectError(media["_constructor"].string)
+			}
+		} catch (e: IllegalStateException) {
+			println(message.toPrettyJson())
+			throw e
+		} catch (e: NoSuchElementException) {
+			println(message.toPrettyJson())
+			throw e
 		}
 		
-		/*if (media is TLMessageMediaPhoto) {
-			return PhotoFileManager(m, u, file_base)
-		} else if (media is TLMessageMediaDocument) {
-			val d = DocumentFileManager(m, u, file_base)
-			return if (d.isSticker) {
-				StickerFileManager(m, u, file_base)
-			} else d
-		} else if (media is TLMessageMediaGeo) {
-			return GeoFileManager(m, u, file_base, settings)
-		} else if (media is TLMessageMediaEmpty) {
-			return UnsupportedFileManager(m, u, file_base, "empty")
-		} else if (media is TLMessageMediaUnsupported) {
-			return UnsupportedFileManager(m, u, file_base, "unsupported")
-		} else if (media is TLMessageMediaWebPage) {
-			return UnsupportedFileManager(m, u, file_base, "webpage")
-		} else if (media is TLMessageMediaContact) {
-			return UnsupportedFileManager(m, u, file_base, "contact")
-		} else if (media is TLMessageMediaVenue) {
-			return UnsupportedFileManager(m, u, file_base, "venue")
-		} else {
-			AbstractMediaFileManager.throwUnexpectedObjectError(media)
-		}*/
 		return null
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt

@@ -25,9 +25,12 @@ import de.fabianonline.telegram_backup.Settings
 
 import java.io.IOException
 import java.io.File
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.Utils
 
-class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String, val settings: Settings?) : AbstractMediaFileManager(msg, user, file_base) {
+class GeoFileManager(message: JsonObject, file_base: String, val settings: Settings?) : AbstractMediaFileManager(message, file_base) {
-	protected lateinit var geo: TLGeoPoint
+	//protected lateinit var geo: TLGeoPoint
 
 	// We don't know the size, so we just guess.
 	override val size: Int
@@ -43,7 +46,10 @@ class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String, val s
 	override val name = "geo"
 	override val description = "Geolocation"
 	
+	val json = message["media"]["geo"].obj
+
 	init {
+		/*
 		val g = (msg.getMedia() as TLMessageMediaGeo).getGeo()
 		if (g is TLGeoPoint) {
 			this.geo = g
@@ -51,16 +57,16 @@ class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String, val s
 			this.isEmpty = true
 		} else {
 			throwUnexpectedObjectError(g)
-		}
+		}*/
 	}
 
 	@Throws(IOException::class)
 	override fun download(): Boolean {
 		val url = "https://maps.googleapis.com/maps/api/staticmap?" +
-			"center=${geo.getLat()},${geo.getLong()}&" +
+			"center=${json["lat"].float},${json["_long"].float}&" +
-			"markers=color:red|${geo.getLat()},${geo.getLong()}&" +
+			"markers=color:red|${json["lat"].float},${json["_long"].float}&" +
 			"zoom=14&size=300x150&scale=2&format=png&" +
-			"key=" + (settings?.gmaps_key ?: Config.SECRET_GMAPS)
+			"key=" + (settings?.gmaps_key)
 		return DownloadManager.downloadExternalFile(targetPathAndFilename, url)
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt

@@ -25,7 +25,11 @@ import com.github.badoualy.telegram.tl.exception.RpcErrorException
 import java.io.IOException
 import java.util.concurrent.TimeoutException
 
-class PhotoFileManager(json: JsonObject, user: UserManager, file_base: String) : AbstractMediaFileManager(json, user, file_base) {
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.*
+
+class PhotoFileManager(message: JsonObject, file_base: String) : AbstractMediaFileManager(message, file_base) {
 	//private lateinit var photo: TLPhoto
 	override var size = 0
 	//private lateinit var photo_size: TLPhotoSize
@@ -35,22 +39,33 @@ class PhotoFileManager(json: JsonObject, user: UserManager, file_base: String) :
 	override val name = "photo"
 	override val description = "Photo"
 	
-	var biggestSize: JsonObject?
+	val biggestSize: JsonObject
 	var biggestSizeW = 0
 	var biggestSizeH = 0
 	
+	val json = message["media"]["photo"].obj
+	override var isEmpty = json.isA("photoEmpty")
+
 	init {
 		/*val p = (msg.getMedia() as TLMessageMediaPhoto).getPhoto()*/
+		if (!isEmpty) {
+			var bsTemp: JsonObject? = null
 		
-		for (elm in json["sizes"]) {
+			for (elm in json["sizes"].array) {
-			val s = elm.obj
+				val s = elm.obj
-			if (biggestSize == null || (s["w"].int > biggestSizeW && s["h"].int > biggestSizeH)) {
+				if (!s.isA("photoSize")) continue
-				biggestSize = s
+				if (bsTemp == null || (s["w"].int > biggestSizeW && s["h"].int > biggestSizeH)) {
-				biggestSizeW = s["w"].int
+					bsTemp = s
-				biggestSizeH = s["h"].int
+					biggestSizeW = s["w"].int
-				size = s["size"].int // file size
+					biggestSizeH = s["h"].int
+					size = s["size"].int // file size
+				}
 			}
-			if (biggestSize == null) throw RuntimeException("Could not find a size for a photo.")
+
+			if (bsTemp == null) throw RuntimeException("Could not find a size for a photo.")
+			biggestSize = bsTemp
+		} else {
+			biggestSize = JsonObject()
 		}
 			
 		/*} else if (p is TLPhotoEmpty) {
@@ -64,6 +79,7 @@ class PhotoFileManager(json: JsonObject, user: UserManager, file_base: String) :
 	override fun download(): Boolean {
 		/*if (isEmpty) return true*/
 		//val loc = photo_size.getLocation() as TLFileLocation
+		
 		val loc = biggestSize["location"].obj
 		DownloadManager.downloadFile(targetPathAndFilename, size, loc["dcId"].int, loc["volumeId"].long, loc["localId"].int, loc["secret"].long)
 		return true

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt

@@ -49,29 +49,23 @@ import java.util.concurrent.TimeoutException
 
 import org.apache.commons.io.FileUtils
 
-class StickerFileManager(msg: TLMessage, user: UserManager, file_base: String) : DocumentFileManager(msg, user, file_base) {
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.*
+
+class StickerFileManager(message: JsonObject, file_base: String) : DocumentFileManager(message, file_base) {
 
 	override val isSticker = true
 	
+	val json = message["media"]["document"].obj
+	val sticker = json["attributes"].array.first{it.obj.isA("documentAttributeSticker")}.obj
+	override var isEmpty = sticker["stickerset"].obj.isA("inputStickerSetEmpty")
+
 	private val filenameBase: String
 		get() {
-			var sticker: TLDocumentAttributeSticker? = null
+			val set = sticker["stickerset"].obj.get("shortName").nullString ?: sticker["stickerset"].obj.get("id").string
-			for (attr in doc!!.getAttributes()) {
+			val hash = sticker["alt"].string.hashCode()
-				if (attr is TLDocumentAttributeSticker) {
+			return "${set}_${hash}"
-					sticker = attr
-				}
-			}
-
-			val file = StringBuilder()
-			val set = sticker!!.getStickerset()
-			if (set is TLInputStickerSetShortName) {
-				file.append(set.getShortName())
-			} else if (set is TLInputStickerSetID) {
-				file.append(set.getId())
-			}
-			file.append("_")
-			file.append(sticker.getAlt().hashCode())
-			return file.toString()
 		}
 
 	override val targetFilename: String

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt

@@ -19,8 +19,9 @@ package de.fabianonline.telegram_backup.mediafilemanager
 import de.fabianonline.telegram_backup.UserManager
 
 import com.github.badoualy.telegram.tl.api.*
+import com.google.gson.JsonObject
 
-class UnsupportedFileManager(msg: TLMessage, user: UserManager, type: String, file_base: String) : AbstractMediaFileManager(msg, user, file_base) {
+class UnsupportedFileManager(json: JsonObject, file_base: String, type: String) : AbstractMediaFileManager(json, file_base) {
 	override var name = type
 	override val targetFilename = ""
 	override val targetPath = ""