fabian
/
telegram_backup
mirror of https://github.com/fabianonline/telegram_backup.git
1
0
Fork 0
Code Issues Releases Wiki Activity

MediaFileManagers now use JSON instead of the TLMessage objects.

This commit is contained in:
Fabian Schlenz 2018-04-17 06:36:39 +02:00
parent 6b5b9a669b
commit bb2a291d4f
12 changed files with 143 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/kotlin/de/fabianonline/telegram_backup/Database.kt
View File

@ -43,6 +43,8 @@ import java.nio.file.Files
import java.nio.file.StandardCopyOption
import java.nio.file.FileAlreadyExistsException
import java.text.SimpleDateFormat
import com.google.gson.*
import com.github.salomonbrys.kotson.*
import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager
import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory
@ -110,14 +112,16 @@ class Database constructor(val file_base: String, val user_manager: UserManager)
fun getMessagesWithMediaCount() = queryInt("SELECT COUNT(*) FROM messages WHERE has_media=1")
fun getMessagesWithMedia(limit: Int = 0, offset: Int = 0): LinkedList<TLMessage?> {
fun getMessagesWithMedia(limit: Int = 0, offset: Int = 0): LinkedList<Pair<Int, JsonObject>> {
try {
val list = LinkedList<TLMessage?>()
var query = "SELECT data FROM messages WHERE has_media=1 ORDER BY id"
val list = LinkedList<Pair<Int, JsonObject>>()
var query = "SELECT id, json FROM messages WHERE has_media=1 AND json IS NOT NULL ORDER BY id"
if (limit > 0) query += " LIMIT ${limit} OFFSET ${offset}"
val rs = executeQuery(query)
val parser = JsonParser()
while (rs.next()) {
list.add(bytesToTLMessage(rs.getBytes(1)))
val obj = parser.parse(rs.getString(2)).obj
list.add(Pair<Int, JsonObject>(rs.getInt(1), obj))
}
rs.close()
return list
@ -325,7 +329,7 @@ class Database constructor(val file_base: String, val user_manager: UserManager)
}
ps.setString(7, text)
ps.setString(8, "" + msg.getDate())
val f = FileManagerFactory.getFileManager(msg, user_manager, file_base, settings)
val f = FileManagerFactory.getFileManager(msg, file_base, settings)
if (f == null) {
ps.setNull(9, Types.BOOLEAN)
ps.setNull(10, Types.VARCHAR)

2
src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt
View File

@ -316,7 +316,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn
} else {
ps.setInt(1, msg.getFwdFrom().getFromId())
}
val f = FileManagerFactory.getFileManager(msg, db.user_manager, db.file_base, settings = null)
val f = FileManagerFactory.getFileManager(msg, db.file_base, settings = null)
if (f == null) {
ps.setNull(2, Types.VARCHAR)
ps.setNull(3, Types.VARCHAR)

21
src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt
View File

@ -36,6 +36,7 @@ import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
import org.slf4j.LoggerFactory
import org.slf4j.Logger
import com.google.gson.Gson
import com.github.salomonbrys.kotson.*
import com.github.kittinunf.fuel.Fuel
import com.github.kittinunf.result.Result
import com.github.kittinunf.result.getAs
@ -243,20 +244,21 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte
offset += limit
logger.debug("Database returned {} messages with media", messages.size)
prog.onMediaDownloadStart(messages.size)
for (msg in messages) {
if (msg == null) continue
val m = FileManagerFactory.getFileManager(msg, user_manager, file_base, settings=settings)
for (pair in messages) {
val id = pair.first
val json = pair.second
try {
val m = FileManagerFactory.getFileManager(json, file_base, settings=settings)!!
logger.trace("message {}, {}, {}, {}, {}",
msg.getId(),
msg.getMedia().javaClass.getSimpleName().replace("TLMessageMedia", "…"),
m!!.javaClass.getSimpleName(),
id,
m.javaClass.getSimpleName(),
if (m.isEmpty) "empty" else "non-empty",
if (m.downloaded) "downloaded" else "not downloaded")
if (m.isEmpty) {
prog.onMediaDownloadedEmpty()
} else if (m.downloaded) {
prog.onMediaAlreadyPresent(m)
} else if (settings.max_file_age>0 && (System.currentTimeMillis() / 1000) - msg.date > settings.max_file_age * 24 * 60 * 60) {
} else if (settings.max_file_age>0 && (System.currentTimeMillis() / 1000) - json["date"].int > settings.max_file_age * 24 * 60 * 60) {
prog.onMediaTooOld()
} else {
try {
@ -269,8 +271,13 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte
} catch (e: TimeoutException) {
// do nothing - skip this file
prog.onMediaSkipped()
}
}
} catch (e: IllegalStateException) {
println(json.toPrettyJson())
throw e
}
}
}
prog.onMediaDownloadFinished()

2
src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt
View File

@ -129,7 +129,7 @@ internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Data
db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings)
System.out.print('.')
if (abs_msg is TLMessage && settings.download_media==true) {
val fm = FileManagerFactory.getFileManager(abs_msg, user_manager, file_base, settings)
val fm = FileManagerFactory.getFileManager(abs_msg, file_base, settings)
if (fm != null && !fm.isEmpty && !fm.downloaded) {
try {
fm.download()

3
src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt
View File

@ -224,6 +224,9 @@ fun String.anonymize(): String {
fun Any.toJson(): String = Gson().toJson(this)
fun Any.toPrettyJson(): String = GsonBuilder().setPrettyPrinting().create().toJson(this)
fun JsonObject.isA(name: String): Boolean = this.contains("_constructor") && this["_constructor"].string.startsWith(name + "#")
fun JsonElement.isA(name: String): Boolean = this.obj.isA(name)
class MaxTriesExceededException(): RuntimeException("Max tries exceeded") {}
fun TLAbsMessage.toJson(): String {

19
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt
View File

@ -26,14 +26,17 @@ import de.fabianonline.telegram_backup.Settings
import java.io.IOException
import java.io.File
import java.util.concurrent.TimeoutException
import com.google.gson.*
import com.github.salomonbrys.kotson.*
import de.fabianonline.telegram_backup.*
abstract class AbstractMediaFileManager(protected var message: TLMessage, protected var user: UserManager, val file_base: String) {
abstract class AbstractMediaFileManager(private var json: JsonObject, val file_base: String) {
open var isEmpty = false
abstract val size: Int
abstract val extension: String
open val downloaded: Boolean
get() = File(targetPathAndFilename).isFile()
get() = !isEmpty && File(targetPathAndFilename).isFile()
val downloading: Boolean
get() = File("${targetPathAndFilename}.downloading").isFile()
@ -47,10 +50,10 @@ abstract class AbstractMediaFileManager(protected var message: TLMessage, protec
open val targetFilename: String
get() {
val message_id = message.getId()
var to = message.getToId()
if (to is TLPeerChannel) {
val channel_id = to.getChannelId()
val message_id = json["id"].int
var to = json["toId"].obj
if (to.isA("peerChannel")) {
val channel_id = to["channelId"].int
return "channel_${channel_id}_${message_id}.$extension"
} else return "${message_id}.$extension"
}
@ -77,8 +80,8 @@ abstract class AbstractMediaFileManager(protected var message: TLMessage, protec
}
companion object {
fun throwUnexpectedObjectError(o: Any) {
throw RuntimeException("Unexpected " + o.javaClass.getName())
fun throwUnexpectedObjectError(constructor: String) {
throw RuntimeException("Unexpected ${constructor}")
}
}
}

40
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt
View File

@ -24,44 +24,38 @@ import com.github.badoualy.telegram.tl.exception.RpcErrorException
import java.io.IOException
import java.util.concurrent.TimeoutException
import com.google.gson.*
import com.github.salomonbrys.kotson.*
import de.fabianonline.telegram_backup.*
open class DocumentFileManager(msg: TLMessage, user: UserManager, file_base: String) : AbstractMediaFileManager(msg, user, file_base) {
protected var doc: TLDocument? = null
open class DocumentFileManager(message: JsonObject, file_base: String) : AbstractMediaFileManager(message, file_base) {
//protected var doc: TLDocument? = null
override lateinit var extension: String
open val isSticker: Boolean
get() {
if (this.isEmpty || doc == null) return false
return doc!!.getAttributes()?.filter { it is TLDocumentAttributeSticker }?.isNotEmpty() ?: false
}
get() = json.get("attributes")?.array?.any{it.obj.isA("documentAttributeSticker")} ?: false
override val size: Int
get() = if (doc != null) doc!!.getSize() else 0
get() = json["size"].int
open override val letter: String = "d"
open override val name: String = "document"
open override val description: String = "Document"
private val json = message["media"]["document"].obj
init {
val d = (msg.getMedia() as TLMessageMediaDocument).getDocument()
if (d is TLDocument) {
this.doc = d
} else if (d is TLDocumentEmpty) {
this.isEmpty = true
} else {
throwUnexpectedObjectError(d)
}
extension = processExtension()
}
private fun processExtension(): String {
if (doc == null) return "empty"
//if (doc == null) return "empty"
var ext: String? = null
var original_filename: String? = null
if (doc!!.getAttributes() != null)
for (attr in doc!!.getAttributes()) {
if (attr is TLDocumentAttributeFilename) {
original_filename = attr.getFileName()
if (json.contains("attributes"))
for (attr in json["attributes"].array) {
if (attr.obj["_constructor"].string.startsWith("documentAttributeFilename")) {
original_filename = attr.obj["fileName"].string
}
}
if (original_filename != null) {
@ -70,7 +64,7 @@ open class DocumentFileManager(msg: TLMessage, user: UserManager, file_base: Str
}
if (ext == null) {
ext = extensionFromMimetype(doc!!.getMimeType())
ext = extensionFromMimetype(json["mimeType"].string)
}
// Sometimes, extensions contain a trailing double quote. Remove this. Fixes #12.
@ -81,9 +75,7 @@ open class DocumentFileManager(msg: TLMessage, user: UserManager, file_base: Str
@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
override fun download(): Boolean {
if (doc != null) {
DownloadManager.downloadFile(targetPathAndFilename, size, doc!!.getDcId(), doc!!.getId(), doc!!.getAccessHash())
}
DownloadManager.downloadFile(targetPathAndFilename, size, json["dcId"].int, json["id"].long, json["accessHash"].long)
return true
}
}

67
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt
View File

@ -36,50 +36,55 @@ import java.io.File
import java.io.FileOutputStream
import java.util.ArrayList
import java.util.LinkedList
import java.util.NoSuchElementException
import java.net.URL
import java.util.concurrent.TimeoutException
import com.google.gson.*
import com.github.salomonbrys.kotson.*
import de.fabianonline.telegram_backup.*
import org.apache.commons.io.FileUtils
object FileManagerFactory {
fun getFileManager(m: TLMessage?, u: UserManager, file_base: String, settings: Settings?): AbstractMediaFileManager? {
fun getFileManager(m: TLMessage?, file_base: String, settings: Settings?): AbstractMediaFileManager? {
if (m == null) return null
val json = Gson().toJsonTree(m).obj
return getFileManager(json, u, file_base, settings)
return getFileManager(json, file_base, settings)
}
fun getFileManager(m: JsonObject?, u: UserManager, file_base: String, settings: Settings?): AbstractMediaFileManager? {
if (m == null) return null
val media = m.get("media")?.obj ?: return null
if (media.contains("photo")) {
return PhotoFileManager(media["photo"].obj, u, file_base)
fun getFileManager(message: JsonObject?, file_base: String, settings: Settings?): AbstractMediaFileManager? {
if (message == null) return null
try {
val media = message.get("media")?.obj ?: return null
if (media.isA("messageMediaPhoto")) {
return PhotoFileManager(message, file_base)
} else if (media.isA("messageMediaDocument")) {
val d = DocumentFileManager(message, file_base)
return if (d.isSticker) StickerFileManager(message, file_base) else d
} else if (media.isA("messageMediaGeo")) {
return GeoFileManager(message, file_base, settings)
} else if (media.isA("messageMediaEmpty")) {
return UnsupportedFileManager(message, file_base, "empty")
} else if (media.isA("messageMediaUnsupported")) {
return UnsupportedFileManager(message, file_base, "unsupported")
} else if (media.isA("messageMediaWebPage")) {
return UnsupportedFileManager(message, file_base, "webpage")
} else if (media.isA("messageMediaContact")) {
return UnsupportedFileManager(message, file_base, "contact")
} else if (media.isA("messageMediaVenue")) {
return UnsupportedFileManager(message, file_base, "venue")
} else {
AbstractMediaFileManager.throwUnexpectedObjectError(media["_constructor"].string)
}
} catch (e: IllegalStateException) {
println(message.toPrettyJson())
throw e
} catch (e: NoSuchElementException) {
println(message.toPrettyJson())
throw e
}
/*if (media is TLMessageMediaPhoto) {
return PhotoFileManager(m, u, file_base)
} else if (media is TLMessageMediaDocument) {
val d = DocumentFileManager(m, u, file_base)
return if (d.isSticker) {
StickerFileManager(m, u, file_base)
} else d
} else if (media is TLMessageMediaGeo) {
return GeoFileManager(m, u, file_base, settings)
} else if (media is TLMessageMediaEmpty) {
return UnsupportedFileManager(m, u, file_base, "empty")
} else if (media is TLMessageMediaUnsupported) {
return UnsupportedFileManager(m, u, file_base, "unsupported")
} else if (media is TLMessageMediaWebPage) {
return UnsupportedFileManager(m, u, file_base, "webpage")
} else if (media is TLMessageMediaContact) {
return UnsupportedFileManager(m, u, file_base, "contact")
} else if (media is TLMessageMediaVenue) {
return UnsupportedFileManager(m, u, file_base, "venue")
} else {
AbstractMediaFileManager.throwUnexpectedObjectError(media)
}*/
return null
}
}

18
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt
View File

@ -25,9 +25,12 @@ import de.fabianonline.telegram_backup.Settings
import java.io.IOException
import java.io.File
import com.google.gson.*
import com.github.salomonbrys.kotson.*
import de.fabianonline.telegram_backup.Utils
class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String, val settings: Settings?) : AbstractMediaFileManager(msg, user, file_base) {
protected lateinit var geo: TLGeoPoint
class GeoFileManager(message: JsonObject, file_base: String, val settings: Settings?) : AbstractMediaFileManager(message, file_base) {
//protected lateinit var geo: TLGeoPoint
// We don't know the size, so we just guess.
override val size: Int
@ -42,8 +45,11 @@ class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String, val s
override val letter = "g"
override val name = "geo"
override val description = "Geolocation"
val json = message["media"]["geo"].obj
init {
/*
val g = (msg.getMedia() as TLMessageMediaGeo).getGeo()
if (g is TLGeoPoint) {
this.geo = g
@ -51,16 +57,16 @@ class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String, val s
this.isEmpty = true
} else {
throwUnexpectedObjectError(g)
}
}*/
}
@Throws(IOException::class)
override fun download(): Boolean {
val url = "https://maps.googleapis.com/maps/api/staticmap?" +
"center=${geo.getLat()},${geo.getLong()}&" +
"markers=color:red|${geo.getLat()},${geo.getLong()}&" +
"center=${json["lat"].float},${json["_long"].float}&" +
"markers=color:red|${json["lat"].float},${json["_long"].float}&" +
"zoom=14&size=300x150&scale=2&format=png&" +
"key=" + (settings?.gmaps_key ?: Config.SECRET_GMAPS)
"key=" + (settings?.gmaps_key)
return DownloadManager.downloadExternalFile(targetPathAndFilename, url)
}
}

36
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt
View File

@ -25,7 +25,11 @@ import com.github.badoualy.telegram.tl.exception.RpcErrorException
import java.io.IOException
import java.util.concurrent.TimeoutException
class PhotoFileManager(json: JsonObject, user: UserManager, file_base: String) : AbstractMediaFileManager(json, user, file_base) {
import com.google.gson.*
import com.github.salomonbrys.kotson.*
import de.fabianonline.telegram_backup.*
class PhotoFileManager(message: JsonObject, file_base: String) : AbstractMediaFileManager(message, file_base) {
//private lateinit var photo: TLPhoto
override var size = 0
//private lateinit var photo_size: TLPhotoSize
@ -35,22 +39,33 @@ class PhotoFileManager(json: JsonObject, user: UserManager, file_base: String) :
override val name = "photo"
override val description = "Photo"
var biggestSize: JsonObject?
val biggestSize: JsonObject
var biggestSizeW = 0
var biggestSizeH = 0
val json = message["media"]["photo"].obj
override var isEmpty = json.isA("photoEmpty")
init {
/*val p = (msg.getMedia() as TLMessageMediaPhoto).getPhoto()*/
if (!isEmpty) {
var bsTemp: JsonObject? = null
for (elm in json["sizes"]) {
val s = elm.obj
if (biggestSize == null || (s["w"].int > biggestSizeW && s["h"].int > biggestSizeH)) {
biggestSize = s
biggestSizeW = s["w"].int
biggestSizeH = s["h"].int
size = s["size"].int // file size
for (elm in json["sizes"].array) {
val s = elm.obj
if (!s.isA("photoSize")) continue
if (bsTemp == null || (s["w"].int > biggestSizeW && s["h"].int > biggestSizeH)) {
bsTemp = s
biggestSizeW = s["w"].int
biggestSizeH = s["h"].int
size = s["size"].int // file size
}
}
if (biggestSize == null) throw RuntimeException("Could not find a size for a photo.")
if (bsTemp == null) throw RuntimeException("Could not find a size for a photo.")
biggestSize = bsTemp
} else {
biggestSize = JsonObject()
}
/*} else if (p is TLPhotoEmpty) {
@ -64,6 +79,7 @@ class PhotoFileManager(json: JsonObject, user: UserManager, file_base: String) :
override fun download(): Boolean {
/*if (isEmpty) return true*/
//val loc = photo_size.getLocation() as TLFileLocation
val loc = biggestSize["location"].obj
DownloadManager.downloadFile(targetPathAndFilename, size, loc["dcId"].int, loc["volumeId"].long, loc["localId"].int, loc["secret"].long)
return true

30
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt
View File

@ -49,29 +49,23 @@ import java.util.concurrent.TimeoutException
import org.apache.commons.io.FileUtils
class StickerFileManager(msg: TLMessage, user: UserManager, file_base: String) : DocumentFileManager(msg, user, file_base) {
import com.google.gson.*
import com.github.salomonbrys.kotson.*
import de.fabianonline.telegram_backup.*
class StickerFileManager(message: JsonObject, file_base: String) : DocumentFileManager(message, file_base) {
override val isSticker = true
val json = message["media"]["document"].obj
val sticker = json["attributes"].array.first{it.obj.isA("documentAttributeSticker")}.obj
override var isEmpty = sticker["stickerset"].obj.isA("inputStickerSetEmpty")
private val filenameBase: String
get() {
var sticker: TLDocumentAttributeSticker? = null
for (attr in doc!!.getAttributes()) {
if (attr is TLDocumentAttributeSticker) {
sticker = attr
}
}
val file = StringBuilder()
val set = sticker!!.getStickerset()
if (set is TLInputStickerSetShortName) {
file.append(set.getShortName())
} else if (set is TLInputStickerSetID) {
file.append(set.getId())
}
file.append("_")
file.append(sticker.getAlt().hashCode())
return file.toString()
val set = sticker["stickerset"].obj.get("shortName").nullString ?: sticker["stickerset"].obj.get("id").string
val hash = sticker["alt"].string.hashCode()
return "${set}_${hash}"
}
override val targetFilename: String

3
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt
View File

@ -19,8 +19,9 @@ package de.fabianonline.telegram_backup.mediafilemanager
import de.fabianonline.telegram_backup.UserManager
import com.github.badoualy.telegram.tl.api.*
import com.google.gson.JsonObject
class UnsupportedFileManager(msg: TLMessage, user: UserManager, type: String, file_base: String) : AbstractMediaFileManager(msg, user, file_base) {
class UnsupportedFileManager(json: JsonObject, file_base: String, type: String) : AbstractMediaFileManager(json, file_base) {
override var name = type
override val targetFilename = ""
override val targetPath = ""