From 24023560132a24d2651c33711647c52ee2ee5786 Mon Sep 17 00:00:00 2001
From: Fabian Schlenz <mail@fabianonline.de>
Date: Thu, 12 Apr 2018 06:54:14 +0200
Subject: [PATCH] Added a DatabaseUpdate to convert the data to json.

---
 build.gradle                                  |  3 +-
 .../telegram_backup/DatabaseUpdates.kt        | 40 +++++++++++++++++++
 .../de/fabianonline/telegram_backup/Utils.kt  | 24 +++++++++++
 3 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 9017a1b..34616be 100644
--- a/build.gradle
+++ b/build.gradle
@@ -38,7 +38,8 @@ dependencies {
 	compile 'com.github.spullara.mustache.java:compiler:0.9.5'
 	compile 'org.slf4j:slf4j-api:1.7.21'
 	compile 'ch.qos.logback:logback-classic:1.1.7'
-	compile 'com.google.code.gson:gson:2.5'
+	compile 'com.google.code.gson:gson:2.8.0'
+	compile 'com.github.salomonbrys.kotson:kotson:2.5.0'
 	compile 'com.github.kittinunf.fuel:fuel:1.12.0'
 	
 	testCompile 'junit:junit:4.12'
diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt
index 7ac15ba..5111c01 100644
--- a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt
+++ b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt
@@ -15,6 +15,8 @@ import org.slf4j.LoggerFactory
 import org.slf4j.Logger
 import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory
 import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager
+import com.github.salomonbrys.kotson.*
+import com.google.gson.*
 
 class DatabaseUpdates(protected var conn: Connection, protected var db: Database) {
 
@@ -33,6 +35,7 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
 		register(DB_Update_8(conn, db))
 		register(DB_Update_9(conn, db))
 		register(DB_Update_10(conn, db))
+		register(DB_Update_11(conn, db))
 	}
 
 	fun doUpdates() {
@@ -451,3 +454,40 @@ internal class DB_Update_10(conn: Connection, db: Database) : DatabaseUpdate(con
 		execute("CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)")
 	}
 }
+
+internal class DB_Update_11(conn: Connection, db: Database) : DatabaseUpdate(conn, db) {
+	override val version = 11
+	val logger = LoggerFactory.getLogger(DB_Update_11::class.java)
+	
+	override fun _doUpdate() {
+		execute("ALTER TABLE messages ADD COLUMN json TEXT NULL")
+		val limit = 5000
+		var offset = 0
+		var i = 0
+		val ps = conn.prepareStatement("UPDATE messages SET json=? WHERE id=?")
+		do {
+			i = 0
+			logger.debug("Querying with limit $limit and offset $offset")
+			val rs = db.executeQuery("SELECT id, data FROM messages WHERE json IS NULL AND api_layer=53 LIMIT $limit")
+			while (rs.next()) {
+				i++
+				val id = rs.getInt(1)
+				val msg = Database.bytesToTLMessage(rs.getBytes(2))
+				if (msg == null) continue
+				val json = msg.toJson()
+				ps.setString(1, json)
+				ps.setInt(2, id)
+				ps.addBatch()
+			}
+			rs.close()
+			conn.setAutoCommit(false)
+			ps.executeBatch()
+			ps.clearBatch()
+			conn.commit()
+			conn.setAutoCommit(true)
+			                                    
+			print(".")
+		} while (i >= limit)
+		ps.close()
+	}
+}
diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt
index 2381427..9a2dd17 100644
--- a/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt
+++ b/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt
@@ -17,11 +17,13 @@
 package de.fabianonline.telegram_backup
 
 import com.github.badoualy.telegram.tl.exception.RpcErrorException
+import com.github.badoualy.telegram.tl.api.TLMessage
 import java.io.File
 import java.util.Vector
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.TimeoutException
 import com.google.gson.*
+import com.github.salomonbrys.kotson.*
 import java.net.URL
 import org.apache.commons.io.IOUtils
 import de.fabianonline.telegram_backup.Version
@@ -223,3 +225,25 @@ fun Any.toJson(): String = Gson().toJson(this)
 fun Any.toPrettyJson(): String = GsonBuilder().setPrettyPrinting().create().toJson(this)
 
 class MaxTriesExceededException(): RuntimeException("Max tries exceeded") {}
+
+fun TLMessage.toJson(): String {
+	val json = Gson().toJsonTree(this)
+	cleanUpMessageJson(json)
+	return json.toString()
+}
+
+fun cleanUpMessageJson(json : JsonElement) {
+	if (json.isJsonArray) {
+		json.array.forEach {cleanUpMessageJson(it)}
+		return
+	} else if (!json.isJsonObject) {
+		return
+	}
+	if (json.obj.has("bytes")) {
+		json.obj -= "bytes"
+		return
+	}
+	json.obj.forEach {_: String, elm: JsonElement ->
+		if (elm.isJsonObject || elm.isJsonArray) cleanUpMessageJson(elm)
+	}
+}