From f5431e805eb4513883f7df42de0a51a410535d27 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 5 Mar 2018 06:36:47 +0100 Subject: [PATCH 1/6] Don't start the daemon mode if an exception happened earlier. --- .../de/fabianonline/telegram_backup/CommandLineController.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 4286ccd..332f25a 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -143,6 +143,8 @@ class CommandLineController { } catch (e: Exception) { e.printStackTrace() logger.error("Exception caught!", e) + // If we encountered an exception, we definitely don't want to start the daemon mode now. + CommandLineOptions.cmd_daemon = false } finally { if (CommandLineOptions.cmd_daemon) { handler.activate() From eb1f731b9d9a180dd16590b2e52c55187febe737 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 5 Mar 2018 06:39:30 +0100 Subject: [PATCH 2/6] DB_Update_9 now uses chunks of messages to (massively) reduce the amount of heap space needed. --- .../telegram_backup/DatabaseUpdates.kt | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt index 583b4a9..4ddfe59 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt @@ -381,21 +381,30 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn override fun _doUpdate() { val logger = LoggerFactory.getLogger(DB_Update_9::class.java) println(" Updating supergroup channel message data (this might take some time)...") - val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53") - val messages = TLVector() - val messages_to_delete = mutableListOf() + val count = db.queryInt("SELECT COUNT(*) FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53") + logger.debug("Found $count candidates for conversion") + val limit = 5000 + var offset = 0 var i = 0 - while (rs.next()) { - i++ - val msg = Database.bytesToTLMessage(rs.getBytes(2)) - if (msg!!.getFromId() != null) { - messages.add(msg) - messages_to_delete.add(rs.getInt(1)) + while (offset + 1 < count) { + logger.debug("Querying with limit $limit and offset $offset") + val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 LIMIT ${limit} OFFSET ${offset}") + val messages = TLVector() + val messages_to_delete = mutableListOf() + while (rs.next()) { + val msg = Database.bytesToTLMessage(rs.getBytes(2)) + if (msg!!.getFromId() != null) { + i++ + messages.add(msg) + messages_to_delete.add(rs.getInt(1)) + } } + db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP) + execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")") + + offset += limit } - logger.info("Converted ${messages.size} of ${i} messages.") - db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP) - execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")") + logger.info("Converted ${i} of ${count} messages.") println(" Cleaning up the database (this might also take some time, sorry)...") execute("VACUUM") } From dae603a95e35defeb6b8fb9182820994b1e225dd Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 5 Mar 2018 06:40:51 +0100 Subject: [PATCH 3/6] Removed building on JDK 6 and 8 from .travis.yml. Mainly because it didn't work. ;-) --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index fbe8433..650a6cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,4 @@ language: java -jdk: - - oraclejdk8 - - oraclejdk9 - - openjdk7 before_cache: - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock From cee9fad0ddea13142483c928b5e900a3c590826e Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 5 Mar 2018 18:04:46 +0100 Subject: [PATCH 4/6] Travis: Build only master. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 650a6cf..a6f94ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ language: java +branches: + only: + - master before_cache: - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ From bc63cfaea1e26d3f486318cc9e8bf04cbaf09d7d Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 5 Mar 2018 18:28:04 +0100 Subject: [PATCH 5/6] Typo in the GMaps key. Aaarg. --- src/main/kotlin/de/fabianonline/telegram_backup/Config.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt index 614ccd5..8be0bc0 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt @@ -46,7 +46,7 @@ object Config { var DEFAULT_PAGINATION = 5_000 - val SECRET_GMAPS = "AI" + "za" + "SyD_2" + "c0DKsfCXqgG" + "z0Sip7KHsBCU-paBfeJk" + val SECRET_GMAPS = "AI" + "za" + "SyD_" + "c0DKsfCXqgG" + "z0Sip7KHsBCU-paBfeJk" init { val p = Properties() From 6592e15b09a926ef6509c3febec7088cd2e7c468 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 5 Mar 2018 18:28:54 +0100 Subject: [PATCH 6/6] Bumped the version to 1.1.2 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e9e6e98..54bf189 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM openjdk:8 -ENV JAR_VERSION 1.1.1 +ENV JAR_VERSION 1.1.2 ENV JAR_DOWNLOAD_URL https://github.com/fabianonline/telegram_backup/releases/download/${JAR_VERSION}/telegram_backup.jar RUN apt-get update -y && apt-get install -y curl && \