From 42dc500514bafb3821673e625dafbf3079c90f0f Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Thu, 22 Feb 2018 07:55:53 +0100 Subject: [PATCH 01/24] Added Settings. --- .../fabianonline/telegram_backup/Database.kt | 22 ++++++- .../telegram_backup/DatabaseUpdates.kt | 12 ++++ .../fabianonline/telegram_backup/Settings.kt | 60 +++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index 8c35302..6f0a971 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -539,7 +539,27 @@ class Database private constructor(var client: TelegramClient) { e.printStackTrace() throw RuntimeException("Exception shown above happened.") } - + } + + fun settingsSetValue(key: String, value: String?) { + val ps = conn!!.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)") + ps.setString(1, key) + if (value==null) { + ps.setNull(2, Types.VARCHAR) + } else { + ps.setString(2, value) + } + ps.execute() + } + + fun settingsGetSettings(): Map { + val rs = stmt!!.executeQuery("SELECT key, value FROM settings") + val result = mutableMapOf() + while (rs.next()) { + result.set(rs.getString(1), rs.getString(2)) + } + rs.close() + return result } fun getIdsFromQuery(query: String): LinkedList { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt index c448681..3e6d780 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt @@ -32,6 +32,7 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database register(DB_Update_7(conn, db)) register(DB_Update_8(conn, db)) register(DB_Update_9(conn, db)) + register(DB_Update_10(conn, db)) } fun doUpdates() { @@ -398,3 +399,14 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")") } } + +internal class DB_Update_10(conn: Connection, db: Database) : DatabaseUpdate(conn, db) { + override val version: Int + get() = 10 + + @Throws(SQLException::class) + override fun _doUpdate() { + val logger = LoggerFactory.getLogger(DB_Update_10::class.java) + execute("CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)") + } +} diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt new file mode 100644 index 0000000..ddd31de --- /dev/null +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt @@ -0,0 +1,60 @@ +package de.fabianonline.telegram_backup + +class Settings() { + + val options = arrayOf( + Option("gmaps_api_key", "abc", "This contains the api key bla foo bar.") + ) + + init { + var all_settings = Database.getInstance().settingsGetSettings() + for ((key, value) in all_settings) { + val opt = options.find{it.key == key} + if (opt==null) throw IllegalArgumentException("Setting with key ${key} is unknown.") + if (value!=null) opt.value = value + } + } + + fun print() { + val modifiedSettings = options.filter{!it.isDefaultValue()} + val defaultSettings = options.filter{it.isDefaultValue()} + + if (modifiedSettings.count() > 0) { + println("Modified settings:") + modifiedSettings.forEach{it.print()} + println("\n") + } + + if (defaultSettings.count() > 0) { + println("Settings with default value:") + defaultSettings.forEach{it.print()} + println("\n") + } + } + + inner open class HiddenOption(val key: String, val default: String) { + var value: String? = null + + fun save() { + if (isDefaultValue()) { + Database.getInstance().settingsSetValue(key, null) + } else { + Database.getInstance().settingsSetValue(key, value) + } + } + + open fun print() { + println("%-30s %-30s".format(key, value)) + } + + fun isDefaultValue(): Boolean = (value==null || value==default) + + } + inner class Option(key: String, default: String, val descr: String): HiddenOption(key, default) { + override fun print() { + println("%-30s %-30s %s".format(key, value, descr)) + } + } +} + + From 75786e39b4e211cf5ef83c9b6d6b39c24b82f950 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 5 Mar 2018 06:55:24 +0100 Subject: [PATCH 02/24] Modified Settings class to only represent internal Settings. Also, renamed it to DbSettings. --- .../fabianonline/telegram_backup/Database.kt | 18 +++--- .../fabianonline/telegram_backup/Settings.kt | 60 ++----------------- 2 files changed, 13 insertions(+), 65 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index 6f0a971..1666fa2 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -541,7 +541,13 @@ class Database private constructor(var client: TelegramClient) { } } - fun settingsSetValue(key: String, value: String?) { + fun fetchSetting(key: String): String? { + val rs = stmt!!.executeQuery("SELECT value FROM settings WHERE key='${key}'") + rs.next() + return rs.getString(1) + } + + fun saveSetting(key: String, value: String?) { val ps = conn!!.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)") ps.setString(1, key) if (value==null) { @@ -552,16 +558,6 @@ class Database private constructor(var client: TelegramClient) { ps.execute() } - fun settingsGetSettings(): Map { - val rs = stmt!!.executeQuery("SELECT key, value FROM settings") - val result = mutableMapOf() - while (rs.next()) { - result.set(rs.getString(1), rs.getString(2)) - } - rs.close() - return result - } - fun getIdsFromQuery(query: String): LinkedList { try { val list = LinkedList() diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt index ddd31de..7cc36fd 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt @@ -1,60 +1,12 @@ package de.fabianonline.telegram_backup -class Settings() { - - val options = arrayOf( - Option("gmaps_api_key", "abc", "This contains the api key bla foo bar.") - ) +class DbSettings() { + private fun fetchValue(name: String): String? = Database.getInstance().fetchSetting(name) + private fun saveValue(name: String, value: String?) = Database.getInstance().saveSetting(name, value) - init { - var all_settings = Database.getInstance().settingsGetSettings() - for ((key, value) in all_settings) { - val opt = options.find{it.key == key} - if (opt==null) throw IllegalArgumentException("Setting with key ${key} is unknown.") - if (value!=null) opt.value = value - } - } - - fun print() { - val modifiedSettings = options.filter{!it.isDefaultValue()} - val defaultSettings = options.filter{it.isDefaultValue()} - - if (modifiedSettings.count() > 0) { - println("Modified settings:") - modifiedSettings.forEach{it.print()} - println("\n") - } - - if (defaultSettings.count() > 0) { - println("Settings with default value:") - defaultSettings.forEach{it.print()} - println("\n") - } - } - - inner open class HiddenOption(val key: String, val default: String) { - var value: String? = null - - fun save() { - if (isDefaultValue()) { - Database.getInstance().settingsSetValue(key, null) - } else { - Database.getInstance().settingsSetValue(key, value) - } - } - - open fun print() { - println("%-30s %-30s".format(key, value)) - } - - fun isDefaultValue(): Boolean = (value==null || value==default) - - } - inner class Option(key: String, default: String, val descr: String): HiddenOption(key, default) { - override fun print() { - println("%-30s %-30s %s".format(key, value, descr)) - } - } + var pts: String? + get() = fetchValue("pts") + set(x: String?) = saveValue("pts", x) } From 4e74b4c30b66a7259dbf874976c2ac77191bd69e Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 5 Mar 2018 06:56:38 +0100 Subject: [PATCH 03/24] Renamed Settings.kt to DbSettings.kt --- .../fabianonline/telegram_backup/{Settings.kt => DbSettings.kt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/kotlin/de/fabianonline/telegram_backup/{Settings.kt => DbSettings.kt} (100%) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt similarity index 100% rename from src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt rename to src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt From 9678aaaee8aa62752a75a4b11fcfb4ed3e332ad2 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 5 Mar 2018 18:26:53 +0100 Subject: [PATCH 04/24] Started IniSettings. (WIP!) --- .../de/fabianonline/telegram_backup/IniSettings.kt | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt new file mode 100644 index 0000000..1cfd7eb --- /dev/null +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -0,0 +1,13 @@ +package de.fabianonline.telegram_backup + +object IniSettings { + val logger = LoggerFactory.getLogger(IniSettings::class) + + init { + loadIni(UserManager.getInstance().fileBase + "config.ini") + } + + fun loadIni(filename: String) { + + } +} From 07572c061814dd304d41a3f15df521cf3b4403a0 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 6 Mar 2018 06:27:31 +0100 Subject: [PATCH 05/24] Extended IniSettings to be able to parse ini files. Yay. But: Still WIP! --- .../telegram_backup/IniSettings.kt | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt index 1cfd7eb..0e8c09f 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -1,13 +1,53 @@ package de.fabianonline.telegram_backup +import java.io.File +import org.slf4j.LoggerFactory +import org.slf4j.Logger + object IniSettings { - val logger = LoggerFactory.getLogger(IniSettings::class) + val logger = LoggerFactory.getLogger(IniSettings::class.java) + var settings = mutableMapOf>() init { loadIni(UserManager.getInstance().fileBase + "config.ini") + copySampleIni(UserManager.getInstance().fileBase + "config.ini.sample") } fun loadIni(filename: String) { - + val file = File(filename) + logger.trace("Checking ini file {}", filename.anonymize()) + if (!file.exists()) return + logger.debug("Loading ini file {}", filename.anonymize()) + file.forEachLine { parseLine(it) } } + + fun parseLine(original_line: String) { + logger.trace("Parsing line: {}", original_line) + var line = original_line.trim().replaceAfter("#", "").removeSuffix("#") + logger.trace("After cleaning: {}", line) + if (line == "") return + val parts: List = line.split("=", limit=2).map{it.trim()} + + if (parts.size < 2) throw RuntimeException("Invalid config setting: $line") + + val (key, value) = parts + if (value=="") { + settings.remove(key) + } else { + var map = settings.get(key) + if (map == null) { + map = mutableListOf() + settings.put(key, map) + } + map.add(value) + } + } + + fun copySampleIni(filename: String) { + + } + + fun println() = println(settings) + + } From 53fcd36e667ed670cdce253c38edbf398a8d4b25 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Wed, 7 Mar 2018 06:07:16 +0100 Subject: [PATCH 06/24] config.sample.ini is copied into the data directory on every run. --- .../telegram_backup/IniSettings.kt | 6 +++-- src/main/resources/config.sample.ini | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 src/main/resources/config.sample.ini diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt index 0e8c09f..2d96006 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -10,7 +10,7 @@ object IniSettings { init { loadIni(UserManager.getInstance().fileBase + "config.ini") - copySampleIni(UserManager.getInstance().fileBase + "config.ini.sample") + copySampleIni(UserManager.getInstance().fileBase + "config.sample.ini") } fun loadIni(filename: String) { @@ -44,7 +44,9 @@ object IniSettings { } fun copySampleIni(filename: String) { - + val stream = Config::class.java.getResourceAsStream("/config.sample.ini") + File(filename).outputStream().use { stream.copyTo(it) } + stream.close() } fun println() = println(settings) diff --git a/src/main/resources/config.sample.ini b/src/main/resources/config.sample.ini new file mode 100644 index 0000000..d734c61 --- /dev/null +++ b/src/main/resources/config.sample.ini @@ -0,0 +1,23 @@ +# Config file for telegram_backup +# Copy it to config.ini to use it. This sample file be overwritten on every run. +# +# Lines starting with '#' are ignored. +# Settings have the form 'key = value' +# To unset a setting, use 'key =' (or just don't use that key at all) +# Some settings may appear more than once. + + + +## GMaps key to use. Leave empty / don't set a value to use the built in key. +# gmaps_key = mysecretgmapskey + +## Use pagination in the HTML export? +# pagination = true +# pagination_size = 5000 + +## Don't download media files +# no_media = false + +## Don't download supergroups and channels by default +# download_channels = false +# download_supergroups = false From 7fa89ab1b1dfdc4b3f29262007e611633b80cef7 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Wed, 7 Mar 2018 06:09:51 +0100 Subject: [PATCH 07/24] IniSettings class is now finished and also already used for GMAPS_KEY. --- .../kotlin/de/fabianonline/telegram_backup/IniSettings.kt | 5 +++++ .../telegram_backup/mediafilemanager/GeoFileManager.kt | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt index 2d96006..8f811fb 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -51,5 +51,10 @@ object IniSettings { fun println() = println(settings) + fun get(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default + fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default + fun getArray(key: String): List = settings.get(key) ?: listOf() + val gmaps_key: String + get() = get("gmaps_key", default=Config.SECRET_GMAPS)!! } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt index 15726c8..67c5647 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt @@ -21,7 +21,7 @@ import de.fabianonline.telegram_backup.Database import de.fabianonline.telegram_backup.StickerConverter import de.fabianonline.telegram_backup.DownloadProgressInterface import de.fabianonline.telegram_backup.DownloadManager -import de.fabianonline.telegram_backup.Config +import de.fabianonline.telegram_backup.IniSettings import com.github.badoualy.telegram.api.TelegramClient import com.github.badoualy.telegram.tl.core.TLIntVector @@ -76,7 +76,7 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) val url = "https://maps.googleapis.com/maps/api/staticmap?" + "center=" + geo.getLat() + "," + geo.getLong() + "&" + "zoom=14&size=300x150&scale=2&format=png&" + - "key=" + Config.SECRET_GMAPS + "key=" + IniSettings.gmaps_key return DownloadManager.downloadExternalFile(targetPathAndFilename, url) } } From 89cca3940918b5b35690211b684e733afd0deeb8 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Wed, 7 Mar 2018 06:12:37 +0100 Subject: [PATCH 08/24] IniSettings will be initialized at startup. --- .../telegram_backup/CommandLineController.kt | 4 ++++ .../de/fabianonline/telegram_backup/IniSettings.kt | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 4286ccd..11ab5b0 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -93,6 +93,10 @@ class CommandLineController { throw RuntimeException("Account / User mismatch") } } + + // Load the ini file. + IniSettings.load() + logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login) if (CommandLineOptions.cmd_login) { cmd_login(account) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt index 8f811fb..ffa5bf2 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -13,7 +13,10 @@ object IniSettings { copySampleIni(UserManager.getInstance().fileBase + "config.sample.ini") } - fun loadIni(filename: String) { + // Dummy function that can be called to force this object to run its init-code. + fun load() { } + + private fun loadIni(filename: String) { val file = File(filename) logger.trace("Checking ini file {}", filename.anonymize()) if (!file.exists()) return @@ -21,7 +24,7 @@ object IniSettings { file.forEachLine { parseLine(it) } } - fun parseLine(original_line: String) { + private fun parseLine(original_line: String) { logger.trace("Parsing line: {}", original_line) var line = original_line.trim().replaceAfter("#", "").removeSuffix("#") logger.trace("After cleaning: {}", line) @@ -43,7 +46,7 @@ object IniSettings { } } - fun copySampleIni(filename: String) { + private fun copySampleIni(filename: String) { val stream = Config::class.java.getResourceAsStream("/config.sample.ini") File(filename).outputStream().use { stream.copyTo(it) } stream.close() From c99766a71efca85ff989df01163662e25fa4093e Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Thu, 8 Mar 2018 22:31:27 +0100 Subject: [PATCH 09/24] deploy.sh: Create relases as published, not as draft. --- deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index db62fd4..167cff2 100755 --- a/deploy.sh +++ b/deploy.sh @@ -56,7 +56,7 @@ echo "Building it..." gradle build || error "Build failed. What did you do?!" echo "Generating a release on Github..." -json=$(ruby -e "require 'json'; puts({tag_name: '$VERSION', name: '$VERSION', draft: true, body: \$stdin.read}.to_json)" <<< "$release_notes") || error "Couldn't generate JSON for Github" +json=$(ruby -e "require 'json'; puts({tag_name: '$VERSION', name: '$VERSION', body: \$stdin.read}.to_json)" <<< "$release_notes") || error "Couldn't generate JSON for Github" json=$(curl $CURL_OPTS https://api.github.com/repos/fabianonline/telegram_backup/releases -XPOST -d "$json") || error "Github failure" From d66834c3d5053c15d1e208a12a57b763baaf783a Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Thu, 8 Mar 2018 22:32:18 +0100 Subject: [PATCH 10/24] deploy.sh: Use the html_url of the new release. --- deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index 167cff2..ee0d528 100755 --- a/deploy.sh +++ b/deploy.sh @@ -63,7 +63,7 @@ json=$(curl $CURL_OPTS https://api.github.com/repos/fabianonline/telegram_backup echo "Uploading telegram_backup.jar to Github..." upload_url=$(jq -r ".upload_url" <<< "$json") || error "Could not parse JSON from Github" upload_url=$(sed 's/{.*}//' <<< "$upload_url") -release_url=$(jq -r ".url" <<< "$json") || error "Could not parse JSON from Github" +release_url=$(jq -r ".html_url" <<< "$json") || error "Could not parse JSON from Github" curl $CURL_OPTS --header "Content-Type: application/zip" "${upload_url}?name=telegram_backup.jar" --upload-file build/libs/telegram_backup.jar || error "Asset upload to github failed" echo "Building the docker image..." From a8944125b6672d98c2fd111114c3291814b5ad67 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Thu, 8 Mar 2018 22:32:51 +0100 Subject: [PATCH 11/24] deploy.sh: Check out master after running the tool. --- deploy.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy.sh b/deploy.sh index ee0d528..be3715d 100755 --- a/deploy.sh +++ b/deploy.sh @@ -81,4 +81,7 @@ curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<- echo "Cleaning release_notes.txt..." > release_notes.txt +echo "Checking out master..." +git checkout master + echo "Done." From b0fa297a6158c4338ead5c0d7ebeca3265edc455 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Thu, 8 Mar 2018 23:11:14 +0100 Subject: [PATCH 12/24] deploy.sh: Moved the gradle build task to earlier in the process to better be able to catch errors. --- deploy.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy.sh b/deploy.sh index be3715d..ea3e0a9 100755 --- a/deploy.sh +++ b/deploy.sh @@ -40,6 +40,9 @@ git commit -m "Bumping the version to $VERSION" Dockerfile echo "Tagging the new version..." git tag -a "$VERSION" -m "Version $VERSION" || error +echo "Building it..." +gradle build || error "Build failed. What did you do?!" + echo "Checking out stable..." git checkout stable || error @@ -52,9 +55,6 @@ git push --all || error echo "Pushing tags to Github..." git push --tags || error -echo "Building it..." -gradle build || error "Build failed. What did you do?!" - echo "Generating a release on Github..." json=$(ruby -e "require 'json'; puts({tag_name: '$VERSION', name: '$VERSION', body: \$stdin.read}.to_json)" <<< "$release_notes") || error "Couldn't generate JSON for Github" From d796cb1bf0b811aaeb8d4607650e2a4fbe456894 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Thu, 8 Mar 2018 23:11:48 +0100 Subject: [PATCH 13/24] deploy.sh: Telegram message is now HTML formatted. --- deploy.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy.sh b/deploy.sh index ea3e0a9..730c01c 100755 --- a/deploy.sh +++ b/deploy.sh @@ -73,10 +73,10 @@ echo "Pushing the docker image..." docker push fabianonline/telegram_backup echo "Notifying the Telegram group..." -release_notes=$(sed 's/\* /• /' <<< "$release_notes") -message="Version $VERSION released"$'\n'$'\n'"$release_notes"$'\n'$'\n'"$release_url" +release_notes=$(sed 's/\* /• /' | sed 's/&/&/g' | sed 's//\>/g' <<< "$release_notes") +message="Version $VERSION was just released"$'\n'$'\n'"$release_notes"$'\n'$'\n'"$release_url" -curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" <<< "$message" +curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" --form-string "parse_mode=HTML" --form-string "disable_web_page_preview=true" <<< "$message" echo "Cleaning release_notes.txt..." > release_notes.txt From 19973818f8920fc8c62577d2cc0d34712ea54a38 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Sat, 10 Mar 2018 23:26:03 +0100 Subject: [PATCH 14/24] Moved more settings to IniSettings. --- .../telegram_backup/CommandLineController.kt | 11 +++-------- .../telegram_backup/CommandLineOptions.kt | 10 ---------- .../fabianonline/telegram_backup/DownloadManager.kt | 4 ++-- .../de/fabianonline/telegram_backup/IniSettings.kt | 11 +++++++++++ .../telegram_backup/exporter/HTMLExporter.kt | 9 ++------- src/main/resources/config.sample.ini | 4 ++-- 6 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 11ab5b0..4d82291 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -137,12 +137,12 @@ class CommandLineController { val d = DownloadManager(client, CommandLineDownloadProgress()) logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages) d.downloadMessages(CommandLineOptions.val_limit_messages) - logger.debug("CommandLineOptions.cmd_no_media: {}", CommandLineOptions.cmd_no_media) - if (!CommandLineOptions.cmd_no_media) { + logger.debug("IniSettings.download_media: {}", IniSettings.download_media) + if (IniSettings.download_media) { logger.debug("Calling DownloadManager.downloadMedia") d.downloadMedia() } else { - println("Skipping media download because --no-media is set.") + println("Skipping media download because download_media is set to false.") } } catch (e: Exception) { e.printStackTrace() @@ -277,17 +277,12 @@ class CommandLineController { println(" --trace-telegram Shows lots of debug messages from the library used to access Telegram.") println(" -A, --list-accounts List all existing accounts ") println(" --limit-messages Downloads at most the most recent messages.") - println(" --no-media Do not download media files.") println(" -t, --target Target directory for the files.") println(" -e, --export Export the database. Valid formats are:") println(" html - Creates HTML files.") - println(" --pagination Splits the HTML export into multiple HTML pages with messages per page. Default is 5000.") - println(" --no-pagination Disables pagination.") println(" --license Displays the license of this program.") println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.") println(" --stats Print some usage statistics.") - println(" --with-channels Backup channels as well.") - println(" --with-supergroups Backup supergroups as well.") } private fun list_accounts() { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index 32ba425..e06dd53 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -26,18 +26,13 @@ internal object CommandLineOptions { var cmd_version = false var cmd_license = false var cmd_daemon = false - var cmd_no_media = false var cmd_anonymize = false var cmd_stats = false - var cmd_channels = false - var cmd_supergroups = false - var cmd_no_pagination = false var val_account: String? = null var val_limit_messages: Int? = null var val_target: String? = null var val_export: String? = null var val_test: Int? = null - var val_pagination: Int = Config.DEFAULT_PAGINATION @JvmStatic fun parseOptions(args: Array) { var last_cmd: String? = null @@ -49,7 +44,6 @@ internal object CommandLineOptions { "--target" -> val_target = arg "--export" -> val_export = arg "--test" -> val_test = Integer.parseInt(arg) - "--pagination" -> val_pagination = Integer.parseInt(arg) } last_cmd = null continue @@ -83,18 +77,14 @@ internal object CommandLineOptions { last_cmd = "--pagination" continue@loop } - "--no-pagination" -> cmd_no_pagination = true "--license" -> cmd_license = true "-d", "--daemon" -> cmd_daemon = true - "--no-media" -> cmd_no_media = true "--test" -> { last_cmd = "--test" continue@loop } "--anonymize" -> cmd_anonymize = true "--stats" -> cmd_stats = true - "--with-channels" -> cmd_channels = true - "--with-supergroups" -> cmd_supergroups = true else -> throw RuntimeException("Unknown command " + arg) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt index 4b745a1..489d95f 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt @@ -178,7 +178,7 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI } */ - if (CommandLineOptions.cmd_channels || CommandLineOptions.cmd_supergroups) { + if (IniSettings.download_channels || IniSettings.download_supergroups) { System.out.println("Processing channels and/or supergroups...") System.out.println("Please note that only channels/supergroups in the last 100 active chats are processed.") @@ -210,7 +210,7 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI // If this is a channel and we don't want to download channels OR // it is a supergroups and we don't want to download supergroups, then - if (channels.contains(channel_id) && !CommandLineOptions.cmd_channels || supergroups.contains(channel_id) && !CommandLineOptions.cmd_supergroups) { + if (channels.contains(channel_id) && !IniSettings.download_channels || supergroups.contains(channel_id) && !IniSettings.download_supergroups) { // Skip this chat. continue } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt index ffa5bf2..8a37ef1 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -56,8 +56,19 @@ object IniSettings { fun get(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default + fun getBoolean(key: String, default: Boolean = false): Boolean = settings.get(key)?.last() == "true" fun getArray(key: String): List = settings.get(key) ?: listOf() val gmaps_key: String get() = get("gmaps_key", default=Config.SECRET_GMAPS)!! + val pagination: Boolean + get() = getBoolean("pagination", default=true) + val pagination_size: Int + get() = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!! + val download_media: Boolean + get() = getBoolean("download_media", default=true) + val download_channels: Boolean + get() = getBoolean("download_channels", default=false) + val download_supergroups: Boolean + get() = getBoolean("download_supergroups", default=false) } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt b/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt index e4e8fae..1053946 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt @@ -16,12 +16,6 @@ package de.fabianonline.telegram_backup.exporter -import de.fabianonline.telegram_backup.UserManager -import de.fabianonline.telegram_backup.Database -import de.fabianonline.telegram_backup.anonymize -import de.fabianonline.telegram_backup.toPrettyJson -import de.fabianonline.telegram_backup.CommandLineOptions - import java.io.File import java.io.PrintWriter import java.io.OutputStreamWriter @@ -38,6 +32,7 @@ import java.util.HashMap import com.github.mustachejava.DefaultMustacheFactory import com.github.mustachejava.Mustache import com.github.mustachejava.MustacheFactory +import de.fabianonline.telegram_backup.* import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -49,7 +44,7 @@ class HTMLExporter { @Throws(IOException::class) fun export() { try { - val pagination = if (CommandLineOptions.cmd_no_pagination) -1 else CommandLineOptions.val_pagination + val pagination = if (IniSettings.pagination) IniSettings.pagination_size else -1 // Create base dir logger.debug("Creating base dir") diff --git a/src/main/resources/config.sample.ini b/src/main/resources/config.sample.ini index d734c61..d7606ea 100644 --- a/src/main/resources/config.sample.ini +++ b/src/main/resources/config.sample.ini @@ -15,8 +15,8 @@ # pagination = true # pagination_size = 5000 -## Don't download media files -# no_media = false +## Download media files +# download_media = true ## Don't download supergroups and channels by default # download_channels = false From e75aa2101eebf3a5ad089c82fd2beeda4b32a624 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 12 Mar 2018 22:01:47 +0100 Subject: [PATCH 15/24] Fixed lots of unclosed ResultSets in Database and DatabaseUpdates. --- .../fabianonline/telegram_backup/Database.kt | 46 ++++++++++++------- .../telegram_backup/DatabaseUpdates.kt | 17 +++---- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index 8c35302..20cbda6 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -56,7 +56,9 @@ class Database private constructor(var client: TelegramClient) { try { val rs = stmt!!.executeQuery("SELECT MAX(message_id) FROM messages WHERE source_type IN ('group', 'dialog')") rs.next() - return rs.getInt(1) + val result = rs.getInt(1) + rs.close() + return result } catch (e: SQLException) { return 0 } @@ -87,6 +89,7 @@ class Database private constructor(var client: TelegramClient) { missing.add(i) } } + rs.close() return missing } catch (e: SQLException) { e.printStackTrace() @@ -111,16 +114,7 @@ class Database private constructor(var client: TelegramClient) { } - fun getMessagesFromUserCount(): Int { - try { - val rs = stmt!!.executeQuery("SELECT COUNT(*) FROM messages WHERE sender_id=" + user_manager.user!!.getId()) - rs.next() - return rs.getInt(1) - } catch (e: SQLException) { - throw RuntimeException(e) - } - - } + fun getMessagesFromUserCount() = queryInt("SELECT COUNT(*) FROM messages WHERE sender_id=" + user_manager.user!!.getId()) fun getMessageTypesWithCount(): HashMap = getMessageTypesWithCount(GlobalChat()) @@ -164,7 +158,9 @@ class Database private constructor(var client: TelegramClient) { try { val rs = stmt!!.executeQuery("PRAGMA encoding") rs.next() - return rs.getString(1) + val result = rs.getString(1) + rs.close() + return result } catch (e: SQLException) { logger.debug("SQLException: {}", e) return "unknown" @@ -255,9 +251,7 @@ class Database private constructor(var client: TelegramClient) { } - fun getTopMessageIDForChannel(id: Int): Int { - return queryInt("SELECT MAX(message_id) FROM messages WHERE source_id=$id AND source_type IN('channel', 'supergroup')") - } + fun getTopMessageIDForChannel(id: Int): Int = queryInt("SELECT MAX(message_id) FROM messages WHERE source_id=$id AND source_type IN('channel', 'supergroup')") fun logRun(start_id: Int, end_id: Int, count: Int) { try { @@ -269,6 +263,7 @@ class Database private constructor(var client: TelegramClient) { ps.setInt(2, end_id) ps.setInt(3, count) ps.execute() + ps.close() } catch (e: SQLException) { } @@ -278,7 +273,9 @@ class Database private constructor(var client: TelegramClient) { try { val rs = stmt!!.executeQuery(query) rs.next() - return rs.getInt(1) + val result = rs.getInt(1) + rs.close() + return result } catch (e: SQLException) { throw RuntimeException("Could not get count of messages.") } @@ -431,6 +428,9 @@ class Database private constructor(var client: TelegramClient) { ps_insert_or_ignore.clearBatch() conn!!.commit() conn!!.setAutoCommit(true) + + ps.close() + ps_insert_or_ignore.close() } catch (e: Exception) { e.printStackTrace() throw RuntimeException("Exception shown above happened.") @@ -486,6 +486,9 @@ class Database private constructor(var client: TelegramClient) { ps_insert_or_replace.clearBatch() conn!!.commit() conn!!.setAutoCommit(true) + + ps_insert_or_ignore.close() + ps_insert_or_replace.close() } catch (e: Exception) { e.printStackTrace() throw RuntimeException("Exception shown above happened.") @@ -535,6 +538,9 @@ class Database private constructor(var client: TelegramClient) { ps_insert_or_replace.clearBatch() conn!!.commit() conn!!.setAutoCommit(true) + + ps_insert_or_ignore.close() + ps_insert_or_replace.close() } catch (e: Exception) { e.printStackTrace() throw RuntimeException("Exception shown above happened.") @@ -564,6 +570,7 @@ class Database private constructor(var client: TelegramClient) { while (rs.next()) { map.put("count.messages.type." + rs.getString(1), rs.getInt(2)) } + rs.close() return map } catch (e: Exception) { throw RuntimeException(e) @@ -586,6 +593,7 @@ class Database private constructor(var client: TelegramClient) { map.put("count.messages.media_type.$s", rs.getInt(2)) } map.put("count.messages.media_type.any", count) + rs.close() return map } catch (e: Exception) { throw RuntimeException(e) @@ -625,6 +633,7 @@ class Database private constructor(var client: TelegramClient) { } map.put("authors.count.others", count_others) map.put("authors.all", all_data) + rs.close() return map } catch (e: Exception) { throw RuntimeException(e) @@ -634,7 +643,9 @@ class Database private constructor(var client: TelegramClient) { fun getMessageCountForExport(c: AbstractChat): Int { val rs = stmt!!.executeQuery("SELECT COUNT(*) FROM messages WHERE " + c.query); rs.next() - return rs.getInt(1) + val result = rs.getInt(1) + rs.close() + return result } fun getMessageTimesMatrix(c: AbstractChat): Array { @@ -647,6 +658,7 @@ class Database private constructor(var client: TelegramClient) { while (rs.next()) { result[if (rs.getInt(1) == 0) 6 else rs.getInt(1) - 1][rs.getInt(2)] = rs.getInt(3) } + rs.close() return result } catch (e: Exception) { throw RuntimeException(e) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt index 0f8d12c..d3b3d99 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt @@ -37,25 +37,19 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database fun doUpdates() { try { val stmt = conn.createStatement() - var rs: ResultSet logger.debug("DatabaseUpdate.doUpdates running") logger.debug("Getting current database version") var version: Int logger.debug("Checking if table database_versions exists") - rs = stmt.executeQuery("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions'") - rs.next() - if (rs.getInt(1) == 0) { + val table_count = db.queryInt("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions'") + if (table_count == 0) { logger.debug("Table does not exist") version = 0 } else { logger.debug("Table exists. Checking max version") - rs.close() - rs = stmt.executeQuery("SELECT MAX(version) FROM database_versions") - rs.next() - version = rs.getInt(1) + version = db.queryInt("SELECT MAX(version) FROM database_versions") } - rs.close() logger.debug("version: {}", version) System.out.println("Database version: " + version) logger.debug("Max available database version is {}", maxPossibleVersion) @@ -106,6 +100,9 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database } catch (e: SQLException) { throw RuntimeException(e) } + + println("Cleaning up the database (this might take some time)...") + try { stmt.executeUpdate("VACUUM") } catch (t: Throwable) { logger.debug("Exception during VACUUMing: {}", t) } } else { logger.debug("No update necessary.") @@ -441,7 +438,5 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn } println() logger.info("Converted ${i} of ${count} messages.") - println(" Cleaning up the database (this might also take some time, sorry)...") - execute("VACUUM") } } From c79336618c809fbda35f2d6148bc8ea4bbb66864 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 12 Mar 2018 22:02:14 +0100 Subject: [PATCH 16/24] deploy_beta.sh: A small script to deploy the current beta version to the telegram group. --- deploy_beta.sh | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 deploy_beta.sh diff --git a/deploy_beta.sh b/deploy_beta.sh new file mode 100755 index 0000000..747b846 --- /dev/null +++ b/deploy_beta.sh @@ -0,0 +1,35 @@ +#!/bin/bash +error() { + echo "Error: $1" + exit 1 +} + +release_notes="$(cat release_notes.txt 2>/dev/null)" + +source "deploy.secret.sh" +[ -z "$BOT_TOKEN" ] && error "BOT_TOKEN is not set or empty." +[ -z "$CHAT_ID" ] && error "CHAT_ID is not set or empty." + +version=$(git describe --tags --dirty) + +echo "Enter additional notes, end with Ctrl-D." +additional_notes="$(cat)" + +echo "Building it..." +gradle build || error "Build failed. What did you do?!" + +echo "Copying it to files.fabianonline.de..." +filename="telegram_backup.beta_${version}.jar" +cp --no-preserve "mode,ownership,timestamps" build/libs/telegram_backup.jar /data/containers/nginx/www/files/${filename} + +echo "Notifying the Telegram group..." +release_notes=$(sed 's/\* /• /' | sed 's/&/&/g' | sed 's//\>/g' <<< "$release_notes") +message="New beta release $version"$'\n'$'\n'"This is a beta release. There may be bugs included that might destroy your data. Only use this beta release if you know what you're doing. AND MAKE A BACKUP BEFORE USING IT!"$'\n'$'\n'"$additional_notes"$'\n'"$release_notes"$'\n'$'\n'"https://files.fabianonline.de/${filename}" + +result=$(curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" --form-string "parse_mode=HTML" --form-string "disable_web_page_preview=true" <<< "$message") +message_id=$(jq -r '.result.message_id' <<< "$result") + +echo "Pinning the new message..." +curl https://api.telegram.org/bot${BOT_TOKEN}/pinChatMessage -XPOST --form "chat_id=${CHAT_ID}" --form "message_id=${message_id}" --form "disable_notification=true" + +echo "Done." From ecb225ef60a85f7874136a51a45fc8a3e9d7171c Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 12 Mar 2018 22:02:43 +0100 Subject: [PATCH 17/24] Extended .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 953f7e4..34f0d86 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ dev/ todo deploy.secret.sh release_notes.txt +out From a8149dfce9eceb26ff7cd207e651ad25ec78f6b8 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 13 Mar 2018 06:09:52 +0100 Subject: [PATCH 18/24] deploy_beta.sh: Fixed a line causing the script to wait for STDIN (don't really understand, why) and modified the "this is just for testing, backup your data" warning. --- deploy_beta.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/deploy_beta.sh b/deploy_beta.sh index 747b846..1466c51 100755 --- a/deploy_beta.sh +++ b/deploy_beta.sh @@ -23,8 +23,13 @@ filename="telegram_backup.beta_${version}.jar" cp --no-preserve "mode,ownership,timestamps" build/libs/telegram_backup.jar /data/containers/nginx/www/files/${filename} echo "Notifying the Telegram group..." -release_notes=$(sed 's/\* /• /' | sed 's/&/&/g' | sed 's//\>/g' <<< "$release_notes") -message="New beta release $version"$'\n'$'\n'"This is a beta release. There may be bugs included that might destroy your data. Only use this beta release if you know what you're doing. AND MAKE A BACKUP BEFORE USING IT!"$'\n'$'\n'"$additional_notes"$'\n'"$release_notes"$'\n'$'\n'"https://files.fabianonline.de/${filename}" +release_notes=$(echo "$release_notes" | sed 's/\* /• /' | sed 's/&/&/g' | sed 's//\>/g') +message="New beta release $version"$'\n\n' +message="${message}${additional_notes}"$'\n\n' +message="${message}${release_notes}"$'\n\n' +message="${message}This is a release for testing purposes only. There may be bugs included that might destroy your data. Only use this beta release if you know what you're doing. AND MAKE A BACKUP OF YOUR BACKUP BEFORE USING IT!"$'\n\n' +message="${message}Please report back if you used this release and encountered a bug. Also report back, if you used it and IT WORKED, please. Thank you."$'\n\n' +message="${message}https://files.fabianonline.de/${filename}" result=$(curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" --form-string "parse_mode=HTML" --form-string "disable_web_page_preview=true" <<< "$message") message_id=$(jq -r '.result.message_id' <<< "$result") From 077cbcebcad3993165a340a4f4d4d0cade27d764 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 13 Mar 2018 06:10:44 +0100 Subject: [PATCH 19/24] deploy_beta.sh: Make clear that the changelog is since the last real release. --- deploy_beta.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy_beta.sh b/deploy_beta.sh index 1466c51..b189ede 100755 --- a/deploy_beta.sh +++ b/deploy_beta.sh @@ -26,7 +26,7 @@ echo "Notifying the Telegram group..." release_notes=$(echo "$release_notes" | sed 's/\* /• /' | sed 's/&/&/g' | sed 's//\>/g') message="New beta release $version"$'\n\n' message="${message}${additional_notes}"$'\n\n' -message="${message}${release_notes}"$'\n\n' +message="${message}Changes since the last real release:"$'\n'"${release_notes}"$'\n\n' message="${message}This is a release for testing purposes only. There may be bugs included that might destroy your data. Only use this beta release if you know what you're doing. AND MAKE A BACKUP OF YOUR BACKUP BEFORE USING IT!"$'\n\n' message="${message}Please report back if you used this release and encountered a bug. Also report back, if you used it and IT WORKED, please. Thank you."$'\n\n' message="${message}https://files.fabianonline.de/${filename}" From 25a01fae4bc032aa379c95719559043624b59b94 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 13 Mar 2018 06:30:39 +0100 Subject: [PATCH 20/24] You can now restrict the downloading of channels and supergroups by defining black- and whitelists in config.ini --- .../telegram_backup/DownloadManager.kt | 131 +++++++++--------- .../telegram_backup/IniSettings.kt | 11 +- src/main/resources/config.sample.ini | 39 +++++- 3 files changed, 117 insertions(+), 64 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt index 489d95f..f3a8728 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt @@ -106,19 +106,14 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI @Throws(RpcErrorException::class, IOException::class, TimeoutException::class) fun _downloadMessages(limit: Int?) { logger.info("This is _downloadMessages with limit {}", limit) - val dialog_limit = 100 - logger.info("Downloading the last {} dialogs", dialog_limit) + logger.info("Downloading the last dialogs") System.out.println("Downloading most recent dialogs... ") var max_message_id = 0 - val dialogs = client!!.messagesGetDialogs( - 0, - 0, - TLInputPeerEmpty(), - dialog_limit) - logger.debug("Got {} dialogs", dialogs.getDialogs().size) + val chats = getChats() + logger.debug("Got {} dialogs, {} supergoups, {} channels", chats.dialogs.size, chats.supergroups.size, chats.channels.size) - for (d in dialogs.getDialogs()) { - if (d.getTopMessage() > max_message_id && d.getPeer() !is TLPeerChannel) { + for (d in chats.dialogs) { + if (d.getTopMessage() > max_message_id) { logger.trace("Updating top message id: {} => {}. Dialog type: {}", max_message_id, d.getTopMessage(), d.getPeer().javaClass) max_message_id = d.getTopMessage() } @@ -179,63 +174,32 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI */ if (IniSettings.download_channels || IniSettings.download_supergroups) { - System.out.println("Processing channels and/or supergroups...") - System.out.println("Please note that only channels/supergroups in the last 100 active chats are processed.") - - val channel_access_hashes = HashMap() - val channel_names = HashMap() - val channels = LinkedList() - val supergroups = LinkedList() - // TODO Add chat title (and other stuff?) to the database - for (c in dialogs.getChats()) { - if (c is TLChannel) { - channel_access_hashes.put(c.getId(), c.getAccessHash()) - channel_names.put(c.getId(), c.getTitle()) - if (c.getMegagroup()) { - supergroups.add(c.getId()) - } else { - channels.add(c.getId()) - } - // Channel: TLChannel - // Supergroup: getMegagroup()==true - } + + if (IniSettings.download_channels) { + println("Checking channels...") + for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) } } - - - - for (d in dialogs.getDialogs()) { - if (d.getPeer() is TLPeerChannel) { - val channel_id = (d.getPeer() as TLPeerChannel).getChannelId() - - // If this is a channel and we don't want to download channels OR - // it is a supergroups and we don't want to download supergroups, then - if (channels.contains(channel_id) && !IniSettings.download_channels || supergroups.contains(channel_id) && !IniSettings.download_supergroups) { - // Skip this chat. - continue - } - val max_known_id = db!!.getTopMessageIDForChannel(channel_id) - if (d.getTopMessage() > max_known_id) { - val ids = makeIdList(max_known_id + 1, d.getTopMessage()) - val access_hash = channel_access_hashes.get(channel_id) ?: throw RuntimeException("AccessHash for Channel missing.") - var channel_name = channel_names.get(channel_id) - if (channel_name == null) { - channel_name = "?" - } - val channel = TLInputChannel(channel_id, access_hash) - val source_type = if (supergroups.contains(channel_id)) { - MessageSource.SUPERGROUP - } else if (channels.contains(channel_id)) { - MessageSource.CHANNEL - } else { - throw RuntimeException("chat is neither in channels nor in supergroups...") - } - downloadMessages(ids, channel, source_type=source_type, source_name=channel_name) - } - } + + if (IniSettings.download_supergroups) { + println("Checking supergroups...") + for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) } } } } + + private fun downloadMessagesFromChannel(channel: Channel) { + val obj = channel.obj + val max_known_id = db!!.getTopMessageIDForChannel(channel.id) + if (obj.getTopMessage() > max_known_id) { + val ids = makeIdList(max_known_id + 1, obj.getTopMessage()) + var channel_name = channel.title + + val input_channel = TLInputChannel(channel.id, channel.access_hash) + val source_type = channel.message_source + downloadMessages(ids, input_channel, source_type=source_type, source_name=channel_name) + } + } @Throws(RpcErrorException::class, IOException::class) private fun downloadMessages(ids: MutableList, channel: TLInputChannel?, source_type: MessageSource = MessageSource.NORMAL, source_name: String? = null) { @@ -385,6 +349,49 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI for (i in start..end) a.add(i) return a } + + fun getChats(): ChatList { + val cl = ChatList() + logger.trace("Calling messagesGetDialogs") + val dialogs = client!!.messagesGetDialogs(0, 0, TLInputPeerEmpty(), 100) + logger.trace("Got {} dialogs back", dialogs.getDialogs().size) + + // Add dialogs + cl.dialogs.addAll(dialogs.getDialogs().filter{it.getPeer() !is TLPeerChannel}) + + // Add supergoups and channels + for (tl_channel in dialogs.getChats().filter{it is TLChannel}.map{it as TLChannel}) { + val tl_peer_channel = dialogs.getDialogs().find{var p = it.getPeer() ; p is TLPeerChannel && p.getChannelId()==tl_channel.getId()} + + if (tl_peer_channel == null) continue + + var download = true + if (IniSettings.whitelist_channels != null) { + download = IniSettings.whitelist_channels!!.contains(tl_channel.getId().toString()) + } else if (IniSettings.blacklist_channels != null) { + download = !IniSettings.blacklist_channels!!.contains(tl_channel.getId().toString()) + } + val channel = Channel(id=tl_channel.getId(), access_hash=tl_channel.getAccessHash(), title=tl_channel.getTitle(), obj=tl_peer_channel, download=download) + if (tl_channel.getMegagroup()) { + channel.message_source = MessageSource.SUPERGROUP + cl.supergroups.add(channel) + } else { + channel.message_source = MessageSource.CHANNEL + cl.channels.add(channel) + } + } + return cl + } + + class ChatList { + val dialogs = mutableListOf() + val supergroups = mutableListOf() + val channels = mutableListOf() + } + + class Channel(val id: Int, val access_hash: Long, val title: String, val obj: TLDialog, val download: Boolean) { + lateinit var message_source: MessageSource + } companion object { internal var download_client: TelegramClient? = null diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt index 8a37ef1..462793c 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -55,8 +55,13 @@ object IniSettings { fun println() = println(settings) fun get(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default + fun getStringList(key: String): List? = settings.get(key) fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default - fun getBoolean(key: String, default: Boolean = false): Boolean = settings.get(key)?.last() == "true" + fun getBoolean(key: String, default: Boolean = false): Boolean { + val value = settings.get(key)?.last() + if (value==null) return default + return value=="true" + } fun getArray(key: String): List = settings.get(key) ?: listOf() val gmaps_key: String @@ -71,4 +76,8 @@ object IniSettings { get() = getBoolean("download_channels", default=false) val download_supergroups: Boolean get() = getBoolean("download_supergroups", default=false) + val whitelist_channels: List? + get() = getStringList("whitelist_channels") + val blacklist_channels: List? + get() = getStringList("blacklist_channels") } diff --git a/src/main/resources/config.sample.ini b/src/main/resources/config.sample.ini index d7606ea..38e9c4c 100644 --- a/src/main/resources/config.sample.ini +++ b/src/main/resources/config.sample.ini @@ -18,6 +18,43 @@ ## Download media files # download_media = true -## Don't download supergroups and channels by default + + +## Downloads of channels and supergroups +## Here you can specify which channels and supergroups +## should be downloaded. The rules for this are: +## 1. Channels and supergroups are NEVER downloaded unless download_channels and/or +## download_supergroups is set to true. +## 2. If there is at least one entry called whitelist_channels, ONLY channels and/or +## supergroups that are listed in the whitelist will be downloaded. +## 3. Only if there are NO channels whitelisted, entrys listed as blacklist_channels +## will not be downloaded, all other channels / supergroups will be. +## +## In other words: +## * Set download_channels and/or download_supergroups to true if you want to include +## those types in your backup. +## * If you use neither black- nor whitelist, all channels (if you set download_channels +## to true) / supergroups (if you set download_supergroups to true) get downloaded. +## * If you set a whitelist, only listed channels / supergroups (there is no distinction +## made here) will be loaded. +## * If you set a blacklist, everything except the listed channels / supergroups (again, +## although the entry is called whitelist_channels it affects channels AND supergroups) +## will be loaded. +## * If you set a whitelist AND a blacklist, the blacklist will be ignored. +## +## Call the app with `--list-channels` to list the available channels and supergroups +## with their IDs. That list will also tell you if a channel / supergroup will be +## be downloaded according to your black- and whitelists. +## +## You can have more than one whitelist_channels and/or blacklist_channels entries +## to build your list. One ID per entry. + # download_channels = false # download_supergroups = false + +# blacklist_channels = 12345678 +# blacklist_channels = 8886542 +# blacklist_channels = 715952 + +# whitelist_channels = 238572935 +# whitelist_channels = 23857623 From dd99612bedc0f17eb96af97c896823a4ea801c62 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 13 Mar 2018 06:31:38 +0100 Subject: [PATCH 21/24] Added command line switch `--list-channels` to list channels and supergroups with their ID to add them to black- and whitelists in config.ini --- .../telegram_backup/CommandLineController.kt | 26 +++++++++++++++++++ .../telegram_backup/CommandLineOptions.kt | 2 ++ 2 files changed, 28 insertions(+) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 4d82291..5fbba7f 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -135,6 +135,31 @@ class CommandLineController { } logger.info("Initializing Download Manager") val d = DownloadManager(client, CommandLineDownloadProgress()) + + if (CommandLineOptions.cmd_list_channels) { + val chats = d.getChats() + val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) } + val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")} + var download: Boolean + + println("Channels:") + download = IniSettings.download_channels + if (!download) println("Download of channels is disabled - see download_channels in config.ini") + print_header(download) + for (c in chats.channels) { + println(format(c, download)) + } + println() + println("Supergroups:") + download = IniSettings.download_supergroups + if (!download) println("Download of supergroups is disabled - see download_supergroups in config.ini") + print_header(download) + for (c in chats.supergroups) { + println(format(c, download)) + } + System.exit(0) + } + logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages) d.downloadMessages(CommandLineOptions.val_limit_messages) logger.debug("IniSettings.download_media: {}", IniSettings.download_media) @@ -283,6 +308,7 @@ class CommandLineController { println(" --license Displays the license of this program.") println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.") println(" --stats Print some usage statistics.") + println(" --list-channels Lists all channels together with their ID") } private fun list_accounts() { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index e06dd53..0997cf4 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -28,6 +28,7 @@ internal object CommandLineOptions { var cmd_daemon = false var cmd_anonymize = false var cmd_stats = false + var cmd_list_channels = false var val_account: String? = null var val_limit_messages: Int? = null var val_target: String? = null @@ -85,6 +86,7 @@ internal object CommandLineOptions { } "--anonymize" -> cmd_anonymize = true "--stats" -> cmd_stats = true + "--list-channels" -> cmd_list_channels = true else -> throw RuntimeException("Unknown command " + arg) } } From 2295ced528e412d329a0db7b538f2d17193f5b87 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 13 Mar 2018 06:42:39 +0100 Subject: [PATCH 22/24] Fixed a bug in IniSettings that prevented new accounts from being created. --- .../kotlin/de/fabianonline/telegram_backup/IniSettings.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt index 462793c..365173b 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -9,8 +9,10 @@ object IniSettings { var settings = mutableMapOf>() init { - loadIni(UserManager.getInstance().fileBase + "config.ini") - copySampleIni(UserManager.getInstance().fileBase + "config.sample.ini") + if (UserManager.getInstance().user != null) { + loadIni(UserManager.getInstance().fileBase + "config.ini") + copySampleIni(UserManager.getInstance().fileBase + "config.sample.ini") + } } // Dummy function that can be called to force this object to run its init-code. From f8984b25b11d1b1ee4f2ce1cc1770fdd2d1e14bb Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 13 Mar 2018 06:44:33 +0100 Subject: [PATCH 23/24] Renamed IniSettings#get to IniSettings#getString. --- .../kotlin/de/fabianonline/telegram_backup/IniSettings.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt index 365173b..5c6e962 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -56,7 +56,7 @@ object IniSettings { fun println() = println(settings) - fun get(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default + fun getString(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default fun getStringList(key: String): List? = settings.get(key) fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default fun getBoolean(key: String, default: Boolean = false): Boolean { @@ -67,7 +67,7 @@ object IniSettings { fun getArray(key: String): List = settings.get(key) ?: listOf() val gmaps_key: String - get() = get("gmaps_key", default=Config.SECRET_GMAPS)!! + get() = getString("gmaps_key", default=Config.SECRET_GMAPS)!! val pagination: Boolean get() = getBoolean("pagination", default=true) val pagination_size: Int From 6276651b84be4646150fd54c0bf46c5636728f86 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 13 Mar 2018 06:46:59 +0100 Subject: [PATCH 24/24] Fixed anonymization of database backup debug messages. Fixes #98. --- src/main/kotlin/de/fabianonline/telegram_backup/Database.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index 0a7d839..cef1f85 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -238,7 +238,7 @@ class Database private constructor(var client: TelegramClient) { try { val src = user_manager.fileBase + Config.FILE_NAME_DB val dst = user_manager.fileBase + filename - logger.debug("Copying {} to {}", src, dst) + logger.debug("Copying {} to {}", src.anonymize(), dst.anonymize()) Files.copy( File(src).toPath(), File(dst).toPath())