From 253b334fc3bca384ab66abebbd73bfa6516f07e3 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Fri, 6 Apr 2018 06:19:49 +0200 Subject: [PATCH] More rewriting. Also lots of fun with git stash and wrong branches... --- gradle/wrapper/gradle-wrapper.properties | 2 +- .../telegram_backup/CommandLineController.kt | 68 ++++---- .../telegram_backup/CommandLineOptions.kt | 89 +++-------- .../telegram_backup/CommandLineRunner.kt | 5 + .../fabianonline/telegram_backup/Database.kt | 12 +- .../fabianonline/telegram_backup/Settings.kt | 151 ++++++++++++++++++ .../mediafilemanager/FileManagerFactory.kt | 20 +-- 7 files changed, 225 insertions(+), 122 deletions(-) create mode 100644 src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index b72ab3d..5bdf9af 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,4 +1,4 @@ -#Thu Oct 06 11:24:39 CEST 2016 +#Fri Mar 23 06:07:28 CET 2018 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 95fe9a3..47a9929 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -42,16 +42,17 @@ class CommandLineController(val options: CommandLineOptions) { var client: TelegramClient val user_manager: UserManager val inisettings: IniSettings + val settings: Settings val database: Database logger.info("CommandLineController started. App version {}", Config.APP_APPVER) printHeader() - if (options.cmd_version) { + if (options.booleans.contains("version")) { System.exit(0) - } else if (options.cmd_help) { + } else if (options.booleans.contains("help")) { show_help() System.exit(0) - } else if (options.cmd_license) { + } else if (options.booleans.contains("license")) { show_license() System.exit(0) } @@ -62,26 +63,26 @@ class CommandLineController(val options: CommandLineOptions) { // Setup file_base logger.debug("Target dir from Config: {}", Config.TARGET_DIR.anonymize()) - target_dir = options.val_target ?: Config.TARGET_DIR + target_dir = options.values.get("target")?.last() ?: Config.TARGET_DIR logger.debug("Target dir after options: {}", target_dir) println("Base directory for files: ${target_dir.anonymize()}") - if (options.cmd_list_accounts) { + if (options.booleans.contains("list_accounts")) { Utils.print_accounts(target_dir) System.exit(0) } - if (options.cmd_login) { - cmd_login(app, target_dir, options.val_account) + if (options.booleans.contains("login")) { + cmd_login(app, target_dir, options.values.get("account")?.last()) } logger.trace("Checking accounts") - phone_number = try { selectAccount(target_dir, options.val_account) + phone_number = try { selectAccount(target_dir, options.values.get("account")?.last()) } catch(e: AccountNotFoundException) { show_error("The specified account could not be found.") } catch(e: NoAccountsException) { println("No accounts found. Starting login process...") - cmd_login(app, target_dir, options.val_account) + cmd_login(app, target_dir, options.values.get("account")?.last()) } // TODO: Create a new TelegramApp if the user set his/her own TelegramApp credentials @@ -112,33 +113,22 @@ class CommandLineController(val options: CommandLineOptions) { show_error("Account / User mismatch") } - // Load the ini file. - inisettings = IniSettings(file_base) - // If we reach this point, we can assume that there is an account and a database can be loaded / created. database = Database(file_base, user_manager) - if (options.cmd_stats) { + // Load the settings and stuff. + settings = Settings(file_base, database, options) + + if (options.booleans.contains("stats")) { cmd_stats(file_base, database) System.exit(0) } - if (options.val_test != null) { - if (options.val_test == 1) { - TestFeatures(database).test1() - } else if (options.val_test == 2) { - TestFeatures(database).test2() - } else { - System.out.println("Unknown test " + options.val_test) - } - System.exit(1) - } - - val export = options.val_export + val export = options.values.get("export")?.last() logger.debug("options.val_export: {}", export) if (export != null) { - if (export.toLowerCase().equals("html")) { - HTMLExporter(database, user_manager, ini=inisettings, file_base=file_base).export() + if (export.toLowerCase() == "html") { + HTMLExporter(database, user_manager, settings=settings, file_base=file_base).export() System.exit(0) } else { show_error("Unknown export format '${export}'.") @@ -150,7 +140,7 @@ class CommandLineController(val options: CommandLineOptions) { logger.info("Initializing Download Manager") val d = DownloadManager(client, CommandLineDownloadProgress(), database, user_manager, inisettings) - if (options.cmd_list_channels) { + if (options.booleans.contains("list_channels")) { val chats = d.getChats() val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) } val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")} @@ -174,8 +164,8 @@ class CommandLineController(val options: CommandLineOptions) { System.exit(0) } - logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.val_limit_messages) - d.downloadMessages(options.val_limit_messages) + logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.values.get("limit_messages")?.last()) + d.downloadMessages(options.values.get("limit_messages")?.last()?.toInt()) logger.debug("IniSettings#download_media: {}", inisettings.download_media) if (inisettings.download_media) { logger.debug("Calling DownloadManager.downloadMedia") @@ -184,7 +174,7 @@ class CommandLineController(val options: CommandLineOptions) { println("Skipping media download because download_media is set to false.") } - if (options.cmd_daemon) { + if (options.boolean.contains("daemon")) { logger.info("Initializing TelegramUpdateHandler") handler = TelegramUpdateHandler(user_manager, database) client.close() @@ -193,7 +183,7 @@ class CommandLineController(val options: CommandLineOptions) { println("DAEMON mode requested - keeping running.") } } catch (e: Throwable) { - println("An error occured!") + println("An error occurred!") e.printStackTrace() logger.error("Exception caught!", e) } finally { @@ -274,19 +264,19 @@ class CommandLineController(val options: CommandLineOptions) { private fun show_help() { println("Valid options are:") - println(" -h, --help Shows this help.") - println(" -a, --account Use account .") - println(" -l, --login Login to an existing telegram account.") + println(" --help Shows this help.") + println(" --account Use account .") + println(" --login Login to an existing telegram account.") println(" --debug Shows some debug information.") println(" --trace Shows lots of debug information. Overrides --debug.") println(" --trace-telegram Shows lots of debug messages from the library used to access Telegram.") - println(" -A, --list-accounts List all existing accounts ") + println(" --list-accounts List all existing accounts ") println(" --limit-messages Downloads at most the most recent messages.") - println(" -t, --target Target directory for the files.") - println(" -e, --export Export the database. Valid formats are:") + println(" --target Target directory for the files.") + println(" --export Export the database. Valid formats are:") println(" html - Creates HTML files.") println(" --license Displays the license of this program.") - println(" -d, --daemon Keep running after the backup and automatically save new messages.") + println(" --daemon Keep running after the backup and automatically save new messages.") println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.") println(" --stats Print some usage statistics.") println(" --list-channels Lists all channels together with their ID") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index 072d486..7808202 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -16,81 +16,28 @@ package de.fabianonline.telegram_backup class CommandLineOptions(args: Array) { - var cmd_console = false - var cmd_help = false - var cmd_login = false - var cmd_debug = false - var cmd_trace = false - var cmd_trace_telegram = false - var cmd_list_accounts = false - var cmd_version = false - var cmd_license = false - var cmd_daemon = false - var cmd_stats = false - var cmd_list_channels = false - var val_account: String? = null - var val_limit_messages: Int? = null - var val_target: String? = null - var val_export: String? = null - var val_test: Int? = null - + val booleans = mutableListOf() + val values = mutableMapOf>() + var last_key: String? = null + init { - var last_cmd: String? = null - loop@ for (arg in args) { - if (last_cmd != null) { - when (last_cmd) { - "--account" -> val_account = arg - "--limit-messages" -> val_limit_messages = Integer.parseInt(arg) - "--target" -> val_target = arg - "--export" -> val_export = arg - "--test" -> val_test = Integer.parseInt(arg) + for(arg in args) { + if (arg.starts_with("--")) { + if (last_key!=null) { + booleans.add(last_key) } - last_cmd = null - continue - } - when (arg) { - "-a", "--account" -> { - last_cmd = "--account" - continue@loop + last_key = arg.substr(2) + } else { + if (last_key==null) throw RuntimeException("Unexpected parameter without switch: $arg") + var list = values.get(last_key) + if (list==null) { + list = mutableListOf() + values.add(last_key, list) } - "-h", "--help" -> cmd_help = true - "-l", "--login" -> cmd_login = true - "--debug" -> cmd_debug = true - "--trace" -> cmd_trace = true - "--trace-telegram" -> cmd_trace_telegram = true - "-A", "--list-accounts" -> cmd_list_accounts = true - "--limit-messages" -> { - last_cmd = arg - continue@loop - } - "--console" -> cmd_console = true - "-t", "--target" -> { - last_cmd = "--target" - continue@loop - } - "-V", "--version" -> cmd_version = true - "-e", "--export" -> { - last_cmd = "--export" - continue@loop - } - "--pagination" -> { - last_cmd = "--pagination" - continue@loop - } - "--license" -> cmd_license = true - "-d", "--daemon" -> cmd_daemon = true - "--test" -> { - last_cmd = "--test" - continue@loop - } - "--anonymize" -> Utils.anonymize = true - "--stats" -> cmd_stats = true - "--list-channels" -> cmd_list_channels = true - else -> throw RuntimeException("Unknown command " + arg) + list.add(arg) + last_key = null } } - if (last_cmd != null) { - CommandLineController.show_error("Command $last_cmd had no parameter set.") - } + if (last_key!=null) booleans.add(last_key) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt index 2f6d334..5728c32 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt @@ -53,6 +53,11 @@ class CommandLineRunner(args: Array) { } fun setupLogging() { + if (options.booleans.hasKey('anonymize')) { + Utils.anonymize = true + } + + logger.trace("Setting up Loggers...") val rootLogger = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) as Logger val rootContext = rootLogger.getLoggerContext() diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index c09e5bf..b71cb2c 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -240,6 +240,16 @@ class Database constructor(val file_base: String, val user_manager: UserManager) rs.close() return result } + + fun queryStringMap(query: String): Map { + val map = mutableMapOf() + val rs = stmt.executeQuery(query) + while(rs.next()) { + map.add(rs.getString(1), rs.getString(2)) + } + rs.close() + return map + } @Synchronized fun saveMessages(all: TLVector, api_layer: Int, source_type: MessageSource = MessageSource.NORMAL) { @@ -489,7 +499,7 @@ class Database constructor(val file_base: String, val user_manager: UserManager) ps_insert_or_replace.close() } - fun fetchSetting(key: String): String? = queryString("SELECT value FROM settings WHERE key='${key}'") + fun fetchSettings() = queryStringMap("SELECT key, value FROM settings WHERE key='${key}'") fun saveSetting(key: String, value: String?) { val ps = conn.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt new file mode 100644 index 0000000..9cce5eb --- /dev/null +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt @@ -0,0 +1,151 @@ +package de.fabianonline.telegram_backup + +import java.io.File +import org.slf4j.LoggerFactory + +class Settings(val file_base: String, val database: Database, val cli_settings: CommandLineOptions) { + val logger = LoggerFactory.getLogger(Settings::class.java) + val settings = mutableMapOf() + + val gmaps_key: Setting + + private val db_settings: Map + + private var ini_settings: Map> + + init { + db_settings = database.fetchSettings() + ini_settings = load_ini("config.ini") + copy_sample_ini("config.sample.ini") + + // Merging CLI and INI settings + gmaps_key = get_setting("gmaps_key", default=Config.SECRET_GMAPS) + } + + private fun get_setting(name: String, default: String? = null): Setting { + return Setting.build(name, ini_settings[name], cli_settings.values[name], default) + } + + private fun load_ini(filename: String): Map> { + val map = mutableMapOf>() + val file = File(file_base + filename) + logger.trace("Checking ini file {}", filename.anonymize()) + if (!file.exists()) return map + logger.debug("Loading ini file {}", filename.anonymize()) + file.forEachLine { parseLine(it, map) } + return map + } + + private fun parseLine(original_line: String, map: MutableMap>) { + logger.trace("Parsing line: {}", original_line) + var line = original_line.trim().replaceAfter("#", "").removeSuffix("#") + logger.trace("After cleaning: {}", line) + if (line == "") return + val parts: List = line.split("=", limit=2).map{it.trim()} + + if (parts.size < 2) throw RuntimeException("Invalid config setting: $line") + + val (key, value) = parts + if (value=="") { + map.remove(key) + } else { + var list = map.get(key) + if (list == null) { + list = mutableListOf() + map.put(key, list) + } + list.add(value) + } + } + + private fun copy_sample_ini(filename: String) { + val stream = Config::class.java.getResourceAsStream("/config.sample.ini") + File(filename).outputStream().use { stream.copyTo(it) } + stream.close() + } + + + open class Setting(name: String, value: List?, source: SettingSource) { + companion object { + val all = mutableListOf() + + fun build(name: String, ini_value: List?, cli_value: List?, default: String?): Setting { + var value: List? + var source: SettingSource + if (cli_value != null) { + value=cli_value + source=SettingSource.CLI + } else if (ini_value != null) { + value=ini_value + source=SettingSource.INI + } else { + if (default!=null) { + value = listOf(default) + } else { + value = null + } + source=SettingSource.DEFAULT + } + return Setting(name, value, source); + } + } + } + + enum class SettingSource { + INI, + CLI, + DEFAULT + } +} +/* +class DbSettings(val database: Database) { + private fun fetchValue(name: String): String? = database.fetchSetting(name) + private fun saveValue(name: String, value: String?) = database.saveSetting(name, value) + + var pts: String? + get() = fetchValue("pts") + set(x: String?) = saveValue("pts", x) +} + + +package de.fabianonline.telegram_backup + + + +class Settings(val file_base: String) { + val logger = LoggerFactory.getLogger(Settings::class.java) + var settings = mutableMapOf>() + + init { + loadIni(file_base + "config.ini") + copySampleIni(file_base + "config.sample.ini") + } + + + + + + + + fun println() = println(settings) + + fun getString(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default + fun getStringList(key: String): List? = settings.get(key) + fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default + fun getBoolean(key: String, default: Boolean = false): Boolean { + val value = settings.get(key)?.last() + if (value==null) return default + return value=="true" + } + fun getArray(key: String): List = settings.get(key) ?: listOf() + + val gmaps_key = getString("gmaps_key", default=Config.SECRET_GMAPS)!! + val pagination = getBoolean("pagination", default=true) + val pagination_size = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!! + val download_media = getBoolean("download_media", default=true) + val download_channels = getBoolean("download_channels", default=false) + val download_supergroups = getBoolean("download_supergroups", default=false) + val whitelist_channels = getStringList("whitelist_channels") + val blacklist_channels = getStringList("blacklist_channels") +} +*/ diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt index 568f617..28e4154 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt @@ -41,29 +41,29 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils object FileManagerFactory { - fun getFileManager(m: TLMessage?, u: UserManager): AbstractMediaFileManager? { + fun getFileManager(m: TLMessage?, u: UserManager, c: TelegramClient): AbstractMediaFileManager? { if (m == null) return null val media = m.getMedia() ?: return null if (media is TLMessageMediaPhoto) { - return PhotoFileManager(m, u) + return PhotoFileManager(m, u, file_base) } else if (media is TLMessageMediaDocument) { - val d = DocumentFileManager(m, u) + val d = DocumentFileManager(m, u, file_base) return if (d.isSticker) { - StickerFileManager(m, u) + StickerFileManager(m, u, file_base) } else d } else if (media is TLMessageMediaGeo) { - return GeoFileManager(m, u) + return GeoFileManager(m, u, file_base) } else if (media is TLMessageMediaEmpty) { - return UnsupportedFileManager(m, u, "empty") + return UnsupportedFileManager(m, u, file_base, "empty") } else if (media is TLMessageMediaUnsupported) { - return UnsupportedFileManager(m, u, "unsupported") + return UnsupportedFileManager(m, u, file_base, "unsupported") } else if (media is TLMessageMediaWebPage) { - return UnsupportedFileManager(m, u, "webpage") + return UnsupportedFileManager(m, u, file_base, "webpage") } else if (media is TLMessageMediaContact) { - return UnsupportedFileManager(m, u, "contact") + return UnsupportedFileManager(m, u, file_base, "contact") } else if (media is TLMessageMediaVenue) { - return UnsupportedFileManager(m, u, "venue") + return UnsupportedFileManager(m, u, file_base, "venue") } else { AbstractMediaFileManager.throwUnexpectedObjectError(media) }