From 97cf26b46d8893f1438aa64aac8dbaf529827444 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Wed, 14 Mar 2018 06:08:07 +0100 Subject: [PATCH 01/16] WIP: Started rewriting the code to make it more null-safe. --- .../fabianonline/telegram_backup/Account.kt | 5 + .../telegram_backup/CommandLineController.kt | 107 ++++++++-------- .../de/fabianonline/telegram_backup/Config.kt | 2 +- .../fabianonline/telegram_backup/Database.kt | 115 ++++++++---------- .../de/fabianonline/telegram_backup/Utils.kt | 18 ++- 5 files changed, 125 insertions(+), 122 deletions(-) create mode 100644 src/main/kotlin/de/fabianonline/telegram_backup/Account.kt diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Account.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Account.kt new file mode 100644 index 0000000..c976c59 --- /dev/null +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Account.kt @@ -0,0 +1,5 @@ +package de.fabianonline.telegram_backup + +class Account(val file_base: String, val phone_number: String) { + +} \ No newline at end of file diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 7e15981..5fcc149 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -31,7 +31,11 @@ import org.slf4j.Logger class CommandLineController { private val storage: ApiStorage - var app: TelegramApp + val app: TelegramApp + val target_dir: String + val file_base: String + val phone_number: String + val account_file_base: String private fun getLine(): String { if (System.console() != null) { @@ -53,25 +57,45 @@ class CommandLineController { init { logger.info("CommandLineController started. App version {}", Config.APP_APPVER) - this.printHeader() + printHeader() if (CommandLineOptions.cmd_version) { System.exit(0) } else if (CommandLineOptions.cmd_help) { - this.show_help() + show_help() System.exit(0) } else if (CommandLineOptions.cmd_license) { - CommandLineController.show_license() + show_license() System.exit(0) } - this.setupFileBase() + + // Setup file_base + logger.debug("Target dir from Config: {}", Config.TARGET_DIR.anonymize()) + target_dir = CommandLineOptions.val_target ?: Config.TARGET_DIR + logger.debug("Target dir after options: {}", target_dir) + println("Base directory for files: ${target_dir.anonymize()}") + if (CommandLineOptions.cmd_list_accounts) { - this.list_accounts() + Utils.print_accounts(target_dir) System.exit(0) } + + logger.trace("Checking accounts") + try { + phone_number = selectAccount(target_dir, CommandLineOptions.val_account) + } catch(e: AccountNotFoundException) { + show_error("The specified account could not be found.") + } catch(e: NoAccountsException) { + println("No accounts found. Starting login process...") + CommandLineOptions.cmd_login = true + } + + file_base = target_dir + File.separatorChar + account_to_use + + account = Account(file_base, account_to_use) + logger.debug("Initializing TelegramApp") app = TelegramApp(Config.APP_ID, Config.APP_HASH, Config.APP_MODEL, Config.APP_SYSVER, Config.APP_APPVER, Config.APP_LANG) - logger.trace("Checking accounts") - val account = this.selectAccount() + logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login) logger.info("Initializing ApiStorage") storage = ApiStorage(account) @@ -196,44 +220,22 @@ class CommandLineController { println() } - private fun setupFileBase() { - logger.debug("Target dir at startup: {}", Config.FILE_BASE.anonymize()) - if (CommandLineOptions.val_target != null) { - Config.FILE_BASE = CommandLineOptions.val_target!! - } - logger.debug("Target dir after options: {}", Config.FILE_BASE.anonymize()) - System.out.println("Base directory for files: " + Config.FILE_BASE.anonymize()) - } - - private fun selectAccount(): String? { - var account = "none" - val accounts = Utils.getAccounts() - if (CommandLineOptions.cmd_login) { - logger.debug("Login requested, doing nothing.") - // do nothing - } else if (CommandLineOptions.val_account != null) { - logger.debug("Account requested: {}", CommandLineOptions.val_account!!.anonymize()) + private fun selectAccount(file_base: String, requested_account: String?): String { + val found_account: String? + val accounts = Utils.getAccounts(file_base) + if (requested_account != null) { + logger.debug("Account requested: {}", requested_account.anonymize()) logger.trace("Checking accounts for match.") - var found = false - for (acc in accounts) { - logger.trace("Checking {}", acc.anonymize()) - if (acc == CommandLineOptions.val_account) { - found = true - logger.trace("Matches.") - break - } + found_account = accounts.find{it == requested_account} + + if (found_account == null) { + throw AccountNotFoundException() } - if (!found) { - show_error("Couldn't find account '" + CommandLineOptions.val_account!!.anonymize() + "'. Maybe you want to use '--login' first?") - } - account = CommandLineOptions.val_account!! } else if (accounts.size == 0) { - println("No accounts found. Starting login process...") - CommandLineOptions.cmd_login = true - return null + throw NoAccountsException() } else if (accounts.size == 1) { - account = accounts.firstElement() - System.out.println("Using only available account: " + account.anonymize()) + found_account = accounts.firstElement() + println("Using only available account: " + account.anonymize()) } else { show_error(("You didn't specify which account to use.\n" + "Use '--account ' to use account .\n" + @@ -241,8 +243,8 @@ class CommandLineController { System.exit(1) } logger.debug("accounts.size: {}", accounts.size) - logger.debug("account: {}", account.anonymize()) - return account + logger.debug("account: {}", found_account.anonymize()) + return found_account!! } private fun cmd_stats() { @@ -315,20 +317,6 @@ class CommandLineController { println(" --list-channels Lists all channels together with their ID") } - private fun list_accounts() { - println("List of available accounts:") - val accounts = Utils.getAccounts() - if (accounts.size > 0) { - for (str in accounts) { - System.out.println(" " + str.anonymize()) - } - println("Use '--account ' to use one of those accounts.") - } else { - println("NO ACCOUNTS FOUND") - println("Use '--login' to login to a telegram account.") - } - } - companion object { private val logger = LoggerFactory.getLogger(CommandLineController::class.java) @@ -342,4 +330,7 @@ class CommandLineController { println("TODO: Print the GPL.") } } + + class AccountNotFoundException() : Exception("Account not found") {} + class NoAccountsException() : Exception("No accounts found") {} } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt index 8be0bc0..26bba17 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt @@ -29,7 +29,7 @@ object Config { val APP_APPVER: String val APP_LANG = "en" - var FILE_BASE = System.getProperty("user.home") + File.separatorChar + ".telegram_backup" + var TARGET_DIR = System.getProperty("user.home") + File.separatorChar + ".telegram_backup" val FILE_NAME_AUTH_KEY = "auth.dat" val FILE_NAME_DC = "dc.dat" val FILE_NAME_DB = "database.sqlite" diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index cef1f85..4d5adeb 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -51,6 +51,33 @@ class Database private constructor(var client: TelegramClient) { private var conn: Connection? = null private var stmt: Statement? = null var user_manager: UserManager + + init { + private val logger = LoggerFactory.getLogger(Database::class.java) + + this.user_manager = UserManager.getInstance() + System.out.println("Opening database...") + try { + Class.forName("org.sqlite.JDBC") + } catch (e: ClassNotFoundException) { + CommandLineController.show_error("Could not load jdbc-sqlite class.") + } + + val path = "jdbc:sqlite:${user_manager.fileBase}${Config.FILE_NAME_DB}" + + try { + conn = DriverManager.getConnection(path) + stmt = conn!!.createStatement() + } catch (e: SQLException) { + CommandLineController.show_error("Could not connect to SQLITE database.") + } + + // Run updates + val updates = DatabaseUpdates(conn!!, this) + updates.doUpdates() + + System.out.println("Database is ready.") + } fun getTopMessageID(): Int { try { @@ -69,34 +96,32 @@ class Database private constructor(var client: TelegramClient) { fun getChatCount(): Int = queryInt("SELECT COUNT(*) FROM chats") fun getUserCount(): Int = queryInt("SELECT COUNT(*) FROM users") - val missingIDs: LinkedList - get() { - try { - val missing = LinkedList() - val max = getTopMessageID() - val rs = stmt!!.executeQuery("SELECT message_id FROM messages WHERE source_type IN ('group', 'dialog') ORDER BY id") - rs.next() - var id = rs.getInt(1) - for (i in 1..max) { - if (i == id) { - rs.next() - if (rs.isClosed()) { - id = Integer.MAX_VALUE - } else { - id = rs.getInt(1) - } - } else if (i < id) { - missing.add(i) + fun getMissingIDs: LinkedList() + try { + val missing = LinkedList() + val max = getTopMessageID() + val rs = stmt!!.executeQuery("SELECT message_id FROM messages WHERE source_type IN ('group', 'dialog') ORDER BY id") + rs.next() + var id = rs.getInt(1) + for (i in 1..max) { + if (i == id) { + rs.next() + if (rs.isClosed()) { + id = Integer.MAX_VALUE + } else { + id = rs.getInt(1) } + } else if (i < id) { + missing.add(i) } - rs.close() - return missing - } catch (e: SQLException) { - e.printStackTrace() - throw RuntimeException("Could not get list of ids.") } - + rs.close() + return missing + } catch (e: SQLException) { + e.printStackTrace() + throw RuntimeException("Could not get list of ids.") } + } fun getMessagesWithMedia(): LinkedList { try { @@ -207,30 +232,7 @@ class Database private constructor(var client: TelegramClient) { } - init { - this.user_manager = UserManager.getInstance() - System.out.println("Opening database...") - try { - Class.forName("org.sqlite.JDBC") - } catch (e: ClassNotFoundException) { - CommandLineController.show_error("Could not load jdbc-sqlite class.") - } - - val path = "jdbc:sqlite:${user_manager.fileBase}${Config.FILE_NAME_DB}" - - try { - conn = DriverManager.getConnection(path) - stmt = conn!!.createStatement() - } catch (e: SQLException) { - CommandLineController.show_error("Could not connect to SQLITE database.") - } - - // Run updates - val updates = DatabaseUpdates(conn!!, this) - updates.doUpdates() - - System.out.println("Database is ready.") - } + fun backupDatabase(currentVersion: Int) { val filename = String.format(Config.FILE_NAME_DB_BACKUP, currentVersion) @@ -550,7 +552,9 @@ class Database private constructor(var client: TelegramClient) { fun fetchSetting(key: String): String? { val rs = stmt!!.executeQuery("SELECT value FROM settings WHERE key='${key}'") rs.next() - return rs.getString(1) + val result = rs.getString(1) + rs.close() + return result } fun saveSetting(key: String, value: String?) { @@ -562,6 +566,7 @@ class Database private constructor(var client: TelegramClient) { ps.setString(2, value) } ps.execute() + ps.close() } fun getIdsFromQuery(query: String): LinkedList { @@ -787,18 +792,6 @@ class Database private constructor(var client: TelegramClient) { } companion object { - private val logger = LoggerFactory.getLogger(Database::class.java) - internal var instance: Database? = null - - fun init(c: TelegramClient) { - instance = Database(c) - } - - fun getInstance(): Database { - if (instance == null) throw RuntimeException("Database is not initialized but getInstance() was called.") - return instance!! - } - fun bytesToTLMessage(b: ByteArray?): TLMessage? { try { if (b == null) return null diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt index 3b99292..00de1d9 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt @@ -34,9 +34,23 @@ object Utils { private val logger = LoggerFactory.getLogger(Utils::class.java) as Logger - fun getAccounts(): Vector { + fun print_accounts(file_base: String) { + println("List of available accounts:") + val accounts = getAccounts(file_base) + if (accounts.size > 0) { + for (str in accounts) { + println(" " + str.anonymize()) + } + println("Use '--account ' to use one of those accounts.") + } else { + println("NO ACCOUNTS FOUND") + println("Use '--login' to login to a telegram account.") + } + } + + fun getAccounts(file_base: String): Vector { val accounts = Vector() - val folder = File(Config.FILE_BASE) + val folder = File(file_base) val files = folder.listFiles() if (files != null) for (f in files) { From 2d409352bc3204056e54eb24c9af8b8d2aecf7f7 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Wed, 14 Mar 2018 06:09:24 +0100 Subject: [PATCH 02/16] WIP: Some more rewriting. --- .../telegram_backup/CommandLineController.kt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 5fcc149..442065d 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -78,6 +78,10 @@ class CommandLineController { Utils.print_accounts(target_dir) System.exit(0) } + + if (CommandLineOptions.cmd_login) { + cmd_login(target_dir, CommandLineOptions.val_account) + } logger.trace("Checking accounts") try { @@ -86,7 +90,7 @@ class CommandLineController { show_error("The specified account could not be found.") } catch(e: NoAccountsException) { println("No accounts found. Starting login process...") - CommandLineOptions.cmd_login = true + cmd_login(target_dir, CommandLineOptions.val_account) } file_base = target_dir + File.separatorChar + account_to_use @@ -273,9 +277,7 @@ class CommandLineController { } } - @Throws(RpcErrorException::class, IOException::class) - private fun cmd_login(phoneToUse: String?) { - val user = UserManager.getInstance() + private fun cmd_login(target_dir: String, phoneToUse: String?): Nothing { val phone: String if (phoneToUse == null) { println("Please enter your phone number in international format.") @@ -284,6 +286,7 @@ class CommandLineController { } else { phone = phoneToUse } + user.sendCodeToPhoneNumber(phone) println("Telegram sent you a code. Please enter it here.") val code = getLine() From eea08a55593a6202ad70def76a5c6545629b5073 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Fri, 16 Mar 2018 06:59:18 +0100 Subject: [PATCH 03/16] WIP: Lots and lots of rewriting. --- .../telegram_backup/ApiStorage.kt | 118 +-- .../telegram_backup/CommandLineController.kt | 232 ++--- .../telegram_backup/CommandLineOptions.kt | 12 +- .../telegram_backup/CommandLineRunner.kt | 42 +- .../fabianonline/telegram_backup/Database.kt | 901 ++++++++---------- .../telegram_backup/DatabaseUpdates.kt | 2 +- .../telegram_backup/DbSettings.kt | 6 +- .../telegram_backup/DownloadManager.kt | 39 +- .../telegram_backup/GUIController.kt | 85 -- .../telegram_backup/IniSettings.kt | 35 +- .../telegram_backup/LoginManager.kt | 87 ++ .../telegram_backup/UserManager.kt | 119 +-- .../AbstractMediaFileManager.kt | 2 +- .../mediafilemanager/DocumentFileManager.kt | 2 +- .../mediafilemanager/FileManagerFactory.kt | 20 +- .../mediafilemanager/GeoFileManager.kt | 2 +- .../mediafilemanager/PhotoFileManager.kt | 2 +- .../mediafilemanager/StickerFileManager.kt | 2 +- .../UnsupportedFileManager.kt | 2 +- 19 files changed, 713 insertions(+), 997 deletions(-) delete mode 100644 src/main/kotlin/de/fabianonline/telegram_backup/GUIController.kt create mode 100644 src/main/kotlin/de/fabianonline/telegram_backup/LoginManager.kt diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/ApiStorage.kt b/src/main/kotlin/de/fabianonline/telegram_backup/ApiStorage.kt index ce99e44..0cace86 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/ApiStorage.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/ApiStorage.kt @@ -22,120 +22,64 @@ import com.github.badoualy.telegram.mtproto.auth.AuthKey import com.github.badoualy.telegram.mtproto.model.MTSession import org.apache.commons.io.FileUtils +import org.slf4j.LoggerFactory +import org.slf4j.Logger import java.io.File import java.io.FileNotFoundException import java.io.IOException -internal class ApiStorage(prefix: String?) : TelegramApiStorage { - private var prefix: String? = null - private var do_save = false - private var auth_key: AuthKey? = null - private var dc: DataCenter? = null - private var file_auth_key: File? = null - private var file_dc: File? = null - +internal class ApiStorage(val base_dir: String) : TelegramApiStorage { + var auth_key: AuthKey? = null + var dc: DataCenter? = null + val file_auth_key: File + val file_dc: File + val logger = LoggerFactory.getLogger(ApiStorage::class.java) + + init { - this.setPrefix(prefix) - } - - fun setPrefix(prefix: String?) { - this.prefix = prefix - this.do_save = this.prefix != null - if (this.do_save) { - val base = Config.FILE_BASE + - File.separatorChar + - this.prefix + - File.separatorChar - this.file_auth_key = File(base + Config.FILE_NAME_AUTH_KEY) - this.file_dc = File(base + Config.FILE_NAME_DC) - this._saveAuthKey() - this._saveDc() - } else { - this.file_auth_key = null - this.file_dc = null - } + file_auth_key = File(base_dir + Config.FILE_NAME_AUTH_KEY) + file_dc = File(base_dir + Config.FILE_NAME_DC) } override fun saveAuthKey(authKey: AuthKey) { - this.auth_key = authKey - this._saveAuthKey() - } - - private fun _saveAuthKey() { - if (this.do_save && this.auth_key != null) { - try { - FileUtils.writeByteArrayToFile(this.file_auth_key, this.auth_key!!.key) - } catch (e: IOException) { - e.printStackTrace() - } - - } + FileUtils.writeByteArrayToFile(file_auth_key, authKey.key) } override fun loadAuthKey(): AuthKey? { - if (this.auth_key != null) return this.auth_key - if (this.file_auth_key != null) { - try { - return AuthKey(FileUtils.readFileToByteArray(this.file_auth_key)) - } catch (e: IOException) { - if (e !is FileNotFoundException) e.printStackTrace() - } - + try { + return AuthKey(FileUtils.readFileToByteArray(file_auth_key)) + } catch (e: FileNotFoundException) { + return null } - - return null } override fun saveDc(dataCenter: DataCenter) { - this.dc = dataCenter - this._saveDc() - } - - private fun _saveDc() { - if (this.do_save && this.dc != null) { - try { - FileUtils.write(this.file_dc, this.dc!!.toString()) - } catch (e: IOException) { - e.printStackTrace() - } - - } + FileUtils.write(file_dc, dataCenter.toString()) } override fun loadDc(): DataCenter? { - if (this.dc != null) return this.dc - if (this.file_dc != null) { - try { - val infos = FileUtils.readFileToString(this.file_dc).split(":") - return DataCenter(infos[0], Integer.parseInt(infos[1])) - } catch (e: IOException) { - if (e !is FileNotFoundException) e.printStackTrace() - } - + try { + val infos = FileUtils.readFileToString(this.file_dc).split(":") + return DataCenter(infos[0], Integer.parseInt(infos[1])) + } catch (e: FileNotFoundException) { + return null } - return null } override fun deleteAuthKey() { - if (this.do_save) { - try { - FileUtils.forceDelete(this.file_auth_key) - } catch (e: IOException) { - e.printStackTrace() - } - + try { + FileUtils.forceDelete(file_auth_key) + } catch (e: IOException) { + logger.warn("Exception in deleteAuthKey(): {}", e) } } override fun deleteDc() { - if (this.do_save) { - try { - FileUtils.forceDelete(this.file_dc) - } catch (e: IOException) { - e.printStackTrace() - } - + try { + FileUtils.forceDelete(file_dc) + } catch (e: IOException) { + logger.warn("Exception in deleteDc(): {}", e) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 442065d..25e96fb 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -29,149 +29,136 @@ import java.util.HashMap import org.slf4j.LoggerFactory import org.slf4j.Logger -class CommandLineController { - private val storage: ApiStorage - val app: TelegramApp - val target_dir: String - val file_base: String - val phone_number: String - val account_file_base: String - - private fun getLine(): String { - if (System.console() != null) { - return System.console().readLine("> ") - } else { - print("> ") - return Scanner(System.`in`).nextLine() - } - } - - private fun getPassword(): String { - if (System.console() != null) { - return String(System.console().readPassword("> ")) - } else { - return getLine() - } - } - +class CommandLineController(val options: CommandLineOptions) { init { + val storage: ApiStorage + val app: TelegramApp + val target_dir: String + val file_base: String + val phone_number: String + val handler: TelegramUpdateHandler + val client: TelegramClient + val user_manager: UserManager + val inisettings: IniSettings + val database: Database logger.info("CommandLineController started. App version {}", Config.APP_APPVER) printHeader() - if (CommandLineOptions.cmd_version) { + if (options.cmd_version) { System.exit(0) - } else if (CommandLineOptions.cmd_help) { + } else if (options.cmd_help) { show_help() System.exit(0) - } else if (CommandLineOptions.cmd_license) { + } else if (options.cmd_license) { show_license() System.exit(0) } + + // Setup TelegramApp + logger.debug("Initializing TelegramApp") + app = TelegramApp(Config.APP_ID, Config.APP_HASH, Config.APP_MODEL, Config.APP_SYSVER, Config.APP_APPVER, Config.APP_LANG) // Setup file_base logger.debug("Target dir from Config: {}", Config.TARGET_DIR.anonymize()) - target_dir = CommandLineOptions.val_target ?: Config.TARGET_DIR + target_dir = options.val_target ?: Config.TARGET_DIR logger.debug("Target dir after options: {}", target_dir) println("Base directory for files: ${target_dir.anonymize()}") - if (CommandLineOptions.cmd_list_accounts) { + if (options.cmd_list_accounts) { Utils.print_accounts(target_dir) System.exit(0) } - if (CommandLineOptions.cmd_login) { - cmd_login(target_dir, CommandLineOptions.val_account) + if (options.cmd_login) { + cmd_login(app, target_dir, options.val_account) } logger.trace("Checking accounts") - try { - phone_number = selectAccount(target_dir, CommandLineOptions.val_account) + phone_number = try { selectAccount(target_dir, options.val_account) } catch(e: AccountNotFoundException) { show_error("The specified account could not be found.") } catch(e: NoAccountsException) { println("No accounts found. Starting login process...") - cmd_login(target_dir, CommandLineOptions.val_account) + cmd_login(app, target_dir, options.val_account) } + + // TODO: Create a new TelegramApp if the user set his/her own TelegramApp credentials - file_base = target_dir + File.separatorChar + account_to_use + // At this point we can assume that the selected user account ("phone_number") exists. + // So we can create some objects: + file_base = build_file_base(target_dir, phone_number) - account = Account(file_base, account_to_use) - - logger.debug("Initializing TelegramApp") - app = TelegramApp(Config.APP_ID, Config.APP_HASH, Config.APP_MODEL, Config.APP_SYSVER, Config.APP_APPVER, Config.APP_LANG) - - logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login) logger.info("Initializing ApiStorage") - storage = ApiStorage(account) + storage = ApiStorage(file_base) + logger.info("Initializing TelegramUpdateHandler") - val handler = TelegramUpdateHandler() + handler = TelegramUpdateHandler() + logger.info("Creating Client") - val client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, handler) + client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, handler) + + // From now on we have a new catch-all-block that will terminate it's TelegramClient when an exception happens. try { logger.info("Initializing UserManager") - UserManager.init(client) - val user = UserManager.getInstance() - if (!CommandLineOptions.cmd_login && !user.loggedIn) { + user_manager = UserManager(client) + + // TODO + /*if (!options.cmd_login && !user.loggedIn) { println("Your authorization data is invalid or missing. You will have to login with Telegram again.") - CommandLineOptions.cmd_login = true - } - if (account != null && user.loggedIn) { - if (account != "+" + user.user!!.getPhone()) { - logger.error("Account: {}, user.user!!.getPhone(): +{}", account.anonymize(), user.user!!.getPhone().anonymize()) - throw RuntimeException("Account / User mismatch") - } + options.cmd_login = true + }*/ + + if (phone_number != user_manager.phone) { + logger.error("phone_number: {}, user_manager.phone: {}", phone_number.anonymize(), user_manager.phone.anonymize()) + show_error("Account / User mismatch") } // Load the ini file. - IniSettings.load() + inisettings = IniSettings(file_base) - logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login) - if (CommandLineOptions.cmd_login) { - cmd_login(CommandLineOptions.val_account) - System.exit(0) - } // If we reach this point, we can assume that there is an account and a database can be loaded / created. - Database.init(client) - if (CommandLineOptions.cmd_stats) { - cmd_stats() + database = Database(file_base, user_manager) + + if (options.cmd_stats) { + cmd_stats(file_base, database) System.exit(0) } - if (CommandLineOptions.val_test != null) { - if (CommandLineOptions.val_test == 1) { + + if (options.val_test != null) { + if (options.val_test == 1) { TestFeatures.test1() - } else if (CommandLineOptions.val_test == 2) { + } else if (options.val_test == 2) { TestFeatures.test2() } else { - System.out.println("Unknown test " + CommandLineOptions.val_test) + System.out.println("Unknown test " + options.val_test) } System.exit(1) } - logger.debug("CommandLineOptions.val_export: {}", CommandLineOptions.val_export) - if (CommandLineOptions.val_export != null) { - if (CommandLineOptions.val_export!!.toLowerCase().equals("html")) { - (HTMLExporter()).export() + + val export = options.val_export + logger.debug("options.val_export: {}", export) + if (export != null) { + if (export.toLowerCase().equals("html")) { + HTMLExporter().export() System.exit(0) } else { - show_error("Unknown export format.") + show_error("Unknown export format '${export}'.") } } - if (user.loggedIn) { - System.out.println("You are logged in as ${user.userString.anonymize()}") - } else { - println("You are not logged in.") - System.exit(1) - } - logger.info("Initializing Download Manager") - val d = DownloadManager(client, CommandLineDownloadProgress()) - if (CommandLineOptions.cmd_list_channels) { + println("You are logged in as ${user_manager.toString().anonymize()}") + + logger.info("Initializing Download Manager") + val d = DownloadManager(client, CommandLineDownloadProgress(), database, user_manager, inisettings) + + if (options.cmd_list_channels) { val chats = d.getChats() val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) } val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")} var download: Boolean println("Channels:") - download = IniSettings.download_channels + download = inisettings.download_channels if (!download) println("Download of channels is disabled - see download_channels in config.ini") print_header(download) for (c in chats.channels) { @@ -179,7 +166,7 @@ class CommandLineController { } println() println("Supergroups:") - download = IniSettings.download_supergroups + download = inisettings.download_supergroups if (!download) println("Download of supergroups is disabled - see download_supergroups in config.ini") print_header(download) for (c in chats.supergroups) { @@ -188,10 +175,10 @@ class CommandLineController { System.exit(0) } - logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages) - d.downloadMessages(CommandLineOptions.val_limit_messages) - logger.debug("IniSettings.download_media: {}", IniSettings.download_media) - if (IniSettings.download_media) { + logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.val_limit_messages) + d.downloadMessages(options.val_limit_messages) + logger.debug("IniSettings#download_media: {}", inisettings.download_media) + if (inisettings.download_media) { logger.debug("Calling DownloadManager.downloadMedia") d.downloadMedia() } else { @@ -202,9 +189,9 @@ class CommandLineController { e.printStackTrace() logger.error("Exception caught!", e) // If we encountered an exception, we definitely don't want to start the daemon mode now. - CommandLineOptions.cmd_daemon = false + options.cmd_daemon = false } finally { - if (CommandLineOptions.cmd_daemon) { + if (options.cmd_daemon) { handler.activate() println("DAEMON mode requested - keeping running.") } else { @@ -225,79 +212,63 @@ class CommandLineController { } private fun selectAccount(file_base: String, requested_account: String?): String { - val found_account: String? + var found_account: String? = null val accounts = Utils.getAccounts(file_base) if (requested_account != null) { logger.debug("Account requested: {}", requested_account.anonymize()) logger.trace("Checking accounts for match.") found_account = accounts.find{it == requested_account} - - if (found_account == null) { - throw AccountNotFoundException() - } } else if (accounts.size == 0) { throw NoAccountsException() } else if (accounts.size == 1) { found_account = accounts.firstElement() - println("Using only available account: " + account.anonymize()) + println("Using only available account: " + found_account.anonymize()) } else { - show_error(("You didn't specify which account to use.\n" + + show_error(("You have more than one account but didn't specify which one to use.\n" + "Use '--account ' to use account .\n" + "Use '--list-accounts' to see all available accounts.")) System.exit(1) } + + if (found_account == null) { + throw AccountNotFoundException() + } + logger.debug("accounts.size: {}", accounts.size) logger.debug("account: {}", found_account.anonymize()) - return found_account!! + return found_account } - private fun cmd_stats() { + private fun cmd_stats(file_base: String, db: Database) { println() println("Stats:") val format = "%40s: %d%n" - System.out.format(format, "Number of accounts", Utils.getAccounts().size) - System.out.format(format, "Number of messages", Database.getInstance().getMessageCount()) - System.out.format(format, "Number of chats", Database.getInstance().getChatCount()) - System.out.format(format, "Number of users", Database.getInstance().getUserCount()) - System.out.format(format, "Top message ID", Database.getInstance().getTopMessageID()) + System.out.format(format, "Number of accounts", Utils.getAccounts(file_base).size) + System.out.format(format, "Number of messages", db.getMessageCount()) + System.out.format(format, "Number of chats", db.getChatCount()) + System.out.format(format, "Number of users", db.getUserCount()) + System.out.format(format, "Top message ID", db.getTopMessageID()) println() println("Media Types:") - for ((key, value) in Database.getInstance().getMessageMediaTypesWithCount()) { + for ((key, value) in db.getMessageMediaTypesWithCount()) { System.out.format(format, key, value) } println() println("Api layers of messages:") - for ((key, value) in Database.getInstance().getMessageApiLayerWithCount()) { + for ((key, value) in db.getMessageApiLayerWithCount()) { System.out.format(format, key, value) } println() println("Message source types:") - for ((key, value) in Database.getInstance().getMessageSourceTypeWithCount()) { + for ((key, value) in db.getMessageSourceTypeWithCount()) { System.out.format(format, key, value) } } - private fun cmd_login(target_dir: String, phoneToUse: String?): Nothing { - val phone: String - if (phoneToUse == null) { - println("Please enter your phone number in international format.") - println("Example: +4917077651234") - phone = getLine() - } else { - phone = phoneToUse - } - - user.sendCodeToPhoneNumber(phone) - println("Telegram sent you a code. Please enter it here.") - val code = getLine() - user.verifyCode(code) - if (user.isPasswordNeeded) { - println("We also need your account password. Please enter it now. It should not be printed, so it's okay if you see nothing while typing it.") - val pw = getPassword() - user.verifyPassword(pw) - } - storage.setPrefix("+" + user.user!!.getPhone()) - System.out.println("Everything seems fine. Please run this tool again with '--account +" + user.user!!.getPhone().anonymize() + " to use this account.") + private fun cmd_login(app: TelegramApp, target_dir: String, phoneToUse: String?): Nothing { + LoginManager(app, target_dir, phoneToUse).run() + System.exit(0) + throw RuntimeException("Code never reaches this. This exists just to keep the Kotlin compiler happy.") } private fun show_help() { @@ -323,15 +294,18 @@ class CommandLineController { companion object { private val logger = LoggerFactory.getLogger(CommandLineController::class.java) - public fun show_error(error: String) { + public fun show_error(error: String): Nothing { logger.error(error) println("ERROR: " + error) System.exit(1) + throw RuntimeException("Code never reaches this. This exists just to keep the Kotlin compiler happy.") } fun show_license() { println("TODO: Print the GPL.") } + + fun build_file_base(target_dir: String, account_to_use: String) = target_dir + File.separatorChar + account_to_use + File.separatorChar } class AccountNotFoundException() : Exception("Account not found") {} diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index 0997cf4..2971a6b 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -15,10 +15,10 @@ * along with this program. If not, see . */ package de.fabianonline.telegram_backup -internal object CommandLineOptions { - public var cmd_console = false - public var cmd_help = false - public var cmd_login = false +class CommandLineOptions(args: Array) { + var cmd_console = false + var cmd_help = false + var cmd_login = false var cmd_debug = false var cmd_trace = false var cmd_trace_telegram = false @@ -34,8 +34,8 @@ internal object CommandLineOptions { var val_target: String? = null var val_export: String? = null var val_test: Int? = null - @JvmStatic - fun parseOptions(args: Array) { + + init { var last_cmd: String? = null loop@ for (arg in args) { if (last_cmd != null) { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt index a922902..2f6d334 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt @@ -29,24 +29,30 @@ import ch.qos.logback.core.ConsoleAppender import ch.qos.logback.classic.Level fun main(args: Array) { - CommandLineOptions.parseOptions(args) - - CommandLineRunner.setupLogging() - CommandLineRunner.checkVersion() - - - - if (true || CommandLineOptions.cmd_console) { - // Always use the console for now. - CommandLineController() - } else { - GUIController() - } + val clr = CommandLineRunner(args) + + clr.setupLogging() + clr.checkVersion() + clr.run() } -object CommandLineRunner { +class CommandLineRunner(args: Array) { + val logger = LoggerFactory.getLogger(CommandLineRunner::class.java) as Logger + val options = CommandLineOptions(args) + + fun run() { + // Always use the console for now. + try { + CommandLineController(options) + } catch (e: Throwable) { + println("An error occured!") + e.printStackTrace() + logger.error("Exception caught!", e) + System.exit(1) + } + } + fun setupLogging() { - val logger = LoggerFactory.getLogger(CommandLineRunner::class.java) as Logger logger.trace("Setting up Loggers...") val rootLogger = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) as Logger val rootContext = rootLogger.getLoggerContext() @@ -65,13 +71,13 @@ object CommandLineRunner { rootLogger.addAppender(appender) rootLogger.setLevel(Level.OFF) - if (CommandLineOptions.cmd_trace) { + if (options.cmd_trace) { (LoggerFactory.getLogger("de.fabianonline.telegram_backup") as Logger).setLevel(Level.TRACE) - } else if (CommandLineOptions.cmd_debug) { + } else if (options.cmd_debug) { (LoggerFactory.getLogger("de.fabianonline.telegram_backup") as Logger).setLevel(Level.DEBUG) } - if (CommandLineOptions.cmd_trace_telegram) { + if (options.cmd_trace_telegram) { (LoggerFactory.getLogger("com.github.badoualy") as Logger).setLevel(Level.TRACE) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index 4d5adeb..3e3270c 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -21,7 +21,6 @@ import com.github.badoualy.telegram.tl.core.TLVector import com.github.badoualy.telegram.api.TelegramClient import org.slf4j.LoggerFactory import org.slf4j.Logger - import java.sql.Connection import java.sql.DriverManager import java.sql.Statement @@ -47,60 +46,45 @@ import java.text.SimpleDateFormat import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory -class Database private constructor(var client: TelegramClient) { - private var conn: Connection? = null - private var stmt: Statement? = null - var user_manager: UserManager +class Database constructor(val file_base: String, val user_manager: UserManager) { + val conn: Connection + val stmt: Statement + val logger = LoggerFactory.getLogger(Database::class.java) init { - private val logger = LoggerFactory.getLogger(Database::class.java) - - this.user_manager = UserManager.getInstance() - System.out.println("Opening database...") + println("Opening database...") try { Class.forName("org.sqlite.JDBC") } catch (e: ClassNotFoundException) { - CommandLineController.show_error("Could not load jdbc-sqlite class.") + throw RuntimeException("Could not load jdbc-sqlite class.") } - val path = "jdbc:sqlite:${user_manager.fileBase}${Config.FILE_NAME_DB}" + val path = "jdbc:sqlite:${file_base}${Config.FILE_NAME_DB}" try { - conn = DriverManager.getConnection(path) - stmt = conn!!.createStatement() + conn = DriverManager.getConnection(path)!! + stmt = conn.createStatement() } catch (e: SQLException) { - CommandLineController.show_error("Could not connect to SQLITE database.") + throw RuntimeException("Could not connect to SQLITE database.") } // Run updates val updates = DatabaseUpdates(conn!!, this) updates.doUpdates() - System.out.println("Database is ready.") - } - - fun getTopMessageID(): Int { - try { - val rs = stmt!!.executeQuery("SELECT MAX(message_id) FROM messages WHERE source_type IN ('group', 'dialog')") - rs.next() - val result = rs.getInt(1) - rs.close() - return result - } catch (e: SQLException) { - return 0 - } - + println("Database is ready.") } + fun getTopMessageID(): Int = queryInt("SELECT MAX(message_id) FROM messages WHERE source_type IN ('group', 'dialog')") fun getMessageCount(): Int = queryInt("SELECT COUNT(*) FROM messages") fun getChatCount(): Int = queryInt("SELECT COUNT(*) FROM chats") fun getUserCount(): Int = queryInt("SELECT COUNT(*) FROM users") - fun getMissingIDs: LinkedList() + fun getMissingIDs(): LinkedList { try { val missing = LinkedList() val max = getTopMessageID() - val rs = stmt!!.executeQuery("SELECT message_id FROM messages WHERE source_type IN ('group', 'dialog') ORDER BY id") + val rs = stmt.executeQuery("SELECT message_id FROM messages WHERE source_type IN ('group', 'dialog') ORDER BY id") rs.next() var id = rs.getInt(1) for (i in 1..max) { @@ -126,7 +110,7 @@ class Database private constructor(var client: TelegramClient) { fun getMessagesWithMedia(): LinkedList { try { val list = LinkedList() - val rs = stmt!!.executeQuery("SELECT data FROM messages WHERE has_media=1") + val rs = stmt.executeQuery("SELECT data FROM messages WHERE has_media=1") while (rs.next()) { list.add(bytesToTLMessage(rs.getBytes(1))) } @@ -139,7 +123,7 @@ class Database private constructor(var client: TelegramClient) { } - fun getMessagesFromUserCount() = queryInt("SELECT COUNT(*) FROM messages WHERE sender_id=" + user_manager.user!!.getId()) + fun getMessagesFromUserCount() = queryInt("SELECT COUNT(*) FROM messages WHERE sender_id=" + user_manager.id) fun getMessageTypesWithCount(): HashMap = getMessageTypesWithCount(GlobalChat()) @@ -148,7 +132,7 @@ class Database private constructor(var client: TelegramClient) { fun getMessageSourceTypeWithCount(): HashMap { val map = HashMap() try { - val rs = stmt!!.executeQuery("SELECT COUNT(id), source_type FROM messages GROUP BY source_type ORDER BY source_type") + val rs = stmt.executeQuery("SELECT COUNT(id), source_type FROM messages GROUP BY source_type ORDER BY source_type") while (rs.next()) { val source_type = rs.getString(2) ?: "null" map.put("count.messages.source_type.${source_type}", rs.getInt(1)) @@ -163,7 +147,7 @@ class Database private constructor(var client: TelegramClient) { fun getMessageApiLayerWithCount(): HashMap { val map = HashMap() try { - val rs = stmt!!.executeQuery("SELECT COUNT(id), api_layer FROM messages GROUP BY api_layer ORDER BY api_layer") + val rs = stmt.executeQuery("SELECT COUNT(id), api_layer FROM messages GROUP BY api_layer ORDER BY api_layer") while (rs.next()) { var layer = rs.getInt(2) map.put("count.messages.api_layer.$layer", rs.getInt(1)) @@ -179,67 +163,41 @@ class Database private constructor(var client: TelegramClient) { fun getMessageTimesMatrix(): Array = getMessageTimesMatrix(GlobalChat()) - fun getEncoding(): String { - try { - val rs = stmt!!.executeQuery("PRAGMA encoding") - rs.next() - val result = rs.getString(1) - rs.close() - return result - } catch (e: SQLException) { - logger.debug("SQLException: {}", e) - return "unknown" - } - - } - + fun getEncoding(): String = queryString("PRAGMA encoding") fun getListOfChatsForExport(): LinkedList { val list = LinkedList() - try { - val rs = stmt!!.executeQuery("SELECT chats.id, chats.name, COUNT(messages.id) as c " + - "FROM chats, messages WHERE messages.source_type IN('group', 'supergroup', 'channel') AND messages.source_id=chats.id " + - "GROUP BY chats.id ORDER BY c DESC") - while (rs.next()) { - list.add(Chat(rs.getInt(1), rs.getString(2), rs.getInt(3))) - } - rs.close() - return list - } catch (e: Exception) { - e.printStackTrace() - throw RuntimeException("Exception above!") - } + val rs = stmt.executeQuery("SELECT chats.id, chats.name, COUNT(messages.id) as c " + + "FROM chats, messages WHERE messages.source_type IN('group', 'supergroup', 'channel') AND messages.source_id=chats.id " + + "GROUP BY chats.id ORDER BY c DESC") + while (rs.next()) { + list.add(Chat(rs.getInt(1), rs.getString(2), rs.getInt(3))) + } + rs.close() + return list } fun getListOfDialogsForExport(): LinkedList { val list = LinkedList() - try { - val rs = stmt!!.executeQuery( - "SELECT users.id, first_name, last_name, username, COUNT(messages.id) as c " + - "FROM users, messages WHERE messages.source_type='dialog' AND messages.source_id=users.id " + - "GROUP BY users.id ORDER BY c DESC") - while (rs.next()) { - list.add(Dialog(rs.getInt(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getInt(5))) - } - rs.close() - return list - } catch (e: Exception) { - e.printStackTrace() - throw RuntimeException("Exception above!") + val rs = stmt.executeQuery( + "SELECT users.id, first_name, last_name, username, COUNT(messages.id) as c " + + "FROM users, messages WHERE messages.source_type='dialog' AND messages.source_id=users.id " + + "GROUP BY users.id ORDER BY c DESC") + while (rs.next()) { + list.add(Dialog(rs.getInt(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getInt(5))) } - + rs.close() + return list } - - fun backupDatabase(currentVersion: Int) { val filename = String.format(Config.FILE_NAME_DB_BACKUP, currentVersion) System.out.println(" Creating a backup of your database as " + filename) try { - val src = user_manager.fileBase + Config.FILE_NAME_DB - val dst = user_manager.fileBase + filename + val src = file_base + Config.FILE_NAME_DB + val dst = file_base + filename logger.debug("Copying {} to {}", src.anonymize(), dst.anonymize()) Files.copy( File(src).toPath(), @@ -256,301 +214,283 @@ class Database private constructor(var client: TelegramClient) { fun getTopMessageIDForChannel(id: Int): Int = queryInt("SELECT MAX(message_id) FROM messages WHERE source_id=$id AND source_type IN('channel', 'supergroup')") fun logRun(start_id: Int, end_id: Int, count: Int) { - try { - val ps = conn!!.prepareStatement("INSERT INTO runs " + - "(time, start_id, end_id, count_missing) " + - "VALUES " + - "(DateTime('now'), ?, ?, ? )") - ps.setInt(1, start_id) - ps.setInt(2, end_id) - ps.setInt(3, count) - ps.execute() - ps.close() - } catch (e: SQLException) { - } - + val ps = conn.prepareStatement("INSERT INTO runs " + + "(time, start_id, end_id, count_missing) " + + "VALUES " + + "(DateTime('now'), ?, ?, ? )") + ps.setInt(1, start_id) + ps.setInt(2, end_id) + ps.setInt(3, count) + ps.execute() + ps.close() } fun queryInt(query: String): Int { - try { - val rs = stmt!!.executeQuery(query) - rs.next() - val result = rs.getInt(1) - rs.close() - return result - } catch (e: SQLException) { - throw RuntimeException("Could not get count of messages.") - } - + val rs = stmt.executeQuery(query) + rs.next() + val result = rs.getInt(1) + rs.close() + return result + } + + fun queryString(query: String): String { + val rs = stmt.executeQuery(query) + rs.next() + val result = rs.getString(1) + rs.close() + return result } @Synchronized fun saveMessages(all: TLVector, api_layer: Int, source_type: MessageSource = MessageSource.NORMAL) { - try { - //"(id, dialog_id, from_id, from_type, text, time, has_media, data, sticker, type) " + - //"VALUES " + - //"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); - val columns = "(message_id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer) " + - "VALUES " + - "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" - //1 2 3 4 5 6 7 8 9 10 11 12 13 14 - val ps = conn!!.prepareStatement("INSERT OR REPLACE INTO messages " + columns) - val ps_insert_or_ignore = conn!!.prepareStatement("INSERT OR IGNORE INTO messages " + columns) + //"(id, dialog_id, from_id, from_type, text, time, has_media, data, sticker, type) " + + //"VALUES " + + //"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + val columns = "(message_id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer) " + + "VALUES " + + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + //1 2 3 4 5 6 7 8 9 10 11 12 13 14 + val ps = conn.prepareStatement("INSERT OR REPLACE INTO messages " + columns) + val ps_insert_or_ignore = conn.prepareStatement("INSERT OR IGNORE INTO messages " + columns) - for (msg in all) { - if (msg is TLMessage) { - ps.setInt(1, msg.getId()) - ps.setString(2, "message") - val peer = msg.getToId() - if (peer is TLPeerChat) { - ps.setString(3, "group") - ps.setInt(4, peer.getChatId()) - } else if (peer is TLPeerUser) { - var id = peer.getUserId() - if (id == this.user_manager.user!!.getId()) { - id = msg.getFromId() - } - ps.setString(3, "dialog") - ps.setInt(4, id) - } else if (peer is TLPeerChannel) { - if (source_type == MessageSource.CHANNEL) { - ps.setString(3, "channel") - } else if (source_type == MessageSource.SUPERGROUP) { - ps.setString(3, "supergroup") - } else { - throw RuntimeException("Got a TLPeerChannel, but were expecting $source_type") - } - ps.setInt(4, peer.getChannelId()) + for (msg in all) { + if (msg is TLMessage) { + ps.setInt(1, msg.getId()) + ps.setString(2, "message") + val peer = msg.getToId() + if (peer is TLPeerChat) { + ps.setString(3, "group") + ps.setInt(4, peer.getChatId()) + } else if (peer is TLPeerUser) { + var id = peer.getUserId() + if (id == user_manager.id) { + id = msg.getFromId() + } + ps.setString(3, "dialog") + ps.setInt(4, id) + } else if (peer is TLPeerChannel) { + if (source_type == MessageSource.CHANNEL) { + ps.setString(3, "channel") + } else if (source_type == MessageSource.SUPERGROUP) { + ps.setString(3, "supergroup") } else { - throw RuntimeException("Unexpected Peer type: " + peer.javaClass) + throw RuntimeException("Got a TLPeerChannel, but were expecting $source_type") } - - if (peer is TLPeerChannel && msg.getFromId() == null) { - ps.setNull(5, Types.INTEGER) - } else { - ps.setInt(5, msg.getFromId()) - } - - if (msg.getFwdFrom() != null && msg.getFwdFrom().getFromId() != null) { - ps.setInt(6, msg.getFwdFrom().getFromId()) - } else { - ps.setNull(6, Types.INTEGER) - } - - var text = msg.getMessage() - if ((text == null || text.equals("")) && msg.getMedia() != null) { - val media = msg.getMedia() - if (media is TLMessageMediaDocument) { - text = media.getCaption() - } else if (media is TLMessageMediaPhoto) { - text = media.getCaption() - } - } - ps.setString(7, text) - ps.setString(8, "" + msg.getDate()) - val f = FileManagerFactory.getFileManager(msg, user_manager, client) - if (f == null) { - ps.setNull(9, Types.BOOLEAN) - ps.setNull(10, Types.VARCHAR) - ps.setNull(11, Types.VARCHAR) - ps.setNull(12, Types.INTEGER) - } else { - ps.setBoolean(9, true) - ps.setString(10, f.name) - ps.setString(11, f.targetFilename) - ps.setInt(12, f.size) - } - val stream = ByteArrayOutputStream() - msg.serializeBody(stream) - ps.setBytes(13, stream.toByteArray()) - ps.setInt(14, api_layer) - ps.addBatch() - } else if (msg is TLMessageService) { - ps_insert_or_ignore.setInt(1, msg.getId()) - ps_insert_or_ignore.setString(2, "service_message") - - val peer = msg.getToId() - if (peer is TLPeerChat) { - ps.setString(3, "group") - ps.setInt(4, peer.getChatId()) - } else if (peer is TLPeerUser) { - var id = peer.getUserId() - if (id == this.user_manager.user!!.getId()) { - id = msg.getFromId() - } - ps.setString(3, "dialog") - ps.setInt(4, id) - } else if (peer is TLPeerChannel) { - // Messages in channels don't have a sender. - if (msg.getFromId() == null) { - ps.setString(3, "channel") - } else { - ps.setString(3, "supergroup") - } - ps.setInt(4, peer.getChannelId()) - } else { - throw RuntimeException("Unexpected Peer type: " + peer.javaClass) - } - - ps_insert_or_ignore.setNull(5, Types.INTEGER) - ps_insert_or_ignore.setNull(6, Types.INTEGER) - ps_insert_or_ignore.setNull(7, Types.VARCHAR) - ps_insert_or_ignore.setNull(8, Types.INTEGER) - ps_insert_or_ignore.setNull(9, Types.BOOLEAN) - ps_insert_or_ignore.setNull(10, Types.VARCHAR) - ps_insert_or_ignore.setNull(11, Types.VARCHAR) - ps_insert_or_ignore.setNull(12, Types.INTEGER) - ps_insert_or_ignore.setNull(13, Types.BLOB) - ps_insert_or_ignore.setInt(14, api_layer) - ps_insert_or_ignore.addBatch() - } else if (msg is TLMessageEmpty) { - ps_insert_or_ignore.setInt(1, msg.getId()) - ps_insert_or_ignore.setString(2, "empty_message") - ps_insert_or_ignore.setNull(3, Types.INTEGER) - ps_insert_or_ignore.setNull(4, Types.INTEGER) - ps_insert_or_ignore.setNull(5, Types.INTEGER) - ps_insert_or_ignore.setNull(6, Types.INTEGER) - ps_insert_or_ignore.setNull(7, Types.VARCHAR) - ps_insert_or_ignore.setNull(8, Types.INTEGER) - ps_insert_or_ignore.setNull(9, Types.BOOLEAN) - ps_insert_or_ignore.setNull(10, Types.VARCHAR) - ps_insert_or_ignore.setNull(11, Types.VARCHAR) - ps_insert_or_ignore.setNull(12, Types.INTEGER) - ps_insert_or_ignore.setNull(13, Types.BLOB) - ps_insert_or_ignore.setInt(14, api_layer) - ps_insert_or_ignore.addBatch() + ps.setInt(4, peer.getChannelId()) } else { - throw RuntimeException("Unexpected Message type: " + msg.javaClass) + throw RuntimeException("Unexpected Peer type: " + peer.javaClass) } - } - conn!!.setAutoCommit(false) - ps.executeBatch() - ps.clearBatch() - ps_insert_or_ignore.executeBatch() - ps_insert_or_ignore.clearBatch() - conn!!.commit() - conn!!.setAutoCommit(true) - - ps.close() - ps_insert_or_ignore.close() - } catch (e: Exception) { - e.printStackTrace() - throw RuntimeException("Exception shown above happened.") - } + if (peer is TLPeerChannel && msg.getFromId() == null) { + ps.setNull(5, Types.INTEGER) + } else { + ps.setInt(5, msg.getFromId()) + } + + if (msg.getFwdFrom() != null && msg.getFwdFrom().getFromId() != null) { + ps.setInt(6, msg.getFwdFrom().getFromId()) + } else { + ps.setNull(6, Types.INTEGER) + } + + var text = msg.getMessage() + if ((text == null || text.equals("")) && msg.getMedia() != null) { + val media = msg.getMedia() + if (media is TLMessageMediaDocument) { + text = media.getCaption() + } else if (media is TLMessageMediaPhoto) { + text = media.getCaption() + } + } + ps.setString(7, text) + ps.setString(8, "" + msg.getDate()) + val f = FileManagerFactory.getFileManager(msg, user_manager) + if (f == null) { + ps.setNull(9, Types.BOOLEAN) + ps.setNull(10, Types.VARCHAR) + ps.setNull(11, Types.VARCHAR) + ps.setNull(12, Types.INTEGER) + } else { + ps.setBoolean(9, true) + ps.setString(10, f.name) + ps.setString(11, f.targetFilename) + ps.setInt(12, f.size) + } + val stream = ByteArrayOutputStream() + msg.serializeBody(stream) + ps.setBytes(13, stream.toByteArray()) + ps.setInt(14, api_layer) + ps.addBatch() + } else if (msg is TLMessageService) { + ps_insert_or_ignore.setInt(1, msg.getId()) + ps_insert_or_ignore.setString(2, "service_message") + + val peer = msg.getToId() + if (peer is TLPeerChat) { + ps.setString(3, "group") + ps.setInt(4, peer.getChatId()) + } else if (peer is TLPeerUser) { + var id = peer.getUserId() + if (id == user_manager.id) { + id = msg.getFromId() + } + ps.setString(3, "dialog") + ps.setInt(4, id) + } else if (peer is TLPeerChannel) { + // Messages in channels don't have a sender. + if (msg.getFromId() == null) { + ps.setString(3, "channel") + } else { + ps.setString(3, "supergroup") + } + ps.setInt(4, peer.getChannelId()) + } else { + throw RuntimeException("Unexpected Peer type: " + peer.javaClass) + } + + ps_insert_or_ignore.setNull(5, Types.INTEGER) + ps_insert_or_ignore.setNull(6, Types.INTEGER) + ps_insert_or_ignore.setNull(7, Types.VARCHAR) + ps_insert_or_ignore.setNull(8, Types.INTEGER) + ps_insert_or_ignore.setNull(9, Types.BOOLEAN) + ps_insert_or_ignore.setNull(10, Types.VARCHAR) + ps_insert_or_ignore.setNull(11, Types.VARCHAR) + ps_insert_or_ignore.setNull(12, Types.INTEGER) + ps_insert_or_ignore.setNull(13, Types.BLOB) + ps_insert_or_ignore.setInt(14, api_layer) + ps_insert_or_ignore.addBatch() + } else if (msg is TLMessageEmpty) { + ps_insert_or_ignore.setInt(1, msg.getId()) + ps_insert_or_ignore.setString(2, "empty_message") + ps_insert_or_ignore.setNull(3, Types.INTEGER) + ps_insert_or_ignore.setNull(4, Types.INTEGER) + ps_insert_or_ignore.setNull(5, Types.INTEGER) + ps_insert_or_ignore.setNull(6, Types.INTEGER) + ps_insert_or_ignore.setNull(7, Types.VARCHAR) + ps_insert_or_ignore.setNull(8, Types.INTEGER) + ps_insert_or_ignore.setNull(9, Types.BOOLEAN) + ps_insert_or_ignore.setNull(10, Types.VARCHAR) + ps_insert_or_ignore.setNull(11, Types.VARCHAR) + ps_insert_or_ignore.setNull(12, Types.INTEGER) + ps_insert_or_ignore.setNull(13, Types.BLOB) + ps_insert_or_ignore.setInt(14, api_layer) + ps_insert_or_ignore.addBatch() + } else { + throw RuntimeException("Unexpected Message type: " + msg.javaClass) + } + } + conn.setAutoCommit(false) + ps.executeBatch() + ps.clearBatch() + ps_insert_or_ignore.executeBatch() + ps_insert_or_ignore.clearBatch() + conn.commit() + conn.setAutoCommit(true) + + ps.close() + ps_insert_or_ignore.close() } @Synchronized fun saveChats(all: TLVector) { - try { - val ps_insert_or_replace = conn!!.prepareStatement( - "INSERT OR REPLACE INTO chats " + - "(id, name, type) " + - "VALUES " + - "(?, ?, ?)") - val ps_insert_or_ignore = conn!!.prepareStatement( - "INSERT OR IGNORE INTO chats " + - "(id, name, type) " + - "VALUES " + - "(?, ?, ?)") + val ps_insert_or_replace = conn.prepareStatement( + "INSERT OR REPLACE INTO chats " + + "(id, name, type) " + + "VALUES " + + "(?, ?, ?)") + val ps_insert_or_ignore = conn.prepareStatement( + "INSERT OR IGNORE INTO chats " + + "(id, name, type) " + + "VALUES " + + "(?, ?, ?)") - for (abs in all) { - ps_insert_or_replace.setInt(1, abs.getId()) - ps_insert_or_ignore.setInt(1, abs.getId()) - if (abs is TLChatEmpty) { - ps_insert_or_ignore.setNull(2, Types.VARCHAR) - ps_insert_or_ignore.setString(3, "empty_chat") - ps_insert_or_ignore.addBatch() - } else if (abs is TLChatForbidden) { - ps_insert_or_replace.setString(2, abs.getTitle()) - ps_insert_or_replace.setString(3, "chat") - ps_insert_or_replace.addBatch() - } else if (abs is TLChannelForbidden) { - ps_insert_or_replace.setString(2, abs.getTitle()) - ps_insert_or_replace.setString(3, "channel") - ps_insert_or_replace.addBatch() - } else if (abs is TLChat) { - ps_insert_or_replace.setString(2, abs.getTitle()) - ps_insert_or_replace.setString(3, "chat") - ps_insert_or_replace.addBatch() - } else if (abs is TLChannel) { - ps_insert_or_replace.setString(2, abs.getTitle()) - ps_insert_or_replace.setString(3, "channel") - ps_insert_or_replace.addBatch() - } else { - throw RuntimeException("Unexpected " + abs.javaClass) - } + for (abs in all) { + ps_insert_or_replace.setInt(1, abs.getId()) + ps_insert_or_ignore.setInt(1, abs.getId()) + if (abs is TLChatEmpty) { + ps_insert_or_ignore.setNull(2, Types.VARCHAR) + ps_insert_or_ignore.setString(3, "empty_chat") + ps_insert_or_ignore.addBatch() + } else if (abs is TLChatForbidden) { + ps_insert_or_replace.setString(2, abs.getTitle()) + ps_insert_or_replace.setString(3, "chat") + ps_insert_or_replace.addBatch() + } else if (abs is TLChannelForbidden) { + ps_insert_or_replace.setString(2, abs.getTitle()) + ps_insert_or_replace.setString(3, "channel") + ps_insert_or_replace.addBatch() + } else if (abs is TLChat) { + ps_insert_or_replace.setString(2, abs.getTitle()) + ps_insert_or_replace.setString(3, "chat") + ps_insert_or_replace.addBatch() + } else if (abs is TLChannel) { + ps_insert_or_replace.setString(2, abs.getTitle()) + ps_insert_or_replace.setString(3, "channel") + ps_insert_or_replace.addBatch() + } else { + throw RuntimeException("Unexpected " + abs.javaClass) } - conn!!.setAutoCommit(false) - ps_insert_or_ignore.executeBatch() - ps_insert_or_ignore.clearBatch() - ps_insert_or_replace.executeBatch() - ps_insert_or_replace.clearBatch() - conn!!.commit() - conn!!.setAutoCommit(true) - - ps_insert_or_ignore.close() - ps_insert_or_replace.close() - } catch (e: Exception) { - e.printStackTrace() - throw RuntimeException("Exception shown above happened.") } - + conn.setAutoCommit(false) + ps_insert_or_ignore.executeBatch() + ps_insert_or_ignore.clearBatch() + ps_insert_or_replace.executeBatch() + ps_insert_or_replace.clearBatch() + conn.commit() + conn.setAutoCommit(true) + + ps_insert_or_ignore.close() + ps_insert_or_replace.close() } @Synchronized fun saveUsers(all: TLVector) { - try { - val ps_insert_or_replace = conn!!.prepareStatement( - "INSERT OR REPLACE INTO users " + - "(id, first_name, last_name, username, type, phone) " + - "VALUES " + - "(?, ?, ?, ?, ?, ?)") - val ps_insert_or_ignore = conn!!.prepareStatement( - "INSERT OR IGNORE INTO users " + - "(id, first_name, last_name, username, type, phone) " + - "VALUES " + - "(?, ?, ?, ?, ?, ?)") - for (abs in all) { - if (abs is TLUser) { - val user = abs - ps_insert_or_replace.setInt(1, user.getId()) - ps_insert_or_replace.setString(2, user.getFirstName()) - ps_insert_or_replace.setString(3, user.getLastName()) - ps_insert_or_replace.setString(4, user.getUsername()) - ps_insert_or_replace.setString(5, "user") - ps_insert_or_replace.setString(6, user.getPhone()) - ps_insert_or_replace.addBatch() - } else if (abs is TLUserEmpty) { - ps_insert_or_ignore.setInt(1, abs.getId()) - ps_insert_or_ignore.setNull(2, Types.VARCHAR) - ps_insert_or_ignore.setNull(3, Types.VARCHAR) - ps_insert_or_ignore.setNull(4, Types.VARCHAR) - ps_insert_or_ignore.setString(5, "empty_user") - ps_insert_or_ignore.setNull(6, Types.VARCHAR) - ps_insert_or_ignore.addBatch() - } else { - throw RuntimeException("Unexpected " + abs.javaClass) - } + val ps_insert_or_replace = conn.prepareStatement( + "INSERT OR REPLACE INTO users " + + "(id, first_name, last_name, username, type, phone) " + + "VALUES " + + "(?, ?, ?, ?, ?, ?)") + val ps_insert_or_ignore = conn.prepareStatement( + "INSERT OR IGNORE INTO users " + + "(id, first_name, last_name, username, type, phone) " + + "VALUES " + + "(?, ?, ?, ?, ?, ?)") + for (abs in all) { + if (abs is TLUser) { + val user = abs + ps_insert_or_replace.setInt(1, user.getId()) + ps_insert_or_replace.setString(2, user.getFirstName()) + ps_insert_or_replace.setString(3, user.getLastName()) + ps_insert_or_replace.setString(4, user.getUsername()) + ps_insert_or_replace.setString(5, "user") + ps_insert_or_replace.setString(6, user.getPhone()) + ps_insert_or_replace.addBatch() + } else if (abs is TLUserEmpty) { + ps_insert_or_ignore.setInt(1, abs.getId()) + ps_insert_or_ignore.setNull(2, Types.VARCHAR) + ps_insert_or_ignore.setNull(3, Types.VARCHAR) + ps_insert_or_ignore.setNull(4, Types.VARCHAR) + ps_insert_or_ignore.setString(5, "empty_user") + ps_insert_or_ignore.setNull(6, Types.VARCHAR) + ps_insert_or_ignore.addBatch() + } else { + throw RuntimeException("Unexpected " + abs.javaClass) } - conn!!.setAutoCommit(false) - ps_insert_or_ignore.executeBatch() - ps_insert_or_ignore.clearBatch() - ps_insert_or_replace.executeBatch() - ps_insert_or_replace.clearBatch() - conn!!.commit() - conn!!.setAutoCommit(true) - - ps_insert_or_ignore.close() - ps_insert_or_replace.close() - } catch (e: Exception) { - e.printStackTrace() - throw RuntimeException("Exception shown above happened.") } + conn.setAutoCommit(false) + ps_insert_or_ignore.executeBatch() + ps_insert_or_ignore.clearBatch() + ps_insert_or_replace.executeBatch() + ps_insert_or_replace.clearBatch() + conn.commit() + conn.setAutoCommit(true) + + ps_insert_or_ignore.close() + ps_insert_or_replace.close() } fun fetchSetting(key: String): String? { - val rs = stmt!!.executeQuery("SELECT value FROM settings WHERE key='${key}'") + val rs = stmt.executeQuery("SELECT value FROM settings WHERE key='${key}'") rs.next() val result = rs.getString(1) rs.close() @@ -558,7 +498,7 @@ class Database private constructor(var client: TelegramClient) { } fun saveSetting(key: String, value: String?) { - val ps = conn!!.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)") + val ps = conn.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)") ps.setString(1, key) if (value==null) { ps.setNull(2, Types.VARCHAR) @@ -570,56 +510,37 @@ class Database private constructor(var client: TelegramClient) { } fun getIdsFromQuery(query: String): LinkedList { - try { - val list = LinkedList() - val rs = stmt!!.executeQuery(query) - while (rs.next()) { - list.add(rs.getInt(1)) - } - rs.close() - return list - } catch (e: SQLException) { - throw RuntimeException(e) + val list = LinkedList() + val rs = stmt.executeQuery(query) + while (rs.next()) { + list.add(rs.getInt(1)) } - + rs.close() + return list } fun getMessageTypesWithCount(c: AbstractChat): HashMap { val map = HashMap() - try { - val rs = stmt!!.executeQuery("SELECT message_type, COUNT(message_id) FROM messages WHERE " + c.query + " GROUP BY message_type") - while (rs.next()) { - map.put("count.messages.type." + rs.getString(1), rs.getInt(2)) - } - rs.close() - return map - } catch (e: Exception) { - throw RuntimeException(e) + val rs = stmt.executeQuery("SELECT message_type, COUNT(message_id) FROM messages WHERE " + c.query + " GROUP BY message_type") + while (rs.next()) { + map.put("count.messages.type." + rs.getString(1), rs.getInt(2)) } - + rs.close() + return map } fun getMessageMediaTypesWithCount(c: AbstractChat): HashMap { val map = HashMap() - try { - var count = 0 - val rs = stmt!!.executeQuery("SELECT media_type, COUNT(message_id) FROM messages WHERE " + c.query + " GROUP BY media_type") - while (rs.next()) { - var s = rs.getString(1) - if (s == null) { - s = "null" - } else { - count += rs.getInt(2) - } - map.put("count.messages.media_type.$s", rs.getInt(2)) - } - map.put("count.messages.media_type.any", count) - rs.close() - return map - } catch (e: Exception) { - throw RuntimeException(e) + var count = 0 + val rs = stmt.executeQuery("SELECT media_type, COUNT(message_id) FROM messages WHERE " + c.query + " GROUP BY media_type") + while (rs.next()) { + var type = rs.getString(1) ?: "null" + if (type != "null") count += rs.getInt(2) + map.put("count.messages.media_type.${type}", rs.getInt(2)) } - + map.put("count.messages.media_type.any", count) + rs.close() + return map } fun getMessageAuthorsWithCount(c: AbstractChat): HashMap { @@ -629,121 +550,100 @@ class Database private constructor(var client: TelegramClient) { // Set a default value for 'me' to fix the charts for channels - cause I // possibly didn't send any messages there. map.put("authors.count.me", 0) - try { - val rs = stmt!!.executeQuery("SELECT users.id, users.first_name, users.last_name, users.username, COUNT(messages.id) " + - "FROM messages " + - "LEFT JOIN users ON users.id=messages.sender_id " + - "WHERE " + c.query + " GROUP BY sender_id") - while (rs.next()) { - val u: User - val data = HashMap() - if (rs.getString(2) != null || rs.getString(3) != null || rs.getString(4) != null) { - u = User(rs.getInt(1), rs.getString(2), rs.getString(3)) - } else { - u = User(rs.getInt(1), "Unknown", "") - } - if (u.isMe) { - map.put("authors.count.me", rs.getInt(5)) - } else { - count_others += rs.getInt(5) - data.put("name", u.name) - data.put("count", ""+rs.getInt(5)) - all_data.add(data) - } - + val rs = stmt.executeQuery("SELECT users.id, users.first_name, users.last_name, users.username, COUNT(messages.id) " + + "FROM messages " + + "LEFT JOIN users ON users.id=messages.sender_id " + + "WHERE " + c.query + " GROUP BY sender_id") + while (rs.next()) { + val u: User + val data = HashMap() + if (rs.getString(2) != null || rs.getString(3) != null || rs.getString(4) != null) { + u = User(rs.getInt(1), rs.getString(2), rs.getString(3)) + } else { + u = User(rs.getInt(1), "Unknown", "") } - map.put("authors.count.others", count_others) - map.put("authors.all", all_data) - rs.close() - return map - } catch (e: Exception) { - throw RuntimeException(e) + if (u.isMe) { + map.put("authors.count.me", rs.getInt(5)) + } else { + count_others += rs.getInt(5) + data.put("name", u.name) + data.put("count", ""+rs.getInt(5)) + all_data.add(data) + } + } + map.put("authors.count.others", count_others) + map.put("authors.all", all_data) + rs.close() + return map } - fun getMessageCountForExport(c: AbstractChat): Int { - val rs = stmt!!.executeQuery("SELECT COUNT(*) FROM messages WHERE " + c.query); - rs.next() - val result = rs.getInt(1) + fun getMessageCountForExport(c: AbstractChat): Int = queryInt("SELECT COUNT(*) FROM messages WHERE " + c.query) + + fun getMessageTimesMatrix(c: AbstractChat): Array { + val result = Array(7) { IntArray(24) } + val rs = stmt.executeQuery("SELECT STRFTIME('%w', time, 'unixepoch') as DAY, " + + "STRFTIME('%H', time, 'unixepoch') AS hour, " + + "COUNT(id) FROM messages WHERE " + c.query + " GROUP BY hour, day " + + "ORDER BY hour, day") + while (rs.next()) { + result[if (rs.getInt(1) == 0) 6 else rs.getInt(1) - 1][rs.getInt(2)] = rs.getInt(3) + } rs.close() return result } - fun getMessageTimesMatrix(c: AbstractChat): Array { - val result = Array(7) { IntArray(24) } - try { - val rs = stmt!!.executeQuery("SELECT STRFTIME('%w', time, 'unixepoch') as DAY, " + - "STRFTIME('%H', time, 'unixepoch') AS hour, " + - "COUNT(id) FROM messages WHERE " + c.query + " GROUP BY hour, day " + - "ORDER BY hour, day") - while (rs.next()) { - result[if (rs.getInt(1) == 0) 6 else rs.getInt(1) - 1][rs.getInt(2)] = rs.getInt(3) - } - rs.close() - return result - } catch (e: Exception) { - throw RuntimeException(e) - } - - } - fun getMessagesForExport(c: AbstractChat, limit: Int=-1, offset: Int=0): LinkedList> { - try { - var query = "SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " + - "media_type, media_file, media_size, users.first_name as user_first_name, users.last_name as user_last_name, " + - "users.username as user_username, users.id as user_id, " + - "users_fwd.first_name as user_fwd_first_name, users_fwd.last_name as user_fwd_last_name, users_fwd.username as user_fwd_username " + - "FROM messages " + - "LEFT JOIN users ON users.id=messages.sender_id " + - "LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " + - c.query + " " + - "ORDER BY messages.message_id" - - if ( limit != -1 ) { - query = query + " LIMIT ${limit} OFFSET ${offset}" - } - - val rs = stmt!!.executeQuery(query) - - val format_time = SimpleDateFormat("HH:mm:ss") - val format_date = SimpleDateFormat("d MMM yy") - val meta = rs.getMetaData() - val columns = meta.getColumnCount() - val list = LinkedList>() + var query = "SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " + + "media_type, media_file, media_size, users.first_name as user_first_name, users.last_name as user_last_name, " + + "users.username as user_username, users.id as user_id, " + + "users_fwd.first_name as user_fwd_first_name, users_fwd.last_name as user_fwd_last_name, users_fwd.username as user_fwd_username " + + "FROM messages " + + "LEFT JOIN users ON users.id=messages.sender_id " + + "LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " + + c.query + " " + + "ORDER BY messages.message_id" + + if ( limit != -1 ) { + query = query + " LIMIT ${limit} OFFSET ${offset}" + } + + val rs = stmt.executeQuery(query) + + val format_time = SimpleDateFormat("HH:mm:ss") + val format_date = SimpleDateFormat("d MMM yy") + val meta = rs.getMetaData() + val columns = meta.getColumnCount() + val list = LinkedList>() - var count = 0 - var old_date: String? = null - var old_user = 0 - while (rs.next()) { - val h = HashMap(columns) - for (i in 1..columns) { - h.put(meta.getColumnName(i), rs.getObject(i)) - } - // Additional values to make up for Mustache's inability to format dates - val d = rs.getTime("time") - val date = format_date.format(d) - h.put("formatted_time", format_time.format(d)) - h.put("formatted_date", date) - if (rs.getString("media_type") != null) { - h.put("media_" + rs.getString("media_type"), true) - } - h.put("from_me", rs.getInt("user_id") == user_manager.user!!.getId()) - h.put("is_new_date", !date.equals(old_date)) - h.put("odd_even", if (count % 2 == 0) "even" else "odd") - h.put("same_user", old_user != 0 && rs.getInt("user_id") == old_user) - old_user = rs.getInt("user_id") - old_date = date - - list.add(h) - count++ + var count = 0 + var old_date: String? = null + var old_user = 0 + while (rs.next()) { + val h = HashMap(columns) + for (i in 1..columns) { + h.put(meta.getColumnName(i), rs.getObject(i)) } - rs.close() - return list - } catch (e: Exception) { - e.printStackTrace() - throw RuntimeException("Exception above!") - } + // Additional values to make up for Mustache's inability to format dates + val d = rs.getTime("time") + val date = format_date.format(d) + h.put("formatted_time", format_time.format(d)) + h.put("formatted_date", date) + if (rs.getString("media_type") != null) { + h.put("media_" + rs.getString("media_type"), true) + } + h.put("from_me", rs.getInt("user_id") == user_manager.id) + h.put("is_new_date", !date.equals(old_date)) + h.put("odd_even", if (count % 2 == 0) "even" else "odd") + h.put("same_user", old_user != 0 && rs.getInt("user_id") == old_user) + old_user = rs.getInt("user_id") + old_date = date + list.add(h) + count++ + } + rs.close() + return list } @@ -753,21 +653,13 @@ class Database private constructor(var client: TelegramClient) { } inner class Dialog(var id: Int, var first_name: String?, var last_name: String?, var username: String?, var count: Int?) : AbstractChat() { - - override val query: String - get() = "source_type='dialog' AND source_id=" + id - - override val type: String - get() = "dialog" + override val query = "source_type='dialog' AND source_id=" + id + override val type = "dialog" } inner class Chat(var id: Int, var name: String?, var count: Int?) : AbstractChat() { - - override val query: String - get() = "source_type IN('group', 'supergroup', 'channel') AND source_id=" + id - - override val type: String - get() = "chat" + override val query = "source_type IN('group', 'supergroup', 'channel') AND source_id=" + id + override val type = "chat" } inner class User(id: Int, first_name: String?, last_name: String?) { @@ -775,7 +667,7 @@ class Database private constructor(var client: TelegramClient) { var isMe: Boolean = false init { - isMe = id == user_manager.user!!.getId() + isMe = id == user_manager.id val s = StringBuilder() if (first_name != null) s.append(first_name + " ") if (last_name != null) s.append(last_name) @@ -784,11 +676,8 @@ class Database private constructor(var client: TelegramClient) { } inner class GlobalChat : AbstractChat() { - override val query: String - get() = "1=1" - - override val type: String - get() = "GlobalChat" + override val query = "1=1" + override val type = "GlobalChat" } companion object { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt index 713670c..b0e5676 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt @@ -313,7 +313,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn } else { ps.setInt(1, msg.getFwdFrom().getFromId()) } - val f = FileManagerFactory.getFileManager(msg, db.user_manager, db.client) + val f = FileManagerFactory.getFileManager(msg, db.user_manager) if (f == null) { ps.setNull(2, Types.VARCHAR) ps.setNull(3, Types.VARCHAR) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt index 7cc36fd..26d4344 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt @@ -1,8 +1,8 @@ package de.fabianonline.telegram_backup -class DbSettings() { - private fun fetchValue(name: String): String? = Database.getInstance().fetchSetting(name) - private fun saveValue(name: String, value: String?) = Database.getInstance().saveSetting(name, value) +class DbSettings(val database: Database) { + private fun fetchValue(name: String): String? = database.fetchSetting(name) + private fun saveValue(name: String, value: String?) = database.saveSetting(name, value) var pts: String? get() = fetchValue("pts") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt index b583212..b953e4d 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt @@ -61,18 +61,9 @@ enum class MessageSource(val descr: String) { SUPERGROUP("supergroup") } -class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressInterface) { - internal var user: UserManager? = null - internal var db: Database? = null - internal var prog: DownloadProgressInterface? = null +class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInterface, val db: Database, val user_manager: UserManager, val inisettings: IniSettings) { internal var has_seen_flood_wait_message = false - init { - this.user = UserManager.getInstance() - this.prog = p - this.db = Database.getInstance() - } - @Throws(RpcErrorException::class, IOException::class) fun downloadMessages(limit: Int?) { var completed: Boolean @@ -173,18 +164,14 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI } */ - if (IniSettings.download_channels || IniSettings.download_supergroups) { - // TODO Add chat title (and other stuff?) to the database + if (inisettings.download_channels) { + println("Checking channels...") + for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) } + } - if (IniSettings.download_channels) { - println("Checking channels...") - for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) } - } - - if (IniSettings.download_supergroups) { - println("Checking supergroups...") - for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) } - } + if (inisettings.download_supergroups) { + println("Checking supergroups...") + for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) } } } @@ -315,7 +302,7 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI prog!!.onMediaDownloadStart(messages.size) for (msg in messages) { if (msg == null) continue - val m = FileManagerFactory.getFileManager(msg, user!!, client!!) + val m = FileManagerFactory.getFileManager(msg, user_manager) logger.trace("message {}, {}, {}, {}, {}", msg.getId(), msg.getMedia().javaClass.getSimpleName().replace("TLMessageMedia", "…"), @@ -366,10 +353,10 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI if (tl_peer_channel == null) continue var download = true - if (IniSettings.whitelist_channels != null) { - download = IniSettings.whitelist_channels!!.contains(tl_channel.getId().toString()) - } else if (IniSettings.blacklist_channels != null) { - download = !IniSettings.blacklist_channels!!.contains(tl_channel.getId().toString()) + if (inisettings.whitelist_channels != null) { + download = inisettings.whitelist_channels.contains(tl_channel.getId().toString()) + } else if (inisettings.blacklist_channels != null) { + download = !inisettings.blacklist_channels.contains(tl_channel.getId().toString()) } val channel = Channel(id=tl_channel.getId(), access_hash=tl_channel.getAccessHash(), title=tl_channel.getTitle(), obj=tl_peer_channel, download=download) if (tl_channel.getMegagroup()) { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/GUIController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/GUIController.kt deleted file mode 100644 index b88acf6..0000000 --- a/src/main/kotlin/de/fabianonline/telegram_backup/GUIController.kt +++ /dev/null @@ -1,85 +0,0 @@ -/* Telegram_Backup - * Copyright (C) 2016 Fabian Schlenz - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . */ - -package de.fabianonline.telegram_backup - -import javax.swing.* -import javax.swing.event.ListSelectionEvent -import javax.swing.event.ListSelectionListener -import java.awt.* -import java.awt.event.ActionEvent -import java.awt.event.ActionListener -import java.util.Vector - -class GUIController() { - init { - showAccountChooserDialog() - } - - private fun showAccountChooserDialog() { - val accountChooser = JDialog() - accountChooser.setTitle("Choose account") - accountChooser.setSize(400, 200) - val vert = JPanel() - vert.setLayout(BorderLayout()) - vert.add(JLabel("Please select the account to use or create a new one."), BorderLayout.NORTH) - val accounts = Utils.getAccounts() - val list = JList(accounts) - list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION) - vert.add(list, BorderLayout.CENTER) - val bottom = JPanel(GridLayout(1, 2)) - val btnAddAccount = JButton("Add account") - bottom.add(btnAddAccount) - val btnLogin = JButton("Login") - btnLogin.setEnabled(false) - bottom.add(btnLogin) - vert.add(bottom, BorderLayout.SOUTH) - accountChooser.add(vert) - accountChooser.setVisible(true) - accountChooser.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE) - } - - private fun addAccountDialog() { - val loginDialog = JDialog() - loginDialog.setTitle("Add an account") - loginDialog.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE) - - val sections = JPanel() - sections.setLayout(BoxLayout(sections, BoxLayout.Y_AXIS)) - - val top = JPanel() - top.setLayout(BoxLayout(top, BoxLayout.Y_AXIS)) - top.add(JLabel("Please enter your phone number in international format:")) - top.add(JTextField("+49123773212")) - - sections.add(top) - sections.add(Box.createVerticalStrut(5)) - sections.add(JSeparator(SwingConstants.HORIZONTAL)) - - val middle = JPanel() - middle.setLayout(BoxLayout(middle, BoxLayout.Y_AXIS)) - middle.add(JLabel("Telegram sent you a code. Enter it here:")) - middle.add(JTextField()) - middle.setEnabled(false) - - sections.add(middle) - sections.add(Box.createVerticalStrut(5)) - sections.add(JSeparator(SwingConstants.HORIZONTAL)) - - loginDialog.add(sections) - loginDialog.setVisible(true) - } -} diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt index 5c6e962..3fe4ba9 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -4,20 +4,15 @@ import java.io.File import org.slf4j.LoggerFactory import org.slf4j.Logger -object IniSettings { +class IniSettings(val file_base: String) { val logger = LoggerFactory.getLogger(IniSettings::class.java) var settings = mutableMapOf>() init { - if (UserManager.getInstance().user != null) { - loadIni(UserManager.getInstance().fileBase + "config.ini") - copySampleIni(UserManager.getInstance().fileBase + "config.sample.ini") - } + loadIni(file_base + "config.ini") + copySampleIni(file_base + "config.sample.ini") } - // Dummy function that can be called to force this object to run its init-code. - fun load() { } - private fun loadIni(filename: String) { val file = File(filename) logger.trace("Checking ini file {}", filename.anonymize()) @@ -66,20 +61,12 @@ object IniSettings { } fun getArray(key: String): List = settings.get(key) ?: listOf() - val gmaps_key: String - get() = getString("gmaps_key", default=Config.SECRET_GMAPS)!! - val pagination: Boolean - get() = getBoolean("pagination", default=true) - val pagination_size: Int - get() = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!! - val download_media: Boolean - get() = getBoolean("download_media", default=true) - val download_channels: Boolean - get() = getBoolean("download_channels", default=false) - val download_supergroups: Boolean - get() = getBoolean("download_supergroups", default=false) - val whitelist_channels: List? - get() = getStringList("whitelist_channels") - val blacklist_channels: List? - get() = getStringList("blacklist_channels") + val gmaps_key = getString("gmaps_key", default=Config.SECRET_GMAPS)!! + val pagination = getBoolean("pagination", default=true) + val pagination_size = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!! + val download_media = getBoolean("download_media", default=true) + val download_channels = getBoolean("download_channels", default=false) + val download_supergroups = getBoolean("download_supergroups", default=false) + val whitelist_channels = getStringList("whitelist_channels") + val blacklist_channels = getStringList("blacklist_channels") } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/LoginManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/LoginManager.kt new file mode 100644 index 0000000..0c3a299 --- /dev/null +++ b/src/main/kotlin/de/fabianonline/telegram_backup/LoginManager.kt @@ -0,0 +1,87 @@ +package de.fabianonline.telegram_backup + +import com.github.badoualy.telegram.tl.api.auth.TLSentCode + +class LoginManager(val app: TelegramApp, val target_dir: String, val phoneToUse: String?) { + fun run() { + var phone: String + + if (phoneToUse == null) { + println("Please enter your phone number in international format.") + println("Example: +4917077651234") + phone = getLine() + } else { + phone = phoneToUse + } + + val file_base = CommandLineController.build_file_base(target_dir, phone) + + // We now have an account, so we can create an ApiStorage and TelegramClient. + val storage = ApiStorage(file_base) + val client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, null) + + val sent_code = send_code_to_phone_number(client, phone) + println("Telegram sent you a code. Please enter it here.") + val code = getLine() + + try { + verify_code(client, phone, sent_code, code) + } catch(e: PasswordNeededException) { + println("We also need your account password. Please enter it now. It should not be printed, so it's okay if you see nothing while typing it.") + val pw = getPassword() + verify_password(client, pw) + } + System.out.println("Everything seems fine. Please run this tool again with '--account +" + user.user!!.getPhone().anonymize() + " to use this account.") + } + + private fun send_code_to_phone_number(client: TelegramClient, phone: String): TLSentCode { + return client.authSendCode(false, phone, true) + } + + private fun verify_code(client: TelegramClient, phone: String, sent_code: TLSentCode, code: String): TLUser { + try { + val auth = client.authSignIn(phone, sent_code.getPhoneCodeHash(), code) + return auth.getUser().getAsUser() + } catch (e: RpcErrorException) { + if (e.getCode() == 401 && e.getTag()=="SESSION_PASSWORD_NEEDED") { + throw PasswordNeededException() + } else { + throw e + } + } + } + + private fun verify_password(client: TelegramClient, password: String): TLUser { + val pw = password.toByteArray(charset = Charsets.UTF_8) + val salt = (client.accountGetPassword() as TLPassword).getCurrentSalt().getData() + val md = MessageDigest.getInstance("SHA-256") + + val salted = ByteArray(2 * salt.size + pw.size) + System.arraycopy(salt, 0, salted, 0, salt.size) + System.arraycopy(pw, 0, salted, salt.size, pw.size) + System.arraycopy(salt, 0, salted, salt.size + pw.size, salt.size) + val hash = md.digest(salted) + val auth = client.authCheckPassword(TLBytes(hash)) + return auth.getUser().getAsUser() + } + + + private fun getLine(): String { + if (System.console() != null) { + return System.console().readLine("> ") + } else { + print("> ") + return Scanner(System.`in`).nextLine() + } + } + + private fun getPassword(): String { + if (System.console() != null) { + return String(System.console().readPassword("> ")) + } else { + return getLine() + } + } +} + +class PasswordNeededException: Exception("A password is needed to be able to login to this account.") {} diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/UserManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/UserManager.kt index a89b5fd..02e83fb 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/UserManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/UserManager.kt @@ -34,107 +34,34 @@ import java.io.File import org.slf4j.LoggerFactory import org.slf4j.Logger -class UserManager @Throws(IOException::class) -private constructor(c: TelegramClient) { - var user: TLUser? = null - var phone: String? = null - private var code: String? = null - private val client: TelegramClient - private var sent_code: TLSentCode? = null - private var auth: TLAuthorization? = null - var isPasswordNeeded = false - private set - - val loggedIn: Boolean - get() = user != null - - val userString: String - get() { - if (this.user == null) return "Not logged in" - val sb = StringBuilder() - if (this.user!!.getFirstName() != null) { - sb.append(this.user!!.getFirstName()) - } - if (this.user!!.getLastName() != null) { - sb.append(" ") - sb.append(this.user!!.getLastName()) - } - if (this.user!!.getUsername() != null) { - sb.append(" (@") - sb.append(this.user!!.getUsername()) - sb.append(")") - } - return sb.toString() - } - - val fileBase: String - get() = Config.FILE_BASE + File.separatorChar + "+" + this.user!!.getPhone() + File.separatorChar - +class UserManager(val client: TelegramClient) { + val tl_user: TLUser + val logger = LoggerFactory.getLogger(UserManager::class.java) + val phone: String + get() = "+" + tl_user.getPhone() + val id: Int + get() = tl_user.getId() + init { - this.client = c logger.debug("Calling getFullUser") - try { - val full_user = this.client.usersGetFullUser(TLInputUserSelf()) - this.user = full_user.getUser().getAsUser() - } catch (e: RpcErrorException) { - // This may happen. Ignoring it. - logger.debug("Ignoring exception:", e) - } - + val full_user = client.usersGetFullUser(TLInputUserSelf()) + tl_user = full_user.getUser().getAsUser() } - @Throws(RpcErrorException::class, IOException::class) - fun sendCodeToPhoneNumber(number: String) { - this.phone = number - this.sent_code = this.client.authSendCode(false, number, true) - } - - @Throws(RpcErrorException::class, IOException::class) - fun verifyCode(code: String) { - this.code = code - try { - this.auth = client.authSignIn(phone, this.sent_code!!.getPhoneCodeHash(), this.code) - this.user = auth!!.getUser().getAsUser() - } catch (e: RpcErrorException) { - if (e.getCode() != 401 || !e.getTag().equals("SESSION_PASSWORD_NEEDED")) throw e - this.isPasswordNeeded = true + fun toString(): String { + val sb = StringBuilder() + sb.append(tl_user.getFirstName() ?: "") + if (tl_user.getLastName() != null) { + sb.append(" ") + sb.append(tl_user.getLastName()) } - - } - - @Throws(RpcErrorException::class, IOException::class) - fun verifyPassword(pw: String) { - val password = pw.toByteArray(charset = Charsets.UTF_8) - val salt = (client.accountGetPassword() as TLPassword).getCurrentSalt().getData() - var md: MessageDigest - try { - md = MessageDigest.getInstance("SHA-256") - } catch (e: NoSuchAlgorithmException) { - e.printStackTrace() - return - } - - val salted = ByteArray(2 * salt.size + password.size) - System.arraycopy(salt, 0, salted, 0, salt.size) - System.arraycopy(password, 0, salted, salt.size, password.size) - System.arraycopy(salt, 0, salted, salt.size + password.size, salt.size) - val hash = md.digest(salted) - auth = client.authCheckPassword(TLBytes(hash)) - this.user = auth!!.getUser().getAsUser() - } - - companion object { - private val logger = LoggerFactory.getLogger(UserManager::class.java) - internal var instance: UserManager? = null - - @Throws(IOException::class) - fun init(c: TelegramClient) { - instance = UserManager(c) - } - - fun getInstance(): UserManager { - if (instance == null) throw RuntimeException("UserManager is not yet initialized.") - return instance!! + if (tl_user.getUsername() != null) { + sb.append(" (@") + sb.append(tl_user.getUsername()) + sb.append(")") } + return sb.toString() } + + } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt index 68d495a..b557b64 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt @@ -43,7 +43,7 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils -abstract class AbstractMediaFileManager(protected var message: TLMessage, protected var user: UserManager, protected var client: TelegramClient) { +abstract class AbstractMediaFileManager(protected var message: TLMessage, protected var user: UserManager) { open var isEmpty = false abstract val size: Int abstract val extension: String diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt index e032d9c..6fcf340 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt @@ -42,7 +42,7 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils -open class DocumentFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : AbstractMediaFileManager(msg, user, client) { +open class DocumentFileManager(msg: TLMessage, user: UserManager) : AbstractMediaFileManager(msg, user) { protected var doc: TLDocument? = null override lateinit var extension: String diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt index b48a687..d3fb4f8 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt @@ -42,29 +42,29 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils object FileManagerFactory { - fun getFileManager(m: TLMessage?, u: UserManager, c: TelegramClient): AbstractMediaFileManager? { + fun getFileManager(m: TLMessage?, u: UserManager): AbstractMediaFileManager? { if (m == null) return null val media = m.getMedia() ?: return null if (media is TLMessageMediaPhoto) { - return PhotoFileManager(m, u, c) + return PhotoFileManager(m, u) } else if (media is TLMessageMediaDocument) { - val d = DocumentFileManager(m, u, c) + val d = DocumentFileManager(m, u) return if (d.isSticker) { - StickerFileManager(m, u, c) + StickerFileManager(m, u) } else d } else if (media is TLMessageMediaGeo) { - return GeoFileManager(m, u, c) + return GeoFileManager(m, u) } else if (media is TLMessageMediaEmpty) { - return UnsupportedFileManager(m, u, c, "empty") + return UnsupportedFileManager(m, u, "empty") } else if (media is TLMessageMediaUnsupported) { - return UnsupportedFileManager(m, u, c, "unsupported") + return UnsupportedFileManager(m, u, "unsupported") } else if (media is TLMessageMediaWebPage) { - return UnsupportedFileManager(m, u, c, "webpage") + return UnsupportedFileManager(m, u, "webpage") } else if (media is TLMessageMediaContact) { - return UnsupportedFileManager(m, u, c, "contact") + return UnsupportedFileManager(m, u, "contact") } else if (media is TLMessageMediaVenue) { - return UnsupportedFileManager(m, u, c, "venue") + return UnsupportedFileManager(m, u, "venue") } else { AbstractMediaFileManager.throwUnexpectedObjectError(media) } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt index 718761b..03f2ebe 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt @@ -43,7 +43,7 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils -class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : AbstractMediaFileManager(msg, user, client) { +class GeoFileManager(msg: TLMessage, user: UserManager) : AbstractMediaFileManager(msg, user) { protected lateinit var geo: TLGeoPoint // We don't know the size, so we just guess. diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt index e5c5c77..6ff750d 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt @@ -42,7 +42,7 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils -class PhotoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : AbstractMediaFileManager(msg, user, client) { +class PhotoFileManager(msg: TLMessage, user: UserManager) : AbstractMediaFileManager(msg, user) { private lateinit var photo: TLPhoto override var size = 0 private lateinit var photo_size: TLPhotoSize diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt index 04f08b9..619dd34 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt @@ -50,7 +50,7 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils -class StickerFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : DocumentFileManager(msg, user, client) { +class StickerFileManager(msg: TLMessage, user: UserManager) : DocumentFileManager(msg, user) { override val isSticker = true diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt index dec94cf..8038cd9 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt @@ -43,7 +43,7 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils -class UnsupportedFileManager(msg: TLMessage, user: UserManager, client: TelegramClient, type: String) : AbstractMediaFileManager(msg, user, client) { +class UnsupportedFileManager(msg: TLMessage, user: UserManager, type: String) : AbstractMediaFileManager(msg, user) { override var name = type override val targetFilename = "" override val targetPath = "" From ebff71b208a896e30c07ce87fda66e1b3f2449b0 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 20 Mar 2018 06:44:36 +0100 Subject: [PATCH 04/16] WIP: Even moar rewriting. --- .../telegram_backup/CommandLineController.kt | 39 +++++++------ .../telegram_backup/CommandLineOptions.kt | 3 +- .../fabianonline/telegram_backup/Database.kt | 10 +--- .../telegram_backup/DownloadManager.kt | 1 - .../telegram_backup/LoginManager.kt | 11 +++- .../telegram_backup/StickerConverter.kt | 52 ----------------- .../telegram_backup/TelegramUpdateHandler.kt | 58 ++++++++----------- .../telegram_backup/TestFeatures.kt | 34 ++++------- .../telegram_backup/UserManager.kt | 2 +- .../de/fabianonline/telegram_backup/Utils.kt | 4 +- .../telegram_backup/exporter/HTMLExporter.kt | 9 +-- .../AbstractMediaFileManager.kt | 21 +------ .../mediafilemanager/DocumentFileManager.kt | 19 +----- .../mediafilemanager/FileManagerFactory.kt | 1 - .../mediafilemanager/GeoFileManager.kt | 20 +------ .../mediafilemanager/PhotoFileManager.kt | 19 +----- .../mediafilemanager/StickerFileManager.kt | 18 +----- .../UnsupportedFileManager.kt | 25 +------- 18 files changed, 83 insertions(+), 263 deletions(-) delete mode 100644 src/main/kotlin/de/fabianonline/telegram_backup/StickerConverter.kt diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 25e96fb..95fe9a3 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -30,6 +30,8 @@ import org.slf4j.LoggerFactory import org.slf4j.Logger class CommandLineController(val options: CommandLineOptions) { + val logger = LoggerFactory.getLogger(CommandLineController::class.java) + init { val storage: ApiStorage val app: TelegramApp @@ -37,7 +39,7 @@ class CommandLineController(val options: CommandLineOptions) { val file_base: String val phone_number: String val handler: TelegramUpdateHandler - val client: TelegramClient + var client: TelegramClient val user_manager: UserManager val inisettings: IniSettings val database: Database @@ -91,11 +93,8 @@ class CommandLineController(val options: CommandLineOptions) { logger.info("Initializing ApiStorage") storage = ApiStorage(file_base) - logger.info("Initializing TelegramUpdateHandler") - handler = TelegramUpdateHandler() - logger.info("Creating Client") - client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, handler) + client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, null) // From now on we have a new catch-all-block that will terminate it's TelegramClient when an exception happens. try { @@ -126,9 +125,9 @@ class CommandLineController(val options: CommandLineOptions) { if (options.val_test != null) { if (options.val_test == 1) { - TestFeatures.test1() + TestFeatures(database).test1() } else if (options.val_test == 2) { - TestFeatures.test2() + TestFeatures(database).test2() } else { System.out.println("Unknown test " + options.val_test) } @@ -139,7 +138,7 @@ class CommandLineController(val options: CommandLineOptions) { logger.debug("options.val_export: {}", export) if (export != null) { if (export.toLowerCase().equals("html")) { - HTMLExporter().export() + HTMLExporter(database, user_manager, ini=inisettings, file_base=file_base).export() System.exit(0) } else { show_error("Unknown export format '${export}'.") @@ -184,22 +183,24 @@ class CommandLineController(val options: CommandLineOptions) { } else { println("Skipping media download because download_media is set to false.") } + + if (options.cmd_daemon) { + logger.info("Initializing TelegramUpdateHandler") + handler = TelegramUpdateHandler(user_manager, database) + client.close() + logger.info("Creating new client") + client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, handler) + println("DAEMON mode requested - keeping running.") + } } catch (e: Throwable) { println("An error occured!") e.printStackTrace() logger.error("Exception caught!", e) - // If we encountered an exception, we definitely don't want to start the daemon mode now. - options.cmd_daemon = false } finally { - if (options.cmd_daemon) { - handler.activate() - println("DAEMON mode requested - keeping running.") - } else { - client.close() - println() - println("----- EXIT -----") - System.exit(0) - } + client.close() + println() + println("----- EXIT -----") + System.exit(0) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index 2971a6b..072d486 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -26,7 +26,6 @@ class CommandLineOptions(args: Array) { var cmd_version = false var cmd_license = false var cmd_daemon = false - var cmd_anonymize = false var cmd_stats = false var cmd_list_channels = false var val_account: String? = null @@ -84,7 +83,7 @@ class CommandLineOptions(args: Array) { last_cmd = "--test" continue@loop } - "--anonymize" -> cmd_anonymize = true + "--anonymize" -> Utils.anonymize = true "--stats" -> cmd_stats = true "--list-channels" -> cmd_list_channels = true else -> throw RuntimeException("Unknown command " + arg) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index 3e3270c..c09e5bf 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -69,7 +69,7 @@ class Database constructor(val file_base: String, val user_manager: UserManager) } // Run updates - val updates = DatabaseUpdates(conn!!, this) + val updates = DatabaseUpdates(conn, this) updates.doUpdates() println("Database is ready.") @@ -489,13 +489,7 @@ class Database constructor(val file_base: String, val user_manager: UserManager) ps_insert_or_replace.close() } - fun fetchSetting(key: String): String? { - val rs = stmt.executeQuery("SELECT value FROM settings WHERE key='${key}'") - rs.next() - val result = rs.getString(1) - rs.close() - return result - } + fun fetchSetting(key: String): String? = queryString("SELECT value FROM settings WHERE key='${key}'") fun saveSetting(key: String, value: String?) { val ps = conn.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt index b953e4d..9913a51 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt @@ -18,7 +18,6 @@ package de.fabianonline.telegram_backup import de.fabianonline.telegram_backup.UserManager import de.fabianonline.telegram_backup.Database -import de.fabianonline.telegram_backup.StickerConverter import de.fabianonline.telegram_backup.DownloadProgressInterface import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/LoginManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/LoginManager.kt index 0c3a299..86aefe0 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/LoginManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/LoginManager.kt @@ -1,6 +1,15 @@ package de.fabianonline.telegram_backup +import com.github.badoualy.telegram.api.Kotlogram +import com.github.badoualy.telegram.api.TelegramApp +import com.github.badoualy.telegram.api.TelegramClient +import com.github.badoualy.telegram.tl.api.TLUser +import com.github.badoualy.telegram.tl.api.account.TLPassword import com.github.badoualy.telegram.tl.api.auth.TLSentCode +import com.github.badoualy.telegram.tl.core.TLBytes +import com.github.badoualy.telegram.tl.exception.RpcErrorException +import java.security.MessageDigest +import java.util.* class LoginManager(val app: TelegramApp, val target_dir: String, val phoneToUse: String?) { fun run() { @@ -31,7 +40,7 @@ class LoginManager(val app: TelegramApp, val target_dir: String, val phoneToUse: val pw = getPassword() verify_password(client, pw) } - System.out.println("Everything seems fine. Please run this tool again with '--account +" + user.user!!.getPhone().anonymize() + " to use this account.") + System.out.println("Everything seems fine. Please run this tool again with '--account ${phone} to use this account.") } private fun send_code_to_phone_number(client: TelegramClient, phone: String): TLSentCode { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/StickerConverter.kt b/src/main/kotlin/de/fabianonline/telegram_backup/StickerConverter.kt deleted file mode 100644 index ad77560..0000000 --- a/src/main/kotlin/de/fabianonline/telegram_backup/StickerConverter.kt +++ /dev/null @@ -1,52 +0,0 @@ -/* Telegram_Backup - * Copyright (C) 2016 Fabian Schlenz - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . */ - -package de.fabianonline.telegram_backup - -import com.github.badoualy.telegram.tl.api.* -import java.lang.StringBuilder -import java.io.File - -object StickerConverter { - fun makeFilenameWithPath(attr: TLDocumentAttributeSticker): String { - val file = StringBuilder() - file.append(makePath()) - file.append(makeFilename(attr)) - return file.toString() - } - - fun makeFilename(attr: TLDocumentAttributeSticker): String { - val file = StringBuilder() - if (attr.getStickerset() is TLInputStickerSetShortName) { - file.append((attr.getStickerset() as TLInputStickerSetShortName).getShortName()) - } else if (attr.getStickerset() is TLInputStickerSetID) { - file.append((attr.getStickerset() as TLInputStickerSetID).getId()) - } - file.append("_") - file.append(attr.getAlt().hashCode()) - file.append(".webp") - return file.toString() - } - - fun makePath(): String { - val path = Config.FILE_BASE + - File.separatorChar + - Config.FILE_STICKER_BASE + - File.separatorChar - File(path).mkdirs() - return path - } -} diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt b/src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt index ef42d13..0b3073a 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt @@ -26,48 +26,39 @@ import de.fabianonline.telegram_backup.Database import de.fabianonline.telegram_backup.UserManager import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory +import org.slf4j.LoggerFactory -internal class TelegramUpdateHandler : UpdateCallback { - private var user: UserManager? = null - private var db: Database? = null - var debug = false - - fun activate() { - this.user = UserManager.getInstance() - this.db = Database.getInstance() - } +internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Database) : UpdateCallback { + val logger = LoggerFactory.getLogger(TelegramUpdateHandler::class.java) override fun onUpdates(client: TelegramClient, updates: TLUpdates) { - if (db == null) return - if (debug) System.out.println("onUpdates - " + updates.getUpdates().size + " Updates, " + updates.getUsers().size + " Users, " + updates.getChats().size + " Chats") + + logger.debug("onUpdates - " + updates.getUpdates().size + " Updates, " + updates.getUsers().size + " Users, " + updates.getChats().size + " Chats") for (update in updates.getUpdates()) { processUpdate(update, client) - if (debug) System.out.println(" " + update.javaClass.getName()) + logger.debug(" " + update.javaClass.getName()) } - db!!.saveUsers(updates.getUsers()) - db!!.saveChats(updates.getChats()) + db.saveUsers(updates.getUsers()) + db.saveChats(updates.getChats()) } override fun onUpdatesCombined(client: TelegramClient, updates: TLUpdatesCombined) { - if (db == null) return - if (debug) System.out.println("onUpdatesCombined") + logger.debug("onUpdatesCombined") for (update in updates.getUpdates()) { processUpdate(update, client) } - db!!.saveUsers(updates.getUsers()) - db!!.saveChats(updates.getChats()) + db.saveUsers(updates.getUsers()) + db.saveChats(updates.getChats()) } override fun onUpdateShort(client: TelegramClient, update: TLUpdateShort) { - if (db == null) return - if (debug) System.out.println("onUpdateShort") + logger.debug("onUpdateShort") processUpdate(update.getUpdate(), client) - if (debug) System.out.println(" " + update.getUpdate().javaClass.getName()) + logger.debug(" " + update.getUpdate().javaClass.getName()) } override fun onShortChatMessage(client: TelegramClient, message: TLUpdateShortChatMessage) { - if (db == null) return - if (debug) System.out.println("onShortChatMessage - " + message.getMessage()) + logger.debug("onShortChatMessage - " + message.getMessage()) val msg = TLMessage( message.getOut(), message.getMentioned(), @@ -85,21 +76,20 @@ internal class TelegramUpdateHandler : UpdateCallback { message.getEntities(), null, null) val vector = TLVector(TLAbsMessage::class.java) vector.add(msg) - db!!.saveMessages(vector, Kotlogram.API_LAYER) + db.saveMessages(vector, Kotlogram.API_LAYER) System.out.print('.') } override fun onShortMessage(client: TelegramClient, message: TLUpdateShortMessage) { val m = message - if (db == null) return - if (debug) System.out.println("onShortMessage - " + m.getOut() + " - " + m.getUserId() + " - " + m.getMessage()) + logger.debug("onShortMessage - " + m.getOut() + " - " + m.getUserId() + " - " + m.getMessage()) val from_id: Int val to_id: Int if (m.getOut() == true) { - from_id = user!!.user!!.getId() + from_id = user_manager.id to_id = m.getUserId() } else { - to_id = user!!.user!!.getId() + to_id = user_manager.id from_id = m.getUserId() } val msg = TLMessage( @@ -119,18 +109,16 @@ internal class TelegramUpdateHandler : UpdateCallback { m.getEntities(), null, null) val vector = TLVector(TLAbsMessage::class.java) vector.add(msg) - db!!.saveMessages(vector, Kotlogram.API_LAYER) + db.saveMessages(vector, Kotlogram.API_LAYER) System.out.print('.') } override fun onShortSentMessage(client: TelegramClient, message: TLUpdateShortSentMessage) { - if (db == null) return - System.out.println("onShortSentMessage") + logger.debug("onShortSentMessage") } override fun onUpdateTooLong(client: TelegramClient) { - if (db == null) return - System.out.println("onUpdateTooLong") + logger.debug("onUpdateTooLong") } private fun processUpdate(update: TLAbsUpdate, client: TelegramClient) { @@ -138,10 +126,10 @@ internal class TelegramUpdateHandler : UpdateCallback { val abs_msg = update.getMessage() val vector = TLVector(TLAbsMessage::class.java) vector.add(abs_msg) - db!!.saveMessages(vector, Kotlogram.API_LAYER) + db.saveMessages(vector, Kotlogram.API_LAYER) System.out.print('.') if (abs_msg is TLMessage) { - val fm = FileManagerFactory.getFileManager(abs_msg, user!!, client) + val fm = FileManagerFactory.getFileManager(abs_msg, user_manager) if (fm != null && !fm.isEmpty && !fm.downloaded) { try { fm.download() diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/TestFeatures.kt b/src/main/kotlin/de/fabianonline/telegram_backup/TestFeatures.kt index cc79f64..ccdf1f3 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/TestFeatures.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/TestFeatures.kt @@ -10,7 +10,7 @@ import java.sql.ResultSet import java.io.IOException import java.nio.charset.Charset -internal object TestFeatures { +internal class TestFeatures(val db: Database) { fun test1() { // Tests entries in a cache4.db in the current working directory for compatibility try { @@ -24,31 +24,22 @@ internal object TestFeatures { var conn: Connection var stmt: Statement? = null - try { - conn = DriverManager.getConnection(path) - stmt = conn.createStatement() - } catch (e: SQLException) { - CommandLineController.show_error("Could not connect to SQLITE database.") - } + conn = DriverManager.getConnection(path) + stmt = conn.createStatement() var unsupported_constructor = 0 var success = 0 - try { - val rs = stmt!!.executeQuery("SELECT data FROM messages") - while (rs.next()) { - try { - TLApiContext.getInstance().deserializeMessage(rs.getBytes(1)) - } catch (e: com.github.badoualy.telegram.tl.exception.UnsupportedConstructorException) { - unsupported_constructor++ - } catch (e: IOException) { - System.out.println("IOException: " + e) - } - - success++ + val rs = stmt.executeQuery("SELECT data FROM messages") + while (rs.next()) { + try { + TLApiContext.getInstance().deserializeMessage(rs.getBytes(1)) + } catch (e: com.github.badoualy.telegram.tl.exception.UnsupportedConstructorException) { + unsupported_constructor++ + } catch (e: IOException) { + System.out.println("IOException: " + e) } - } catch (e: SQLException) { - System.out.println("SQL exception: " + e) + success++ } System.out.println("Success: " + success) @@ -59,7 +50,6 @@ internal object TestFeatures { // Prints system.encoding and default charset System.out.println("Default Charset: " + Charset.defaultCharset()) System.out.println("file.encoding: " + System.getProperty("file.encoding")) - val db = Database.getInstance() System.out.println("Database encoding: " + db.getEncoding()) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/UserManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/UserManager.kt index 02e83fb..0786793 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/UserManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/UserManager.kt @@ -48,7 +48,7 @@ class UserManager(val client: TelegramClient) { tl_user = full_user.getUser().getAsUser() } - fun toString(): String { + override fun toString(): String { val sb = StringBuilder() sb.append(tl_user.getFirstName() ?: "") if (tl_user.getLastName() != null) { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt index 00de1d9..8644c39 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt @@ -32,6 +32,8 @@ object Utils { @JvmField public val VERSION_1_NEWER = 1 @JvmField public val VERSION_2_NEWER = 2 + var anonymize = false + private val logger = LoggerFactory.getLogger(Utils::class.java) as Logger fun print_accounts(file_base: String) { @@ -193,7 +195,7 @@ object Utils { } fun String.anonymize(): String { - return if (!CommandLineOptions.cmd_anonymize) this else this.replace(Regex("[0-9]"), "1").replace(Regex("[A-Z]"), "A").replace(Regex("[a-z]"), "a") + " (ANONYMIZED)" + return if (Utils.anonymize) this else this.replace(Regex("[0-9]"), "1").replace(Regex("[A-Z]"), "A").replace(Regex("[a-z]"), "a") + " (ANONYMIZED)" } fun Any.toJson(): String = Gson().toJson(this) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt b/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt index 1053946..586c37f 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt @@ -37,18 +37,15 @@ import de.fabianonline.telegram_backup.* import org.slf4j.Logger import org.slf4j.LoggerFactory -class HTMLExporter { - val db = Database.getInstance() - val user = UserManager.getInstance() - +class HTMLExporter(val db: Database, val user: UserManager, val ini: IniSettings, val file_base: String) { @Throws(IOException::class) fun export() { try { - val pagination = if (IniSettings.pagination) IniSettings.pagination_size else -1 + val pagination = if (ini.pagination) ini.pagination_size else -1 // Create base dir logger.debug("Creating base dir") - val base = user.fileBase + "files" + File.separatorChar + val base = file_base + "files" + File.separatorChar File(base).mkdirs() File(base + "dialogs").mkdirs() diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt index b557b64..f082df0 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt @@ -17,33 +17,16 @@ package de.fabianonline.telegram_backup.mediafilemanager import de.fabianonline.telegram_backup.UserManager -import de.fabianonline.telegram_backup.Database -import de.fabianonline.telegram_backup.StickerConverter -import de.fabianonline.telegram_backup.DownloadProgressInterface import de.fabianonline.telegram_backup.Config -import de.fabianonline.telegram_backup.DownloadManager -import com.github.badoualy.telegram.api.TelegramClient -import com.github.badoualy.telegram.tl.core.TLIntVector -import com.github.badoualy.telegram.tl.core.TLObject -import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages -import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs import com.github.badoualy.telegram.tl.api.* -import com.github.badoualy.telegram.tl.api.upload.TLFile import com.github.badoualy.telegram.tl.exception.RpcErrorException -import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile import java.io.IOException import java.io.File -import java.io.FileOutputStream -import java.util.ArrayList -import java.util.LinkedList -import java.net.URL import java.util.concurrent.TimeoutException -import org.apache.commons.io.FileUtils - -abstract class AbstractMediaFileManager(protected var message: TLMessage, protected var user: UserManager) { +abstract class AbstractMediaFileManager(protected var message: TLMessage, protected var user: UserManager, val file_base: String) { open var isEmpty = false abstract val size: Int abstract val extension: String @@ -56,7 +39,7 @@ abstract class AbstractMediaFileManager(protected var message: TLMessage, protec open val targetPath: String get() { - val path = user.fileBase + Config.FILE_FILES_BASE + File.separatorChar + val path = file_base + Config.FILE_FILES_BASE + File.separatorChar File(path).mkdirs() return path } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt index 6fcf340..fc351d4 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt @@ -17,32 +17,15 @@ package de.fabianonline.telegram_backup.mediafilemanager import de.fabianonline.telegram_backup.UserManager -import de.fabianonline.telegram_backup.Database -import de.fabianonline.telegram_backup.StickerConverter -import de.fabianonline.telegram_backup.DownloadProgressInterface import de.fabianonline.telegram_backup.DownloadManager -import com.github.badoualy.telegram.api.TelegramClient -import com.github.badoualy.telegram.tl.core.TLIntVector -import com.github.badoualy.telegram.tl.core.TLObject -import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages -import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs import com.github.badoualy.telegram.tl.api.* -import com.github.badoualy.telegram.tl.api.upload.TLFile import com.github.badoualy.telegram.tl.exception.RpcErrorException -import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile import java.io.IOException -import java.io.File -import java.io.FileOutputStream -import java.util.ArrayList -import java.util.LinkedList -import java.net.URL import java.util.concurrent.TimeoutException -import org.apache.commons.io.FileUtils - -open class DocumentFileManager(msg: TLMessage, user: UserManager) : AbstractMediaFileManager(msg, user) { +open class DocumentFileManager(msg: TLMessage, user: UserManager, file_base: String) : AbstractMediaFileManager(msg, user, file_base) { protected var doc: TLDocument? = null override lateinit var extension: String diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt index d3fb4f8..568f617 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt @@ -18,7 +18,6 @@ package de.fabianonline.telegram_backup.mediafilemanager import de.fabianonline.telegram_backup.UserManager import de.fabianonline.telegram_backup.Database -import de.fabianonline.telegram_backup.StickerConverter import de.fabianonline.telegram_backup.DownloadProgressInterface import com.github.badoualy.telegram.api.TelegramClient diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt index 03f2ebe..2d68c3c 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt @@ -17,33 +17,15 @@ package de.fabianonline.telegram_backup.mediafilemanager import de.fabianonline.telegram_backup.UserManager -import de.fabianonline.telegram_backup.Database -import de.fabianonline.telegram_backup.StickerConverter -import de.fabianonline.telegram_backup.DownloadProgressInterface import de.fabianonline.telegram_backup.DownloadManager import de.fabianonline.telegram_backup.IniSettings -import com.github.badoualy.telegram.api.TelegramClient -import com.github.badoualy.telegram.tl.core.TLIntVector -import com.github.badoualy.telegram.tl.core.TLObject -import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages -import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs import com.github.badoualy.telegram.tl.api.* -import com.github.badoualy.telegram.tl.api.upload.TLFile -import com.github.badoualy.telegram.tl.exception.RpcErrorException -import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile import java.io.IOException import java.io.File -import java.io.FileOutputStream -import java.util.ArrayList -import java.util.LinkedList -import java.net.URL -import java.util.concurrent.TimeoutException -import org.apache.commons.io.FileUtils - -class GeoFileManager(msg: TLMessage, user: UserManager) : AbstractMediaFileManager(msg, user) { +class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String) : AbstractMediaFileManager(msg, user, file_base) { protected lateinit var geo: TLGeoPoint // We don't know the size, so we just guess. diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt index 6ff750d..8421398 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt @@ -17,32 +17,15 @@ package de.fabianonline.telegram_backup.mediafilemanager import de.fabianonline.telegram_backup.UserManager -import de.fabianonline.telegram_backup.Database -import de.fabianonline.telegram_backup.StickerConverter -import de.fabianonline.telegram_backup.DownloadProgressInterface import de.fabianonline.telegram_backup.DownloadManager -import com.github.badoualy.telegram.api.TelegramClient -import com.github.badoualy.telegram.tl.core.TLIntVector -import com.github.badoualy.telegram.tl.core.TLObject -import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages -import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs import com.github.badoualy.telegram.tl.api.* -import com.github.badoualy.telegram.tl.api.upload.TLFile import com.github.badoualy.telegram.tl.exception.RpcErrorException -import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile import java.io.IOException -import java.io.File -import java.io.FileOutputStream -import java.util.ArrayList -import java.util.LinkedList -import java.net.URL import java.util.concurrent.TimeoutException -import org.apache.commons.io.FileUtils - -class PhotoFileManager(msg: TLMessage, user: UserManager) : AbstractMediaFileManager(msg, user) { +class PhotoFileManager(msg: TLMessage, user: UserManager, file_base: String) : AbstractMediaFileManager(msg, user, file_base) { private lateinit var photo: TLPhoto override var size = 0 private lateinit var photo_size: TLPhotoSize diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt index 619dd34..6985d39 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt @@ -18,7 +18,6 @@ package de.fabianonline.telegram_backup.mediafilemanager import de.fabianonline.telegram_backup.UserManager import de.fabianonline.telegram_backup.Database -import de.fabianonline.telegram_backup.StickerConverter import de.fabianonline.telegram_backup.DownloadProgressInterface import de.fabianonline.telegram_backup.DownloadManager import de.fabianonline.telegram_backup.Config @@ -50,7 +49,7 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils -class StickerFileManager(msg: TLMessage, user: UserManager) : DocumentFileManager(msg, user) { +class StickerFileManager(msg: TLMessage, user: UserManager, file_base: String) : DocumentFileManager(msg, user, file_base) { override val isSticker = true @@ -80,7 +79,7 @@ class StickerFileManager(msg: TLMessage, user: UserManager) : DocumentFileManage override val targetPath: String get() { - val path = user.fileBase + Config.FILE_FILES_BASE + File.separatorChar + Config.FILE_STICKER_BASE + File.separatorChar + val path = file_base + Config.FILE_FILES_BASE + File.separatorChar + Config.FILE_STICKER_BASE + File.separatorChar File(path).mkdirs() return path } @@ -94,19 +93,6 @@ class StickerFileManager(msg: TLMessage, user: UserManager) : DocumentFileManage override val description: String get() = "Sticker" - @Throws(RpcErrorException::class, IOException::class, TimeoutException::class) - override fun download(): Boolean { - val old_file = Config.FILE_BASE + File.separatorChar + Config.FILE_STICKER_BASE + File.separatorChar + targetFilename - - logger.trace("Old filename exists: {}", File(old_file).exists()) - - if (File(old_file).exists()) { - Files.copy(Paths.get(old_file), Paths.get(targetPathAndFilename), StandardCopyOption.REPLACE_EXISTING) - return true - } - return super.download() - } - companion object { private val logger = LoggerFactory.getLogger(StickerFileManager::class.java) } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt index 8038cd9..66ac8d2 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt @@ -17,33 +17,10 @@ package de.fabianonline.telegram_backup.mediafilemanager import de.fabianonline.telegram_backup.UserManager -import de.fabianonline.telegram_backup.Database -import de.fabianonline.telegram_backup.StickerConverter -import de.fabianonline.telegram_backup.DownloadProgressInterface -import de.fabianonline.telegram_backup.DownloadManager -import de.fabianonline.telegram_backup.Config -import com.github.badoualy.telegram.api.TelegramClient -import com.github.badoualy.telegram.tl.core.TLIntVector -import com.github.badoualy.telegram.tl.core.TLObject -import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages -import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs import com.github.badoualy.telegram.tl.api.* -import com.github.badoualy.telegram.tl.api.upload.TLFile -import com.github.badoualy.telegram.tl.exception.RpcErrorException -import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile -import java.io.IOException -import java.io.File -import java.io.FileOutputStream -import java.util.ArrayList -import java.util.LinkedList -import java.net.URL -import java.util.concurrent.TimeoutException - -import org.apache.commons.io.FileUtils - -class UnsupportedFileManager(msg: TLMessage, user: UserManager, type: String) : AbstractMediaFileManager(msg, user) { +class UnsupportedFileManager(msg: TLMessage, user: UserManager, type: String, file_base: String) : AbstractMediaFileManager(msg, user, file_base) { override var name = type override val targetFilename = "" override val targetPath = "" From 253b334fc3bca384ab66abebbd73bfa6516f07e3 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Fri, 6 Apr 2018 06:19:49 +0200 Subject: [PATCH 05/16] More rewriting. Also lots of fun with git stash and wrong branches... --- gradle/wrapper/gradle-wrapper.properties | 2 +- .../telegram_backup/CommandLineController.kt | 68 ++++---- .../telegram_backup/CommandLineOptions.kt | 89 +++-------- .../telegram_backup/CommandLineRunner.kt | 5 + .../fabianonline/telegram_backup/Database.kt | 12 +- .../fabianonline/telegram_backup/Settings.kt | 151 ++++++++++++++++++ .../mediafilemanager/FileManagerFactory.kt | 20 +-- 7 files changed, 225 insertions(+), 122 deletions(-) create mode 100644 src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index b72ab3d..5bdf9af 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,4 +1,4 @@ -#Thu Oct 06 11:24:39 CEST 2016 +#Fri Mar 23 06:07:28 CET 2018 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 95fe9a3..47a9929 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -42,16 +42,17 @@ class CommandLineController(val options: CommandLineOptions) { var client: TelegramClient val user_manager: UserManager val inisettings: IniSettings + val settings: Settings val database: Database logger.info("CommandLineController started. App version {}", Config.APP_APPVER) printHeader() - if (options.cmd_version) { + if (options.booleans.contains("version")) { System.exit(0) - } else if (options.cmd_help) { + } else if (options.booleans.contains("help")) { show_help() System.exit(0) - } else if (options.cmd_license) { + } else if (options.booleans.contains("license")) { show_license() System.exit(0) } @@ -62,26 +63,26 @@ class CommandLineController(val options: CommandLineOptions) { // Setup file_base logger.debug("Target dir from Config: {}", Config.TARGET_DIR.anonymize()) - target_dir = options.val_target ?: Config.TARGET_DIR + target_dir = options.values.get("target")?.last() ?: Config.TARGET_DIR logger.debug("Target dir after options: {}", target_dir) println("Base directory for files: ${target_dir.anonymize()}") - if (options.cmd_list_accounts) { + if (options.booleans.contains("list_accounts")) { Utils.print_accounts(target_dir) System.exit(0) } - if (options.cmd_login) { - cmd_login(app, target_dir, options.val_account) + if (options.booleans.contains("login")) { + cmd_login(app, target_dir, options.values.get("account")?.last()) } logger.trace("Checking accounts") - phone_number = try { selectAccount(target_dir, options.val_account) + phone_number = try { selectAccount(target_dir, options.values.get("account")?.last()) } catch(e: AccountNotFoundException) { show_error("The specified account could not be found.") } catch(e: NoAccountsException) { println("No accounts found. Starting login process...") - cmd_login(app, target_dir, options.val_account) + cmd_login(app, target_dir, options.values.get("account")?.last()) } // TODO: Create a new TelegramApp if the user set his/her own TelegramApp credentials @@ -112,33 +113,22 @@ class CommandLineController(val options: CommandLineOptions) { show_error("Account / User mismatch") } - // Load the ini file. - inisettings = IniSettings(file_base) - // If we reach this point, we can assume that there is an account and a database can be loaded / created. database = Database(file_base, user_manager) - if (options.cmd_stats) { + // Load the settings and stuff. + settings = Settings(file_base, database, options) + + if (options.booleans.contains("stats")) { cmd_stats(file_base, database) System.exit(0) } - if (options.val_test != null) { - if (options.val_test == 1) { - TestFeatures(database).test1() - } else if (options.val_test == 2) { - TestFeatures(database).test2() - } else { - System.out.println("Unknown test " + options.val_test) - } - System.exit(1) - } - - val export = options.val_export + val export = options.values.get("export")?.last() logger.debug("options.val_export: {}", export) if (export != null) { - if (export.toLowerCase().equals("html")) { - HTMLExporter(database, user_manager, ini=inisettings, file_base=file_base).export() + if (export.toLowerCase() == "html") { + HTMLExporter(database, user_manager, settings=settings, file_base=file_base).export() System.exit(0) } else { show_error("Unknown export format '${export}'.") @@ -150,7 +140,7 @@ class CommandLineController(val options: CommandLineOptions) { logger.info("Initializing Download Manager") val d = DownloadManager(client, CommandLineDownloadProgress(), database, user_manager, inisettings) - if (options.cmd_list_channels) { + if (options.booleans.contains("list_channels")) { val chats = d.getChats() val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) } val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")} @@ -174,8 +164,8 @@ class CommandLineController(val options: CommandLineOptions) { System.exit(0) } - logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.val_limit_messages) - d.downloadMessages(options.val_limit_messages) + logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.values.get("limit_messages")?.last()) + d.downloadMessages(options.values.get("limit_messages")?.last()?.toInt()) logger.debug("IniSettings#download_media: {}", inisettings.download_media) if (inisettings.download_media) { logger.debug("Calling DownloadManager.downloadMedia") @@ -184,7 +174,7 @@ class CommandLineController(val options: CommandLineOptions) { println("Skipping media download because download_media is set to false.") } - if (options.cmd_daemon) { + if (options.boolean.contains("daemon")) { logger.info("Initializing TelegramUpdateHandler") handler = TelegramUpdateHandler(user_manager, database) client.close() @@ -193,7 +183,7 @@ class CommandLineController(val options: CommandLineOptions) { println("DAEMON mode requested - keeping running.") } } catch (e: Throwable) { - println("An error occured!") + println("An error occurred!") e.printStackTrace() logger.error("Exception caught!", e) } finally { @@ -274,19 +264,19 @@ class CommandLineController(val options: CommandLineOptions) { private fun show_help() { println("Valid options are:") - println(" -h, --help Shows this help.") - println(" -a, --account Use account .") - println(" -l, --login Login to an existing telegram account.") + println(" --help Shows this help.") + println(" --account Use account .") + println(" --login Login to an existing telegram account.") println(" --debug Shows some debug information.") println(" --trace Shows lots of debug information. Overrides --debug.") println(" --trace-telegram Shows lots of debug messages from the library used to access Telegram.") - println(" -A, --list-accounts List all existing accounts ") + println(" --list-accounts List all existing accounts ") println(" --limit-messages Downloads at most the most recent messages.") - println(" -t, --target Target directory for the files.") - println(" -e, --export Export the database. Valid formats are:") + println(" --target Target directory for the files.") + println(" --export Export the database. Valid formats are:") println(" html - Creates HTML files.") println(" --license Displays the license of this program.") - println(" -d, --daemon Keep running after the backup and automatically save new messages.") + println(" --daemon Keep running after the backup and automatically save new messages.") println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.") println(" --stats Print some usage statistics.") println(" --list-channels Lists all channels together with their ID") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index 072d486..7808202 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -16,81 +16,28 @@ package de.fabianonline.telegram_backup class CommandLineOptions(args: Array) { - var cmd_console = false - var cmd_help = false - var cmd_login = false - var cmd_debug = false - var cmd_trace = false - var cmd_trace_telegram = false - var cmd_list_accounts = false - var cmd_version = false - var cmd_license = false - var cmd_daemon = false - var cmd_stats = false - var cmd_list_channels = false - var val_account: String? = null - var val_limit_messages: Int? = null - var val_target: String? = null - var val_export: String? = null - var val_test: Int? = null - + val booleans = mutableListOf() + val values = mutableMapOf>() + var last_key: String? = null + init { - var last_cmd: String? = null - loop@ for (arg in args) { - if (last_cmd != null) { - when (last_cmd) { - "--account" -> val_account = arg - "--limit-messages" -> val_limit_messages = Integer.parseInt(arg) - "--target" -> val_target = arg - "--export" -> val_export = arg - "--test" -> val_test = Integer.parseInt(arg) + for(arg in args) { + if (arg.starts_with("--")) { + if (last_key!=null) { + booleans.add(last_key) } - last_cmd = null - continue - } - when (arg) { - "-a", "--account" -> { - last_cmd = "--account" - continue@loop + last_key = arg.substr(2) + } else { + if (last_key==null) throw RuntimeException("Unexpected parameter without switch: $arg") + var list = values.get(last_key) + if (list==null) { + list = mutableListOf() + values.add(last_key, list) } - "-h", "--help" -> cmd_help = true - "-l", "--login" -> cmd_login = true - "--debug" -> cmd_debug = true - "--trace" -> cmd_trace = true - "--trace-telegram" -> cmd_trace_telegram = true - "-A", "--list-accounts" -> cmd_list_accounts = true - "--limit-messages" -> { - last_cmd = arg - continue@loop - } - "--console" -> cmd_console = true - "-t", "--target" -> { - last_cmd = "--target" - continue@loop - } - "-V", "--version" -> cmd_version = true - "-e", "--export" -> { - last_cmd = "--export" - continue@loop - } - "--pagination" -> { - last_cmd = "--pagination" - continue@loop - } - "--license" -> cmd_license = true - "-d", "--daemon" -> cmd_daemon = true - "--test" -> { - last_cmd = "--test" - continue@loop - } - "--anonymize" -> Utils.anonymize = true - "--stats" -> cmd_stats = true - "--list-channels" -> cmd_list_channels = true - else -> throw RuntimeException("Unknown command " + arg) + list.add(arg) + last_key = null } } - if (last_cmd != null) { - CommandLineController.show_error("Command $last_cmd had no parameter set.") - } + if (last_key!=null) booleans.add(last_key) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt index 2f6d334..5728c32 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt @@ -53,6 +53,11 @@ class CommandLineRunner(args: Array) { } fun setupLogging() { + if (options.booleans.hasKey('anonymize')) { + Utils.anonymize = true + } + + logger.trace("Setting up Loggers...") val rootLogger = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) as Logger val rootContext = rootLogger.getLoggerContext() diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index c09e5bf..b71cb2c 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -240,6 +240,16 @@ class Database constructor(val file_base: String, val user_manager: UserManager) rs.close() return result } + + fun queryStringMap(query: String): Map { + val map = mutableMapOf() + val rs = stmt.executeQuery(query) + while(rs.next()) { + map.add(rs.getString(1), rs.getString(2)) + } + rs.close() + return map + } @Synchronized fun saveMessages(all: TLVector, api_layer: Int, source_type: MessageSource = MessageSource.NORMAL) { @@ -489,7 +499,7 @@ class Database constructor(val file_base: String, val user_manager: UserManager) ps_insert_or_replace.close() } - fun fetchSetting(key: String): String? = queryString("SELECT value FROM settings WHERE key='${key}'") + fun fetchSettings() = queryStringMap("SELECT key, value FROM settings WHERE key='${key}'") fun saveSetting(key: String, value: String?) { val ps = conn.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt new file mode 100644 index 0000000..9cce5eb --- /dev/null +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt @@ -0,0 +1,151 @@ +package de.fabianonline.telegram_backup + +import java.io.File +import org.slf4j.LoggerFactory + +class Settings(val file_base: String, val database: Database, val cli_settings: CommandLineOptions) { + val logger = LoggerFactory.getLogger(Settings::class.java) + val settings = mutableMapOf() + + val gmaps_key: Setting + + private val db_settings: Map + + private var ini_settings: Map> + + init { + db_settings = database.fetchSettings() + ini_settings = load_ini("config.ini") + copy_sample_ini("config.sample.ini") + + // Merging CLI and INI settings + gmaps_key = get_setting("gmaps_key", default=Config.SECRET_GMAPS) + } + + private fun get_setting(name: String, default: String? = null): Setting { + return Setting.build(name, ini_settings[name], cli_settings.values[name], default) + } + + private fun load_ini(filename: String): Map> { + val map = mutableMapOf>() + val file = File(file_base + filename) + logger.trace("Checking ini file {}", filename.anonymize()) + if (!file.exists()) return map + logger.debug("Loading ini file {}", filename.anonymize()) + file.forEachLine { parseLine(it, map) } + return map + } + + private fun parseLine(original_line: String, map: MutableMap>) { + logger.trace("Parsing line: {}", original_line) + var line = original_line.trim().replaceAfter("#", "").removeSuffix("#") + logger.trace("After cleaning: {}", line) + if (line == "") return + val parts: List = line.split("=", limit=2).map{it.trim()} + + if (parts.size < 2) throw RuntimeException("Invalid config setting: $line") + + val (key, value) = parts + if (value=="") { + map.remove(key) + } else { + var list = map.get(key) + if (list == null) { + list = mutableListOf() + map.put(key, list) + } + list.add(value) + } + } + + private fun copy_sample_ini(filename: String) { + val stream = Config::class.java.getResourceAsStream("/config.sample.ini") + File(filename).outputStream().use { stream.copyTo(it) } + stream.close() + } + + + open class Setting(name: String, value: List?, source: SettingSource) { + companion object { + val all = mutableListOf() + + fun build(name: String, ini_value: List?, cli_value: List?, default: String?): Setting { + var value: List? + var source: SettingSource + if (cli_value != null) { + value=cli_value + source=SettingSource.CLI + } else if (ini_value != null) { + value=ini_value + source=SettingSource.INI + } else { + if (default!=null) { + value = listOf(default) + } else { + value = null + } + source=SettingSource.DEFAULT + } + return Setting(name, value, source); + } + } + } + + enum class SettingSource { + INI, + CLI, + DEFAULT + } +} +/* +class DbSettings(val database: Database) { + private fun fetchValue(name: String): String? = database.fetchSetting(name) + private fun saveValue(name: String, value: String?) = database.saveSetting(name, value) + + var pts: String? + get() = fetchValue("pts") + set(x: String?) = saveValue("pts", x) +} + + +package de.fabianonline.telegram_backup + + + +class Settings(val file_base: String) { + val logger = LoggerFactory.getLogger(Settings::class.java) + var settings = mutableMapOf>() + + init { + loadIni(file_base + "config.ini") + copySampleIni(file_base + "config.sample.ini") + } + + + + + + + + fun println() = println(settings) + + fun getString(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default + fun getStringList(key: String): List? = settings.get(key) + fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default + fun getBoolean(key: String, default: Boolean = false): Boolean { + val value = settings.get(key)?.last() + if (value==null) return default + return value=="true" + } + fun getArray(key: String): List = settings.get(key) ?: listOf() + + val gmaps_key = getString("gmaps_key", default=Config.SECRET_GMAPS)!! + val pagination = getBoolean("pagination", default=true) + val pagination_size = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!! + val download_media = getBoolean("download_media", default=true) + val download_channels = getBoolean("download_channels", default=false) + val download_supergroups = getBoolean("download_supergroups", default=false) + val whitelist_channels = getStringList("whitelist_channels") + val blacklist_channels = getStringList("blacklist_channels") +} +*/ diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt index 568f617..28e4154 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt @@ -41,29 +41,29 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils object FileManagerFactory { - fun getFileManager(m: TLMessage?, u: UserManager): AbstractMediaFileManager? { + fun getFileManager(m: TLMessage?, u: UserManager, c: TelegramClient): AbstractMediaFileManager? { if (m == null) return null val media = m.getMedia() ?: return null if (media is TLMessageMediaPhoto) { - return PhotoFileManager(m, u) + return PhotoFileManager(m, u, file_base) } else if (media is TLMessageMediaDocument) { - val d = DocumentFileManager(m, u) + val d = DocumentFileManager(m, u, file_base) return if (d.isSticker) { - StickerFileManager(m, u) + StickerFileManager(m, u, file_base) } else d } else if (media is TLMessageMediaGeo) { - return GeoFileManager(m, u) + return GeoFileManager(m, u, file_base) } else if (media is TLMessageMediaEmpty) { - return UnsupportedFileManager(m, u, "empty") + return UnsupportedFileManager(m, u, file_base, "empty") } else if (media is TLMessageMediaUnsupported) { - return UnsupportedFileManager(m, u, "unsupported") + return UnsupportedFileManager(m, u, file_base, "unsupported") } else if (media is TLMessageMediaWebPage) { - return UnsupportedFileManager(m, u, "webpage") + return UnsupportedFileManager(m, u, file_base, "webpage") } else if (media is TLMessageMediaContact) { - return UnsupportedFileManager(m, u, "contact") + return UnsupportedFileManager(m, u, file_base, "contact") } else if (media is TLMessageMediaVenue) { - return UnsupportedFileManager(m, u, "venue") + return UnsupportedFileManager(m, u, file_base, "venue") } else { AbstractMediaFileManager.throwUnexpectedObjectError(media) } From 77efde1136a8c89d9ab652acfde73e3fcfbff28c Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Fri, 6 Apr 2018 06:21:39 +0200 Subject: [PATCH 06/16] WIP: Still having fun with git stash. -.- --- .../telegram_backup/mediafilemanager/FileManagerFactory.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt index 28e4154..fb1224d 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt @@ -41,7 +41,7 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils object FileManagerFactory { - fun getFileManager(m: TLMessage?, u: UserManager, c: TelegramClient): AbstractMediaFileManager? { + fun getFileManager(m: TLMessage?, u: UserManager, file_base: String): AbstractMediaFileManager? { if (m == null) return null val media = m.getMedia() ?: return null From ec4097e77721958efcf8d3f737dc231c8d99b954 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 9 Apr 2018 06:07:53 +0200 Subject: [PATCH 07/16] The code is now compiling, but still largely untested. So it's still kinda WIP. --- .../telegram_backup/CommandLineController.kt | 28 ++++---- .../telegram_backup/CommandLineOptions.kt | 19 +++-- .../telegram_backup/CommandLineRunner.kt | 8 +-- .../fabianonline/telegram_backup/Database.kt | 8 +-- .../telegram_backup/DatabaseUpdates.kt | 4 +- .../telegram_backup/DbSettings.kt | 12 ---- .../telegram_backup/DownloadManager.kt | 62 ++++++++-------- .../telegram_backup/IniSettings.kt | 72 ------------------- .../fabianonline/telegram_backup/Settings.kt | 64 +++++++---------- .../telegram_backup/TelegramUpdateHandler.kt | 18 ++--- .../telegram_backup/exporter/HTMLExporter.kt | 4 +- .../AbstractMediaFileManager.kt | 1 + .../mediafilemanager/FileManagerFactory.kt | 5 +- .../mediafilemanager/GeoFileManager.kt | 7 +- 14 files changed, 105 insertions(+), 207 deletions(-) delete mode 100644 src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt delete mode 100644 src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 47a9929..ef68648 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -41,7 +41,6 @@ class CommandLineController(val options: CommandLineOptions) { val handler: TelegramUpdateHandler var client: TelegramClient val user_manager: UserManager - val inisettings: IniSettings val settings: Settings val database: Database logger.info("CommandLineController started. App version {}", Config.APP_APPVER) @@ -63,7 +62,7 @@ class CommandLineController(val options: CommandLineOptions) { // Setup file_base logger.debug("Target dir from Config: {}", Config.TARGET_DIR.anonymize()) - target_dir = options.values.get("target")?.last() ?: Config.TARGET_DIR + target_dir = options.values.get("target") ?: Config.TARGET_DIR logger.debug("Target dir after options: {}", target_dir) println("Base directory for files: ${target_dir.anonymize()}") @@ -73,16 +72,16 @@ class CommandLineController(val options: CommandLineOptions) { } if (options.booleans.contains("login")) { - cmd_login(app, target_dir, options.values.get("account")?.last()) + cmd_login(app, target_dir, options.values.get("account")) } logger.trace("Checking accounts") - phone_number = try { selectAccount(target_dir, options.values.get("account")?.last()) + phone_number = try { selectAccount(target_dir, options.values.get("account")) } catch(e: AccountNotFoundException) { show_error("The specified account could not be found.") } catch(e: NoAccountsException) { println("No accounts found. Starting login process...") - cmd_login(app, target_dir, options.values.get("account")?.last()) + cmd_login(app, target_dir, options.values.get("account")) } // TODO: Create a new TelegramApp if the user set his/her own TelegramApp credentials @@ -124,7 +123,7 @@ class CommandLineController(val options: CommandLineOptions) { System.exit(0) } - val export = options.values.get("export")?.last() + val export = options.values["export"] logger.debug("options.val_export: {}", export) if (export != null) { if (export.toLowerCase() == "html") { @@ -138,7 +137,7 @@ class CommandLineController(val options: CommandLineOptions) { println("You are logged in as ${user_manager.toString().anonymize()}") logger.info("Initializing Download Manager") - val d = DownloadManager(client, CommandLineDownloadProgress(), database, user_manager, inisettings) + val d = DownloadManager(client, CommandLineDownloadProgress(), database, user_manager, settings, file_base) if (options.booleans.contains("list_channels")) { val chats = d.getChats() @@ -147,7 +146,7 @@ class CommandLineController(val options: CommandLineOptions) { var download: Boolean println("Channels:") - download = inisettings.download_channels + download = settings.download_channels if (!download) println("Download of channels is disabled - see download_channels in config.ini") print_header(download) for (c in chats.channels) { @@ -155,7 +154,7 @@ class CommandLineController(val options: CommandLineOptions) { } println() println("Supergroups:") - download = inisettings.download_supergroups + download = settings.download_supergroups if (!download) println("Download of supergroups is disabled - see download_supergroups in config.ini") print_header(download) for (c in chats.supergroups) { @@ -166,17 +165,17 @@ class CommandLineController(val options: CommandLineOptions) { logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.values.get("limit_messages")?.last()) d.downloadMessages(options.values.get("limit_messages")?.last()?.toInt()) - logger.debug("IniSettings#download_media: {}", inisettings.download_media) - if (inisettings.download_media) { + logger.debug("IniSettings#download_media: {}", settings.download_media) + if (settings.download_media) { logger.debug("Calling DownloadManager.downloadMedia") d.downloadMedia() } else { println("Skipping media download because download_media is set to false.") } - if (options.boolean.contains("daemon")) { + if (options.booleans.contains("daemon")) { logger.info("Initializing TelegramUpdateHandler") - handler = TelegramUpdateHandler(user_manager, database) + handler = TelegramUpdateHandler(user_manager, database, file_base, settings) client.close() logger.info("Creating new client") client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, handler) @@ -203,7 +202,7 @@ class CommandLineController(val options: CommandLineOptions) { } private fun selectAccount(file_base: String, requested_account: String?): String { - var found_account: String? = null + var found_account: String? val accounts = Utils.getAccounts(file_base) if (requested_account != null) { logger.debug("Account requested: {}", requested_account.anonymize()) @@ -218,7 +217,6 @@ class CommandLineController(val options: CommandLineOptions) { show_error(("You have more than one account but didn't specify which one to use.\n" + "Use '--account ' to use account .\n" + "Use '--list-accounts' to see all available accounts.")) - System.exit(1) } if (found_account == null) { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index 7808202..e9947a5 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -17,27 +17,24 @@ package de.fabianonline.telegram_backup class CommandLineOptions(args: Array) { val booleans = mutableListOf() - val values = mutableMapOf>() + val values = mutableMapOf() var last_key: String? = null init { for(arg in args) { - if (arg.starts_with("--")) { + if (arg.startsWith("--")) { if (last_key!=null) { - booleans.add(last_key) + booleans.add(last_key!!) } - last_key = arg.substr(2) + last_key = arg.substring(2) } else { - if (last_key==null) throw RuntimeException("Unexpected parameter without switch: $arg") - var list = values.get(last_key) - if (list==null) { - list = mutableListOf() - values.add(last_key, list) + if (last_key==null) { + throw RuntimeException("Unexpected unnamed parameter ${arg}") } - list.add(arg) + values.put(last_key!!, arg) last_key = null } } - if (last_key!=null) booleans.add(last_key) + if (last_key!=null) booleans.add(last_key!!) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt index 5728c32..e517da7 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt @@ -53,7 +53,7 @@ class CommandLineRunner(args: Array) { } fun setupLogging() { - if (options.booleans.hasKey('anonymize')) { + if (options.booleans.contains("anonymize")) { Utils.anonymize = true } @@ -76,13 +76,13 @@ class CommandLineRunner(args: Array) { rootLogger.addAppender(appender) rootLogger.setLevel(Level.OFF) - if (options.cmd_trace) { + if (options.booleans.contains("trace")) { (LoggerFactory.getLogger("de.fabianonline.telegram_backup") as Logger).setLevel(Level.TRACE) - } else if (options.cmd_debug) { + } else if (options.booleans.contains("debug")) { (LoggerFactory.getLogger("de.fabianonline.telegram_backup") as Logger).setLevel(Level.DEBUG) } - if (options.cmd_trace_telegram) { + if (options.booleans.contains("trace_telegram")) { (LoggerFactory.getLogger("com.github.badoualy") as Logger).setLevel(Level.TRACE) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index b71cb2c..12b915b 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -245,14 +245,14 @@ class Database constructor(val file_base: String, val user_manager: UserManager) val map = mutableMapOf() val rs = stmt.executeQuery(query) while(rs.next()) { - map.add(rs.getString(1), rs.getString(2)) + map.put(rs.getString(1), rs.getString(2)) } rs.close() return map } @Synchronized - fun saveMessages(all: TLVector, api_layer: Int, source_type: MessageSource = MessageSource.NORMAL) { + fun saveMessages(all: TLVector, api_layer: Int, source_type: MessageSource = MessageSource.NORMAL, settings: Settings?) { //"(id, dialog_id, from_id, from_type, text, time, has_media, data, sticker, type) " + //"VALUES " + //"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); @@ -314,7 +314,7 @@ class Database constructor(val file_base: String, val user_manager: UserManager) } ps.setString(7, text) ps.setString(8, "" + msg.getDate()) - val f = FileManagerFactory.getFileManager(msg, user_manager) + val f = FileManagerFactory.getFileManager(msg, user_manager, file_base, settings) if (f == null) { ps.setNull(9, Types.BOOLEAN) ps.setNull(10, Types.VARCHAR) @@ -499,7 +499,7 @@ class Database constructor(val file_base: String, val user_manager: UserManager) ps_insert_or_replace.close() } - fun fetchSettings() = queryStringMap("SELECT key, value FROM settings WHERE key='${key}'") + fun fetchSettings() = queryStringMap("SELECT key, value FROM settings") fun saveSetting(key: String, value: String?) { val ps = conn.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt index b0e5676..7ac15ba 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt @@ -313,7 +313,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn } else { ps.setInt(1, msg.getFwdFrom().getFromId()) } - val f = FileManagerFactory.getFileManager(msg, db.user_manager) + val f = FileManagerFactory.getFileManager(msg, db.user_manager, db.file_base, settings = null) if (f == null) { ps.setNull(2, Types.VARCHAR) ps.setNull(3, Types.VARCHAR) @@ -431,7 +431,7 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn } } rs.close() - db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP) + db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP, settings=null) execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")") print(".") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt deleted file mode 100644 index 26d4344..0000000 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt +++ /dev/null @@ -1,12 +0,0 @@ -package de.fabianonline.telegram_backup - -class DbSettings(val database: Database) { - private fun fetchValue(name: String): String? = database.fetchSetting(name) - private fun saveValue(name: String, value: String?) = database.saveSetting(name, value) - - var pts: String? - get() = fetchValue("pts") - set(x: String?) = saveValue("pts", x) -} - - diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt index 9913a51..3e2f7e4 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt @@ -60,7 +60,7 @@ enum class MessageSource(val descr: String) { SUPERGROUP("supergroup") } -class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInterface, val db: Database, val user_manager: UserManager, val inisettings: IniSettings) { +class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInterface, val db: Database, val user_manager: UserManager, val settings: Settings, val file_base: String) { internal var has_seen_flood_wait_message = false @Throws(RpcErrorException::class, IOException::class) @@ -109,7 +109,7 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte } } System.out.println("Top message ID is " + max_message_id) - var max_database_id = db!!.getTopMessageID() + var max_database_id = db.getTopMessageID() System.out.println("Top message ID in database is " + max_database_id) if (limit != null) { System.out.println("Limit is set to " + limit) @@ -137,8 +137,8 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte logger.info("Searching for missing messages in the db") System.out.println("Checking message database for completeness...") - val db_count = db!!.getMessageCount() - val db_max = db!!.getTopMessageID() + val db_count = db.getMessageCount() + val db_max = db.getTopMessageID() logger.debug("db_count: {}", db_count) logger.debug("db_max: {}", db_max) @@ -163,12 +163,12 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte } */ - if (inisettings.download_channels) { + if (settings.download_channels) { println("Checking channels...") for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) } } - if (inisettings.download_supergroups) { + if (settings.download_supergroups) { println("Checking supergroups...") for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) } } @@ -176,7 +176,7 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte private fun downloadMessagesFromChannel(channel: Channel) { val obj = channel.obj - val max_known_id = db!!.getTopMessageIDForChannel(channel.id) + val max_known_id = db.getTopMessageIDForChannel(channel.id) if (obj.getTopMessage() > max_known_id) { val ids = makeIdList(max_known_id + 1, obj.getTopMessage()) var channel_name = channel.title @@ -196,7 +196,7 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte } else { "${source_type.descr} $source_name" } - prog!!.onMessageDownloadStart(ids.size, source_string) + prog.onMessageDownloadStart(ids.size, source_string) logger.debug("Entering download loop") while (ids.size > 0) { @@ -221,9 +221,9 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte tries++ try { if (channel == null) { - response = client!!.messagesGetMessages(vector) + response = client.messagesGetMessages(vector) } else { - response = client!!.channelsGetMessages(channel, vector) + response = client.channelsGetMessages(channel, vector) } break } catch (e: RpcErrorException) { @@ -241,10 +241,10 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte CommandLineController.show_error("Requested ${vector.size} messages, but got ${response.getMessages().size}. That is unexpected. Quitting.") } - prog!!.onMessageDownloaded(response.getMessages().size) - db!!.saveMessages(response.getMessages(), Kotlogram.API_LAYER, source_type=source_type) - db!!.saveChats(response.getChats()) - db!!.saveUsers(response.getUsers()) + prog.onMessageDownloaded(response.getMessages().size) + db.saveMessages(response.getMessages(), Kotlogram.API_LAYER, source_type=source_type, settings=settings) + db.saveChats(response.getChats()) + db.saveUsers(response.getUsers()) logger.trace("Sleeping") try { TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_MESSAGES) @@ -254,12 +254,12 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte } logger.debug("Finished.") - prog!!.onMessageDownloadFinished() + prog.onMessageDownloadFinished() } @Throws(RpcErrorException::class, IOException::class) fun downloadMedia() { - download_client = client!!.getDownloaderClient() + download_client = client.getDownloaderClient() var completed: Boolean do { completed = true @@ -289,19 +289,19 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte private fun _downloadMedia() { logger.info("This is _downloadMedia") logger.info("Checking if there are messages in the DB with a too old API layer") - val ids = db!!.getIdsFromQuery("SELECT id FROM messages WHERE has_media=1 AND api_layer<" + Kotlogram.API_LAYER) + val ids = db.getIdsFromQuery("SELECT id FROM messages WHERE has_media=1 AND api_layer<" + Kotlogram.API_LAYER) if (ids.size > 0) { System.out.println("You have ${ids.size} messages in your db that need an update. Doing that now.") logger.debug("Found {} messages", ids.size) downloadMessages(ids, null, source_type=MessageSource.NORMAL) } - val messages = this.db!!.getMessagesWithMedia() + val messages = db.getMessagesWithMedia() logger.debug("Database returned {} messages with media", messages.size) - prog!!.onMediaDownloadStart(messages.size) + prog.onMediaDownloadStart(messages.size) for (msg in messages) { if (msg == null) continue - val m = FileManagerFactory.getFileManager(msg, user_manager) + val m = FileManagerFactory.getFileManager(msg, user_manager, file_base, settings=settings) logger.trace("message {}, {}, {}, {}, {}", msg.getId(), msg.getMedia().javaClass.getSimpleName().replace("TLMessageMedia", "…"), @@ -309,25 +309,25 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte if (m.isEmpty) "empty" else "non-empty", if (m.downloaded) "downloaded" else "not downloaded") if (m.isEmpty) { - prog!!.onMediaDownloadedEmpty() + prog.onMediaDownloadedEmpty() } else if (m.downloaded) { - prog!!.onMediaAlreadyPresent(m) + prog.onMediaAlreadyPresent(m) } else { try { val result = m.download() if (result) { - prog!!.onMediaDownloaded(m) + prog.onMediaDownloaded(m) } else { - prog!!.onMediaSkipped() + prog.onMediaSkipped() } } catch (e: TimeoutException) { // do nothing - skip this file - prog!!.onMediaSkipped() + prog.onMediaSkipped() } } } - prog!!.onMediaDownloadFinished() + prog.onMediaDownloadFinished() } private fun makeIdList(start: Int, end: Int): MutableList { @@ -339,7 +339,7 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte fun getChats(): ChatList { val cl = ChatList() logger.trace("Calling messagesGetDialogs") - val dialogs = client!!.messagesGetDialogs(0, 0, TLInputPeerEmpty(), 100) + val dialogs = client.messagesGetDialogs(0, 0, TLInputPeerEmpty(), 100) logger.trace("Got {} dialogs back", dialogs.getDialogs().size) // Add dialogs @@ -352,10 +352,10 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte if (tl_peer_channel == null) continue var download = true - if (inisettings.whitelist_channels != null) { - download = inisettings.whitelist_channels.contains(tl_channel.getId().toString()) - } else if (inisettings.blacklist_channels != null) { - download = !inisettings.blacklist_channels.contains(tl_channel.getId().toString()) + if (settings.whitelist_channels != null) { + download = settings.whitelist_channels.contains(tl_channel.getId().toString()) + } else if (settings.blacklist_channels != null) { + download = !settings.blacklist_channels.contains(tl_channel.getId().toString()) } val channel = Channel(id=tl_channel.getId(), access_hash=tl_channel.getAccessHash(), title=tl_channel.getTitle(), obj=tl_peer_channel, download=download) if (tl_channel.getMegagroup()) { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt deleted file mode 100644 index 3fe4ba9..0000000 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ /dev/null @@ -1,72 +0,0 @@ -package de.fabianonline.telegram_backup - -import java.io.File -import org.slf4j.LoggerFactory -import org.slf4j.Logger - -class IniSettings(val file_base: String) { - val logger = LoggerFactory.getLogger(IniSettings::class.java) - var settings = mutableMapOf>() - - init { - loadIni(file_base + "config.ini") - copySampleIni(file_base + "config.sample.ini") - } - - private fun loadIni(filename: String) { - val file = File(filename) - logger.trace("Checking ini file {}", filename.anonymize()) - if (!file.exists()) return - logger.debug("Loading ini file {}", filename.anonymize()) - file.forEachLine { parseLine(it) } - } - - private fun parseLine(original_line: String) { - logger.trace("Parsing line: {}", original_line) - var line = original_line.trim().replaceAfter("#", "").removeSuffix("#") - logger.trace("After cleaning: {}", line) - if (line == "") return - val parts: List = line.split("=", limit=2).map{it.trim()} - - if (parts.size < 2) throw RuntimeException("Invalid config setting: $line") - - val (key, value) = parts - if (value=="") { - settings.remove(key) - } else { - var map = settings.get(key) - if (map == null) { - map = mutableListOf() - settings.put(key, map) - } - map.add(value) - } - } - - private fun copySampleIni(filename: String) { - val stream = Config::class.java.getResourceAsStream("/config.sample.ini") - File(filename).outputStream().use { stream.copyTo(it) } - stream.close() - } - - fun println() = println(settings) - - fun getString(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default - fun getStringList(key: String): List? = settings.get(key) - fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default - fun getBoolean(key: String, default: Boolean = false): Boolean { - val value = settings.get(key)?.last() - if (value==null) return default - return value=="true" - } - fun getArray(key: String): List = settings.get(key) ?: listOf() - - val gmaps_key = getString("gmaps_key", default=Config.SECRET_GMAPS)!! - val pagination = getBoolean("pagination", default=true) - val pagination_size = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!! - val download_media = getBoolean("download_media", default=true) - val download_channels = getBoolean("download_channels", default=false) - val download_supergroups = getBoolean("download_supergroups", default=false) - val whitelist_channels = getStringList("whitelist_channels") - val blacklist_channels = getStringList("blacklist_channels") -} diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt index 9cce5eb..0afc136 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt @@ -5,25 +5,41 @@ import org.slf4j.LoggerFactory class Settings(val file_base: String, val database: Database, val cli_settings: CommandLineOptions) { val logger = LoggerFactory.getLogger(Settings::class.java) - val settings = mutableMapOf() - - val gmaps_key: Setting private val db_settings: Map - private var ini_settings: Map> + val ini_settings: Map> init { db_settings = database.fetchSettings() ini_settings = load_ini("config.ini") copy_sample_ini("config.sample.ini") - + } // Merging CLI and INI settings - gmaps_key = get_setting("gmaps_key", default=Config.SECRET_GMAPS) + val gmaps_key = get_setting_string("gmaps_key", default=Config.SECRET_GMAPS) + val pagination = get_setting_boolean("pagination", default=true) + val pagination_size = get_setting_int("pagination_size", default=Config.DEFAULT_PAGINATION) + val download_media = get_setting_boolean("download_media", default=true) + val download_channels = get_setting_boolean("download_channels", default=false) + val download_supergroups = get_setting_boolean("download_supergroups", default=false) + val whitelist_channels = get_setting_list("whitelist_channels") + val blacklist_channels = get_setting_list("blacklist_channels") + + + private fun get_setting_string(name: String, default: String): String { + return ini_settings[name]?.last() ?: cli_settings.values[name] ?: default } - private fun get_setting(name: String, default: String? = null): Setting { - return Setting.build(name, ini_settings[name], cli_settings.values[name], default) + private fun get_setting_int(name: String, default: Int): Int { + return ini_settings[name]?.last()?.toInt() ?: cli_settings.values[name]?.toInt() ?: default + } + + private fun get_setting_boolean(name: String, default: Boolean): Boolean { + return ini_settings[name]?.last()?.toBoolean() ?: cli_settings.values[name]?.toBoolean() ?: default + } + + private fun get_setting_list(name: String): List? { + return ini_settings[name] } private fun load_ini(filename: String): Map> { @@ -63,39 +79,7 @@ class Settings(val file_base: String, val database: Database, val cli_settings: File(filename).outputStream().use { stream.copyTo(it) } stream.close() } - - - open class Setting(name: String, value: List?, source: SettingSource) { - companion object { - val all = mutableListOf() - fun build(name: String, ini_value: List?, cli_value: List?, default: String?): Setting { - var value: List? - var source: SettingSource - if (cli_value != null) { - value=cli_value - source=SettingSource.CLI - } else if (ini_value != null) { - value=ini_value - source=SettingSource.INI - } else { - if (default!=null) { - value = listOf(default) - } else { - value = null - } - source=SettingSource.DEFAULT - } - return Setting(name, value, source); - } - } - } - - enum class SettingSource { - INI, - CLI, - DEFAULT - } } /* class DbSettings(val database: Database) { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt b/src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt index 0b3073a..f65b71e 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt @@ -28,14 +28,14 @@ import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory import org.slf4j.LoggerFactory -internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Database) : UpdateCallback { +internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Database, val file_base: String, val settings: Settings) : UpdateCallback { val logger = LoggerFactory.getLogger(TelegramUpdateHandler::class.java) override fun onUpdates(client: TelegramClient, updates: TLUpdates) { logger.debug("onUpdates - " + updates.getUpdates().size + " Updates, " + updates.getUsers().size + " Users, " + updates.getChats().size + " Chats") for (update in updates.getUpdates()) { - processUpdate(update, client) + processUpdate(update) logger.debug(" " + update.javaClass.getName()) } db.saveUsers(updates.getUsers()) @@ -45,7 +45,7 @@ internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Data override fun onUpdatesCombined(client: TelegramClient, updates: TLUpdatesCombined) { logger.debug("onUpdatesCombined") for (update in updates.getUpdates()) { - processUpdate(update, client) + processUpdate(update) } db.saveUsers(updates.getUsers()) db.saveChats(updates.getChats()) @@ -53,7 +53,7 @@ internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Data override fun onUpdateShort(client: TelegramClient, update: TLUpdateShort) { logger.debug("onUpdateShort") - processUpdate(update.getUpdate(), client) + processUpdate(update.getUpdate()) logger.debug(" " + update.getUpdate().javaClass.getName()) } @@ -76,7 +76,7 @@ internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Data message.getEntities(), null, null) val vector = TLVector(TLAbsMessage::class.java) vector.add(msg) - db.saveMessages(vector, Kotlogram.API_LAYER) + db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings) System.out.print('.') } @@ -109,7 +109,7 @@ internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Data m.getEntities(), null, null) val vector = TLVector(TLAbsMessage::class.java) vector.add(msg) - db.saveMessages(vector, Kotlogram.API_LAYER) + db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings) System.out.print('.') } @@ -121,15 +121,15 @@ internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Data logger.debug("onUpdateTooLong") } - private fun processUpdate(update: TLAbsUpdate, client: TelegramClient) { + private fun processUpdate(update: TLAbsUpdate) { if (update is TLUpdateNewMessage) { val abs_msg = update.getMessage() val vector = TLVector(TLAbsMessage::class.java) vector.add(abs_msg) - db.saveMessages(vector, Kotlogram.API_LAYER) + db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings) System.out.print('.') if (abs_msg is TLMessage) { - val fm = FileManagerFactory.getFileManager(abs_msg, user_manager) + val fm = FileManagerFactory.getFileManager(abs_msg, user_manager, file_base, settings) if (fm != null && !fm.isEmpty && !fm.downloaded) { try { fm.download() diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt b/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt index 586c37f..ee61236 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt @@ -37,11 +37,11 @@ import de.fabianonline.telegram_backup.* import org.slf4j.Logger import org.slf4j.LoggerFactory -class HTMLExporter(val db: Database, val user: UserManager, val ini: IniSettings, val file_base: String) { +class HTMLExporter(val db: Database, val user: UserManager, val settings: Settings, val file_base: String) { @Throws(IOException::class) fun export() { try { - val pagination = if (ini.pagination) ini.pagination_size else -1 + val pagination = if (settings.pagination) settings.pagination_size else -1 // Create base dir logger.debug("Creating base dir") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt index f082df0..bc2034e 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt @@ -21,6 +21,7 @@ import de.fabianonline.telegram_backup.Config import com.github.badoualy.telegram.tl.api.* import com.github.badoualy.telegram.tl.exception.RpcErrorException +import de.fabianonline.telegram_backup.Settings import java.io.IOException import java.io.File diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt index fb1224d..8248f15 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt @@ -29,6 +29,7 @@ import com.github.badoualy.telegram.tl.api.* import com.github.badoualy.telegram.tl.api.upload.TLFile import com.github.badoualy.telegram.tl.exception.RpcErrorException import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile +import de.fabianonline.telegram_backup.Settings import java.io.IOException import java.io.File @@ -41,7 +42,7 @@ import java.util.concurrent.TimeoutException import org.apache.commons.io.FileUtils object FileManagerFactory { - fun getFileManager(m: TLMessage?, u: UserManager, file_base: String): AbstractMediaFileManager? { + fun getFileManager(m: TLMessage?, u: UserManager, file_base: String, settings: Settings?): AbstractMediaFileManager? { if (m == null) return null val media = m.getMedia() ?: return null @@ -53,7 +54,7 @@ object FileManagerFactory { StickerFileManager(m, u, file_base) } else d } else if (media is TLMessageMediaGeo) { - return GeoFileManager(m, u, file_base) + return GeoFileManager(m, u, file_base, settings) } else if (media is TLMessageMediaEmpty) { return UnsupportedFileManager(m, u, file_base, "empty") } else if (media is TLMessageMediaUnsupported) { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt index 2d68c3c..a47041c 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt @@ -18,14 +18,15 @@ package de.fabianonline.telegram_backup.mediafilemanager import de.fabianonline.telegram_backup.UserManager import de.fabianonline.telegram_backup.DownloadManager -import de.fabianonline.telegram_backup.IniSettings import com.github.badoualy.telegram.tl.api.* +import de.fabianonline.telegram_backup.Config +import de.fabianonline.telegram_backup.Settings import java.io.IOException import java.io.File -class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String) : AbstractMediaFileManager(msg, user, file_base) { +class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String, val settings: Settings?) : AbstractMediaFileManager(msg, user, file_base) { protected lateinit var geo: TLGeoPoint // We don't know the size, so we just guess. @@ -59,7 +60,7 @@ class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String) : Abs "center=${geo.getLat()},${geo.getLong()}&" + "markers=color:red|${geo.getLat()},${geo.getLong()}&" + "zoom=14&size=300x150&scale=2&format=png&" + - "key=" + IniSettings.gmaps_key + "key=" + (settings?.gmaps_key ?: Config.SECRET_GMAPS) return DownloadManager.downloadExternalFile(targetPathAndFilename, url) } } From e5da546386053727136f562c09c2eb405773d6e2 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 10 Apr 2018 06:11:01 +0200 Subject: [PATCH 08/16] CommandLineOptions will now insert booleans into values with an value of "true" as well. --- .../de/fabianonline/telegram_backup/CommandLineOptions.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index e9947a5..94f2412 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -25,6 +25,7 @@ class CommandLineOptions(args: Array) { if (arg.startsWith("--")) { if (last_key!=null) { booleans.add(last_key!!) + values.put(last_key!!, "true") } last_key = arg.substring(2) } else { @@ -35,6 +36,9 @@ class CommandLineOptions(args: Array) { last_key = null } } - if (last_key!=null) booleans.add(last_key!!) + if (last_key!=null) { + booleans.add(last_key!!) + values.put(last_key!!, "true") + } } } From 5c466131d3c13037ab65211f6a5fcbd977a3bbae Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 10 Apr 2018 06:12:02 +0200 Subject: [PATCH 09/16] Rewritten Settings. --- .../telegram_backup/DownloadManager.kt | 4 +- .../fabianonline/telegram_backup/Settings.kt | 117 ++++++++---------- 2 files changed, 55 insertions(+), 66 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt index 3e2f7e4..8cda106 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt @@ -352,9 +352,9 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte if (tl_peer_channel == null) continue var download = true - if (settings.whitelist_channels != null) { + if (settings.whitelist_channels.isNotEmpty()) { download = settings.whitelist_channels.contains(tl_channel.getId().toString()) - } else if (settings.blacklist_channels != null) { + } else if (settings.blacklist_channels.isNotEmpty()) { download = !settings.blacklist_channels.contains(tl_channel.getId().toString()) } val channel = Channel(id=tl_channel.getId(), access_hash=tl_channel.getAccessHash(), title=tl_channel.getTitle(), obj=tl_peer_channel, download=download) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt index 0afc136..2822f0e 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt @@ -1,6 +1,7 @@ package de.fabianonline.telegram_backup import java.io.File +import java.util.LinkedList import org.slf4j.LoggerFactory class Settings(val file_base: String, val database: Database, val cli_settings: CommandLineOptions) { @@ -16,27 +17,16 @@ class Settings(val file_base: String, val database: Database, val cli_settings: copy_sample_ini("config.sample.ini") } // Merging CLI and INI settings - val gmaps_key = get_setting_string("gmaps_key", default=Config.SECRET_GMAPS) - val pagination = get_setting_boolean("pagination", default=true) - val pagination_size = get_setting_int("pagination_size", default=Config.DEFAULT_PAGINATION) - val download_media = get_setting_boolean("download_media", default=true) - val download_channels = get_setting_boolean("download_channels", default=false) - val download_supergroups = get_setting_boolean("download_supergroups", default=false) - val whitelist_channels = get_setting_list("whitelist_channels") - val blacklist_channels = get_setting_list("blacklist_channels") - - - private fun get_setting_string(name: String, default: String): String { - return ini_settings[name]?.last() ?: cli_settings.values[name] ?: default - } - - private fun get_setting_int(name: String, default: Int): Int { - return ini_settings[name]?.last()?.toInt() ?: cli_settings.values[name]?.toInt() ?: default - } - - private fun get_setting_boolean(name: String, default: Boolean): Boolean { - return ini_settings[name]?.last()?.toBoolean() ?: cli_settings.values[name]?.toBoolean() ?: default - } + + val sf = SettingsFactory(ini_settings, cli_settings) + val gmaps_key = sf.getString("gmaps_key", default=Config.SECRET_GMAPS) + val pagination = sf.getBoolean("pagination", default=true) + val pagination_size = sf.getInt("pagination_size", default=Config.DEFAULT_PAGINATION) + val download_media = sf.getBoolean("download_media", default=true) + val download_channels = sf.getBoolean("download_channels", default=false) + val download_supergroups = sf.getBoolean("download_supergroups", default=false) + val whitelist_channels = sf.getStringList("whitelist_channels", default=LinkedList()) + val blacklist_channels = sf.getStringList("blacklist_channels", default=LinkedList()) private fun get_setting_list(name: String): List? { return ini_settings[name] @@ -79,57 +69,56 @@ class Settings(val file_base: String, val database: Database, val cli_settings: File(filename).outputStream().use { stream.copyTo(it) } stream.close() } - } -/* -class DbSettings(val database: Database) { - private fun fetchValue(name: String): String? = database.fetchSetting(name) - private fun saveValue(name: String, value: String?) = database.saveSetting(name, value) + +class SettingsFactory(val ini: Map>, val cli: CommandLineOptions) { + fun getInt(name: String, default: Int) = getSetting(name, listOf(default.toString())).get().toInt() + fun getBoolean(name: String, default: Boolean) = getSetting(name, listOf(default.toString())).get().toBoolean() + fun getString(name: String, default: String) = getSetting(name, listOf(default)).get() + fun getStringList(name: String, default: List) = getSetting(name, default).getList() - var pts: String? - get() = fetchValue("pts") - set(x: String?) = saveValue("pts", x) + fun getSetting(name: String, default: List) = Setting(ini, cli, name, default) } - -package de.fabianonline.telegram_backup - - - -class Settings(val file_base: String) { - val logger = LoggerFactory.getLogger(Settings::class.java) - var settings = mutableMapOf>() +class Setting(val ini: Map>, val cli: CommandLineOptions, val name: String, val default: List) { + val values: List + val source: SettingSource + val logger = LoggerFactory.getLogger(Setting::class.java) init { - loadIni(file_base + "config.ini") - copySampleIni(file_base + "config.sample.ini") + if (getIni(name) != null) { + values = getIni(name)!! + source = SettingSource.INI + } else if (getCli(name) != null) { + values = listOf(getCli(name)!!) + source = SettingSource.CLI + } else { + values = default + source = SettingSource.DEFAULT + } + + logger.debug("Setting ${name} loaded. Source: ${source}. Value: ${values.toString().anonymize()}") + + all_settings.add(this) + } + fun get(): String = values.last() + fun getList(): List = values + + fun getIni(name: String): List? { + return ini[name] } - - - - - - - fun println() = println(settings) - - fun getString(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default - fun getStringList(key: String): List? = settings.get(key) - fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default - fun getBoolean(key: String, default: Boolean = false): Boolean { - val value = settings.get(key)?.last() - if (value==null) return default - return value=="true" + fun getCli(name: String): String? { + return cli.values[name] } - fun getArray(key: String): List = settings.get(key) ?: listOf() - val gmaps_key = getString("gmaps_key", default=Config.SECRET_GMAPS)!! - val pagination = getBoolean("pagination", default=true) - val pagination_size = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!! - val download_media = getBoolean("download_media", default=true) - val download_channels = getBoolean("download_channels", default=false) - val download_supergroups = getBoolean("download_supergroups", default=false) - val whitelist_channels = getStringList("whitelist_channels") - val blacklist_channels = getStringList("blacklist_channels") + companion object { + val all_settings = LinkedList() + } +} + +enum class SettingSource { + INI, + CLI, + DEFAULT } -*/ From be1cf8ba913ef6c11ec05becd2a01b24c548b924 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 10 Apr 2018 06:13:29 +0200 Subject: [PATCH 10/16] Fixed a bug in Utils.anonymize --- src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt index 8644c39..d1e2a31 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt @@ -195,7 +195,7 @@ object Utils { } fun String.anonymize(): String { - return if (Utils.anonymize) this else this.replace(Regex("[0-9]"), "1").replace(Regex("[A-Z]"), "A").replace(Regex("[a-z]"), "a") + " (ANONYMIZED)" + return if (!Utils.anonymize) this else this.replace(Regex("[0-9]"), "1").replace(Regex("[A-Z]"), "A").replace(Regex("[a-z]"), "a") + " (ANONYMIZED)" } fun Any.toJson(): String = Gson().toJson(this) From 9affb471307ac15cd31d2f1898471afca7997d11 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 10 Apr 2018 06:26:46 +0200 Subject: [PATCH 11/16] Reworked the obeyFloodLimit-Stuff to now use lambdas. --- .../telegram_backup/DownloadManager.kt | 116 +++++------------- .../de/fabianonline/telegram_backup/Utils.kt | 67 ++++++---- 2 files changed, 74 insertions(+), 109 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt index 8cda106..3a41b86 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt @@ -61,36 +61,11 @@ enum class MessageSource(val descr: String) { } class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInterface, val db: Database, val user_manager: UserManager, val settings: Settings, val file_base: String) { - internal var has_seen_flood_wait_message = false - @Throws(RpcErrorException::class, IOException::class) fun downloadMessages(limit: Int?) { - var completed: Boolean - do { - completed = true - try { - _downloadMessages(limit) - } catch (e: RpcErrorException) { - if (e.getCode() == 420) { // FLOOD_WAIT - completed = false - Utils.obeyFloodWaitException(e) - } else { - throw e - } - } catch (e: TimeoutException) { - completed = false - System.out.println("") - System.out.println("Telegram took too long to respond to our request.") - System.out.println("I'm going to wait a minute and then try again.") - try { - TimeUnit.MINUTES.sleep(1) - } catch (e2: InterruptedException) { - } - - System.out.println("") - } - - } while (!completed) + Utils.obeyFloodWait() { + _downloadMessages(limit) + } } @Throws(RpcErrorException::class, IOException::class, TimeoutException::class) @@ -211,31 +186,19 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte logger.trace("vector.size(): {}", vector.size) logger.trace("ids.size(): {}", ids.size) - var response: TLAbsMessages - var tries = 0 - while (true) { - logger.trace("Trying getMessages(), tries={}", tries) - if (tries >= 5) { - CommandLineController.show_error("Couldn't getMessages after 5 tries. Quitting.") - } - tries++ - try { + var resp: TLAbsMessages? = null + try { + Utils.obeyFloodWait(max_tries=5) { if (channel == null) { - response = client.messagesGetMessages(vector) + resp = client.messagesGetMessages(vector) } else { - response = client.channelsGetMessages(channel, vector) - } - break - } catch (e: RpcErrorException) { - if (e.getCode() == 420) { // FLOOD_WAIT - Utils.obeyFloodWaitException(e, has_seen_flood_wait_message) - has_seen_flood_wait_message = true - } else { - throw e + resp = client.channelsGetMessages(channel, vector) } } - + } catch (e: MaxTriesExceededException) { + CommandLineController.show_error("Couldn't getMessages after 5 tries. Quitting.") } + val response = resp!! logger.trace("response.getMessages().size(): {}", response.getMessages().size) if (response.getMessages().size != vector.size) { CommandLineController.show_error("Requested ${vector.size} messages, but got ${response.getMessages().size}. That is unexpected. Quitting.") @@ -260,29 +223,9 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte @Throws(RpcErrorException::class, IOException::class) fun downloadMedia() { download_client = client.getDownloaderClient() - var completed: Boolean - do { - completed = true - try { - _downloadMedia() - } catch (e: RpcErrorException) { - if (e.getCode() == 420) { // FLOOD_WAIT - completed = false - Utils.obeyFloodWaitException(e) - } else { - throw e - } - } - /*catch (TimeoutException e) { - completed = false; - System.out.println(""); - System.out.println("Telegram took too long to respond to our request."); - System.out.println("I'm going to wait a minute and then try again."); - logger.warn("TimeoutException caught", e); - try { TimeUnit.MINUTES.sleep(1); } catch(InterruptedException e2) {} - System.out.println(""); - }*/ - } while (!completed) + Utils.obeyFloodWait() { + _downloadMedia() + } } @Throws(RpcErrorException::class, IOException::class) @@ -423,31 +366,30 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte logger.trace("offset: {} block_size: {} size: {}", offset, size, size) val req = TLRequestUploadGetFile(loc, offset, size) try { - response = download_client!!.executeRpcQuery(req, dcID) as TLFile - } catch (e: RpcErrorException) { - if (e.getCode() == 420) { // FLOOD_WAIT - try_again = true - Utils.obeyFloodWaitException(e) - continue // response is null since we didn't actually receive any data. Skip the rest of this iteration and try again. - } else if (e.getCode() == 400) { - //Somehow this file is broken. No idea why. Let's skip it for now - return false - } else { - throw e + Utils.obeyFloodWait() { + response = download_client!!.executeRpcQuery(req, dcID) as TLFile } + } catch (e: RpcErrorException) { + if (e.getCode() == 400) { + // Somehow this file is broken. No idea why. Let's skip it for now. + return false + } + throw e } + + val resp = response!! + + offset += resp.getBytes().getData().size + logger.trace("response: {} total size: {}", resp.getBytes().getData().size, offset) - offset += response.getBytes().getData().size - logger.trace("response: {} total size: {}", response.getBytes().getData().size, offset) - - fos.write(response.getBytes().getData()) + fos.write(resp.getBytes().getData()) fos.flush() try { TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_FILE) } catch (e: InterruptedException) { } - } while (offset < size && (try_again || response!!.getBytes().getData().size > 0)) + } while (offset < size && (try_again || resp.getBytes().getData().size > 0)) fos.close() if (offset < size) { System.out.println("Requested file $target with $size bytes, but got only $offset bytes.") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt index d1e2a31..2381427 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt @@ -20,6 +20,7 @@ import com.github.badoualy.telegram.tl.exception.RpcErrorException import java.io.File import java.util.Vector import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException import com.google.gson.* import java.net.URL import org.apache.commons.io.IOUtils @@ -31,6 +32,8 @@ object Utils { @JvmField public val VERSIONS_EQUAL = 0 @JvmField public val VERSION_1_NEWER = 1 @JvmField public val VERSION_2_NEWER = 2 + + var hasSeenFloodWaitMessage = false var anonymize = false @@ -96,30 +99,48 @@ object Utils { } } - - @Throws(RpcErrorException::class) - @JvmOverloads internal fun obeyFloodWaitException(e: RpcErrorException?, silent: Boolean = false) { - if (e == null || e.getCode() != 420) return - - val delay: Long = e.getTagInteger()!!.toLong() - if (!silent) { - System.out.println("") - System.out.println( - "Telegram complained about us (okay, me) making too many requests in too short time by\n" + - "sending us \"" + e.getTag() + "\" as an error. So we now have to wait a bit. Telegram\n" + - "asked us to wait for " + delay + " seconds.\n" + + + fun obeyFloodWait(max_tries: Int = -1, method: () -> Unit) { + var tries = 0 + while (true) { + tries++ + if (max_tries>0 && tries>max_tries) throw MaxTriesExceededException() + logger.trace("This is try ${tries}.") + try { + method.invoke() + // If we reach this, the method has returned successfully -> we are done + return + } catch(e: RpcErrorException) { + // If we got something else than a FLOOD_WAIT error, we just rethrow it + if (e.getCode() != 420) throw e + + val delay = e.getTagInteger()!!.toLong() + + if (!hasSeenFloodWaitMessage) { + println( + "\n" + + "Telegram complained about us (okay, me) making too many requests in too short time by\n" + + "sending us \"${e.getTag()}\" as an error. So we now have to wait a bit. Telegram\n" + + "asked us to wait for ${delay} seconds.\n" + + "\n" + + "So I'm going to do just that for now. If you don't want to wait, you can quit by pressing\n" + + "Ctrl+C. You can restart me at any time and I will just continue to download your\n" + + "messages and media. But be advised that just restarting me is not going to change\n" + + "the fact that Telegram won't talk to me until then." + + "\n") + } + hasSeenFloodWaitMessage = true + + try { TimeUnit.SECONDS.sleep(delay + 1) } catch (e: InterruptedException) { } + } catch (e: TimeoutException) { + println( "\n" + - "So I'm going to do just that for now. If you don't want to wait, you can quit by pressing\n" + - "Ctrl+C. You can restart me at any time and I will just continue to download your\n" + - "messages and media. But be advised that just restarting me is not going to change\n" + - "the fact that Telegram won't talk to me until then.") - System.out.println("") + "Telegram took too long to respond to our request.\n" + + "I'm going to wait a minute and then try again." + + "\n") + try { TimeUnit.MINUTES.sleep(1) } catch (e: InterruptedException) { } + } } - try { - TimeUnit.SECONDS.sleep(delay + 1) - } catch (e2: InterruptedException) { - } - } @JvmStatic @@ -200,3 +221,5 @@ fun String.anonymize(): String { fun Any.toJson(): String = Gson().toJson(this) fun Any.toPrettyJson(): String = GsonBuilder().setPrettyPrinting().create().toJson(this) + +class MaxTriesExceededException(): RuntimeException("Max tries exceeded") {} From 069799cbaf22b16a27729caf534b0077fe568810 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 10 Apr 2018 06:33:15 +0200 Subject: [PATCH 12/16] Resorted the code in DownloadManager - no more need for downloadMessages calling _downloadMessages or downloadMedia calling _downloadMedia. --- .../telegram_backup/DownloadManager.kt | 92 ++++++++----------- 1 file changed, 38 insertions(+), 54 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt index 3a41b86..dea773e 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt @@ -63,18 +63,18 @@ enum class MessageSource(val descr: String) { class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInterface, val db: Database, val user_manager: UserManager, val settings: Settings, val file_base: String) { @Throws(RpcErrorException::class, IOException::class) fun downloadMessages(limit: Int?) { - Utils.obeyFloodWait() { - _downloadMessages(limit) - } - } - - @Throws(RpcErrorException::class, IOException::class, TimeoutException::class) - fun _downloadMessages(limit: Int?) { - logger.info("This is _downloadMessages with limit {}", limit) + logger.info("This is downloadMessages with limit {}", limit) logger.info("Downloading the last dialogs") System.out.println("Downloading most recent dialogs... ") var max_message_id = 0 - val chats = getChats() + var result: ChatList? = null + + Utils.obeyFloodWait() { + result = getChats() + } + + val chats = result!! + logger.debug("Got {} dialogs, {} supergoups, {} channels", chats.dialogs.size, chats.supergroups.size, chats.channels.size) for (d in chats.dialogs) { @@ -101,7 +101,7 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte if (max_database_id == max_message_id) { System.out.println("No new messages to download.") } else if (max_database_id > max_message_id) { - throw RuntimeException("max_database_id is bigger then max_message_id. This shouldn't happen. But the telegram api nonetheless does that sometimes. Just ignore this error, wait a few seconds and then try again.") + throw RuntimeException("max_database_id is bigger than max_message_id. This shouldn't happen. But the telegram api nonetheless does that sometimes. Just ignore this error, wait a few seconds and then try again.") } else { val start_id = max_database_id + 1 val end_id = max_message_id @@ -117,38 +117,38 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte logger.debug("db_count: {}", db_count) logger.debug("db_max: {}", db_max) - /*if (db_count != db_max) { - if (limit != null) { - System.out.println("You are missing messages in your database. But since you're using '--limit-messages', I won't download these now."); - } else { - LinkedList all_missing_ids = db.getMissingIDs(); - LinkedList downloadable_missing_ids = new LinkedList(); - for (Integer id : all_missing_ids) { - if (id > max_message_id - 1000000) downloadable_missing_ids.add(id); + /*if (db_count != db_max) { + if (limit != null) { + System.out.println("You are missing messages in your database. But since you're using '--limit-messages', I won't download these now."); + } else { + LinkedList all_missing_ids = db.getMissingIDs(); + LinkedList downloadable_missing_ids = new LinkedList(); + for (Integer id : all_missing_ids) { + if (id > max_message_id - 1000000) downloadable_missing_ids.add(id); + } + count_missing = all_missing_ids.size(); + System.out.println("" + all_missing_ids.size() + " messages are missing in your Database."); + System.out.println("I can (and will) download " + downloadable_missing_ids.size() + " of them."); + + downloadMessages(downloadable_missing_ids, null); } - count_missing = all_missing_ids.size(); - System.out.println("" + all_missing_ids.size() + " messages are missing in your Database."); - System.out.println("I can (and will) download " + downloadable_missing_ids.size() + " of them."); - downloadMessages(downloadable_missing_ids, null); + logger.info("Logging this run"); + db.logRun(Math.min(max_database_id + 1, max_message_id), max_message_id, count_missing); } - - logger.info("Logging this run"); - db.logRun(Math.min(max_database_id + 1, max_message_id), max_message_id, count_missing); - } - */ + */ if (settings.download_channels) { println("Checking channels...") for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) } } - + if (settings.download_supergroups) { println("Checking supergroups...") for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) } } } - + private fun downloadMessagesFromChannel(channel: Channel) { val obj = channel.obj val max_known_id = db.getTopMessageIDForChannel(channel.id) @@ -209,11 +209,7 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte db.saveChats(response.getChats()) db.saveUsers(response.getUsers()) logger.trace("Sleeping") - try { - TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_MESSAGES) - } catch (e: InterruptedException) { - } - + try { TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_MESSAGES) } catch (e: InterruptedException) { } } logger.debug("Finished.") @@ -223,13 +219,6 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte @Throws(RpcErrorException::class, IOException::class) fun downloadMedia() { download_client = client.getDownloaderClient() - Utils.obeyFloodWait() { - _downloadMedia() - } - } - - @Throws(RpcErrorException::class, IOException::class) - private fun _downloadMedia() { logger.info("This is _downloadMedia") logger.info("Checking if there are messages in the DB with a too old API layer") val ids = db.getIdsFromQuery("SELECT id FROM messages WHERE has_media=1 AND api_layer<" + Kotlogram.API_LAYER) @@ -359,15 +348,13 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte } logger.trace("offset before the loop is {}", offset) fos = FileOutputStream(temp_filename, true) - var response: TLFile? = null - var try_again: Boolean do { - try_again = false logger.trace("offset: {} block_size: {} size: {}", offset, size, size) val req = TLRequestUploadGetFile(loc, offset, size) + var resp: TLFile? = null try { Utils.obeyFloodWait() { - response = download_client!!.executeRpcQuery(req, dcID) as TLFile + resp = download_client!!.executeRpcQuery(req, dcID) as TLFile } } catch (e: RpcErrorException) { if (e.getCode() == 400) { @@ -377,19 +364,16 @@ class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInte throw e } - val resp = response!! + val response = resp!! - offset += resp.getBytes().getData().size - logger.trace("response: {} total size: {}", resp.getBytes().getData().size, offset) + offset += response.getBytes().getData().size + logger.trace("response: {} total size: {}", response.getBytes().getData().size, offset) - fos.write(resp.getBytes().getData()) + fos.write(response.getBytes().getData()) fos.flush() - try { - TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_FILE) - } catch (e: InterruptedException) { - } + try { TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_FILE) } catch (e: InterruptedException) { } - } while (offset < size && (try_again || resp.getBytes().getData().size > 0)) + } while (offset < size && response.getBytes().getData().size > 0) fos.close() if (offset < size) { System.out.println("Requested file $target with $size bytes, but got only $offset bytes.") From a9444e7813703c70d8945905b83a7bfd1a5d0b0f Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 10 Apr 2018 06:36:18 +0200 Subject: [PATCH 13/16] Rewrote CommandLineOptions: The code is now better readable and accepts parameters like `--target=/data` and accepts some short parameters like -t for --target. --- .../telegram_backup/CommandLineOptions.kt | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index 94f2412..d26d99f 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -19,26 +19,42 @@ class CommandLineOptions(args: Array) { val booleans = mutableListOf() val values = mutableMapOf() var last_key: String? = null + val substitutions = mapOf("-t" to "--target") init { - for(arg in args) { - if (arg.startsWith("--")) { - if (last_key!=null) { - booleans.add(last_key!!) - values.put(last_key!!, "true") - } - last_key = arg.substring(2) - } else { - if (last_key==null) { - throw RuntimeException("Unexpected unnamed parameter ${arg}") - } - values.put(last_key!!, arg) - last_key = null + val list = args.toMutableList() + + while (list.isNotEmpty()) { + + var current_arg = list.removeAt(0) + if (!current_arg.startsWith("-")) throw RuntimeException("Unexpected unnamed parameter ${current_arg}") + + var next_arg: String? = null + + if (current_arg.contains("=")) { + val parts = current_arg.split("=", limit=2) + current_arg = parts[0] + next_arg = parts[1] + } else if (list.isNotEmpty() && !list[0].startsWith("--")) { + next_arg = list.removeAt(0) + } + + if (!current_arg.startsWith("--") && current_arg.startsWith("-")) { + val replacement = substitutions.get(current_arg) + if (replacement == null) throw RuntimeException("Unknown short parameter ${current_arg}") + current_arg = replacement + } + + current_arg = current_arg.substring(2) + + if (next_arg == null) { + // current_arg seems to be a boolean value + booleans.add(current_arg) + values.put(current_arg, "true") + } else { + // current_arg has the value next_arg + values.put(current_arg, next_arg) } - } - if (last_key!=null) { - booleans.add(last_key!!) - values.put(last_key!!, "true") } } } From f24e66271fea3cb2322fdf6d8d9296245b6674a0 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 10 Apr 2018 06:38:28 +0200 Subject: [PATCH 14/16] Added command '--settings', which will print the currently active settings. Settings can now be marked as secret - if that is true, the settings value will only be printed if it is not the default value. --- .../telegram_backup/CommandLineController.kt | 3 +++ .../fabianonline/telegram_backup/Settings.kt | 24 +++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index ef68648..c17e9cb 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -121,6 +121,9 @@ class CommandLineController(val options: CommandLineOptions) { if (options.booleans.contains("stats")) { cmd_stats(file_base, database) System.exit(0) + } else if (options.booleans.contains("settings")) { + settings.print() + System.exit(0) } val export = options.values["export"] diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt index 2822f0e..cfae075 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt @@ -19,7 +19,7 @@ class Settings(val file_base: String, val database: Database, val cli_settings: // Merging CLI and INI settings val sf = SettingsFactory(ini_settings, cli_settings) - val gmaps_key = sf.getString("gmaps_key", default=Config.SECRET_GMAPS) + val gmaps_key = sf.getString("gmaps_key", default=Config.SECRET_GMAPS, secret=true) val pagination = sf.getBoolean("pagination", default=true) val pagination_size = sf.getInt("pagination_size", default=Config.DEFAULT_PAGINATION) val download_media = sf.getBoolean("download_media", default=true) @@ -69,18 +69,24 @@ class Settings(val file_base: String, val database: Database, val cli_settings: File(filename).outputStream().use { stream.copyTo(it) } stream.close() } + + fun print() { + println() + Setting.all_settings.forEach { it.print() } + println() + } } class SettingsFactory(val ini: Map>, val cli: CommandLineOptions) { - fun getInt(name: String, default: Int) = getSetting(name, listOf(default.toString())).get().toInt() - fun getBoolean(name: String, default: Boolean) = getSetting(name, listOf(default.toString())).get().toBoolean() - fun getString(name: String, default: String) = getSetting(name, listOf(default)).get() - fun getStringList(name: String, default: List) = getSetting(name, default).getList() + fun getInt(name: String, default: Int, secret: Boolean = false) = getSetting(name, listOf(default.toString()), secret).get().toInt() + fun getBoolean(name: String, default: Boolean, secret: Boolean = false) = getSetting(name, listOf(default.toString()), secret).get().toBoolean() + fun getString(name: String, default: String, secret: Boolean = false) = getSetting(name, listOf(default), secret).get() + fun getStringList(name: String, default: List, secret: Boolean = false) = getSetting(name, default, secret).getList() - fun getSetting(name: String, default: List) = Setting(ini, cli, name, default) + fun getSetting(name: String, default: List, secret: Boolean) = Setting(ini, cli, name, default, secret) } -class Setting(val ini: Map>, val cli: CommandLineOptions, val name: String, val default: List) { +class Setting(val ini: Map>, val cli: CommandLineOptions, val name: String, val default: List, val secret: Boolean) { val values: List val source: SettingSource val logger = LoggerFactory.getLogger(Setting::class.java) @@ -112,6 +118,10 @@ class Setting(val ini: Map>, val cli: CommandLineOptions, v return cli.values[name] } + fun print() { + println("%-25s %-10s %s".format(name, source, (if (secret && source==SettingSource.DEFAULT) "[REDACTED]" else values))) + } + companion object { val all_settings = LinkedList() } From 38fce0ee5c1af7cf97a0c2c59aa44550f504e1b0 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Wed, 11 Apr 2018 05:50:26 +0200 Subject: [PATCH 15/16] Some more refactoring of Settings. --- .../telegram_backup/CommandLineController.kt | 32 +++++++++---------- .../telegram_backup/CommandLineOptions.kt | 12 +++++-- .../telegram_backup/CommandLineRunner.kt | 8 ++--- .../fabianonline/telegram_backup/Settings.kt | 2 +- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index c17e9cb..d2291c3 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -46,12 +46,12 @@ class CommandLineController(val options: CommandLineOptions) { logger.info("CommandLineController started. App version {}", Config.APP_APPVER) printHeader() - if (options.booleans.contains("version")) { + if (options.isSet("version")) { System.exit(0) - } else if (options.booleans.contains("help")) { + } else if (options.isSet("help")) { show_help() System.exit(0) - } else if (options.booleans.contains("license")) { + } else if (options.isSet("license")) { show_license() System.exit(0) } @@ -62,26 +62,26 @@ class CommandLineController(val options: CommandLineOptions) { // Setup file_base logger.debug("Target dir from Config: {}", Config.TARGET_DIR.anonymize()) - target_dir = options.values.get("target") ?: Config.TARGET_DIR + target_dir = options.get("target") ?: Config.TARGET_DIR logger.debug("Target dir after options: {}", target_dir) println("Base directory for files: ${target_dir.anonymize()}") - if (options.booleans.contains("list_accounts")) { + if (options.isSet("list_accounts")) { Utils.print_accounts(target_dir) System.exit(0) } - if (options.booleans.contains("login")) { - cmd_login(app, target_dir, options.values.get("account")) + if (options.isSet("login")) { + cmd_login(app, target_dir, options.get("account")) } logger.trace("Checking accounts") - phone_number = try { selectAccount(target_dir, options.values.get("account")) + phone_number = try { selectAccount(target_dir, options.get("account")) } catch(e: AccountNotFoundException) { show_error("The specified account could not be found.") } catch(e: NoAccountsException) { println("No accounts found. Starting login process...") - cmd_login(app, target_dir, options.values.get("account")) + cmd_login(app, target_dir, options.get("account")) } // TODO: Create a new TelegramApp if the user set his/her own TelegramApp credentials @@ -118,15 +118,15 @@ class CommandLineController(val options: CommandLineOptions) { // Load the settings and stuff. settings = Settings(file_base, database, options) - if (options.booleans.contains("stats")) { + if (options.isSet("stats")) { cmd_stats(file_base, database) System.exit(0) - } else if (options.booleans.contains("settings")) { + } else if (options.isSet("settings")) { settings.print() System.exit(0) } - val export = options.values["export"] + val export = options.get("export") logger.debug("options.val_export: {}", export) if (export != null) { if (export.toLowerCase() == "html") { @@ -142,7 +142,7 @@ class CommandLineController(val options: CommandLineOptions) { logger.info("Initializing Download Manager") val d = DownloadManager(client, CommandLineDownloadProgress(), database, user_manager, settings, file_base) - if (options.booleans.contains("list_channels")) { + if (options.isSet("list_channels")) { val chats = d.getChats() val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) } val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")} @@ -166,8 +166,8 @@ class CommandLineController(val options: CommandLineOptions) { System.exit(0) } - logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.values.get("limit_messages")?.last()) - d.downloadMessages(options.values.get("limit_messages")?.last()?.toInt()) + logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.get("limit_messages")) + d.downloadMessages(options.get("limit_messages")?.toInt()) logger.debug("IniSettings#download_media: {}", settings.download_media) if (settings.download_media) { logger.debug("Calling DownloadManager.downloadMedia") @@ -176,7 +176,7 @@ class CommandLineController(val options: CommandLineOptions) { println("Skipping media download because download_media is set to false.") } - if (options.booleans.contains("daemon")) { + if (options.isSet("daemon")) { logger.info("Initializing TelegramUpdateHandler") handler = TelegramUpdateHandler(user_manager, database, file_base, settings) client.close() diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index d26d99f..bd16d5d 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -16,8 +16,7 @@ package de.fabianonline.telegram_backup class CommandLineOptions(args: Array) { - val booleans = mutableListOf() - val values = mutableMapOf() + private val values = mutableMapOf() var last_key: String? = null val substitutions = mapOf("-t" to "--target") @@ -49,12 +48,19 @@ class CommandLineOptions(args: Array) { if (next_arg == null) { // current_arg seems to be a boolean value - booleans.add(current_arg) values.put(current_arg, "true") + if (current_arg.startsWith("no-")) { + current_arg = current_arg.substring(3) + values.put(current_arg, "false") + } } else { // current_arg has the value next_arg values.put(current_arg, next_arg) } } + println(values) } + + operator fun get(name: String): String? = values[name] + fun isSet(name: String): Boolean = values[name]=="true" } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt index e517da7..9c58b0f 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt @@ -53,7 +53,7 @@ class CommandLineRunner(args: Array) { } fun setupLogging() { - if (options.booleans.contains("anonymize")) { + if (options.isSet("anonymize")) { Utils.anonymize = true } @@ -76,13 +76,13 @@ class CommandLineRunner(args: Array) { rootLogger.addAppender(appender) rootLogger.setLevel(Level.OFF) - if (options.booleans.contains("trace")) { + if (options.isSet("trace")) { (LoggerFactory.getLogger("de.fabianonline.telegram_backup") as Logger).setLevel(Level.TRACE) - } else if (options.booleans.contains("debug")) { + } else if (options.isSet("debug")) { (LoggerFactory.getLogger("de.fabianonline.telegram_backup") as Logger).setLevel(Level.DEBUG) } - if (options.booleans.contains("trace_telegram")) { + if (options.isSet("trace_telegram")) { (LoggerFactory.getLogger("com.github.badoualy") as Logger).setLevel(Level.TRACE) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt index cfae075..c479e21 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt @@ -115,7 +115,7 @@ class Setting(val ini: Map>, val cli: CommandLineOptions, v } fun getCli(name: String): String? { - return cli.values[name] + return cli.get(name) } fun print() { From ff9163c1bb79fce3c1fe9eb53c17b6fb2d6c7de9 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Wed, 11 Apr 2018 05:51:00 +0200 Subject: [PATCH 16/16] Settings: Is CLI is set, use it instead of INI. --- .../kotlin/de/fabianonline/telegram_backup/Settings.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt index c479e21..74982ab 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt @@ -92,12 +92,12 @@ class Setting(val ini: Map>, val cli: CommandLineOptions, v val logger = LoggerFactory.getLogger(Setting::class.java) init { - if (getIni(name) != null) { - values = getIni(name)!! - source = SettingSource.INI - } else if (getCli(name) != null) { + if (getCli(name) != null) { values = listOf(getCli(name)!!) source = SettingSource.CLI + } else if (getIni(name) != null) { + values = getIni(name)!! + source = SettingSource.INI } else { values = default source = SettingSource.DEFAULT