More rewriting. Also lots of fun with git stash and wrong branches...

gradle/wrapper/gradle-wrapper.properties

@@ -1,4 +1,4 @@
-#Thu Oct 06 11:24:39 CEST 2016
+#Fri Mar 23 06:07:28 CET 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt

@@ -42,16 +42,17 @@ class CommandLineController(val options: CommandLineOptions) {
 		var client: TelegramClient
 		val user_manager: UserManager
 		val inisettings: IniSettings
+		val settings: Settings
 		val database: Database
 		logger.info("CommandLineController started. App version {}", Config.APP_APPVER)
 		
 		printHeader()
-		if (options.cmd_version) {
+		if (options.booleans.contains("version")) {
 			System.exit(0)
-		} else if (options.cmd_help) {
+		} else if (options.booleans.contains("help")) {
 			show_help()
 			System.exit(0)
-		} else if (options.cmd_license) {
+		} else if (options.booleans.contains("license")) {
 			show_license()
 			System.exit(0)
 		}
@@ -62,26 +63,26 @@ class CommandLineController(val options: CommandLineOptions) {
 
 		// Setup file_base
 		logger.debug("Target dir from Config: {}", Config.TARGET_DIR.anonymize())
-		target_dir = options.val_target ?: Config.TARGET_DIR
+		target_dir = options.values.get("target")?.last() ?: Config.TARGET_DIR
 		logger.debug("Target dir after options: {}", target_dir)
 		println("Base directory for files: ${target_dir.anonymize()}")
 
-		if (options.cmd_list_accounts) {
+		if (options.booleans.contains("list_accounts")) {
 			Utils.print_accounts(target_dir)
 			System.exit(0)
 		}
 		
-		if (options.cmd_login) {
+		if (options.booleans.contains("login")) {
-			cmd_login(app, target_dir, options.val_account)
+			cmd_login(app, target_dir, options.values.get("account")?.last())
 		}
 
 		logger.trace("Checking accounts")
-		phone_number = try { selectAccount(target_dir, options.val_account)
+		phone_number = try { selectAccount(target_dir, options.values.get("account")?.last())
 		} catch(e: AccountNotFoundException) {
 			show_error("The specified account could not be found.")
 		} catch(e: NoAccountsException) {
 			println("No accounts found. Starting login process...")
-			cmd_login(app, target_dir, options.val_account)
+			cmd_login(app, target_dir, options.values.get("account")?.last())
 		}
 		
 		// TODO: Create a new TelegramApp if the user set his/her own TelegramApp credentials
@@ -112,33 +113,22 @@ class CommandLineController(val options: CommandLineOptions) {
 				show_error("Account / User mismatch")
 			}
 			
-			// Load the ini file.
-			inisettings = IniSettings(file_base)
-			
 			// If we reach this point, we can assume that there is an account and a database can be loaded / created.
 			database = Database(file_base, user_manager)
 			
-			if (options.cmd_stats) {
+			// Load the settings and stuff.
+			settings = Settings(file_base, database, options)
+			
+			if (options.booleans.contains("stats")) {
 				cmd_stats(file_base, database)
 				System.exit(0)
 			}
 			
-			if (options.val_test != null) {
+			val export = options.values.get("export")?.last()
-				if (options.val_test == 1) {
-					TestFeatures(database).test1()
-				} else if (options.val_test == 2) {
-					TestFeatures(database).test2()
-				} else {
-					System.out.println("Unknown test " + options.val_test)
-				}
-				System.exit(1)
-			}
-			
-			val export = options.val_export
 			logger.debug("options.val_export: {}", export)
 			if (export != null) {
-				if (export.toLowerCase().equals("html")) {
+				if (export.toLowerCase() == "html") {
-					HTMLExporter(database, user_manager, ini=inisettings, file_base=file_base).export()
+					HTMLExporter(database, user_manager, settings=settings, file_base=file_base).export()
 					System.exit(0)
 				} else {
 					show_error("Unknown export format '${export}'.")
@@ -150,7 +140,7 @@ class CommandLineController(val options: CommandLineOptions) {
 			logger.info("Initializing Download Manager")
 			val d = DownloadManager(client, CommandLineDownloadProgress(), database, user_manager, inisettings)
 			
-			if (options.cmd_list_channels) {
+			if (options.booleans.contains("list_channels")) {
 				val chats = d.getChats()
 				val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) }
 				val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")}
@@ -174,8 +164,8 @@ class CommandLineController(val options: CommandLineOptions) {
 				System.exit(0)
 			}
 			
-			logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.val_limit_messages)
+			logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.values.get("limit_messages")?.last())
-			d.downloadMessages(options.val_limit_messages)
+			d.downloadMessages(options.values.get("limit_messages")?.last()?.toInt())
 			logger.debug("IniSettings#download_media: {}", inisettings.download_media)
 			if (inisettings.download_media) {
 				logger.debug("Calling DownloadManager.downloadMedia")
@@ -184,7 +174,7 @@ class CommandLineController(val options: CommandLineOptions) {
 				println("Skipping media download because download_media is set to false.")
 			}
 
-			if (options.cmd_daemon) {
+			if (options.boolean.contains("daemon")) {
 				logger.info("Initializing TelegramUpdateHandler")
 				handler = TelegramUpdateHandler(user_manager, database)
 				client.close()
@@ -193,7 +183,7 @@ class CommandLineController(val options: CommandLineOptions) {
 				println("DAEMON mode requested - keeping running.")
 			}
 		} catch (e: Throwable) {
-			println("An error occured!")
+			println("An error occurred!")
 			e.printStackTrace()
 			logger.error("Exception caught!", e)
 		} finally {
@@ -274,19 +264,19 @@ class CommandLineController(val options: CommandLineOptions) {
 
 	private fun show_help() {
 		println("Valid options are:")
-		println(" -h, --help            Shows this help.")
+		println(" --help                Shows this help.")
-		println(" -a, --account <x>     Use account <x>.")
+		println(" --account <x>         Use account <x>.")
-		println(" -l, --login           Login to an existing telegram account.")
+		println(" --login               Login to an existing telegram account.")
 		println(" --debug               Shows some debug information.")
 		println(" --trace               Shows lots of debug information. Overrides --debug.")
 		println(" --trace-telegram      Shows lots of debug messages from the library used to access Telegram.")
-		println(" -A, --list-accounts   List all existing accounts ")
+		println(" --list-accounts       List all existing accounts ")
 		println(" --limit-messages <x>  Downloads at most the most recent <x> messages.")
-		println(" -t, --target <x>      Target directory for the files.")
+		println(" --target <x>          Target directory for the files.")
-		println(" -e, --export <format> Export the database. Valid formats are:")
+		println(" --export <format>     Export the database. Valid formats are:")
 		println("                html - Creates HTML files.")
 		println(" --license             Displays the license of this program.")
-		println(" -d, --daemon          Keep running after the backup and automatically save new messages.")
+		println(" --daemon              Keep running after the backup and automatically save new messages.")
 		println(" --anonymize           (Try to) Remove all sensitive information from output. Useful for requesting support.")
 		println(" --stats               Print some usage statistics.")
 		println(" --list-channels       Lists all channels together with their ID")

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt

@@ -16,81 +16,28 @@
 package de.fabianonline.telegram_backup
 
 class CommandLineOptions(args: Array<String>) {
-	var cmd_console = false
+	val booleans = mutableListOf<String>()
-	var cmd_help = false
+	val values = mutableMapOf<String, MutableList<String>>()
-	var cmd_login = false
+	var last_key: String? = null
-	var cmd_debug = false
+	
-	var cmd_trace = false
-	var cmd_trace_telegram = false
-	var cmd_list_accounts = false
-	var cmd_version = false
-	var cmd_license = false
-	var cmd_daemon = false
-	var cmd_stats = false
-	var cmd_list_channels = false
-	var val_account: String? = null
-	var val_limit_messages: Int? = null
-	var val_target: String? = null
-	var val_export: String? = null
-	var val_test: Int? = null
-
 	init {
-		var last_cmd: String? = null
+		for(arg in args) {
-		loop@ for (arg in args) {
+			if (arg.starts_with("--")) {
-			if (last_cmd != null) {
+				if (last_key!=null) {
-				when (last_cmd) {
+					booleans.add(last_key)
-					"--account" -> val_account = arg
-					"--limit-messages" -> val_limit_messages = Integer.parseInt(arg)
-					"--target" -> val_target = arg
-					"--export" -> val_export = arg
-					"--test" -> val_test = Integer.parseInt(arg)
 				}
-				last_cmd = null
+				last_key = arg.substr(2)
-				continue
+			} else {
-			}
+				if (last_key==null) throw RuntimeException("Unexpected parameter without switch: $arg")
-			when (arg) {
+				var list = values.get(last_key)
-				"-a", "--account" -> {
+				if (list==null) {
-					last_cmd = "--account"
+					list = mutableListOf<String>()
-					continue@loop
+					values.add(last_key, list)
 				}
-				"-h", "--help" -> cmd_help = true
+				list.add(arg)
-				"-l", "--login" -> cmd_login = true
+				last_key = null
-				"--debug" -> cmd_debug = true
-				"--trace" -> cmd_trace = true
-				"--trace-telegram" -> cmd_trace_telegram = true
-				"-A", "--list-accounts" -> cmd_list_accounts = true
-				"--limit-messages" -> {
-					last_cmd = arg
-					continue@loop
-				}
-				"--console" -> cmd_console = true
-				"-t", "--target" -> {
-					last_cmd = "--target"
-					continue@loop
-				}
-				"-V", "--version" -> cmd_version = true
-				"-e", "--export" -> {
-					last_cmd = "--export"
-					continue@loop
-				}
-				"--pagination" -> {
-					last_cmd = "--pagination"
-					continue@loop
-				}
-				"--license" -> cmd_license = true
-				"-d", "--daemon" -> cmd_daemon = true
-				"--test" -> {
-					last_cmd = "--test"
-					continue@loop
-				}
-				"--anonymize" -> Utils.anonymize = true
-				"--stats" -> cmd_stats = true
-				"--list-channels" -> cmd_list_channels = true
-				else -> throw RuntimeException("Unknown command " + arg)
 			}
 		}
-		if (last_cmd != null) {
+		if (last_key!=null) booleans.add(last_key)
-			CommandLineController.show_error("Command $last_cmd had no parameter set.")
-		}
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt

@@ -53,6 +53,11 @@ class CommandLineRunner(args: Array<String>) {
 	}
 	
 	fun setupLogging() {
+		if (options.booleans.hasKey('anonymize')) {
+			Utils.anonymize = true
+		}
+	
+	
 		logger.trace("Setting up Loggers...")
 		val rootLogger = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) as Logger
 		val rootContext = rootLogger.getLoggerContext()

src/main/kotlin/de/fabianonline/telegram_backup/Database.kt

@@ -240,6 +240,16 @@ class Database constructor(val file_base: String, val user_manager: UserManager)
 		rs.close()
 		return result
 	}
+	
+	fun queryStringMap(query: String): Map<String, String> {
+		val map = mutableMapOf<String, String>()
+		val rs = stmt.executeQuery(query)
+		while(rs.next()) {
+			map.add(rs.getString(1), rs.getString(2))
+		}
+		rs.close()
+		return map
+	}
 
 	@Synchronized
 	fun saveMessages(all: TLVector<TLAbsMessage>, api_layer: Int, source_type: MessageSource = MessageSource.NORMAL) {
@@ -489,7 +499,7 @@ class Database constructor(val file_base: String, val user_manager: UserManager)
 		ps_insert_or_replace.close()
 	}
 	
-	fun fetchSetting(key: String): String? = queryString("SELECT value FROM settings WHERE key='${key}'")
+	fun fetchSettings() = queryStringMap("SELECT key, value FROM settings WHERE key='${key}'")
 	
 	fun saveSetting(key: String, value: String?) {
 		val ps = conn.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)")

src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt

@@ -0,0 +1,151 @@
+package de.fabianonline.telegram_backup
+
+import java.io.File
+import org.slf4j.LoggerFactory
+
+class Settings(val file_base: String, val database: Database, val cli_settings: CommandLineOptions) {
+	val logger = LoggerFactory.getLogger(Settings::class.java)
+	val settings = mutableMapOf<String, Setting>()
+
+	val gmaps_key: Setting
+
+	private val db_settings: Map<String, String>
+
+	private var ini_settings: Map<String, List<String>>
+
+	init {
+		db_settings = database.fetchSettings()
+		ini_settings = load_ini("config.ini")
+		copy_sample_ini("config.sample.ini")
+
+		// Merging CLI and INI settings
+		gmaps_key = get_setting("gmaps_key", default=Config.SECRET_GMAPS)
+	}
+
+	private fun get_setting(name: String, default: String? = null): Setting {
+		return Setting.build(name, ini_settings[name], cli_settings.values[name], default)
+	}
+
+	private fun load_ini(filename: String): Map<String, List<String>> {
+		val map = mutableMapOf<String, MutableList<String>>()
+		val file = File(file_base + filename)
+		logger.trace("Checking ini file {}", filename.anonymize())
+		if (!file.exists()) return map
+		logger.debug("Loading ini file {}", filename.anonymize())
+		file.forEachLine { parseLine(it, map) }
+		return map
+	}
+
+	private fun parseLine(original_line: String, map: MutableMap<String, MutableList<String>>) {
+		logger.trace("Parsing line: {}", original_line)
+		var line = original_line.trim().replaceAfter("#", "").removeSuffix("#")
+		logger.trace("After cleaning: {}", line)
+		if (line == "") return
+		val parts: List<String> = line.split("=", limit=2).map{it.trim()}
+
+		if (parts.size < 2) throw RuntimeException("Invalid config setting: $line")
+
+		val (key, value) = parts
+		if (value=="") {
+			map.remove(key)
+		} else {
+			var list = map.get(key)
+			if (list == null) {
+				list = mutableListOf<String>()
+				map.put(key, list)
+			}
+			list.add(value)
+		}
+	}
+
+	private fun copy_sample_ini(filename: String) {
+		val stream = Config::class.java.getResourceAsStream("/config.sample.ini")
+		File(filename).outputStream().use { stream.copyTo(it) }
+		stream.close()
+	}
+	
+	
+	open class Setting(name: String, value: List<String>?, source: SettingSource) {
+		companion object {
+			val all = mutableListOf<Setting>()
+
+			fun build(name: String, ini_value: List<String>?, cli_value: List<String>?, default: String?): Setting {
+				var value: List<String>?
+				var source: SettingSource
+				if (cli_value != null) {
+					value=cli_value
+					source=SettingSource.CLI
+				} else if (ini_value != null) {
+					value=ini_value
+					source=SettingSource.INI
+				} else {
+					if (default!=null) {
+						value = listOf(default)
+					} else {
+						value = null
+					}
+					source=SettingSource.DEFAULT
+				}
+				return Setting(name, value, source);
+			}
+		}
+	}
+
+	enum class SettingSource {
+		INI,
+		CLI,
+		DEFAULT
+	}
+}
+/*
+class DbSettings(val database: Database) {
+	private fun fetchValue(name: String): String? = database.fetchSetting(name)
+	private fun saveValue(name: String, value: String?) = database.saveSetting(name, value)
+	
+	var pts: String?
+		get() = fetchValue("pts")
+		set(x: String?) = saveValue("pts", x)
+}
+
+
+package de.fabianonline.telegram_backup
+
+
+
+class Settings(val file_base: String) {
+	val logger = LoggerFactory.getLogger(Settings::class.java)
+	var settings = mutableMapOf<String, MutableList<String>>()
+	
+	init {
+		loadIni(file_base + "config.ini")
+		copySampleIni(file_base + "config.sample.ini")
+	}
+	
+
+	
+
+	
+
+	
+	fun println() = println(settings)
+	
+	fun getString(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default
+	fun getStringList(key: String): List<String>? = settings.get(key)
+	fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default
+	fun getBoolean(key: String, default: Boolean = false): Boolean {
+		val value = settings.get(key)?.last()
+		if (value==null) return default
+		return value=="true"
+	}
+	fun getArray(key: String): List<String> = settings.get(key) ?: listOf<String>()
+	
+	val gmaps_key = getString("gmaps_key", default=Config.SECRET_GMAPS)!!
+	val pagination = getBoolean("pagination", default=true)
+	val pagination_size = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!!
+	val download_media = getBoolean("download_media", default=true)
+	val download_channels = getBoolean("download_channels", default=false)
+	val download_supergroups = getBoolean("download_supergroups", default=false)
+	val whitelist_channels = getStringList("whitelist_channels")
+	val blacklist_channels = getStringList("blacklist_channels")
+}
+*/

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt

@@ -41,29 +41,29 @@ import java.util.concurrent.TimeoutException
 import org.apache.commons.io.FileUtils
 
 object FileManagerFactory {
-	fun getFileManager(m: TLMessage?, u: UserManager): AbstractMediaFileManager? {
+	fun getFileManager(m: TLMessage?, u: UserManager, c: TelegramClient): AbstractMediaFileManager? {
 		if (m == null) return null
 		val media = m.getMedia() ?: return null
 
 		if (media is TLMessageMediaPhoto) {
-			return PhotoFileManager(m, u)
+			return PhotoFileManager(m, u, file_base)
 		} else if (media is TLMessageMediaDocument) {
-			val d = DocumentFileManager(m, u)
+			val d = DocumentFileManager(m, u, file_base)
 			return if (d.isSticker) {
-				StickerFileManager(m, u)
+				StickerFileManager(m, u, file_base)
 			} else d
 		} else if (media is TLMessageMediaGeo) {
-			return GeoFileManager(m, u)
+			return GeoFileManager(m, u, file_base)
 		} else if (media is TLMessageMediaEmpty) {
-			return UnsupportedFileManager(m, u, "empty")
+			return UnsupportedFileManager(m, u, file_base, "empty")
 		} else if (media is TLMessageMediaUnsupported) {
-			return UnsupportedFileManager(m, u, "unsupported")
+			return UnsupportedFileManager(m, u, file_base, "unsupported")
 		} else if (media is TLMessageMediaWebPage) {
-			return UnsupportedFileManager(m, u, "webpage")
+			return UnsupportedFileManager(m, u, file_base, "webpage")
 		} else if (media is TLMessageMediaContact) {
-			return UnsupportedFileManager(m, u, "contact")
+			return UnsupportedFileManager(m, u, file_base, "contact")
 		} else if (media is TLMessageMediaVenue) {
-			return UnsupportedFileManager(m, u, "venue")
+			return UnsupportedFileManager(m, u, file_base, "venue")
 		} else {
 			AbstractMediaFileManager.throwUnexpectedObjectError(media)
 		}