Updated README.md to reflect the state of the project.

Corrects the path given in README.md.

.gitignore

@@ -10,3 +10,7 @@ src/main/main.iml
 cache4.*
 src/test/test.iml
 dev/
+todo
+deploy.secret.sh
+release_notes.txt
+out

.travis.yml

@@ -11,13 +11,3 @@ cache:
   directories:
     - $HOME/.gradle/caches/
     - $HOME/.gradle/wrapper/
-
-deploy:
-  provider: pages
-  skip-cleanup: true
-  github-token: $github_token
-  keep-history: true
-  on:
-    branch: master
-  local-dir: build/libs
-  target-branch: gh-pages

DEPLOY.md

@@ -2,6 +2,7 @@
 
 * Update the version in the Dockerfile to the coming version.
 * Commit the new Dockerfile.
+* Merge into stable: `git checkout stable && git merge --no-ff master`
 * Create a new tag for the new version: `git tag -a <version>`.
 * Push everything to github: `git push --all && git push --tags`.
 * Build it: `gradle build`.

Dockerfile

@@ -1,10 +1,15 @@
 FROM openjdk:8
 
-ENV JAR_VERSION 1.1.2
+ENV JAR_VERSION 1.1.3
 ENV JAR_DOWNLOAD_URL https://github.com/fabianonline/telegram_backup/releases/download/${JAR_VERSION}/telegram_backup.jar
 
-RUN apt-get update -y && apt-get install -y curl && \
+RUN apt-get update -y && \
+	apt-get install --no-install-recommends -y curl && \
     curl -L "https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64" -o /bin/dumb-init && \
-    curl -L $JAR_DOWNLOAD_URL -o telegram_backup.jar && mkdir /data/ && chmod +x /bin/dumb-init
+    curl -L $JAR_DOWNLOAD_URL -o telegram_backup.jar && mkdir /data/ && \
+    chmod +x /bin/dumb-init && \
+    apt-get remove -y curl && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
 
 ENTRYPOINT ["/bin/dumb-init", "--", "java", "-jar", "telegram_backup.jar", "--target", "/data/"]

README.md

@@ -2,6 +2,16 @@
 Copyright 2016 Fabian Schlenz  
 Licensed under GPLv3
 
+## State of this project
+The tool is working, but not really as intended: Media files in most cases can't be downloaded, message downloads are hit with 30 second delays after every 200 messages. Some users reported getting banned by Telegram without reason after using this tool (with many users not getting banned at the same time, so this could theoretically just be a coincidence).
+
+At the same time, the official Telegram client has an official way to download one's data, which is a) officially supported and b) much, much, much faster than this tool.
+
+Fixing this tool to at least get it to work again as planned would require more or less a complete rewrite of this code. Since I'm quite happy with the possibilities given by the official clients and don't have enough free time to spare to continue developing this project, I've decided to officially archive this tool. This is not an easy step for me, because this was my most used project and quite a lot of people wrote me nice messages and thanked me. But just keeping the user's hopes up for an update without really being able to do something doesn't seem fair. So...
+
+So long, and thanks for all the fish. ;-) \
+Fabian
+
 ## Description
 This is a small Java app that allows you to download all your history from
 Telegram's servers and keep a local copy of them.
@@ -21,7 +31,7 @@ You can find the whole app packed into one fat jar file under
 
 ## Limitations
 This tool relies on Telegram's API. They started rate limiting the calls
-made by this tool some time ago. As of february 2017, downloading messages
+made by this tool some time ago. As of February 2017, downloading messages
 is limited to 400 messages every 30 seconds, resulting in 48,000 messages
 per hour. Media download is not throttled right now, so it should be a lot
 quicker.
@@ -85,7 +95,7 @@ this will be detected at the next run of this program and then tried again.
 The files are being saved in your User directory in a folder named
 `telegram_backup`. Under windows, this would typically be under
 `C:\Users\<username>\telegram_backup`. Linux users should look unter
-`/home/<username>/telegram_backup`.
+`/home/<username>/.telegram_backup`.
 
 You can change this directory by supplying `--target <dir>` when calling
 Telegram_Backup.

build.gradle

@@ -38,7 +38,8 @@ dependencies {
 	compile 'com.github.spullara.mustache.java:compiler:0.9.5'
 	compile 'org.slf4j:slf4j-api:1.7.21'
 	compile 'ch.qos.logback:logback-classic:1.1.7'
-	compile 'com.google.code.gson:gson:2.5'
+	compile 'com.google.code.gson:gson:2.8.0'
+	compile 'com.github.salomonbrys.kotson:kotson:2.5.0'
 	compile 'com.github.kittinunf.fuel:fuel:1.12.0'
 	
 	testCompile 'junit:junit:4.12'

deploy.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+error() {
+	echo "Error: $1"
+	exit 1
+}
+
+[ -z "$1" ] && error "Parameter's missing. Expecting version number like '1.2.3' as first and only parameter."
+
+if [ "$1" == "--help" ]; then
+	echo "Usage: `basename "$0"` 1.2.3"
+	exit 1
+fi
+
+release_notes="$(cat release_notes.txt 2>/dev/null)"
+[ -z "$release_notes" ] && error "release_notes.txt is empty"
+
+VERSION="$1"
+
+source "deploy.secret.sh"
+[ -z "$BOT_TOKEN" ] && error "BOT_TOKEN is not set or empty."
+[ -z "$CHAT_ID" ] && error "CHAT_ID is not set or empty."
+[ -z "$TOKEN" ] && error "TOKEN is not set or empty."
+
+CURL_OPTS="-u fabianonline:$TOKEN"
+
+git diff-files --quiet --ignore-submodules -- || error "You have changes in your working tree."
+
+git diff-index --cached --quiet HEAD --ignore-submodules -- || error "You have uncommited changes."
+
+branch_name=$(git symbolic-ref HEAD 2>/dev/null)
+branch_name=${branch_name##refs/heads/}
+[ "$branch_name" == "master" ] || error "Current branch is $branch_name, not master."
+
+echo "Updating the Dockerfile..."
+sed -i "s/ENV JAR_VERSION .\+/ENV JAR_VERSION $VERSION/g" Dockerfile || error "Couldn't modify Dockerfile."
+
+echo "Committing the new Dockerfile..."
+git commit -m "Bumping the version to $VERSION" Dockerfile
+
+echo "Tagging the new version..."
+git tag -a "$VERSION" -m "Version $VERSION" || error
+
+echo "Building it..."
+gradle build || error "Build failed. What did you do?!"
+
+echo "Getting git stats..."
+git_stats=$(git diff --shortstat stable..)
+
+echo "Checking out stable..."
+git checkout stable || error
+
+echo "Merging master into stable..."
+git merge --no-ff -m "Merging master into stable for version $VERSION" master || error
+
+echo "Checking out master again..."
+git checkout master || error
+
+echo "Pushing all to Github..."
+git push --all || error
+
+echo "Pushing tags to Github..."
+git push --tags || error
+
+echo "Generating a release on Github..."
+json=$(ruby -e "require 'json'; puts({tag_name: '$VERSION', name: '$VERSION', body: \$stdin.read}.to_json)" <<< "$release_notes") || error "Couldn't generate JSON for Github"
+
+json=$(curl $CURL_OPTS https://api.github.com/repos/fabianonline/telegram_backup/releases -XPOST -d "$json") || error "Github failure"
+
+echo "Uploading telegram_backup.jar to Github..."
+upload_url=$(jq -r ".upload_url" <<< "$json") || error "Could not parse JSON from Github"
+upload_url=$(sed 's/{.*}//' <<< "$upload_url")
+release_url=$(jq -r ".html_url" <<< "$json") || error "Could not parse JSON from Github"
+curl $CURL_OPTS --header "Content-Type: application/zip" "${upload_url}?name=telegram_backup.jar" --upload-file build/libs/telegram_backup.jar || error "Asset upload to github failed"
+
+echo "Building the docker image..."
+docker build -t fabianonline/telegram_backup:$VERSION -t fabianonline/telegram_backup:latest - < Dockerfile
+
+echo "Pushing the docker image..."
+docker push fabianonline/telegram_backup
+
+echo "Notifying the Telegram group..."
+release_notes=$(sed 's/\* /• /' | sed 's/&/&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g' <<< "$release_notes")
+message="<b>Version $VERSION was just released</b>"$'\n'"$git_stats"$'\n'$'\n'"$release_notes"$'\n'$'\n'"$release_url"
+
+curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" --form-string "parse_mode=HTML" --form-string "disable_web_page_preview=true" <<< "$message"
+
+echo "Cleaning release_notes.txt..."
+> release_notes.txt
+
+echo "Checking out master..."
+git checkout master
+
+echo "Done."

deploy_beta.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+error() {
+	echo "Error: $1"
+	exit 1
+}
+
+release_notes="$(cat release_notes.txt 2>/dev/null)"
+
+source "deploy.secret.sh"
+[ -z "$BOT_TOKEN" ] && error "BOT_TOKEN is not set or empty."
+[ -z "$CHAT_ID" ] && error "CHAT_ID is not set or empty."
+
+version=$(git describe --tags --dirty)
+
+echo "Enter additional notes, end with Ctrl-D."
+additional_notes="$(cat)"
+
+echo "Building it..."
+gradle build || error "Build failed. What did you do?!"
+
+echo "Getting git stats..."
+git_stats=$(git diff --shortstat stable..)
+
+echo "Copying it to files.fabianonline.de..."
+filename="telegram_backup.beta_${version}.jar"
+cp --no-preserve "mode,ownership,timestamps" build/libs/telegram_backup.jar /data/containers/nginx/www/files/${filename}
+
+echo "Notifying the Telegram group..."
+release_notes=$(echo "$release_notes" | sed 's/\* /• /' | sed 's/&/&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g')
+message="<b>New beta release $version</b>"$'\n'
+message="${message}${git_stats}"$'\n\n'
+message="${message}${additional_notes}"$'\n\n'
+message="${message}Changes since the last <i>real</i> release:"$'\n'"${release_notes}"$'\n\n'
+message="${message}<b>This is a release for testing purposes only. There may be bugs included that might destroy your data. Only use this beta release if you know what you're doing. AND MAKE A BACKUP OF YOUR BACKUP BEFORE USING IT!</b>"$'\n\n'
+message="${message}Please report back if you used this release and encountered a bug. Also report back, if you used it and IT WORKED, please. Thank you."$'\n\n'
+message="${message}https://files.fabianonline.de/${filename}"
+
+result=$(curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" --form-string "parse_mode=HTML" --form-string "disable_web_page_preview=true" <<< "$message")
+message_id=$(jq -r '.result.message_id' <<< "$result")
+
+echo "Pinning the new message..."
+curl https://api.telegram.org/bot${BOT_TOKEN}/pinChatMessage -XPOST --form "chat_id=${CHAT_ID}" --form "message_id=${message_id}" --form "disable_notification=true"
+
+echo "Done."

gradle/wrapper/gradle-wrapper.properties

@@ -1,4 +1,4 @@
-#Thu Oct 06 11:24:39 CEST 2016
+#Fri Mar 23 06:07:28 CET 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME

src/main/kotlin/de/fabianonline/telegram_backup/Account.kt

@@ -0,0 +1,5 @@
+package de.fabianonline.telegram_backup
+
+class Account(val file_base: String, val phone_number: String) {
+
+}

src/main/kotlin/de/fabianonline/telegram_backup/ApiStorage.kt

@@ -22,120 +22,64 @@ import com.github.badoualy.telegram.mtproto.auth.AuthKey
 import com.github.badoualy.telegram.mtproto.model.MTSession
 
 import org.apache.commons.io.FileUtils
+import org.slf4j.LoggerFactory
+import org.slf4j.Logger
 
 import java.io.File
 import java.io.FileNotFoundException
 import java.io.IOException
 
-internal class ApiStorage(prefix: String?) : TelegramApiStorage {
-	private var prefix: String? = null
-	private var do_save = false
-	private var auth_key: AuthKey? = null
-	private var dc: DataCenter? = null
-	private var file_auth_key: File? = null
-	private var file_dc: File? = null
-
+internal class ApiStorage(val base_dir: String) : TelegramApiStorage {
+	var auth_key: AuthKey? = null
+	var dc: DataCenter? = null
+	val file_auth_key: File
+	val file_dc: File
+	val logger = LoggerFactory.getLogger(ApiStorage::class.java)
+	                                   
+	
 	init {
-		this.setPrefix(prefix)
-	}
-
-	fun setPrefix(prefix: String?) {
-		this.prefix = prefix
-		this.do_save = this.prefix != null
-		if (this.do_save) {
-			val base = Config.FILE_BASE +
-				File.separatorChar +
-				this.prefix +
-				File.separatorChar
-			this.file_auth_key = File(base + Config.FILE_NAME_AUTH_KEY)
-			this.file_dc = File(base + Config.FILE_NAME_DC)
-			this._saveAuthKey()
-			this._saveDc()
-		} else {
-			this.file_auth_key = null
-			this.file_dc = null
-		}
+		file_auth_key = File(base_dir + Config.FILE_NAME_AUTH_KEY)
+		file_dc = File(base_dir + Config.FILE_NAME_DC)
 	}
 
 	override fun saveAuthKey(authKey: AuthKey) {
-		this.auth_key = authKey
-		this._saveAuthKey()
-	}
-
-	private fun _saveAuthKey() {
-		if (this.do_save && this.auth_key != null) {
-			try {
-				FileUtils.writeByteArrayToFile(this.file_auth_key, this.auth_key!!.key)
-			} catch (e: IOException) {
-				e.printStackTrace()
-			}
-
-		}
+		FileUtils.writeByteArrayToFile(file_auth_key, authKey.key)
 	}
 
 	override fun loadAuthKey(): AuthKey? {
-		if (this.auth_key != null) return this.auth_key
-		if (this.file_auth_key != null) {
-			try {
-				return AuthKey(FileUtils.readFileToByteArray(this.file_auth_key))
-			} catch (e: IOException) {
-				if (e !is FileNotFoundException) e.printStackTrace()
-			}
-
+		try {
+			return AuthKey(FileUtils.readFileToByteArray(file_auth_key))
+		} catch (e: FileNotFoundException) {
+			return null
 		}
-
-		return null
 	}
 
 	override fun saveDc(dataCenter: DataCenter) {
-		this.dc = dataCenter
-		this._saveDc()
-	}
-
-	private fun _saveDc() {
-		if (this.do_save && this.dc != null) {
-			try {
-				FileUtils.write(this.file_dc, this.dc!!.toString())
-			} catch (e: IOException) {
-				e.printStackTrace()
-			}
-
-		}
+		FileUtils.write(file_dc, dataCenter.toString())
 	}
 
 	override fun loadDc(): DataCenter? {
-		if (this.dc != null) return this.dc
-		if (this.file_dc != null) {
-			try {
-				val infos = FileUtils.readFileToString(this.file_dc).split(":")
-				return DataCenter(infos[0], Integer.parseInt(infos[1]))
-			} catch (e: IOException) {
-				if (e !is FileNotFoundException) e.printStackTrace()
-			}
-
+		try {
+			val infos = FileUtils.readFileToString(this.file_dc).split(":")
+			return DataCenter(infos[0], Integer.parseInt(infos[1]))
+		} catch (e: FileNotFoundException) {
+			return null
 		}
-		return null
 	}
 
 	override fun deleteAuthKey() {
-		if (this.do_save) {
-			try {
-				FileUtils.forceDelete(this.file_auth_key)
-			} catch (e: IOException) {
-				e.printStackTrace()
-			}
-
+		try {
+			FileUtils.forceDelete(file_auth_key)
+		} catch (e: IOException) {
+			logger.warn("Exception in deleteAuthKey(): {}", e)
 		}
 	}
 
 	override fun deleteDc() {
-		if (this.do_save) {
-			try {
-				FileUtils.forceDelete(this.file_dc)
-			} catch (e: IOException) {
-				e.printStackTrace()
-			}
-
+		try {
+			FileUtils.forceDelete(file_dc)
+		} catch (e: IOException) {
+			logger.warn("Exception in deleteDc(): {}", e)
 		}
 	}
 

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt

@@ -16,7 +16,7 @@
 package de.fabianonline.telegram_backup
 
 import de.fabianonline.telegram_backup.TelegramUpdateHandler
-import de.fabianonline.telegram_backup.exporter.HTMLExporter
+import de.fabianonline.telegram_backup.exporter.*
 import com.github.badoualy.telegram.api.Kotlogram
 import com.github.badoualy.telegram.api.TelegramApp
 import com.github.badoualy.telegram.api.TelegramClient
@@ -29,132 +29,174 @@ import java.util.HashMap
 import org.slf4j.LoggerFactory
 import org.slf4j.Logger
 
-class CommandLineController {
-	private val storage: ApiStorage
-	var app: TelegramApp
-
-	private fun getLine(): String {
-		if (System.console() != null) {
-			return System.console().readLine("> ")
-		} else {
-			print("> ")
-			return Scanner(System.`in`).nextLine()
-		}
-	}
-
-	private fun getPassword(): String {
-		if (System.console() != null) {
-			return String(System.console().readPassword("> "))
-		} else {
-			return getLine()
-		}
-	}
+class CommandLineController(val options: CommandLineOptions) {
+	val logger = LoggerFactory.getLogger(CommandLineController::class.java)
 
 	init {
+		val storage: ApiStorage
+		val app: TelegramApp
+		val target_dir: String
+		val file_base: String
+		val phone_number: String
+		val handler: TelegramUpdateHandler
+		var client: TelegramClient
+		val user_manager: UserManager
+		val settings: Settings
+		val database: Database
 		logger.info("CommandLineController started. App version {}", Config.APP_APPVER)
 		
-		this.printHeader()
-		if (CommandLineOptions.cmd_version) {
+		printHeader()
+		if (options.isSet("version")) {
 			System.exit(0)
-		} else if (CommandLineOptions.cmd_help) {
-			this.show_help()
+		} else if (options.isSet("help")) {
+			show_help()
 			System.exit(0)
-		} else if (CommandLineOptions.cmd_license) {
-			CommandLineController.show_license()
-			System.exit(0)
-		}
-		this.setupFileBase()
-		if (CommandLineOptions.cmd_list_accounts) {
-			this.list_accounts()
+		} else if (options.isSet("license")) {
+			show_license()
 			System.exit(0)
 		}
+		
+		// Setup TelegramApp
 		logger.debug("Initializing TelegramApp")
 		app = TelegramApp(Config.APP_ID, Config.APP_HASH, Config.APP_MODEL, Config.APP_SYSVER, Config.APP_APPVER, Config.APP_LANG)
+
+		// Setup file_base
+		logger.debug("Target dir from Config: {}", Config.TARGET_DIR.anonymize())
+		target_dir = options.get("target") ?: Config.TARGET_DIR
+		logger.debug("Target dir after options: {}", target_dir.anonymize())
+		println("Base directory for files: ${target_dir.anonymize()}")
+
+		if (options.isSet("list_accounts")) {
+			Utils.print_accounts(target_dir)
+			System.exit(0)
+		}
+		
+		if (options.isSet("login")) {
+			cmd_login(app, target_dir, options.get("account"))
+		}
+
 		logger.trace("Checking accounts")
-		val account = this.selectAccount()
-		logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login)
+		phone_number = try { selectAccount(target_dir, options.get("account"))
+		} catch(e: AccountNotFoundException) {
+			show_error("The specified account could not be found.")
+		} catch(e: NoAccountsException) {
+			println("No accounts found. Starting login process...")
+			cmd_login(app, target_dir, options.get("account"))
+		}
+		
+		// TODO: Create a new TelegramApp if the user set his/her own TelegramApp credentials
+
+		// At this point we can assume that the selected user account ("phone_number") exists.
+		// So we can create some objects:
+		file_base = build_file_base(target_dir, phone_number)
+
 		logger.info("Initializing ApiStorage")
-		storage = ApiStorage(account)
-		logger.info("Initializing TelegramUpdateHandler")
-		val handler = TelegramUpdateHandler()
+		storage = ApiStorage(file_base)
+		
 		logger.info("Creating Client")
-		val client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, handler)
+		client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, null)
+		
+		// From now on we have a new catch-all-block that will terminate it's TelegramClient when an exception happens.
 		try {
 			logger.info("Initializing UserManager")
-			UserManager.init(client)
-			val user = UserManager.getInstance()
-			if (!CommandLineOptions.cmd_login && !user.loggedIn) {
+			user_manager = UserManager(client)
+		
+			// TODO
+			/*if (!options.cmd_login && !user.loggedIn) {
 				println("Your authorization data is invalid or missing. You will have to login with Telegram again.")
-				CommandLineOptions.cmd_login = true
-			}
-			if (account != null && user.loggedIn) {
-				if (account != "+" + user.user!!.getPhone()) {
-					logger.error("Account: {}, user.user!!.getPhone(): +{}", account.anonymize(), user.user!!.getPhone().anonymize())
-					throw RuntimeException("Account / User mismatch")
-				}
-			}
-			logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login)
-			if (CommandLineOptions.cmd_login) {
-				cmd_login(account)
-				System.exit(0)
+				options.cmd_login = true
+			}*/
+			
+			if (phone_number != user_manager.phone) {
+				logger.error("phone_number: {}, user_manager.phone: {}", phone_number.anonymize(), user_manager.phone.anonymize())
+				show_error("Account / User mismatch")
 			}
+			
 			// If we reach this point, we can assume that there is an account and a database can be loaded / created.
-			Database.init(client)
-			if (CommandLineOptions.cmd_stats) {
-				cmd_stats()
+			database = Database(file_base, user_manager)
+			
+			Runtime.getRuntime().addShutdownHook(Thread() {
+				database.close()
+			})
+			
+			// Load the settings and stuff.
+			settings = Settings(file_base, database, options)
+			
+			if (options.isSet("stats")) {
+				cmd_stats(file_base, database)
+				System.exit(0)
+			} else if (options.isSet("settings")) {
+				settings.print()
 				System.exit(0)
 			}
-			if (CommandLineOptions.val_test != null) {
-				if (CommandLineOptions.val_test == 1) {
-					TestFeatures.test1()
-				} else if (CommandLineOptions.val_test == 2) {
-					TestFeatures.test2()
-				} else {
-					System.out.println("Unknown test " + CommandLineOptions.val_test)
-				}
-				System.exit(1)
-			}
-			logger.debug("CommandLineOptions.val_export: {}", CommandLineOptions.val_export)
-			if (CommandLineOptions.val_export != null) {
-				if (CommandLineOptions.val_export!!.toLowerCase().equals("html")) {
-					(HTMLExporter()).export()
-					System.exit(0)
-				} else {
-					show_error("Unknown export format.")
-				}
-			}
-			if (user.loggedIn) {
-				System.out.println("You are logged in as ${user.userString.anonymize()}")
-			} else {
-				println("You are not logged in.")
-				System.exit(1)
+			
+			val export = options.get("export")?.toLowerCase()
+			logger.debug("options.export: {}", export)
+			when(export) {
+				"html" -> { HTMLExporter(database, user_manager, settings=settings, file_base=file_base).export() ; System.exit(0) }
+				"csv" -> { CSVExporter(database, file_base, settings).export(); System.exit(0) }
+				"csv_links" -> { CSVLinkExporter(database, file_base, settings).export() ; System.exit(0) }
+				null -> { /* No export whished -> do nothing. */ }
+				else -> show_error("Unknown export format '${export}'.")
 			}
+			
+			println("You are logged in as ${user_manager.toString().anonymize()}")
+
 			logger.info("Initializing Download Manager")
-			val d = DownloadManager(client, CommandLineDownloadProgress())
-			logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages)
-			d.downloadMessages(CommandLineOptions.val_limit_messages)
-			logger.debug("CommandLineOptions.cmd_no_media: {}", CommandLineOptions.cmd_no_media)
-			if (!CommandLineOptions.cmd_no_media) {
+			val d = DownloadManager(client, CommandLineDownloadProgress(), database, user_manager, settings, file_base)
+			
+			if (options.isSet("list_channels")) {
+				val chats = d.getChats()
+				val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) }
+				val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")}
+				var download: Boolean
+
+				println("Channels:")
+				download = settings.download_channels
+				if (!download) println("Download of channels is disabled - see download_channels in config.ini")
+				print_header(download)
+				for (c in chats.channels) {
+					println(format(c, download))
+				}
+				println()
+				println("Supergroups:")
+				download = settings.download_supergroups
+				if (!download) println("Download of supergroups is disabled - see download_supergroups in config.ini")
+				print_header(download)
+				for (c in chats.supergroups) {
+					println(format(c, download))
+				}
+				System.exit(0)
+			}
+			
+			logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.get("limit_messages"))
+			d.downloadMessages(options.get("limit_messages")?.toInt())
+			logger.debug("IniSettings#download_media: {}", settings.download_media)
+			if (settings.download_media) {
 				logger.debug("Calling DownloadManager.downloadMedia")
 				d.downloadMedia()
 			} else {
-				println("Skipping media download because --no-media is set.")
+				println("Skipping media download because download_media is set to false.")
 			}
-		} catch (e: Exception) {
+
+			if (options.isSet("daemon")) {
+				logger.info("Initializing TelegramUpdateHandler")
+				handler = TelegramUpdateHandler(user_manager, database, file_base, settings)
+				client.close()
+				logger.info("Creating new client")
+				client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, handler)
+				println("DAEMON mode requested - keeping running.")
+			}
+		} catch (e: Throwable) {
+			println("An error occurred!")
 			e.printStackTrace()
 			logger.error("Exception caught!", e)
-			// If we encountered an exception, we definitely don't want to start the daemon mode now.
-			CommandLineOptions.cmd_daemon = false
+			System.exit(1)
 		} finally {
-			if (CommandLineOptions.cmd_daemon) {
-				handler.activate()
-				println("DAEMON mode requested - keeping running.")
-			} else {
-				client.close()
-				println()
-				println("----- EXIT -----")
-				System.exit(0)
-			}
+			client.close()
+			println()
+			println("----- EXIT -----")
+			System.exit(0)
 		}
 	}
 
@@ -166,153 +208,103 @@ class CommandLineController {
 		println()
 	}
 
-	private fun setupFileBase() {
-		logger.debug("Target dir at startup: {}", Config.FILE_BASE.anonymize())
-		if (CommandLineOptions.val_target != null) {
-			Config.FILE_BASE = CommandLineOptions.val_target!!
-		}
-		logger.debug("Target dir after options: {}", Config.FILE_BASE.anonymize())
-		System.out.println("Base directory for files: " + Config.FILE_BASE.anonymize())
-	}
-
-	private fun selectAccount(): String? {
-		var account = "none"
-		val accounts = Utils.getAccounts()
-		if (CommandLineOptions.cmd_login) {
-			logger.debug("Login requested, doing nothing.")
-			// do nothing
-		} else if (CommandLineOptions.val_account != null) {
-			logger.debug("Account requested: {}", CommandLineOptions.val_account!!.anonymize())
+	private fun selectAccount(file_base: String, requested_account: String?): String {
+		var found_account: String?
+		val accounts = Utils.getAccounts(file_base)
+		if (requested_account != null) {
+			logger.debug("Account requested: {}", requested_account.anonymize())
 			logger.trace("Checking accounts for match.")
-			var found = false
-			for (acc in accounts) {
-				logger.trace("Checking {}", acc.anonymize())
-				if (acc == CommandLineOptions.val_account) {
-					found = true
-					logger.trace("Matches.")
-					break
-				}
-			}
-			if (!found) {
-				show_error("Couldn't find account '" + CommandLineOptions.val_account!!.anonymize() + "'. Maybe you want to use '--login' first?")
-			}
-			account = CommandLineOptions.val_account!!
+			found_account = accounts.find{it == requested_account}
 		} else if (accounts.size == 0) {
-			println("No accounts found. Starting login process...")
-			CommandLineOptions.cmd_login = true
-			return null
+			throw NoAccountsException()
 		} else if (accounts.size == 1) {
-			account = accounts.firstElement()
-			System.out.println("Using only available account: " + account.anonymize())
+			found_account = accounts.firstElement()
+			println("Using only available account: " + found_account.anonymize())
 		} else {
-			show_error(("You didn't specify which account to use.\n" +
+			show_error(("You have more than one account but didn't specify which one to use.\n" +
 				"Use '--account <x>' to use account <x>.\n" +
 				"Use '--list-accounts' to see all available accounts."))
-			System.exit(1)
 		}
+		
+		if (found_account == null) {
+			throw AccountNotFoundException()
+		}
+		
 		logger.debug("accounts.size: {}", accounts.size)
-		logger.debug("account: {}", account.anonymize())
-		return account
+		logger.debug("account: {}", found_account.anonymize())
+		return found_account
 	}
 
-	private fun cmd_stats() {
+	private fun cmd_stats(file_base: String, db: Database) {
 		println()
 		println("Stats:")
 		val format = "%40s: %d%n"
-		System.out.format(format, "Number of accounts", Utils.getAccounts().size)
-		System.out.format(format, "Number of messages", Database.getInstance().getMessageCount())
-		System.out.format(format, "Number of chats", Database.getInstance().getChatCount())
-		System.out.format(format, "Number of users", Database.getInstance().getUserCount())
-		System.out.format(format, "Top message ID", Database.getInstance().getTopMessageID())
+		System.out.format(format, "Number of accounts", Utils.getAccounts(file_base).size)
+		System.out.format(format, "Number of messages", db.getMessageCount())
+		System.out.format(format, "Number of chats", db.getChatCount())
+		System.out.format(format, "Number of users", db.getUserCount())
+		System.out.format(format, "Top message ID", db.getTopMessageID())
 		println()
 		println("Media Types:")
-		for ((key, value) in Database.getInstance().getMessageMediaTypesWithCount()) {
+		for ((key, value) in db.getMessageMediaTypesWithCount()) {
 			System.out.format(format, key, value)
 		}
 		println()
 		println("Api layers of messages:")
-		for ((key, value) in Database.getInstance().getMessageApiLayerWithCount()) {
+		for ((key, value) in db.getMessageApiLayerWithCount()) {
 			System.out.format(format, key, value)
 		}
 		println()
 		println("Message source types:")
-		for ((key, value) in Database.getInstance().getMessageSourceTypeWithCount()) {
+		for ((key, value) in db.getMessageSourceTypeWithCount()) {
 			System.out.format(format, key, value)
 		}
 	}
 
-	@Throws(RpcErrorException::class, IOException::class)
-	private fun cmd_login(phoneToUse: String?) {
-		val user = UserManager.getInstance()
-		val phone: String
-		if (phoneToUse == null) {
-			println("Please enter your phone number in international format.")
-			println("Example: +4917077651234")
-			phone = getLine()
-		} else {
-			phone = phoneToUse
-		}
-		user.sendCodeToPhoneNumber(phone)
-		println("Telegram sent you a code. Please enter it here.")
-		val code = getLine()
-		user.verifyCode(code)
-		if (user.isPasswordNeeded) {
-			println("We also need your account password. Please enter it now. It should not be printed, so it's okay if you see nothing while typing it.")
-			val pw = getPassword()
-			user.verifyPassword(pw)
-		}
-		storage.setPrefix("+" + user.user!!.getPhone())
-		System.out.println("Everything seems fine. Please run this tool again with '--account +" + user.user!!.getPhone().anonymize() + " to use this account.")
+	private fun cmd_login(app: TelegramApp, target_dir: String, phoneToUse: String?): Nothing {
+		LoginManager(app, target_dir, phoneToUse).run()
+		System.exit(0)
+		throw RuntimeException("Code never reaches this. This exists just to keep the Kotlin compiler happy.")
 	}
 
 	private fun show_help() {
 		println("Valid options are:")
-		println(" -h, --help            Shows this help.")
-		println(" -a, --account <x>     Use account <x>.")
-		println(" -l, --login           Login to an existing telegram account.")
+		println(" --help                Shows this help.")
+		println(" --account <x>         Use account <x>.")
+		println(" --login               Login to an existing telegram account.")
 		println(" --debug               Shows some debug information.")
 		println(" --trace               Shows lots of debug information. Overrides --debug.")
 		println(" --trace-telegram      Shows lots of debug messages from the library used to access Telegram.")
-		println(" -A, --list-accounts   List all existing accounts ")
+		println(" --list-accounts       List all existing accounts ")
 		println(" --limit-messages <x>  Downloads at most the most recent <x> messages.")
-		println(" --no-media            Do not download media files.")
-		println(" -t, --target <x>      Target directory for the files.")
-		println(" -e, --export <format> Export the database. Valid formats are:")
+		println(" --target <x>          Target directory for the files.")
+		println(" --export <format>     Export the database. Valid formats are:")
 		println("                html - Creates HTML files.")
-		println(" --pagination <x>      Splits the HTML export into multiple HTML pages with <x> messages per page. Default is 5000.")
-		println(" --no-pagination       Disables pagination.")
+		println("                csv  - Creates daily CSV files for the last 7 days. Set max_file_age to change the number of days.")
 		println(" --license             Displays the license of this program.")
+		println(" --daemon              Keep running after the backup and automatically save new messages.")
 		println(" --anonymize           (Try to) Remove all sensitive information from output. Useful for requesting support.")
 		println(" --stats               Print some usage statistics.")
-		println(" --with-channels       Backup channels as well.")
-		println(" --with-supergroups    Backup supergroups as well.")
-	}
-
-	private fun list_accounts() {
-		println("List of available accounts:")
-		val accounts = Utils.getAccounts()
-		if (accounts.size > 0) {
-			for (str in accounts) {
-				System.out.println(" " + str.anonymize())
-			}
-			println("Use '--account <x>' to use one of those accounts.")
-		} else {
-			println("NO ACCOUNTS FOUND")
-			println("Use '--login' to login to a telegram account.")
-		}
+		println(" --list-channels       Lists all channels together with their ID")
 	}
 
 	companion object {
 		private val logger = LoggerFactory.getLogger(CommandLineController::class.java)
 
-		public fun show_error(error: String) {
+		public fun show_error(error: String): Nothing {
 			logger.error(error)
 			println("ERROR: " + error)
 			System.exit(1)
+			throw RuntimeException("Code never reaches this. This exists just to keep the Kotlin compiler happy.")
 		}
 
 		fun show_license() {
 			println("TODO: Print the GPL.")
 		}
+		
+		fun build_file_base(target_dir: String, account_to_use: String) = target_dir + File.separatorChar + account_to_use + File.separatorChar
 	}
+
+	class AccountNotFoundException() : Exception("Account not found") {}
+	class NoAccountsException() : Exception("No accounts found") {}
 }

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineDownloadProgress.kt

@@ -23,6 +23,8 @@ import de.fabianonline.telegram_backup.Utils
 internal class CommandLineDownloadProgress : DownloadProgressInterface {
 	private var mediaCount = 0
 	private var i = 0
+	private var step = 0
+	private val chars = arrayOf("|", "/", "-", "\\")
 
 	override fun onMessageDownloadStart(count: Int, source: String?) {
 		i = 0
@@ -51,13 +53,29 @@ internal class CommandLineDownloadProgress : DownloadProgressInterface {
 		println("'S' - Sticker       'A' - Audio         'G' - Geolocation")
 		println("'.' - Previously downloaded file        'e' - Empty file")
 		println("' ' - Ignored media type (weblinks or contacts, for example)")
-		println("'x' - File skipped because of errors - will be tried again at next run")
+		println("'x' - File skipped (because of max_file_age or max_file_size)")
+		println("'!' - Download failed. Will be tried again at next run.")
 		println("" + count + " Files to check / download")
 	}
 
 	override fun onMediaDownloaded(file_manager: AbstractMediaFileManager) {
 		show(file_manager.letter.toUpperCase())
 	}
+	
+	override fun onMediaFileDownloadStarted() {
+		step = 0
+		print(chars[step % chars.size])
+	}
+	
+	override fun onMediaFileDownloadStep() {
+		step++
+		print("\b")
+		print(chars[step % chars.size])
+	}
+	
+	override fun onMediaFileDownloadFinished() {
+		print("\b")
+	}
 
 	override fun onMediaDownloadedEmpty() {
 		show("e")
@@ -75,6 +93,8 @@ internal class CommandLineDownloadProgress : DownloadProgressInterface {
 		showNewLine()
 		println("Done.")
 	}
+	
+	override fun onMediaFailed() = show("!")
 
 	private fun show(letter: String) {
 		print(letter)

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt

@@ -15,91 +15,51 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>. */
 package de.fabianonline.telegram_backup
 
-internal object CommandLineOptions {
-	public var cmd_console = false
-	public var cmd_help = false
-	public var cmd_login = false
-	var cmd_debug = false
-	var cmd_trace = false
-	var cmd_trace_telegram = false
-	var cmd_list_accounts = false
-	var cmd_version = false
-	var cmd_license = false
-	var cmd_daemon = false
-	var cmd_no_media = false
-	var cmd_anonymize = false
-	var cmd_stats = false
-	var cmd_channels = false
-	var cmd_supergroups = false
-	var cmd_no_pagination = false
-	var val_account: String? = null
-	var val_limit_messages: Int? = null
-	var val_target: String? = null
-	var val_export: String? = null
-	var val_test: Int? = null
-	var val_pagination: Int = Config.DEFAULT_PAGINATION
-	@JvmStatic
-	fun parseOptions(args: Array<String>) {
-		var last_cmd: String? = null
-		loop@ for (arg in args) {
-			if (last_cmd != null) {
-				when (last_cmd) {
-					"--account" -> val_account = arg
-					"--limit-messages" -> val_limit_messages = Integer.parseInt(arg)
-					"--target" -> val_target = arg
-					"--export" -> val_export = arg
-					"--test" -> val_test = Integer.parseInt(arg)
-					"--pagination" -> val_pagination = Integer.parseInt(arg)
-				}
-				last_cmd = null
-				continue
+class CommandLineOptions(args: Array<String>) {
+	private val values = mutableMapOf<String, String>()
+	var last_key: String? = null
+	val substitutions = mapOf("-t" to "--target")
+	
+	init {
+		val list = args.toMutableList()
+		
+		while (list.isNotEmpty()) {
+			
+			var current_arg = list.removeAt(0)
+			if (!current_arg.startsWith("-")) throw RuntimeException("Unexpected unnamed parameter ${current_arg}")
+			
+			var next_arg: String? = null
+			
+			if (current_arg.contains("=")) {
+				val parts = current_arg.split("=", limit=2)
+				current_arg = parts[0]
+				next_arg = parts[1]
+			} else if (list.isNotEmpty() && !list[0].startsWith("--")) {
+				next_arg = list.removeAt(0)
 			}
-			when (arg) {
-				"-a", "--account" -> {
-					last_cmd = "--account"
-					continue@loop
-				}
-				"-h", "--help" -> cmd_help = true
-				"-l", "--login" -> cmd_login = true
-				"--debug" -> cmd_debug = true
-				"--trace" -> cmd_trace = true
-				"--trace-telegram" -> cmd_trace_telegram = true
-				"-A", "--list-accounts" -> cmd_list_accounts = true
-				"--limit-messages" -> {
-					last_cmd = arg
-					continue@loop
-				}
-				"--console" -> cmd_console = true
-				"-t", "--target" -> {
-					last_cmd = "--target"
-					continue@loop
-				}
-				"-V", "--version" -> cmd_version = true
-				"-e", "--export" -> {
-					last_cmd = "--export"
-					continue@loop
-				}
-				"--pagination" -> {
-					last_cmd = "--pagination"
-					continue@loop
-				}
-				"--no-pagination" -> cmd_no_pagination = true
-				"--license" -> cmd_license = true
-				"-d", "--daemon" -> cmd_daemon = true
-				"--no-media" -> cmd_no_media = true
-				"--test" -> {
-					last_cmd = "--test"
-					continue@loop
-				}
-				"--anonymize" -> cmd_anonymize = true
-				"--stats" -> cmd_stats = true
-				"--with-channels" -> cmd_channels = true
-				"--with-supergroups" -> cmd_supergroups = true
-				else -> throw RuntimeException("Unknown command " + arg)
+			
+			if (!current_arg.startsWith("--") && current_arg.startsWith("-")) {
+				val replacement = substitutions.get(current_arg)
+				if (replacement == null) throw RuntimeException("Unknown short parameter ${current_arg}")
+				current_arg = replacement
+			}
+			
+			current_arg = current_arg.substring(2)
+				
+			if (next_arg == null) {
+				// current_arg seems to be a boolean value
+				values.put(current_arg, "true")
+				if (current_arg.startsWith("no-")) {
+					current_arg = current_arg.substring(3)
+					values.put(current_arg, "false")
+				}
+			} else {
+				// current_arg has the value next_arg
+				values.put(current_arg, next_arg)
 			}
-		}
-		if (last_cmd != null) {
-			CommandLineController.show_error("Command $last_cmd had no parameter set.")
 		}
 	}
+	
+	operator fun get(name: String): String? = values[name]
+	fun isSet(name: String): Boolean = values[name]=="true"
 }

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt

@@ -19,6 +19,7 @@ package de.fabianonline.telegram_backup
 import de.fabianonline.telegram_backup.CommandLineController
 import de.fabianonline.telegram_backup.Utils
 import de.fabianonline.telegram_backup.Version
+import java.util.concurrent.TimeUnit
 import org.slf4j.LoggerFactory
 import ch.qos.logback.classic.Logger
 import ch.qos.logback.classic.LoggerContext
@@ -28,24 +29,35 @@ import ch.qos.logback.core.ConsoleAppender
 import ch.qos.logback.classic.Level
 
 fun main(args: Array<String>) {
-	CommandLineOptions.parseOptions(args)
-
-	CommandLineRunner.setupLogging()
-	CommandLineRunner.checkVersion()
-
-
-
-	if (true || CommandLineOptions.cmd_console) {
-		// Always use the console for now.
-		CommandLineController()
-	} else {
-		GUIController()
-	}
+	val clr = CommandLineRunner(args)
+	
+	clr.setupLogging()
+	clr.checkVersion()
+	clr.run()
 }
 
-object CommandLineRunner {
+class CommandLineRunner(args: Array<String>) {
+	val logger = LoggerFactory.getLogger(CommandLineRunner::class.java) as Logger
+	val options = CommandLineOptions(args)
+	
+	fun run() {
+		// Always use the console for now.
+		try {
+			CommandLineController(options)
+		} catch (e: Throwable) {
+			println("An error occured!")
+			e.printStackTrace()
+			logger.error("Exception caught!", e)
+			System.exit(1)
+		}
+	}
+	
 	fun setupLogging() {
-		val logger = LoggerFactory.getLogger(CommandLineRunner::class.java) as Logger
+		if (options.isSet("anonymize")) {
+			Utils.anonymize = true
+		}
+	
+	
 		logger.trace("Setting up Loggers...")
 		val rootLogger = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) as Logger
 		val rootContext = rootLogger.getLoggerContext()
@@ -64,30 +76,39 @@ object CommandLineRunner {
 		rootLogger.addAppender(appender)
 		rootLogger.setLevel(Level.OFF)
 
-		if (CommandLineOptions.cmd_trace) {
+		if (options.isSet("trace")) {
 			(LoggerFactory.getLogger("de.fabianonline.telegram_backup") as Logger).setLevel(Level.TRACE)
-		} else if (CommandLineOptions.cmd_debug) {
+		} else if (options.isSet("debug")) {
 			(LoggerFactory.getLogger("de.fabianonline.telegram_backup") as Logger).setLevel(Level.DEBUG)
 		}
 
-		if (CommandLineOptions.cmd_trace_telegram) {
+		if (options.isSet("trace_telegram")) {
 			(LoggerFactory.getLogger("com.github.badoualy") as Logger).setLevel(Level.TRACE)
 		}
 	}
 
-	fun checkVersion(): Boolean {
+	fun checkVersion() {
+		if (Config.APP_APPVER.contains("-")) {
+			println("Your version ${Config.APP_APPVER} seems to be a development version. Version check is disabled.")
+			return
+		}
 		val v = Utils.getNewestVersion()
 		if (v != null && v.isNewer) {
-			System.out.println("A newer version is vailable!")
-			System.out.println("You are using: " + Config.APP_APPVER)
-			System.out.println("Available:     " + v.version)
-			System.out.println("Get it here:   " + v.url)
-			System.out.println()
-			System.out.println("Changes in this version:")
-			System.out.println(v.body)
-			System.out.println()
-			return false
+			println()
+			println()
+			println()
+			println("A newer version is vailable!")
+			println("You are using: " + Config.APP_APPVER)
+			println("Available:     " + v.version)
+			println("Get it here:   " + v.url)
+			println()
+			println()
+			println("Changes in this version:")
+			println(v.body)
+			println()
+			println()
+			println()
+			TimeUnit.SECONDS.sleep(5)
 		}
-		return true
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/Config.kt

@@ -29,7 +29,7 @@ object Config {
 	val APP_APPVER: String
 	val APP_LANG = "en"
 
-	var FILE_BASE = System.getProperty("user.home") + File.separatorChar + ".telegram_backup"
+	var TARGET_DIR = System.getProperty("user.home") + File.separatorChar + ".telegram_backup"
 	val FILE_NAME_AUTH_KEY = "auth.dat"
 	val FILE_NAME_DC = "dc.dat"
 	val FILE_NAME_DB = "database.sqlite"

src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt

@@ -15,6 +15,8 @@ import org.slf4j.LoggerFactory
 import org.slf4j.Logger
 import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory
 import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager
+import com.github.salomonbrys.kotson.*
+import com.google.gson.*
 
 class DatabaseUpdates(protected var conn: Connection, protected var db: Database) {
 
@@ -32,34 +34,50 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
 		register(DB_Update_7(conn, db))
 		register(DB_Update_8(conn, db))
 		register(DB_Update_9(conn, db))
+		register(DB_Update_10(conn, db))
+		register(DB_Update_11(conn, db))
 	}
 
 	fun doUpdates() {
 		try {
 			val stmt = conn.createStatement()
-			var rs: ResultSet
 			logger.debug("DatabaseUpdate.doUpdates running")
 
 			logger.debug("Getting current database version")
-			val version: Int
+			var version: Int
 			logger.debug("Checking if table database_versions exists")
-			rs = stmt.executeQuery("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions'")
-			rs.next()
-			if (rs.getInt(1) == 0) {
+			val table_count = db.queryInt("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions'")
+			if (table_count == 0) {
 				logger.debug("Table does not exist")
 				version = 0
 			} else {
 				logger.debug("Table exists. Checking max version")
-				rs.close()
-				rs = stmt.executeQuery("SELECT MAX(version) FROM database_versions")
-				rs.next()
-				version = rs.getInt(1)
+				version = db.queryInt("SELECT MAX(version) FROM database_versions")
 			}
-			rs.close()
 			logger.debug("version: {}", version)
 			System.out.println("Database version: " + version)
 			logger.debug("Max available database version is {}", maxPossibleVersion)
 
+			if (version == 0) {
+				logger.debug("Looking for DatabaseUpdate with create_query...")
+				// This is a fresh database - so we search for the latest available version with a create_query
+				// and use this as a shortcut.
+				var update: DatabaseUpdate? = null
+				for (i in maxPossibleVersion downTo 1) {
+					update = getUpdateToVersion(i)
+					logger.trace("Looking at DatabaseUpdate version {}", update.version)
+					if (update.create_query != null) break
+					update = null
+				}
+
+				if (update != null) {
+					logger.debug("Found DatabaseUpdate version {} with create_query.", update.version)
+					for (query in update.create_query!!) stmt.execute(query)
+					stmt.execute("INSERT INTO database_versions (version) VALUES (${update.version})")
+					version = update.version
+				}
+			}
+
 			if (version < maxPossibleVersion) {
 				logger.debug("Update is necessary. {} => {}.", version, maxPossibleVersion)
 				var backup = false
@@ -86,6 +104,9 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
 				} catch (e: SQLException) {
 					throw RuntimeException(e)
 				}
+				
+				println("Cleaning up the database (this might take some time)...")
+				try { stmt.executeUpdate("VACUUM") } catch (t: Throwable) { logger.debug("Exception during VACUUMing: {}", t) }
 
 			} else {
 				logger.debug("No update necessary.")
@@ -151,6 +172,8 @@ internal abstract class DatabaseUpdate(protected var conn: Connection, protected
 	companion object {
 		protected val logger = LoggerFactory.getLogger(DatabaseUpdate::class.java)
 	}
+
+	open val create_query: List<String>? = null
 }
 
 internal class DB_Update_1(conn: Connection, db: Database) : DatabaseUpdate(conn, db) {
@@ -293,7 +316,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn
 			} else {
 				ps.setInt(1, msg.getFwdFrom().getFromId())
 			}
-			val f = FileManagerFactory.getFileManager(msg, db.user_manager, db.client)
+			val f = FileManagerFactory.getFileManager(msg, db.file_base, settings = null)
 			if (f == null) {
 				ps.setNull(2, Types.VARCHAR)
 				ps.setNull(3, Types.VARCHAR)
@@ -308,6 +331,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn
 		rs.close()
 		conn.setAutoCommit(false)
 		ps.executeBatch()
+		ps.close()
 		conn.commit()
 		conn.setAutoCommit(true)
 		stmt.executeUpdate("DROP TABLE messages")
@@ -376,19 +400,29 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
 	override val version: Int
 		get() = 9
 	override val needsBackup = true
+
+	override val create_query = listOf(
+		"CREATE TABLE \"chats\" (id INTEGER PRIMARY KEY ASC, name TEXT, type TEXT);",
+		"CREATE TABLE \"users\" (id INTEGER PRIMARY KEY ASC, first_name TEXT, last_name TEXT, username TEXT, type TEXT, phone TEXT);",
+		"CREATE TABLE database_versions (version INTEGER);",
+		"CREATE TABLE runs (id INTEGER PRIMARY KEY ASC, time INTEGER, start_id INTEGER, end_id INTEGER, count_missing INTEGER);",
+		"CREATE TABLE \"messages\" (id INTEGER PRIMARY KEY AUTOINCREMENT,message_id INTEGER,message_type TEXT,source_type TEXT,source_id INTEGER,sender_id INTEGER,fwd_from_id INTEGER,text TEXT,time INTEGER,has_media BOOLEAN,media_type TEXT,media_file TEXT,media_size INTEGER,media_json TEXT,markup_json TEXT,data BLOB,api_layer INTEGER);",
+		"CREATE UNIQUE INDEX unique_messages ON messages (source_type, source_id, message_id);"
+	)
 	
 	@Throws(SQLException::class)
 	override fun _doUpdate() {
 		val logger = LoggerFactory.getLogger(DB_Update_9::class.java)
 		println("    Updating supergroup channel message data (this might take some time)...")
+		print("    ")
 		val count = db.queryInt("SELECT COUNT(*) FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53")
 		logger.debug("Found $count candidates for conversion")
 		val limit = 5000
 		var offset = 0
 		var i = 0
-		while (offset + 1 < count) {
+		while (offset < count) {
 			logger.debug("Querying with limit $limit and offset $offset")
-			val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 LIMIT ${limit} OFFSET ${offset}")
+			val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 ORDER BY id LIMIT ${limit} OFFSET ${offset}")
 			val messages = TLVector<TLAbsMessage>()
 			val messages_to_delete = mutableListOf<Int>()
 			while (rs.next()) {
@@ -399,13 +433,67 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
 					messages_to_delete.add(rs.getInt(1))
 				}
 			}
-			db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP)
+			rs.close()
+			db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP, settings=null)
 			execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")")
+			print(".")
 			
 			offset += limit
 		}
+		println()
 		logger.info("Converted ${i} of ${count} messages.")
-		println("    Cleaning up the database (this might also take some time, sorry)...")
-		execute("VACUUM")
+	}
+}
+
+internal class DB_Update_10(conn: Connection, db: Database) : DatabaseUpdate(conn, db) {
+	override val version: Int
+		get() = 10
+	
+	@Throws(SQLException::class)
+	override fun _doUpdate() {
+		execute("CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)")
+	}
+}
+
+internal class DB_Update_11(conn: Connection, db: Database) : DatabaseUpdate(conn, db) {
+	override val version = 11
+	val logger = LoggerFactory.getLogger(DB_Update_11::class.java)
+	
+	override fun _doUpdate() {
+		execute("ALTER TABLE messages ADD COLUMN json TEXT NULL")
+		execute("ALTER TABLE chats ADD COLUMN json TEXT NULL")
+		execute("ALTER TABLE chats ADD COLUMN api_layer INTEGER NULL")
+		execute("ALTER TABLE users ADD COLUMN json TEXT NULL")
+		execute("ALTER TABLE users ADD COLUMN api_layer INTEGER NULL")
+		val limit = 5000
+		var offset = 0
+		var i: Int
+		val ps = conn.prepareStatement("UPDATE messages SET json=? WHERE id=?")
+		println("    Updating messages to add their JSON representation to the database. This might take a few moments...")
+		print("    ")
+		do {
+			i = 0
+			logger.debug("Querying with limit $limit, offset is now $offset")
+			val rs = db.executeQuery("SELECT id, data FROM messages WHERE json IS NULL AND api_layer=53 LIMIT $limit")
+			while (rs.next()) {
+				i++
+				val id = rs.getInt(1)
+				val msg = Database.bytesToTLMessage(rs.getBytes(2))
+				val json = if (msg==null) Gson().toJson(null) else msg.toJson()
+				ps.setString(1, json)
+				ps.setInt(2, id)
+				ps.addBatch()
+			}
+			rs.close()
+			conn.setAutoCommit(false)
+			ps.executeBatch()
+			ps.clearBatch()
+			conn.commit()
+			conn.setAutoCommit(true)
+			offset += limit
+			print(".")
+		} while (i >= limit)
+		println()
+		ps.close()
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt

@@ -18,7 +18,6 @@ package de.fabianonline.telegram_backup
 
 import de.fabianonline.telegram_backup.UserManager
 import de.fabianonline.telegram_backup.Database
-import de.fabianonline.telegram_backup.StickerConverter
 import de.fabianonline.telegram_backup.DownloadProgressInterface
 import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory
 import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager
@@ -29,6 +28,7 @@ import com.github.badoualy.telegram.tl.core.TLIntVector
 import com.github.badoualy.telegram.tl.core.TLObject
 import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
 import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
+import com.github.badoualy.telegram.tl.api.messages.TLDialogsSlice
 import com.github.badoualy.telegram.tl.api.*
 import com.github.badoualy.telegram.tl.api.upload.TLFile
 import com.github.badoualy.telegram.tl.exception.RpcErrorException
@@ -36,6 +36,7 @@ import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
 import org.slf4j.LoggerFactory
 import org.slf4j.Logger
 import com.google.gson.Gson
+import com.github.salomonbrys.kotson.*
 import com.github.kittinunf.fuel.Fuel
 import com.github.kittinunf.result.Result
 import com.github.kittinunf.result.getAs
@@ -61,70 +62,31 @@ enum class MessageSource(val descr: String) {
 	SUPERGROUP("supergroup")
 }
 
-class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressInterface) {
-	internal var user: UserManager? = null
-	internal var db: Database? = null
-	internal var prog: DownloadProgressInterface? = null
-	internal var has_seen_flood_wait_message = false
-
-	init {
-		this.user = UserManager.getInstance()
-		this.prog = p
-		this.db = Database.getInstance()
-	}
-
+class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInterface, val db: Database, val user_manager: UserManager, val settings: Settings, val file_base: String) {
 	@Throws(RpcErrorException::class, IOException::class)
 	fun downloadMessages(limit: Int?) {
-		var completed: Boolean
-		do {
-			completed = true
-			try {
-				_downloadMessages(limit)
-			} catch (e: RpcErrorException) {
-				if (e.getCode() == 420) { // FLOOD_WAIT
-					completed = false
-					Utils.obeyFloodWaitException(e)
-				} else {
-					throw e
-				}
-			} catch (e: TimeoutException) {
-				completed = false
-				System.out.println("")
-				System.out.println("Telegram took too long to respond to our request.")
-				System.out.println("I'm going to wait a minute and then try again.")
-				try {
-					TimeUnit.MINUTES.sleep(1)
-				} catch (e2: InterruptedException) {
-				}
-
-				System.out.println("")
-			}
-
-		} while (!completed)
-	}
-
-	@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
-	fun _downloadMessages(limit: Int?) {
-		logger.info("This is _downloadMessages with limit {}", limit)
-		val dialog_limit = 100
-		logger.info("Downloading the last {} dialogs", dialog_limit)
+		logger.info("This is downloadMessages with limit {}", limit)
+		logger.info("Downloading the last dialogs")
 		System.out.println("Downloading most recent dialogs... ")
 		var max_message_id = 0
-		val dialogs = client!!.messagesGetDialogs(
-			0,
-			0,
-			TLInputPeerEmpty(),
-			dialog_limit)
-		logger.debug("Got {} dialogs", dialogs.getDialogs().size)
+		var result: ChatList? = null
+		
+		Utils.obeyFloodWait() {
+			result = getChats()
+		}
+		
+		val chats = result!!
+		
+		logger.debug("Got {} dialogs, {} supergoups, {} channels", chats.dialogs.size, chats.supergroups.size, chats.channels.size)
 
-		for (d in dialogs.getDialogs()) {
-			if (d.getTopMessage() > max_message_id && d.getPeer() !is TLPeerChannel) {
+		for (d in chats.dialogs) {
+			if (d.getTopMessage() > max_message_id) {
 				logger.trace("Updating top message id: {} => {}. Dialog type: {}", max_message_id, d.getTopMessage(), d.getPeer().javaClass)
 				max_message_id = d.getTopMessage()
 			}
 		}
 		System.out.println("Top message ID is " + max_message_id)
-		var max_database_id = db!!.getTopMessageID()
+		var max_database_id = db.getTopMessageID()
 		System.out.println("Top message ID in database is " + max_database_id)
 		if (limit != null) {
 			System.out.println("Limit is set to " + limit)
@@ -141,7 +103,7 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 		if (max_database_id == max_message_id) {
 			System.out.println("No new messages to download.")
 		} else if (max_database_id > max_message_id) {
-			throw RuntimeException("max_database_id is bigger then max_message_id. This shouldn't happen. But the telegram api nonetheless does that sometimes. Just ignore this error, wait a few seconds and then try again.")
+			throw RuntimeException("max_database_id is bigger than max_message_id. This shouldn't happen. But the telegram api nonetheless does that sometimes. Just ignore this error, wait a few seconds and then try again.")
 		} else {
 			val start_id = max_database_id + 1
 			val end_id = max_message_id
@@ -152,88 +114,56 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 
 		logger.info("Searching for missing messages in the db")
 		System.out.println("Checking message database for completeness...")
-		val db_count = db!!.getMessageCount()
-		val db_max = db!!.getTopMessageID()
+		val db_count = db.getMessageCount()
+		val db_max = db.getTopMessageID()
 		logger.debug("db_count: {}", db_count)
 		logger.debug("db_max: {}", db_max)
 
-		/*if (db_count != db_max) {
-			if (limit != null) {
-				System.out.println("You are missing messages in your database. But since you're using '--limit-messages', I won't download these now.");
-			} else {
-				LinkedList<Integer> all_missing_ids = db.getMissingIDs();
-				LinkedList<Integer> downloadable_missing_ids = new LinkedList<Integer>();
-				for (Integer id : all_missing_ids) {
-					if (id > max_message_id - 1000000) downloadable_missing_ids.add(id);
+			/*if (db_count != db_max) {
+				if (limit != null) {
+					System.out.println("You are missing messages in your database. But since you're using '--limit-messages', I won't download these now.");
+				} else {
+					LinkedList<Integer> all_missing_ids = db.getMissingIDs();
+					LinkedList<Integer> downloadable_missing_ids = new LinkedList<Integer>();
+					for (Integer id : all_missing_ids) {
+						if (id > max_message_id - 1000000) downloadable_missing_ids.add(id);
+					}
+					count_missing = all_missing_ids.size();
+					System.out.println("" + all_missing_ids.size() + " messages are missing in your Database.");
+					System.out.println("I can (and will) download " + downloadable_missing_ids.size() + " of them.");
+
+					downloadMessages(downloadable_missing_ids, null);
 				}
-				count_missing = all_missing_ids.size();
-				System.out.println("" + all_missing_ids.size() + " messages are missing in your Database.");
-				System.out.println("I can (and will) download " + downloadable_missing_ids.size() + " of them.");
 
-				downloadMessages(downloadable_missing_ids, null);
+				logger.info("Logging this run");
+				db.logRun(Math.min(max_database_id + 1, max_message_id), max_message_id, count_missing);
 			}
+			*/
 
-			logger.info("Logging this run");
-			db.logRun(Math.min(max_database_id + 1, max_message_id), max_message_id, count_missing);
+		if (settings.download_channels) {
+			println("Checking channels...")
+			for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel, limit) }
 		}
-		*/
+				
+		if (settings.download_supergroups) {
+			println("Checking supergroups...")
+			for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup, limit) }
+		}
+	}
 
-		if (CommandLineOptions.cmd_channels || CommandLineOptions.cmd_supergroups) {
-			System.out.println("Processing channels and/or supergroups...")
-			System.out.println("Please note that only channels/supergroups in the last 100 active chats are processed.")
-
-			val channel_access_hashes = HashMap<Int, Long>()
-			val channel_names = HashMap<Int, String>()
-			val channels = LinkedList<Int>()
-			val supergroups = LinkedList<Int>()
-
-			// TODO Add chat title (and other stuff?) to the database
-			for (c in dialogs.getChats()) {
-				if (c is TLChannel) {
-					channel_access_hashes.put(c.getId(), c.getAccessHash())
-					channel_names.put(c.getId(), c.getTitle())
-					if (c.getMegagroup()) {
-						supergroups.add(c.getId())
-					} else {
-						channels.add(c.getId())
-					}
-					// Channel: TLChannel
-					// Supergroup: getMegagroup()==true
-				}
+	private fun downloadMessagesFromChannel(channel: Channel, limit: Int?) {
+		val obj = channel.obj
+		var max_known_id = db.getTopMessageIDForChannel(channel.id)
+		if (obj.getTopMessage() > max_known_id) {
+			if (limit != null) {
+				max_known_id = Math.max(max_known_id, obj.getTopMessage() - limit)
 			}
+			val ids = makeIdList(max_known_id + 1, obj.getTopMessage())
+			var channel_name = channel.title
 
-
-
-			for (d in dialogs.getDialogs()) {
-				if (d.getPeer() is TLPeerChannel) {
-					val channel_id = (d.getPeer() as TLPeerChannel).getChannelId()
-
-					// If this is a channel and we don't want to download channels OR
-					// it is a supergroups and we don't want to download supergroups, then
-					if (channels.contains(channel_id) && !CommandLineOptions.cmd_channels || supergroups.contains(channel_id) && !CommandLineOptions.cmd_supergroups) {
-						// Skip this chat.
-						continue
-					}
-					val max_known_id = db!!.getTopMessageIDForChannel(channel_id)
-					if (d.getTopMessage() > max_known_id) {
-						val ids = makeIdList(max_known_id + 1, d.getTopMessage())
-						val access_hash = channel_access_hashes.get(channel_id) ?: throw RuntimeException("AccessHash for Channel missing.")
-						var channel_name = channel_names.get(channel_id)
-						if (channel_name == null) {
-							channel_name = "?"
-						}
-						val channel = TLInputChannel(channel_id, access_hash)
-						val source_type = if (supergroups.contains(channel_id)) {
-							MessageSource.SUPERGROUP
-						} else if (channels.contains(channel_id)) {
-							MessageSource.CHANNEL
-						} else {
-							throw RuntimeException("chat is neither in channels nor in supergroups...")
-						}
-						downloadMessages(ids, channel, source_type=source_type, source_name=channel_name)
-					}
-				}
-			}
+			val input_channel = TLInputChannel(channel.id, channel.access_hash)
+			val source_type = channel.message_source
+			downloadMessages(ids, input_channel, source_type=source_type, source_name=channel_name)
 		}
 	}
 
@@ -246,7 +176,7 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 		} else {
 			"${source_type.descr} $source_name"
 		}
-		prog!!.onMessageDownloadStart(ids.size, source_string)
+		prog.onMessageDownloadStart(ids.size, source_string)
 
 		logger.debug("Entering download loop")
 		while (ids.size > 0) {
@@ -261,123 +191,98 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 			logger.trace("vector.size(): {}", vector.size)
 			logger.trace("ids.size(): {}", ids.size)
 
-			var response: TLAbsMessages
-			var tries = 0
-			while (true) {
-				logger.trace("Trying getMessages(), tries={}", tries)
-				if (tries >= 5) {
-					CommandLineController.show_error("Couldn't getMessages after 5 tries. Quitting.")
-				}
-				tries++
-				try {
+			var resp: TLAbsMessages? = null
+			try {
+				Utils.obeyFloodWait(max_tries=5) {
 					if (channel == null) {
-						response = client!!.messagesGetMessages(vector)
+						resp = client.messagesGetMessages(vector)
 					} else {
-						response = client!!.channelsGetMessages(channel, vector)
-					}
-					break
-				} catch (e: RpcErrorException) {
-					if (e.getCode() == 420) { // FLOOD_WAIT
-						Utils.obeyFloodWaitException(e, has_seen_flood_wait_message)
-						has_seen_flood_wait_message = true
-					} else {
-						throw e
+						resp = client.channelsGetMessages(channel, vector)
 					}
 				}
-
+			} catch (e: MaxTriesExceededException) {
+				CommandLineController.show_error("Couldn't getMessages after 5 tries. Quitting.")
 			}
+			val response = resp!!
 			logger.trace("response.getMessages().size(): {}", response.getMessages().size)
 			if (response.getMessages().size != vector.size) {
 				CommandLineController.show_error("Requested ${vector.size} messages, but got ${response.getMessages().size}. That is unexpected. Quitting.")
 			}
 
-			prog!!.onMessageDownloaded(response.getMessages().size)
-			db!!.saveMessages(response.getMessages(), Kotlogram.API_LAYER, source_type=source_type)
-			db!!.saveChats(response.getChats())
-			db!!.saveUsers(response.getUsers())
+			prog.onMessageDownloaded(response.getMessages().size)
+			db.saveMessages(response.getMessages(), Kotlogram.API_LAYER, source_type=source_type, settings=settings)
+			db.saveChats(response.getChats())
+			db.saveUsers(response.getUsers())
 			logger.trace("Sleeping")
-			try {
-				TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_MESSAGES)
-			} catch (e: InterruptedException) {
-			}
-
+			try { TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_MESSAGES) } catch (e: InterruptedException) { }
 		}
 		logger.debug("Finished.")
 
-		prog!!.onMessageDownloadFinished()
+		prog.onMessageDownloadFinished()
 	}
 
 	@Throws(RpcErrorException::class, IOException::class)
 	fun downloadMedia() {
-		download_client = client!!.getDownloaderClient()
-		var completed: Boolean
-		do {
-			completed = true
-			try {
-				_downloadMedia()
-			} catch (e: RpcErrorException) {
-				if (e.getCode() == 420) { // FLOOD_WAIT
-					completed = false
-					Utils.obeyFloodWaitException(e)
-				} else {
-					throw e
-				}
-			}
-			/*catch (TimeoutException e) {
-				completed = false;
-				System.out.println("");
-				System.out.println("Telegram took too long to respond to our request.");
-				System.out.println("I'm going to wait a minute and then try again.");
-				logger.warn("TimeoutException caught", e);
-				try { TimeUnit.MINUTES.sleep(1); } catch(InterruptedException e2) {}
-				System.out.println("");
-			}*/
-		} while (!completed)
-	}
-
-	@Throws(RpcErrorException::class, IOException::class)
-	private fun _downloadMedia() {
+		download_client = client.getDownloaderClient()
 		logger.info("This is _downloadMedia")
 		logger.info("Checking if there are messages in the DB with a too old API layer")
-		val ids = db!!.getIdsFromQuery("SELECT id FROM messages WHERE has_media=1 AND api_layer<" + Kotlogram.API_LAYER)
+		val ids = db.getIdsFromQuery("SELECT id FROM messages WHERE has_media=1 AND api_layer<" + Kotlogram.API_LAYER)
 		if (ids.size > 0) {
 			System.out.println("You have ${ids.size} messages in your db that need an update. Doing that now.")
 			logger.debug("Found {} messages", ids.size)
 			downloadMessages(ids, null, source_type=MessageSource.NORMAL)
 		}
 
-		val messages = this.db!!.getMessagesWithMedia()
-		logger.debug("Database returned {} messages with media", messages.size)
-		prog!!.onMediaDownloadStart(messages.size)
-		for (msg in messages) {
-			if (msg == null) continue
-			val m = FileManagerFactory.getFileManager(msg, user!!, client!!)
-			logger.trace("message {}, {}, {}, {}, {}",
-				msg.getId(),
-				msg.getMedia().javaClass.getSimpleName().replace("TLMessageMedia", "…"),
-				m!!.javaClass.getSimpleName(),
-				if (m.isEmpty) "empty" else "non-empty",
-				if (m.downloaded) "downloaded" else "not downloaded")
-			if (m.isEmpty) {
-				prog!!.onMediaDownloadedEmpty()
-			} else if (m.downloaded) {
-				prog!!.onMediaAlreadyPresent(m)
-			} else {
+		val message_count = db.getMessagesWithMediaCount()
+		prog.onMediaDownloadStart(message_count)
+		var offset = 0
+		val limit = 1000
+		while (true) {
+			logger.debug("Querying messages with media, limit={}, offset={}", limit, offset)
+			val messages = db.getMessagesWithMedia(limit, offset)
+			if (messages.size == 0) break
+			offset += limit
+			logger.debug("Database returned {} messages with media", messages.size)
+			for (pair in messages) {
+				val id = pair.first
+				val json = pair.second
 				try {
-					val result = m.download()
-					if (result) {
-						prog!!.onMediaDownloaded(m)
-					} else {
-						prog!!.onMediaSkipped()
+				val m = FileManagerFactory.getFileManager(json, file_base, settings=settings)!!
+				logger.trace("message {}, {}, {}, {}, {}",
+					id,
+					m.javaClass.getSimpleName(),
+					if (m.isEmpty) "empty" else "non-empty",
+					if (m.downloaded) "downloaded" else "not downloaded")
+				if (m.isEmpty) {
+					prog.onMediaDownloadedEmpty()
+				} else if (m.downloaded) {
+					prog.onMediaAlreadyPresent(m)
+				} else if (settings.max_file_age>0 && (System.currentTimeMillis() / 1000) - json["date"].int > settings.max_file_age * 24 * 60 * 60) {
+					prog.onMediaSkipped()
+				} else if (settings.max_file_size>0 && settings.max_file_size*1024*1024 > m.size) {
+					prog.onMediaSkipped()
+				} else if (settings.blacklist_extensions.contains(m.extension)) {
+					prog.onMediaSkipped()
+				} else {
+					try {
+						val result = m.download(prog)
+						if (result) {
+							prog.onMediaDownloaded(m)
+						} else {
+							prog.onMediaFailed()
+						}
+					} catch (e: TimeoutException) {
+						// do nothing - skip this file
+						prog.onMediaFailed()
 					}
-				} catch (e: TimeoutException) {
-					// do nothing - skip this file
-					prog!!.onMediaSkipped()
 				}
-
+				} catch (e: IllegalStateException) {
+					println(json.toPrettyJson())
+					throw e
+				}
 			}
 		}
-		prog!!.onMediaDownloadFinished()
+		prog.onMediaDownloadFinished()
 	}
 
 	private fun makeIdList(start: Int, end: Int): MutableList<Int> {
@@ -385,6 +290,72 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 		for (i in start..end) a.add(i)
 		return a
 	}
+	
+	fun getChats(): ChatList {
+		val cl = ChatList()
+		logger.debug("Getting list of chats...")
+		val limit = 100
+		var offset = 0
+		while (true) {
+			var temp: TLAbsDialogs? = null
+			logger.trace("Calling messagesGetDialogs with offset {}", offset)
+			Utils.obeyFloodWait {
+				temp = client.messagesGetDialogs(offset, 0, TLInputPeerEmpty(), limit)
+			}
+			val dialogs = temp!!
+			val last_message = dialogs.messages.filter{ it is TLMessage || it is TLMessageService }.last()
+			offset = when(last_message) {
+				is TLMessage -> last_message.date
+				is TLMessageService -> last_message.date
+				else -> throw RuntimeException("Unexpected last_message type ${last_message.javaClass}")
+			}
+			logger.trace("Got {} dialogs back", dialogs.dialogs.size)
+			logger.trace("New offset will be {}", offset)
+			
+			// Add dialogs
+			cl.dialogs.addAll(dialogs.getDialogs().filter{it.getPeer() !is TLPeerChannel})
+			
+			// Add supergoups and channels
+			for (tl_channel in dialogs.getChats().filter{it is TLChannel}.map{it as TLChannel}) {
+				val tl_peer_channel = dialogs.getDialogs().find{var p = it.getPeer() ; p is TLPeerChannel && p.getChannelId()==tl_channel.getId()}
+				
+				if (tl_peer_channel == null) continue
+				
+				var download = true
+				if (settings.whitelist_channels.isNotEmpty()) {
+					download = settings.whitelist_channels.contains(tl_channel.getId().toString())
+				} else if (settings.blacklist_channels.isNotEmpty()) {
+					download = !settings.blacklist_channels.contains(tl_channel.getId().toString())
+				}
+				val channel = Channel(id=tl_channel.getId(), access_hash=tl_channel.getAccessHash(), title=tl_channel.getTitle(), obj=tl_peer_channel, download=download)
+				if (tl_channel.getMegagroup()) {
+					channel.message_source = MessageSource.SUPERGROUP
+					cl.supergroups.add(channel)
+				} else {
+					channel.message_source = MessageSource.CHANNEL
+					cl.channels.add(channel)
+				}
+			}
+			
+			if (dialogs.dialogs.size < limit) {
+				logger.debug("Got only ${dialogs.dialogs.size} back instead of ${limit}. Stopping the loop.")
+				logger.debug("Got ${cl.dialogs.size} groups, ${cl.channels.size} channels and ${cl.supergroups.size} supergroups.")
+				break;
+			}
+		}
+		
+		return cl
+	}
+	
+	class ChatList {
+		val dialogs = mutableListOf<TLDialog>()
+		val supergroups = mutableListOf<Channel>()
+		val channels = mutableListOf<Channel>()
+	}
+	
+	class Channel(val id: Int, val access_hash: Long, val title: String, val obj: TLDialog, val download: Boolean) {
+		lateinit var message_source: MessageSource
+	}
 
 	companion object {
 		internal var download_client: TelegramClient? = null
@@ -392,19 +363,19 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 		internal val logger = LoggerFactory.getLogger(DownloadManager::class.java)
 
 		@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
-		fun downloadFile(targetFilename: String, size: Int, dcId: Int, volumeId: Long, localId: Int, secret: Long) {
+		fun downloadFile(targetFilename: String, size: Int, dcId: Int, volumeId: Long, localId: Int, secret: Long, prog: DownloadProgressInterface?) {
 			val loc = TLInputFileLocation(volumeId, localId, secret)
-			downloadFileFromDc(targetFilename, loc, dcId, size)
+			downloadFileFromDc(targetFilename, loc, dcId, size, prog)
 		}
 
 		@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
-		fun downloadFile(targetFilename: String, size: Int, dcId: Int, id: Long, accessHash: Long) {
+		fun downloadFile(targetFilename: String, size: Int, dcId: Int, id: Long, accessHash: Long, prog: DownloadProgressInterface?) {
 			val loc = TLInputDocumentFileLocation(id, accessHash)
-			downloadFileFromDc(targetFilename, loc, dcId, size)
+			downloadFileFromDc(targetFilename, loc, dcId, size, prog)
 		}
 
 		@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
-		private fun downloadFileFromDc(target: String, loc: TLAbsInputFileLocation, dcID: Int, size: Int): Boolean {
+		private fun downloadFileFromDc(target: String, loc: TLAbsInputFileLocation, dcID: Int, size: Int, prog: DownloadProgressInterface?): Boolean {
 			var fos: FileOutputStream? = null
 			try {
 				val temp_filename = target + ".downloading"
@@ -423,38 +394,34 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 				}
 				logger.trace("offset before the loop is {}", offset)
 				fos = FileOutputStream(temp_filename, true)
-				var response: TLFile? = null
-				var try_again: Boolean
+				if (prog != null) prog.onMediaFileDownloadStarted()
 				do {
-					try_again = false
 					logger.trace("offset: {} block_size: {} size: {}", offset, size, size)
 					val req = TLRequestUploadGetFile(loc, offset, size)
+					var resp: TLFile? = null
 					try {
-						response = download_client!!.executeRpcQuery(req, dcID) as TLFile
-					} catch (e: RpcErrorException) {
-						if (e.getCode() == 420) { // FLOOD_WAIT
-							try_again = true
-							Utils.obeyFloodWaitException(e)
-							continue // response is null since we didn't actually receive any data. Skip the rest of this iteration and try again.
-						} else if (e.getCode() == 400) {
-							//Somehow this file is broken. No idea why. Let's skip it for now
-							return false
-						} else {
-							throw e
+						Utils.obeyFloodWait() {
+							resp = download_client!!.executeRpcQuery(req, dcID) as TLFile
 						}
+					} catch (e: RpcErrorException) {
+						if (e.getCode() == 400) {
+							// Somehow this file is broken. No idea why. Let's skip it for now.
+							return false
+						}
+						throw e
 					}
-
+					
+					val response = resp!!
+					if (prog!=null) prog.onMediaFileDownloadStep()
 					offset += response.getBytes().getData().size
 					logger.trace("response: {} total size: {}", response.getBytes().getData().size, offset)
 
 					fos.write(response.getBytes().getData())
 					fos.flush()
-					try {
-						TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_FILE)
-					} catch (e: InterruptedException) {
-					}
+					try { TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_FILE) } catch (e: InterruptedException) { }
 
-				} while (offset < size && (try_again || response!!.getBytes().getData().size > 0))
+				} while (offset < size && response.getBytes().getData().size > 0)
+				if (prog != null) prog.onMediaFileDownloadFinished()
 				fos.close()
 				if (offset < size) {
 					System.out.println("Requested file $target with $size bytes, but got only $offset bytes.")
@@ -509,13 +476,18 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 		}
 
 		@Throws(IOException::class)
-		fun downloadExternalFile(target: String, url: String): Boolean {
-			val (_, response, result) = Fuel.get(url).response()
-			if (result is Result.Success) {
-				File(target).writeBytes(response.data)
-				return true
+		fun downloadExternalFile(target: String, url: String, prog: DownloadProgressInterface?): Boolean {
+			if (prog != null) prog.onMediaFileDownloadStarted()
+			var success = true
+			Fuel.download(url).destination { _, _ ->
+				File(target)
+			}.progress { _, _ ->
+				if (prog != null) prog.onMediaFileDownloadStep()
+			}.response { _, _, result ->
+				success = (result is Result.Success)
 			}
-			return false
+			if (prog != null) prog.onMediaFileDownloadFinished()
+			return success
 		}
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/DownloadProgressInterface.kt

@@ -29,4 +29,8 @@ interface DownloadProgressInterface {
 	fun onMediaSkipped()
 	fun onMediaAlreadyPresent(file_manager: AbstractMediaFileManager)
 	fun onMediaDownloadFinished()
+	fun onMediaFailed()
+	fun onMediaFileDownloadStarted()
+	fun onMediaFileDownloadStep()
+	fun onMediaFileDownloadFinished()
 }

src/main/kotlin/de/fabianonline/telegram_backup/GUIController.kt

@@ -1,85 +0,0 @@
-/* Telegram_Backup
- * Copyright (C) 2016 Fabian Schlenz
- * 
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- * 
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>. */
-
-package de.fabianonline.telegram_backup
-
-import javax.swing.*
-import javax.swing.event.ListSelectionEvent
-import javax.swing.event.ListSelectionListener
-import java.awt.*
-import java.awt.event.ActionEvent
-import java.awt.event.ActionListener
-import java.util.Vector
-
-class GUIController() {
-	init {
-		showAccountChooserDialog()
-	}
-
-	private fun showAccountChooserDialog() {
-		val accountChooser = JDialog()
-		accountChooser.setTitle("Choose account")
-		accountChooser.setSize(400, 200)
-		val vert = JPanel()
-		vert.setLayout(BorderLayout())
-		vert.add(JLabel("Please select the account to use or create a new one."), BorderLayout.NORTH)
-		val accounts = Utils.getAccounts()
-		val list = JList<String>(accounts)
-		list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION)
-		vert.add(list, BorderLayout.CENTER)
-		val bottom = JPanel(GridLayout(1, 2))
-		val btnAddAccount = JButton("Add account")
-		bottom.add(btnAddAccount)
-		val btnLogin = JButton("Login")
-		btnLogin.setEnabled(false)
-		bottom.add(btnLogin)
-		vert.add(bottom, BorderLayout.SOUTH)
-		accountChooser.add(vert)
-		accountChooser.setVisible(true)
-		accountChooser.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE)
-	}
-
-	private fun addAccountDialog() {
-		val loginDialog = JDialog()
-		loginDialog.setTitle("Add an account")
-		loginDialog.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE)
-
-		val sections = JPanel()
-		sections.setLayout(BoxLayout(sections, BoxLayout.Y_AXIS))
-
-		val top = JPanel()
-		top.setLayout(BoxLayout(top, BoxLayout.Y_AXIS))
-		top.add(JLabel("Please enter your phone number in international format:"))
-		top.add(JTextField("+49123773212"))
-
-		sections.add(top)
-		sections.add(Box.createVerticalStrut(5))
-		sections.add(JSeparator(SwingConstants.HORIZONTAL))
-
-		val middle = JPanel()
-		middle.setLayout(BoxLayout(middle, BoxLayout.Y_AXIS))
-		middle.add(JLabel("Telegram sent you a code. Enter it here:"))
-		middle.add(JTextField())
-		middle.setEnabled(false)
-
-		sections.add(middle)
-		sections.add(Box.createVerticalStrut(5))
-		sections.add(JSeparator(SwingConstants.HORIZONTAL))
-
-		loginDialog.add(sections)
-		loginDialog.setVisible(true)
-	}
-}

src/main/kotlin/de/fabianonline/telegram_backup/LoginManager.kt

@@ -0,0 +1,96 @@
+package de.fabianonline.telegram_backup
+
+import com.github.badoualy.telegram.api.Kotlogram
+import com.github.badoualy.telegram.api.TelegramApp
+import com.github.badoualy.telegram.api.TelegramClient
+import com.github.badoualy.telegram.tl.api.TLUser
+import com.github.badoualy.telegram.tl.api.account.TLPassword
+import com.github.badoualy.telegram.tl.api.auth.TLSentCode
+import com.github.badoualy.telegram.tl.core.TLBytes
+import com.github.badoualy.telegram.tl.exception.RpcErrorException
+import java.security.MessageDigest
+import java.util.*
+
+class LoginManager(val app: TelegramApp, val target_dir: String, val phoneToUse: String?) {
+	fun run() {
+		var phone: String
+		
+		if (phoneToUse == null) {
+			println("Please enter your phone number in international format.")
+			println("Example: +4917077651234")
+			phone = getLine()
+		} else {
+			phone = phoneToUse
+		}
+		
+		val file_base = CommandLineController.build_file_base(target_dir, phone)
+		
+		// We now have an account, so we can create an ApiStorage and TelegramClient.
+		val storage = ApiStorage(file_base)
+		val client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, null)
+		
+		val sent_code = send_code_to_phone_number(client, phone)
+		println("Telegram sent you a code. Please enter it here.")
+		val code = getLine()
+		
+		try {
+			verify_code(client, phone, sent_code, code)
+		} catch(e: PasswordNeededException) {
+			println("We also need your account password. Please enter it now. It should not be printed, so it's okay if you see nothing while typing it.")
+			val pw = getPassword()
+			verify_password(client, pw)
+		}
+		System.out.println("Everything seems fine. Please run this tool again with '--account ${phone} to use this account.")
+	}
+	
+	private fun send_code_to_phone_number(client: TelegramClient, phone: String): TLSentCode {
+		return client.authSendCode(false, phone, true)
+	}
+	
+	private fun verify_code(client: TelegramClient, phone: String, sent_code: TLSentCode, code: String): TLUser {
+		try {
+			val auth = client.authSignIn(phone, sent_code.getPhoneCodeHash(), code)
+			return auth.getUser().getAsUser()
+		} catch (e: RpcErrorException) {
+			if (e.getCode() == 401 && e.getTag()=="SESSION_PASSWORD_NEEDED") {
+				throw PasswordNeededException()
+			} else {
+				throw e
+			}
+		}
+	}
+	
+	private fun verify_password(client: TelegramClient, password: String): TLUser {
+		val pw = password.toByteArray(charset = Charsets.UTF_8)
+		val salt = (client.accountGetPassword() as TLPassword).getCurrentSalt().getData()
+		val md = MessageDigest.getInstance("SHA-256")
+		
+		val salted = ByteArray(2 * salt.size + pw.size)
+		System.arraycopy(salt, 0, salted, 0, salt.size)
+		System.arraycopy(pw, 0, salted, salt.size, pw.size)
+		System.arraycopy(salt, 0, salted, salt.size + pw.size, salt.size)
+		val hash = md.digest(salted)
+		val auth = client.authCheckPassword(TLBytes(hash))
+		return auth.getUser().getAsUser()
+	}
+			
+	
+	private fun getLine(): String {
+		if (System.console() != null) {
+			return System.console().readLine("> ")
+		} else {
+			print("> ")
+			return Scanner(System.`in`).nextLine()
+		}
+	}
+
+	private fun getPassword(): String {
+		if (System.console() != null) {
+			return String(System.console().readPassword("> "))
+		} else {
+			return getLine()
+		}
+	}
+}
+
+class PasswordNeededException: Exception("A password is needed to be able to login to this account.") {}

src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt

@@ -0,0 +1,137 @@
+package de.fabianonline.telegram_backup
+
+import java.io.File
+import java.util.LinkedList
+import org.slf4j.LoggerFactory
+
+class Settings(val file_base: String, val database: Database, val cli_settings: CommandLineOptions) {
+	val logger = LoggerFactory.getLogger(Settings::class.java)
+
+	private val db_settings: Map<String, String>
+
+	val ini_settings: Map<String, List<String>>
+
+	init {
+		db_settings = database.fetchSettings()
+		ini_settings = load_ini("config.ini")
+		copy_sample_ini("config.sample.ini")
+	}
+		// Merging CLI and INI settings
+		
+	val sf = SettingsFactory(ini_settings, cli_settings)
+	val gmaps_key = sf.getString("gmaps_key", default=Config.SECRET_GMAPS, secret=true)
+	val pagination = sf.getBoolean("pagination", default=true)
+	val pagination_size = sf.getInt("pagination_size", default=Config.DEFAULT_PAGINATION)
+	val download_media = sf.getBoolean("download_media", default=true)
+	val download_channels = sf.getBoolean("download_channels", default=false)
+	val download_supergroups = sf.getBoolean("download_supergroups", default=false)
+	val whitelist_channels = sf.getStringList("whitelist_channels", default=LinkedList<String>())
+	val blacklist_channels = sf.getStringList("blacklist_channels", default=LinkedList<String>())
+	val max_file_age = sf.getInt("max_file_age", default=-1)
+	val max_file_size = sf.getInt("max_file_size", default=-1)
+	val blacklist_extensions = sf.getStringList("blacklist_extensions", default=LinkedList<String>())
+
+	private fun get_setting_list(name: String): List<String>? {
+		return ini_settings[name]
+	}
+
+	private fun load_ini(filename: String): Map<String, List<String>> {
+		val map = mutableMapOf<String, MutableList<String>>()
+		val file = File(file_base + filename)
+		logger.trace("Checking ini file {}", filename.anonymize())
+		if (!file.exists()) return map
+		logger.debug("Loading ini file {}", filename.anonymize())
+		file.forEachLine { parseLine(it, map) }
+		return map
+	}
+
+	private fun parseLine(original_line: String, map: MutableMap<String, MutableList<String>>) {
+		logger.trace("Parsing line: {}", original_line)
+		var line = original_line.trim().replaceAfter("#", "").removeSuffix("#")
+		logger.trace("After cleaning: {}", line)
+		if (line == "") return
+		val parts: List<String> = line.split("=", limit=2).map{it.trim()}
+
+		if (parts.size < 2) throw RuntimeException("Invalid config setting: $line")
+
+		val (key, value) = parts
+		if (value=="") {
+			map.remove(key)
+		} else {
+			var list = map.get(key)
+			if (list == null) {
+				list = mutableListOf<String>()
+				map.put(key, list)
+			}
+			list.add(value)
+		}
+	}
+
+	private fun copy_sample_ini(filename: String) {
+		val stream = Config::class.java.getResourceAsStream("/config.sample.ini")
+		File(file_base + filename).outputStream().use { stream.copyTo(it) }
+		stream.close()
+	}
+	
+	fun print() {
+		println()
+		Setting.all_settings.forEach { it.print() }
+		println()
+	}
+}
+
+class SettingsFactory(val ini: Map<String, List<String>>, val cli: CommandLineOptions) {
+	fun getInt(name: String, default: Int, secret: Boolean = false) = getSetting(name, listOf(default.toString()), secret).get().toInt()
+	fun getBoolean(name: String, default: Boolean, secret: Boolean = false) = getSetting(name, listOf(default.toString()), secret).get().toBoolean()
+	fun getString(name: String, default: String, secret: Boolean = false) = getSetting(name, listOf(default), secret).get()
+	fun getStringList(name: String, default: List<String>, secret: Boolean = false) = getSetting(name, default, secret).getList()
+	
+	fun getSetting(name: String, default: List<String>, secret: Boolean) = Setting(ini, cli, name, default, secret)
+}
+
+class Setting(val ini: Map<String, List<String>>, val cli: CommandLineOptions, val name: String, val default: List<String>, val secret: Boolean) {
+	val values: List<String>
+	val source: SettingSource
+	val logger = LoggerFactory.getLogger(Setting::class.java)
+	
+	init {
+		if (getCli(name) != null) {
+			values = listOf(getCli(name)!!)
+			source = SettingSource.CLI
+		} else if (getIni(name) != null) {
+			values = getIni(name)!!
+			source = SettingSource.INI
+		} else {
+			values = default
+			source = SettingSource.DEFAULT
+		}
+		
+		logger.debug("Setting ${name} loaded. Source: ${source}. Value: ${values.toString().anonymize()}")
+		
+		all_settings.add(this)
+	}
+	fun get(): String = values.last()
+	fun getList(): List<String> = values
+	
+	fun getIni(name: String): List<String>? {
+		return ini[name]
+	}
+	
+	fun getCli(name: String): String? {
+		return cli.get(name)
+	}
+	
+	fun print() {
+		println("%-25s %-10s %s".format(name, source, (if (secret && source==SettingSource.DEFAULT) "[REDACTED]" else values)))
+	}
+	
+	companion object {
+		val all_settings = LinkedList<Setting>()
+	}
+}
+
+enum class SettingSource {
+	INI,
+	CLI,
+	DEFAULT
+}

src/main/kotlin/de/fabianonline/telegram_backup/StickerConverter.kt

@@ -1,52 +0,0 @@
-/* Telegram_Backup
- * Copyright (C) 2016 Fabian Schlenz
- * 
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- * 
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>. */
-
-package de.fabianonline.telegram_backup
-
-import com.github.badoualy.telegram.tl.api.*
-import java.lang.StringBuilder
-import java.io.File
-
-object StickerConverter {
-	fun makeFilenameWithPath(attr: TLDocumentAttributeSticker): String {
-		val file = StringBuilder()
-		file.append(makePath())
-		file.append(makeFilename(attr))
-		return file.toString()
-	}
-
-	fun makeFilename(attr: TLDocumentAttributeSticker): String {
-		val file = StringBuilder()
-		if (attr.getStickerset() is TLInputStickerSetShortName) {
-			file.append((attr.getStickerset() as TLInputStickerSetShortName).getShortName())
-		} else if (attr.getStickerset() is TLInputStickerSetID) {
-			file.append((attr.getStickerset() as TLInputStickerSetID).getId())
-		}
-		file.append("_")
-		file.append(attr.getAlt().hashCode())
-		file.append(".webp")
-		return file.toString()
-	}
-
-	fun makePath(): String {
-		val path = Config.FILE_BASE +
-			File.separatorChar +
-			Config.FILE_STICKER_BASE +
-			File.separatorChar
-		File(path).mkdirs()
-		return path
-	}
-}

src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt

@@ -26,48 +26,39 @@ import de.fabianonline.telegram_backup.Database
 import de.fabianonline.telegram_backup.UserManager
 import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager
 import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory
+import org.slf4j.LoggerFactory
 
-internal class TelegramUpdateHandler : UpdateCallback {
-	private var user: UserManager? = null
-	private var db: Database? = null
-	var debug = false
-
-	fun activate() {
-		this.user = UserManager.getInstance()
-		this.db = Database.getInstance()
-	}
+internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Database, val file_base: String, val settings: Settings) : UpdateCallback {
+	val logger = LoggerFactory.getLogger(TelegramUpdateHandler::class.java)
 
 	override fun onUpdates(client: TelegramClient, updates: TLUpdates) {
-		if (db == null) return
-		if (debug) System.out.println("onUpdates - " + updates.getUpdates().size + " Updates, " + updates.getUsers().size + " Users, " + updates.getChats().size + " Chats")
+
+		logger.debug("onUpdates - " + updates.getUpdates().size + " Updates, " + updates.getUsers().size + " Users, " + updates.getChats().size + " Chats")
 		for (update in updates.getUpdates()) {
-			processUpdate(update, client)
-			if (debug) System.out.println("  " + update.javaClass.getName())
+			processUpdate(update)
+			logger.debug("  " + update.javaClass.getName())
 		}
-		db!!.saveUsers(updates.getUsers())
-		db!!.saveChats(updates.getChats())
+		db.saveUsers(updates.getUsers())
+		db.saveChats(updates.getChats())
 	}
 
 	override fun onUpdatesCombined(client: TelegramClient, updates: TLUpdatesCombined) {
-		if (db == null) return
-		if (debug) System.out.println("onUpdatesCombined")
+		logger.debug("onUpdatesCombined")
 		for (update in updates.getUpdates()) {
-			processUpdate(update, client)
+			processUpdate(update)
 		}
-		db!!.saveUsers(updates.getUsers())
-		db!!.saveChats(updates.getChats())
+		db.saveUsers(updates.getUsers())
+		db.saveChats(updates.getChats())
 	}
 
 	override fun onUpdateShort(client: TelegramClient, update: TLUpdateShort) {
-		if (db == null) return
-		if (debug) System.out.println("onUpdateShort")
-		processUpdate(update.getUpdate(), client)
-		if (debug) System.out.println("  " + update.getUpdate().javaClass.getName())
+		logger.debug("onUpdateShort")
+		processUpdate(update.getUpdate())
+		logger.debug("  " + update.getUpdate().javaClass.getName())
 	}
 
 	override fun onShortChatMessage(client: TelegramClient, message: TLUpdateShortChatMessage) {
-		if (db == null) return
-		if (debug) System.out.println("onShortChatMessage - " + message.getMessage())
+		logger.debug("onShortChatMessage - " + message.getMessage())
 		val msg = TLMessage(
 			message.getOut(),
 			message.getMentioned(),
@@ -85,21 +76,20 @@ internal class TelegramUpdateHandler : UpdateCallback {
 			message.getEntities(), null, null)
 		val vector = TLVector<TLAbsMessage>(TLAbsMessage::class.java)
 		vector.add(msg)
-		db!!.saveMessages(vector, Kotlogram.API_LAYER)
+		db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings)
 		System.out.print('.')
 	}
 
 	override fun onShortMessage(client: TelegramClient, message: TLUpdateShortMessage) {
 		val m = message
-		if (db == null) return
-		if (debug) System.out.println("onShortMessage - " + m.getOut() + " - " + m.getUserId() + " - " + m.getMessage())
+		logger.debug("onShortMessage - " + m.getOut() + " - " + m.getUserId() + " - " + m.getMessage())
 		val from_id: Int
 		val to_id: Int
 		if (m.getOut() == true) {
-			from_id = user!!.user!!.getId()
+			from_id = user_manager.id
 			to_id = m.getUserId()
 		} else {
-			to_id = user!!.user!!.getId()
+			to_id = user_manager.id
 			from_id = m.getUserId()
 		}
 		val msg = TLMessage(
@@ -119,29 +109,27 @@ internal class TelegramUpdateHandler : UpdateCallback {
 			m.getEntities(), null, null)
 		val vector = TLVector<TLAbsMessage>(TLAbsMessage::class.java)
 		vector.add(msg)
-		db!!.saveMessages(vector, Kotlogram.API_LAYER)
+		db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings)
 		System.out.print('.')
 	}
 
 	override fun onShortSentMessage(client: TelegramClient, message: TLUpdateShortSentMessage) {
-		if (db == null) return
-		System.out.println("onShortSentMessage")
+		logger.debug("onShortSentMessage")
 	}
 
 	override fun onUpdateTooLong(client: TelegramClient) {
-		if (db == null) return
-		System.out.println("onUpdateTooLong")
+		logger.debug("onUpdateTooLong")
 	}
 
-	private fun processUpdate(update: TLAbsUpdate, client: TelegramClient) {
+	private fun processUpdate(update: TLAbsUpdate) {
 		if (update is TLUpdateNewMessage) {
 			val abs_msg = update.getMessage()
 			val vector = TLVector<TLAbsMessage>(TLAbsMessage::class.java)
 			vector.add(abs_msg)
-			db!!.saveMessages(vector, Kotlogram.API_LAYER)
+			db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings)
 			System.out.print('.')
-			if (abs_msg is TLMessage) {
-				val fm = FileManagerFactory.getFileManager(abs_msg, user!!, client)
+			if (abs_msg is TLMessage && settings.download_media==true) {
+				val fm = FileManagerFactory.getFileManager(abs_msg, file_base, settings)
 				if (fm != null && !fm.isEmpty && !fm.downloaded) {
 					try {
 						fm.download()

src/main/kotlin/de/fabianonline/telegram_backup/TestFeatures.kt

@@ -10,7 +10,7 @@ import java.sql.ResultSet
 import java.io.IOException
 import java.nio.charset.Charset
 
-internal object TestFeatures {
+internal class TestFeatures(val db: Database) {
 	fun test1() {
 		// Tests entries in a cache4.db in the current working directory for compatibility
 		try {
@@ -24,31 +24,22 @@ internal object TestFeatures {
 		var conn: Connection
 		var stmt: Statement? = null
 
-		try {
-			conn = DriverManager.getConnection(path)
-			stmt = conn.createStatement()
-		} catch (e: SQLException) {
-			CommandLineController.show_error("Could not connect to SQLITE database.")
-		}
+		conn = DriverManager.getConnection(path)
+		stmt = conn.createStatement()
 
 		var unsupported_constructor = 0
 		var success = 0
 
-		try {
-			val rs = stmt!!.executeQuery("SELECT data FROM messages")
-			while (rs.next()) {
-				try {
-					TLApiContext.getInstance().deserializeMessage(rs.getBytes(1))
-				} catch (e: com.github.badoualy.telegram.tl.exception.UnsupportedConstructorException) {
-					unsupported_constructor++
-				} catch (e: IOException) {
-					System.out.println("IOException: " + e)
-				}
-
-				success++
+		val rs = stmt.executeQuery("SELECT data FROM messages")
+		while (rs.next()) {
+			try {
+				TLApiContext.getInstance().deserializeMessage(rs.getBytes(1))
+			} catch (e: com.github.badoualy.telegram.tl.exception.UnsupportedConstructorException) {
+				unsupported_constructor++
+			} catch (e: IOException) {
+				System.out.println("IOException: " + e)
 			}
-		} catch (e: SQLException) {
-			System.out.println("SQL exception: " + e)
+			success++
 		}
 
 		System.out.println("Success:                 " + success)
@@ -59,7 +50,6 @@ internal object TestFeatures {
 		// Prints system.encoding and default charset
 		System.out.println("Default Charset:   " + Charset.defaultCharset())
 		System.out.println("file.encoding:     " + System.getProperty("file.encoding"))
-		val db = Database.getInstance()
 		System.out.println("Database encoding: " + db.getEncoding())
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/UserManager.kt

@@ -34,107 +34,34 @@ import java.io.File
 import org.slf4j.LoggerFactory
 import org.slf4j.Logger
 
-class UserManager @Throws(IOException::class)
-private constructor(c: TelegramClient) {
-	var user: TLUser? = null
-	var phone: String? = null
-	private var code: String? = null
-	private val client: TelegramClient
-	private var sent_code: TLSentCode? = null
-	private var auth: TLAuthorization? = null
-	var isPasswordNeeded = false
-		private set
-
-	val loggedIn: Boolean
-		get() = user != null
-
-	val userString: String
-		get() {
-			if (this.user == null) return "Not logged in"
-			val sb = StringBuilder()
-			if (this.user!!.getFirstName() != null) {
-				sb.append(this.user!!.getFirstName())
-			}
-			if (this.user!!.getLastName() != null) {
-				sb.append(" ")
-				sb.append(this.user!!.getLastName())
-			}
-			if (this.user!!.getUsername() != null) {
-				sb.append(" (@")
-				sb.append(this.user!!.getUsername())
-				sb.append(")")
-			}
-			return sb.toString()
-		}
-
-	val fileBase: String
-		get() = Config.FILE_BASE + File.separatorChar + "+" + this.user!!.getPhone() + File.separatorChar
-
+class UserManager(val client: TelegramClient) {
+	val tl_user: TLUser
+	val logger = LoggerFactory.getLogger(UserManager::class.java)
+	val phone: String
+		get() = "+" + tl_user.getPhone()
+	val id: Int
+		get() = tl_user.getId()
+	
 	init {
-		this.client = c
 		logger.debug("Calling getFullUser")
-		try {
-			val full_user = this.client.usersGetFullUser(TLInputUserSelf())
-			this.user = full_user.getUser().getAsUser()
-		} catch (e: RpcErrorException) {
-			// This may happen. Ignoring it.
-			logger.debug("Ignoring exception:", e)
-		}
-
+		val full_user = client.usersGetFullUser(TLInputUserSelf())
+		tl_user = full_user.getUser().getAsUser()
 	}
 
-	@Throws(RpcErrorException::class, IOException::class)
-	fun sendCodeToPhoneNumber(number: String) {
-		this.phone = number
-		this.sent_code = this.client.authSendCode(false, number, true)
-	}
-
-	@Throws(RpcErrorException::class, IOException::class)
-	fun verifyCode(code: String) {
-		this.code = code
-		try {
-			this.auth = client.authSignIn(phone, this.sent_code!!.getPhoneCodeHash(), this.code)
-			this.user = auth!!.getUser().getAsUser()
-		} catch (e: RpcErrorException) {
-			if (e.getCode() != 401 || !e.getTag().equals("SESSION_PASSWORD_NEEDED")) throw e
-			this.isPasswordNeeded = true
+	override fun toString(): String {
+		val sb = StringBuilder()
+		sb.append(tl_user.getFirstName() ?: "")
+		if (tl_user.getLastName() != null) {
+			sb.append(" ")
+			sb.append(tl_user.getLastName())
 		}
-
-	}
-
-	@Throws(RpcErrorException::class, IOException::class)
-	fun verifyPassword(pw: String) {
-		val password = pw.toByteArray(charset = Charsets.UTF_8)
-		val salt = (client.accountGetPassword() as TLPassword).getCurrentSalt().getData()
-		var md: MessageDigest
-		try {
-			md = MessageDigest.getInstance("SHA-256")
-		} catch (e: NoSuchAlgorithmException) {
-			e.printStackTrace()
-			return
-		}
-
-		val salted = ByteArray(2 * salt.size + password.size)
-		System.arraycopy(salt, 0, salted, 0, salt.size)
-		System.arraycopy(password, 0, salted, salt.size, password.size)
-		System.arraycopy(salt, 0, salted, salt.size + password.size, salt.size)
-		val hash = md.digest(salted)
-		auth = client.authCheckPassword(TLBytes(hash))
-		this.user = auth!!.getUser().getAsUser()
-	}
-
-	companion object {
-		private val logger = LoggerFactory.getLogger(UserManager::class.java)
-		internal var instance: UserManager? = null
-
-		@Throws(IOException::class)
-		fun init(c: TelegramClient) {
-			instance = UserManager(c)
-		}
-
-		fun getInstance(): UserManager {
-			if (instance == null) throw RuntimeException("UserManager is not yet initialized.")
-			return instance!!
+		if (tl_user.getUsername() != null) {
+			sb.append(" (@")
+			sb.append(tl_user.getUsername())
+			sb.append(")")
 		}
+		return sb.toString()
 	}
+	
+	
 }

src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt

@@ -17,10 +17,16 @@
 package de.fabianonline.telegram_backup
 
 import com.github.badoualy.telegram.tl.exception.RpcErrorException
+import com.github.badoualy.telegram.tl.api.TLAbsMessage
+import com.github.badoualy.telegram.tl.api.TLAbsUser
+import com.github.badoualy.telegram.tl.api.TLAbsChat
+import com.github.badoualy.telegram.api.Kotlogram
 import java.io.File
 import java.util.Vector
 import java.util.concurrent.TimeUnit
+import java.util.concurrent.TimeoutException
 import com.google.gson.*
+import com.github.salomonbrys.kotson.*
 import java.net.URL
 import org.apache.commons.io.IOUtils
 import de.fabianonline.telegram_backup.Version
@@ -31,12 +37,30 @@ object Utils {
 	@JvmField public val VERSIONS_EQUAL = 0
 	@JvmField public val VERSION_1_NEWER = 1
 	@JvmField public val VERSION_2_NEWER = 2
+	
+	var hasSeenFloodWaitMessage = false
+
+	var anonymize = false
 
 	private val logger = LoggerFactory.getLogger(Utils::class.java) as Logger
 
-	fun getAccounts(): Vector<String> {
+	fun print_accounts(file_base: String) {
+		println("List of available accounts:")
+		val accounts = getAccounts(file_base)
+		if (accounts.size > 0) {
+			for (str in accounts) {
+				println(" " + str.anonymize())
+			}
+			println("Use '--account <x>' to use one of those accounts.")
+		} else {
+			println("NO ACCOUNTS FOUND")
+			println("Use '--login' to login to a telegram account.")
+		}
+	}
+
+	fun getAccounts(file_base: String): Vector<String> {
 		val accounts = Vector<String>()
-		val folder = File(Config.FILE_BASE)
+		val folder = File(file_base)
 		val files = folder.listFiles()
 		if (files != null)
 			for (f in files) {
@@ -80,30 +104,56 @@ object Utils {
 		}
 
 	}
-
-	@Throws(RpcErrorException::class)
-	@JvmOverloads internal fun obeyFloodWaitException(e: RpcErrorException?, silent: Boolean = false) {
-		if (e == null || e.getCode() != 420) return
-
-		val delay: Long = e.getTagInteger()!!.toLong()
-		if (!silent) {
-			System.out.println("")
-			System.out.println(
-				"Telegram complained about us (okay, me) making too many requests in too short time by\n" +
-					"sending us \"" + e.getTag() + "\" as an error. So we now have to wait a bit. Telegram\n" +
-					"asked us to wait for " + delay + " seconds.\n" +
+	
+	fun obeyFloodWait(max_tries: Int = -1, method: () -> Unit) {
+		var tries = 0
+		while (true) {
+			tries++
+			if (max_tries>0 && tries>max_tries) throw MaxTriesExceededException()
+			logger.trace("This is try ${tries}.")
+			try {
+				method.invoke()
+				// If we reach this, the method has returned successfully -> we are done
+				return
+			} catch(e: RpcErrorException) {
+				// If we got something else than a FLOOD_WAIT error, we just rethrow it
+				if (e.getCode() != 420) throw e
+				
+				val delay = e.getTagInteger()!!.toLong()
+				
+				if (!hasSeenFloodWaitMessage) {
+					println(
+						"\n" +
+						"Telegram complained about us (okay, me) making too many requests in too short time by\n" +
+						"sending us \"${e.getTag()}\" as an error. So we now have to wait a bit. Telegram\n" +
+						"asked us to wait for ${delay} seconds.\n" +
+						"\n" +
+						"So I'm going to do just that for now. If you don't want to wait, you can quit by pressing\n" +
+						"Ctrl+C. You can restart me at any time and I will just continue to download your\n" +
+						"messages and media. But be advised that just restarting me is not going to change\n" +
+						"the fact that Telegram won't talk to me until then." +
+						"\n")
+				} else {
+					print(" Waiting...")
+				}
+				
+				try { TimeUnit.SECONDS.sleep(delay + 1) } catch (e: InterruptedException) { }
+				
+				if (hasSeenFloodWaitMessage) {
+					//    "  W a i t i n g . . ."
+					print("\b\b\b\b\b\b\b\b\b\b\b")
+				}
+				
+				hasSeenFloodWaitMessage = true
+			} catch (e: TimeoutException) {
+				println(
 					"\n" +
-					"So I'm going to do just that for now. If you don't want to wait, you can quit by pressing\n" +
-					"Ctrl+C. You can restart me at any time and I will just continue to download your\n" +
-					"messages and media. But be advised that just restarting me is not going to change\n" +
-					"the fact that Telegram won't talk to me until then.")
-			System.out.println("")
+					"Telegram took too long to respond to our request.\n" +
+					"I'm going to wait a minute and then try again." +
+					"\n")
+				try { TimeUnit.MINUTES.sleep(1) } catch (e: InterruptedException) { }
+			}
 		}
-		try {
-			TimeUnit.SECONDS.sleep(delay + 1)
-		} catch (e2: InterruptedException) {
-		}
-
 	}
 
 	@JvmStatic
@@ -179,8 +229,48 @@ object Utils {
 }
 
 fun String.anonymize(): String {
-	return if (!CommandLineOptions.cmd_anonymize) this else this.replace(Regex("[0-9]"), "1").replace(Regex("[A-Z]"), "A").replace(Regex("[a-z]"), "a") + " (ANONYMIZED)"
+	return if (!Utils.anonymize) this else this.replace(Regex("[0-9]"), "1").replace(Regex("[A-Z]"), "A").replace(Regex("[a-z]"), "a") + " (ANONYMIZED)"
 }
 
 fun Any.toJson(): String = Gson().toJson(this)
 fun Any.toPrettyJson(): String = GsonBuilder().setPrettyPrinting().create().toJson(this)
+
+fun JsonObject.isA(name: String): Boolean = this.contains("_constructor") && this["_constructor"].string.startsWith(name + "#")
+fun JsonElement.isA(name: String): Boolean = this.obj.isA(name)
+
+class MaxTriesExceededException(): RuntimeException("Max tries exceeded") {}
+
+fun TLAbsMessage.toJson(): String {
+	val json = Gson().toJsonTree(this).obj
+	cleanUpMessageJson(json)
+	json["api_layer"] = Kotlogram.API_LAYER
+	return json.toString()
+}
+
+fun TLAbsChat.toJson(): String {
+	val json = Gson().toJsonTree(this).obj
+	json["api_layer"] = Kotlogram.API_LAYER
+	return json.toString()
+}
+
+fun TLAbsUser.toJson(): String {
+	val json = Gson().toJsonTree(this).obj
+	json["api_layer"] = Kotlogram.API_LAYER
+	return json.toString()
+}
+
+fun cleanUpMessageJson(json : JsonElement) {
+	if (json.isJsonArray) {
+		json.array.forEach {cleanUpMessageJson(it)}
+		return
+	} else if (!json.isJsonObject) {
+		return
+	}
+	if (json.obj.has("bytes")) {
+		json.obj -= "bytes"
+		return
+	}
+	json.obj.forEach {_: String, elm: JsonElement ->
+		if (elm.isJsonObject || elm.isJsonArray) cleanUpMessageJson(elm)
+	}
+}

src/main/kotlin/de/fabianonline/telegram_backup/exporter/CSVExporter.kt

@@ -0,0 +1,111 @@
+/* Telegram_Backup
+ * Copyright (C) 2016 Fabian Schlenz
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. */
+
+package de.fabianonline.telegram_backup.exporter
+
+import java.io.File
+import java.io.PrintWriter
+import java.io.OutputStreamWriter
+import java.io.FileOutputStream
+import java.nio.charset.Charset
+import java.io.FileWriter
+import java.io.IOException
+import java.io.FileNotFoundException
+import java.net.URL
+import org.apache.commons.io.FileUtils
+import java.util.LinkedList
+import java.util.HashMap
+import java.time.LocalDate
+import java.time.LocalTime
+import java.time.LocalDateTime
+import java.time.ZoneOffset
+import java.time.format.DateTimeFormatter 
+import java.sql.Time
+import java.text.SimpleDateFormat
+
+import com.github.mustachejava.DefaultMustacheFactory
+import com.github.mustachejava.Mustache
+import com.github.mustachejava.MustacheFactory
+import de.fabianonline.telegram_backup.*
+import com.github.badoualy.telegram.tl.api.*
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+       
+
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+class CSVExporter(val db: Database, val file_base: String, val settings: Settings) {
+	val logger = LoggerFactory.getLogger(CSVExporter::class.java)
+	val mustache = DefaultMustacheFactory().compile("templates/csv/messages.csv")
+	val dialogs = db.getListOfDialogsForExport()
+	val chats = db.getListOfChatsForExport()
+	val datetime_format = SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
+	val base = file_base + "files" + File.separatorChar
+	
+	fun export() {
+		val today = LocalDateTime.of(LocalDate.now(), LocalTime.MIDNIGHT)
+		val timezone = ZoneOffset.systemDefault()
+		val days = if (settings.max_file_age==-1) 7 else settings.max_file_age
+
+		// Create base dir
+		logger.debug("Creating base dir")
+		File(base).mkdirs()
+
+		if (days > 0) {
+			for (dayOffset in days downTo 1) {
+				val day = today.minusDays(dayOffset.toLong())
+
+				val start = day.toEpochSecond(timezone.rules.getOffset(day))
+				val end = start + 24 * 60 * 60
+				val filename = base + "messages.${day.format(DateTimeFormatter.ISO_LOCAL_DATE)}.csv"
+				if (!File(file_base + filename).exists()) {
+					logger.debug("Range: {} to {}", start, end)
+					println("Processing messages for ${day}...")
+					exportToFile(start, end, filename)
+				}
+			}
+		} else {
+			println("Processing all messages...")
+			exportToFile(0, Long.MAX_VALUE, base + "messages.all.csv")
+		}
+	}
+
+	fun exportToFile(start: Long, end: Long, filename: String) {
+		val list = mutableListOf<Map<String, String?>>()
+		db.getMessagesForCSVExport(start, end) {data: HashMap<String, Any> ->
+			val scope = HashMap<String, String?>()
+			val timestamp = data["time"] as Time
+			scope.put("time", datetime_format.format(timestamp))
+			scope.put("username", if (data["user_username"]!=null) data["user_username"] as String else null)
+			if (data["source_type"]=="dialog") {
+				scope.put("chat_name", "@" + (dialogs.firstOrNull{it.id==data["source_id"]}?.username ?: ""))
+			} else {
+				scope.put("chat_name", chats.firstOrNull{it.id==data["source_id"]}?.name)
+			}
+			scope.put("message", data["message"] as String)
+			list.add(scope)
+		}
+		val writer = getWriter(filename)
+		mustache.execute(writer, mapOf("messages" to list))
+		writer.close()
+	}
+	
+	private fun getWriter(filename: String): OutputStreamWriter {
+		logger.trace("Creating writer for file {}", filename.anonymize())
+		return OutputStreamWriter(FileOutputStream(filename), Charset.forName("UTF-8").newEncoder())
+	}
+}

src/main/kotlin/de/fabianonline/telegram_backup/exporter/CSVLinkExporter.kt

@@ -0,0 +1,139 @@
+/* Telegram_Backup
+ * Copyright (C) 2016 Fabian Schlenz
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. */
+
+package de.fabianonline.telegram_backup.exporter
+
+import java.io.File
+import java.io.PrintWriter
+import java.io.OutputStreamWriter
+import java.io.FileOutputStream
+import java.nio.charset.Charset
+import java.io.FileWriter
+import java.io.IOException
+import java.io.FileNotFoundException
+import java.net.URL
+import org.apache.commons.io.FileUtils
+import java.util.LinkedList
+import java.util.HashMap
+import java.time.LocalDate
+import java.time.LocalTime
+import java.time.LocalDateTime
+import java.time.ZoneOffset
+import java.time.format.DateTimeFormatter 
+import java.sql.Time
+import java.text.SimpleDateFormat
+
+import com.github.mustachejava.DefaultMustacheFactory
+import com.github.mustachejava.Mustache
+import com.github.mustachejava.MustacheFactory
+import de.fabianonline.telegram_backup.*
+import com.github.badoualy.telegram.tl.api.*
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+       
+
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+class CSVLinkExporter(val db: Database, val file_base: String, val settings: Settings) {
+	val logger = LoggerFactory.getLogger(CSVLinkExporter::class.java)
+	val mustache = DefaultMustacheFactory().compile("templates/csv/links.csv")
+	val dialogs = db.getListOfDialogsForExport()
+	val chats = db.getListOfChatsForExport()
+	val datetime_format = SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
+	val base = file_base + "files" + File.separatorChar
+	
+	val invalid_entity_index = "[INVALID ENTITY INDEX]"
+
+	fun export() {
+		val today = LocalDateTime.of(LocalDate.now(), LocalTime.MIDNIGHT)
+		val timezone = ZoneOffset.systemDefault()
+		val days = if (settings.max_file_age==-1) 7 else settings.max_file_age
+
+		// Create base dir
+		logger.debug("Creating base dir")
+		File(base).mkdirs()
+
+		if (days > 0) {
+			for (dayOffset in days downTo 1) {
+				val day = today.minusDays(dayOffset.toLong())
+
+				val start = day.toEpochSecond(timezone.rules.getOffset(day))
+				val end = start + 24 * 60 * 60
+				val filename = base + "links.${day.format(DateTimeFormatter.ISO_LOCAL_DATE)}.csv"
+				if (!File(file_base + filename).exists()) {
+					logger.debug("Range: {} to {}", start, end)
+					println("Processing messages for ${day}...")
+					exportToFile(start, end, filename)
+				}
+			}
+		} else {
+			println("Processing all messages...")
+			exportToFile(0, Long.MAX_VALUE, base + "links.all.csv")
+		}
+	}
+
+	fun exportToFile(start: Long, end: Long, filename: String) {
+
+		//val messages: List<Map<String, Any>> = db.getMessagesForCSVExport(start, end)
+		val list = mutableListOf<Map<String, String?>>()
+		val parser = JsonParser()
+		//logger.debug("Got {} messages", messages.size)
+		db.getMessagesForCSVExport(start, end) {data: HashMap<String, Any> ->
+			//val msg: TLMessage = data.get("message_object") as TLMessage
+			val json = parser.parse(data.get("json") as String).obj
+			if (!json.contains("entities")) return@getMessagesForCSVExport
+			
+			val urls: List<String>? = json["entities"].array.filter{it.obj.isA("messageEntityTextUrl") || it.obj.isA("messageEntityUrl")}?.map {
+				var url: String
+				try {
+					url = if (it.obj.contains("url")) it["url"].string else json["message"].string.substring(it["offset"].int, it["offset"].int + it["length"].int)
+					if (!url.toLowerCase().startsWith("http:") && !url.toLowerCase().startsWith("https://")) url = "http://${url}"
+				} catch (e: StringIndexOutOfBoundsException) {
+					url = invalid_entity_index
+				}
+				url
+			}
+
+			if (urls != null) for(url in urls) {
+				val scope = HashMap<String, String?>()
+				scope.put("url", url)
+				if (url == invalid_entity_index) {
+					scope.put("host", invalid_entity_index)
+				} else {
+					scope.put("host", URL(url).getHost())
+				}
+				val timestamp = data["time"] as Time
+				scope.put("time", datetime_format.format(timestamp))
+				scope.put("username", if (data["user_username"]!=null) data["user_username"] as String else null)
+				if (data["source_type"]=="dialog") {
+					scope.put("chat_name", "@" + (dialogs.firstOrNull{it.id==data["source_id"]}?.username ?: ""))
+				} else {
+					scope.put("chat_name", chats.firstOrNull{it.id==data["source_id"]}?.name)
+				}
+				list.add(scope)
+			}
+		}
+		val writer = getWriter(filename)
+		mustache.execute(writer, mapOf("links" to list))
+		writer.close()
+	}
+	
+	private fun getWriter(filename: String): OutputStreamWriter {
+		logger.trace("Creating writer for file {}", filename.anonymize())
+		return OutputStreamWriter(FileOutputStream(filename), Charset.forName("UTF-8").newEncoder())
+	}
+}

src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt

@@ -16,12 +16,6 @@
 
 package de.fabianonline.telegram_backup.exporter
 
-import de.fabianonline.telegram_backup.UserManager
-import de.fabianonline.telegram_backup.Database
-import de.fabianonline.telegram_backup.anonymize
-import de.fabianonline.telegram_backup.toPrettyJson
-import de.fabianonline.telegram_backup.CommandLineOptions
-
 import java.io.File
 import java.io.PrintWriter
 import java.io.OutputStreamWriter
@@ -38,22 +32,20 @@ import java.util.HashMap
 import com.github.mustachejava.DefaultMustacheFactory
 import com.github.mustachejava.Mustache
 import com.github.mustachejava.MustacheFactory
+import de.fabianonline.telegram_backup.*
 
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
-class HTMLExporter {
-	val db = Database.getInstance()
-	val user = UserManager.getInstance()
-
+class HTMLExporter(val db: Database, val user: UserManager, val settings: Settings, val file_base: String) {
 	@Throws(IOException::class)
 	fun export() {
 		try {
-			val pagination = if (CommandLineOptions.cmd_no_pagination) -1 else CommandLineOptions.val_pagination
+			val pagination = if (settings.pagination) settings.pagination_size else -1
 			
 			// Create base dir
 			logger.debug("Creating base dir")
-			val base = user.fileBase + "files" + File.separatorChar
+			val base = file_base + "files" + File.separatorChar
 			File(base).mkdirs()
 			File(base + "dialogs").mkdirs()
 

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt

@@ -17,56 +17,43 @@
 package de.fabianonline.telegram_backup.mediafilemanager
 
 import de.fabianonline.telegram_backup.UserManager
-import de.fabianonline.telegram_backup.Database
-import de.fabianonline.telegram_backup.StickerConverter
-import de.fabianonline.telegram_backup.DownloadProgressInterface
 import de.fabianonline.telegram_backup.Config
-import de.fabianonline.telegram_backup.DownloadManager
 
-import com.github.badoualy.telegram.api.TelegramClient
-import com.github.badoualy.telegram.tl.core.TLIntVector
-import com.github.badoualy.telegram.tl.core.TLObject
-import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
-import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
 import com.github.badoualy.telegram.tl.api.*
-import com.github.badoualy.telegram.tl.api.upload.TLFile
 import com.github.badoualy.telegram.tl.exception.RpcErrorException
-import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
+import de.fabianonline.telegram_backup.Settings
 
 import java.io.IOException
 import java.io.File
-import java.io.FileOutputStream
-import java.util.ArrayList
-import java.util.LinkedList
-import java.net.URL
 import java.util.concurrent.TimeoutException
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.*
 
-import org.apache.commons.io.FileUtils
-
-abstract class AbstractMediaFileManager(protected var message: TLMessage, protected var user: UserManager, protected var client: TelegramClient) {
+abstract class AbstractMediaFileManager(private var json: JsonObject, val file_base: String) {
 	open var isEmpty = false
 	abstract val size: Int
 	abstract val extension: String
 
 	open val downloaded: Boolean
-		get() = File(targetPathAndFilename).isFile()
+		get() = !isEmpty && File(targetPathAndFilename).isFile()
 
 	val downloading: Boolean
 		get() = File("${targetPathAndFilename}.downloading").isFile()
 
 	open val targetPath: String
 		get() {
-			val path = user.fileBase + Config.FILE_FILES_BASE + File.separatorChar
+			val path = file_base + Config.FILE_FILES_BASE + File.separatorChar
 			File(path).mkdirs()
 			return path
 		}
 
 	open val targetFilename: String
 		get() {
-			val message_id = message.getId()
-			var to = message.getToId()
-			if (to is TLPeerChannel) {
-				val channel_id = to.getChannelId()
+			val message_id = json["id"].int
+			var to = json["toId"].obj
+			if (to.isA("peerChannel")) {
+				val channel_id = to["channelId"].int
 				return "channel_${channel_id}_${message_id}.$extension"
 			} else return "${message_id}.$extension"
 		}
@@ -78,7 +65,7 @@ abstract class AbstractMediaFileManager(protected var message: TLMessage, protec
 	abstract val name: String
 	abstract val description: String
 	@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
-	abstract fun download(): Boolean
+	abstract fun download(prog: DownloadProgressInterface? = null): Boolean
 
 	protected fun extensionFromMimetype(mime: String): String {
 		when (mime) {
@@ -93,8 +80,8 @@ abstract class AbstractMediaFileManager(protected var message: TLMessage, protec
 	}
 
 	companion object {
-		fun throwUnexpectedObjectError(o: Any) {
-			throw RuntimeException("Unexpected " + o.javaClass.getName())
+		fun throwUnexpectedObjectError(constructor: String) {
+			throw RuntimeException("Unexpected ${constructor}")
 		}
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt

@@ -17,68 +17,45 @@
 package de.fabianonline.telegram_backup.mediafilemanager
 
 import de.fabianonline.telegram_backup.UserManager
-import de.fabianonline.telegram_backup.Database
-import de.fabianonline.telegram_backup.StickerConverter
-import de.fabianonline.telegram_backup.DownloadProgressInterface
 import de.fabianonline.telegram_backup.DownloadManager
-
-import com.github.badoualy.telegram.api.TelegramClient
-import com.github.badoualy.telegram.tl.core.TLIntVector
-import com.github.badoualy.telegram.tl.core.TLObject
-import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
-import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
+import de.fabianonline.telegram_backup.DownloadProgressInterface
 import com.github.badoualy.telegram.tl.api.*
-import com.github.badoualy.telegram.tl.api.upload.TLFile
 import com.github.badoualy.telegram.tl.exception.RpcErrorException
-import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
 
 import java.io.IOException
-import java.io.File
-import java.io.FileOutputStream
-import java.util.ArrayList
-import java.util.LinkedList
-import java.net.URL
 import java.util.concurrent.TimeoutException
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.*
 
-import org.apache.commons.io.FileUtils
-
-open class DocumentFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : AbstractMediaFileManager(msg, user, client) {
-	protected var doc: TLDocument? = null
+open class DocumentFileManager(message: JsonObject, file_base: String) : AbstractMediaFileManager(message, file_base) {
+	//protected var doc: TLDocument? = null
 	override lateinit var extension: String
 
 	open val isSticker: Boolean
-		get() {
-			if (this.isEmpty || doc == null) return false
-			return doc!!.getAttributes()?.filter { it is TLDocumentAttributeSticker }?.isNotEmpty() ?: false
-		}
+		get() = json.get("attributes")?.array?.any{it.obj.isA("documentAttributeSticker")} ?: false
 
 	override val size: Int
-		get() = if (doc != null) doc!!.getSize() else 0
+		get() = json["size"].int
 
 	open override val letter: String = "d"
 	open override val name: String = "document"
 	open override val description: String = "Document"
+	
+	private val json = message["media"]["document"].obj
 
 	init {
-		val d = (msg.getMedia() as TLMessageMediaDocument).getDocument()
-		if (d is TLDocument) {
-			this.doc = d
-		} else if (d is TLDocumentEmpty) {
-			this.isEmpty = true
-		} else {
-			throwUnexpectedObjectError(d)
-		}
 		extension = processExtension()
 	}
 
 	private fun processExtension(): String {
-		if (doc == null) return "empty"
+		//if (doc == null) return "empty"
 		var ext: String? = null
 		var original_filename: String? = null
-		if (doc!!.getAttributes() != null)
-			for (attr in doc!!.getAttributes()) {
-				if (attr is TLDocumentAttributeFilename) {
-					original_filename = attr.getFileName()
+		if (json.contains("attributes"))
+			for (attr in json["attributes"].array) {
+				if (attr.obj["_constructor"].string.startsWith("documentAttributeFilename")) {
+					original_filename = attr.obj["fileName"].string
 				}
 			}
 		if (original_filename != null) {
@@ -87,7 +64,7 @@ open class DocumentFileManager(msg: TLMessage, user: UserManager, client: Telegr
 
 		}
 		if (ext == null) {
-			ext = extensionFromMimetype(doc!!.getMimeType())
+			ext = extensionFromMimetype(json["mimeType"].string)
 		}
 
 		// Sometimes, extensions contain a trailing double quote. Remove this. Fixes #12.
@@ -97,10 +74,8 @@ open class DocumentFileManager(msg: TLMessage, user: UserManager, client: Telegr
 	}
 
 	@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
-	override fun download(): Boolean {
-		if (doc != null) {
-			DownloadManager.downloadFile(targetPathAndFilename, size, doc!!.getDcId(), doc!!.getId(), doc!!.getAccessHash())
-		}
+	override fun download(prog: DownloadProgressInterface?): Boolean {
+		DownloadManager.downloadFile(targetPathAndFilename, size, json["dcId"].int, json["id"].long, json["accessHash"].long, prog)
 		return true
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt

@@ -18,7 +18,6 @@ package de.fabianonline.telegram_backup.mediafilemanager
 
 import de.fabianonline.telegram_backup.UserManager
 import de.fabianonline.telegram_backup.Database
-import de.fabianonline.telegram_backup.StickerConverter
 import de.fabianonline.telegram_backup.DownloadProgressInterface
 
 import com.github.badoualy.telegram.api.TelegramClient
@@ -30,44 +29,62 @@ import com.github.badoualy.telegram.tl.api.*
 import com.github.badoualy.telegram.tl.api.upload.TLFile
 import com.github.badoualy.telegram.tl.exception.RpcErrorException
 import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
+import de.fabianonline.telegram_backup.Settings
 
 import java.io.IOException
 import java.io.File
 import java.io.FileOutputStream
 import java.util.ArrayList
 import java.util.LinkedList
+import java.util.NoSuchElementException
 import java.net.URL
 import java.util.concurrent.TimeoutException
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.*
 
 import org.apache.commons.io.FileUtils
 
 object FileManagerFactory {
-	fun getFileManager(m: TLMessage?, u: UserManager, c: TelegramClient): AbstractMediaFileManager? {
+	fun getFileManager(m: TLMessage?, file_base: String, settings: Settings?): AbstractMediaFileManager? {
 		if (m == null) return null
-		val media = m.getMedia() ?: return null
-
-		if (media is TLMessageMediaPhoto) {
-			return PhotoFileManager(m, u, c)
-		} else if (media is TLMessageMediaDocument) {
-			val d = DocumentFileManager(m, u, c)
-			return if (d.isSticker) {
-				StickerFileManager(m, u, c)
-			} else d
-		} else if (media is TLMessageMediaGeo) {
-			return GeoFileManager(m, u, c)
-		} else if (media is TLMessageMediaEmpty) {
-			return UnsupportedFileManager(m, u, c, "empty")
-		} else if (media is TLMessageMediaUnsupported) {
-			return UnsupportedFileManager(m, u, c, "unsupported")
-		} else if (media is TLMessageMediaWebPage) {
-			return UnsupportedFileManager(m, u, c, "webpage")
-		} else if (media is TLMessageMediaContact) {
-			return UnsupportedFileManager(m, u, c, "contact")
-		} else if (media is TLMessageMediaVenue) {
-			return UnsupportedFileManager(m, u, c, "venue")
-		} else {
-			AbstractMediaFileManager.throwUnexpectedObjectError(media)
+		val json = Gson().toJsonTree(m).obj
+		return getFileManager(json, file_base, settings)
+	}
+	            
+	fun getFileManager(message: JsonObject?, file_base: String, settings: Settings?): AbstractMediaFileManager? {
+		if (message == null) return null
+		try {
+			val media = message.get("media")?.obj ?: return null
+			
+			if (media.isA("messageMediaPhoto")) {
+				return PhotoFileManager(message, file_base)
+			} else if (media.isA("messageMediaDocument")) {
+				val d = DocumentFileManager(message, file_base)
+				return if (d.isSticker) StickerFileManager(message, file_base) else d
+			} else if (media.isA("messageMediaGeo")) {
+				return GeoFileManager(message, file_base, settings)
+			} else if (media.isA("messageMediaEmpty")) {
+				return UnsupportedFileManager(message, file_base, "empty")
+			} else if (media.isA("messageMediaUnsupported")) {
+				return UnsupportedFileManager(message, file_base, "unsupported")
+			} else if (media.isA("messageMediaWebPage")) {
+				return UnsupportedFileManager(message, file_base, "webpage")
+			} else if (media.isA("messageMediaContact")) {
+				return UnsupportedFileManager(message, file_base, "contact")
+			} else if (media.isA("messageMediaVenue")) {
+				return UnsupportedFileManager(message, file_base, "venue")
+			} else {
+				AbstractMediaFileManager.throwUnexpectedObjectError(media["_constructor"].string)
+			}
+		} catch (e: IllegalStateException) {
+			println(message.toPrettyJson())
+			throw e
+		} catch (e: NoSuchElementException) {
+			println(message.toPrettyJson())
+			throw e
 		}
+		
 		return null
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt

@@ -17,34 +17,20 @@
 package de.fabianonline.telegram_backup.mediafilemanager
 
 import de.fabianonline.telegram_backup.UserManager
-import de.fabianonline.telegram_backup.Database
-import de.fabianonline.telegram_backup.StickerConverter
-import de.fabianonline.telegram_backup.DownloadProgressInterface
 import de.fabianonline.telegram_backup.DownloadManager
-import de.fabianonline.telegram_backup.Config
-
-import com.github.badoualy.telegram.api.TelegramClient
-import com.github.badoualy.telegram.tl.core.TLIntVector
-import com.github.badoualy.telegram.tl.core.TLObject
-import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
-import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
+import de.fabianonline.telegram_backup.DownloadProgressInterface
 import com.github.badoualy.telegram.tl.api.*
-import com.github.badoualy.telegram.tl.api.upload.TLFile
-import com.github.badoualy.telegram.tl.exception.RpcErrorException
-import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
+import de.fabianonline.telegram_backup.Config
+import de.fabianonline.telegram_backup.Settings
 
 import java.io.IOException
 import java.io.File
-import java.io.FileOutputStream
-import java.util.ArrayList
-import java.util.LinkedList
-import java.net.URL
-import java.util.concurrent.TimeoutException
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.Utils
 
-import org.apache.commons.io.FileUtils
-
-class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : AbstractMediaFileManager(msg, user, client) {
-	protected lateinit var geo: TLGeoPoint
+class GeoFileManager(message: JsonObject, file_base: String, val settings: Settings?) : AbstractMediaFileManager(message, file_base) {
+	//protected lateinit var geo: TLGeoPoint
 
 	// We don't know the size, so we just guess.
 	override val size: Int
@@ -59,8 +45,11 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient)
 	override val letter = "g"
 	override val name = "geo"
 	override val description = "Geolocation"
+	
+	val json = message["media"]["geo"].obj
 
 	init {
+		/*
 		val g = (msg.getMedia() as TLMessageMediaGeo).getGeo()
 		if (g is TLGeoPoint) {
 			this.geo = g
@@ -68,15 +57,16 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient)
 			this.isEmpty = true
 		} else {
 			throwUnexpectedObjectError(g)
-		}
+		}*/
 	}
 
 	@Throws(IOException::class)
-	override fun download(): Boolean {
+	override fun download(prog: DownloadProgressInterface?): Boolean {
 		val url = "https://maps.googleapis.com/maps/api/staticmap?" +
-			"center=" + geo.getLat() + "," + geo.getLong() + "&" +
+			"center=${json["lat"].float},${json["_long"].float}&" +
+			"markers=color:red|${json["lat"].float},${json["_long"].float}&" +
 			"zoom=14&size=300x150&scale=2&format=png&" +
-			"key=" + Config.SECRET_GMAPS
-		return DownloadManager.downloadExternalFile(targetPathAndFilename, url)
+			"key=" + (settings?.gmaps_key)
+		return DownloadManager.downloadExternalFile(targetPathAndFilename, url, prog)
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt

@@ -17,70 +17,71 @@
 package de.fabianonline.telegram_backup.mediafilemanager
 
 import de.fabianonline.telegram_backup.UserManager
-import de.fabianonline.telegram_backup.Database
-import de.fabianonline.telegram_backup.StickerConverter
-import de.fabianonline.telegram_backup.DownloadProgressInterface
 import de.fabianonline.telegram_backup.DownloadManager
-
-import com.github.badoualy.telegram.api.TelegramClient
-import com.github.badoualy.telegram.tl.core.TLIntVector
-import com.github.badoualy.telegram.tl.core.TLObject
-import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
-import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
+import de.fabianonline.telegram_backup.DownloadProgressInterface
 import com.github.badoualy.telegram.tl.api.*
-import com.github.badoualy.telegram.tl.api.upload.TLFile
 import com.github.badoualy.telegram.tl.exception.RpcErrorException
-import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
 
 import java.io.IOException
-import java.io.File
-import java.io.FileOutputStream
-import java.util.ArrayList
-import java.util.LinkedList
-import java.net.URL
 import java.util.concurrent.TimeoutException
 
-import org.apache.commons.io.FileUtils
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.*
 
-class PhotoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : AbstractMediaFileManager(msg, user, client) {
-	private lateinit var photo: TLPhoto
+class PhotoFileManager(message: JsonObject, file_base: String) : AbstractMediaFileManager(message, file_base) {
+	//private lateinit var photo: TLPhoto
 	override var size = 0
-	private lateinit var photo_size: TLPhotoSize
+	//private lateinit var photo_size: TLPhotoSize
 
 	override val extension = "jpg"
 	override val letter = "p"
 	override val name = "photo"
 	override val description = "Photo"
+	
+	val biggestSize: JsonObject
+	var biggestSizeW = 0
+	var biggestSizeH = 0
+	
+	val json = message["media"]["photo"].obj
+	override var isEmpty = json.isA("photoEmpty")
 
 	init {
-		val p = (msg.getMedia() as TLMessageMediaPhoto).getPhoto()
-		if (p is TLPhoto) {
-			this.photo = p
-
-			var biggest: TLPhotoSize? = null
-			for (s in photo.getSizes())
-				if (s is TLPhotoSize) {
-					if (biggest == null || s.getW() > biggest.getW() && s.getH() > biggest.getH()) {
-						biggest = s
-					}
+		/*val p = (msg.getMedia() as TLMessageMediaPhoto).getPhoto()*/
+		if (!isEmpty) {
+			var bsTemp: JsonObject? = null
+		
+			for (elm in json["sizes"].array) {
+				val s = elm.obj
+				if (!s.isA("photoSize")) continue
+				if (bsTemp == null || (s["w"].int > biggestSizeW && s["h"].int > biggestSizeH)) {
+					bsTemp = s
+					biggestSizeW = s["w"].int
+					biggestSizeH = s["h"].int
+					size = s["size"].int // file size
 				}
-			if (biggest == null) {
-				throw RuntimeException("Could not find a size for a photo.")
 			}
-			this.photo_size = biggest
-			this.size = biggest.getSize()
-		} else if (p is TLPhotoEmpty) {
+
+			if (bsTemp == null) throw RuntimeException("Could not find a size for a photo.")
+			biggestSize = bsTemp
+		} else {
+			biggestSize = JsonObject()
+		}
+			
+		/*} else if (p is TLPhotoEmpty) {
 			this.isEmpty = true
 		} else {
 			throwUnexpectedObjectError(p)
-		}
+		}*/
 	}
 
 	@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
-	override fun download(): Boolean {
-		if (isEmpty) return true
-		val loc = photo_size.getLocation() as TLFileLocation
-		DownloadManager.downloadFile(targetPathAndFilename, size, loc.getDcId(), loc.getVolumeId(), loc.getLocalId(), loc.getSecret())
+	override fun download(prog: DownloadProgressInterface?): Boolean {
+		/*if (isEmpty) return true*/
+		//val loc = photo_size.getLocation() as TLFileLocation
+		
+		val loc = biggestSize["location"].obj
+		DownloadManager.downloadFile(targetPathAndFilename, size, loc["dcId"].int, loc["volumeId"].long, loc["localId"].int, loc["secret"].long, prog)
 		return true
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt

@@ -18,7 +18,6 @@ package de.fabianonline.telegram_backup.mediafilemanager
 
 import de.fabianonline.telegram_backup.UserManager
 import de.fabianonline.telegram_backup.Database
-import de.fabianonline.telegram_backup.StickerConverter
 import de.fabianonline.telegram_backup.DownloadProgressInterface
 import de.fabianonline.telegram_backup.DownloadManager
 import de.fabianonline.telegram_backup.Config
@@ -50,29 +49,23 @@ import java.util.concurrent.TimeoutException
 
 import org.apache.commons.io.FileUtils
 
-class StickerFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : DocumentFileManager(msg, user, client) {
+import com.google.gson.*
+import com.github.salomonbrys.kotson.*
+import de.fabianonline.telegram_backup.*
+
+class StickerFileManager(message: JsonObject, file_base: String) : DocumentFileManager(message, file_base) {
 
 	override val isSticker = true
+	
+	val json = message["media"]["document"].obj
+	val sticker = json["attributes"].array.first{it.obj.isA("documentAttributeSticker")}.obj
+	override var isEmpty = sticker["stickerset"].obj.isA("inputStickerSetEmpty")
 
 	private val filenameBase: String
 		get() {
-			var sticker: TLDocumentAttributeSticker? = null
-			for (attr in doc!!.getAttributes()) {
-				if (attr is TLDocumentAttributeSticker) {
-					sticker = attr
-				}
-			}
-
-			val file = StringBuilder()
-			val set = sticker!!.getStickerset()
-			if (set is TLInputStickerSetShortName) {
-				file.append(set.getShortName())
-			} else if (set is TLInputStickerSetID) {
-				file.append(set.getId())
-			}
-			file.append("_")
-			file.append(sticker.getAlt().hashCode())
-			return file.toString()
+			val set = sticker["stickerset"].obj.get("shortName").nullString ?: sticker["stickerset"].obj.get("id").string
+			val hash = sticker["alt"].string.hashCode()
+			return "${set}_${hash}"
 		}
 
 	override val targetFilename: String
@@ -80,7 +73,7 @@ class StickerFileManager(msg: TLMessage, user: UserManager, client: TelegramClie
 
 	override val targetPath: String
 		get() {
-			val path = user.fileBase + Config.FILE_FILES_BASE + File.separatorChar + Config.FILE_STICKER_BASE + File.separatorChar
+			val path = file_base + Config.FILE_FILES_BASE + File.separatorChar + Config.FILE_STICKER_BASE + File.separatorChar
 			File(path).mkdirs()
 			return path
 		}
@@ -94,19 +87,6 @@ class StickerFileManager(msg: TLMessage, user: UserManager, client: TelegramClie
 	override val description: String
 		get() = "Sticker"
 
-	@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
-	override fun download(): Boolean {
-		val old_file = Config.FILE_BASE + File.separatorChar + Config.FILE_STICKER_BASE + File.separatorChar + targetFilename
-
-		logger.trace("Old filename exists: {}", File(old_file).exists())
-
-		if (File(old_file).exists()) {
-			Files.copy(Paths.get(old_file), Paths.get(targetPathAndFilename), StandardCopyOption.REPLACE_EXISTING)
-			return true
-		}
-		return super.download()
-	}
-
 	companion object {
 		private val logger = LoggerFactory.getLogger(StickerFileManager::class.java)
 	}

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt

@@ -17,33 +17,11 @@
 package de.fabianonline.telegram_backup.mediafilemanager
 
 import de.fabianonline.telegram_backup.UserManager
-import de.fabianonline.telegram_backup.Database
-import de.fabianonline.telegram_backup.StickerConverter
 import de.fabianonline.telegram_backup.DownloadProgressInterface
-import de.fabianonline.telegram_backup.DownloadManager
-import de.fabianonline.telegram_backup.Config
-
-import com.github.badoualy.telegram.api.TelegramClient
-import com.github.badoualy.telegram.tl.core.TLIntVector
-import com.github.badoualy.telegram.tl.core.TLObject
-import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
-import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
 import com.github.badoualy.telegram.tl.api.*
-import com.github.badoualy.telegram.tl.api.upload.TLFile
-import com.github.badoualy.telegram.tl.exception.RpcErrorException
-import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
+import com.google.gson.JsonObject
 
-import java.io.IOException
-import java.io.File
-import java.io.FileOutputStream
-import java.util.ArrayList
-import java.util.LinkedList
-import java.net.URL
-import java.util.concurrent.TimeoutException
-
-import org.apache.commons.io.FileUtils
-
-class UnsupportedFileManager(msg: TLMessage, user: UserManager, client: TelegramClient, type: String) : AbstractMediaFileManager(msg, user, client) {
+class UnsupportedFileManager(json: JsonObject, file_base: String, type: String) : AbstractMediaFileManager(json, file_base) {
 	override var name = type
 	override val targetFilename = ""
 	override val targetPath = ""
@@ -54,5 +32,5 @@ class UnsupportedFileManager(msg: TLMessage, user: UserManager, client: Telegram
 	override val letter = " "
 	override val description = "Unsupported / non-downloadable Media"
 
-	override fun download(): Boolean = true
+	override fun download(prog: DownloadProgressInterface?): Boolean = true
 }

src/main/resources/config.sample.ini

@@ -0,0 +1,71 @@
+# Config file for telegram_backup
+# Copy it to config.ini to use it. This sample file be overwritten on every run.
+#
+# Lines starting with '#' are ignored.
+# Settings have the form 'key = value'
+# To unset a setting, use 'key =' (or just don't use that key at all)
+# Some settings may appear more than once. 
+
+
+
+## GMaps key to use. Leave empty / don't set a value to use the built in key.
+# gmaps_key = mysecretgmapskey
+
+## Use pagination in the HTML export?
+# pagination = true
+# pagination_size = 5000
+
+## Download media files
+# download_media = true
+
+## Only download media files from messages that are never than x days.
+## Leave unset to download all media files.
+# max_file_age = 7
+
+## Only download media files that are smaller than x MB.
+## Leave unset to download media files regardless of their size.
+# max_file_size = 5
+
+## Don't download media files having these extensions.
+## You can add this line multiple times to blacklist more than one extension.
+# blacklist_extensions = jpg
+# blacklist_extensions = avi
+
+## Downloads of channels and supergroups
+## Here you can specify which channels and supergroups
+## should be downloaded. The rules for this are:
+## 1. Channels and supergroups are NEVER downloaded unless download_channels and/or
+##    download_supergroups is set to true.
+## 2. If there is at least one entry called whitelist_channels, ONLY channels and/or
+##    supergroups that are listed in the whitelist will be downloaded.
+## 3. Only if there are NO channels whitelisted, entrys listed as blacklist_channels
+##    will not be downloaded, all other channels / supergroups will be.
+##
+## In other words:
+## * Set download_channels and/or download_supergroups to true if you want to include
+##   those types in your backup.
+## * If you use neither black- nor whitelist, all channels (if you set download_channels
+##   to true) / supergroups (if you set download_supergroups to true) get downloaded.
+## * If you set a whitelist, only listed channels / supergroups (there is no distinction
+##   made here) will be loaded.
+## * If you set a blacklist, everything except the listed channels / supergroups (again,
+##   although the entry is called whitelist_channels it affects channels AND supergroups)
+##   will be loaded.
+## * If you set a whitelist AND a blacklist, the blacklist will be ignored.
+##
+## Call the app with `--list-channels` to list the available channels and supergroups
+## with their IDs. That list will also tell you if a channel / supergroup will be
+## be downloaded according to your black- and whitelists.
+##
+## You can have more than one whitelist_channels and/or blacklist_channels entries
+## to build your list. One ID per entry.
+
+# download_channels = false
+# download_supergroups = false
+
+# blacklist_channels = 12345678
+# blacklist_channels = 8886542
+# blacklist_channels = 715952
+
+# whitelist_channels = 238572935
+# whitelist_channels = 23857623

src/main/resources/templates/csv/links.csv

@@ -0,0 +1,3 @@
+{{#links}}
+"{{time}}","{{url}}","{{host}}","{{username}}","{{chat_name}}"
+{{/links}}

src/main/resources/templates/csv/messages.csv

@@ -0,0 +1,3 @@
+{{#messages}}
+"{{time}}","{{username}}","{{chat_name}}","{{message}}"
+{{/messages}}

src/main/resources/templates/html/_messages.mustache

@@ -12,6 +12,7 @@
 			{{#media_sticker}}<span class="sticker"><img src="../stickers/{{media_file}}" /></span>{{/media_sticker}}
 			{{#media_photo}}<span class="photo"><img src="../{{media_file}}" /></span>{{/media_photo}}
 			{{#media_document}}<span class="document"><a href="../{{media_file}}">{{media_file}}</a></span>{{/media_document}}
+			{{#media_geo}}<span class="geo"><img src="../{{media_file}}" width="300" height="150"></span>{{/media_geo}}
 		</li>
 	{{/messages}}
 </ul>