Merge branch 'master' into issue_94

deploy.sh

@@ -40,6 +40,9 @@ git commit -m "Bumping the version to $VERSION" Dockerfile
 echo "Tagging the new version..."
 git tag -a "$VERSION" -m "Version $VERSION" || error
 
+echo "Building it..."
+gradle build || error "Build failed. What did you do?!"
+
 echo "Checking out stable..."
 git checkout stable || error
 
@@ -52,18 +55,15 @@ git push --all || error
 echo "Pushing tags to Github..."
 git push --tags || error
 
-echo "Building it..."
-gradle build || error "Build failed. What did you do?!"
-
 echo "Generating a release on Github..."
-json=$(ruby -e "require 'json'; puts({tag_name: '$VERSION', name: '$VERSION', draft: true, body: \$stdin.read}.to_json)" <<< "$release_notes") || error "Couldn't generate JSON for Github"
+json=$(ruby -e "require 'json'; puts({tag_name: '$VERSION', name: '$VERSION', body: \$stdin.read}.to_json)" <<< "$release_notes") || error "Couldn't generate JSON for Github"
 
 json=$(curl $CURL_OPTS https://api.github.com/repos/fabianonline/telegram_backup/releases -XPOST -d "$json") || error "Github failure"
 
 echo "Uploading telegram_backup.jar to Github..."
 upload_url=$(jq -r ".upload_url" <<< "$json") || error "Could not parse JSON from Github"
 upload_url=$(sed 's/{.*}//' <<< "$upload_url")
-release_url=$(jq -r ".url" <<< "$json") || error "Could not parse JSON from Github"
+release_url=$(jq -r ".html_url" <<< "$json") || error "Could not parse JSON from Github"
 curl $CURL_OPTS --header "Content-Type: application/zip" "${upload_url}?name=telegram_backup.jar" --upload-file build/libs/telegram_backup.jar || error "Asset upload to github failed"
 
 echo "Building the docker image..."
@@ -73,12 +73,15 @@ echo "Pushing the docker image..."
 docker push fabianonline/telegram_backup
 
 echo "Notifying the Telegram group..."
-release_notes=$(sed 's/\* /• /' <<< "$release_notes")
-message="Version $VERSION released"$'\n'$'\n'"$release_notes"$'\n'$'\n'"$release_url"
+release_notes=$(sed 's/\* /• /' | sed 's/&/&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g' <<< "$release_notes")
+message="<b>Version $VERSION was just released</b>"$'\n'$'\n'"$release_notes"$'\n'$'\n'"$release_url"
 
-curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" <<< "$message"
+curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" --form-string "parse_mode=HTML" --form-string "disable_web_page_preview=true" <<< "$message"
 
 echo "Cleaning release_notes.txt..."
 > release_notes.txt
 
+echo "Checking out master..."
+git checkout master
+
 echo "Done."

deploy_beta.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+error() {
+	echo "Error: $1"
+	exit 1
+}
+
+release_notes="$(cat release_notes.txt 2>/dev/null)"
+
+source "deploy.secret.sh"
+[ -z "$BOT_TOKEN" ] && error "BOT_TOKEN is not set or empty."
+[ -z "$CHAT_ID" ] && error "CHAT_ID is not set or empty."
+
+version=$(git describe --tags --dirty)
+
+echo "Enter additional notes, end with Ctrl-D."
+additional_notes="$(cat)"
+
+echo "Building it..."
+gradle build || error "Build failed. What did you do?!"
+
+echo "Copying it to files.fabianonline.de..."
+filename="telegram_backup.beta_${version}.jar"
+cp --no-preserve "mode,ownership,timestamps" build/libs/telegram_backup.jar /data/containers/nginx/www/files/${filename}
+
+echo "Notifying the Telegram group..."
+release_notes=$(echo "$release_notes" | sed 's/\* /• /' | sed 's/&/&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g')
+message="<b>New beta release $version</b>"$'\n\n'
+message="${message}${additional_notes}"$'\n\n'
+message="${message}Changes since the last <i>real</i> release:"$'\n'"${release_notes}"$'\n\n'
+message="${message}<b>This is a release for testing purposes only. There may be bugs included that might destroy your data. Only use this beta release if you know what you're doing. AND MAKE A BACKUP OF YOUR BACKUP BEFORE USING IT!</b>"$'\n\n'
+message="${message}Please report back if you used this release and encountered a bug. Also report back, if you used it and IT WORKED, please. Thank you."$'\n\n'
+message="${message}https://files.fabianonline.de/${filename}"
+
+result=$(curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" --form-string "parse_mode=HTML" --form-string "disable_web_page_preview=true" <<< "$message")
+message_id=$(jq -r '.result.message_id' <<< "$result")
+
+echo "Pinning the new message..."
+curl https://api.telegram.org/bot${BOT_TOKEN}/pinChatMessage -XPOST --form "chat_id=${CHAT_ID}" --form "message_id=${message_id}" --form "disable_notification=true"
+
+echo "Done."

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt

@@ -93,6 +93,10 @@ class CommandLineController {
 					throw RuntimeException("Account / User mismatch")
 				}
 			}
+			
+			// Load the ini file.
+			IniSettings.load()
+			
 			logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login)
 			if (CommandLineOptions.cmd_login) {
 				cmd_login(CommandLineOptions.val_account)
@@ -131,14 +135,39 @@ class CommandLineController {
 			}
 			logger.info("Initializing Download Manager")
 			val d = DownloadManager(client, CommandLineDownloadProgress())
+			
+			if (CommandLineOptions.cmd_list_channels) {
+				val chats = d.getChats()
+				val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) }
+				val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")}
+				var download: Boolean
+
+				println("Channels:")
+				download = IniSettings.download_channels
+				if (!download) println("Download of channels is disabled - see download_channels in config.ini")
+				print_header(download)
+				for (c in chats.channels) {
+					println(format(c, download))
+				}
+				println()
+				println("Supergroups:")
+				download = IniSettings.download_supergroups
+				if (!download) println("Download of supergroups is disabled - see download_supergroups in config.ini")
+				print_header(download)
+				for (c in chats.supergroups) {
+					println(format(c, download))
+				}
+				System.exit(0)
+			}
+			
 			logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages)
 			d.downloadMessages(CommandLineOptions.val_limit_messages)
-			logger.debug("CommandLineOptions.cmd_no_media: {}", CommandLineOptions.cmd_no_media)
-			if (!CommandLineOptions.cmd_no_media) {
+			logger.debug("IniSettings.download_media: {}", IniSettings.download_media)
+			if (IniSettings.download_media) {
 				logger.debug("Calling DownloadManager.downloadMedia")
 				d.downloadMedia()
 			} else {
-				println("Skipping media download because --no-media is set.")
+				println("Skipping media download because download_media is set to false.")
 			}
 		} catch (e: Throwable) {
 			println("An error occured!")
@@ -276,18 +305,14 @@ class CommandLineController {
 		println(" --trace-telegram      Shows lots of debug messages from the library used to access Telegram.")
 		println(" -A, --list-accounts   List all existing accounts ")
 		println(" --limit-messages <x>  Downloads at most the most recent <x> messages.")
-		println(" --no-media            Do not download media files.")
 		println(" -t, --target <x>      Target directory for the files.")
 		println(" -e, --export <format> Export the database. Valid formats are:")
 		println("                html - Creates HTML files.")
-		println(" --pagination <x>      Splits the HTML export into multiple HTML pages with <x> messages per page. Default is 5000.")
-		println(" --no-pagination       Disables pagination.")
 		println(" --license             Displays the license of this program.")
 		println(" -d, --daemon          Keep running after the backup and automatically save new messages.")
 		println(" --anonymize           (Try to) Remove all sensitive information from output. Useful for requesting support.")
 		println(" --stats               Print some usage statistics.")
-		println(" --with-channels       Backup channels as well.")
-		println(" --with-supergroups    Backup supergroups as well.")
+		println(" --list-channels       Lists all channels together with their ID")
 	}
 
 	private fun list_accounts() {

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt

@@ -26,18 +26,14 @@ internal object CommandLineOptions {
 	var cmd_version = false
 	var cmd_license = false
 	var cmd_daemon = false
-	var cmd_no_media = false
 	var cmd_anonymize = false
 	var cmd_stats = false
-	var cmd_channels = false
-	var cmd_supergroups = false
-	var cmd_no_pagination = false
+	var cmd_list_channels = false
 	var val_account: String? = null
 	var val_limit_messages: Int? = null
 	var val_target: String? = null
 	var val_export: String? = null
 	var val_test: Int? = null
-	var val_pagination: Int = Config.DEFAULT_PAGINATION
 	@JvmStatic
 	fun parseOptions(args: Array<String>) {
 		var last_cmd: String? = null
@@ -49,7 +45,6 @@ internal object CommandLineOptions {
 					"--target" -> val_target = arg
 					"--export" -> val_export = arg
 					"--test" -> val_test = Integer.parseInt(arg)
-					"--pagination" -> val_pagination = Integer.parseInt(arg)
 				}
 				last_cmd = null
 				continue
@@ -83,18 +78,15 @@ internal object CommandLineOptions {
 					last_cmd = "--pagination"
 					continue@loop
 				}
-				"--no-pagination" -> cmd_no_pagination = true
 				"--license" -> cmd_license = true
 				"-d", "--daemon" -> cmd_daemon = true
-				"--no-media" -> cmd_no_media = true
 				"--test" -> {
 					last_cmd = "--test"
 					continue@loop
 				}
 				"--anonymize" -> cmd_anonymize = true
 				"--stats" -> cmd_stats = true
-				"--with-channels" -> cmd_channels = true
-				"--with-supergroups" -> cmd_supergroups = true
+				"--list-channels" -> cmd_list_channels = true
 				else -> throw RuntimeException("Unknown command " + arg)
 			}
 		}

src/main/kotlin/de/fabianonline/telegram_backup/Database.kt

@@ -56,7 +56,9 @@ class Database private constructor(var client: TelegramClient) {
 		try {
 			val rs = stmt!!.executeQuery("SELECT MAX(message_id) FROM messages WHERE source_type IN ('group', 'dialog')")
 			rs.next()
-			return rs.getInt(1)
+			val result = rs.getInt(1)
+			rs.close()
+			return result
 		} catch (e: SQLException) {
 			return 0
 		}
@@ -87,6 +89,7 @@ class Database private constructor(var client: TelegramClient) {
 						missing.add(i)
 					}
 				}
+				rs.close()
 				return missing
 			} catch (e: SQLException) {
 				e.printStackTrace()
@@ -111,16 +114,7 @@ class Database private constructor(var client: TelegramClient) {
 
 	}
 
-	fun getMessagesFromUserCount(): Int {
-		try {
-			val rs = stmt!!.executeQuery("SELECT COUNT(*) FROM messages WHERE sender_id=" + user_manager.user!!.getId())
-			rs.next()
-			return rs.getInt(1)
-		} catch (e: SQLException) {
-			throw RuntimeException(e)
-		}
-
-	}
+	fun getMessagesFromUserCount() = queryInt("SELECT COUNT(*) FROM messages WHERE sender_id=" + user_manager.user!!.getId())
 
 	fun getMessageTypesWithCount(): HashMap<String, Int> = getMessageTypesWithCount(GlobalChat())
 
@@ -164,7 +158,9 @@ class Database private constructor(var client: TelegramClient) {
 		try {
 			val rs = stmt!!.executeQuery("PRAGMA encoding")
 			rs.next()
-			return rs.getString(1)
+			val result = rs.getString(1)
+			rs.close()
+			return result
 		} catch (e: SQLException) {
 			logger.debug("SQLException: {}", e)
 			return "unknown"
@@ -242,7 +238,7 @@ class Database private constructor(var client: TelegramClient) {
 		try {
 			val src = user_manager.fileBase + Config.FILE_NAME_DB
 			val dst = user_manager.fileBase + filename
-			logger.debug("Copying {} to {}", src, dst)
+			logger.debug("Copying {} to {}", src.anonymize(), dst.anonymize())
 			Files.copy(
 				File(src).toPath(),
 				File(dst).toPath())
@@ -255,9 +251,7 @@ class Database private constructor(var client: TelegramClient) {
 
 	}
 
-	fun getTopMessageIDForChannel(id: Int): Int {
-		return queryInt("SELECT MAX(message_id) FROM messages WHERE source_id=$id AND source_type IN('channel', 'supergroup')")
-	}
+	fun getTopMessageIDForChannel(id: Int): Int = queryInt("SELECT MAX(message_id) FROM messages WHERE source_id=$id AND source_type IN('channel', 'supergroup')")
 
 	fun logRun(start_id: Int, end_id: Int, count: Int) {
 		try {
@@ -269,6 +263,7 @@ class Database private constructor(var client: TelegramClient) {
 			ps.setInt(2, end_id)
 			ps.setInt(3, count)
 			ps.execute()
+			ps.close()
 		} catch (e: SQLException) {
 		}
 
@@ -278,10 +273,9 @@ class Database private constructor(var client: TelegramClient) {
 		try {
 			val rs = stmt!!.executeQuery(query)
 			rs.next()
-
 			val result = rs.getInt(1)
-
 			rs.close()
+
 			return result
 		} catch (e: SQLException) {
 			throw RuntimeException("Could not get count of messages.")
@@ -434,6 +428,9 @@ class Database private constructor(var client: TelegramClient) {
 			ps_insert_or_ignore.clearBatch()
 			conn!!.commit()
 			conn!!.setAutoCommit(true)
+			
+			ps.close()
+			ps_insert_or_ignore.close()
 		} catch (e: Exception) {
 			e.printStackTrace()
 			throw RuntimeException("Exception shown above happened.")
@@ -489,6 +486,9 @@ class Database private constructor(var client: TelegramClient) {
 			ps_insert_or_replace.clearBatch()
 			conn!!.commit()
 			conn!!.setAutoCommit(true)
+			
+			ps_insert_or_ignore.close()
+			ps_insert_or_replace.close()
 		} catch (e: Exception) {
 			e.printStackTrace()
 			throw RuntimeException("Exception shown above happened.")
@@ -538,13 +538,32 @@ class Database private constructor(var client: TelegramClient) {
 			ps_insert_or_replace.clearBatch()
 			conn!!.commit()
 			conn!!.setAutoCommit(true)
+			
+			ps_insert_or_ignore.close()
+			ps_insert_or_replace.close()
 		} catch (e: Exception) {
 			e.printStackTrace()
 			throw RuntimeException("Exception shown above happened.")
 		}
-
 	}
-
+	
+	fun fetchSetting(key: String): String? {
+		val rs = stmt!!.executeQuery("SELECT value FROM settings WHERE key='${key}'")
+		rs.next()
+		return rs.getString(1)
+	}
+	
+	fun saveSetting(key: String, value: String?) {
+		val ps = conn!!.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)")
+		ps.setString(1, key)
+		if (value==null) {
+			ps.setNull(2, Types.VARCHAR)
+		} else {
+			ps.setString(2, value)
+		}
+		ps.execute()
+	}
+	
 	fun getIdsFromQuery(query: String): LinkedList<Int> {
 		try {
 			val list = LinkedList<Int>()
@@ -567,6 +586,7 @@ class Database private constructor(var client: TelegramClient) {
 			while (rs.next()) {
 				map.put("count.messages.type." + rs.getString(1), rs.getInt(2))
 			}
+			rs.close()
 			return map
 		} catch (e: Exception) {
 			throw RuntimeException(e)
@@ -589,6 +609,7 @@ class Database private constructor(var client: TelegramClient) {
 				map.put("count.messages.media_type.$s", rs.getInt(2))
 			}
 			map.put("count.messages.media_type.any", count)
+			rs.close()
 			return map
 		} catch (e: Exception) {
 			throw RuntimeException(e)
@@ -628,6 +649,7 @@ class Database private constructor(var client: TelegramClient) {
 			}
 			map.put("authors.count.others", count_others)
 			map.put("authors.all", all_data)
+			rs.close()
 			return map
 		} catch (e: Exception) {
 			throw RuntimeException(e)
@@ -637,7 +659,9 @@ class Database private constructor(var client: TelegramClient) {
 	fun getMessageCountForExport(c: AbstractChat): Int {
 		val rs = stmt!!.executeQuery("SELECT COUNT(*) FROM messages WHERE " + c.query);
 		rs.next()
-		return rs.getInt(1)
+		val result = rs.getInt(1)
+		rs.close()
+		return result
 	}
 
 	fun getMessageTimesMatrix(c: AbstractChat): Array<IntArray> {
@@ -650,6 +674,7 @@ class Database private constructor(var client: TelegramClient) {
 			while (rs.next()) {
 				result[if (rs.getInt(1) == 0) 6 else rs.getInt(1) - 1][rs.getInt(2)] = rs.getInt(3)
 			}
+			rs.close()
 			return result
 		} catch (e: Exception) {
 			throw RuntimeException(e)

src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt

@@ -32,30 +32,25 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
 		register(DB_Update_7(conn, db))
 		register(DB_Update_8(conn, db))
 		register(DB_Update_9(conn, db))
+		register(DB_Update_10(conn, db))
 	}
 
 	fun doUpdates() {
 		try {
 			val stmt = conn.createStatement()
-			var rs: ResultSet
 			logger.debug("DatabaseUpdate.doUpdates running")
 
 			logger.debug("Getting current database version")
 			var version: Int
 			logger.debug("Checking if table database_versions exists")
-			rs = stmt.executeQuery("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions'")
-			rs.next()
-			if (rs.getInt(1) == 0) {
+			val table_count = db.queryInt("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions'")
+			if (table_count == 0) {
 				logger.debug("Table does not exist")
 				version = 0
 			} else {
 				logger.debug("Table exists. Checking max version")
-				rs.close()
-				rs = stmt.executeQuery("SELECT MAX(version) FROM database_versions")
-				rs.next()
-				version = rs.getInt(1)
+				version = db.queryInt("SELECT MAX(version) FROM database_versions")
 			}
-			rs.close()
 			logger.debug("version: {}", version)
 			System.out.println("Database version: " + version)
 			logger.debug("Max available database version is {}", maxPossibleVersion)
@@ -106,6 +101,9 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
 				} catch (e: SQLException) {
 					throw RuntimeException(e)
 				}
+				
+				println("Cleaning up the database (this might take some time)...")
+				try { stmt.executeUpdate("VACUUM") } catch (t: Throwable) { logger.debug("Exception during VACUUMing: {}", t) }
 
 			} else {
 				logger.debug("No update necessary.")
@@ -442,7 +440,15 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
 		}
 		println()
 		logger.info("Converted ${i} of ${count} messages.")
-		println("    Cleaning up the database (this might also take some time, sorry)...")
-		execute("VACUUM")
+	}
+}
+
+internal class DB_Update_10(conn: Connection, db: Database) : DatabaseUpdate(conn, db) {
+	override val version: Int
+		get() = 10
+	
+	@Throws(SQLException::class)
+	override fun _doUpdate() {
+		execute("CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)")
 	}
 }

src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt

@@ -0,0 +1,12 @@
+package de.fabianonline.telegram_backup
+
+class DbSettings() {
+	private fun fetchValue(name: String): String? = Database.getInstance().fetchSetting(name)
+	private fun saveValue(name: String, value: String?) = Database.getInstance().saveSetting(name, value)
+	
+	var pts: String?
+		get() = fetchValue("pts")
+		set(x: String?) = saveValue("pts", x)
+}
+
+

src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt

@@ -106,19 +106,14 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 	@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
 	fun _downloadMessages(limit: Int?) {
 		logger.info("This is _downloadMessages with limit {}", limit)
-		val dialog_limit = 100
-		logger.info("Downloading the last {} dialogs", dialog_limit)
+		logger.info("Downloading the last dialogs")
 		System.out.println("Downloading most recent dialogs... ")
 		var max_message_id = 0
-		val dialogs = client!!.messagesGetDialogs(
-			0,
-			0,
-			TLInputPeerEmpty(),
-			dialog_limit)
-		logger.debug("Got {} dialogs", dialogs.getDialogs().size)
+		val chats = getChats()
+		logger.debug("Got {} dialogs, {} supergoups, {} channels", chats.dialogs.size, chats.supergroups.size, chats.channels.size)
 
-		for (d in dialogs.getDialogs()) {
-			if (d.getTopMessage() > max_message_id && d.getPeer() !is TLPeerChannel) {
+		for (d in chats.dialogs) {
+			if (d.getTopMessage() > max_message_id) {
 				logger.trace("Updating top message id: {} => {}. Dialog type: {}", max_message_id, d.getTopMessage(), d.getPeer().javaClass)
 				max_message_id = d.getTopMessage()
 			}
@@ -178,64 +173,33 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 		}
 		*/
 
-		if (CommandLineOptions.cmd_channels || CommandLineOptions.cmd_supergroups) {
-			System.out.println("Processing channels and/or supergroups...")
-			System.out.println("Please note that only channels/supergroups in the last 100 active chats are processed.")
-
-			val channel_access_hashes = HashMap<Int, Long>()
-			val channel_names = HashMap<Int, String>()
-			val channels = LinkedList<Int>()
-			val supergroups = LinkedList<Int>()
-
+		if (IniSettings.download_channels || IniSettings.download_supergroups) {
 			// TODO Add chat title (and other stuff?) to the database
-			for (c in dialogs.getChats()) {
-				if (c is TLChannel) {
-					channel_access_hashes.put(c.getId(), c.getAccessHash())
-					channel_names.put(c.getId(), c.getTitle())
-					if (c.getMegagroup()) {
-						supergroups.add(c.getId())
-					} else {
-						channels.add(c.getId())
-					}
-					// Channel: TLChannel
-					// Supergroup: getMegagroup()==true
-				}
+			
+			if (IniSettings.download_channels) {
+				println("Checking channels...")
+				for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) }
 			}
-
-
-
-			for (d in dialogs.getDialogs()) {
-				if (d.getPeer() is TLPeerChannel) {
-					val channel_id = (d.getPeer() as TLPeerChannel).getChannelId()
-
-					// If this is a channel and we don't want to download channels OR
-					// it is a supergroups and we don't want to download supergroups, then
-					if (channels.contains(channel_id) && !CommandLineOptions.cmd_channels || supergroups.contains(channel_id) && !CommandLineOptions.cmd_supergroups) {
-						// Skip this chat.
-						continue
-					}
-					val max_known_id = db!!.getTopMessageIDForChannel(channel_id)
-					if (d.getTopMessage() > max_known_id) {
-						val ids = makeIdList(max_known_id + 1, d.getTopMessage())
-						val access_hash = channel_access_hashes.get(channel_id) ?: throw RuntimeException("AccessHash for Channel missing.")
-						var channel_name = channel_names.get(channel_id)
-						if (channel_name == null) {
-							channel_name = "?"
-						}
-						val channel = TLInputChannel(channel_id, access_hash)
-						val source_type = if (supergroups.contains(channel_id)) {
-							MessageSource.SUPERGROUP
-						} else if (channels.contains(channel_id)) {
-							MessageSource.CHANNEL
-						} else {
-							throw RuntimeException("chat is neither in channels nor in supergroups...")
-						}
-						downloadMessages(ids, channel, source_type=source_type, source_name=channel_name)
-					}
-				}
+			
+			if (IniSettings.download_supergroups) {
+				println("Checking supergroups...")
+				for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) }
 			}
 		}
 	}
+	
+	private fun downloadMessagesFromChannel(channel: Channel) {
+		val obj = channel.obj
+		val max_known_id = db!!.getTopMessageIDForChannel(channel.id)
+		if (obj.getTopMessage() > max_known_id) {
+			val ids = makeIdList(max_known_id + 1, obj.getTopMessage())
+			var channel_name = channel.title
+
+			val input_channel = TLInputChannel(channel.id, channel.access_hash)
+			val source_type = channel.message_source
+			downloadMessages(ids, input_channel, source_type=source_type, source_name=channel_name)
+		}
+	}
 
 	@Throws(RpcErrorException::class, IOException::class)
 	private fun downloadMessages(ids: MutableList<Int>, channel: TLInputChannel?, source_type: MessageSource = MessageSource.NORMAL, source_name: String? = null) {
@@ -385,6 +349,49 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
 		for (i in start..end) a.add(i)
 		return a
 	}
+	
+	fun getChats(): ChatList {
+		val cl = ChatList()
+		logger.trace("Calling messagesGetDialogs")
+		val dialogs = client!!.messagesGetDialogs(0, 0, TLInputPeerEmpty(), 100)
+		logger.trace("Got {} dialogs back", dialogs.getDialogs().size)
+		
+		// Add dialogs
+		cl.dialogs.addAll(dialogs.getDialogs().filter{it.getPeer() !is TLPeerChannel})
+		
+		// Add supergoups and channels
+		for (tl_channel in dialogs.getChats().filter{it is TLChannel}.map{it as TLChannel}) {
+			val tl_peer_channel = dialogs.getDialogs().find{var p = it.getPeer() ; p is TLPeerChannel && p.getChannelId()==tl_channel.getId()}
+			
+			if (tl_peer_channel == null) continue
+			
+			var download = true
+			if (IniSettings.whitelist_channels != null) {
+				download = IniSettings.whitelist_channels!!.contains(tl_channel.getId().toString())
+			} else if (IniSettings.blacklist_channels != null) {
+				download = !IniSettings.blacklist_channels!!.contains(tl_channel.getId().toString())
+			}
+			val channel = Channel(id=tl_channel.getId(), access_hash=tl_channel.getAccessHash(), title=tl_channel.getTitle(), obj=tl_peer_channel, download=download)
+			if (tl_channel.getMegagroup()) {
+				channel.message_source = MessageSource.SUPERGROUP
+				cl.supergroups.add(channel)
+			} else {
+				channel.message_source = MessageSource.CHANNEL
+				cl.channels.add(channel)
+			}
+		}
+		return cl
+	}
+	
+	class ChatList {
+		val dialogs = mutableListOf<TLDialog>()
+		val supergroups = mutableListOf<Channel>()
+		val channels = mutableListOf<Channel>()
+	}
+	
+	class Channel(val id: Int, val access_hash: Long, val title: String, val obj: TLDialog, val download: Boolean) {
+		lateinit var message_source: MessageSource
+	}
 
 	companion object {
 		internal var download_client: TelegramClient? = null

src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt

@@ -0,0 +1,85 @@
+package de.fabianonline.telegram_backup
+
+import java.io.File
+import org.slf4j.LoggerFactory
+import org.slf4j.Logger
+
+object IniSettings {
+	val logger = LoggerFactory.getLogger(IniSettings::class.java)
+	var settings = mutableMapOf<String, MutableList<String>>()
+	
+	init {
+		if (UserManager.getInstance().user != null) {
+			loadIni(UserManager.getInstance().fileBase + "config.ini")
+			copySampleIni(UserManager.getInstance().fileBase + "config.sample.ini")
+		}
+	}
+	
+	// Dummy function that can be called to force this object to run its init-code.
+	fun load() { }
+	
+	private fun loadIni(filename: String) {
+		val file = File(filename)
+		logger.trace("Checking ini file {}", filename.anonymize())
+		if (!file.exists()) return
+		logger.debug("Loading ini file {}", filename.anonymize())
+		file.forEachLine { parseLine(it) }
+	}
+	
+	private fun parseLine(original_line: String) {
+		logger.trace("Parsing line: {}", original_line)
+		var line = original_line.trim().replaceAfter("#", "").removeSuffix("#")
+		logger.trace("After cleaning: {}", line)
+		if (line == "") return
+		val parts: List<String> = line.split("=", limit=2).map{it.trim()}
+
+		if (parts.size < 2) throw RuntimeException("Invalid config setting: $line")
+		
+		val (key, value) = parts
+		if (value=="") {
+			settings.remove(key)
+		} else {
+			var map = settings.get(key)
+			if (map == null) {
+				map = mutableListOf<String>()
+				settings.put(key, map)
+			}
+			map.add(value)
+		}
+	}
+	
+	private fun copySampleIni(filename: String) {
+		val stream = Config::class.java.getResourceAsStream("/config.sample.ini")
+		File(filename).outputStream().use { stream.copyTo(it) }
+		stream.close()
+	}
+	
+	fun println() = println(settings)
+	
+	fun getString(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default
+	fun getStringList(key: String): List<String>? = settings.get(key)
+	fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default
+	fun getBoolean(key: String, default: Boolean = false): Boolean {
+		val value = settings.get(key)?.last()
+		if (value==null) return default
+		return value=="true"
+	}
+	fun getArray(key: String): List<String> = settings.get(key) ?: listOf<String>()
+	
+	val gmaps_key: String
+		get() = getString("gmaps_key", default=Config.SECRET_GMAPS)!!
+	val pagination: Boolean
+		get() = getBoolean("pagination", default=true)
+	val pagination_size: Int
+		get() = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!!
+	val download_media: Boolean
+		get() = getBoolean("download_media", default=true)
+	val download_channels: Boolean
+		get() = getBoolean("download_channels", default=false)
+	val download_supergroups: Boolean
+		get() = getBoolean("download_supergroups", default=false)
+	val whitelist_channels: List<String>?
+		get() = getStringList("whitelist_channels")
+	val blacklist_channels: List<String>?
+		get() = getStringList("blacklist_channels")
+}

src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt

@@ -16,12 +16,6 @@
 
 package de.fabianonline.telegram_backup.exporter
 
-import de.fabianonline.telegram_backup.UserManager
-import de.fabianonline.telegram_backup.Database
-import de.fabianonline.telegram_backup.anonymize
-import de.fabianonline.telegram_backup.toPrettyJson
-import de.fabianonline.telegram_backup.CommandLineOptions
-
 import java.io.File
 import java.io.PrintWriter
 import java.io.OutputStreamWriter
@@ -38,6 +32,7 @@ import java.util.HashMap
 import com.github.mustachejava.DefaultMustacheFactory
 import com.github.mustachejava.Mustache
 import com.github.mustachejava.MustacheFactory
+import de.fabianonline.telegram_backup.*
 
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -49,7 +44,7 @@ class HTMLExporter {
 	@Throws(IOException::class)
 	fun export() {
 		try {
-			val pagination = if (CommandLineOptions.cmd_no_pagination) -1 else CommandLineOptions.val_pagination
+			val pagination = if (IniSettings.pagination) IniSettings.pagination_size else -1
 			
 			// Create base dir
 			logger.debug("Creating base dir")

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt

@@ -21,7 +21,7 @@ import de.fabianonline.telegram_backup.Database
 import de.fabianonline.telegram_backup.StickerConverter
 import de.fabianonline.telegram_backup.DownloadProgressInterface
 import de.fabianonline.telegram_backup.DownloadManager
-import de.fabianonline.telegram_backup.Config
+import de.fabianonline.telegram_backup.IniSettings
 
 import com.github.badoualy.telegram.api.TelegramClient
 import com.github.badoualy.telegram.tl.core.TLIntVector
@@ -77,7 +77,7 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient)
 			"center=${geo.getLat()},${geo.getLong()}&" +
 			"markers=color:red|${geo.getLat()},${geo.getLong()}&" +
 			"zoom=14&size=300x150&scale=2&format=png&" +
-			"key=" + Config.SECRET_GMAPS
+			"key=" + IniSettings.gmaps_key
 		return DownloadManager.downloadExternalFile(targetPathAndFilename, url)
 	}
 }

src/main/resources/config.sample.ini

@@ -0,0 +1,60 @@
+# Config file for telegram_backup
+# Copy it to config.ini to use it. This sample file be overwritten on every run.
+#
+# Lines starting with '#' are ignored.
+# Settings have the form 'key = value'
+# To unset a setting, use 'key =' (or just don't use that key at all)
+# Some settings may appear more than once. 
+
+
+
+## GMaps key to use. Leave empty / don't set a value to use the built in key.
+# gmaps_key = mysecretgmapskey
+
+## Use pagination in the HTML export?
+# pagination = true
+# pagination_size = 5000
+
+## Download media files
+# download_media = true
+
+
+
+## Downloads of channels and supergroups
+## Here you can specify which channels and supergroups
+## should be downloaded. The rules for this are:
+## 1. Channels and supergroups are NEVER downloaded unless download_channels and/or
+##    download_supergroups is set to true.
+## 2. If there is at least one entry called whitelist_channels, ONLY channels and/or
+##    supergroups that are listed in the whitelist will be downloaded.
+## 3. Only if there are NO channels whitelisted, entrys listed as blacklist_channels
+##    will not be downloaded, all other channels / supergroups will be.
+##
+## In other words:
+## * Set download_channels and/or download_supergroups to true if you want to include
+##   those types in your backup.
+## * If you use neither black- nor whitelist, all channels (if you set download_channels
+##   to true) / supergroups (if you set download_supergroups to true) get downloaded.
+## * If you set a whitelist, only listed channels / supergroups (there is no distinction
+##   made here) will be loaded.
+## * If you set a blacklist, everything except the listed channels / supergroups (again,
+##   although the entry is called whitelist_channels it affects channels AND supergroups)
+##   will be loaded.
+## * If you set a whitelist AND a blacklist, the blacklist will be ignored.
+##
+## Call the app with `--list-channels` to list the available channels and supergroups
+## with their IDs. That list will also tell you if a channel / supergroup will be
+## be downloaded according to your black- and whitelists.
+##
+## You can have more than one whitelist_channels and/or blacklist_channels entries
+## to build your list. One ID per entry.
+
+# download_channels = false
+# download_supergroups = false
+
+# blacklist_channels = 12345678
+# blacklist_channels = 8886542
+# blacklist_channels = 715952
+
+# whitelist_channels = 238572935
+# whitelist_channels = 23857623