diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 70969bc..7e15981 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -93,6 +93,10 @@ class CommandLineController { throw RuntimeException("Account / User mismatch") } } + + // Load the ini file. + IniSettings.load() + logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login) if (CommandLineOptions.cmd_login) { cmd_login(CommandLineOptions.val_account) @@ -131,14 +135,39 @@ class CommandLineController { } logger.info("Initializing Download Manager") val d = DownloadManager(client, CommandLineDownloadProgress()) + + if (CommandLineOptions.cmd_list_channels) { + val chats = d.getChats() + val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) } + val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")} + var download: Boolean + + println("Channels:") + download = IniSettings.download_channels + if (!download) println("Download of channels is disabled - see download_channels in config.ini") + print_header(download) + for (c in chats.channels) { + println(format(c, download)) + } + println() + println("Supergroups:") + download = IniSettings.download_supergroups + if (!download) println("Download of supergroups is disabled - see download_supergroups in config.ini") + print_header(download) + for (c in chats.supergroups) { + println(format(c, download)) + } + System.exit(0) + } + logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages) d.downloadMessages(CommandLineOptions.val_limit_messages) - logger.debug("CommandLineOptions.cmd_no_media: {}", CommandLineOptions.cmd_no_media) - if (!CommandLineOptions.cmd_no_media) { + logger.debug("IniSettings.download_media: {}", IniSettings.download_media) + if (IniSettings.download_media) { logger.debug("Calling DownloadManager.downloadMedia") d.downloadMedia() } else { - println("Skipping media download because --no-media is set.") + println("Skipping media download because download_media is set to false.") } } catch (e: Throwable) { println("An error occured!") @@ -276,18 +305,14 @@ class CommandLineController { println(" --trace-telegram Shows lots of debug messages from the library used to access Telegram.") println(" -A, --list-accounts List all existing accounts ") println(" --limit-messages Downloads at most the most recent messages.") - println(" --no-media Do not download media files.") println(" -t, --target Target directory for the files.") println(" -e, --export Export the database. Valid formats are:") println(" html - Creates HTML files.") - println(" --pagination Splits the HTML export into multiple HTML pages with messages per page. Default is 5000.") - println(" --no-pagination Disables pagination.") println(" --license Displays the license of this program.") println(" -d, --daemon Keep running after the backup and automatically save new messages.") println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.") println(" --stats Print some usage statistics.") - println(" --with-channels Backup channels as well.") - println(" --with-supergroups Backup supergroups as well.") + println(" --list-channels Lists all channels together with their ID") } private fun list_accounts() { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index 32ba425..0997cf4 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -26,18 +26,14 @@ internal object CommandLineOptions { var cmd_version = false var cmd_license = false var cmd_daemon = false - var cmd_no_media = false var cmd_anonymize = false var cmd_stats = false - var cmd_channels = false - var cmd_supergroups = false - var cmd_no_pagination = false + var cmd_list_channels = false var val_account: String? = null var val_limit_messages: Int? = null var val_target: String? = null var val_export: String? = null var val_test: Int? = null - var val_pagination: Int = Config.DEFAULT_PAGINATION @JvmStatic fun parseOptions(args: Array) { var last_cmd: String? = null @@ -49,7 +45,6 @@ internal object CommandLineOptions { "--target" -> val_target = arg "--export" -> val_export = arg "--test" -> val_test = Integer.parseInt(arg) - "--pagination" -> val_pagination = Integer.parseInt(arg) } last_cmd = null continue @@ -83,18 +78,15 @@ internal object CommandLineOptions { last_cmd = "--pagination" continue@loop } - "--no-pagination" -> cmd_no_pagination = true "--license" -> cmd_license = true "-d", "--daemon" -> cmd_daemon = true - "--no-media" -> cmd_no_media = true "--test" -> { last_cmd = "--test" continue@loop } "--anonymize" -> cmd_anonymize = true "--stats" -> cmd_stats = true - "--with-channels" -> cmd_channels = true - "--with-supergroups" -> cmd_supergroups = true + "--list-channels" -> cmd_list_channels = true else -> throw RuntimeException("Unknown command " + arg) } } diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index 20cbda6..0a7d839 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -545,9 +545,25 @@ class Database private constructor(var client: TelegramClient) { e.printStackTrace() throw RuntimeException("Exception shown above happened.") } - } - + + fun fetchSetting(key: String): String? { + val rs = stmt!!.executeQuery("SELECT value FROM settings WHERE key='${key}'") + rs.next() + return rs.getString(1) + } + + fun saveSetting(key: String, value: String?) { + val ps = conn!!.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)") + ps.setString(1, key) + if (value==null) { + ps.setNull(2, Types.VARCHAR) + } else { + ps.setString(2, value) + } + ps.execute() + } + fun getIdsFromQuery(query: String): LinkedList { try { val list = LinkedList() diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt index d3b3d99..713670c 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt @@ -32,6 +32,7 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database register(DB_Update_7(conn, db)) register(DB_Update_8(conn, db)) register(DB_Update_9(conn, db)) + register(DB_Update_10(conn, db)) } fun doUpdates() { @@ -440,3 +441,13 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn logger.info("Converted ${i} of ${count} messages.") } } + +internal class DB_Update_10(conn: Connection, db: Database) : DatabaseUpdate(conn, db) { + override val version: Int + get() = 10 + + @Throws(SQLException::class) + override fun _doUpdate() { + execute("CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)") + } +} diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt new file mode 100644 index 0000000..7cc36fd --- /dev/null +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt @@ -0,0 +1,12 @@ +package de.fabianonline.telegram_backup + +class DbSettings() { + private fun fetchValue(name: String): String? = Database.getInstance().fetchSetting(name) + private fun saveValue(name: String, value: String?) = Database.getInstance().saveSetting(name, value) + + var pts: String? + get() = fetchValue("pts") + set(x: String?) = saveValue("pts", x) +} + + diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt index 9766ae4..b583212 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt @@ -106,19 +106,14 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI @Throws(RpcErrorException::class, IOException::class, TimeoutException::class) fun _downloadMessages(limit: Int?) { logger.info("This is _downloadMessages with limit {}", limit) - val dialog_limit = 100 - logger.info("Downloading the last {} dialogs", dialog_limit) + logger.info("Downloading the last dialogs") System.out.println("Downloading most recent dialogs... ") var max_message_id = 0 - val dialogs = client!!.messagesGetDialogs( - 0, - 0, - TLInputPeerEmpty(), - dialog_limit) - logger.debug("Got {} dialogs", dialogs.getDialogs().size) + val chats = getChats() + logger.debug("Got {} dialogs, {} supergoups, {} channels", chats.dialogs.size, chats.supergroups.size, chats.channels.size) - for (d in dialogs.getDialogs()) { - if (d.getTopMessage() > max_message_id && d.getPeer() !is TLPeerChannel) { + for (d in chats.dialogs) { + if (d.getTopMessage() > max_message_id) { logger.trace("Updating top message id: {} => {}. Dialog type: {}", max_message_id, d.getTopMessage(), d.getPeer().javaClass) max_message_id = d.getTopMessage() } @@ -178,64 +173,33 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI } */ - if (CommandLineOptions.cmd_channels || CommandLineOptions.cmd_supergroups) { - System.out.println("Processing channels and/or supergroups...") - System.out.println("Please note that only channels/supergroups in the last 100 active chats are processed.") - - val channel_access_hashes = HashMap() - val channel_names = HashMap() - val channels = LinkedList() - val supergroups = LinkedList() - + if (IniSettings.download_channels || IniSettings.download_supergroups) { // TODO Add chat title (and other stuff?) to the database - for (c in dialogs.getChats()) { - if (c is TLChannel) { - channel_access_hashes.put(c.getId(), c.getAccessHash()) - channel_names.put(c.getId(), c.getTitle()) - if (c.getMegagroup()) { - supergroups.add(c.getId()) - } else { - channels.add(c.getId()) - } - // Channel: TLChannel - // Supergroup: getMegagroup()==true - } + + if (IniSettings.download_channels) { + println("Checking channels...") + for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) } } - - - - for (d in dialogs.getDialogs()) { - if (d.getPeer() is TLPeerChannel) { - val channel_id = (d.getPeer() as TLPeerChannel).getChannelId() - - // If this is a channel and we don't want to download channels OR - // it is a supergroups and we don't want to download supergroups, then - if (channels.contains(channel_id) && !CommandLineOptions.cmd_channels || supergroups.contains(channel_id) && !CommandLineOptions.cmd_supergroups) { - // Skip this chat. - continue - } - val max_known_id = db!!.getTopMessageIDForChannel(channel_id) - if (d.getTopMessage() > max_known_id) { - val ids = makeIdList(max_known_id + 1, d.getTopMessage()) - val access_hash = channel_access_hashes.get(channel_id) ?: throw RuntimeException("AccessHash for Channel missing.") - var channel_name = channel_names.get(channel_id) - if (channel_name == null) { - channel_name = "?" - } - val channel = TLInputChannel(channel_id, access_hash) - val source_type = if (supergroups.contains(channel_id)) { - MessageSource.SUPERGROUP - } else if (channels.contains(channel_id)) { - MessageSource.CHANNEL - } else { - throw RuntimeException("chat is neither in channels nor in supergroups...") - } - downloadMessages(ids, channel, source_type=source_type, source_name=channel_name) - } - } + + if (IniSettings.download_supergroups) { + println("Checking supergroups...") + for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) } } } } + + private fun downloadMessagesFromChannel(channel: Channel) { + val obj = channel.obj + val max_known_id = db!!.getTopMessageIDForChannel(channel.id) + if (obj.getTopMessage() > max_known_id) { + val ids = makeIdList(max_known_id + 1, obj.getTopMessage()) + var channel_name = channel.title + + val input_channel = TLInputChannel(channel.id, channel.access_hash) + val source_type = channel.message_source + downloadMessages(ids, input_channel, source_type=source_type, source_name=channel_name) + } + } @Throws(RpcErrorException::class, IOException::class) private fun downloadMessages(ids: MutableList, channel: TLInputChannel?, source_type: MessageSource = MessageSource.NORMAL, source_name: String? = null) { @@ -385,6 +349,49 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI for (i in start..end) a.add(i) return a } + + fun getChats(): ChatList { + val cl = ChatList() + logger.trace("Calling messagesGetDialogs") + val dialogs = client!!.messagesGetDialogs(0, 0, TLInputPeerEmpty(), 100) + logger.trace("Got {} dialogs back", dialogs.getDialogs().size) + + // Add dialogs + cl.dialogs.addAll(dialogs.getDialogs().filter{it.getPeer() !is TLPeerChannel}) + + // Add supergoups and channels + for (tl_channel in dialogs.getChats().filter{it is TLChannel}.map{it as TLChannel}) { + val tl_peer_channel = dialogs.getDialogs().find{var p = it.getPeer() ; p is TLPeerChannel && p.getChannelId()==tl_channel.getId()} + + if (tl_peer_channel == null) continue + + var download = true + if (IniSettings.whitelist_channels != null) { + download = IniSettings.whitelist_channels!!.contains(tl_channel.getId().toString()) + } else if (IniSettings.blacklist_channels != null) { + download = !IniSettings.blacklist_channels!!.contains(tl_channel.getId().toString()) + } + val channel = Channel(id=tl_channel.getId(), access_hash=tl_channel.getAccessHash(), title=tl_channel.getTitle(), obj=tl_peer_channel, download=download) + if (tl_channel.getMegagroup()) { + channel.message_source = MessageSource.SUPERGROUP + cl.supergroups.add(channel) + } else { + channel.message_source = MessageSource.CHANNEL + cl.channels.add(channel) + } + } + return cl + } + + class ChatList { + val dialogs = mutableListOf() + val supergroups = mutableListOf() + val channels = mutableListOf() + } + + class Channel(val id: Int, val access_hash: Long, val title: String, val obj: TLDialog, val download: Boolean) { + lateinit var message_source: MessageSource + } companion object { internal var download_client: TelegramClient? = null diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt new file mode 100644 index 0000000..462793c --- /dev/null +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -0,0 +1,83 @@ +package de.fabianonline.telegram_backup + +import java.io.File +import org.slf4j.LoggerFactory +import org.slf4j.Logger + +object IniSettings { + val logger = LoggerFactory.getLogger(IniSettings::class.java) + var settings = mutableMapOf>() + + init { + loadIni(UserManager.getInstance().fileBase + "config.ini") + copySampleIni(UserManager.getInstance().fileBase + "config.sample.ini") + } + + // Dummy function that can be called to force this object to run its init-code. + fun load() { } + + private fun loadIni(filename: String) { + val file = File(filename) + logger.trace("Checking ini file {}", filename.anonymize()) + if (!file.exists()) return + logger.debug("Loading ini file {}", filename.anonymize()) + file.forEachLine { parseLine(it) } + } + + private fun parseLine(original_line: String) { + logger.trace("Parsing line: {}", original_line) + var line = original_line.trim().replaceAfter("#", "").removeSuffix("#") + logger.trace("After cleaning: {}", line) + if (line == "") return + val parts: List = line.split("=", limit=2).map{it.trim()} + + if (parts.size < 2) throw RuntimeException("Invalid config setting: $line") + + val (key, value) = parts + if (value=="") { + settings.remove(key) + } else { + var map = settings.get(key) + if (map == null) { + map = mutableListOf() + settings.put(key, map) + } + map.add(value) + } + } + + private fun copySampleIni(filename: String) { + val stream = Config::class.java.getResourceAsStream("/config.sample.ini") + File(filename).outputStream().use { stream.copyTo(it) } + stream.close() + } + + fun println() = println(settings) + + fun get(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default + fun getStringList(key: String): List? = settings.get(key) + fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default + fun getBoolean(key: String, default: Boolean = false): Boolean { + val value = settings.get(key)?.last() + if (value==null) return default + return value=="true" + } + fun getArray(key: String): List = settings.get(key) ?: listOf() + + val gmaps_key: String + get() = get("gmaps_key", default=Config.SECRET_GMAPS)!! + val pagination: Boolean + get() = getBoolean("pagination", default=true) + val pagination_size: Int + get() = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!! + val download_media: Boolean + get() = getBoolean("download_media", default=true) + val download_channels: Boolean + get() = getBoolean("download_channels", default=false) + val download_supergroups: Boolean + get() = getBoolean("download_supergroups", default=false) + val whitelist_channels: List? + get() = getStringList("whitelist_channels") + val blacklist_channels: List? + get() = getStringList("blacklist_channels") +} diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt b/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt index e4e8fae..1053946 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt @@ -16,12 +16,6 @@ package de.fabianonline.telegram_backup.exporter -import de.fabianonline.telegram_backup.UserManager -import de.fabianonline.telegram_backup.Database -import de.fabianonline.telegram_backup.anonymize -import de.fabianonline.telegram_backup.toPrettyJson -import de.fabianonline.telegram_backup.CommandLineOptions - import java.io.File import java.io.PrintWriter import java.io.OutputStreamWriter @@ -38,6 +32,7 @@ import java.util.HashMap import com.github.mustachejava.DefaultMustacheFactory import com.github.mustachejava.Mustache import com.github.mustachejava.MustacheFactory +import de.fabianonline.telegram_backup.* import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -49,7 +44,7 @@ class HTMLExporter { @Throws(IOException::class) fun export() { try { - val pagination = if (CommandLineOptions.cmd_no_pagination) -1 else CommandLineOptions.val_pagination + val pagination = if (IniSettings.pagination) IniSettings.pagination_size else -1 // Create base dir logger.debug("Creating base dir") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt index 41d1f1c..718761b 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt @@ -21,7 +21,7 @@ import de.fabianonline.telegram_backup.Database import de.fabianonline.telegram_backup.StickerConverter import de.fabianonline.telegram_backup.DownloadProgressInterface import de.fabianonline.telegram_backup.DownloadManager -import de.fabianonline.telegram_backup.Config +import de.fabianonline.telegram_backup.IniSettings import com.github.badoualy.telegram.api.TelegramClient import com.github.badoualy.telegram.tl.core.TLIntVector @@ -77,7 +77,7 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) "center=${geo.getLat()},${geo.getLong()}&" + "markers=color:red|${geo.getLat()},${geo.getLong()}&" + "zoom=14&size=300x150&scale=2&format=png&" + - "key=" + Config.SECRET_GMAPS + "key=" + IniSettings.gmaps_key return DownloadManager.downloadExternalFile(targetPathAndFilename, url) } } diff --git a/src/main/resources/config.sample.ini b/src/main/resources/config.sample.ini new file mode 100644 index 0000000..38e9c4c --- /dev/null +++ b/src/main/resources/config.sample.ini @@ -0,0 +1,60 @@ +# Config file for telegram_backup +# Copy it to config.ini to use it. This sample file be overwritten on every run. +# +# Lines starting with '#' are ignored. +# Settings have the form 'key = value' +# To unset a setting, use 'key =' (or just don't use that key at all) +# Some settings may appear more than once. + + + +## GMaps key to use. Leave empty / don't set a value to use the built in key. +# gmaps_key = mysecretgmapskey + +## Use pagination in the HTML export? +# pagination = true +# pagination_size = 5000 + +## Download media files +# download_media = true + + + +## Downloads of channels and supergroups +## Here you can specify which channels and supergroups +## should be downloaded. The rules for this are: +## 1. Channels and supergroups are NEVER downloaded unless download_channels and/or +## download_supergroups is set to true. +## 2. If there is at least one entry called whitelist_channels, ONLY channels and/or +## supergroups that are listed in the whitelist will be downloaded. +## 3. Only if there are NO channels whitelisted, entrys listed as blacklist_channels +## will not be downloaded, all other channels / supergroups will be. +## +## In other words: +## * Set download_channels and/or download_supergroups to true if you want to include +## those types in your backup. +## * If you use neither black- nor whitelist, all channels (if you set download_channels +## to true) / supergroups (if you set download_supergroups to true) get downloaded. +## * If you set a whitelist, only listed channels / supergroups (there is no distinction +## made here) will be loaded. +## * If you set a blacklist, everything except the listed channels / supergroups (again, +## although the entry is called whitelist_channels it affects channels AND supergroups) +## will be loaded. +## * If you set a whitelist AND a blacklist, the blacklist will be ignored. +## +## Call the app with `--list-channels` to list the available channels and supergroups +## with their IDs. That list will also tell you if a channel / supergroup will be +## be downloaded according to your black- and whitelists. +## +## You can have more than one whitelist_channels and/or blacklist_channels entries +## to build your list. One ID per entry. + +# download_channels = false +# download_supergroups = false + +# blacklist_channels = 12345678 +# blacklist_channels = 8886542 +# blacklist_channels = 715952 + +# whitelist_channels = 238572935 +# whitelist_channels = 23857623