From 25a01fae4bc032aa379c95719559043624b59b94 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 13 Mar 2018 06:30:39 +0100 Subject: [PATCH] You can now restrict the downloading of channels and supergroups by defining black- and whitelists in config.ini --- .../telegram_backup/DownloadManager.kt | 131 +++++++++--------- .../telegram_backup/IniSettings.kt | 11 +- src/main/resources/config.sample.ini | 39 +++++- 3 files changed, 117 insertions(+), 64 deletions(-) diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt index 489d95f..f3a8728 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt @@ -106,19 +106,14 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI @Throws(RpcErrorException::class, IOException::class, TimeoutException::class) fun _downloadMessages(limit: Int?) { logger.info("This is _downloadMessages with limit {}", limit) - val dialog_limit = 100 - logger.info("Downloading the last {} dialogs", dialog_limit) + logger.info("Downloading the last dialogs") System.out.println("Downloading most recent dialogs... ") var max_message_id = 0 - val dialogs = client!!.messagesGetDialogs( - 0, - 0, - TLInputPeerEmpty(), - dialog_limit) - logger.debug("Got {} dialogs", dialogs.getDialogs().size) + val chats = getChats() + logger.debug("Got {} dialogs, {} supergoups, {} channels", chats.dialogs.size, chats.supergroups.size, chats.channels.size) - for (d in dialogs.getDialogs()) { - if (d.getTopMessage() > max_message_id && d.getPeer() !is TLPeerChannel) { + for (d in chats.dialogs) { + if (d.getTopMessage() > max_message_id) { logger.trace("Updating top message id: {} => {}. Dialog type: {}", max_message_id, d.getTopMessage(), d.getPeer().javaClass) max_message_id = d.getTopMessage() } @@ -179,63 +174,32 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI */ if (IniSettings.download_channels || IniSettings.download_supergroups) { - System.out.println("Processing channels and/or supergroups...") - System.out.println("Please note that only channels/supergroups in the last 100 active chats are processed.") - - val channel_access_hashes = HashMap() - val channel_names = HashMap() - val channels = LinkedList() - val supergroups = LinkedList() - // TODO Add chat title (and other stuff?) to the database - for (c in dialogs.getChats()) { - if (c is TLChannel) { - channel_access_hashes.put(c.getId(), c.getAccessHash()) - channel_names.put(c.getId(), c.getTitle()) - if (c.getMegagroup()) { - supergroups.add(c.getId()) - } else { - channels.add(c.getId()) - } - // Channel: TLChannel - // Supergroup: getMegagroup()==true - } + + if (IniSettings.download_channels) { + println("Checking channels...") + for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) } } - - - - for (d in dialogs.getDialogs()) { - if (d.getPeer() is TLPeerChannel) { - val channel_id = (d.getPeer() as TLPeerChannel).getChannelId() - - // If this is a channel and we don't want to download channels OR - // it is a supergroups and we don't want to download supergroups, then - if (channels.contains(channel_id) && !IniSettings.download_channels || supergroups.contains(channel_id) && !IniSettings.download_supergroups) { - // Skip this chat. - continue - } - val max_known_id = db!!.getTopMessageIDForChannel(channel_id) - if (d.getTopMessage() > max_known_id) { - val ids = makeIdList(max_known_id + 1, d.getTopMessage()) - val access_hash = channel_access_hashes.get(channel_id) ?: throw RuntimeException("AccessHash for Channel missing.") - var channel_name = channel_names.get(channel_id) - if (channel_name == null) { - channel_name = "?" - } - val channel = TLInputChannel(channel_id, access_hash) - val source_type = if (supergroups.contains(channel_id)) { - MessageSource.SUPERGROUP - } else if (channels.contains(channel_id)) { - MessageSource.CHANNEL - } else { - throw RuntimeException("chat is neither in channels nor in supergroups...") - } - downloadMessages(ids, channel, source_type=source_type, source_name=channel_name) - } - } + + if (IniSettings.download_supergroups) { + println("Checking supergroups...") + for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) } } } } + + private fun downloadMessagesFromChannel(channel: Channel) { + val obj = channel.obj + val max_known_id = db!!.getTopMessageIDForChannel(channel.id) + if (obj.getTopMessage() > max_known_id) { + val ids = makeIdList(max_known_id + 1, obj.getTopMessage()) + var channel_name = channel.title + + val input_channel = TLInputChannel(channel.id, channel.access_hash) + val source_type = channel.message_source + downloadMessages(ids, input_channel, source_type=source_type, source_name=channel_name) + } + } @Throws(RpcErrorException::class, IOException::class) private fun downloadMessages(ids: MutableList, channel: TLInputChannel?, source_type: MessageSource = MessageSource.NORMAL, source_name: String? = null) { @@ -385,6 +349,49 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI for (i in start..end) a.add(i) return a } + + fun getChats(): ChatList { + val cl = ChatList() + logger.trace("Calling messagesGetDialogs") + val dialogs = client!!.messagesGetDialogs(0, 0, TLInputPeerEmpty(), 100) + logger.trace("Got {} dialogs back", dialogs.getDialogs().size) + + // Add dialogs + cl.dialogs.addAll(dialogs.getDialogs().filter{it.getPeer() !is TLPeerChannel}) + + // Add supergoups and channels + for (tl_channel in dialogs.getChats().filter{it is TLChannel}.map{it as TLChannel}) { + val tl_peer_channel = dialogs.getDialogs().find{var p = it.getPeer() ; p is TLPeerChannel && p.getChannelId()==tl_channel.getId()} + + if (tl_peer_channel == null) continue + + var download = true + if (IniSettings.whitelist_channels != null) { + download = IniSettings.whitelist_channels!!.contains(tl_channel.getId().toString()) + } else if (IniSettings.blacklist_channels != null) { + download = !IniSettings.blacklist_channels!!.contains(tl_channel.getId().toString()) + } + val channel = Channel(id=tl_channel.getId(), access_hash=tl_channel.getAccessHash(), title=tl_channel.getTitle(), obj=tl_peer_channel, download=download) + if (tl_channel.getMegagroup()) { + channel.message_source = MessageSource.SUPERGROUP + cl.supergroups.add(channel) + } else { + channel.message_source = MessageSource.CHANNEL + cl.channels.add(channel) + } + } + return cl + } + + class ChatList { + val dialogs = mutableListOf() + val supergroups = mutableListOf() + val channels = mutableListOf() + } + + class Channel(val id: Int, val access_hash: Long, val title: String, val obj: TLDialog, val download: Boolean) { + lateinit var message_source: MessageSource + } companion object { internal var download_client: TelegramClient? = null diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt index 8a37ef1..462793c 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt @@ -55,8 +55,13 @@ object IniSettings { fun println() = println(settings) fun get(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default + fun getStringList(key: String): List? = settings.get(key) fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default - fun getBoolean(key: String, default: Boolean = false): Boolean = settings.get(key)?.last() == "true" + fun getBoolean(key: String, default: Boolean = false): Boolean { + val value = settings.get(key)?.last() + if (value==null) return default + return value=="true" + } fun getArray(key: String): List = settings.get(key) ?: listOf() val gmaps_key: String @@ -71,4 +76,8 @@ object IniSettings { get() = getBoolean("download_channels", default=false) val download_supergroups: Boolean get() = getBoolean("download_supergroups", default=false) + val whitelist_channels: List? + get() = getStringList("whitelist_channels") + val blacklist_channels: List? + get() = getStringList("blacklist_channels") } diff --git a/src/main/resources/config.sample.ini b/src/main/resources/config.sample.ini index d7606ea..38e9c4c 100644 --- a/src/main/resources/config.sample.ini +++ b/src/main/resources/config.sample.ini @@ -18,6 +18,43 @@ ## Download media files # download_media = true -## Don't download supergroups and channels by default + + +## Downloads of channels and supergroups +## Here you can specify which channels and supergroups +## should be downloaded. The rules for this are: +## 1. Channels and supergroups are NEVER downloaded unless download_channels and/or +## download_supergroups is set to true. +## 2. If there is at least one entry called whitelist_channels, ONLY channels and/or +## supergroups that are listed in the whitelist will be downloaded. +## 3. Only if there are NO channels whitelisted, entrys listed as blacklist_channels +## will not be downloaded, all other channels / supergroups will be. +## +## In other words: +## * Set download_channels and/or download_supergroups to true if you want to include +## those types in your backup. +## * If you use neither black- nor whitelist, all channels (if you set download_channels +## to true) / supergroups (if you set download_supergroups to true) get downloaded. +## * If you set a whitelist, only listed channels / supergroups (there is no distinction +## made here) will be loaded. +## * If you set a blacklist, everything except the listed channels / supergroups (again, +## although the entry is called whitelist_channels it affects channels AND supergroups) +## will be loaded. +## * If you set a whitelist AND a blacklist, the blacklist will be ignored. +## +## Call the app with `--list-channels` to list the available channels and supergroups +## with their IDs. That list will also tell you if a channel / supergroup will be +## be downloaded according to your black- and whitelists. +## +## You can have more than one whitelist_channels and/or blacklist_channels entries +## to build your list. One ID per entry. + # download_channels = false # download_supergroups = false + +# blacklist_channels = 12345678 +# blacklist_channels = 8886542 +# blacklist_channels = 715952 + +# whitelist_channels = 238572935 +# whitelist_channels = 23857623