fabian
/
telegram_backup
mirror of https://github.com/fabianonline/telegram_backup.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'feature-settings'

This commit is contained in:
Fabian Schlenz 2018-03-13 06:32:06 +01:00
parent 077cbcebca dd99612bed
commit aec609e6c4
10 changed files with 293 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt
View File

@ -93,6 +93,10 @@ class CommandLineController {
throw RuntimeException("Account / User mismatch")
}
}
// Load the ini file.
IniSettings.load()
logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login)
if (CommandLineOptions.cmd_login) {
cmd_login(CommandLineOptions.val_account)
@ -131,14 +135,39 @@ class CommandLineController {
}
logger.info("Initializing Download Manager")
val d = DownloadManager(client, CommandLineDownloadProgress())
if (CommandLineOptions.cmd_list_channels) {
val chats = d.getChats()
val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) }
val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")}
var download: Boolean
println("Channels:")
download = IniSettings.download_channels
if (!download) println("Download of channels is disabled - see download_channels in config.ini")
print_header(download)
for (c in chats.channels) {
println(format(c, download))
}
println()
println("Supergroups:")
download = IniSettings.download_supergroups
if (!download) println("Download of supergroups is disabled - see download_supergroups in config.ini")
print_header(download)
for (c in chats.supergroups) {
println(format(c, download))
}
System.exit(0)
}
logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages)
d.downloadMessages(CommandLineOptions.val_limit_messages)
logger.debug("CommandLineOptions.cmd_no_media: {}", CommandLineOptions.cmd_no_media)
if (!CommandLineOptions.cmd_no_media) {
logger.debug("IniSettings.download_media: {}", IniSettings.download_media)
if (IniSettings.download_media) {
logger.debug("Calling DownloadManager.downloadMedia")
d.downloadMedia()
} else {
println("Skipping media download because --no-media is set.")
println("Skipping media download because download_media is set to false.")
}
} catch (e: Throwable) {
println("An error occured!")
@ -276,18 +305,14 @@ class CommandLineController {
println(" --trace-telegram Shows lots of debug messages from the library used to access Telegram.")
println(" -A, --list-accounts List all existing accounts ")
println(" --limit-messages <x> Downloads at most the most recent <x> messages.")
println(" --no-media Do not download media files.")
println(" -t, --target <x> Target directory for the files.")
println(" -e, --export <format> Export the database. Valid formats are:")
println(" html - Creates HTML files.")
println(" --pagination <x> Splits the HTML export into multiple HTML pages with <x> messages per page. Default is 5000.")
println(" --no-pagination Disables pagination.")
println(" --license Displays the license of this program.")
println(" -d, --daemon Keep running after the backup and automatically save new messages.")
println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.")
println(" --stats Print some usage statistics.")
println(" --with-channels Backup channels as well.")
println(" --with-supergroups Backup supergroups as well.")
println(" --list-channels Lists all channels together with their ID")
}
private fun list_accounts() {

12
src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt
View File

@ -26,18 +26,14 @@ internal object CommandLineOptions {
var cmd_version = false
var cmd_license = false
var cmd_daemon = false
var cmd_no_media = false
var cmd_anonymize = false
var cmd_stats = false
var cmd_channels = false
var cmd_supergroups = false
var cmd_no_pagination = false
var cmd_list_channels = false
var val_account: String? = null
var val_limit_messages: Int? = null
var val_target: String? = null
var val_export: String? = null
var val_test: Int? = null
var val_pagination: Int = Config.DEFAULT_PAGINATION
@JvmStatic
fun parseOptions(args: Array<String>) {
var last_cmd: String? = null
@ -49,7 +45,6 @@ internal object CommandLineOptions {
"--target" -> val_target = arg
"--export" -> val_export = arg
"--test" -> val_test = Integer.parseInt(arg)
"--pagination" -> val_pagination = Integer.parseInt(arg)
}
last_cmd = null
continue
@ -83,18 +78,15 @@ internal object CommandLineOptions {
last_cmd = "--pagination"
continue@loop
}
"--no-pagination" -> cmd_no_pagination = true
"--license" -> cmd_license = true
"-d", "--daemon" -> cmd_daemon = true
"--no-media" -> cmd_no_media = true
"--test" -> {
last_cmd = "--test"
continue@loop
}
"--anonymize" -> cmd_anonymize = true
"--stats" -> cmd_stats = true
"--with-channels" -> cmd_channels = true
"--with-supergroups" -> cmd_supergroups = true
"--list-channels" -> cmd_list_channels = true
else -> throw RuntimeException("Unknown command " + arg)
}
}

20
src/main/kotlin/de/fabianonline/telegram_backup/Database.kt
View File

@ -545,9 +545,25 @@ class Database private constructor(var client: TelegramClient) {
e.printStackTrace()
throw RuntimeException("Exception shown above happened.")
}
}
fun fetchSetting(key: String): String? {
val rs = stmt!!.executeQuery("SELECT value FROM settings WHERE key='${key}'")
rs.next()
return rs.getString(1)
}
fun saveSetting(key: String, value: String?) {
val ps = conn!!.prepareStatement("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)")
ps.setString(1, key)
if (value==null) {
ps.setNull(2, Types.VARCHAR)
} else {
ps.setString(2, value)
}
ps.execute()
}
fun getIdsFromQuery(query: String): LinkedList<Int> {
try {
val list = LinkedList<Int>()

11
src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt
View File

@ -32,6 +32,7 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
register(DB_Update_7(conn, db))
register(DB_Update_8(conn, db))
register(DB_Update_9(conn, db))
register(DB_Update_10(conn, db))
}
fun doUpdates() {
@ -440,3 +441,13 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
logger.info("Converted ${i} of ${count} messages.")
}
}
internal class DB_Update_10(conn: Connection, db: Database) : DatabaseUpdate(conn, db) {
override val version: Int
get() = 10
@Throws(SQLException::class)
override fun _doUpdate() {
execute("CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)")
}
}

12
src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt Normal file
View File

@ -0,0 +1,12 @@
package de.fabianonline.telegram_backup
class DbSettings() {
private fun fetchValue(name: String): String? = Database.getInstance().fetchSetting(name)
private fun saveValue(name: String, value: String?) = Database.getInstance().saveSetting(name, value)
var pts: String?
get() = fetchValue("pts")
set(x: String?) = saveValue("pts", x)
}

133
src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt
View File

@ -106,19 +106,14 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
fun _downloadMessages(limit: Int?) {
logger.info("This is _downloadMessages with limit {}", limit)
val dialog_limit = 100
logger.info("Downloading the last {} dialogs", dialog_limit)
logger.info("Downloading the last dialogs")
System.out.println("Downloading most recent dialogs... ")
var max_message_id = 0
val dialogs = client!!.messagesGetDialogs(
0,
0,
TLInputPeerEmpty(),
dialog_limit)
logger.debug("Got {} dialogs", dialogs.getDialogs().size)
val chats = getChats()
logger.debug("Got {} dialogs, {} supergoups, {} channels", chats.dialogs.size, chats.supergroups.size, chats.channels.size)
for (d in dialogs.getDialogs()) {
if (d.getTopMessage() > max_message_id && d.getPeer() !is TLPeerChannel) {
for (d in chats.dialogs) {
if (d.getTopMessage() > max_message_id) {
logger.trace("Updating top message id: {} => {}. Dialog type: {}", max_message_id, d.getTopMessage(), d.getPeer().javaClass)
max_message_id = d.getTopMessage()
}
@ -178,64 +173,33 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
}
*/
if (CommandLineOptions.cmd_channels || CommandLineOptions.cmd_supergroups) {
System.out.println("Processing channels and/or supergroups...")
System.out.println("Please note that only channels/supergroups in the last 100 active chats are processed.")
val channel_access_hashes = HashMap<Int, Long>()
val channel_names = HashMap<Int, String>()
val channels = LinkedList<Int>()
val supergroups = LinkedList<Int>()
if (IniSettings.download_channels || IniSettings.download_supergroups) {
// TODO Add chat title (and other stuff?) to the database
for (c in dialogs.getChats()) {
if (c is TLChannel) {
channel_access_hashes.put(c.getId(), c.getAccessHash())
channel_names.put(c.getId(), c.getTitle())
if (c.getMegagroup()) {
supergroups.add(c.getId())
} else {
channels.add(c.getId())
}
// Channel: TLChannel
// Supergroup: getMegagroup()==true
}
if (IniSettings.download_channels) {
println("Checking channels...")
for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) }
}
for (d in dialogs.getDialogs()) {
if (d.getPeer() is TLPeerChannel) {
val channel_id = (d.getPeer() as TLPeerChannel).getChannelId()
// If this is a channel and we don't want to download channels OR
// it is a supergroups and we don't want to download supergroups, then
if (channels.contains(channel_id) && !CommandLineOptions.cmd_channels || supergroups.contains(channel_id) && !CommandLineOptions.cmd_supergroups) {
// Skip this chat.
continue
}
val max_known_id = db!!.getTopMessageIDForChannel(channel_id)
if (d.getTopMessage() > max_known_id) {
val ids = makeIdList(max_known_id + 1, d.getTopMessage())
val access_hash = channel_access_hashes.get(channel_id) ?: throw RuntimeException("AccessHash for Channel missing.")
var channel_name = channel_names.get(channel_id)
if (channel_name == null) {
channel_name = "?"
}
val channel = TLInputChannel(channel_id, access_hash)
val source_type = if (supergroups.contains(channel_id)) {
MessageSource.SUPERGROUP
} else if (channels.contains(channel_id)) {
MessageSource.CHANNEL
} else {
throw RuntimeException("chat is neither in channels nor in supergroups...")
}
downloadMessages(ids, channel, source_type=source_type, source_name=channel_name)
}
}
if (IniSettings.download_supergroups) {
println("Checking supergroups...")
for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) }
}
}
}
private fun downloadMessagesFromChannel(channel: Channel) {
val obj = channel.obj
val max_known_id = db!!.getTopMessageIDForChannel(channel.id)
if (obj.getTopMessage() > max_known_id) {
val ids = makeIdList(max_known_id + 1, obj.getTopMessage())
var channel_name = channel.title
val input_channel = TLInputChannel(channel.id, channel.access_hash)
val source_type = channel.message_source
downloadMessages(ids, input_channel, source_type=source_type, source_name=channel_name)
}
}
@Throws(RpcErrorException::class, IOException::class)
private fun downloadMessages(ids: MutableList<Int>, channel: TLInputChannel?, source_type: MessageSource = MessageSource.NORMAL, source_name: String? = null) {
@ -385,6 +349,49 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
for (i in start..end) a.add(i)
return a
}
fun getChats(): ChatList {
val cl = ChatList()
logger.trace("Calling messagesGetDialogs")
val dialogs = client!!.messagesGetDialogs(0, 0, TLInputPeerEmpty(), 100)
logger.trace("Got {} dialogs back", dialogs.getDialogs().size)
// Add dialogs
cl.dialogs.addAll(dialogs.getDialogs().filter{it.getPeer() !is TLPeerChannel})
// Add supergoups and channels
for (tl_channel in dialogs.getChats().filter{it is TLChannel}.map{it as TLChannel}) {
val tl_peer_channel = dialogs.getDialogs().find{var p = it.getPeer() ; p is TLPeerChannel && p.getChannelId()==tl_channel.getId()}
if (tl_peer_channel == null) continue
var download = true
if (IniSettings.whitelist_channels != null) {
download = IniSettings.whitelist_channels!!.contains(tl_channel.getId().toString())
} else if (IniSettings.blacklist_channels != null) {
download = !IniSettings.blacklist_channels!!.contains(tl_channel.getId().toString())
}
val channel = Channel(id=tl_channel.getId(), access_hash=tl_channel.getAccessHash(), title=tl_channel.getTitle(), obj=tl_peer_channel, download=download)
if (tl_channel.getMegagroup()) {
channel.message_source = MessageSource.SUPERGROUP
cl.supergroups.add(channel)
} else {
channel.message_source = MessageSource.CHANNEL
cl.channels.add(channel)
}
}
return cl
}
class ChatList {
val dialogs = mutableListOf<TLDialog>()
val supergroups = mutableListOf<Channel>()
val channels = mutableListOf<Channel>()
}
class Channel(val id: Int, val access_hash: Long, val title: String, val obj: TLDialog, val download: Boolean) {
lateinit var message_source: MessageSource
}
companion object {
internal var download_client: TelegramClient? = null

83
src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt Normal file
View File

@ -0,0 +1,83 @@
package de.fabianonline.telegram_backup
import java.io.File
import org.slf4j.LoggerFactory
import org.slf4j.Logger
object IniSettings {
val logger = LoggerFactory.getLogger(IniSettings::class.java)
var settings = mutableMapOf<String, MutableList<String>>()
init {
loadIni(UserManager.getInstance().fileBase + "config.ini")
copySampleIni(UserManager.getInstance().fileBase + "config.sample.ini")
}
// Dummy function that can be called to force this object to run its init-code.
fun load() { }
private fun loadIni(filename: String) {
val file = File(filename)
logger.trace("Checking ini file {}", filename.anonymize())
if (!file.exists()) return
logger.debug("Loading ini file {}", filename.anonymize())
file.forEachLine { parseLine(it) }
}
private fun parseLine(original_line: String) {
logger.trace("Parsing line: {}", original_line)
var line = original_line.trim().replaceAfter("#", "").removeSuffix("#")
logger.trace("After cleaning: {}", line)
if (line == "") return
val parts: List<String> = line.split("=", limit=2).map{it.trim()}
if (parts.size < 2) throw RuntimeException("Invalid config setting: $line")
val (key, value) = parts
if (value=="") {
settings.remove(key)
} else {
var map = settings.get(key)
if (map == null) {
map = mutableListOf<String>()
settings.put(key, map)
}
map.add(value)
}
}
private fun copySampleIni(filename: String) {
val stream = Config::class.java.getResourceAsStream("/config.sample.ini")
File(filename).outputStream().use { stream.copyTo(it) }
stream.close()
}
fun println() = println(settings)
fun get(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default
fun getStringList(key: String): List<String>? = settings.get(key)
fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default
fun getBoolean(key: String, default: Boolean = false): Boolean {
val value = settings.get(key)?.last()
if (value==null) return default
return value=="true"
}
fun getArray(key: String): List<String> = settings.get(key) ?: listOf<String>()
val gmaps_key: String
get() = get("gmaps_key", default=Config.SECRET_GMAPS)!!
val pagination: Boolean
get() = getBoolean("pagination", default=true)
val pagination_size: Int
get() = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!!
val download_media: Boolean
get() = getBoolean("download_media", default=true)
val download_channels: Boolean
get() = getBoolean("download_channels", default=false)
val download_supergroups: Boolean
get() = getBoolean("download_supergroups", default=false)
val whitelist_channels: List<String>?
get() = getStringList("whitelist_channels")
val blacklist_channels: List<String>?
get() = getStringList("blacklist_channels")
}

9
src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt
View File

@ -16,12 +16,6 @@
package de.fabianonline.telegram_backup.exporter
import de.fabianonline.telegram_backup.UserManager
import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.anonymize
import de.fabianonline.telegram_backup.toPrettyJson
import de.fabianonline.telegram_backup.CommandLineOptions
import java.io.File
import java.io.PrintWriter
import java.io.OutputStreamWriter
@ -38,6 +32,7 @@ import java.util.HashMap
import com.github.mustachejava.DefaultMustacheFactory
import com.github.mustachejava.Mustache
import com.github.mustachejava.MustacheFactory
import de.fabianonline.telegram_backup.*
import org.slf4j.Logger
import org.slf4j.LoggerFactory
@ -49,7 +44,7 @@ class HTMLExporter {
@Throws(IOException::class)
fun export() {
try {
val pagination = if (CommandLineOptions.cmd_no_pagination) -1 else CommandLineOptions.val_pagination
val pagination = if (IniSettings.pagination) IniSettings.pagination_size else -1
// Create base dir
logger.debug("Creating base dir")

4
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt
View File

@ -21,7 +21,7 @@ import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.StickerConverter
import de.fabianonline.telegram_backup.DownloadProgressInterface
import de.fabianonline.telegram_backup.DownloadManager
import de.fabianonline.telegram_backup.Config
import de.fabianonline.telegram_backup.IniSettings
import com.github.badoualy.telegram.api.TelegramClient
import com.github.badoualy.telegram.tl.core.TLIntVector
@ -77,7 +77,7 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient)
"center=${geo.getLat()},${geo.getLong()}&" +
"markers=color:red|${geo.getLat()},${geo.getLong()}&" +
"zoom=14&size=300x150&scale=2&format=png&" +
"key=" + Config.SECRET_GMAPS
"key=" + IniSettings.gmaps_key
return DownloadManager.downloadExternalFile(targetPathAndFilename, url)
}
}

60
src/main/resources/config.sample.ini Normal file
View File

@ -0,0 +1,60 @@
# Config file for telegram_backup
# Copy it to config.ini to use it. This sample file be overwritten on every run.
#
# Lines starting with '#' are ignored.
# Settings have the form 'key = value'
# To unset a setting, use 'key =' (or just don't use that key at all)
# Some settings may appear more than once.
## GMaps key to use. Leave empty / don't set a value to use the built in key.
# gmaps_key = mysecretgmapskey
## Use pagination in the HTML export?
# pagination = true
# pagination_size = 5000
## Download media files
# download_media = true
## Downloads of channels and supergroups
## Here you can specify which channels and supergroups
## should be downloaded. The rules for this are:
## 1. Channels and supergroups are NEVER downloaded unless download_channels and/or
## download_supergroups is set to true.
## 2. If there is at least one entry called whitelist_channels, ONLY channels and/or
## supergroups that are listed in the whitelist will be downloaded.
## 3. Only if there are NO channels whitelisted, entrys listed as blacklist_channels
## will not be downloaded, all other channels / supergroups will be.
##
## In other words:
## * Set download_channels and/or download_supergroups to true if you want to include
## those types in your backup.
## * If you use neither black- nor whitelist, all channels (if you set download_channels
## to true) / supergroups (if you set download_supergroups to true) get downloaded.
## * If you set a whitelist, only listed channels / supergroups (there is no distinction
## made here) will be loaded.
## * If you set a blacklist, everything except the listed channels / supergroups (again,
## although the entry is called whitelist_channels it affects channels AND supergroups)
## will be loaded.
## * If you set a whitelist AND a blacklist, the blacklist will be ignored.
##
## Call the app with `--list-channels` to list the available channels and supergroups
## with their IDs. That list will also tell you if a channel / supergroup will be
## be downloaded according to your black- and whitelists.
##
## You can have more than one whitelist_channels and/or blacklist_channels entries
## to build your list. One ID per entry.
# download_channels = false
# download_supergroups = false
# blacklist_channels = 12345678
# blacklist_channels = 8886542
# blacklist_channels = 715952
# whitelist_channels = 238572935
# whitelist_channels = 23857623