fabian/telegram_backup
1
0
Fork 0
mirror of https://github.com/fabianonline/telegram_backup.git synced 2024-11-22 08:46:15 +00:00
Code Issues Releases Wiki Activity

Merge branch 'feature-rewrite'

Browse Source
This commit is contained in:
Fabian Schlenz 2018-04-11 06:11:07 +02:00
commit 1ff540977e
28 changed files with 1177 additions and 1657 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,4 +1,4 @@
#Thu Oct 06 11:24:39 CEST 2016
#Fri Mar 23 06:07:28 CET 2018
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME

5
src/main/kotlin/de/fabianonline/telegram_backup/Account.kt Normal file
View File

@ -0,0 +1,5 @@
package de.fabianonline.telegram_backup
class Account(val file_base: String, val phone_number: String) {
}

118
src/main/kotlin/de/fabianonline/telegram_backup/ApiStorage.kt
View File

@ -22,120 +22,64 @@ import com.github.badoualy.telegram.mtproto.auth.AuthKey
import com.github.badoualy.telegram.mtproto.model.MTSession
import org.apache.commons.io.FileUtils
import org.slf4j.LoggerFactory
import org.slf4j.Logger
import java.io.File
import java.io.FileNotFoundException
import java.io.IOException
internal class ApiStorage(prefix: String?) : TelegramApiStorage {
private var prefix: String? = null
private var do_save = false
private var auth_key: AuthKey? = null
private var dc: DataCenter? = null
private var file_auth_key: File? = null
private var file_dc: File? = null
internal class ApiStorage(val base_dir: String) : TelegramApiStorage {
var auth_key: AuthKey? = null
var dc: DataCenter? = null
val file_auth_key: File
val file_dc: File
val logger = LoggerFactory.getLogger(ApiStorage::class.java)
init {
this.setPrefix(prefix)
}
fun setPrefix(prefix: String?) {
this.prefix = prefix
this.do_save = this.prefix != null
if (this.do_save) {
val base = Config.FILE_BASE +
File.separatorChar +
this.prefix +
File.separatorChar
this.file_auth_key = File(base + Config.FILE_NAME_AUTH_KEY)
this.file_dc = File(base + Config.FILE_NAME_DC)
this._saveAuthKey()
this._saveDc()
} else {
this.file_auth_key = null
this.file_dc = null
}
file_auth_key = File(base_dir + Config.FILE_NAME_AUTH_KEY)
file_dc = File(base_dir + Config.FILE_NAME_DC)
}
override fun saveAuthKey(authKey: AuthKey) {
this.auth_key = authKey
this._saveAuthKey()
}
private fun _saveAuthKey() {
if (this.do_save && this.auth_key != null) {
try {
FileUtils.writeByteArrayToFile(this.file_auth_key, this.auth_key!!.key)
} catch (e: IOException) {
e.printStackTrace()
}
}
FileUtils.writeByteArrayToFile(file_auth_key, authKey.key)
}
override fun loadAuthKey(): AuthKey? {
if (this.auth_key != null) return this.auth_key
if (this.file_auth_key != null) {
try {
return AuthKey(FileUtils.readFileToByteArray(this.file_auth_key))
} catch (e: IOException) {
if (e !is FileNotFoundException) e.printStackTrace()
}
try {
return AuthKey(FileUtils.readFileToByteArray(file_auth_key))
} catch (e: FileNotFoundException) {
return null
}
return null
}
override fun saveDc(dataCenter: DataCenter) {
this.dc = dataCenter
this._saveDc()
}
private fun _saveDc() {
if (this.do_save && this.dc != null) {
try {
FileUtils.write(this.file_dc, this.dc!!.toString())
} catch (e: IOException) {
e.printStackTrace()
}
}
FileUtils.write(file_dc, dataCenter.toString())
}
override fun loadDc(): DataCenter? {
if (this.dc != null) return this.dc
if (this.file_dc != null) {
try {
val infos = FileUtils.readFileToString(this.file_dc).split(":")
return DataCenter(infos[0], Integer.parseInt(infos[1]))
} catch (e: IOException) {
if (e !is FileNotFoundException) e.printStackTrace()
}
try {
val infos = FileUtils.readFileToString(this.file_dc).split(":")
return DataCenter(infos[0], Integer.parseInt(infos[1]))
} catch (e: FileNotFoundException) {
return null
}
return null
}
override fun deleteAuthKey() {
if (this.do_save) {
try {
FileUtils.forceDelete(this.file_auth_key)
} catch (e: IOException) {
e.printStackTrace()
}
try {
FileUtils.forceDelete(file_auth_key)
} catch (e: IOException) {
logger.warn("Exception in deleteAuthKey(): {}", e)
}
}
override fun deleteDc() {
if (this.do_save) {
try {
FileUtils.forceDelete(this.file_dc)
} catch (e: IOException) {
e.printStackTrace()
}
try {
FileUtils.forceDelete(file_dc)
} catch (e: IOException) {
logger.warn("Exception in deleteDc(): {}", e)
}
}

344
src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt
View File

@ -29,121 +29,127 @@ import java.util.HashMap
import org.slf4j.LoggerFactory
import org.slf4j.Logger
class CommandLineController {
private val storage: ApiStorage
var app: TelegramApp
private fun getLine(): String {
if (System.console() != null) {
return System.console().readLine("> ")
} else {
print("> ")
return Scanner(System.`in`).nextLine()
}
}
private fun getPassword(): String {
if (System.console() != null) {
return String(System.console().readPassword("> "))
} else {
return getLine()
}
}
class CommandLineController(val options: CommandLineOptions) {
val logger = LoggerFactory.getLogger(CommandLineController::class.java)
init {
val storage: ApiStorage
val app: TelegramApp
val target_dir: String
val file_base: String
val phone_number: String
val handler: TelegramUpdateHandler
var client: TelegramClient
val user_manager: UserManager
val settings: Settings
val database: Database
logger.info("CommandLineController started. App version {}", Config.APP_APPVER)
this.printHeader()
if (CommandLineOptions.cmd_version) {
printHeader()
if (options.isSet("version")) {
System.exit(0)
} else if (CommandLineOptions.cmd_help) {
this.show_help()
} else if (options.isSet("help")) {
show_help()
System.exit(0)
} else if (CommandLineOptions.cmd_license) {
CommandLineController.show_license()
System.exit(0)
}
this.setupFileBase()
if (CommandLineOptions.cmd_list_accounts) {
this.list_accounts()
} else if (options.isSet("license")) {
show_license()
System.exit(0)
}
// Setup TelegramApp
logger.debug("Initializing TelegramApp")
app = TelegramApp(Config.APP_ID, Config.APP_HASH, Config.APP_MODEL, Config.APP_SYSVER, Config.APP_APPVER, Config.APP_LANG)
// Setup file_base
logger.debug("Target dir from Config: {}", Config.TARGET_DIR.anonymize())
target_dir = options.get("target") ?: Config.TARGET_DIR
logger.debug("Target dir after options: {}", target_dir)
println("Base directory for files: ${target_dir.anonymize()}")
if (options.isSet("list_accounts")) {
Utils.print_accounts(target_dir)
System.exit(0)
}
if (options.isSet("login")) {
cmd_login(app, target_dir, options.get("account"))
}
logger.trace("Checking accounts")
val account = this.selectAccount()
logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login)
phone_number = try { selectAccount(target_dir, options.get("account"))
} catch(e: AccountNotFoundException) {
show_error("The specified account could not be found.")
} catch(e: NoAccountsException) {
println("No accounts found. Starting login process...")
cmd_login(app, target_dir, options.get("account"))
}
// TODO: Create a new TelegramApp if the user set his/her own TelegramApp credentials
// At this point we can assume that the selected user account ("phone_number") exists.
// So we can create some objects:
file_base = build_file_base(target_dir, phone_number)
logger.info("Initializing ApiStorage")
storage = ApiStorage(account)
logger.info("Initializing TelegramUpdateHandler")
val handler = TelegramUpdateHandler()
storage = ApiStorage(file_base)
logger.info("Creating Client")
val client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, handler)
client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, null)
// From now on we have a new catch-all-block that will terminate it's TelegramClient when an exception happens.
try {
logger.info("Initializing UserManager")
UserManager.init(client)
val user = UserManager.getInstance()
if (!CommandLineOptions.cmd_login && !user.loggedIn) {
user_manager = UserManager(client)
// TODO
/*if (!options.cmd_login && !user.loggedIn) {
println("Your authorization data is invalid or missing. You will have to login with Telegram again.")
CommandLineOptions.cmd_login = true
}
if (account != null && user.loggedIn) {
if (account != "+" + user.user!!.getPhone()) {
logger.error("Account: {}, user.user!!.getPhone(): +{}", account.anonymize(), user.user!!.getPhone().anonymize())
throw RuntimeException("Account / User mismatch")
}
options.cmd_login = true
}*/
if (phone_number != user_manager.phone) {
logger.error("phone_number: {}, user_manager.phone: {}", phone_number.anonymize(), user_manager.phone.anonymize())
show_error("Account / User mismatch")
}
// Load the ini file.
IniSettings.load()
logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login)
if (CommandLineOptions.cmd_login) {
cmd_login(CommandLineOptions.val_account)
System.exit(0)
}
// If we reach this point, we can assume that there is an account and a database can be loaded / created.
Database.init(client)
if (CommandLineOptions.cmd_stats) {
cmd_stats()
database = Database(file_base, user_manager)
// Load the settings and stuff.
settings = Settings(file_base, database, options)
if (options.isSet("stats")) {
cmd_stats(file_base, database)
System.exit(0)
} else if (options.isSet("settings")) {
settings.print()
System.exit(0)
}
if (CommandLineOptions.val_test != null) {
if (CommandLineOptions.val_test == 1) {
TestFeatures.test1()
} else if (CommandLineOptions.val_test == 2) {
TestFeatures.test2()
} else {
System.out.println("Unknown test " + CommandLineOptions.val_test)
}
System.exit(1)
}
logger.debug("CommandLineOptions.val_export: {}", CommandLineOptions.val_export)
if (CommandLineOptions.val_export != null) {
if (CommandLineOptions.val_export!!.toLowerCase().equals("html")) {
(HTMLExporter()).export()
val export = options.get("export")
logger.debug("options.val_export: {}", export)
if (export != null) {
if (export.toLowerCase() == "html") {
HTMLExporter(database, user_manager, settings=settings, file_base=file_base).export()
System.exit(0)
} else {
show_error("Unknown export format.")
show_error("Unknown export format '${export}'.")
}
}
if (user.loggedIn) {
System.out.println("You are logged in as ${user.userString.anonymize()}")
} else {
println("You are not logged in.")
System.exit(1)
}
logger.info("Initializing Download Manager")
val d = DownloadManager(client, CommandLineDownloadProgress())
if (CommandLineOptions.cmd_list_channels) {
println("You are logged in as ${user_manager.toString().anonymize()}")
logger.info("Initializing Download Manager")
val d = DownloadManager(client, CommandLineDownloadProgress(), database, user_manager, settings, file_base)
if (options.isSet("list_channels")) {
val chats = d.getChats()
val print_header = {download: Boolean -> println("%-15s %-40s %s".format("ID", "Title", if (download) "Download" else "")); println("-".repeat(65)) }
val format = {c: DownloadManager.Channel, download: Boolean -> "%-15s %-40s %s".format(c.id.toString().anonymize(), c.title.anonymize(), if (download) (if(c.download) "YES" else "no") else "")}
var download: Boolean
println("Channels:")
download = IniSettings.download_channels
download = settings.download_channels
if (!download) println("Download of channels is disabled - see download_channels in config.ini")
print_header(download)
for (c in chats.channels) {
@ -151,7 +157,7 @@ class CommandLineController {
}
println()
println("Supergroups:")
download = IniSettings.download_supergroups
download = settings.download_supergroups
if (!download) println("Download of supergroups is disabled - see download_supergroups in config.ini")
print_header(download)
for (c in chats.supergroups) {
@ -160,31 +166,33 @@ class CommandLineController {
System.exit(0)
}
logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages)
d.downloadMessages(CommandLineOptions.val_limit_messages)
logger.debug("IniSettings.download_media: {}", IniSettings.download_media)
if (IniSettings.download_media) {
logger.debug("Calling DownloadManager.downloadMessages with limit {}", options.get("limit_messages"))
d.downloadMessages(options.get("limit_messages")?.toInt())
logger.debug("IniSettings#download_media: {}", settings.download_media)
if (settings.download_media) {
logger.debug("Calling DownloadManager.downloadMedia")
d.downloadMedia()
} else {
println("Skipping media download because download_media is set to false.")
}
if (options.isSet("daemon")) {
logger.info("Initializing TelegramUpdateHandler")
handler = TelegramUpdateHandler(user_manager, database, file_base, settings)
client.close()
logger.info("Creating new client")
client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, handler)
println("DAEMON mode requested - keeping running.")
}
} catch (e: Throwable) {
println("An error occured!")
println("An error occurred!")
e.printStackTrace()
logger.error("Exception caught!", e)
// If we encountered an exception, we definitely don't want to start the daemon mode now.
CommandLineOptions.cmd_daemon = false
} finally {
if (CommandLineOptions.cmd_daemon) {
handler.activate()
println("DAEMON mode requested - keeping running.")
} else {
client.close()
println()
println("----- EXIT -----")
System.exit(0)
}
client.close()
println()
println("----- EXIT -----")
System.exit(0)
}
}
@ -196,150 +204,102 @@ class CommandLineController {
println()
}
private fun setupFileBase() {
logger.debug("Target dir at startup: {}", Config.FILE_BASE.anonymize())
if (CommandLineOptions.val_target != null) {
Config.FILE_BASE = CommandLineOptions.val_target!!
}
logger.debug("Target dir after options: {}", Config.FILE_BASE.anonymize())
System.out.println("Base directory for files: " + Config.FILE_BASE.anonymize())
}
private fun selectAccount(): String? {
var account = "none"
val accounts = Utils.getAccounts()
if (CommandLineOptions.cmd_login) {
logger.debug("Login requested, doing nothing.")
// do nothing
} else if (CommandLineOptions.val_account != null) {
logger.debug("Account requested: {}", CommandLineOptions.val_account!!.anonymize())
private fun selectAccount(file_base: String, requested_account: String?): String {
var found_account: String?
val accounts = Utils.getAccounts(file_base)
if (requested_account != null) {
logger.debug("Account requested: {}", requested_account.anonymize())
logger.trace("Checking accounts for match.")
var found = false
for (acc in accounts) {
logger.trace("Checking {}", acc.anonymize())
if (acc == CommandLineOptions.val_account) {
found = true
logger.trace("Matches.")
break
}
}
if (!found) {
show_error("Couldn't find account '" + CommandLineOptions.val_account!!.anonymize() + "'. Maybe you want to use '--login' first?")
}
account = CommandLineOptions.val_account!!
found_account = accounts.find{it == requested_account}
} else if (accounts.size == 0) {
println("No accounts found. Starting login process...")
CommandLineOptions.cmd_login = true
return null
throw NoAccountsException()
} else if (accounts.size == 1) {
account = accounts.firstElement()
System.out.println("Using only available account: " + account.anonymize())
found_account = accounts.firstElement()
println("Using only available account: " + found_account.anonymize())
} else {
show_error(("You didn't specify which account to use.\n" +
show_error(("You have more than one account but didn't specify which one to use.\n" +
"Use '--account <x>' to use account <x>.\n" +
"Use '--list-accounts' to see all available accounts."))
System.exit(1)
}
if (found_account == null) {
throw AccountNotFoundException()
}
logger.debug("accounts.size: {}", accounts.size)
logger.debug("account: {}", account.anonymize())
return account
logger.debug("account: {}", found_account.anonymize())
return found_account
}
private fun cmd_stats() {
private fun cmd_stats(file_base: String, db: Database) {
println()
println("Stats:")
val format = "%40s: %d%n"
System.out.format(format, "Number of accounts", Utils.getAccounts().size)
System.out.format(format, "Number of messages", Database.getInstance().getMessageCount())
System.out.format(format, "Number of chats", Database.getInstance().getChatCount())
System.out.format(format, "Number of users", Database.getInstance().getUserCount())
System.out.format(format, "Top message ID", Database.getInstance().getTopMessageID())
System.out.format(format, "Number of accounts", Utils.getAccounts(file_base).size)
System.out.format(format, "Number of messages", db.getMessageCount())
System.out.format(format, "Number of chats", db.getChatCount())
System.out.format(format, "Number of users", db.getUserCount())
System.out.format(format, "Top message ID", db.getTopMessageID())
println()
println("Media Types:")
for ((key, value) in Database.getInstance().getMessageMediaTypesWithCount()) {
for ((key, value) in db.getMessageMediaTypesWithCount()) {
System.out.format(format, key, value)
}
println()
println("Api layers of messages:")
for ((key, value) in Database.getInstance().getMessageApiLayerWithCount()) {
for ((key, value) in db.getMessageApiLayerWithCount()) {
System.out.format(format, key, value)
}
println()
println("Message source types:")
for ((key, value) in Database.getInstance().getMessageSourceTypeWithCount()) {
for ((key, value) in db.getMessageSourceTypeWithCount()) {
System.out.format(format, key, value)
}
}
@Throws(RpcErrorException::class, IOException::class)
private fun cmd_login(phoneToUse: String?) {
val user = UserManager.getInstance()
val phone: String
if (phoneToUse == null) {
println("Please enter your phone number in international format.")
println("Example: +4917077651234")
phone = getLine()
} else {
phone = phoneToUse
}
user.sendCodeToPhoneNumber(phone)
println("Telegram sent you a code. Please enter it here.")
val code = getLine()
user.verifyCode(code)
if (user.isPasswordNeeded) {
println("We also need your account password. Please enter it now. It should not be printed, so it's okay if you see nothing while typing it.")
val pw = getPassword()
user.verifyPassword(pw)
}
storage.setPrefix("+" + user.user!!.getPhone())
System.out.println("Everything seems fine. Please run this tool again with '--account +" + user.user!!.getPhone().anonymize() + " to use this account.")
private fun cmd_login(app: TelegramApp, target_dir: String, phoneToUse: String?): Nothing {
LoginManager(app, target_dir, phoneToUse).run()
System.exit(0)
throw RuntimeException("Code never reaches this. This exists just to keep the Kotlin compiler happy.")
}
private fun show_help() {
println("Valid options are:")
println(" -h, --help Shows this help.")
println(" -a, --account <x> Use account <x>.")
println(" -l, --login Login to an existing telegram account.")
println(" --help Shows this help.")
println(" --account <x> Use account <x>.")
println(" --login Login to an existing telegram account.")
println(" --debug Shows some debug information.")
println(" --trace Shows lots of debug information. Overrides --debug.")
println(" --trace-telegram Shows lots of debug messages from the library used to access Telegram.")
println(" -A, --list-accounts List all existing accounts ")
println(" --list-accounts List all existing accounts ")
println(" --limit-messages <x> Downloads at most the most recent <x> messages.")
println(" -t, --target <x> Target directory for the files.")
println(" -e, --export <format> Export the database. Valid formats are:")
println(" --target <x> Target directory for the files.")
println(" --export <format> Export the database. Valid formats are:")
println(" html - Creates HTML files.")
println(" --license Displays the license of this program.")
println(" -d, --daemon Keep running after the backup and automatically save new messages.")
println(" --daemon Keep running after the backup and automatically save new messages.")
println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.")
println(" --stats Print some usage statistics.")
println(" --list-channels Lists all channels together with their ID")
}
private fun list_accounts() {
println("List of available accounts:")
val accounts = Utils.getAccounts()
if (accounts.size > 0) {
for (str in accounts) {
System.out.println(" " + str.anonymize())
}
println("Use '--account <x>' to use one of those accounts.")
} else {
println("NO ACCOUNTS FOUND")
println("Use '--login' to login to a telegram account.")
}
}
companion object {
private val logger = LoggerFactory.getLogger(CommandLineController::class.java)
public fun show_error(error: String) {
public fun show_error(error: String): Nothing {
logger.error(error)
println("ERROR: " + error)
System.exit(1)
throw RuntimeException("Code never reaches this. This exists just to keep the Kotlin compiler happy.")
}
fun show_license() {
println("TODO: Print the GPL.")
}
fun build_file_base(target_dir: String, account_to_use: String) = target_dir + File.separatorChar + account_to_use + File.separatorChar
}
class AccountNotFoundException() : Exception("Account not found") {}
class NoAccountsException() : Exception("No accounts found") {}
}

117
src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt
View File

@ -15,83 +15,52 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. */
package de.fabianonline.telegram_backup
internal object CommandLineOptions {
public var cmd_console = false
public var cmd_help = false
public var cmd_login = false
var cmd_debug = false
var cmd_trace = false
var cmd_trace_telegram = false
var cmd_list_accounts = false
var cmd_version = false
var cmd_license = false
var cmd_daemon = false
var cmd_anonymize = false
var cmd_stats = false
var cmd_list_channels = false
var val_account: String? = null
var val_limit_messages: Int? = null
var val_target: String? = null
var val_export: String? = null
var val_test: Int? = null
@JvmStatic
fun parseOptions(args: Array<String>) {
var last_cmd: String? = null
loop@ for (arg in args) {
if (last_cmd != null) {
when (last_cmd) {
"--account" -> val_account = arg
"--limit-messages" -> val_limit_messages = Integer.parseInt(arg)
"--target" -> val_target = arg
"--export" -> val_export = arg
"--test" -> val_test = Integer.parseInt(arg)
}
last_cmd = null
continue
class CommandLineOptions(args: Array<String>) {
private val values = mutableMapOf<String, String>()
var last_key: String? = null
val substitutions = mapOf("-t" to "--target")
init {
val list = args.toMutableList()
while (list.isNotEmpty()) {
var current_arg = list.removeAt(0)
if (!current_arg.startsWith("-")) throw RuntimeException("Unexpected unnamed parameter ${current_arg}")
var next_arg: String? = null
if (current_arg.contains("=")) {
val parts = current_arg.split("=", limit=2)
current_arg = parts[0]
next_arg = parts[1]
} else if (list.isNotEmpty() && !list[0].startsWith("--")) {
next_arg = list.removeAt(0)
}
when (arg) {
"-a", "--account" -> {
last_cmd = "--account"
continue@loop
if (!current_arg.startsWith("--") && current_arg.startsWith("-")) {
val replacement = substitutions.get(current_arg)
if (replacement == null) throw RuntimeException("Unknown short parameter ${current_arg}")
current_arg = replacement
}
current_arg = current_arg.substring(2)
if (next_arg == null) {
// current_arg seems to be a boolean value
values.put(current_arg, "true")
if (current_arg.startsWith("no-")) {
current_arg = current_arg.substring(3)
values.put(current_arg, "false")
}
"-h", "--help" -> cmd_help = true
"-l", "--login" -> cmd_login = true
"--debug" -> cmd_debug = true
"--trace" -> cmd_trace = true
"--trace-telegram" -> cmd_trace_telegram = true
"-A", "--list-accounts" -> cmd_list_accounts = true
"--limit-messages" -> {
last_cmd = arg
continue@loop
}
"--console" -> cmd_console = true
"-t", "--target" -> {
last_cmd = "--target"
continue@loop
}
"-V", "--version" -> cmd_version = true
"-e", "--export" -> {
last_cmd = "--export"
continue@loop
}
"--pagination" -> {
last_cmd = "--pagination"
continue@loop
}
"--license" -> cmd_license = true
"-d", "--daemon" -> cmd_daemon = true
"--test" -> {
last_cmd = "--test"
continue@loop
}
"--anonymize" -> cmd_anonymize = true
"--stats" -> cmd_stats = true
"--list-channels" -> cmd_list_channels = true
else -> throw RuntimeException("Unknown command " + arg)
} else {
// current_arg has the value next_arg
values.put(current_arg, next_arg)
}
}
if (last_cmd != null) {
CommandLineController.show_error("Command $last_cmd had no parameter set.")
}
println(values)
}
operator fun get(name: String): String? = values[name]
fun isSet(name: String): Boolean = values[name]=="true"
}

47
src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt
View File

@ -29,24 +29,35 @@ import ch.qos.logback.core.ConsoleAppender
import ch.qos.logback.classic.Level
fun main(args: Array<String>) {
CommandLineOptions.parseOptions(args)
CommandLineRunner.setupLogging()
CommandLineRunner.checkVersion()
if (true || CommandLineOptions.cmd_console) {
// Always use the console for now.
CommandLineController()
} else {
GUIController()
}
val clr = CommandLineRunner(args)
clr.setupLogging()
clr.checkVersion()
clr.run()
}
object CommandLineRunner {
class CommandLineRunner(args: Array<String>) {
val logger = LoggerFactory.getLogger(CommandLineRunner::class.java) as Logger
val options = CommandLineOptions(args)
fun run() {
// Always use the console for now.
try {
CommandLineController(options)
} catch (e: Throwable) {
println("An error occured!")
e.printStackTrace()
logger.error("Exception caught!", e)
System.exit(1)
}
}
fun setupLogging() {
val logger = LoggerFactory.getLogger(CommandLineRunner::class.java) as Logger
if (options.isSet("anonymize")) {
Utils.anonymize = true
}
logger.trace("Setting up Loggers...")
val rootLogger = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) as Logger
val rootContext = rootLogger.getLoggerContext()
@ -65,13 +76,13 @@ object CommandLineRunner {
rootLogger.addAppender(appender)
rootLogger.setLevel(Level.OFF)
if (CommandLineOptions.cmd_trace) {
if (options.isSet("trace")) {
(LoggerFactory.getLogger("de.fabianonline.telegram_backup") as Logger).setLevel(Level.TRACE)
} else if (CommandLineOptions.cmd_debug) {
} else if (options.isSet("debug")) {
(LoggerFactory.getLogger("de.fabianonline.telegram_backup") as Logger).setLevel(Level.DEBUG)
}
if (CommandLineOptions.cmd_trace_telegram) {
if (options.isSet("trace_telegram")) {
(LoggerFactory.getLogger("com.github.badoualy") as Logger).setLevel(Level.TRACE)
}
}

2
src/main/kotlin/de/fabianonline/telegram_backup/Config.kt
View File

@ -29,7 +29,7 @@ object Config {
val APP_APPVER: String
val APP_LANG = "en"
var FILE_BASE = System.getProperty("user.home") + File.separatorChar + ".telegram_backup"
var TARGET_DIR = System.getProperty("user.home") + File.separatorChar + ".telegram_backup"
val FILE_NAME_AUTH_KEY = "auth.dat"
val FILE_NAME_DC = "dc.dat"
val FILE_NAME_DB = "database.sqlite"

992
src/main/kotlin/de/fabianonline/telegram_backup/Database.kt
View File

File diff suppressed because it is too large Load Diff

4
src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt
View File

@ -313,7 +313,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn
} else {
ps.setInt(1, msg.getFwdFrom().getFromId())
}
val f = FileManagerFactory.getFileManager(msg, db.user_manager, db.client)
val f = FileManagerFactory.getFileManager(msg, db.user_manager, db.file_base, settings = null)
if (f == null) {
ps.setNull(2, Types.VARCHAR)
ps.setNull(3, Types.VARCHAR)
@ -431,7 +431,7 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
}
}
rs.close()
db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP)
db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP, settings=null)
execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")")
print(".")

12
src/main/kotlin/de/fabianonline/telegram_backup/DbSettings.kt
View File

@ -1,12 +0,0 @@
package de.fabianonline.telegram_backup
class DbSettings() {
private fun fetchValue(name: String): String? = Database.getInstance().fetchSetting(name)
private fun saveValue(name: String, value: String?) = Database.getInstance().saveSetting(name, value)
var pts: String?
get() = fetchValue("pts")
set(x: String?) = saveValue("pts", x)
}

268
src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt
View File

@ -18,7 +18,6 @@ package de.fabianonline.telegram_backup
import de.fabianonline.telegram_backup.UserManager
import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.StickerConverter
import de.fabianonline.telegram_backup.DownloadProgressInterface
import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory
import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager
@ -61,55 +60,21 @@ enum class MessageSource(val descr: String) {
SUPERGROUP("supergroup")
}
class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressInterface) {
internal var user: UserManager? = null
internal var db: Database? = null
internal var prog: DownloadProgressInterface? = null
internal var has_seen_flood_wait_message = false
init {
this.user = UserManager.getInstance()
this.prog = p
this.db = Database.getInstance()
}
class DownloadManager(val client: TelegramClient, val prog: DownloadProgressInterface, val db: Database, val user_manager: UserManager, val settings: Settings, val file_base: String) {
@Throws(RpcErrorException::class, IOException::class)
fun downloadMessages(limit: Int?) {
var completed: Boolean
do {
completed = true
try {
_downloadMessages(limit)
} catch (e: RpcErrorException) {
if (e.getCode() == 420) { // FLOOD_WAIT
completed = false
Utils.obeyFloodWaitException(e)
} else {
throw e
}
} catch (e: TimeoutException) {
completed = false
System.out.println("")
System.out.println("Telegram took too long to respond to our request.")
System.out.println("I'm going to wait a minute and then try again.")
try {
TimeUnit.MINUTES.sleep(1)
} catch (e2: InterruptedException) {
}
System.out.println("")
}
} while (!completed)
}
@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
fun _downloadMessages(limit: Int?) {
logger.info("This is _downloadMessages with limit {}", limit)
logger.info("This is downloadMessages with limit {}", limit)
logger.info("Downloading the last dialogs")
System.out.println("Downloading most recent dialogs... ")
var max_message_id = 0
val chats = getChats()
var result: ChatList? = null
Utils.obeyFloodWait() {
result = getChats()
}
val chats = result!!
logger.debug("Got {} dialogs, {} supergoups, {} channels", chats.dialogs.size, chats.supergroups.size, chats.channels.size)
for (d in chats.dialogs) {
@ -119,7 +84,7 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
}
}
System.out.println("Top message ID is " + max_message_id)
var max_database_id = db!!.getTopMessageID()
var max_database_id = db.getTopMessageID()
System.out.println("Top message ID in database is " + max_database_id)
if (limit != null) {
System.out.println("Limit is set to " + limit)
@ -136,7 +101,7 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
if (max_database_id == max_message_id) {
System.out.println("No new messages to download.")
} else if (max_database_id > max_message_id) {
throw RuntimeException("max_database_id is bigger then max_message_id. This shouldn't happen. But the telegram api nonetheless does that sometimes. Just ignore this error, wait a few seconds and then try again.")
throw RuntimeException("max_database_id is bigger than max_message_id. This shouldn't happen. But the telegram api nonetheless does that sometimes. Just ignore this error, wait a few seconds and then try again.")
} else {
val start_id = max_database_id + 1
val end_id = max_message_id
@ -147,50 +112,46 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
logger.info("Searching for missing messages in the db")
System.out.println("Checking message database for completeness...")
val db_count = db!!.getMessageCount()
val db_max = db!!.getTopMessageID()
val db_count = db.getMessageCount()
val db_max = db.getTopMessageID()
logger.debug("db_count: {}", db_count)
logger.debug("db_max: {}", db_max)
/*if (db_count != db_max) {
if (limit != null) {
System.out.println("You are missing messages in your database. But since you're using '--limit-messages', I won't download these now.");
} else {
LinkedList<Integer> all_missing_ids = db.getMissingIDs();
LinkedList<Integer> downloadable_missing_ids = new LinkedList<Integer>();
for (Integer id : all_missing_ids) {
if (id > max_message_id - 1000000) downloadable_missing_ids.add(id);
/*if (db_count != db_max) {
if (limit != null) {
System.out.println("You are missing messages in your database. But since you're using '--limit-messages', I won't download these now.");
} else {
LinkedList<Integer> all_missing_ids = db.getMissingIDs();
LinkedList<Integer> downloadable_missing_ids = new LinkedList<Integer>();
for (Integer id : all_missing_ids) {
if (id > max_message_id - 1000000) downloadable_missing_ids.add(id);
}
count_missing = all_missing_ids.size();
System.out.println("" + all_missing_ids.size() + " messages are missing in your Database.");
System.out.println("I can (and will) download " + downloadable_missing_ids.size() + " of them.");
downloadMessages(downloadable_missing_ids, null);
}
count_missing = all_missing_ids.size();
System.out.println("" + all_missing_ids.size() + " messages are missing in your Database.");
System.out.println("I can (and will) download " + downloadable_missing_ids.size() + " of them.");
downloadMessages(downloadable_missing_ids, null);
logger.info("Logging this run");
db.logRun(Math.min(max_database_id + 1, max_message_id), max_message_id, count_missing);
}
*/
logger.info("Logging this run");
db.logRun(Math.min(max_database_id + 1, max_message_id), max_message_id, count_missing);
if (settings.download_channels) {
println("Checking channels...")
for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) }
}
*/
if (IniSettings.download_channels || IniSettings.download_supergroups) {
// TODO Add chat title (and other stuff?) to the database
if (IniSettings.download_channels) {
println("Checking channels...")
for (channel in chats.channels) { if (channel.download) downloadMessagesFromChannel(channel) }
}
if (IniSettings.download_supergroups) {
println("Checking supergroups...")
for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) }
}
if (settings.download_supergroups) {
println("Checking supergroups...")
for (supergroup in chats.supergroups) { if (supergroup.download) downloadMessagesFromChannel(supergroup) }
}
}
private fun downloadMessagesFromChannel(channel: Channel) {
val obj = channel.obj
val max_known_id = db!!.getTopMessageIDForChannel(channel.id)
val max_known_id = db.getTopMessageIDForChannel(channel.id)
if (obj.getTopMessage() > max_known_id) {
val ids = makeIdList(max_known_id + 1, obj.getTopMessage())
var channel_name = channel.title
@ -210,7 +171,7 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
} else {
"${source_type.descr} $source_name"
}
prog!!.onMessageDownloadStart(ids.size, source_string)
prog.onMessageDownloadStart(ids.size, source_string)
logger.debug("Entering download loop")
while (ids.size > 0) {
@ -225,105 +186,62 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
logger.trace("vector.size(): {}", vector.size)
logger.trace("ids.size(): {}", ids.size)
var response: TLAbsMessages
var tries = 0
while (true) {
logger.trace("Trying getMessages(), tries={}", tries)
if (tries >= 5) {
CommandLineController.show_error("Couldn't getMessages after 5 tries. Quitting.")
}
tries++
try {
var resp: TLAbsMessages? = null
try {
Utils.obeyFloodWait(max_tries=5) {
if (channel == null) {
response = client!!.messagesGetMessages(vector)
resp = client.messagesGetMessages(vector)
} else {
response = client!!.channelsGetMessages(channel, vector)
}
break
} catch (e: RpcErrorException) {
if (e.getCode() == 420) { // FLOOD_WAIT
Utils.obeyFloodWaitException(e, has_seen_flood_wait_message)
has_seen_flood_wait_message = true
} else {
throw e
resp = client.channelsGetMessages(channel, vector)
}
}
} catch (e: MaxTriesExceededException) {
CommandLineController.show_error("Couldn't getMessages after 5 tries. Quitting.")
}
val response = resp!!
logger.trace("response.getMessages().size(): {}", response.getMessages().size)
if (response.getMessages().size != vector.size) {
CommandLineController.show_error("Requested ${vector.size} messages, but got ${response.getMessages().size}. That is unexpected. Quitting.")
}
prog!!.onMessageDownloaded(response.getMessages().size)
db!!.saveMessages(response.getMessages(), Kotlogram.API_LAYER, source_type=source_type)
db!!.saveChats(response.getChats())
db!!.saveUsers(response.getUsers())
prog.onMessageDownloaded(response.getMessages().size)
db.saveMessages(response.getMessages(), Kotlogram.API_LAYER, source_type=source_type, settings=settings)
db.saveChats(response.getChats())
db.saveUsers(response.getUsers())
logger.trace("Sleeping")
try {
TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_MESSAGES)
} catch (e: InterruptedException) {
}
try { TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_MESSAGES) } catch (e: InterruptedException) { }
}
logger.debug("Finished.")
prog!!.onMessageDownloadFinished()
prog.onMessageDownloadFinished()
}
@Throws(RpcErrorException::class, IOException::class)
fun downloadMedia() {
download_client = client!!.getDownloaderClient()
var completed: Boolean
do {
completed = true
try {
_downloadMedia()
} catch (e: RpcErrorException) {
if (e.getCode() == 420) { // FLOOD_WAIT
completed = false
Utils.obeyFloodWaitException(e)
} else {
throw e
}
}
/*catch (TimeoutException e) {
completed = false;
System.out.println("");
System.out.println("Telegram took too long to respond to our request.");
System.out.println("I'm going to wait a minute and then try again.");
logger.warn("TimeoutException caught", e);
try { TimeUnit.MINUTES.sleep(1); } catch(InterruptedException e2) {}
System.out.println("");
}*/
} while (!completed)
}
@Throws(RpcErrorException::class, IOException::class)
private fun _downloadMedia() {
download_client = client.getDownloaderClient()
logger.info("This is _downloadMedia")
logger.info("Checking if there are messages in the DB with a too old API layer")
val ids = db!!.getIdsFromQuery("SELECT id FROM messages WHERE has_media=1 AND api_layer<" + Kotlogram.API_LAYER)
val ids = db.getIdsFromQuery("SELECT id FROM messages WHERE has_media=1 AND api_layer<" + Kotlogram.API_LAYER)
if (ids.size > 0) {
System.out.println("You have ${ids.size} messages in your db that need an update. Doing that now.")
logger.debug("Found {} messages", ids.size)
downloadMessages(ids, null, source_type=MessageSource.NORMAL)
}
val message_count = db.getMessagesWithMediaCount()
prog.onMediaDownloadStart(message_count)
var offset = 0
val limit = 1000
val message_count = this.db!!.getMessagesWithMediaCount()
prog!!.onMediaDownloadStart(message_count)
while (true) {
logger.debug("Querying messages with media, limit={}, offset={}", limit, offset)
val messages = this.db!!.getMessagesWithMedia(limit=limit, offset=offset)
val messages = db.getMessagesWithMedia(limit, offset)
if (messages.size == 0) break
offset += limit
logger.debug("Database returned {} messages with media", messages.size)
prog.onMediaDownloadStart(messages.size)
for (msg in messages) {
if (msg == null) continue
val m = FileManagerFactory.getFileManager(msg, user!!, client!!)
val m = FileManagerFactory.getFileManager(msg, user_manager, file_base, settings=settings)
logger.trace("message {}, {}, {}, {}, {}",
msg.getId(),
msg.getMedia().javaClass.getSimpleName().replace("TLMessageMedia", "…"),
@ -331,27 +249,27 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
if (m.isEmpty) "empty" else "non-empty",
if (m.downloaded) "downloaded" else "not downloaded")
if (m.isEmpty) {
prog!!.onMediaDownloadedEmpty()
prog.onMediaDownloadedEmpty()
} else if (m.downloaded) {
prog!!.onMediaAlreadyPresent(m)
} else if (IniSettings.max_file_age!=null && (System.currentTimeMillis() / 1000) - msg.date > IniSettings.max_file_age * 24 * 60 * 60) {
prog!!.onMediaTooOld()
prog.onMediaAlreadyPresent(m)
} else if (settings.max_file_age>0 && (System.currentTimeMillis() / 1000) - msg.date > settings.max_file_age * 24 * 60 * 60) {
prog.onMediaTooOld()
} else {
try {
val result = m.download()
if (result) {
prog!!.onMediaDownloaded(m)
prog.onMediaDownloaded(m)
} else {
prog!!.onMediaSkipped()
prog.onMediaSkipped()
}
} catch (e: TimeoutException) {
// do nothing - skip this file
prog!!.onMediaSkipped()
prog.onMediaSkipped()
}
}
}
}
prog!!.onMediaDownloadFinished()
prog.onMediaDownloadFinished()
}
private fun makeIdList(start: Int, end: Int): MutableList<Int> {
@ -363,7 +281,7 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
fun getChats(): ChatList {
val cl = ChatList()
logger.trace("Calling messagesGetDialogs")
val dialogs = client!!.messagesGetDialogs(0, 0, TLInputPeerEmpty(), 100)
val dialogs = client.messagesGetDialogs(0, 0, TLInputPeerEmpty(), 100)
logger.trace("Got {} dialogs back", dialogs.getDialogs().size)
// Add dialogs
@ -376,10 +294,10 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
if (tl_peer_channel == null) continue
var download = true
if (IniSettings.whitelist_channels != null) {
download = IniSettings.whitelist_channels!!.contains(tl_channel.getId().toString())
} else if (IniSettings.blacklist_channels != null) {
download = !IniSettings.blacklist_channels!!.contains(tl_channel.getId().toString())
if (settings.whitelist_channels.isNotEmpty()) {
download = settings.whitelist_channels.contains(tl_channel.getId().toString())
} else if (settings.blacklist_channels.isNotEmpty()) {
download = !settings.blacklist_channels.contains(tl_channel.getId().toString())
}
val channel = Channel(id=tl_channel.getId(), access_hash=tl_channel.getAccessHash(), title=tl_channel.getTitle(), obj=tl_peer_channel, download=download)
if (tl_channel.getMegagroup()) {
@ -440,38 +358,32 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
}
logger.trace("offset before the loop is {}", offset)
fos = FileOutputStream(temp_filename, true)
var response: TLFile? = null
var try_again: Boolean
do {
try_again = false
logger.trace("offset: {} block_size: {} size: {}", offset, size, size)
val req = TLRequestUploadGetFile(loc, offset, size)
var resp: TLFile? = null
try {
response = download_client!!.executeRpcQuery(req, dcID) as TLFile
} catch (e: RpcErrorException) {
if (e.getCode() == 420) { // FLOOD_WAIT
try_again = true
Utils.obeyFloodWaitException(e)
continue // response is null since we didn't actually receive any data. Skip the rest of this iteration and try again.
} else if (e.getCode() == 400) {
//Somehow this file is broken. No idea why. Let's skip it for now
return false
} else {
throw e
Utils.obeyFloodWait() {
resp = download_client!!.executeRpcQuery(req, dcID) as TLFile
}
} catch (e: RpcErrorException) {
if (e.getCode() == 400) {
// Somehow this file is broken. No idea why. Let's skip it for now.
return false
}
throw e
}
val response = resp!!
offset += response.getBytes().getData().size
logger.trace("response: {} total size: {}", response.getBytes().getData().size, offset)
fos.write(response.getBytes().getData())
fos.flush()
try {
TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_FILE)
} catch (e: InterruptedException) {
}
try { TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_FILE) } catch (e: InterruptedException) { }
} while (offset < size && (try_again || response!!.getBytes().getData().size > 0))
} while (offset < size && response.getBytes().getData().size > 0)
fos.close()
if (offset < size) {
System.out.println("Requested file $target with $size bytes, but got only $offset bytes.")

85
src/main/kotlin/de/fabianonline/telegram_backup/GUIController.kt
View File

@ -1,85 +0,0 @@
/* Telegram_Backup
* Copyright (C) 2016 Fabian Schlenz
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. */
package de.fabianonline.telegram_backup
import javax.swing.*
import javax.swing.event.ListSelectionEvent
import javax.swing.event.ListSelectionListener
import java.awt.*
import java.awt.event.ActionEvent
import java.awt.event.ActionListener
import java.util.Vector
class GUIController() {
init {
showAccountChooserDialog()
}
private fun showAccountChooserDialog() {
val accountChooser = JDialog()
accountChooser.setTitle("Choose account")
accountChooser.setSize(400, 200)
val vert = JPanel()
vert.setLayout(BorderLayout())
vert.add(JLabel("Please select the account to use or create a new one."), BorderLayout.NORTH)
val accounts = Utils.getAccounts()
val list = JList<String>(accounts)
list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION)
vert.add(list, BorderLayout.CENTER)
val bottom = JPanel(GridLayout(1, 2))
val btnAddAccount = JButton("Add account")
bottom.add(btnAddAccount)
val btnLogin = JButton("Login")
btnLogin.setEnabled(false)
bottom.add(btnLogin)
vert.add(bottom, BorderLayout.SOUTH)
accountChooser.add(vert)
accountChooser.setVisible(true)
accountChooser.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE)
}
private fun addAccountDialog() {
val loginDialog = JDialog()
loginDialog.setTitle("Add an account")
loginDialog.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE)
val sections = JPanel()
sections.setLayout(BoxLayout(sections, BoxLayout.Y_AXIS))
val top = JPanel()
top.setLayout(BoxLayout(top, BoxLayout.Y_AXIS))
top.add(JLabel("Please enter your phone number in international format:"))
top.add(JTextField("+49123773212"))
sections.add(top)
sections.add(Box.createVerticalStrut(5))
sections.add(JSeparator(SwingConstants.HORIZONTAL))
val middle = JPanel()
middle.setLayout(BoxLayout(middle, BoxLayout.Y_AXIS))
middle.add(JLabel("Telegram sent you a code. Enter it here:"))
middle.add(JTextField())
middle.setEnabled(false)
sections.add(middle)
sections.add(Box.createVerticalStrut(5))
sections.add(JSeparator(SwingConstants.HORIZONTAL))
loginDialog.add(sections)
loginDialog.setVisible(true)
}
}

86
src/main/kotlin/de/fabianonline/telegram_backup/IniSettings.kt
View File

@ -1,86 +0,0 @@
package de.fabianonline.telegram_backup
import java.io.File
import org.slf4j.LoggerFactory
import org.slf4j.Logger
object IniSettings {
val logger = LoggerFactory.getLogger(IniSettings::class.java)
var settings = mutableMapOf<String, MutableList<String>>()
init {
if (UserManager.getInstance().user != null) {
loadIni(UserManager.getInstance().fileBase + "config.ini")
copySampleIni(UserManager.getInstance().fileBase + "config.sample.ini")
}
}
// Dummy function that can be called to force this object to run its init-code.
fun load() { }
private fun loadIni(filename: String) {
val file = File(filename)
logger.trace("Checking ini file {}", filename.anonymize())
if (!file.exists()) return
logger.debug("Loading ini file {}", filename.anonymize())
file.forEachLine { parseLine(it) }
}
private fun parseLine(original_line: String) {
logger.trace("Parsing line: {}", original_line)
var line = original_line.trim().replaceAfter("#", "").removeSuffix("#")
logger.trace("After cleaning: {}", line)
if (line == "") return
val parts: List<String> = line.split("=", limit=2).map{it.trim()}
if (parts.size < 2) throw RuntimeException("Invalid config setting: $line")
val (key, value) = parts
if (value=="") {
settings.remove(key)
} else {
var map = settings.get(key)
if (map == null) {
map = mutableListOf<String>()
settings.put(key, map)
}
map.add(value)
}
}
private fun copySampleIni(filename: String) {
val stream = Config::class.java.getResourceAsStream("/config.sample.ini")
File(filename).outputStream().use { stream.copyTo(it) }
stream.close()
}
fun println() = println(settings)
fun getString(key: String, default: String? = null): String? = settings.get(key)?.last() ?: default
fun getStringList(key: String): List<String>? = settings.get(key)
fun getInt(key: String, default: Int? = null): Int? = try { settings.get(key)?.last()?.toInt() } catch (e: NumberFormatException) { null } ?: default
fun getBoolean(key: String, default: Boolean = false): Boolean {
val value = settings.get(key)?.last()
if (value==null) return default
return value=="true"
}
fun getArray(key: String): List<String> = settings.get(key) ?: listOf<String>()
val gmaps_key: String
get() = getString("gmaps_key", default=Config.SECRET_GMAPS)!!
val pagination: Boolean
get() = getBoolean("pagination", default=true)
val pagination_size: Int
get() = getInt("pagination_size", default=Config.DEFAULT_PAGINATION)!!
val download_media: Boolean
get() = getBoolean("download_media", default=true)
val download_channels: Boolean
get() = getBoolean("download_channels", default=false)
val download_supergroups: Boolean
get() = getBoolean("download_supergroups", default=false)
val whitelist_channels: List<String>?
get() = getStringList("whitelist_channels")
val blacklist_channels: List<String>?
get() = getStringList("blacklist_channels")
val max_file_age = getInt("max_file_age")
}

96
src/main/kotlin/de/fabianonline/telegram_backup/LoginManager.kt Normal file
View File

@ -0,0 +1,96 @@
package de.fabianonline.telegram_backup
import com.github.badoualy.telegram.api.Kotlogram
import com.github.badoualy.telegram.api.TelegramApp
import com.github.badoualy.telegram.api.TelegramClient
import com.github.badoualy.telegram.tl.api.TLUser
import com.github.badoualy.telegram.tl.api.account.TLPassword
import com.github.badoualy.telegram.tl.api.auth.TLSentCode
import com.github.badoualy.telegram.tl.core.TLBytes
import com.github.badoualy.telegram.tl.exception.RpcErrorException
import java.security.MessageDigest
import java.util.*
class LoginManager(val app: TelegramApp, val target_dir: String, val phoneToUse: String?) {
fun run() {
var phone: String
if (phoneToUse == null) {
println("Please enter your phone number in international format.")
println("Example: +4917077651234")
phone = getLine()
} else {
phone = phoneToUse
}
val file_base = CommandLineController.build_file_base(target_dir, phone)
// We now have an account, so we can create an ApiStorage and TelegramClient.
val storage = ApiStorage(file_base)
val client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, null)
val sent_code = send_code_to_phone_number(client, phone)
println("Telegram sent you a code. Please enter it here.")
val code = getLine()
try {
verify_code(client, phone, sent_code, code)
} catch(e: PasswordNeededException) {
println("We also need your account password. Please enter it now. It should not be printed, so it's okay if you see nothing while typing it.")
val pw = getPassword()
verify_password(client, pw)
}
System.out.println("Everything seems fine. Please run this tool again with '--account ${phone} to use this account.")
}
private fun send_code_to_phone_number(client: TelegramClient, phone: String): TLSentCode {
return client.authSendCode(false, phone, true)
}
private fun verify_code(client: TelegramClient, phone: String, sent_code: TLSentCode, code: String): TLUser {
try {
val auth = client.authSignIn(phone, sent_code.getPhoneCodeHash(), code)
return auth.getUser().getAsUser()
} catch (e: RpcErrorException) {
if (e.getCode() == 401 && e.getTag()=="SESSION_PASSWORD_NEEDED") {
throw PasswordNeededException()
} else {
throw e
}
}
}
private fun verify_password(client: TelegramClient, password: String): TLUser {
val pw = password.toByteArray(charset = Charsets.UTF_8)
val salt = (client.accountGetPassword() as TLPassword).getCurrentSalt().getData()
val md = MessageDigest.getInstance("SHA-256")
val salted = ByteArray(2 * salt.size + pw.size)
System.arraycopy(salt, 0, salted, 0, salt.size)
System.arraycopy(pw, 0, salted, salt.size, pw.size)
System.arraycopy(salt, 0, salted, salt.size + pw.size, salt.size)
val hash = md.digest(salted)
val auth = client.authCheckPassword(TLBytes(hash))
return auth.getUser().getAsUser()
}
private fun getLine(): String {
if (System.console() != null) {
return System.console().readLine("> ")
} else {
print("> ")
return Scanner(System.`in`).nextLine()
}
}
private fun getPassword(): String {
if (System.console() != null) {
return String(System.console().readPassword("> "))
} else {
return getLine()
}
}
}
class PasswordNeededException: Exception("A password is needed to be able to login to this account.") {}

135
src/main/kotlin/de/fabianonline/telegram_backup/Settings.kt Normal file
View File

@ -0,0 +1,135 @@
package de.fabianonline.telegram_backup
import java.io.File
import java.util.LinkedList
import org.slf4j.LoggerFactory
class Settings(val file_base: String, val database: Database, val cli_settings: CommandLineOptions) {
val logger = LoggerFactory.getLogger(Settings::class.java)
private val db_settings: Map<String, String>
val ini_settings: Map<String, List<String>>
init {
db_settings = database.fetchSettings()
ini_settings = load_ini("config.ini")
copy_sample_ini("config.sample.ini")
}
// Merging CLI and INI settings
val sf = SettingsFactory(ini_settings, cli_settings)
val gmaps_key = sf.getString("gmaps_key", default=Config.SECRET_GMAPS, secret=true)
val pagination = sf.getBoolean("pagination", default=true)
val pagination_size = sf.getInt("pagination_size", default=Config.DEFAULT_PAGINATION)
val download_media = sf.getBoolean("download_media", default=true)
val download_channels = sf.getBoolean("download_channels", default=false)
val download_supergroups = sf.getBoolean("download_supergroups", default=false)
val whitelist_channels = sf.getStringList("whitelist_channels", default=LinkedList<String>())
val blacklist_channels = sf.getStringList("blacklist_channels", default=LinkedList<String>())
val max_file_age = sf.getInt("max_file_age", default=-1)
private fun get_setting_list(name: String): List<String>? {
return ini_settings[name]
}
private fun load_ini(filename: String): Map<String, List<String>> {
val map = mutableMapOf<String, MutableList<String>>()
val file = File(file_base + filename)
logger.trace("Checking ini file {}", filename.anonymize())
if (!file.exists()) return map
logger.debug("Loading ini file {}", filename.anonymize())
file.forEachLine { parseLine(it, map) }
return map
}
private fun parseLine(original_line: String, map: MutableMap<String, MutableList<String>>) {
logger.trace("Parsing line: {}", original_line)
var line = original_line.trim().replaceAfter("#", "").removeSuffix("#")
logger.trace("After cleaning: {}", line)
if (line == "") return
val parts: List<String> = line.split("=", limit=2).map{it.trim()}
if (parts.size < 2) throw RuntimeException("Invalid config setting: $line")
val (key, value) = parts
if (value=="") {
map.remove(key)
} else {
var list = map.get(key)
if (list == null) {
list = mutableListOf<String>()
map.put(key, list)
}
list.add(value)
}
}
private fun copy_sample_ini(filename: String) {
val stream = Config::class.java.getResourceAsStream("/config.sample.ini")
File(filename).outputStream().use { stream.copyTo(it) }
stream.close()
}
fun print() {
println()
Setting.all_settings.forEach { it.print() }
println()
}
}
class SettingsFactory(val ini: Map<String, List<String>>, val cli: CommandLineOptions) {
fun getInt(name: String, default: Int, secret: Boolean = false) = getSetting(name, listOf(default.toString()), secret).get().toInt()
fun getBoolean(name: String, default: Boolean, secret: Boolean = false) = getSetting(name, listOf(default.toString()), secret).get().toBoolean()
fun getString(name: String, default: String, secret: Boolean = false) = getSetting(name, listOf(default), secret).get()
fun getStringList(name: String, default: List<String>, secret: Boolean = false) = getSetting(name, default, secret).getList()
fun getSetting(name: String, default: List<String>, secret: Boolean) = Setting(ini, cli, name, default, secret)
}
class Setting(val ini: Map<String, List<String>>, val cli: CommandLineOptions, val name: String, val default: List<String>, val secret: Boolean) {
val values: List<String>
val source: SettingSource
val logger = LoggerFactory.getLogger(Setting::class.java)
init {
if (getCli(name) != null) {
values = listOf(getCli(name)!!)
source = SettingSource.CLI
} else if (getIni(name) != null) {
values = getIni(name)!!
source = SettingSource.INI
} else {
values = default
source = SettingSource.DEFAULT
}
logger.debug("Setting ${name} loaded. Source: ${source}. Value: ${values.toString().anonymize()}")
all_settings.add(this)
}
fun get(): String = values.last()
fun getList(): List<String> = values
fun getIni(name: String): List<String>? {
return ini[name]
}
fun getCli(name: String): String? {
return cli.get(name)
}
fun print() {
println("%-25s %-10s %s".format(name, source, (if (secret && source==SettingSource.DEFAULT) "[REDACTED]" else values)))
}
companion object {
val all_settings = LinkedList<Setting>()
}
}
enum class SettingSource {
INI,
CLI,
DEFAULT
}

52
src/main/kotlin/de/fabianonline/telegram_backup/StickerConverter.kt
View File

@ -1,52 +0,0 @@
/* Telegram_Backup
* Copyright (C) 2016 Fabian Schlenz
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. */
package de.fabianonline.telegram_backup
import com.github.badoualy.telegram.tl.api.*
import java.lang.StringBuilder
import java.io.File
object StickerConverter {
fun makeFilenameWithPath(attr: TLDocumentAttributeSticker): String {
val file = StringBuilder()
file.append(makePath())
file.append(makeFilename(attr))
return file.toString()
}
fun makeFilename(attr: TLDocumentAttributeSticker): String {
val file = StringBuilder()
if (attr.getStickerset() is TLInputStickerSetShortName) {
file.append((attr.getStickerset() as TLInputStickerSetShortName).getShortName())
} else if (attr.getStickerset() is TLInputStickerSetID) {
file.append((attr.getStickerset() as TLInputStickerSetID).getId())
}
file.append("_")
file.append(attr.getAlt().hashCode())
file.append(".webp")
return file.toString()
}
fun makePath(): String {
val path = Config.FILE_BASE +
File.separatorChar +
Config.FILE_STICKER_BASE +
File.separatorChar
File(path).mkdirs()
return path
}
}

68
src/main/kotlin/de/fabianonline/telegram_backup/TelegramUpdateHandler.kt
View File

@ -26,48 +26,39 @@ import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.UserManager
import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager
import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory
import org.slf4j.LoggerFactory
internal class TelegramUpdateHandler : UpdateCallback {
private var user: UserManager? = null
private var db: Database? = null
var debug = false
fun activate() {
this.user = UserManager.getInstance()
this.db = Database.getInstance()
}
internal class TelegramUpdateHandler(val user_manager: UserManager, val db: Database, val file_base: String, val settings: Settings) : UpdateCallback {
val logger = LoggerFactory.getLogger(TelegramUpdateHandler::class.java)
override fun onUpdates(client: TelegramClient, updates: TLUpdates) {
if (db == null) return
if (debug) System.out.println("onUpdates - " + updates.getUpdates().size + " Updates, " + updates.getUsers().size + " Users, " + updates.getChats().size + " Chats")
logger.debug("onUpdates - " + updates.getUpdates().size + " Updates, " + updates.getUsers().size + " Users, " + updates.getChats().size + " Chats")
for (update in updates.getUpdates()) {
processUpdate(update, client)
if (debug) System.out.println(" " + update.javaClass.getName())
processUpdate(update)
logger.debug(" " + update.javaClass.getName())
}
db!!.saveUsers(updates.getUsers())
db!!.saveChats(updates.getChats())
db.saveUsers(updates.getUsers())
db.saveChats(updates.getChats())
}
override fun onUpdatesCombined(client: TelegramClient, updates: TLUpdatesCombined) {
if (db == null) return
if (debug) System.out.println("onUpdatesCombined")
logger.debug("onUpdatesCombined")
for (update in updates.getUpdates()) {
processUpdate(update, client)
processUpdate(update)
}
db!!.saveUsers(updates.getUsers())
db!!.saveChats(updates.getChats())
db.saveUsers(updates.getUsers())
db.saveChats(updates.getChats())
}
override fun onUpdateShort(client: TelegramClient, update: TLUpdateShort) {
if (db == null) return
if (debug) System.out.println("onUpdateShort")
processUpdate(update.getUpdate(), client)
if (debug) System.out.println(" " + update.getUpdate().javaClass.getName())
logger.debug("onUpdateShort")
processUpdate(update.getUpdate())
logger.debug(" " + update.getUpdate().javaClass.getName())
}
override fun onShortChatMessage(client: TelegramClient, message: TLUpdateShortChatMessage) {
if (db == null) return
if (debug) System.out.println("onShortChatMessage - " + message.getMessage())
logger.debug("onShortChatMessage - " + message.getMessage())
val msg = TLMessage(
message.getOut(),
message.getMentioned(),
@ -85,21 +76,20 @@ internal class TelegramUpdateHandler : UpdateCallback {
message.getEntities(), null, null)
val vector = TLVector<TLAbsMessage>(TLAbsMessage::class.java)
vector.add(msg)
db!!.saveMessages(vector, Kotlogram.API_LAYER)
db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings)
System.out.print('.')
}
override fun onShortMessage(client: TelegramClient, message: TLUpdateShortMessage) {
val m = message
if (db == null) return
if (debug) System.out.println("onShortMessage - " + m.getOut() + " - " + m.getUserId() + " - " + m.getMessage())
logger.debug("onShortMessage - " + m.getOut() + " - " + m.getUserId() + " - " + m.getMessage())
val from_id: Int
val to_id: Int
if (m.getOut() == true) {
from_id = user!!.user!!.getId()
from_id = user_manager.id
to_id = m.getUserId()
} else {
to_id = user!!.user!!.getId()
to_id = user_manager.id
from_id = m.getUserId()
}
val msg = TLMessage(
@ -119,29 +109,27 @@ internal class TelegramUpdateHandler : UpdateCallback {
m.getEntities(), null, null)
val vector = TLVector<TLAbsMessage>(TLAbsMessage::class.java)
vector.add(msg)
db!!.saveMessages(vector, Kotlogram.API_LAYER)
db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings)
System.out.print('.')
}
override fun onShortSentMessage(client: TelegramClient, message: TLUpdateShortSentMessage) {
if (db == null) return
System.out.println("onShortSentMessage")
logger.debug("onShortSentMessage")
}
override fun onUpdateTooLong(client: TelegramClient) {
if (db == null) return
System.out.println("onUpdateTooLong")
logger.debug("onUpdateTooLong")
}
private fun processUpdate(update: TLAbsUpdate, client: TelegramClient) {
private fun processUpdate(update: TLAbsUpdate) {
if (update is TLUpdateNewMessage) {
val abs_msg = update.getMessage()
val vector = TLVector<TLAbsMessage>(TLAbsMessage::class.java)
vector.add(abs_msg)
db!!.saveMessages(vector, Kotlogram.API_LAYER)
db.saveMessages(vector, Kotlogram.API_LAYER, settings=settings)
System.out.print('.')
if (abs_msg is TLMessage && IniSettings.download_media==true) {
val fm = FileManagerFactory.getFileManager(abs_msg, user!!, client)
if (abs_msg is TLMessage && settings.download_media==true) {
val fm = FileManagerFactory.getFileManager(abs_msg, user_manager, file_base, settings)
if (fm != null && !fm.isEmpty && !fm.downloaded) {
try {
fm.download()

34
src/main/kotlin/de/fabianonline/telegram_backup/TestFeatures.kt
View File

@ -10,7 +10,7 @@ import java.sql.ResultSet
import java.io.IOException
import java.nio.charset.Charset
internal object TestFeatures {
internal class TestFeatures(val db: Database) {
fun test1() {
// Tests entries in a cache4.db in the current working directory for compatibility
try {
@ -24,31 +24,22 @@ internal object TestFeatures {
var conn: Connection
var stmt: Statement? = null
try {
conn = DriverManager.getConnection(path)
stmt = conn.createStatement()
} catch (e: SQLException) {
CommandLineController.show_error("Could not connect to SQLITE database.")
}
conn = DriverManager.getConnection(path)
stmt = conn.createStatement()
var unsupported_constructor = 0
var success = 0
try {
val rs = stmt!!.executeQuery("SELECT data FROM messages")
while (rs.next()) {
try {
TLApiContext.getInstance().deserializeMessage(rs.getBytes(1))
} catch (e: com.github.badoualy.telegram.tl.exception.UnsupportedConstructorException) {
unsupported_constructor++
} catch (e: IOException) {
System.out.println("IOException: " + e)
}
success++
val rs = stmt.executeQuery("SELECT data FROM messages")
while (rs.next()) {
try {
TLApiContext.getInstance().deserializeMessage(rs.getBytes(1))
} catch (e: com.github.badoualy.telegram.tl.exception.UnsupportedConstructorException) {
unsupported_constructor++
} catch (e: IOException) {
System.out.println("IOException: " + e)
}
} catch (e: SQLException) {
System.out.println("SQL exception: " + e)
success++
}
System.out.println("Success: " + success)
@ -59,7 +50,6 @@ internal object TestFeatures {
// Prints system.encoding and default charset
System.out.println("Default Charset: " + Charset.defaultCharset())
System.out.println("file.encoding: " + System.getProperty("file.encoding"))
val db = Database.getInstance()
System.out.println("Database encoding: " + db.getEncoding())
}
}

119
src/main/kotlin/de/fabianonline/telegram_backup/UserManager.kt
View File

@ -34,107 +34,34 @@ import java.io.File
import org.slf4j.LoggerFactory
import org.slf4j.Logger
class UserManager @Throws(IOException::class)
private constructor(c: TelegramClient) {
var user: TLUser? = null
var phone: String? = null
private var code: String? = null
private val client: TelegramClient
private var sent_code: TLSentCode? = null
private var auth: TLAuthorization? = null
var isPasswordNeeded = false
private set
val loggedIn: Boolean
get() = user != null
val userString: String
get() {
if (this.user == null) return "Not logged in"
val sb = StringBuilder()
if (this.user!!.getFirstName() != null) {
sb.append(this.user!!.getFirstName())
}
if (this.user!!.getLastName() != null) {
sb.append(" ")
sb.append(this.user!!.getLastName())
}
if (this.user!!.getUsername() != null) {
sb.append(" (@")
sb.append(this.user!!.getUsername())
sb.append(")")
}
return sb.toString()
}
val fileBase: String
get() = Config.FILE_BASE + File.separatorChar + "+" + this.user!!.getPhone() + File.separatorChar
class UserManager(val client: TelegramClient) {
val tl_user: TLUser
val logger = LoggerFactory.getLogger(UserManager::class.java)
val phone: String
get() = "+" + tl_user.getPhone()
val id: Int
get() = tl_user.getId()
init {
this.client = c
logger.debug("Calling getFullUser")
try {
val full_user = this.client.usersGetFullUser(TLInputUserSelf())
this.user = full_user.getUser().getAsUser()
} catch (e: RpcErrorException) {
// This may happen. Ignoring it.
logger.debug("Ignoring exception:", e)
}
val full_user = client.usersGetFullUser(TLInputUserSelf())
tl_user = full_user.getUser().getAsUser()
}
@Throws(RpcErrorException::class, IOException::class)
fun sendCodeToPhoneNumber(number: String) {
this.phone = number
this.sent_code = this.client.authSendCode(false, number, true)
}
@Throws(RpcErrorException::class, IOException::class)
fun verifyCode(code: String) {
this.code = code
try {
this.auth = client.authSignIn(phone, this.sent_code!!.getPhoneCodeHash(), this.code)
this.user = auth!!.getUser().getAsUser()
} catch (e: RpcErrorException) {
if (e.getCode() != 401 || !e.getTag().equals("SESSION_PASSWORD_NEEDED")) throw e
this.isPasswordNeeded = true
override fun toString(): String {
val sb = StringBuilder()
sb.append(tl_user.getFirstName() ?: "")
if (tl_user.getLastName() != null) {
sb.append(" ")
sb.append(tl_user.getLastName())
}
}
@Throws(RpcErrorException::class, IOException::class)
fun verifyPassword(pw: String) {
val password = pw.toByteArray(charset = Charsets.UTF_8)
val salt = (client.accountGetPassword() as TLPassword).getCurrentSalt().getData()
var md: MessageDigest
try {
md = MessageDigest.getInstance("SHA-256")
} catch (e: NoSuchAlgorithmException) {
e.printStackTrace()
return
}
val salted = ByteArray(2 * salt.size + password.size)
System.arraycopy(salt, 0, salted, 0, salt.size)
System.arraycopy(password, 0, salted, salt.size, password.size)
System.arraycopy(salt, 0, salted, salt.size + password.size, salt.size)
val hash = md.digest(salted)
auth = client.authCheckPassword(TLBytes(hash))
this.user = auth!!.getUser().getAsUser()
}
companion object {
private val logger = LoggerFactory.getLogger(UserManager::class.java)
internal var instance: UserManager? = null
@Throws(IOException::class)
fun init(c: TelegramClient) {
instance = UserManager(c)
}
fun getInstance(): UserManager {
if (instance == null) throw RuntimeException("UserManager is not yet initialized.")
return instance!!
if (tl_user.getUsername() != null) {
sb.append(" (@")
sb.append(tl_user.getUsername())
sb.append(")")
}
return sb.toString()
}
}

89
src/main/kotlin/de/fabianonline/telegram_backup/Utils.kt
View File

@ -20,6 +20,7 @@ import com.github.badoualy.telegram.tl.exception.RpcErrorException
import java.io.File
import java.util.Vector
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import com.google.gson.*
import java.net.URL
import org.apache.commons.io.IOUtils
@ -31,12 +32,30 @@ object Utils {
@JvmField public val VERSIONS_EQUAL = 0
@JvmField public val VERSION_1_NEWER = 1
@JvmField public val VERSION_2_NEWER = 2
var hasSeenFloodWaitMessage = false
var anonymize = false
private val logger = LoggerFactory.getLogger(Utils::class.java) as Logger
fun getAccounts(): Vector<String> {
fun print_accounts(file_base: String) {
println("List of available accounts:")
val accounts = getAccounts(file_base)
if (accounts.size > 0) {
for (str in accounts) {
println(" " + str.anonymize())
}
println("Use '--account <x>' to use one of those accounts.")
} else {
println("NO ACCOUNTS FOUND")
println("Use '--login' to login to a telegram account.")
}
}
fun getAccounts(file_base: String): Vector<String> {
val accounts = Vector<String>()
val folder = File(Config.FILE_BASE)
val folder = File(file_base)
val files = folder.listFiles()
if (files != null)
for (f in files) {
@ -80,30 +99,48 @@ object Utils {
}
}
@Throws(RpcErrorException::class)
@JvmOverloads internal fun obeyFloodWaitException(e: RpcErrorException?, silent: Boolean = false) {
if (e == null || e.getCode() != 420) return
val delay: Long = e.getTagInteger()!!.toLong()
if (!silent) {
System.out.println("")
System.out.println(
"Telegram complained about us (okay, me) making too many requests in too short time by\n" +
"sending us \"" + e.getTag() + "\" as an error. So we now have to wait a bit. Telegram\n" +
"asked us to wait for " + delay + " seconds.\n" +
fun obeyFloodWait(max_tries: Int = -1, method: () -> Unit) {
var tries = 0
while (true) {
tries++
if (max_tries>0 && tries>max_tries) throw MaxTriesExceededException()
logger.trace("This is try ${tries}.")
try {
method.invoke()
// If we reach this, the method has returned successfully -> we are done
return
} catch(e: RpcErrorException) {
// If we got something else than a FLOOD_WAIT error, we just rethrow it
if (e.getCode() != 420) throw e
val delay = e.getTagInteger()!!.toLong()
if (!hasSeenFloodWaitMessage) {
println(
"\n" +
"Telegram complained about us (okay, me) making too many requests in too short time by\n" +
"sending us \"${e.getTag()}\" as an error. So we now have to wait a bit. Telegram\n" +
"asked us to wait for ${delay} seconds.\n" +
"\n" +
"So I'm going to do just that for now. If you don't want to wait, you can quit by pressing\n" +
"Ctrl+C. You can restart me at any time and I will just continue to download your\n" +
"messages and media. But be advised that just restarting me is not going to change\n" +
"the fact that Telegram won't talk to me until then." +
"\n")
}
hasSeenFloodWaitMessage = true
try { TimeUnit.SECONDS.sleep(delay + 1) } catch (e: InterruptedException) { }
} catch (e: TimeoutException) {
println(
"\n" +
"So I'm going to do just that for now. If you don't want to wait, you can quit by pressing\n" +
"Ctrl+C. You can restart me at any time and I will just continue to download your\n" +
"messages and media. But be advised that just restarting me is not going to change\n" +
"the fact that Telegram won't talk to me until then.")
System.out.println("")
"Telegram took too long to respond to our request.\n" +
"I'm going to wait a minute and then try again." +
"\n")
try { TimeUnit.MINUTES.sleep(1) } catch (e: InterruptedException) { }
}
}
try {
TimeUnit.SECONDS.sleep(delay + 1)
} catch (e2: InterruptedException) {
}
}
@JvmStatic
@ -179,8 +216,10 @@ object Utils {
}
fun String.anonymize(): String {
return if (!CommandLineOptions.cmd_anonymize) this else this.replace(Regex("[0-9]"), "1").replace(Regex("[A-Z]"), "A").replace(Regex("[a-z]"), "a") + " (ANONYMIZED)"
return if (!Utils.anonymize) this else this.replace(Regex("[0-9]"), "1").replace(Regex("[A-Z]"), "A").replace(Regex("[a-z]"), "a") + " (ANONYMIZED)"
}
fun Any.toJson(): String = Gson().toJson(this)
fun Any.toPrettyJson(): String = GsonBuilder().setPrettyPrinting().create().toJson(this)
class MaxTriesExceededException(): RuntimeException("Max tries exceeded") {}

9
src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt
View File

@ -37,18 +37,15 @@ import de.fabianonline.telegram_backup.*
import org.slf4j.Logger
import org.slf4j.LoggerFactory
class HTMLExporter {
val db = Database.getInstance()
val user = UserManager.getInstance()
class HTMLExporter(val db: Database, val user: UserManager, val settings: Settings, val file_base: String) {
@Throws(IOException::class)
fun export() {
try {
val pagination = if (IniSettings.pagination) IniSettings.pagination_size else -1
val pagination = if (settings.pagination) settings.pagination_size else -1
// Create base dir
logger.debug("Creating base dir")
val base = user.fileBase + "files" + File.separatorChar
val base = file_base + "files" + File.separatorChar
File(base).mkdirs()
File(base + "dialogs").mkdirs()

22
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.kt
View File

@ -17,33 +17,17 @@
package de.fabianonline.telegram_backup.mediafilemanager
import de.fabianonline.telegram_backup.UserManager
import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.StickerConverter
import de.fabianonline.telegram_backup.DownloadProgressInterface
import de.fabianonline.telegram_backup.Config
import de.fabianonline.telegram_backup.DownloadManager
import com.github.badoualy.telegram.api.TelegramClient
import com.github.badoualy.telegram.tl.core.TLIntVector
import com.github.badoualy.telegram.tl.core.TLObject
import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
import com.github.badoualy.telegram.tl.api.*
import com.github.badoualy.telegram.tl.api.upload.TLFile
import com.github.badoualy.telegram.tl.exception.RpcErrorException
import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
import de.fabianonline.telegram_backup.Settings
import java.io.IOException
import java.io.File
import java.io.FileOutputStream
import java.util.ArrayList
import java.util.LinkedList
import java.net.URL
import java.util.concurrent.TimeoutException
import org.apache.commons.io.FileUtils
abstract class AbstractMediaFileManager(protected var message: TLMessage, protected var user: UserManager, protected var client: TelegramClient) {
abstract class AbstractMediaFileManager(protected var message: TLMessage, protected var user: UserManager, val file_base: String) {
open var isEmpty = false
abstract val size: Int
abstract val extension: String
@ -56,7 +40,7 @@ abstract class AbstractMediaFileManager(protected var message: TLMessage, protec
open val targetPath: String
get() {
val path = user.fileBase + Config.FILE_FILES_BASE + File.separatorChar
val path = file_base + Config.FILE_FILES_BASE + File.separatorChar
File(path).mkdirs()
return path
}

19
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/DocumentFileManager.kt
View File

@ -17,32 +17,15 @@
package de.fabianonline.telegram_backup.mediafilemanager
import de.fabianonline.telegram_backup.UserManager
import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.StickerConverter
import de.fabianonline.telegram_backup.DownloadProgressInterface
import de.fabianonline.telegram_backup.DownloadManager
import com.github.badoualy.telegram.api.TelegramClient
import com.github.badoualy.telegram.tl.core.TLIntVector
import com.github.badoualy.telegram.tl.core.TLObject
import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
import com.github.badoualy.telegram.tl.api.*
import com.github.badoualy.telegram.tl.api.upload.TLFile
import com.github.badoualy.telegram.tl.exception.RpcErrorException
import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
import java.io.IOException
import java.io.File
import java.io.FileOutputStream
import java.util.ArrayList
import java.util.LinkedList
import java.net.URL
import java.util.concurrent.TimeoutException
import org.apache.commons.io.FileUtils
open class DocumentFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : AbstractMediaFileManager(msg, user, client) {
open class DocumentFileManager(msg: TLMessage, user: UserManager, file_base: String) : AbstractMediaFileManager(msg, user, file_base) {
protected var doc: TLDocument? = null
override lateinit var extension: String

22
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/FileManagerFactory.kt
View File

@ -18,7 +18,6 @@ package de.fabianonline.telegram_backup.mediafilemanager
import de.fabianonline.telegram_backup.UserManager
import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.StickerConverter
import de.fabianonline.telegram_backup.DownloadProgressInterface
import com.github.badoualy.telegram.api.TelegramClient
@ -30,6 +29,7 @@ import com.github.badoualy.telegram.tl.api.*
import com.github.badoualy.telegram.tl.api.upload.TLFile
import com.github.badoualy.telegram.tl.exception.RpcErrorException
import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
import de.fabianonline.telegram_backup.Settings
import java.io.IOException
import java.io.File
@ -42,29 +42,29 @@ import java.util.concurrent.TimeoutException
import org.apache.commons.io.FileUtils
object FileManagerFactory {
fun getFileManager(m: TLMessage?, u: UserManager, c: TelegramClient): AbstractMediaFileManager? {
fun getFileManager(m: TLMessage?, u: UserManager, file_base: String, settings: Settings?): AbstractMediaFileManager? {
if (m == null) return null
val media = m.getMedia() ?: return null
if (media is TLMessageMediaPhoto) {
return PhotoFileManager(m, u, c)
return PhotoFileManager(m, u, file_base)
} else if (media is TLMessageMediaDocument) {
val d = DocumentFileManager(m, u, c)
val d = DocumentFileManager(m, u, file_base)
return if (d.isSticker) {
StickerFileManager(m, u, c)
StickerFileManager(m, u, file_base)
} else d
} else if (media is TLMessageMediaGeo) {
return GeoFileManager(m, u, c)
return GeoFileManager(m, u, file_base, settings)
} else if (media is TLMessageMediaEmpty) {
return UnsupportedFileManager(m, u, c, "empty")
return UnsupportedFileManager(m, u, file_base, "empty")
} else if (media is TLMessageMediaUnsupported) {
return UnsupportedFileManager(m, u, c, "unsupported")
return UnsupportedFileManager(m, u, file_base, "unsupported")
} else if (media is TLMessageMediaWebPage) {
return UnsupportedFileManager(m, u, c, "webpage")
return UnsupportedFileManager(m, u, file_base, "webpage")
} else if (media is TLMessageMediaContact) {
return UnsupportedFileManager(m, u, c, "contact")
return UnsupportedFileManager(m, u, file_base, "contact")
} else if (media is TLMessageMediaVenue) {
return UnsupportedFileManager(m, u, c, "venue")
return UnsupportedFileManager(m, u, file_base, "venue")
} else {
AbstractMediaFileManager.throwUnexpectedObjectError(media)
}

25
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt
View File

@ -17,33 +17,16 @@
package de.fabianonline.telegram_backup.mediafilemanager
import de.fabianonline.telegram_backup.UserManager
import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.StickerConverter
import de.fabianonline.telegram_backup.DownloadProgressInterface
import de.fabianonline.telegram_backup.DownloadManager
import de.fabianonline.telegram_backup.IniSettings
import com.github.badoualy.telegram.api.TelegramClient
import com.github.badoualy.telegram.tl.core.TLIntVector
import com.github.badoualy.telegram.tl.core.TLObject
import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
import com.github.badoualy.telegram.tl.api.*
import com.github.badoualy.telegram.tl.api.upload.TLFile
import com.github.badoualy.telegram.tl.exception.RpcErrorException
import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
import de.fabianonline.telegram_backup.Config
import de.fabianonline.telegram_backup.Settings
import java.io.IOException
import java.io.File
import java.io.FileOutputStream
import java.util.ArrayList
import java.util.LinkedList
import java.net.URL
import java.util.concurrent.TimeoutException
import org.apache.commons.io.FileUtils
class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : AbstractMediaFileManager(msg, user, client) {
class GeoFileManager(msg: TLMessage, user: UserManager, file_base: String, val settings: Settings?) : AbstractMediaFileManager(msg, user, file_base) {
protected lateinit var geo: TLGeoPoint
// We don't know the size, so we just guess.
@ -77,7 +60,7 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient)
"center=${geo.getLat()},${geo.getLong()}&" +
"markers=color:red|${geo.getLat()},${geo.getLong()}&" +
"zoom=14&size=300x150&scale=2&format=png&" +
"key=" + IniSettings.gmaps_key
"key=" + (settings?.gmaps_key ?: Config.SECRET_GMAPS)
return DownloadManager.downloadExternalFile(targetPathAndFilename, url)
}
}

19
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/PhotoFileManager.kt
View File

@ -17,32 +17,15 @@
package de.fabianonline.telegram_backup.mediafilemanager
import de.fabianonline.telegram_backup.UserManager
import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.StickerConverter
import de.fabianonline.telegram_backup.DownloadProgressInterface
import de.fabianonline.telegram_backup.DownloadManager
import com.github.badoualy.telegram.api.TelegramClient
import com.github.badoualy.telegram.tl.core.TLIntVector
import com.github.badoualy.telegram.tl.core.TLObject
import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
import com.github.badoualy.telegram.tl.api.*
import com.github.badoualy.telegram.tl.api.upload.TLFile
import com.github.badoualy.telegram.tl.exception.RpcErrorException
import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
import java.io.IOException
import java.io.File
import java.io.FileOutputStream
import java.util.ArrayList
import java.util.LinkedList
import java.net.URL
import java.util.concurrent.TimeoutException
import org.apache.commons.io.FileUtils
class PhotoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : AbstractMediaFileManager(msg, user, client) {
class PhotoFileManager(msg: TLMessage, user: UserManager, file_base: String) : AbstractMediaFileManager(msg, user, file_base) {
private lateinit var photo: TLPhoto
override var size = 0
private lateinit var photo_size: TLPhotoSize

18
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/StickerFileManager.kt
View File

@ -18,7 +18,6 @@ package de.fabianonline.telegram_backup.mediafilemanager
import de.fabianonline.telegram_backup.UserManager
import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.StickerConverter
import de.fabianonline.telegram_backup.DownloadProgressInterface
import de.fabianonline.telegram_backup.DownloadManager
import de.fabianonline.telegram_backup.Config
@ -50,7 +49,7 @@ import java.util.concurrent.TimeoutException
import org.apache.commons.io.FileUtils
class StickerFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) : DocumentFileManager(msg, user, client) {
class StickerFileManager(msg: TLMessage, user: UserManager, file_base: String) : DocumentFileManager(msg, user, file_base) {
override val isSticker = true
@ -80,7 +79,7 @@ class StickerFileManager(msg: TLMessage, user: UserManager, client: TelegramClie
override val targetPath: String
get() {
val path = user.fileBase + Config.FILE_FILES_BASE + File.separatorChar + Config.FILE_STICKER_BASE + File.separatorChar
val path = file_base + Config.FILE_FILES_BASE + File.separatorChar + Config.FILE_STICKER_BASE + File.separatorChar
File(path).mkdirs()
return path
}
@ -94,19 +93,6 @@ class StickerFileManager(msg: TLMessage, user: UserManager, client: TelegramClie
override val description: String
get() = "Sticker"
@Throws(RpcErrorException::class, IOException::class, TimeoutException::class)
override fun download(): Boolean {
val old_file = Config.FILE_BASE + File.separatorChar + Config.FILE_STICKER_BASE + File.separatorChar + targetFilename
logger.trace("Old filename exists: {}", File(old_file).exists())
if (File(old_file).exists()) {
Files.copy(Paths.get(old_file), Paths.get(targetPathAndFilename), StandardCopyOption.REPLACE_EXISTING)
return true
}
return super.download()
}
companion object {
private val logger = LoggerFactory.getLogger(StickerFileManager::class.java)
}

25
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/UnsupportedFileManager.kt
View File

@ -17,33 +17,10 @@
package de.fabianonline.telegram_backup.mediafilemanager
import de.fabianonline.telegram_backup.UserManager
import de.fabianonline.telegram_backup.Database
import de.fabianonline.telegram_backup.StickerConverter
import de.fabianonline.telegram_backup.DownloadProgressInterface
import de.fabianonline.telegram_backup.DownloadManager
import de.fabianonline.telegram_backup.Config
import com.github.badoualy.telegram.api.TelegramClient
import com.github.badoualy.telegram.tl.core.TLIntVector
import com.github.badoualy.telegram.tl.core.TLObject
import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages
import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs
import com.github.badoualy.telegram.tl.api.*
import com.github.badoualy.telegram.tl.api.upload.TLFile
import com.github.badoualy.telegram.tl.exception.RpcErrorException
import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile
import java.io.IOException
import java.io.File
import java.io.FileOutputStream
import java.util.ArrayList
import java.util.LinkedList
import java.net.URL
import java.util.concurrent.TimeoutException
import org.apache.commons.io.FileUtils
class UnsupportedFileManager(msg: TLMessage, user: UserManager, client: TelegramClient, type: String) : AbstractMediaFileManager(msg, user, client) {
class UnsupportedFileManager(msg: TLMessage, user: UserManager, type: String, file_base: String) : AbstractMediaFileManager(msg, user, file_base) {
override var name = type
override val targetFilename = ""
override val targetPath = ""