From 0e2eeab5b9a56f816a99c404e0950f1590a2d8ab Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 30 Jan 2018 18:13:05 +0100 Subject: [PATCH] Added pagination for the output. --- .../telegram_backup/CommandLineController.kt | 35 +++--- .../telegram_backup/CommandLineOptions.kt | 8 ++ .../de/fabianonline/telegram_backup/Config.kt | 2 + .../fabianonline/telegram_backup/Database.kt | 41 +++++-- .../telegram_backup/exporter/HTMLExporter.kt | 108 +++++++++++++----- .../templates/html/_messages.mustache | 17 +++ .../resources/templates/html/_stats.mustache | 2 +- .../resources/templates/html/chat.mustache | 29 ++--- .../resources/templates/html/page.mustache | 33 ++++++ 9 files changed, 200 insertions(+), 75 deletions(-) create mode 100644 src/main/resources/templates/html/_messages.mustache create mode 100644 src/main/resources/templates/html/page.mustache diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index b539c0f..4286ccd 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -265,24 +265,25 @@ class CommandLineController { private fun show_help() { println("Valid options are:") - println(" -h, --help Shows this help.") - println(" -a, --account Use account .") - println(" -l, --login Login to an existing telegram account.") - println(" --debug Shows some debug information.") - println(" --trace Shows lots of debug information. Overrides --debug.") - println(" --trace-telegram Shows lots of debug messages from the library used to access Telegram.") - println(" -A, --list-accounts List all existing accounts ") - println(" --limit-messages Downloads at most the most recent messages.") - println(" --no-media Do not download media files.") - println(" -t, --target Target directory for the files.") + println(" -h, --help Shows this help.") + println(" -a, --account Use account .") + println(" -l, --login Login to an existing telegram account.") + println(" --debug Shows some debug information.") + println(" --trace Shows lots of debug information. Overrides --debug.") + println(" --trace-telegram Shows lots of debug messages from the library used to access Telegram.") + println(" -A, --list-accounts List all existing accounts ") + println(" --limit-messages Downloads at most the most recent messages.") + println(" --no-media Do not download media files.") + println(" -t, --target Target directory for the files.") println(" -e, --export Export the database. Valid formats are:") - println(" html - Creates HTML files.") - println(" --license Displays the license of this program.") - println(" -d, --daemon Keep running and automatically save new messages.") - println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.") - println(" --stats Print some usage statistics.") - println(" --with-channels Backup channels as well.") - println(" --with-supergroups Backup supergroups as well.") + println(" html - Creates HTML files.") + println(" --pagination Splits the HTML export into multiple HTML pages with messages per page. Default is 5000.") + println(" --no-pagination Disables pagination.") + println(" --license Displays the license of this program.") + println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.") + println(" --stats Print some usage statistics.") + println(" --with-channels Backup channels as well.") + println(" --with-supergroups Backup supergroups as well.") } private fun list_accounts() { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt index ba371a0..32ba425 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt @@ -31,11 +31,13 @@ internal object CommandLineOptions { var cmd_stats = false var cmd_channels = false var cmd_supergroups = false + var cmd_no_pagination = false var val_account: String? = null var val_limit_messages: Int? = null var val_target: String? = null var val_export: String? = null var val_test: Int? = null + var val_pagination: Int = Config.DEFAULT_PAGINATION @JvmStatic fun parseOptions(args: Array) { var last_cmd: String? = null @@ -47,6 +49,7 @@ internal object CommandLineOptions { "--target" -> val_target = arg "--export" -> val_export = arg "--test" -> val_test = Integer.parseInt(arg) + "--pagination" -> val_pagination = Integer.parseInt(arg) } last_cmd = null continue @@ -76,6 +79,11 @@ internal object CommandLineOptions { last_cmd = "--export" continue@loop } + "--pagination" -> { + last_cmd = "--pagination" + continue@loop + } + "--no-pagination" -> cmd_no_pagination = true "--license" -> cmd_license = true "-d", "--daemon" -> cmd_daemon = true "--no-media" -> cmd_no_media = true diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt index c0828d4..4a045f7 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Config.kt @@ -43,6 +43,8 @@ object Config { var RENAMING_MAX_TRIES = 5 var RENAMING_DELAY: Long = 1000 + + var DEFAULT_PAGINATION = 5_000 val SECRET_GMAPS = "AIzaSyBEtUDhCQKEH6i2Mn1GAiQ9M_tLN0vxHIs" diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt index 66bb42d..8c35302 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/Database.kt @@ -595,7 +595,7 @@ class Database private constructor(var client: TelegramClient) { fun getMessageAuthorsWithCount(c: AbstractChat): HashMap { val map = HashMap() - val user_map = HashMap() + val all_data = LinkedList>() var count_others = 0 // Set a default value for 'me' to fix the charts for channels - cause I // possibly didn't send any messages there. @@ -607,6 +607,7 @@ class Database private constructor(var client: TelegramClient) { "WHERE " + c.query + " GROUP BY sender_id") while (rs.next()) { val u: User + val data = HashMap() if (rs.getString(2) != null || rs.getString(3) != null || rs.getString(4) != null) { u = User(rs.getInt(1), rs.getString(2), rs.getString(3)) } else { @@ -615,17 +616,25 @@ class Database private constructor(var client: TelegramClient) { if (u.isMe) { map.put("authors.count.me", rs.getInt(5)) } else { - user_map.put(u, rs.getInt(5)) count_others += rs.getInt(5) + data.put("name", u.name) + data.put("count", ""+rs.getInt(5)) + all_data.add(data) } + } map.put("authors.count.others", count_others) - map.put("authors.all", user_map) + map.put("authors.all", all_data) return map } catch (e: Exception) { throw RuntimeException(e) } - + } + + fun getMessageCountForExport(c: AbstractChat): Int { + val rs = stmt!!.executeQuery("SELECT COUNT(*) FROM messages WHERE " + c.query); + rs.next() + return rs.getInt(1) } fun getMessageTimesMatrix(c: AbstractChat): Array { @@ -645,10 +654,9 @@ class Database private constructor(var client: TelegramClient) { } - fun getMessagesForExport(c: AbstractChat): LinkedList> { + fun getMessagesForExport(c: AbstractChat, limit: Int=-1, offset: Int=0): LinkedList> { try { - - val rs = stmt!!.executeQuery("SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " + + var query = "SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " + "media_type, media_file, media_size, users.first_name as user_first_name, users.last_name as user_last_name, " + "users.username as user_username, users.id as user_id, " + "users_fwd.first_name as user_fwd_first_name, users_fwd.last_name as user_fwd_last_name, users_fwd.username as user_fwd_username " + @@ -656,7 +664,14 @@ class Database private constructor(var client: TelegramClient) { "LEFT JOIN users ON users.id=messages.sender_id " + "LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " + c.query + " " + - "ORDER BY messages.message_id") + "ORDER BY messages.message_id" + + if ( limit != -1 ) { + query = query + " LIMIT ${limit} OFFSET ${offset}" + } + + val rs = stmt!!.executeQuery(query) + val format_time = SimpleDateFormat("HH:mm:ss") val format_date = SimpleDateFormat("d MMM yy") val meta = rs.getMetaData() @@ -701,18 +716,25 @@ class Database private constructor(var client: TelegramClient) { abstract inner class AbstractChat { abstract val query: String + abstract val type: String } inner class Dialog(var id: Int, var first_name: String?, var last_name: String?, var username: String?, var count: Int?) : AbstractChat() { override val query: String get() = "source_type='dialog' AND source_id=" + id + + override val type: String + get() = "dialog" } inner class Chat(var id: Int, var name: String?, var count: Int?) : AbstractChat() { override val query: String get() = "source_type IN('group', 'supergroup', 'channel') AND source_id=" + id + + override val type: String + get() = "chat" } inner class User(id: Int, first_name: String?, last_name: String?) { @@ -731,6 +753,9 @@ class Database private constructor(var client: TelegramClient) { inner class GlobalChat : AbstractChat() { override val query: String get() = "1=1" + + override val type: String + get() = "GlobalChat" } companion object { diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt b/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt index f1b5a70..e4e8fae 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt @@ -19,6 +19,8 @@ package de.fabianonline.telegram_backup.exporter import de.fabianonline.telegram_backup.UserManager import de.fabianonline.telegram_backup.Database import de.fabianonline.telegram_backup.anonymize +import de.fabianonline.telegram_backup.toPrettyJson +import de.fabianonline.telegram_backup.CommandLineOptions import java.io.File import java.io.PrintWriter @@ -41,13 +43,14 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory class HTMLExporter { + val db = Database.getInstance() + val user = UserManager.getInstance() @Throws(IOException::class) fun export() { try { - val user = UserManager.getInstance() - val db = Database.getInstance() - + val pagination = if (CommandLineOptions.cmd_no_pagination) -1 else CommandLineOptions.val_pagination + // Create base dir logger.debug("Creating base dir") val base = user.fileBase + "files" + File.separatorChar @@ -95,26 +98,14 @@ class HTMLExporter { w.close() mustache = mf.compile("templates/html/chat.mustache") + val page_mustache = mf.compile("templates/html/page.mustache") var i = 0 println("Generating ${dialogs.size} dialog pages...") for (d in dialogs) { i++ logger.trace("Dialog {}/{}: {}", i, dialogs.size, d.id.toString().anonymize()) - val messages = db.getMessagesForExport(d) - scope.clear() - scope.put("user", user) - scope.put("dialog", d) - scope.put("messages", messages) - - scope.putAll(db.getMessageAuthorsWithCount(d)) - scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(d))) - scope.putAll(db.getMessageTypesWithCount(d)) - scope.putAll(db.getMessageMediaTypesWithCount(d)) - - w = getWriter(base + "dialogs" + File.separatorChar + "user_" + d.id + ".html") - mustache.execute(w, scope) - w.close() + processChat(chat=d, pagination=pagination, index_mustache=mustache, base_dir=base, page_mustache=page_mustache); print(".") if (i % 100 == 0) { println(" - $i/${dialogs.size}") @@ -127,20 +118,7 @@ class HTMLExporter { for (c in chats) { i++ logger.trace("Chat {}/{}: {}", i, chats.size, c.id.toString().anonymize()) - val messages = db.getMessagesForExport(c) - scope.clear() - scope.put("user", user) - scope.put("chat", c) - scope.put("messages", messages) - - scope.putAll(db.getMessageAuthorsWithCount(c)) - scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(c))) - scope.putAll(db.getMessageTypesWithCount(c)) - scope.putAll(db.getMessageMediaTypesWithCount(c)) - - w = getWriter(base + "dialogs" + File.separatorChar + "chat_" + c.id + ".html") - mustache.execute(w, scope) - w.close() + processChat(chat=c, pagination=pagination, index_mustache=mustache, base_dir=base, page_mustache=page_mustache); print(".") if (i % 100 == 0) { println(" - $i/${chats.size}") @@ -163,6 +141,74 @@ class HTMLExporter { } } + + private fun processChat(chat: Database.AbstractChat, pagination: Int, index_mustache: Mustache, base_dir: String, page_mustache: Mustache) { + + val scope = HashMap() + + val count = db.getMessageCountForExport(chat) + + val prefix = if (chat.type == "dialog") "user_" else "chat_" + val id = if (chat is Database.Chat) chat.id else if (chat is Database.Dialog) chat.id else throw IllegalArgumentException("Unexpected unknown id") + + scope.put("user", user) + scope.put(chat.type, chat) + + if (pagination>0 && count>pagination) { // pagination is enabled and we have more messages than allowed on one page + scope.put("paginated", true) + val pages_data = LinkedList>() + + var offset = 0 + var page = 1 + val pages: Int = count / pagination + 1 + val dir = "${base_dir}dialogs${File.separatorChar}" + val filename_base = "${prefix}${id}_p" + while (offset < count) { + val page_scope = HashMap() + val filename = "${filename_base}${page}.html" + + page_scope.put("page", page) + page_scope.put("pages", pages) + page_scope.put(chat.type, chat) + + if (page > 1) page_scope.put("previous_page", "${filename_base}${page-1}.html") + if (page < pages) page_scope.put("next_page", "${filename_base}${page+1}.html") + + val messages = db.getMessagesForExport(chat, limit=pagination, offset=offset) + page_scope.put("messages", messages) + + val w = getWriter(dir + filename) + page_mustache.execute(w, page_scope) + w.close() + + + val data = HashMap() + data.put("page", ""+page) + data.put("filename", "${prefix}${id}_p${page}.html") + data.put("start_time", messages.getFirst().get("formatted_time") as String) + data.put("start_date", messages.getFirst().get("formatted_date") as String) + pages_data.add(data) + page += 1 + offset += pagination + } + scope.put("pages_data", pages_data) + scope.put("pages", pages) + } else { // put all messages on one page + scope.put("paginated", false) + val messages = db.getMessagesForExport(chat) + scope.put("messages", messages) + } + + scope.putAll(db.getMessageAuthorsWithCount(chat)) + scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(chat))) + scope.putAll(db.getMessageTypesWithCount(chat)) + scope.putAll(db.getMessageMediaTypesWithCount(chat)) + + + val w = getWriter(base_dir + "dialogs" + File.separatorChar + prefix + id + ".html") + index_mustache.execute(w, scope) + w.close() + } @Throws(FileNotFoundException::class) private fun getWriter(filename: String): OutputStreamWriter { diff --git a/src/main/resources/templates/html/_messages.mustache b/src/main/resources/templates/html/_messages.mustache new file mode 100644 index 0000000..d25ca2a --- /dev/null +++ b/src/main/resources/templates/html/_messages.mustache @@ -0,0 +1,17 @@ +
    + {{#messages}} + {{#is_new_date}} +
  • + {{formatted_date}} +
    • + {{/is_new_date}} +
  • + {{formatted_time}} + {{user_first_name}} + {{#text}}{{text}}{{/text}} + {{#media_sticker}}{{/media_sticker}} + {{#media_photo}}{{/media_photo}} + {{#media_document}}{{media_file}}{{/media_document}} +
    • + {{/messages}} +
diff --git a/src/main/resources/templates/html/_stats.mustache b/src/main/resources/templates/html/_stats.mustache index d423249..ccf7699 100644 --- a/src/main/resources/templates/html/_stats.mustache +++ b/src/main/resources/templates/html/_stats.mustache @@ -121,7 +121,7 @@ var author_data = [ {{#authors.all}} - ['{{key.name}}', {{value}}], + ['{{name}}', {{count}}], {{/authors.all}} ]; author_data.sort(function(a, b) { return b[1]-a[1]; }); diff --git a/src/main/resources/templates/html/chat.mustache b/src/main/resources/templates/html/chat.mustache index 94489b1..1d87410 100644 --- a/src/main/resources/templates/html/chat.mustache +++ b/src/main/resources/templates/html/chat.mustache @@ -17,24 +17,17 @@ Back to the overview -
    - {{#messages}} - {{#is_new_date}} -
  • - {{formatted_date}} -
    • - {{/is_new_date}} -
  • - {{formatted_time}} - {{user_first_name}} - {{#text}}{{text}}{{/text}} - {{#media_sticker}}{{/media_sticker}} - {{#media_photo}}{{/media_photo}} - {{#media_document}}{{media_file}}{{/media_document}} -
    • - {{/messages}} -
- + {{#paginated}} +

{{pages}} Pages

+
    + {{#pages_data}} +
  • Page {{page}} (starting {{start_date}} {{start_time}})
    • + {{/pages_data}} +
+ {{/paginated}} + {{^paginated}} + {{> _messages }} + {{/paginated}} Back to the overview {{> _stats }} diff --git a/src/main/resources/templates/html/page.mustache b/src/main/resources/templates/html/page.mustache new file mode 100644 index 0000000..4edb4c9 --- /dev/null +++ b/src/main/resources/templates/html/page.mustache @@ -0,0 +1,33 @@ + + + +Telegram Backup for {{user.getUserString}} + + + + + +

Telegram Backup

+{{#dialog}} +

Dialog with {{first_name}} {{last_name}} {{#username}}(@{{username}}){{/username}}

+{{/dialog}} +{{#chat}} +

Chat {{name}}

+{{/chat}} +

Page {{page}} of {{pages}}

+ +{{#previous_page}} + Previous page +{{/previous_page}} +{{#next_page}} + Next page +{{/next_page}} + +{{> _messages }} + +{{#previous_page}} + Previous page +{{/previous_page}} +{{#next_page}} + Next page +{{/next_page}}