mirror of
https://github.com/fabianonline/telegram_backup.git
synced 2024-10-31 23:54:05 +00:00
Added pagination for the output.
This commit is contained in:
parent
ab16c44de5
commit
0e2eeab5b9
@ -265,24 +265,25 @@ class CommandLineController {
|
||||
|
||||
private fun show_help() {
|
||||
println("Valid options are:")
|
||||
println(" -h, --help Shows this help.")
|
||||
println(" -a, --account <x> Use account <x>.")
|
||||
println(" -l, --login Login to an existing telegram account.")
|
||||
println(" --debug Shows some debug information.")
|
||||
println(" --trace Shows lots of debug information. Overrides --debug.")
|
||||
println(" --trace-telegram Shows lots of debug messages from the library used to access Telegram.")
|
||||
println(" -A, --list-accounts List all existing accounts ")
|
||||
println(" --limit-messages <x> Downloads at most the most recent <x> messages.")
|
||||
println(" --no-media Do not download media files.")
|
||||
println(" -t, --target <x> Target directory for the files.")
|
||||
println(" -h, --help Shows this help.")
|
||||
println(" -a, --account <x> Use account <x>.")
|
||||
println(" -l, --login Login to an existing telegram account.")
|
||||
println(" --debug Shows some debug information.")
|
||||
println(" --trace Shows lots of debug information. Overrides --debug.")
|
||||
println(" --trace-telegram Shows lots of debug messages from the library used to access Telegram.")
|
||||
println(" -A, --list-accounts List all existing accounts ")
|
||||
println(" --limit-messages <x> Downloads at most the most recent <x> messages.")
|
||||
println(" --no-media Do not download media files.")
|
||||
println(" -t, --target <x> Target directory for the files.")
|
||||
println(" -e, --export <format> Export the database. Valid formats are:")
|
||||
println(" html - Creates HTML files.")
|
||||
println(" --license Displays the license of this program.")
|
||||
println(" -d, --daemon Keep running and automatically save new messages.")
|
||||
println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.")
|
||||
println(" --stats Print some usage statistics.")
|
||||
println(" --with-channels Backup channels as well.")
|
||||
println(" --with-supergroups Backup supergroups as well.")
|
||||
println(" html - Creates HTML files.")
|
||||
println(" --pagination <x> Splits the HTML export into multiple HTML pages with <x> messages per page. Default is 5000.")
|
||||
println(" --no-pagination Disables pagination.")
|
||||
println(" --license Displays the license of this program.")
|
||||
println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.")
|
||||
println(" --stats Print some usage statistics.")
|
||||
println(" --with-channels Backup channels as well.")
|
||||
println(" --with-supergroups Backup supergroups as well.")
|
||||
}
|
||||
|
||||
private fun list_accounts() {
|
||||
|
@ -31,11 +31,13 @@ internal object CommandLineOptions {
|
||||
var cmd_stats = false
|
||||
var cmd_channels = false
|
||||
var cmd_supergroups = false
|
||||
var cmd_no_pagination = false
|
||||
var val_account: String? = null
|
||||
var val_limit_messages: Int? = null
|
||||
var val_target: String? = null
|
||||
var val_export: String? = null
|
||||
var val_test: Int? = null
|
||||
var val_pagination: Int = Config.DEFAULT_PAGINATION
|
||||
@JvmStatic
|
||||
fun parseOptions(args: Array<String>) {
|
||||
var last_cmd: String? = null
|
||||
@ -47,6 +49,7 @@ internal object CommandLineOptions {
|
||||
"--target" -> val_target = arg
|
||||
"--export" -> val_export = arg
|
||||
"--test" -> val_test = Integer.parseInt(arg)
|
||||
"--pagination" -> val_pagination = Integer.parseInt(arg)
|
||||
}
|
||||
last_cmd = null
|
||||
continue
|
||||
@ -76,6 +79,11 @@ internal object CommandLineOptions {
|
||||
last_cmd = "--export"
|
||||
continue@loop
|
||||
}
|
||||
"--pagination" -> {
|
||||
last_cmd = "--pagination"
|
||||
continue@loop
|
||||
}
|
||||
"--no-pagination" -> cmd_no_pagination = true
|
||||
"--license" -> cmd_license = true
|
||||
"-d", "--daemon" -> cmd_daemon = true
|
||||
"--no-media" -> cmd_no_media = true
|
||||
|
@ -43,6 +43,8 @@ object Config {
|
||||
|
||||
var RENAMING_MAX_TRIES = 5
|
||||
var RENAMING_DELAY: Long = 1000
|
||||
|
||||
var DEFAULT_PAGINATION = 5_000
|
||||
|
||||
val SECRET_GMAPS = "AIzaSyBEtUDhCQKEH6i2Mn1GAiQ9M_tLN0vxHIs"
|
||||
|
||||
|
@ -595,7 +595,7 @@ class Database private constructor(var client: TelegramClient) {
|
||||
|
||||
fun getMessageAuthorsWithCount(c: AbstractChat): HashMap<String, Any> {
|
||||
val map = HashMap<String, Any>()
|
||||
val user_map = HashMap<User, Int>()
|
||||
val all_data = LinkedList<HashMap<String, String>>()
|
||||
var count_others = 0
|
||||
// Set a default value for 'me' to fix the charts for channels - cause I
|
||||
// possibly didn't send any messages there.
|
||||
@ -607,6 +607,7 @@ class Database private constructor(var client: TelegramClient) {
|
||||
"WHERE " + c.query + " GROUP BY sender_id")
|
||||
while (rs.next()) {
|
||||
val u: User
|
||||
val data = HashMap<String, String>()
|
||||
if (rs.getString(2) != null || rs.getString(3) != null || rs.getString(4) != null) {
|
||||
u = User(rs.getInt(1), rs.getString(2), rs.getString(3))
|
||||
} else {
|
||||
@ -615,17 +616,25 @@ class Database private constructor(var client: TelegramClient) {
|
||||
if (u.isMe) {
|
||||
map.put("authors.count.me", rs.getInt(5))
|
||||
} else {
|
||||
user_map.put(u, rs.getInt(5))
|
||||
count_others += rs.getInt(5)
|
||||
data.put("name", u.name)
|
||||
data.put("count", ""+rs.getInt(5))
|
||||
all_data.add(data)
|
||||
}
|
||||
|
||||
}
|
||||
map.put("authors.count.others", count_others)
|
||||
map.put("authors.all", user_map)
|
||||
map.put("authors.all", all_data)
|
||||
return map
|
||||
} catch (e: Exception) {
|
||||
throw RuntimeException(e)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fun getMessageCountForExport(c: AbstractChat): Int {
|
||||
val rs = stmt!!.executeQuery("SELECT COUNT(*) FROM messages WHERE " + c.query);
|
||||
rs.next()
|
||||
return rs.getInt(1)
|
||||
}
|
||||
|
||||
fun getMessageTimesMatrix(c: AbstractChat): Array<IntArray> {
|
||||
@ -645,10 +654,9 @@ class Database private constructor(var client: TelegramClient) {
|
||||
|
||||
}
|
||||
|
||||
fun getMessagesForExport(c: AbstractChat): LinkedList<HashMap<String, Any>> {
|
||||
fun getMessagesForExport(c: AbstractChat, limit: Int=-1, offset: Int=0): LinkedList<HashMap<String, Any>> {
|
||||
try {
|
||||
|
||||
val rs = stmt!!.executeQuery("SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " +
|
||||
var query = "SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " +
|
||||
"media_type, media_file, media_size, users.first_name as user_first_name, users.last_name as user_last_name, " +
|
||||
"users.username as user_username, users.id as user_id, " +
|
||||
"users_fwd.first_name as user_fwd_first_name, users_fwd.last_name as user_fwd_last_name, users_fwd.username as user_fwd_username " +
|
||||
@ -656,7 +664,14 @@ class Database private constructor(var client: TelegramClient) {
|
||||
"LEFT JOIN users ON users.id=messages.sender_id " +
|
||||
"LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " +
|
||||
c.query + " " +
|
||||
"ORDER BY messages.message_id")
|
||||
"ORDER BY messages.message_id"
|
||||
|
||||
if ( limit != -1 ) {
|
||||
query = query + " LIMIT ${limit} OFFSET ${offset}"
|
||||
}
|
||||
|
||||
val rs = stmt!!.executeQuery(query)
|
||||
|
||||
val format_time = SimpleDateFormat("HH:mm:ss")
|
||||
val format_date = SimpleDateFormat("d MMM yy")
|
||||
val meta = rs.getMetaData()
|
||||
@ -701,18 +716,25 @@ class Database private constructor(var client: TelegramClient) {
|
||||
|
||||
abstract inner class AbstractChat {
|
||||
abstract val query: String
|
||||
abstract val type: String
|
||||
}
|
||||
|
||||
inner class Dialog(var id: Int, var first_name: String?, var last_name: String?, var username: String?, var count: Int?) : AbstractChat() {
|
||||
|
||||
override val query: String
|
||||
get() = "source_type='dialog' AND source_id=" + id
|
||||
|
||||
override val type: String
|
||||
get() = "dialog"
|
||||
}
|
||||
|
||||
inner class Chat(var id: Int, var name: String?, var count: Int?) : AbstractChat() {
|
||||
|
||||
override val query: String
|
||||
get() = "source_type IN('group', 'supergroup', 'channel') AND source_id=" + id
|
||||
|
||||
override val type: String
|
||||
get() = "chat"
|
||||
}
|
||||
|
||||
inner class User(id: Int, first_name: String?, last_name: String?) {
|
||||
@ -731,6 +753,9 @@ class Database private constructor(var client: TelegramClient) {
|
||||
inner class GlobalChat : AbstractChat() {
|
||||
override val query: String
|
||||
get() = "1=1"
|
||||
|
||||
override val type: String
|
||||
get() = "GlobalChat"
|
||||
}
|
||||
|
||||
companion object {
|
||||
|
@ -19,6 +19,8 @@ package de.fabianonline.telegram_backup.exporter
|
||||
import de.fabianonline.telegram_backup.UserManager
|
||||
import de.fabianonline.telegram_backup.Database
|
||||
import de.fabianonline.telegram_backup.anonymize
|
||||
import de.fabianonline.telegram_backup.toPrettyJson
|
||||
import de.fabianonline.telegram_backup.CommandLineOptions
|
||||
|
||||
import java.io.File
|
||||
import java.io.PrintWriter
|
||||
@ -41,13 +43,14 @@ import org.slf4j.Logger
|
||||
import org.slf4j.LoggerFactory
|
||||
|
||||
class HTMLExporter {
|
||||
val db = Database.getInstance()
|
||||
val user = UserManager.getInstance()
|
||||
|
||||
@Throws(IOException::class)
|
||||
fun export() {
|
||||
try {
|
||||
val user = UserManager.getInstance()
|
||||
val db = Database.getInstance()
|
||||
|
||||
val pagination = if (CommandLineOptions.cmd_no_pagination) -1 else CommandLineOptions.val_pagination
|
||||
|
||||
// Create base dir
|
||||
logger.debug("Creating base dir")
|
||||
val base = user.fileBase + "files" + File.separatorChar
|
||||
@ -95,26 +98,14 @@ class HTMLExporter {
|
||||
w.close()
|
||||
|
||||
mustache = mf.compile("templates/html/chat.mustache")
|
||||
val page_mustache = mf.compile("templates/html/page.mustache")
|
||||
|
||||
var i = 0
|
||||
println("Generating ${dialogs.size} dialog pages...")
|
||||
for (d in dialogs) {
|
||||
i++
|
||||
logger.trace("Dialog {}/{}: {}", i, dialogs.size, d.id.toString().anonymize())
|
||||
val messages = db.getMessagesForExport(d)
|
||||
scope.clear()
|
||||
scope.put("user", user)
|
||||
scope.put("dialog", d)
|
||||
scope.put("messages", messages)
|
||||
|
||||
scope.putAll(db.getMessageAuthorsWithCount(d))
|
||||
scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(d)))
|
||||
scope.putAll(db.getMessageTypesWithCount(d))
|
||||
scope.putAll(db.getMessageMediaTypesWithCount(d))
|
||||
|
||||
w = getWriter(base + "dialogs" + File.separatorChar + "user_" + d.id + ".html")
|
||||
mustache.execute(w, scope)
|
||||
w.close()
|
||||
processChat(chat=d, pagination=pagination, index_mustache=mustache, base_dir=base, page_mustache=page_mustache);
|
||||
print(".")
|
||||
if (i % 100 == 0) {
|
||||
println(" - $i/${dialogs.size}")
|
||||
@ -127,20 +118,7 @@ class HTMLExporter {
|
||||
for (c in chats) {
|
||||
i++
|
||||
logger.trace("Chat {}/{}: {}", i, chats.size, c.id.toString().anonymize())
|
||||
val messages = db.getMessagesForExport(c)
|
||||
scope.clear()
|
||||
scope.put("user", user)
|
||||
scope.put("chat", c)
|
||||
scope.put("messages", messages)
|
||||
|
||||
scope.putAll(db.getMessageAuthorsWithCount(c))
|
||||
scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(c)))
|
||||
scope.putAll(db.getMessageTypesWithCount(c))
|
||||
scope.putAll(db.getMessageMediaTypesWithCount(c))
|
||||
|
||||
w = getWriter(base + "dialogs" + File.separatorChar + "chat_" + c.id + ".html")
|
||||
mustache.execute(w, scope)
|
||||
w.close()
|
||||
processChat(chat=c, pagination=pagination, index_mustache=mustache, base_dir=base, page_mustache=page_mustache);
|
||||
print(".")
|
||||
if (i % 100 == 0) {
|
||||
println(" - $i/${chats.size}")
|
||||
@ -163,6 +141,74 @@ class HTMLExporter {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private fun processChat(chat: Database.AbstractChat, pagination: Int, index_mustache: Mustache, base_dir: String, page_mustache: Mustache) {
|
||||
|
||||
val scope = HashMap<String, Any>()
|
||||
|
||||
val count = db.getMessageCountForExport(chat)
|
||||
|
||||
val prefix = if (chat.type == "dialog") "user_" else "chat_"
|
||||
val id = if (chat is Database.Chat) chat.id else if (chat is Database.Dialog) chat.id else throw IllegalArgumentException("Unexpected unknown id")
|
||||
|
||||
scope.put("user", user)
|
||||
scope.put(chat.type, chat)
|
||||
|
||||
if (pagination>0 && count>pagination) { // pagination is enabled and we have more messages than allowed on one page
|
||||
scope.put("paginated", true)
|
||||
val pages_data = LinkedList<HashMap<String, String>>()
|
||||
|
||||
var offset = 0
|
||||
var page = 1
|
||||
val pages: Int = count / pagination + 1
|
||||
val dir = "${base_dir}dialogs${File.separatorChar}"
|
||||
val filename_base = "${prefix}${id}_p"
|
||||
while (offset < count) {
|
||||
val page_scope = HashMap<String, Any>()
|
||||
val filename = "${filename_base}${page}.html"
|
||||
|
||||
page_scope.put("page", page)
|
||||
page_scope.put("pages", pages)
|
||||
page_scope.put(chat.type, chat)
|
||||
|
||||
if (page > 1) page_scope.put("previous_page", "${filename_base}${page-1}.html")
|
||||
if (page < pages) page_scope.put("next_page", "${filename_base}${page+1}.html")
|
||||
|
||||
val messages = db.getMessagesForExport(chat, limit=pagination, offset=offset)
|
||||
page_scope.put("messages", messages)
|
||||
|
||||
val w = getWriter(dir + filename)
|
||||
page_mustache.execute(w, page_scope)
|
||||
w.close()
|
||||
|
||||
|
||||
val data = HashMap<String, String>()
|
||||
data.put("page", ""+page)
|
||||
data.put("filename", "${prefix}${id}_p${page}.html")
|
||||
data.put("start_time", messages.getFirst().get("formatted_time") as String)
|
||||
data.put("start_date", messages.getFirst().get("formatted_date") as String)
|
||||
pages_data.add(data)
|
||||
page += 1
|
||||
offset += pagination
|
||||
}
|
||||
scope.put("pages_data", pages_data)
|
||||
scope.put("pages", pages)
|
||||
} else { // put all messages on one page
|
||||
scope.put("paginated", false)
|
||||
val messages = db.getMessagesForExport(chat)
|
||||
scope.put("messages", messages)
|
||||
}
|
||||
|
||||
scope.putAll(db.getMessageAuthorsWithCount(chat))
|
||||
scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(chat)))
|
||||
scope.putAll(db.getMessageTypesWithCount(chat))
|
||||
scope.putAll(db.getMessageMediaTypesWithCount(chat))
|
||||
|
||||
|
||||
val w = getWriter(base_dir + "dialogs" + File.separatorChar + prefix + id + ".html")
|
||||
index_mustache.execute(w, scope)
|
||||
w.close()
|
||||
}
|
||||
|
||||
@Throws(FileNotFoundException::class)
|
||||
private fun getWriter(filename: String): OutputStreamWriter {
|
||||
|
17
src/main/resources/templates/html/_messages.mustache
Normal file
17
src/main/resources/templates/html/_messages.mustache
Normal file
@ -0,0 +1,17 @@
|
||||
<ul class="messages">
|
||||
{{#messages}}
|
||||
{{#is_new_date}}
|
||||
<li class="date">
|
||||
{{formatted_date}}
|
||||
</li>
|
||||
{{/is_new_date}}
|
||||
<li class="message {{#from_me}}from-me{{/from_me}} {{odd_even}} {{#same_user}}same-user{{/same_user}}" data-message-id="{{message_id}}" data-media="{{media_type}}">
|
||||
<span class="time">{{formatted_time}}</span>
|
||||
<span class="sender">{{user_first_name}}</span>
|
||||
{{#text}}<span class="text">{{text}}</span>{{/text}}
|
||||
{{#media_sticker}}<span class="sticker"><img src="../stickers/{{media_file}}" /></span>{{/media_sticker}}
|
||||
{{#media_photo}}<span class="photo"><img src="../{{media_file}}" /></span>{{/media_photo}}
|
||||
{{#media_document}}<span class="document"><a href="../{{media_file}}">{{media_file}}</a></span>{{/media_document}}
|
||||
</li>
|
||||
{{/messages}}
|
||||
</ul>
|
@ -121,7 +121,7 @@
|
||||
|
||||
var author_data = [
|
||||
{{#authors.all}}
|
||||
['{{key.name}}', {{value}}],
|
||||
['{{name}}', {{count}}],
|
||||
{{/authors.all}}
|
||||
];
|
||||
author_data.sort(function(a, b) { return b[1]-a[1]; });
|
||||
|
@ -17,24 +17,17 @@
|
||||
|
||||
<a href="../index.html">Back to the overview</a>
|
||||
|
||||
<ul class="messages">
|
||||
{{#messages}}
|
||||
{{#is_new_date}}
|
||||
<li class="date">
|
||||
{{formatted_date}}
|
||||
</li>
|
||||
{{/is_new_date}}
|
||||
<li class="message {{#from_me}}from-me{{/from_me}} {{odd_even}} {{#same_user}}same-user{{/same_user}}" data-message-id="{{message_id}}" data-media="{{media_type}}">
|
||||
<span class="time">{{formatted_time}}</span>
|
||||
<span class="sender">{{user_first_name}}</span>
|
||||
{{#text}}<span class="text">{{text}}</span>{{/text}}
|
||||
{{#media_sticker}}<span class="sticker"><img src="../stickers/{{media_file}}" /></span>{{/media_sticker}}
|
||||
{{#media_photo}}<span class="photo"><img src="../{{media_file}}" /></span>{{/media_photo}}
|
||||
{{#media_document}}<span class="document"><a href="../{{media_file}}">{{media_file}}</a></span>{{/media_document}}
|
||||
</li>
|
||||
{{/messages}}
|
||||
</ul>
|
||||
|
||||
{{#paginated}}
|
||||
<h3>{{pages}} Pages</h3>
|
||||
<ul>
|
||||
{{#pages_data}}
|
||||
<li><a href="{{filename}}">Page {{page}}</a> (starting {{start_date}} {{start_time}})</li>
|
||||
{{/pages_data}}
|
||||
</ul>
|
||||
{{/paginated}}
|
||||
{{^paginated}}
|
||||
{{> _messages }}
|
||||
{{/paginated}}
|
||||
<a href="../index.html">Back to the overview</a>
|
||||
|
||||
{{> _stats }}
|
||||
|
33
src/main/resources/templates/html/page.mustache
Normal file
33
src/main/resources/templates/html/page.mustache
Normal file
@ -0,0 +1,33 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Telegram Backup for {{user.getUserString}}</title>
|
||||
<meta charset="UTF-8">
|
||||
<link rel="stylesheet" href="../style.css" type="text/css" />
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<h1>Telegram Backup</h1>
|
||||
{{#dialog}}
|
||||
<h2>Dialog with {{first_name}} {{last_name}} {{#username}}(@{{username}}){{/username}}</h2>
|
||||
{{/dialog}}
|
||||
{{#chat}}
|
||||
<h2>Chat {{name}}</h2>
|
||||
{{/chat}}
|
||||
<h3>Page {{page}} of {{pages}}</h3>
|
||||
|
||||
{{#previous_page}}
|
||||
<a href="{{previous_page}}">Previous page</a>
|
||||
{{/previous_page}}
|
||||
{{#next_page}}
|
||||
<a href="{{next_page}}">Next page</a>
|
||||
{{/next_page}}
|
||||
|
||||
{{> _messages }}
|
||||
|
||||
{{#previous_page}}
|
||||
<a href="{{previous_page}}">Previous page</a>
|
||||
{{/previous_page}}
|
||||
{{#next_page}}
|
||||
<a href="{{next_page}}">Next page</a>
|
||||
{{/next_page}}
|
Loading…
Reference in New Issue
Block a user