Added pagination for the output.

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt

@@ -277,8 +277,9 @@ class CommandLineController {
 		println(" -t, --target <x>      Target directory for the files.")
 		println(" -e, --export <format> Export the database. Valid formats are:")
 		println("                html - Creates HTML files.")
+		println(" --pagination <x>      Splits the HTML export into multiple HTML pages with <x> messages per page. Default is 5000.")
+		println(" --no-pagination       Disables pagination.")
 		println(" --license             Displays the license of this program.")
-		println(" -d, --daemon Keep running and automatically save new messages.")
 		println(" --anonymize           (Try to) Remove all sensitive information from output. Useful for requesting support.")
 		println(" --stats               Print some usage statistics.")
 		println(" --with-channels       Backup channels as well.")

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineOptions.kt

@@ -31,11 +31,13 @@ internal object CommandLineOptions {
 	var cmd_stats = false
 	var cmd_channels = false
 	var cmd_supergroups = false
+	var cmd_no_pagination = false
 	var val_account: String? = null
 	var val_limit_messages: Int? = null
 	var val_target: String? = null
 	var val_export: String? = null
 	var val_test: Int? = null
+	var val_pagination: Int = Config.DEFAULT_PAGINATION
 	@JvmStatic
 	fun parseOptions(args: Array<String>) {
 		var last_cmd: String? = null
@@ -47,6 +49,7 @@ internal object CommandLineOptions {
 					"--target" -> val_target = arg
 					"--export" -> val_export = arg
 					"--test" -> val_test = Integer.parseInt(arg)
+					"--pagination" -> val_pagination = Integer.parseInt(arg)
 				}
 				last_cmd = null
 				continue
@@ -76,6 +79,11 @@ internal object CommandLineOptions {
 					last_cmd = "--export"
 					continue@loop
 				}
+				"--pagination" -> {
+					last_cmd = "--pagination"
+					continue@loop
+				}
+				"--no-pagination" -> cmd_no_pagination = true
 				"--license" -> cmd_license = true
 				"-d", "--daemon" -> cmd_daemon = true
 				"--no-media" -> cmd_no_media = true

src/main/kotlin/de/fabianonline/telegram_backup/Config.kt

@@ -44,6 +44,8 @@ object Config {
 	var RENAMING_MAX_TRIES = 5
 	var RENAMING_DELAY: Long = 1000
 	
+	var DEFAULT_PAGINATION = 5_000
+
 	val SECRET_GMAPS = "AIzaSyBEtUDhCQKEH6i2Mn1GAiQ9M_tLN0vxHIs"
 
 	init {

src/main/kotlin/de/fabianonline/telegram_backup/Database.kt

@@ -595,7 +595,7 @@ class Database private constructor(var client: TelegramClient) {
 
 	fun getMessageAuthorsWithCount(c: AbstractChat): HashMap<String, Any> {
 		val map = HashMap<String, Any>()
-		val user_map = HashMap<User, Int>()
+		val all_data = LinkedList<HashMap<String, String>>()
 		var count_others = 0
 		// Set a default value for 'me' to fix the charts for channels - cause I
 		// possibly didn't send any messages there.
@@ -607,6 +607,7 @@ class Database private constructor(var client: TelegramClient) {
 				"WHERE " + c.query + " GROUP BY sender_id")
 			while (rs.next()) {
 				val u: User
+				val data = HashMap<String, String>()
 				if (rs.getString(2) != null || rs.getString(3) != null || rs.getString(4) != null) {
 					u = User(rs.getInt(1), rs.getString(2), rs.getString(3))
 				} else {
@@ -615,17 +616,25 @@ class Database private constructor(var client: TelegramClient) {
 				if (u.isMe) {
 					map.put("authors.count.me", rs.getInt(5))
 				} else {
-					user_map.put(u, rs.getInt(5))
 					count_others += rs.getInt(5)
+					data.put("name", u.name)
+					data.put("count", ""+rs.getInt(5))
+					all_data.add(data)
 				}
+				
 			}
 			map.put("authors.count.others", count_others)
-			map.put("authors.all", user_map)
+			map.put("authors.all", all_data)
 			return map
 		} catch (e: Exception) {
 			throw RuntimeException(e)
 		}
+	}
 	
+	fun getMessageCountForExport(c: AbstractChat): Int {
+		val rs = stmt!!.executeQuery("SELECT COUNT(*) FROM messages WHERE " + c.query);
+		rs.next()
+		return rs.getInt(1)
 	}
 
 	fun getMessageTimesMatrix(c: AbstractChat): Array<IntArray> {
@@ -645,10 +654,9 @@ class Database private constructor(var client: TelegramClient) {
 
 	}
 
-	fun getMessagesForExport(c: AbstractChat): LinkedList<HashMap<String, Any>> {
+	fun getMessagesForExport(c: AbstractChat, limit: Int=-1, offset: Int=0): LinkedList<HashMap<String, Any>> {
 		try {
-
+			var query = "SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " +
-			val rs = stmt!!.executeQuery("SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " +
 				"media_type, media_file, media_size, users.first_name as user_first_name, users.last_name as user_last_name, " +
 				"users.username as user_username, users.id as user_id, " +
 				"users_fwd.first_name as user_fwd_first_name, users_fwd.last_name as user_fwd_last_name, users_fwd.username as user_fwd_username " +
@@ -656,7 +664,14 @@ class Database private constructor(var client: TelegramClient) {
 				"LEFT JOIN users ON users.id=messages.sender_id " +
 				"LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " +
 				c.query + " " +
-				"ORDER BY messages.message_id")
+				"ORDER BY messages.message_id"
+			
+			if ( limit != -1 ) {
+				query = query + " LIMIT ${limit} OFFSET ${offset}"
+			}	
+			
+			val rs = stmt!!.executeQuery(query)
+			
 			val format_time = SimpleDateFormat("HH:mm:ss")
 			val format_date = SimpleDateFormat("d MMM yy")
 			val meta = rs.getMetaData()
@@ -701,18 +716,25 @@ class Database private constructor(var client: TelegramClient) {
 
 	abstract inner class AbstractChat {
 		abstract val query: String
+		abstract val type: String
 	}
 
 	inner class Dialog(var id: Int, var first_name: String?, var last_name: String?, var username: String?, var count: Int?) : AbstractChat() {
 
 		override val query: String
 			get() = "source_type='dialog' AND source_id=" + id
+		
+		override val type: String
+			get() = "dialog"
 	}
 
 	inner class Chat(var id: Int, var name: String?, var count: Int?) : AbstractChat() {
 
 		override val query: String
 			get() = "source_type IN('group', 'supergroup', 'channel') AND source_id=" + id
+		
+		override val type: String
+			get() = "chat"
 	}
 
 	inner class User(id: Int, first_name: String?, last_name: String?) {
@@ -731,6 +753,9 @@ class Database private constructor(var client: TelegramClient) {
 	inner class GlobalChat : AbstractChat() {
 		override val query: String
 			get() = "1=1"
+		
+		override val type: String
+			get() = "GlobalChat"
 	}
 
 	companion object {

src/main/kotlin/de/fabianonline/telegram_backup/exporter/HTMLExporter.kt

@@ -19,6 +19,8 @@ package de.fabianonline.telegram_backup.exporter
 import de.fabianonline.telegram_backup.UserManager
 import de.fabianonline.telegram_backup.Database
 import de.fabianonline.telegram_backup.anonymize
+import de.fabianonline.telegram_backup.toPrettyJson
+import de.fabianonline.telegram_backup.CommandLineOptions
 
 import java.io.File
 import java.io.PrintWriter
@@ -41,12 +43,13 @@ import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
 class HTMLExporter {
+	val db = Database.getInstance()
+	val user = UserManager.getInstance()
 
 	@Throws(IOException::class)
 	fun export() {
 		try {
-			val user = UserManager.getInstance()
+			val pagination = if (CommandLineOptions.cmd_no_pagination) -1 else CommandLineOptions.val_pagination
-			val db = Database.getInstance()
 			
 			// Create base dir
 			logger.debug("Creating base dir")
@@ -95,26 +98,14 @@ class HTMLExporter {
 			w.close()
 
 			mustache = mf.compile("templates/html/chat.mustache")
+			val page_mustache = mf.compile("templates/html/page.mustache")
 
 			var i = 0
 			println("Generating ${dialogs.size} dialog pages...")
 			for (d in dialogs) {
 				i++
 				logger.trace("Dialog {}/{}: {}", i, dialogs.size, d.id.toString().anonymize())
-				val messages = db.getMessagesForExport(d)
+				processChat(chat=d, pagination=pagination, index_mustache=mustache, base_dir=base, page_mustache=page_mustache);
-				scope.clear()
-				scope.put("user", user)
-				scope.put("dialog", d)
-				scope.put("messages", messages)
-
-				scope.putAll(db.getMessageAuthorsWithCount(d))
-				scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(d)))
-				scope.putAll(db.getMessageTypesWithCount(d))
-				scope.putAll(db.getMessageMediaTypesWithCount(d))
-
-				w = getWriter(base + "dialogs" + File.separatorChar + "user_" + d.id + ".html")
-				mustache.execute(w, scope)
-				w.close()
 				print(".")
 				if (i % 100 == 0) {
 					println(" - $i/${dialogs.size}")
@@ -127,20 +118,7 @@ class HTMLExporter {
 			for (c in chats) {
 				i++
 				logger.trace("Chat {}/{}: {}", i, chats.size, c.id.toString().anonymize())
-				val messages = db.getMessagesForExport(c)
+				processChat(chat=c, pagination=pagination, index_mustache=mustache, base_dir=base, page_mustache=page_mustache);
-				scope.clear()
-				scope.put("user", user)
-				scope.put("chat", c)
-				scope.put("messages", messages)
-
-				scope.putAll(db.getMessageAuthorsWithCount(c))
-				scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(c)))
-				scope.putAll(db.getMessageTypesWithCount(c))
-				scope.putAll(db.getMessageMediaTypesWithCount(c))
-
-				w = getWriter(base + "dialogs" + File.separatorChar + "chat_" + c.id + ".html")
-				mustache.execute(w, scope)
-				w.close()
 				print(".")
 				if (i % 100 == 0) {
 					println(" - $i/${chats.size}")
@@ -164,6 +142,74 @@ class HTMLExporter {
 
 	}
 	
+	private fun processChat(chat: Database.AbstractChat, pagination: Int, index_mustache: Mustache, base_dir: String, page_mustache: Mustache) {
+		
+		val scope = HashMap<String, Any>()
+		
+		val count = db.getMessageCountForExport(chat)
+		
+		val prefix = if (chat.type == "dialog") "user_" else "chat_"
+		val id = if (chat is Database.Chat) chat.id else if (chat is Database.Dialog) chat.id else throw IllegalArgumentException("Unexpected unknown id")
+		
+		scope.put("user", user)
+		scope.put(chat.type, chat)
+		
+		if (pagination>0 && count>pagination) { // pagination is enabled and we have more messages than allowed on one page
+			scope.put("paginated", true)
+			val pages_data = LinkedList<HashMap<String, String>>()
+			
+			var offset = 0
+			var page = 1
+			val pages: Int = count / pagination + 1
+			val dir = "${base_dir}dialogs${File.separatorChar}"
+			val filename_base = "${prefix}${id}_p"
+			while (offset < count) {
+				val page_scope = HashMap<String, Any>()
+				val filename = "${filename_base}${page}.html"
+				
+				page_scope.put("page", page)
+				page_scope.put("pages", pages)
+				page_scope.put(chat.type, chat)
+				
+				if (page > 1) page_scope.put("previous_page", "${filename_base}${page-1}.html")
+				if (page < pages) page_scope.put("next_page", "${filename_base}${page+1}.html")
+				
+				val messages = db.getMessagesForExport(chat, limit=pagination, offset=offset)
+				page_scope.put("messages", messages)
+				
+				val w = getWriter(dir + filename)
+				page_mustache.execute(w, page_scope)
+				w.close()
+				
+				
+				val data = HashMap<String, String>()
+				data.put("page", ""+page)
+				data.put("filename", "${prefix}${id}_p${page}.html")
+				data.put("start_time", messages.getFirst().get("formatted_time") as String)
+				data.put("start_date", messages.getFirst().get("formatted_date") as String)
+				pages_data.add(data)
+				page += 1
+				offset += pagination
+			}
+			scope.put("pages_data", pages_data)
+			scope.put("pages", pages)
+		} else { // put all messages on one page
+			scope.put("paginated", false)
+			val messages = db.getMessagesForExport(chat)
+			scope.put("messages", messages)
+		}
+
+		scope.putAll(db.getMessageAuthorsWithCount(chat))
+		scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(chat)))
+		scope.putAll(db.getMessageTypesWithCount(chat))
+		scope.putAll(db.getMessageMediaTypesWithCount(chat))
+
+		
+		val w = getWriter(base_dir + "dialogs" + File.separatorChar + prefix + id + ".html")
+		index_mustache.execute(w, scope)
+		w.close()
+	}
+
 	@Throws(FileNotFoundException::class)
 	private fun getWriter(filename: String): OutputStreamWriter {
 		logger.trace("Creating writer for file {}", filename.anonymize())

src/main/resources/templates/html/_messages.mustache

@@ -0,0 +1,17 @@
+<ul class="messages">
+	{{#messages}}
+		{{#is_new_date}}
+		<li class="date">
+			{{formatted_date}}
+		</li>
+		{{/is_new_date}}
+		<li class="message {{#from_me}}from-me{{/from_me}} {{odd_even}} {{#same_user}}same-user{{/same_user}}" data-message-id="{{message_id}}" data-media="{{media_type}}">
+			<span class="time">{{formatted_time}}</span>
+			<span class="sender">{{user_first_name}}</span>
+			{{#text}}<span class="text">{{text}}</span>{{/text}}
+			{{#media_sticker}}<span class="sticker"><img src="../stickers/{{media_file}}" /></span>{{/media_sticker}}
+			{{#media_photo}}<span class="photo"><img src="../{{media_file}}" /></span>{{/media_photo}}
+			{{#media_document}}<span class="document"><a href="../{{media_file}}">{{media_file}}</a></span>{{/media_document}}
+		</li>
+	{{/messages}}
+</ul>

src/main/resources/templates/html/_stats.mustache

@@ -121,7 +121,7 @@
 		
 		var author_data = [
 			{{#authors.all}}
-				['{{key.name}}', {{value}}],
+				['{{name}}', {{count}}],
 			{{/authors.all}}
 		];
 		author_data.sort(function(a, b) { return b[1]-a[1]; });

src/main/resources/templates/html/chat.mustache

@@ -17,24 +17,17 @@
 	
 	<a href="../index.html">Back to the overview</a>
 	
-	<ul class="messages">
+	{{#paginated}}
-		{{#messages}}
+		<h3>{{pages}} Pages</h3>
-			{{#is_new_date}}
+		<ul>
-			<li class="date">
+		{{#pages_data}}
-				{{formatted_date}}
+			<li><a href="{{filename}}">Page {{page}}</a> (starting {{start_date}} {{start_time}})</li>
-			</li>
+		{{/pages_data}}
-			{{/is_new_date}}
-			<li class="message {{#from_me}}from-me{{/from_me}} {{odd_even}} {{#same_user}}same-user{{/same_user}}" data-message-id="{{message_id}}" data-media="{{media_type}}">
-				<span class="time">{{formatted_time}}</span>
-				<span class="sender">{{user_first_name}}</span>
-				{{#text}}<span class="text">{{text}}</span>{{/text}}
-				{{#media_sticker}}<span class="sticker"><img src="../stickers/{{media_file}}" /></span>{{/media_sticker}}
-				{{#media_photo}}<span class="photo"><img src="../{{media_file}}" /></span>{{/media_photo}}
-				{{#media_document}}<span class="document"><a href="../{{media_file}}">{{media_file}}</a></span>{{/media_document}}
-			</li>
-		{{/messages}}
 		</ul>
-	
+	{{/paginated}}
+	{{^paginated}}
+		{{> _messages }}
+	{{/paginated}}
 	<a href="../index.html">Back to the overview</a>
 	
 	{{> _stats }}

src/main/resources/templates/html/page.mustache

@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>Telegram Backup for {{user.getUserString}}</title>
+<meta charset="UTF-8">
+<link rel="stylesheet" href="../style.css" type="text/css" />
+</head>
+<body>
+
+<h1>Telegram Backup</h1>
+{{#dialog}}
+	<h2>Dialog with {{first_name}} {{last_name}} {{#username}}(@{{username}}){{/username}}</h2>
+{{/dialog}}
+{{#chat}}
+	<h2>Chat {{name}}</h2>
+{{/chat}}
+<h3>Page {{page}} of {{pages}}</h3>
+
+{{#previous_page}}
+	<a href="{{previous_page}}">Previous page</a>
+{{/previous_page}}
+{{#next_page}}
+	<a href="{{next_page}}">Next page</a>
+{{/next_page}}
+
+{{> _messages }}
+
+{{#previous_page}}
+	<a href="{{previous_page}}">Previous page</a>
+{{/previous_page}}
+{{#next_page}}
+	<a href="{{next_page}}">Next page</a>
+{{/next_page}}