From c8e33fa319cc29da75722506192e9915e53395c8 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Tue, 5 Jul 2016 12:55:25 +0200 Subject: [PATCH] Added beginnings of an HTMLExporter. --- .../CommandLineController.java | 16 +- .../telegram_backup/CommandLineOptions.java | 9 +- .../telegram_backup/Database.java | 144 +++++++++++++++++- .../telegram_backup/DownloadManager.java | 37 +++-- .../telegram_backup/HTMLExporter.java | 77 ++++++++++ 5 files changed, 262 insertions(+), 21 deletions(-) create mode 100644 src/main/java/de/fabianonline/telegram_backup/HTMLExporter.java diff --git a/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java b/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java index 95feddb..e42f6de 100644 --- a/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java +++ b/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java @@ -29,9 +29,8 @@ public class CommandLineController { Config.FILE_BASE = options.target; } - System.out.println("Target directory for files: " + Config.FILE_BASE); + System.out.println("Base directory for files: " + Config.FILE_BASE); - if (options.cmd_list_accounts) this.list_accounts(); app = new TelegramApp(Config.APP_ID, Config.APP_HASH, Config.APP_MODEL, Config.APP_SYSVER, Config.APP_APPVER, Config.APP_LANG); @@ -60,12 +59,23 @@ public class CommandLineController { "Use '--list-accounts' to see all available accounts."); } + storage = new ApiStorage(account); TelegramClient client = Kotlogram.getDefaultClient(app, storage); try { user = new UserManager(client); + if (options.export != null) { + if (options.export.toLowerCase().equals("html")) { + (new HTMLExporter()).export(user); + System.exit(0); + } else { + show_error("Unknown export format."); + } + } + + if (options.cmd_login) { System.out.println("Please enter your phone number in international format."); System.out.println("Example: +4917077651234"); @@ -114,6 +124,8 @@ public class CommandLineController { System.out.println(" -A, --list-accounts List all existing accounts "); System.out.println(" --limit-messages Downloads at most the most recent messages."); System.out.println(" -t, --target Target directory for the files."); + System.out.println(" -e, --export Export the database. Valid formats are:"); + System.out.println(" html - Creates HTML files."); System.exit(0); } diff --git a/src/main/java/de/fabianonline/telegram_backup/CommandLineOptions.java b/src/main/java/de/fabianonline/telegram_backup/CommandLineOptions.java index cbba44a..c732148 100644 --- a/src/main/java/de/fabianonline/telegram_backup/CommandLineOptions.java +++ b/src/main/java/de/fabianonline/telegram_backup/CommandLineOptions.java @@ -10,6 +10,7 @@ class CommandLineOptions { public Integer limit_messages = null; public String target = null; public boolean cmd_version = false; + public String export = null; public CommandLineOptions(String[] args) { String last_cmd = null; @@ -26,6 +27,9 @@ class CommandLineOptions { case "--target": this.target = arg; break; + case "--export": + this.export = arg; + break; } last_cmd = null; continue; @@ -57,7 +61,10 @@ class CommandLineOptions { last_cmd = "--target"; continue; case "-V": case "--version": - this.cmd_version = true; continue; + this.cmd_version = true; break; + + case "-e": case "--export": + last_cmd = "--export"; continue; default: throw new RuntimeException("Unknown command " + arg); diff --git a/src/main/java/de/fabianonline/telegram_backup/Database.java b/src/main/java/de/fabianonline/telegram_backup/Database.java index 2300e49..5461e54 100644 --- a/src/main/java/de/fabianonline/telegram_backup/Database.java +++ b/src/main/java/de/fabianonline/telegram_backup/Database.java @@ -10,11 +10,11 @@ import java.sql.SQLException; import java.sql.ResultSet; import java.sql.PreparedStatement; import java.sql.Types; +import java.sql.Time; import java.io.File; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.util.LinkedList; -import java.sql.Array; import de.fabianonline.telegram_backup.UserManager; import de.fabianonline.telegram_backup.StickerConverter; @@ -98,6 +98,21 @@ class Database { stmt.executeUpdate("INSERT INTO database_versions (version) VALUES (2)"); version = 2; } + if (version==2) { + System.out.println(" Updating to version 3..."); + stmt.executeUpdate("ALTER TABLE dialogs RENAME TO 'chats'"); + stmt.executeUpdate("INSERT INTO database_versions (version) VALUES (3)"); + version = 3; + } + if (version==3) { + System.out.println(" Updating to version 4..."); + stmt.executeUpdate("CREATE TABLE messages_new (id INTEGER PRIMARY KEY ASC, dialog_id INTEGER, to_id INTEGER, from_id INTEGER, from_type TEXT, text TEXT, time INTEGER, has_media BOOLEAN, sticker TEXT, data BLOB,type TEXT);"); + stmt.executeUpdate("INSERT INTO messages_new SELECT * FROM messages"); + stmt.executeUpdate("DROP TABLE messages"); + stmt.executeUpdate("ALTER TABLE messages_new RENAME TO 'messages'"); + stmt.executeUpdate("INSERT INTO database_versions (version) VALUES (4)"); + version = 4; + } System.out.println("Database is ready."); } catch (SQLException e) { @@ -264,12 +279,12 @@ class Database { public void saveChats(TLVector all) { try { PreparedStatement ps_insert_or_replace = conn.prepareStatement( - "INSERT OR REPLACE INTO dialogs " + + "INSERT OR REPLACE INTO chats " + "(id, name, type) "+ "VALUES " + "(?, ?, ?)"); PreparedStatement ps_insert_or_ignore = conn.prepareStatement( - "INSERT OR IGNORE INTO dialogs " + + "INSERT OR IGNORE INTO chats " + "(id, name, type) "+ "VALUES " + "(?, ?, ?)"); @@ -378,4 +393,127 @@ class Database { throw new RuntimeException("Exception occured. See above."); } } + + public LinkedList getListOfChatsForExport() { + LinkedList list = new LinkedList(); + try { + ResultSet rs = stmt.executeQuery("SELECT chats.id, chats.name, COUNT(messages.id) as c "+ + "FROM chats, messages WHERE messages.from_type='chat' AND messages.dialog_id=chats.id "+ + "GROUP BY chats.id ORDER BY c DESC"); + while (rs.next()) { + list.add(new Chat(rs.getInt(1), rs.getString(2), rs.getInt(3))); + } + rs.close(); + return list; + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException("Exception above!"); + } + } + + public LinkedList getListOfDialogsForExport() { + LinkedList list = new LinkedList(); + try { + ResultSet rs = stmt.executeQuery( + "SELECT users.id, first_name, last_name, username, COUNT(messages.id) as c " + + "FROM users, messages WHERE messages.from_type='user' AND messages.dialog_id=users.id " + + "GROUP BY users.id ORDER BY c DESC"); + while (rs.next()) { + list.add(new Dialog(rs.getInt(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getInt(5))); + } + rs.close(); + return list; + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException("Exception above!"); + } + } + + public void getMessagesForExport(Dialog d, ChatMessageProcessor p) { + getMessagesForExport("user", d.id, p); + } + + public void getMessagesForExport(Chat c, ChatMessageProcessor p) { + getMessagesForExport("chat", c.id, p); + } + + private void getMessagesForExport(String type, Integer id, ChatMessageProcessor p) { + try { + ResultSet rs = stmt.executeQuery("SELECT messages.id, text, time, has_media, " + + "sticker, first_name, last_name, username FROM messages, users WHERE " + + "users.id=messages.from_id AND dialog_id=" + id + " AND from_type='" + type + "' " + + "ORDER BY messages.id"); + while (rs.next()) { + Message m = new Message( + rs.getInt(1), + rs.getString(2), + rs.getTime(3), + rs.getBoolean(4), + rs.getString(5), + rs.getString(6), + rs.getString(7), + rs.getString(8)); + p.process(m); + } + rs.close(); + } catch(Exception e) { + e.printStackTrace(); + throw new RuntimeException("Exception above!"); + } + } + + + public class Dialog { + public int id; + public String first_name; + public String last_name; + public String username; + public int count; + + public Dialog (int id, String first_name, String last_name, String username, int count) { + this.id = id; + this.first_name = first_name; + this.last_name = last_name; + this.username = username; + this.count = count; + } + } + + public class Chat { + public int id; + public String name; + public int count; + + public Chat(int id, String name, int count) { + this.id = id; + this.name = name; + this.count = count; + } + } + + public class Message { + public int id; + public String text; + public Time time; + public boolean has_media; + public String sticker; + public String first_name; + public String last_name; + public String username; + + public Message(int i, String t, Time t2, boolean m, String st, String n1, String n2, String n3) { + this.id = i; + this.text = t; + this.time = t2; + this.has_media = m; + this.sticker = st; + this.first_name = n1; + this.last_name = n2; + this.username = n3; + } + } + + public interface ChatMessageProcessor { + public void process(Message msg); + } } diff --git a/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java b/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java index b16fd09..e0e1822 100644 --- a/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java +++ b/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java @@ -20,6 +20,9 @@ import java.io.File; import java.io.FileOutputStream; import java.util.ArrayList; import java.util.LinkedList; +import java.net.URL; + +import org.apache.commons.io.FileUtils; class DownloadManager { UserManager user; @@ -84,23 +87,27 @@ class DownloadManager { System.out.println("Checking message database for completeness..."); if (db.getMessageCount() != db.getTopMessageID()) { - LinkedList ids = db.getMissingIDs(); - System.out.println("Downloading " + ids.size() + " messages that are missing in your database."); - prog.onMessageDownloadStart(ids.size()); - while (ids.size()>0) { - TLIntVector vector = new TLIntVector(); - for (int i=0; i<100; i++) { - if (ids.size()==0) break; - vector.add(ids.remove()); + if (limit != null) { + System.out.println("You are missing messages in your database. But since you're using '--limit-messages', I won't download these now."); + } else { + LinkedList ids = db.getMissingIDs(); + System.out.println("Downloading " + ids.size() + " messages that are missing in your database."); + prog.onMessageDownloadStart(ids.size()); + while (ids.size()>0) { + TLIntVector vector = new TLIntVector(); + for (int i=0; i<100; i++) { + if (ids.size()==0) break; + vector.add(ids.remove()); + } + TLAbsMessages response = client.messagesGetMessages(vector); + prog.onMessageDownloaded(response.getMessages().size()); + db.saveMessages(response.getMessages()); + db.saveChats(response.getChats()); + db.saveUsers(response.getUsers()); + try { Thread.sleep(Config.DELAY_AFTER_GET_MESSAGES); } catch (InterruptedException e) {} } - TLAbsMessages response = client.messagesGetMessages(vector); - prog.onMessageDownloaded(response.getMessages().size()); - db.saveMessages(response.getMessages()); - db.saveChats(response.getChats()); - db.saveUsers(response.getUsers()); - try { Thread.sleep(Config.DELAY_AFTER_GET_MESSAGES); } catch (InterruptedException e) {} + prog.onMessageDownloadFinished(); } - prog.onMessageDownloadFinished(); } } diff --git a/src/main/java/de/fabianonline/telegram_backup/HTMLExporter.java b/src/main/java/de/fabianonline/telegram_backup/HTMLExporter.java new file mode 100644 index 0000000..9ae689d --- /dev/null +++ b/src/main/java/de/fabianonline/telegram_backup/HTMLExporter.java @@ -0,0 +1,77 @@ +package de.fabianonline.telegram_backup; + +import de.fabianonline.telegram_backup.UserManager; +import de.fabianonline.telegram_backup.Database; + +import java.io.File; +import java.io.PrintWriter; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.util.LinkedList; + +import org.javatuples.Triplet; + +class HTMLExporter { + public void export(UserManager user) { + try { + Database db = new Database(user); + + // Create base dir + String base = user.getFileBase() + "export" + File.separatorChar + "html" + File.separatorChar; + new File(base).mkdirs(); + new File(base + "dialogs").mkdirs(); + + PrintWriter index = new PrintWriter(new BufferedWriter(new FileWriter(base + "index.html"))); + index.println(""); + index.println(""); + index.println(""); + index.println("Telegram Backup for " + user.getUserString() + ""); + index.println(""); + + index.println(""); + index.println("

Telegram Backup

"); + index.println("

" + user.getUserString() + "

"); + LinkedList dialogs = db.getListOfDialogsForExport(); + index.println("

Dialogs

"); + index.println("
    "); + for (Database.Dialog dialog : dialogs) { + index.println("
  • " + dialog.first_name + " " + (dialog.last_name!=null ? dialog.last_name : "") + " (" + dialog.count + ")
    • "); + String filename = base + "dialogs" + File.separatorChar + "user_" + dialog.id + ".html"; + final PrintWriter chatfile = new PrintWriter(new BufferedWriter(new FileWriter(filename))); + db.getMessagesForExport(dialog, new ChatLineWriter(chatfile)); + chatfile.close(); + } + LinkedList chats = db.getListOfChatsForExport(); + index.println("

    Group chats

    "); + index.println("
      "); + for (Database.Chat chat : chats) { + index.println("
    • " + chat.name + " (" + chat.count + ")
      • "); + String filename = base + "dialogs" + File.separatorChar + "chat_" + chat.id + ".html"; + final PrintWriter chatfile = new PrintWriter(new BufferedWriter(new FileWriter(filename))); + db.getMessagesForExport(chat, new ChatLineWriter(chatfile)); + chatfile.close(); + } + index.println("
    "); + index.println(""); + index.println(""); + index.close(); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("Exception above!"); + } + } + + class ChatLineWriter implements Database.ChatMessageProcessor { + PrintWriter w; + + public ChatLineWriter(PrintWriter w) { + this.w = w; + } + + public void process(Database.Message msg) { + w.println("" + msg.text + "
    "); + } + } + +}