From 07ba594a152c53273c32136ca675329743685a6f Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Mon, 4 Jul 2016 13:33:17 +0200 Subject: [PATCH] Added automatic checks for missing messages in the database complete with download of these mesages. --- .../CommandLineDownloadProgress.java | 2 +- .../telegram_backup/Database.java | 37 ++++++++++ .../telegram_backup/DownloadManager.java | 73 ++++++++++++------- 3 files changed, 86 insertions(+), 26 deletions(-) diff --git a/src/main/java/de/fabianonline/telegram_backup/CommandLineDownloadProgress.java b/src/main/java/de/fabianonline/telegram_backup/CommandLineDownloadProgress.java index 4be0d49..6cb0a0f 100644 --- a/src/main/java/de/fabianonline/telegram_backup/CommandLineDownloadProgress.java +++ b/src/main/java/de/fabianonline/telegram_backup/CommandLineDownloadProgress.java @@ -31,7 +31,7 @@ class CommandLineDownloadProgress implements DownloadProgressInterface { public void onMediaDownloadedEmpty(boolean n) { show(true, 'e'); } public void onMediaDownloadFinished() { showNewLine(); System.out.println("Done."); } - private void show(boolean n, char letter) { System.out.print(n ? letter : '.'); i++; if (i % 50 == 0) showNewLine();} + private void show(boolean n, char letter) { System.out.print(n ? letter : '.'); i++; if (i % 100 == 0) showNewLine();} private void showNewLine() { System.out.println(" - " + i + "/" + mediaCount); } } diff --git a/src/main/java/de/fabianonline/telegram_backup/Database.java b/src/main/java/de/fabianonline/telegram_backup/Database.java index bd2fc2e..2300e49 100644 --- a/src/main/java/de/fabianonline/telegram_backup/Database.java +++ b/src/main/java/de/fabianonline/telegram_backup/Database.java @@ -14,6 +14,7 @@ import java.io.File; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.util.LinkedList; +import java.sql.Array; import de.fabianonline.telegram_backup.UserManager; import de.fabianonline.telegram_backup.StickerConverter; @@ -115,6 +116,42 @@ class Database { } } + public int getMessageCount() { + try { + ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM messages"); + rs.next(); + return rs.getInt(1); + } catch (SQLException e) { + throw new RuntimeException("Could not get count of messages."); + } + } + + public LinkedList getMissingIDs() { + try { + LinkedList missing = new LinkedList(); + int max = getTopMessageID(); + ResultSet rs = stmt.executeQuery("SELECT id FROM messages ORDER BY id"); + rs.next(); + int id=rs.getInt(1); + for (int i=1; i<=max; i++) { + if (i==id) { + rs.next(); + if (rs.isClosed()) { + id = Integer.MAX_VALUE; + } else { + id=rs.getInt(1); + } + } else if (i all) { try { PreparedStatement ps = conn.prepareStatement( diff --git a/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java b/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java index 3012de3..b16fd09 100644 --- a/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java +++ b/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java @@ -50,35 +50,58 @@ class DownloadManager { System.out.println("New top message id 'in database' is " + max_database_id); } - int start_id = max_database_id + 1; - int current_start_id = start_id; - int end_id = max_message_id; - if (start_id > end_id) { + if (max_database_id == max_message_id) { System.out.println("No new messages to download."); - return; - } - - prog.onMessageDownloadStart(end_id - current_start_id + 1); - - while (current_start_id <= end_id) { - int my_end_id = Math.min(current_start_id+99, end_id); - ArrayList a = makeIdList(current_start_id, my_end_id); - TLIntVector ids = new TLIntVector(); - ids.addAll(a); - my_end_id = ids.get(ids.size()-1); - current_start_id = my_end_id + 1; + } else if (max_database_id > max_message_id) { + throw new RuntimeException("max_database_id is bigger then max_message_id. This shouldn't be aple to happen. Ever."); + } else { + int start_id = max_database_id + 1; + int current_start_id = start_id; + int end_id = max_message_id; - TLAbsMessages response = client.messagesGetMessages(ids); - prog.onMessageDownloaded(response.getMessages().size()); - db.saveMessages(response.getMessages()); - db.saveChats(response.getChats()); - db.saveUsers(response.getUsers()); - try { - Thread.sleep(Config.DELAY_AFTER_GET_MESSAGES); - } catch (InterruptedException e) {} + prog.onMessageDownloadStart(end_id - current_start_id + 1); + + while (current_start_id <= end_id) { + int my_end_id = Math.min(current_start_id+99, end_id); + ArrayList a = makeIdList(current_start_id, my_end_id); + TLIntVector ids = new TLIntVector(); + ids.addAll(a); + my_end_id = ids.get(ids.size()-1); + current_start_id = my_end_id + 1; + + TLAbsMessages response = client.messagesGetMessages(ids); + prog.onMessageDownloaded(response.getMessages().size()); + db.saveMessages(response.getMessages()); + db.saveChats(response.getChats()); + db.saveUsers(response.getUsers()); + try { + Thread.sleep(Config.DELAY_AFTER_GET_MESSAGES); + } catch (InterruptedException e) {} + } + + prog.onMessageDownloadFinished(); } - prog.onMessageDownloadFinished(); + System.out.println("Checking message database for completeness..."); + if (db.getMessageCount() != db.getTopMessageID()) { + LinkedList ids = db.getMissingIDs(); + System.out.println("Downloading " + ids.size() + " messages that are missing in your database."); + prog.onMessageDownloadStart(ids.size()); + while (ids.size()>0) { + TLIntVector vector = new TLIntVector(); + for (int i=0; i<100; i++) { + if (ids.size()==0) break; + vector.add(ids.remove()); + } + TLAbsMessages response = client.messagesGetMessages(vector); + prog.onMessageDownloaded(response.getMessages().size()); + db.saveMessages(response.getMessages()); + db.saveChats(response.getChats()); + db.saveUsers(response.getUsers()); + try { Thread.sleep(Config.DELAY_AFTER_GET_MESSAGES); } catch (InterruptedException e) {} + } + prog.onMessageDownloadFinished(); + } } public void downloadMedia() throws RpcErrorException, IOException {