diff --git a/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java b/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java index 591ee2c..c007f16 100644 --- a/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java +++ b/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java @@ -18,6 +18,7 @@ package de.fabianonline.telegram_backup; import de.fabianonline.telegram_backup.TelegramUpdateHandler; import de.fabianonline.telegram_backup.exporter.HTMLExporter; +import de.fabianonline.telegram_backup.models.Message; import com.github.badoualy.telegram.api.Kotlogram; import com.github.badoualy.telegram.api.TelegramApp; @@ -90,16 +91,7 @@ public class CommandLineController { } } - if (CommandLineOptions.val_test != null) { - if (CommandLineOptions.val_test == 1) { - TestFeatures.test1(); - } else if (CommandLineOptions.val_test == 2) { - TestFeatures.test2(user, client); - } else { - System.out.println("Unknown test " + CommandLineOptions.val_test); - } - System.exit(1); - } + logger.debug("CommandLineOptions.val_export: {}", CommandLineOptions.val_export); if (CommandLineOptions.val_export != null) { @@ -126,6 +118,20 @@ public class CommandLineController { logger.info("Initializing Download Manager"); DownloadManager d = new DownloadManager(user, client, new CommandLineDownloadProgress()); + + if (CommandLineOptions.val_test != null) { + if (CommandLineOptions.val_test == 1) { + TestFeatures.test1(); + } else if (CommandLineOptions.val_test == 2) { + TestFeatures.test2(user, client); + } else if (CommandLineOptions.val_test == 3) { + logger.debug(Message.get(39925).getMessage()); + } else { + System.out.println("Unknown test " + CommandLineOptions.val_test); + } + System.exit(1); + } + logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages); d.downloadMessages(CommandLineOptions.val_limit_messages); diff --git a/src/main/java/de/fabianonline/telegram_backup/Database.java b/src/main/java/de/fabianonline/telegram_backup/Database.java index 9641ffa..14f68d7 100644 --- a/src/main/java/de/fabianonline/telegram_backup/Database.java +++ b/src/main/java/de/fabianonline/telegram_backup/Database.java @@ -21,6 +21,7 @@ import com.github.badoualy.telegram.tl.core.TLVector; import com.github.badoualy.telegram.api.TelegramClient; import org.slf4j.LoggerFactory; import org.slf4j.Logger; +import com.google.gson.Gson; import java.sql.Connection; import java.sql.DriverManager; @@ -166,16 +167,16 @@ public class Database { } } - public synchronized void saveMessages(TLVector all, Integer api_layer) { + public synchronized void saveMessages(TLVector all, Integer api_layer, Gson gson) { try { //"(id, dialog_id, from_id, from_type, text, time, has_media, data, sticker, type) " + //"VALUES " + //"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); String columns = - "(id, message_type, dialog_id, chat_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer) "+ + "(id, message_type, dialog_id, chat_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer, json) "+ "VALUES " + - "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - //1 2 3 4 5 6 7 8 9 10 11 12 13 14 + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + //1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 PreparedStatement ps = conn.prepareStatement("INSERT OR REPLACE INTO messages " + columns); PreparedStatement ps_insert_or_ignore = conn.prepareStatement("INSERT OR IGNORE INTO messages " + columns); @@ -232,6 +233,7 @@ public class Database { msg.serializeBody(stream); ps.setBytes(13, stream.toByteArray()); ps.setInt(14, api_layer); + ps.setString(15, gson.toJson(msg)); ps.addBatch(); } else if (abs instanceof TLMessageService) { ps_insert_or_ignore.setInt(1, abs.getId()); @@ -248,6 +250,7 @@ public class Database { ps_insert_or_ignore.setNull(12, Types.INTEGER); ps_insert_or_ignore.setNull(13, Types.BLOB); ps_insert_or_ignore.setInt(14, api_layer); + ps_insert_or_ignore.setString(15, gson.toJson((TLMessageService)abs)); ps_insert_or_ignore.addBatch(); } else if (abs instanceof TLMessageEmpty) { ps_insert_or_ignore.setInt(1, abs.getId()); @@ -264,6 +267,7 @@ public class Database { ps_insert_or_ignore.setNull(12, Types.INTEGER); ps_insert_or_ignore.setNull(13, Types.BLOB); ps_insert_or_ignore.setInt(14, api_layer); + ps_insert_or_ignore.setString(15, gson.toJson((TLMessageEmpty)abs)); ps_insert_or_ignore.addBatch(); } else { throw new RuntimeException("Unexpected Message type: " + abs.getClass().getName()); @@ -282,18 +286,18 @@ public class Database { } } - public synchronized void saveChats(TLVector all) { + public synchronized void saveChats(TLVector all, Gson gson) { try { PreparedStatement ps_insert_or_replace = conn.prepareStatement( "INSERT OR REPLACE INTO chats " + - "(id, name, type) "+ + "(id, name, type, json) "+ "VALUES " + - "(?, ?, ?)"); + "(?, ?, ?, ?)"); PreparedStatement ps_insert_or_ignore = conn.prepareStatement( "INSERT OR IGNORE INTO chats " + - "(id, name, type) "+ + "(id, name, type, json) "+ "VALUES " + - "(?, ?, ?)"); + "(?, ?, ?, ?)"); for(TLAbsChat abs : all) { ps_insert_or_replace.setInt(1, abs.getId()); @@ -301,22 +305,27 @@ public class Database { if (abs instanceof TLChatEmpty) { ps_insert_or_ignore.setNull(2, Types.VARCHAR); ps_insert_or_ignore.setString(3, "empty_chat"); + ps_insert_or_ignore.setString(4, gson.toJson((TLChatEmpty)abs)); ps_insert_or_ignore.addBatch(); } else if (abs instanceof TLChatForbidden) { ps_insert_or_replace.setString(2, ((TLChatForbidden)abs).getTitle()); ps_insert_or_replace.setString(3, "chat"); + ps_insert_or_replace.setString(4, gson.toJson((TLChatForbidden)abs)); ps_insert_or_replace.addBatch(); } else if (abs instanceof TLChannelForbidden) { ps_insert_or_replace.setString(2, ((TLChannelForbidden)abs).getTitle()); ps_insert_or_replace.setString(3, "channel"); + ps_insert_or_replace.setString(4, gson.toJson((TLChannelForbidden)abs)); ps_insert_or_replace.addBatch(); } else if (abs instanceof TLChat) { ps_insert_or_replace.setString(2, ((TLChat) abs).getTitle()); ps_insert_or_replace.setString(3, "chat"); + ps_insert_or_replace.setString(4, gson.toJson((TLChat) abs)); ps_insert_or_replace.addBatch(); } else if (abs instanceof TLChannel) { ps_insert_or_replace.setString(2, ((TLChannel)abs).getTitle()); ps_insert_or_replace.setString(3, "channel"); + ps_insert_or_replace.setString(4, gson.toJson((TLChannel)abs)); ps_insert_or_replace.addBatch(); } else { throw new RuntimeException("Unexpected " + abs.getClass().getName()); @@ -335,18 +344,18 @@ public class Database { } } - public synchronized void saveUsers(TLVector all) { + public synchronized void saveUsers(TLVector all, Gson gson) { try { PreparedStatement ps_insert_or_replace = conn.prepareStatement( "INSERT OR REPLACE INTO users " + - "(id, first_name, last_name, username, type, phone) " + + "(id, first_name, last_name, username, type, phone, json) " + "VALUES " + - "(?, ?, ?, ?, ?, ?)"); + "(?, ?, ?, ?, ?, ?, ?)"); PreparedStatement ps_insert_or_ignore = conn.prepareStatement( "INSERT OR IGNORE INTO users " + - "(id, first_name, last_name, username, type, phone) " + + "(id, first_name, last_name, username, type, phone, json) " + "VALUES " + - "(?, ?, ?, ?, ?, ?)"); + "(?, ?, ?, ?, ?, ?, ?)"); for (TLAbsUser abs : all) { if (abs instanceof TLUser) { TLUser user = (TLUser)abs; @@ -356,6 +365,7 @@ public class Database { ps_insert_or_replace.setString(4, user.getUsername()); ps_insert_or_replace.setString(5, "user"); ps_insert_or_replace.setString(6, user.getPhone()); + ps_insert_or_replace.setString(7, gson.toJson(user)); ps_insert_or_replace.addBatch(); } else if (abs instanceof TLUserEmpty) { ps_insert_or_ignore.setInt(1, abs.getId()); @@ -364,6 +374,7 @@ public class Database { ps_insert_or_ignore.setNull(4, Types.VARCHAR); ps_insert_or_ignore.setString(5, "empty_user"); ps_insert_or_ignore.setNull(6, Types.VARCHAR); + ps_insert_or_replace.setString(7, gson.toJson((TLUserEmpty)abs)); ps_insert_or_ignore.addBatch(); } else { throw new RuntimeException("Unexpected " + abs.getClass().getName()); @@ -397,6 +408,19 @@ public class Database { } } + public String queryString(String query) { + String result = null; + try { + ResultSet rs = stmt.executeQuery(query); + rs.next(); + result = rs.getString(1); + rs.close(); + } catch (Exception e) { + logger.warn("Exception happened in queryString:", e); + } + return result; + } + public int getMessagesFromUserCount() { try { ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM messages WHERE sender_id=" + user_manager.getUser().getId()); diff --git a/src/main/java/de/fabianonline/telegram_backup/DatabaseUpdates.java b/src/main/java/de/fabianonline/telegram_backup/DatabaseUpdates.java index 8de519b..aae9d6d 100644 --- a/src/main/java/de/fabianonline/telegram_backup/DatabaseUpdates.java +++ b/src/main/java/de/fabianonline/telegram_backup/DatabaseUpdates.java @@ -32,6 +32,7 @@ public class DatabaseUpdates { register(new DB_Update_5(conn, db)); register(new DB_Update_6(conn, db)); register(new DB_Update_7(conn, db)); + register(new DB_Update_8(conn, db)); } public void doUpdates() { @@ -301,3 +302,14 @@ class DB_Update_7 extends DatabaseUpdate { stmt.executeUpdate("UPDATE messages SET api_layer=51"); } } + +class DB_Update_8 extends DatabaseUpdate { + public int getVersion() { return 8; } + public DB_Update_8(Connection conn, Database db) { super(conn, db); } + + protected void _doUpdate() throws SQLException { + stmt.executeUpdate("ALTER TABLE messages ADD COLUMN json TEXT"); + stmt.executeUpdate("ALTER TABLE chats ADD COLUMN json TEXT"); + stmt.executeUpdate("ALTER TABLE users ADD COLUMN json TEXT"); + } +} diff --git a/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java b/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java index 444eac4..88f14db 100644 --- a/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java +++ b/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java @@ -212,14 +212,11 @@ public class DownloadManager { //ObjectMapper om = new ObjectMapper(); //String json = om.writerWithDefaultPrettyPrinter().writeValueAsString(response.getMessages().get(1)); Gson gson = Utils.getGson(); - String json = gson.toJson(response.getMessages()); - System.out.println(json); - //System.exit(1); prog.onMessageDownloaded(response.getMessages().size()); - //db.saveMessages(response.getMessages(), Kotlogram.API_LAYER); - //db.saveChats(response.getChats()); - //db.saveUsers(response.getUsers()); + db.saveMessages(response.getMessages(), Kotlogram.API_LAYER, gson); + db.saveChats(response.getChats(), gson); + db.saveUsers(response.getUsers(), gson); logger.trace("Sleeping"); try { TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_MESSAGES); diff --git a/src/main/java/de/fabianonline/telegram_backup/TelegramUpdateHandler.java b/src/main/java/de/fabianonline/telegram_backup/TelegramUpdateHandler.java index 501a10e..f74cfa0 100644 --- a/src/main/java/de/fabianonline/telegram_backup/TelegramUpdateHandler.java +++ b/src/main/java/de/fabianonline/telegram_backup/TelegramUpdateHandler.java @@ -22,7 +22,10 @@ import com.github.badoualy.telegram.api.Kotlogram; import com.github.badoualy.telegram.tl.api.*; import com.github.badoualy.telegram.tl.core.TLVector; +import com.google.gson.Gson; + import de.fabianonline.telegram_backup.Database; +import de.fabianonline.telegram_backup.Utils; import de.fabianonline.telegram_backup.UserManager; import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager; import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory; @@ -31,6 +34,7 @@ class TelegramUpdateHandler implements UpdateCallback { private UserManager user = null; private Database db = null; public boolean debug = false; + private Gson gson = Utils.getGson(); public void setUser(UserManager user, TelegramClient client) { this.user = user; this.db = new Database(user, client, false);} @@ -41,8 +45,8 @@ class TelegramUpdateHandler implements UpdateCallback { processUpdate(update, c); if (debug) System.out.println(" " + update.getClass().getName()); } - db.saveUsers(u.getUsers()); - db.saveChats(u.getChats()); + db.saveUsers(u.getUsers(), gson); + db.saveChats(u.getChats(), gson); } public void onUpdatesCombined(TelegramClient c, TLUpdatesCombined u) { @@ -51,8 +55,8 @@ class TelegramUpdateHandler implements UpdateCallback { for(TLAbsUpdate update : u.getUpdates()) { processUpdate(update, c); } - db.saveUsers(u.getUsers()); - db.saveChats(u.getChats()); + db.saveUsers(u.getUsers(), gson); + db.saveChats(u.getChats(), gson); } public void onUpdateShort(TelegramClient c, TLUpdateShort u) { @@ -86,7 +90,7 @@ class TelegramUpdateHandler implements UpdateCallback { null); TLVector vector = new TLVector(TLAbsMessage.class); vector.add(msg); - db.saveMessages(vector, Kotlogram.API_LAYER); + db.saveMessages(vector, Kotlogram.API_LAYER, gson); System.out.print('.'); } @@ -122,7 +126,7 @@ class TelegramUpdateHandler implements UpdateCallback { null); TLVector vector = new TLVector(TLAbsMessage.class); vector.add(msg); - db.saveMessages(vector, Kotlogram.API_LAYER); + db.saveMessages(vector, Kotlogram.API_LAYER, gson); System.out.print('.'); } @@ -134,7 +138,7 @@ class TelegramUpdateHandler implements UpdateCallback { TLAbsMessage abs_msg = ((TLUpdateNewMessage)update).getMessage(); TLVector vector = new TLVector(TLAbsMessage.class); vector.add(abs_msg); - db.saveMessages(vector, Kotlogram.API_LAYER); + db.saveMessages(vector, Kotlogram.API_LAYER, gson); System.out.print('.'); if (abs_msg instanceof TLMessage) { AbstractMediaFileManager fm = FileManagerFactory.getFileManager((TLMessage)abs_msg, user, client); diff --git a/src/main/java/de/fabianonline/telegram_backup/Utils.java b/src/main/java/de/fabianonline/telegram_backup/Utils.java index 67d621e..bdcafa6 100644 --- a/src/main/java/de/fabianonline/telegram_backup/Utils.java +++ b/src/main/java/de/fabianonline/telegram_backup/Utils.java @@ -180,7 +180,6 @@ public class Utils { public static Gson getGson() { return new GsonBuilder() .registerTypeAdapter(TLBytes.class, new TLBytesSerializer()) - .setPrettyPrinting() .create(); } } diff --git a/src/main/java/de/fabianonline/telegram_backup/models/Message.java b/src/main/java/de/fabianonline/telegram_backup/models/Message.java new file mode 100644 index 0000000..ecab8b3 --- /dev/null +++ b/src/main/java/de/fabianonline/telegram_backup/models/Message.java @@ -0,0 +1,23 @@ +package de.fabianonline.telegram_backup.models; + +import de.fabianonline.telegram_backup.Database; +import com.google.gson.JsonParser; +import com.google.gson.JsonObject; + +public class Message { + protected static String tableName = "messages"; + private JsonObject json; + + public Message(String json) { + this.json = new JsonParser().parse(json).getAsJsonObject(); + } + + public static Message get(int id) { + String json = Database.getInstance().queryString("SELECT json FROM " + tableName + " WHERE id=" + id); + return new Message(json); + } + + public String getMessage() { + return json.getAsJsonPrimitive("message").getAsString(); + } +} diff --git a/src/main/java/de/fabianonline/telegram_backup/models/Message_c09be45f.java b/src/main/java/de/fabianonline/telegram_backup/models/Message_c09be45f.java deleted file mode 100644 index 7bfc333..0000000 --- a/src/main/java/de/fabianonline/telegram_backup/models/Message_c09be45f.java +++ /dev/null @@ -1,15 +0,0 @@ -public class Message_c09be45f { - int flags; - boolean out; - boolean mentioned; - boolean mediaUnread; - boolean silent; - boolean post; - int fromId; - Peer toId; - String _constructor; - int date; - String message; - int id; - JsonElement media; -}