New entries are saved with their JSON equivalent.

src/main/java/de/fabianonline/telegram_backup/CommandLineController.java

@@ -18,6 +18,7 @@ package de.fabianonline.telegram_backup;
 
 import de.fabianonline.telegram_backup.TelegramUpdateHandler;
 import de.fabianonline.telegram_backup.exporter.HTMLExporter;
+import de.fabianonline.telegram_backup.models.Message;
 
 import com.github.badoualy.telegram.api.Kotlogram;
 import com.github.badoualy.telegram.api.TelegramApp;
@@ -90,16 +91,7 @@ public class CommandLineController {
 				}
 			}
 			
-			if (CommandLineOptions.val_test != null) {
+			
-				if (CommandLineOptions.val_test == 1) {
-					TestFeatures.test1();
-				} else if (CommandLineOptions.val_test == 2) {
-					TestFeatures.test2(user, client);
-				} else {
-					System.out.println("Unknown test " + CommandLineOptions.val_test);
-				}
-				System.exit(1);
-			}
 			
 			logger.debug("CommandLineOptions.val_export: {}", CommandLineOptions.val_export);
 			if (CommandLineOptions.val_export != null) {
@@ -126,6 +118,20 @@ public class CommandLineController {
 			
 			logger.info("Initializing Download Manager");
 			DownloadManager d = new DownloadManager(user, client, new CommandLineDownloadProgress());
+			
+			if (CommandLineOptions.val_test != null) {
+				if (CommandLineOptions.val_test == 1) {
+					TestFeatures.test1();
+				} else if (CommandLineOptions.val_test == 2) {
+					TestFeatures.test2(user, client);
+				} else if (CommandLineOptions.val_test == 3) {
+					logger.debug(Message.get(39925).getMessage());
+				} else {
+					System.out.println("Unknown test " + CommandLineOptions.val_test);
+				}
+				System.exit(1);
+			}
+			
 			logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages);
 			d.downloadMessages(CommandLineOptions.val_limit_messages);
 			

src/main/java/de/fabianonline/telegram_backup/Database.java

@@ -21,6 +21,7 @@ import com.github.badoualy.telegram.tl.core.TLVector;
 import com.github.badoualy.telegram.api.TelegramClient;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Logger;
+import com.google.gson.Gson;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -166,16 +167,16 @@ public class Database {
 		}
 	}
 	
-	public synchronized void saveMessages(TLVector<TLAbsMessage> all, Integer api_layer) {
+	public synchronized void saveMessages(TLVector<TLAbsMessage> all, Integer api_layer, Gson gson) {
 		try {
 				//"(id, dialog_id, from_id, from_type, text, time, has_media, data, sticker, type) " +
 				//"VALUES " +
 				//"(?,  ?,         ?,       ?,         ?,    ?,    ?,         ?,    ?,       ?)");
 			String columns =
-				"(id, message_type, dialog_id, chat_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer) "+
+				"(id, message_type, dialog_id, chat_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer, json) "+
 				"VALUES " +
-				"(?,  ?,            ?,         ?,       ?,         ?,           ?,    ?,    ?,         ?,          ?,          ?,          ?,    ?)";
+				"(?,  ?,            ?,         ?,       ?,         ?,           ?,    ?,    ?,         ?,          ?,          ?,          ?,    ?,         ?)";
-				//1   2             3          4        5          6            7     8     9          10          11          12          13    14
+				//1   2             3          4        5          6            7     8     9          10          11          12          13    14         15
 			PreparedStatement ps = conn.prepareStatement("INSERT OR REPLACE INTO messages " + columns);
 			PreparedStatement ps_insert_or_ignore = conn.prepareStatement("INSERT OR IGNORE INTO messages " + columns);
 
@@ -232,6 +233,7 @@ public class Database {
 					msg.serializeBody(stream);
 					ps.setBytes(13, stream.toByteArray());
 					ps.setInt(14, api_layer);
+					ps.setString(15, gson.toJson(msg));
 					ps.addBatch();
 				} else if (abs instanceof TLMessageService) {
 					ps_insert_or_ignore.setInt(1, abs.getId());
@@ -248,6 +250,7 @@ public class Database {
 					ps_insert_or_ignore.setNull(12, Types.INTEGER);
 					ps_insert_or_ignore.setNull(13, Types.BLOB);
 					ps_insert_or_ignore.setInt(14, api_layer);
+					ps_insert_or_ignore.setString(15, gson.toJson((TLMessageService)abs));
 					ps_insert_or_ignore.addBatch();
 				} else if (abs instanceof TLMessageEmpty) {
 					ps_insert_or_ignore.setInt(1, abs.getId());
@@ -264,6 +267,7 @@ public class Database {
 					ps_insert_or_ignore.setNull(12, Types.INTEGER);
 					ps_insert_or_ignore.setNull(13, Types.BLOB);
 					ps_insert_or_ignore.setInt(14, api_layer);
+					ps_insert_or_ignore.setString(15, gson.toJson((TLMessageEmpty)abs));
 					ps_insert_or_ignore.addBatch();
 				} else {
 					throw new RuntimeException("Unexpected Message type: " + abs.getClass().getName());
@@ -282,18 +286,18 @@ public class Database {
 		}
 	}
 
-	public synchronized void saveChats(TLVector<TLAbsChat> all) {
+	public synchronized void saveChats(TLVector<TLAbsChat> all, Gson gson) {
 		try {
 			PreparedStatement ps_insert_or_replace = conn.prepareStatement(
 				"INSERT OR REPLACE INTO chats " +
-					"(id, name, type) "+
+					"(id, name, type, json) "+
 					"VALUES " +
-					"(?,  ?,    ?)");
+					"(?,  ?,    ?,    ?)");
 			PreparedStatement ps_insert_or_ignore = conn.prepareStatement(
 				"INSERT OR IGNORE INTO chats " +
-					"(id, name, type) "+
+					"(id, name, type, json) "+
 					"VALUES " +
-					"(?,  ?,    ?)");
+					"(?,  ?,    ?,    ?)");
 
 			for(TLAbsChat abs : all) {
 				ps_insert_or_replace.setInt(1, abs.getId());
@@ -301,22 +305,27 @@ public class Database {
 				if (abs instanceof TLChatEmpty) {
 					ps_insert_or_ignore.setNull(2, Types.VARCHAR);
 					ps_insert_or_ignore.setString(3, "empty_chat");
+					ps_insert_or_ignore.setString(4, gson.toJson((TLChatEmpty)abs));
 					ps_insert_or_ignore.addBatch();
 				} else if (abs instanceof TLChatForbidden) {
 					ps_insert_or_replace.setString(2, ((TLChatForbidden)abs).getTitle());
 					ps_insert_or_replace.setString(3, "chat");
+					ps_insert_or_replace.setString(4, gson.toJson((TLChatForbidden)abs));
 					ps_insert_or_replace.addBatch();
 				} else if (abs instanceof TLChannelForbidden) {
 					ps_insert_or_replace.setString(2, ((TLChannelForbidden)abs).getTitle());
 					ps_insert_or_replace.setString(3, "channel");
+					ps_insert_or_replace.setString(4, gson.toJson((TLChannelForbidden)abs));
 					ps_insert_or_replace.addBatch();
 				} else if (abs instanceof TLChat) {
 					ps_insert_or_replace.setString(2, ((TLChat) abs).getTitle());
 					ps_insert_or_replace.setString(3, "chat");
+					ps_insert_or_replace.setString(4, gson.toJson((TLChat) abs));
 					ps_insert_or_replace.addBatch();
 				} else if (abs instanceof TLChannel) {
 					ps_insert_or_replace.setString(2, ((TLChannel)abs).getTitle());
 					ps_insert_or_replace.setString(3, "channel");
+					ps_insert_or_replace.setString(4, gson.toJson((TLChannel)abs));
 					ps_insert_or_replace.addBatch();
 				} else {
 					throw new RuntimeException("Unexpected " + abs.getClass().getName());
@@ -335,18 +344,18 @@ public class Database {
 		}
 	}
 
-	public synchronized void saveUsers(TLVector<TLAbsUser> all) {
+	public synchronized void saveUsers(TLVector<TLAbsUser> all, Gson gson) {
 		try {
 			PreparedStatement ps_insert_or_replace = conn.prepareStatement(
 				"INSERT OR REPLACE INTO users " +
-					"(id, first_name, last_name, username, type, phone) " +
+					"(id, first_name, last_name, username, type, phone, json) " +
 					"VALUES " +
-					"(?,  ?,          ?,         ?,        ?,    ?)");
+					"(?,  ?,          ?,         ?,        ?,    ?,     ?)");
 			PreparedStatement ps_insert_or_ignore = conn.prepareStatement(
 				"INSERT OR IGNORE INTO users " +
-					"(id, first_name, last_name, username, type, phone) " +
+					"(id, first_name, last_name, username, type, phone, json) " +
 					"VALUES " +
-					"(?,  ?,          ?,         ?,        ?,    ?)");
+					"(?,  ?,          ?,         ?,        ?,    ?,     ?)");
 			for (TLAbsUser abs : all) {
 				if (abs instanceof TLUser) {
 					TLUser user = (TLUser)abs;
@@ -356,6 +365,7 @@ public class Database {
 					ps_insert_or_replace.setString(4, user.getUsername());
 					ps_insert_or_replace.setString(5, "user");
 					ps_insert_or_replace.setString(6, user.getPhone());
+					ps_insert_or_replace.setString(7, gson.toJson(user));
 					ps_insert_or_replace.addBatch();
 				} else if (abs instanceof TLUserEmpty) {
 					ps_insert_or_ignore.setInt(1, abs.getId());
@@ -364,6 +374,7 @@ public class Database {
 					ps_insert_or_ignore.setNull(4, Types.VARCHAR);
 					ps_insert_or_ignore.setString(5, "empty_user");
 					ps_insert_or_ignore.setNull(6, Types.VARCHAR);
+					ps_insert_or_replace.setString(7, gson.toJson((TLUserEmpty)abs));
 					ps_insert_or_ignore.addBatch();
 				} else {
 					throw new RuntimeException("Unexpected " + abs.getClass().getName());
@@ -397,6 +408,19 @@ public class Database {
 		}
 	}
 	
+	public String queryString(String query) {
+		String result = null;
+		try {
+			ResultSet rs = stmt.executeQuery(query);
+			rs.next();
+			result = rs.getString(1);
+			rs.close();
+		} catch (Exception e) {
+			logger.warn("Exception happened in queryString:", e);
+		}
+		return result;
+	}
+	
 	public int getMessagesFromUserCount() {
 		try {
 			ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM messages WHERE sender_id=" + user_manager.getUser().getId());

src/main/java/de/fabianonline/telegram_backup/DatabaseUpdates.java

@@ -32,6 +32,7 @@ public class DatabaseUpdates {
 		register(new DB_Update_5(conn, db));
 		register(new DB_Update_6(conn, db));
 		register(new DB_Update_7(conn, db));
+		register(new DB_Update_8(conn, db));
 	}
 	
 	public void doUpdates() {
@@ -301,3 +302,14 @@ class DB_Update_7 extends DatabaseUpdate {
 		stmt.executeUpdate("UPDATE messages SET api_layer=51");
 	}
 }
+
+class DB_Update_8 extends DatabaseUpdate {
+	public int getVersion() { return 8; }
+	public DB_Update_8(Connection conn, Database db) { super(conn, db); }
+	
+	protected void _doUpdate() throws SQLException {
+		stmt.executeUpdate("ALTER TABLE messages ADD COLUMN json TEXT");
+		stmt.executeUpdate("ALTER TABLE chats ADD COLUMN json TEXT");
+		stmt.executeUpdate("ALTER TABLE users ADD COLUMN json TEXT");
+	}
+}

src/main/java/de/fabianonline/telegram_backup/DownloadManager.java

@@ -212,14 +212,11 @@ public class DownloadManager {
 			//ObjectMapper om = new ObjectMapper();
 			//String json = om.writerWithDefaultPrettyPrinter().writeValueAsString(response.getMessages().get(1));
 			Gson gson = Utils.getGson();
-			String json = gson.toJson(response.getMessages());
-			System.out.println(json);
-			//System.exit(1);
 			
 			prog.onMessageDownloaded(response.getMessages().size());
-			//db.saveMessages(response.getMessages(), Kotlogram.API_LAYER);
+			db.saveMessages(response.getMessages(), Kotlogram.API_LAYER, gson);
-			//db.saveChats(response.getChats());
+			db.saveChats(response.getChats(), gson);
-			//db.saveUsers(response.getUsers());
+			db.saveUsers(response.getUsers(), gson);
 			logger.trace("Sleeping");
 			try {
 				TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_MESSAGES);

src/main/java/de/fabianonline/telegram_backup/TelegramUpdateHandler.java

@@ -22,7 +22,10 @@ import com.github.badoualy.telegram.api.Kotlogram;
 import com.github.badoualy.telegram.tl.api.*;
 import com.github.badoualy.telegram.tl.core.TLVector;
 
+import com.google.gson.Gson;
+
 import de.fabianonline.telegram_backup.Database;
+import de.fabianonline.telegram_backup.Utils;
 import de.fabianonline.telegram_backup.UserManager;
 import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager;
 import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory;
@@ -31,6 +34,7 @@ class TelegramUpdateHandler implements UpdateCallback {
 	private UserManager user = null;
 	private Database db = null;
 	public boolean debug = false;
+	private Gson gson = Utils.getGson();
 	
 	public void setUser(UserManager user, TelegramClient client) { this.user = user; this.db = new Database(user, client, false);}
 	
@@ -41,8 +45,8 @@ class TelegramUpdateHandler implements UpdateCallback {
 			processUpdate(update, c);
 			if (debug) System.out.println("  " + update.getClass().getName());
 		}
-		db.saveUsers(u.getUsers());
+		db.saveUsers(u.getUsers(), gson);
-		db.saveChats(u.getChats());
+		db.saveChats(u.getChats(), gson);
 	}
 	
 	public void onUpdatesCombined(TelegramClient c, TLUpdatesCombined u) {
@@ -51,8 +55,8 @@ class TelegramUpdateHandler implements UpdateCallback {
 		for(TLAbsUpdate update : u.getUpdates()) {
 			processUpdate(update, c);
 		}
-		db.saveUsers(u.getUsers());
+		db.saveUsers(u.getUsers(), gson);
-		db.saveChats(u.getChats());
+		db.saveChats(u.getChats(), gson);
 	}
 		
 	public void onUpdateShort(TelegramClient c, TLUpdateShort u) {
@@ -86,7 +90,7 @@ class TelegramUpdateHandler implements UpdateCallback {
 			null);
 		TLVector<TLAbsMessage> vector = new TLVector<TLAbsMessage>(TLAbsMessage.class);
 		vector.add(msg);
-		db.saveMessages(vector, Kotlogram.API_LAYER);
+		db.saveMessages(vector, Kotlogram.API_LAYER, gson);
 		System.out.print('.');
 	}
 	
@@ -122,7 +126,7 @@ class TelegramUpdateHandler implements UpdateCallback {
 			null);
 		TLVector<TLAbsMessage> vector = new TLVector<TLAbsMessage>(TLAbsMessage.class);
 		vector.add(msg);
-		db.saveMessages(vector, Kotlogram.API_LAYER);
+		db.saveMessages(vector, Kotlogram.API_LAYER, gson);
 		System.out.print('.');
 	}
 	
@@ -134,7 +138,7 @@ class TelegramUpdateHandler implements UpdateCallback {
 			TLAbsMessage abs_msg = ((TLUpdateNewMessage)update).getMessage();
 			TLVector<TLAbsMessage> vector = new TLVector<TLAbsMessage>(TLAbsMessage.class);
 			vector.add(abs_msg);
-			db.saveMessages(vector, Kotlogram.API_LAYER);
+			db.saveMessages(vector, Kotlogram.API_LAYER, gson);
 			System.out.print('.');
 			if (abs_msg instanceof TLMessage) {
 				AbstractMediaFileManager fm = FileManagerFactory.getFileManager((TLMessage)abs_msg, user, client);

src/main/java/de/fabianonline/telegram_backup/Utils.java

@@ -180,7 +180,6 @@ public class Utils {
 	public static Gson getGson() {
 		return new GsonBuilder()
 			.registerTypeAdapter(TLBytes.class, new TLBytesSerializer())
-			.setPrettyPrinting()
 			.create();
 	}
 }

src/main/java/de/fabianonline/telegram_backup/models/Message.java

@@ -0,0 +1,23 @@
+package de.fabianonline.telegram_backup.models;
+
+import de.fabianonline.telegram_backup.Database;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonObject;
+
+public class Message {
+	protected static String tableName = "messages";
+	private JsonObject json;
+	
+	public Message(String json) {
+		this.json = new JsonParser().parse(json).getAsJsonObject();
+	}
+	
+	public static Message get(int id) {
+		String json = Database.getInstance().queryString("SELECT json FROM " + tableName + " WHERE id=" + id);
+		return new Message(json);
+	}
+	
+	public String getMessage() {
+		return json.getAsJsonPrimitive("message").getAsString();
+	}
+}

src/main/java/de/fabianonline/telegram_backup/models/Message_c09be45f.java

@@ -1,15 +0,0 @@
-public class Message_c09be45f {
-	int flags;
-	boolean out;
-	boolean mentioned;
-	boolean mediaUnread;
-	boolean silent;
-	boolean post;
-	int fromId;
-	Peer toId;
-	String _constructor;
-	int date;
-	String message;
-	int id;
-	JsonElement media;
-}