Download-stuff. Files.

src/main/java/de/fabianonline/telegram_backup/CommandLineController.java

@@ -64,10 +64,11 @@ public class CommandLineController {
 				System.out.println("Next time, please run this tool with '--account " + user.getUser().getPhone() + " to use this account.");
 			}
 			
-			System.out.println("You are now signed in as " + user.getUserString());
+			System.out.println("You are logged in as " + user.getUserString());
 			
-			DownloadManager d = new DownloadManager(user);
+			DownloadManager d = new DownloadManager(user, client);
 			d.downloadMessages();
+			d.downloadMedia();
 		} catch (RpcErrorException e) {
 			e.printStackTrace();
 		} catch (IOException e) {

src/main/java/de/fabianonline/telegram_backup/Config.java

@@ -14,5 +14,8 @@ class Config {
     public static final String FILE_NAME_AUTH_KEY = "auth.dat";
     public static final String FILE_NAME_DC = "dc.dat";
     public static final String FILE_NAME_DB = "database.sqlite";
+    public static final String FILE_FILES_BASE = "files";
+    
+    public static final int FILE_DOWNLOAD_BLOCK_SIZE = 1024*1024;
 }
 

src/main/java/de/fabianonline/telegram_backup/Database.java

@@ -1,13 +1,28 @@
 package de.fabianonline.telegram_backup;
 
 import com.github.badoualy.telegram.tl.api.TLMessage;
+import com.github.badoualy.telegram.tl.api.TLMessageEmpty;
+import com.github.badoualy.telegram.tl.api.TLMessageService;
+import com.github.badoualy.telegram.tl.core.TLVector;
+import com.github.badoualy.telegram.tl.api.TLAbsMessage;
+import com.github.badoualy.telegram.tl.api.TLMessageMediaEmpty;
+import com.github.badoualy.telegram.tl.api.TLAbsPeer;
+import com.github.badoualy.telegram.tl.api.TLPeerUser;
+import com.github.badoualy.telegram.tl.api.TLPeerChat;
+import com.github.badoualy.telegram.tl.api.TLPeerChannel;
+import com.github.badoualy.telegram.tl.api.TLApiContext;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.Statement;
 import java.sql.SQLException;
 import java.sql.ResultSet;
+import java.sql.PreparedStatement;
+import java.sql.Types;
 import java.io.File;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.util.LinkedList;
 
 import de.fabianonline.telegram_backup.UserManager;
 
@@ -15,8 +30,10 @@ import de.fabianonline.telegram_backup.UserManager;
 class Database {
 	private Connection conn;
 	private Statement stmt;
+	private UserManager user_manager;
 	
-	public Database(UserManager user) {
+	public Database(UserManager user_manager) {
+		this.user_manager = user_manager;
 		try {
 			Class.forName("org.sqlite.JDBC");
 		} catch(ClassNotFoundException e) {
@@ -26,7 +43,7 @@ class Database {
 		String path = "jdbc:sqlite:" +
 			Config.FILE_BASE +
 			File.separatorChar +
-			user.getUser().getPhone() +
+			user_manager.getUser().getPhone() +
 			File.separatorChar +
 			Config.FILE_NAME_DB;
 		
@@ -87,11 +104,103 @@ class Database {
 		}
 	}
 	
-	public void save(TLMessage msg) {
+	public int getTopMessageID() {
-		/*BufferArrayOutputStream stream = new BufferArrayOutputStream();
+		try {
+			ResultSet rs = stmt.executeQuery("SELECT MAX(id) FROM messages");
+			rs.next();
+			return rs.getInt(1);
+		} catch (SQLException e) {
+			return 0;
+		}
+	}
+	
+	public void save(TLVector<TLAbsMessage> all) {
+		try {
+			PreparedStatement ps = conn.prepareStatement(
+				"INSERT INTO messages " +
+				"(id, dialog_id, from_id, type, text, time, has_media, data) " + 
+				"VALUES " +
+				"(?,  ?,         ?,       ?,    ?,    ?,    ?,         ?)");
+			for (TLAbsMessage abs : all) {
+				if (abs instanceof TLMessage) {
+					TLMessage msg = (TLMessage) abs;
+					ps.setInt(1, msg.getId());
+					TLAbsPeer peer = msg.getToId();
+					if (peer instanceof TLPeerChat) {
+						ps.setInt(2, ((TLPeerChat)peer).getChatId());
+						ps.setString(4, "chat");
+					} else if (peer instanceof TLPeerChannel) {
+						ps.setInt(2, ((TLPeerChannel)peer).getChannelId());
+						ps.setString(4, "channel");
+					} else if (peer instanceof TLPeerUser) {
+						int id = ((TLPeerUser)peer).getUserId();
+						if (id==this.user_manager.getUser().getId()) {
+							id = msg.getFromId();
+						}
+						ps.setInt(2, id);
+						ps.setString(4, "user");
+					} else {
+						throw new RuntimeException("Unexpected Peer type: " + peer.getClass().getName());
+					}
+					ps.setInt(3, msg.getFromId());
+					String text = msg.getMessage();
+					if ((text==null || text.equals("")) && msg.getMedia()!=null) {
+						if (msg.getMedia() instanceof TLMessageMediaDocument) {
+							text = ((TLMessageMediaDocument)msg.getMedia()).getCaption();
+						} else if (msg.getMedia() instanceof TLMessageMediaPhoto) {
+							text = ((TLMessageMediaPhoto)msg.getMedia()).getCaption();
+						}
+					}
+					ps.setString(5, text);
+					ps.setString(6, ""+msg.getDate());
+					ps.setBoolean(7, msg.getMedia() != null);
+					ByteArrayOutputStream stream = new ByteArrayOutputStream();
 					msg.serializeBody(stream);
-		bytes[] = stream.toByteArray();
+					ps.setBytes(8, stream.toByteArray());
-		PreparedStatement ps = conn.prepareStatement(query);
+					ps.addBatch();
-		ps.setBytes(x, bytes[]);*/
+				} else if (abs instanceof TLMessageService) {
+					// Ignore service messages.
+				} else if (abs instanceof TLMessageEmpty) {
+					TLMessageEmpty msg = (TLMessageEmpty) abs;
+					ps.setInt(1, msg.getId());
+					ps.setNull(2, Types.INTEGER);
+					ps.setNull(3, Types.INTEGER);
+					ps.setNull(4, Types.VARCHAR);
+					ps.setNull(5, Types.VARCHAR);
+					ps.setNull(6, Types.INTEGER);
+					ps.setNull(7, Types.BOOLEAN);
+					ps.setNull(8, Types.BLOB);
+					ps.addBatch();
+				} else {
+					throw new RuntimeException("Unexpected Message type: " + abs.getClass().getName());
+				}
+			}
+			conn.setAutoCommit(false);
+			ps.executeBatch();
+			ps.clearBatch();
+			conn.commit();
+			conn.setAutoCommit(true);
+		} catch (Exception e) {
+			e.printStackTrace();
+			throw new RuntimeException("Exception shown above happened.");
+		}
+	}
+	
+	public LinkedList<TLMessage> getMessagesWithMedia() {
+		try {
+			LinkedList<TLMessage> list = new LinkedList<TLMessage>();
+			ResultSet rs = stmt.executeQuery("SELECT data FROM messages WHERE has_media=1");
+			while (rs.next()) {
+				ByteArrayInputStream stream = new ByteArrayInputStream(rs.getBytes(1));
+				TLMessage msg = new TLMessage();
+				msg.deserializeBody(stream, TLApiContext.getInstance());
+				list.add(msg);
+			}
+			rs.close();
+			return list;
+		} catch (Exception e) {
+			e.printStackTrace();
+			throw new RuntimeException("Exception occured. See above.");
+		}
 	}
 }

src/main/java/de/fabianonline/telegram_backup/DownloadManager.java

@@ -3,16 +3,140 @@ package de.fabianonline.telegram_backup;
 import de.fabianonline.telegram_backup.UserManager;
 import de.fabianonline.telegram_backup.Database;
 
+import com.github.badoualy.telegram.api.TelegramClient;
+import com.github.badoualy.telegram.tl.core.TLIntVector;
+import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages;
+import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs;
+import com.github.badoualy.telegram.tl.api.*;
+import com.github.badoualy.telegram.tl.api.upload.TLFile;
+import com.github.badoualy.telegram.tl.exception.RpcErrorException;
+
+import java.io.IOException;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.util.ArrayList;
+import java.util.LinkedList;
+
 class DownloadManager {
 	UserManager user;
+	TelegramClient client;
 	Database db;
 	
-	public DownloadManager(UserManager u) {
+	public DownloadManager(UserManager u, TelegramClient c) {
 		this.user = u;
+		this.client = c;
 		this.db = new Database(u);
 	}
 	
-	public void downloadMessages() {
+	public void downloadMessages() throws RpcErrorException, IOException {
-		System.out.println("downloading messages... not.");
+		System.out.print("Downloading dialogs... ");
+		TLAbsDialogs dialogs = client.messagesGetDialogs(
+			0,
+			0,
+			new TLInputPeerEmpty(),
+			100);
+		System.out.println("Got " + dialogs.getDialogs().size() + " dialogs.");
+		int max_message_id = -1;
+		for(TLAbsDialog dialog : dialogs.getDialogs()) {
+			max_message_id = Math.max(max_message_id, dialog.getTopMessage());
+		}
+		System.out.println("Top message ID is " + max_message_id);
+		int max_database_id = db.getTopMessageID();
+		System.out.println("Top message ID in database is " + max_database_id);
+		
+		int start_id = max_database_id + 1;
+		int current_start_id = start_id;
+		int end_id = max_message_id;
+		if (start_id > end_id) {
+			System.out.println("No new messages to download.");
+			return;
+		}
+		
+		while (current_start_id <= end_id) {
+			int my_end_id = Math.min(current_start_id+99, end_id);
+			ArrayList<Integer> a = makeIdList(current_start_id, my_end_id);
+			TLIntVector ids = new TLIntVector();
+			ids.addAll(a);
+			my_end_id = ids.get(ids.size()-1);
+			System.out.println("Fetching messages from " + ids.get(0) + " to " + my_end_id + "...");
+			current_start_id = my_end_id + 1;
+			
+			TLAbsMessages response = client.messagesGetMessages(ids);
+			db.save(response.getMessages());
+			try {
+				Thread.sleep(750);
+			} catch (InterruptedException e) {}
+		}
+	}
+	
+	public void downloadMedia() throws RpcErrorException, IOException {
+		LinkedList<TLMessage> messages = this.db.getMessagesWithMedia();
+		for (TLMessage msg : messages) {
+			TLAbsMessageMedia media = msg.getMedia();
+			
+			if (media instanceof TLMessageMediaPhoto) {
+				TLMessageMediaPhoto p = (TLMessageMediaPhoto) media;
+				if (p.getPhoto() instanceof TLPhoto) {
+					TLPhoto photo = (TLPhoto) p.getPhoto();
+					TLPhotoSize size = null;
+					for (TLAbsPhotoSize s : photo.getSizes()) {
+						if (s instanceof TLPhotoSize) {
+							TLPhotoSize s2 = (TLPhotoSize) s;
+							if (size == null || (s2.getW()>size.getW() && s2.getH()>size.getH())) {
+								size = s2;
+							}
+						}
+					}
+					if (size==null) {
+						throw new RuntimeException("Could not find a size for a photo.");
+					}
+					if (size.getLocation() instanceof TLFileLocation) {
+						this.downloadPhoto(msg.getId(), (TLFileLocation)size.getLocation());
+						return;
+					}
+				} else {
+					throw new RuntimeException("Got an unexpected " + p.getPhoto().getClass().getName());
+				}
+			}
+		}
+	}
+	
+	private ArrayList<Integer> makeIdList(int start, int end) {
+		if (start > end) throw new RuntimeException("start and end reversed");
+		ArrayList<Integer> a = new ArrayList<Integer>(end - start + 1);
+		for (int i=0; i<=end-start; i++) a.add(start+i);
+		return a; 
+	}
+	
+	private void downloadPhoto(int msgId, TLFileLocation src) throws RpcErrorException, IOException {
+		TLInputFileLocation loc = new TLInputFileLocation();
+		loc.setVolumeId(src.getVolumeId());
+		loc.setLocalId(src.getLocalId());
+		loc.setSecret(src.getSecret());
+		
+		this.downloadFile(this.makeFilename(msgId, "jpg"), loc);
+	}
+	
+	private String makeFilename(int id, String ext) {
+		String path = Config.FILE_BASE + 
+			File.separatorChar + 
+			Config.FILE_FILES_BASE +
+			File.separatorChar;
+		new File(path).mkdirs();
+		if (ext!=null) return path + id + "." + ext;
+		return path + id + ".dat";
+	}
+		
+	
+	private void downloadFile(String target, TLInputFileLocation loc) throws RpcErrorException, IOException {
+		FileOutputStream fos = new FileOutputStream(target);
+		int offset = 0;
+		TLFile response;
+		do {
+			response = this.client.uploadGetFile(loc, offset, Config.FILE_DOWNLOAD_BLOCK_SIZE);
+			offset += Config.FILE_DOWNLOAD_BLOCK_SIZE;
+			fos.write(response.getBytes().getData());
+		} while(response.getBytes().getLength() == Config.FILE_DOWNLOAD_BLOCK_SIZE);
+		fos.close();
 	}
 }