From d60a56711ac9c0dbea2d79d43d603025de6c461f Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Fri, 1 Jul 2016 07:40:08 +0200 Subject: [PATCH] Download-stuff. Files. --- .../CommandLineController.java | 5 +- .../fabianonline/telegram_backup/Config.java | 3 + .../telegram_backup/Database.java | 125 +++++++++++++++-- .../telegram_backup/DownloadManager.java | 130 +++++++++++++++++- 4 files changed, 250 insertions(+), 13 deletions(-) diff --git a/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java b/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java index d2b1acc..e18cb64 100644 --- a/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java +++ b/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java @@ -64,10 +64,11 @@ public class CommandLineController { System.out.println("Next time, please run this tool with '--account " + user.getUser().getPhone() + " to use this account."); } - System.out.println("You are now signed in as " + user.getUserString()); + System.out.println("You are logged in as " + user.getUserString()); - DownloadManager d = new DownloadManager(user); + DownloadManager d = new DownloadManager(user, client); d.downloadMessages(); + d.downloadMedia(); } catch (RpcErrorException e) { e.printStackTrace(); } catch (IOException e) { diff --git a/src/main/java/de/fabianonline/telegram_backup/Config.java b/src/main/java/de/fabianonline/telegram_backup/Config.java index 35636f4..c6124aa 100644 --- a/src/main/java/de/fabianonline/telegram_backup/Config.java +++ b/src/main/java/de/fabianonline/telegram_backup/Config.java @@ -14,5 +14,8 @@ class Config { public static final String FILE_NAME_AUTH_KEY = "auth.dat"; public static final String FILE_NAME_DC = "dc.dat"; public static final String FILE_NAME_DB = "database.sqlite"; + public static final String FILE_FILES_BASE = "files"; + + public static final int FILE_DOWNLOAD_BLOCK_SIZE = 1024*1024; } diff --git a/src/main/java/de/fabianonline/telegram_backup/Database.java b/src/main/java/de/fabianonline/telegram_backup/Database.java index 1e5e3ad..bd66dd9 100644 --- a/src/main/java/de/fabianonline/telegram_backup/Database.java +++ b/src/main/java/de/fabianonline/telegram_backup/Database.java @@ -1,13 +1,28 @@ package de.fabianonline.telegram_backup; import com.github.badoualy.telegram.tl.api.TLMessage; +import com.github.badoualy.telegram.tl.api.TLMessageEmpty; +import com.github.badoualy.telegram.tl.api.TLMessageService; +import com.github.badoualy.telegram.tl.core.TLVector; +import com.github.badoualy.telegram.tl.api.TLAbsMessage; +import com.github.badoualy.telegram.tl.api.TLMessageMediaEmpty; +import com.github.badoualy.telegram.tl.api.TLAbsPeer; +import com.github.badoualy.telegram.tl.api.TLPeerUser; +import com.github.badoualy.telegram.tl.api.TLPeerChat; +import com.github.badoualy.telegram.tl.api.TLPeerChannel; +import com.github.badoualy.telegram.tl.api.TLApiContext; import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; import java.sql.SQLException; import java.sql.ResultSet; +import java.sql.PreparedStatement; +import java.sql.Types; import java.io.File; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.LinkedList; import de.fabianonline.telegram_backup.UserManager; @@ -15,8 +30,10 @@ import de.fabianonline.telegram_backup.UserManager; class Database { private Connection conn; private Statement stmt; + private UserManager user_manager; - public Database(UserManager user) { + public Database(UserManager user_manager) { + this.user_manager = user_manager; try { Class.forName("org.sqlite.JDBC"); } catch(ClassNotFoundException e) { @@ -26,7 +43,7 @@ class Database { String path = "jdbc:sqlite:" + Config.FILE_BASE + File.separatorChar + - user.getUser().getPhone() + + user_manager.getUser().getPhone() + File.separatorChar + Config.FILE_NAME_DB; @@ -87,11 +104,103 @@ class Database { } } - public void save(TLMessage msg) { - /*BufferArrayOutputStream stream = new BufferArrayOutputStream(); - msg.serializeBody(stream); - bytes[] = stream.toByteArray(); - PreparedStatement ps = conn.prepareStatement(query); - ps.setBytes(x, bytes[]);*/ + public int getTopMessageID() { + try { + ResultSet rs = stmt.executeQuery("SELECT MAX(id) FROM messages"); + rs.next(); + return rs.getInt(1); + } catch (SQLException e) { + return 0; + } + } + + public void save(TLVector all) { + try { + PreparedStatement ps = conn.prepareStatement( + "INSERT INTO messages " + + "(id, dialog_id, from_id, type, text, time, has_media, data) " + + "VALUES " + + "(?, ?, ?, ?, ?, ?, ?, ?)"); + for (TLAbsMessage abs : all) { + if (abs instanceof TLMessage) { + TLMessage msg = (TLMessage) abs; + ps.setInt(1, msg.getId()); + TLAbsPeer peer = msg.getToId(); + if (peer instanceof TLPeerChat) { + ps.setInt(2, ((TLPeerChat)peer).getChatId()); + ps.setString(4, "chat"); + } else if (peer instanceof TLPeerChannel) { + ps.setInt(2, ((TLPeerChannel)peer).getChannelId()); + ps.setString(4, "channel"); + } else if (peer instanceof TLPeerUser) { + int id = ((TLPeerUser)peer).getUserId(); + if (id==this.user_manager.getUser().getId()) { + id = msg.getFromId(); + } + ps.setInt(2, id); + ps.setString(4, "user"); + } else { + throw new RuntimeException("Unexpected Peer type: " + peer.getClass().getName()); + } + ps.setInt(3, msg.getFromId()); + String text = msg.getMessage(); + if ((text==null || text.equals("")) && msg.getMedia()!=null) { + if (msg.getMedia() instanceof TLMessageMediaDocument) { + text = ((TLMessageMediaDocument)msg.getMedia()).getCaption(); + } else if (msg.getMedia() instanceof TLMessageMediaPhoto) { + text = ((TLMessageMediaPhoto)msg.getMedia()).getCaption(); + } + } + ps.setString(5, text); + ps.setString(6, ""+msg.getDate()); + ps.setBoolean(7, msg.getMedia() != null); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + msg.serializeBody(stream); + ps.setBytes(8, stream.toByteArray()); + ps.addBatch(); + } else if (abs instanceof TLMessageService) { + // Ignore service messages. + } else if (abs instanceof TLMessageEmpty) { + TLMessageEmpty msg = (TLMessageEmpty) abs; + ps.setInt(1, msg.getId()); + ps.setNull(2, Types.INTEGER); + ps.setNull(3, Types.INTEGER); + ps.setNull(4, Types.VARCHAR); + ps.setNull(5, Types.VARCHAR); + ps.setNull(6, Types.INTEGER); + ps.setNull(7, Types.BOOLEAN); + ps.setNull(8, Types.BLOB); + ps.addBatch(); + } else { + throw new RuntimeException("Unexpected Message type: " + abs.getClass().getName()); + } + } + conn.setAutoCommit(false); + ps.executeBatch(); + ps.clearBatch(); + conn.commit(); + conn.setAutoCommit(true); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException("Exception shown above happened."); + } + } + + public LinkedList getMessagesWithMedia() { + try { + LinkedList list = new LinkedList(); + ResultSet rs = stmt.executeQuery("SELECT data FROM messages WHERE has_media=1"); + while (rs.next()) { + ByteArrayInputStream stream = new ByteArrayInputStream(rs.getBytes(1)); + TLMessage msg = new TLMessage(); + msg.deserializeBody(stream, TLApiContext.getInstance()); + list.add(msg); + } + rs.close(); + return list; + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException("Exception occured. See above."); + } } } diff --git a/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java b/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java index c1f78a3..45617e0 100644 --- a/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java +++ b/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java @@ -3,16 +3,140 @@ package de.fabianonline.telegram_backup; import de.fabianonline.telegram_backup.UserManager; import de.fabianonline.telegram_backup.Database; +import com.github.badoualy.telegram.api.TelegramClient; +import com.github.badoualy.telegram.tl.core.TLIntVector; +import com.github.badoualy.telegram.tl.api.messages.TLAbsMessages; +import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs; +import com.github.badoualy.telegram.tl.api.*; +import com.github.badoualy.telegram.tl.api.upload.TLFile; +import com.github.badoualy.telegram.tl.exception.RpcErrorException; + +import java.io.IOException; +import java.io.File; +import java.io.FileOutputStream; +import java.util.ArrayList; +import java.util.LinkedList; + class DownloadManager { UserManager user; + TelegramClient client; Database db; - public DownloadManager(UserManager u) { + public DownloadManager(UserManager u, TelegramClient c) { this.user = u; + this.client = c; this.db = new Database(u); } - public void downloadMessages() { - System.out.println("downloading messages... not."); + public void downloadMessages() throws RpcErrorException, IOException { + System.out.print("Downloading dialogs... "); + TLAbsDialogs dialogs = client.messagesGetDialogs( + 0, + 0, + new TLInputPeerEmpty(), + 100); + System.out.println("Got " + dialogs.getDialogs().size() + " dialogs."); + int max_message_id = -1; + for(TLAbsDialog dialog : dialogs.getDialogs()) { + max_message_id = Math.max(max_message_id, dialog.getTopMessage()); + } + System.out.println("Top message ID is " + max_message_id); + int max_database_id = db.getTopMessageID(); + System.out.println("Top message ID in database is " + max_database_id); + + int start_id = max_database_id + 1; + int current_start_id = start_id; + int end_id = max_message_id; + if (start_id > end_id) { + System.out.println("No new messages to download."); + return; + } + + while (current_start_id <= end_id) { + int my_end_id = Math.min(current_start_id+99, end_id); + ArrayList a = makeIdList(current_start_id, my_end_id); + TLIntVector ids = new TLIntVector(); + ids.addAll(a); + my_end_id = ids.get(ids.size()-1); + System.out.println("Fetching messages from " + ids.get(0) + " to " + my_end_id + "..."); + current_start_id = my_end_id + 1; + + TLAbsMessages response = client.messagesGetMessages(ids); + db.save(response.getMessages()); + try { + Thread.sleep(750); + } catch (InterruptedException e) {} + } + } + + public void downloadMedia() throws RpcErrorException, IOException { + LinkedList messages = this.db.getMessagesWithMedia(); + for (TLMessage msg : messages) { + TLAbsMessageMedia media = msg.getMedia(); + + if (media instanceof TLMessageMediaPhoto) { + TLMessageMediaPhoto p = (TLMessageMediaPhoto) media; + if (p.getPhoto() instanceof TLPhoto) { + TLPhoto photo = (TLPhoto) p.getPhoto(); + TLPhotoSize size = null; + for (TLAbsPhotoSize s : photo.getSizes()) { + if (s instanceof TLPhotoSize) { + TLPhotoSize s2 = (TLPhotoSize) s; + if (size == null || (s2.getW()>size.getW() && s2.getH()>size.getH())) { + size = s2; + } + } + } + if (size==null) { + throw new RuntimeException("Could not find a size for a photo."); + } + if (size.getLocation() instanceof TLFileLocation) { + this.downloadPhoto(msg.getId(), (TLFileLocation)size.getLocation()); + return; + } + } else { + throw new RuntimeException("Got an unexpected " + p.getPhoto().getClass().getName()); + } + } + } + } + + private ArrayList makeIdList(int start, int end) { + if (start > end) throw new RuntimeException("start and end reversed"); + ArrayList a = new ArrayList(end - start + 1); + for (int i=0; i<=end-start; i++) a.add(start+i); + return a; + } + + private void downloadPhoto(int msgId, TLFileLocation src) throws RpcErrorException, IOException { + TLInputFileLocation loc = new TLInputFileLocation(); + loc.setVolumeId(src.getVolumeId()); + loc.setLocalId(src.getLocalId()); + loc.setSecret(src.getSecret()); + + this.downloadFile(this.makeFilename(msgId, "jpg"), loc); + } + + private String makeFilename(int id, String ext) { + String path = Config.FILE_BASE + + File.separatorChar + + Config.FILE_FILES_BASE + + File.separatorChar; + new File(path).mkdirs(); + if (ext!=null) return path + id + "." + ext; + return path + id + ".dat"; + } + + + private void downloadFile(String target, TLInputFileLocation loc) throws RpcErrorException, IOException { + FileOutputStream fos = new FileOutputStream(target); + int offset = 0; + TLFile response; + do { + response = this.client.uploadGetFile(loc, offset, Config.FILE_DOWNLOAD_BLOCK_SIZE); + offset += Config.FILE_DOWNLOAD_BLOCK_SIZE; + fos.write(response.getBytes().getData()); + } while(response.getBytes().getLength() == Config.FILE_DOWNLOAD_BLOCK_SIZE); + fos.close(); } }