From 265f3e854acd8c371eba13968985bc5628161c51 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Fri, 1 Jul 2016 11:25:44 +0200 Subject: [PATCH] This looks more or less usable... --- .../CommandLineController.java | 10 +- .../fabianonline/telegram_backup/Config.java | 3 + .../telegram_backup/Database.java | 45 +++-- .../telegram_backup/DownloadManager.java | 184 ++++++++++++++---- .../telegram_backup/StickerConverter.java | 36 ++++ .../telegram_backup/UserManager.java | 5 + 6 files changed, 229 insertions(+), 54 deletions(-) create mode 100644 src/main/java/de/fabianonline/telegram_backup/StickerConverter.java diff --git a/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java b/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java index e18cb64..c2fb818 100644 --- a/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java +++ b/src/main/java/de/fabianonline/telegram_backup/CommandLineController.java @@ -59,7 +59,7 @@ public class CommandLineController { String pw = new Scanner(System.in).nextLine(); user.verifyPassword(pw); } - storage.setPrefix(user.getUser().getPhone()); + storage.setPrefix("+" + user.getUser().getPhone()); System.out.println("Next time, please run this tool with '--account " + user.getUser().getPhone() + " to use this account."); } @@ -67,7 +67,7 @@ public class CommandLineController { System.out.println("You are logged in as " + user.getUserString()); DownloadManager d = new DownloadManager(user, client); - d.downloadMessages(); + d.downloadMessages(options.limit_messages); d.downloadMedia(); } catch (RpcErrorException e) { e.printStackTrace(); @@ -85,7 +85,8 @@ public class CommandLineController { System.out.println(" --account Use account ."); System.out.println(" --login Login to an existing telegram account."); System.out.println(" --debug Show (lots of) debug information."); - System.out.println(" --list-accounts List all existing accounts"); + System.out.println(" --list-accounts List all existing accounts "); + System.out.println(" --limit-messages Downloads at most the most recent messages."); System.exit(0); } @@ -121,6 +122,7 @@ public class CommandLineController { public boolean cmd_login = false; public boolean cmd_debug = false; public boolean cmd_list_accounts = false; + public Integer limit_messages = null; public CommandLineOptions(String[] args) { String last_cmd = null; @@ -129,6 +131,7 @@ public class CommandLineController { if (last_cmd != null) { switch(last_cmd) { case "--account": this.account=arg; break; + case "--limit-messages": this.limit_messages=Integer.parseInt(arg); break; } last_cmd = null; continue; @@ -140,6 +143,7 @@ public class CommandLineController { case "--login": this.cmd_login=true; break; case "--debug": this.cmd_debug=true; break; case "--list-accounts": this.cmd_list_accounts=true; break; + case "--limit-messages": last_cmd=arg; continue; default: throw new RuntimeException("Unknown command " + arg); } } diff --git a/src/main/java/de/fabianonline/telegram_backup/Config.java b/src/main/java/de/fabianonline/telegram_backup/Config.java index c6124aa..256f250 100644 --- a/src/main/java/de/fabianonline/telegram_backup/Config.java +++ b/src/main/java/de/fabianonline/telegram_backup/Config.java @@ -15,7 +15,10 @@ class Config { public static final String FILE_NAME_DC = "dc.dat"; public static final String FILE_NAME_DB = "database.sqlite"; public static final String FILE_FILES_BASE = "files"; + public static final String FILE_STICKER_BASE = "stickers"; public static final int FILE_DOWNLOAD_BLOCK_SIZE = 1024*1024; + + public static final int DELAY_AFTER_GET_FILE = 750; } diff --git a/src/main/java/de/fabianonline/telegram_backup/Database.java b/src/main/java/de/fabianonline/telegram_backup/Database.java index bd66dd9..2f39edd 100644 --- a/src/main/java/de/fabianonline/telegram_backup/Database.java +++ b/src/main/java/de/fabianonline/telegram_backup/Database.java @@ -1,16 +1,7 @@ package de.fabianonline.telegram_backup; -import com.github.badoualy.telegram.tl.api.TLMessage; -import com.github.badoualy.telegram.tl.api.TLMessageEmpty; -import com.github.badoualy.telegram.tl.api.TLMessageService; +import com.github.badoualy.telegram.tl.api.*; import com.github.badoualy.telegram.tl.core.TLVector; -import com.github.badoualy.telegram.tl.api.TLAbsMessage; -import com.github.badoualy.telegram.tl.api.TLMessageMediaEmpty; -import com.github.badoualy.telegram.tl.api.TLAbsPeer; -import com.github.badoualy.telegram.tl.api.TLPeerUser; -import com.github.badoualy.telegram.tl.api.TLPeerChat; -import com.github.badoualy.telegram.tl.api.TLPeerChannel; -import com.github.badoualy.telegram.tl.api.TLApiContext; import java.sql.Connection; import java.sql.DriverManager; @@ -25,7 +16,7 @@ import java.io.ByteArrayOutputStream; import java.util.LinkedList; import de.fabianonline.telegram_backup.UserManager; - +import de.fabianonline.telegram_backup.StickerConverter; class Database { private Connection conn; @@ -41,10 +32,7 @@ class Database { } String path = "jdbc:sqlite:" + - Config.FILE_BASE + - File.separatorChar + - user_manager.getUser().getPhone() + - File.separatorChar + + user_manager.getFileBase() + Config.FILE_NAME_DB; try { @@ -74,6 +62,7 @@ class Database { System.out.println("Database version: " + version); if (version==0) { + System.out.println(" Updating to version 1..."); stmt.executeUpdate("CREATE TABLE messages (" + "id INTEGER PRIMARY KEY ASC, " + "dialog_id INTEGER, " @@ -83,6 +72,7 @@ class Database { + "text TEXT, " + "time TEXT, " + "has_media BOOLEAN, " + + "sticker TEXT, " + "data BLOB)"); stmt.executeUpdate("CREATE TABLE dialogs (" + "id INTEGER PRIMARY KEY ASC, " @@ -98,6 +88,8 @@ class Database { stmt.executeUpdate("INSERT INTO database_versions (version) VALUES (1)"); version = 1; } + + System.out.println("Database is ready."); } catch (SQLException e) { System.out.println(e.getSQLState()); e.printStackTrace(); @@ -118,9 +110,9 @@ class Database { try { PreparedStatement ps = conn.prepareStatement( "INSERT INTO messages " + - "(id, dialog_id, from_id, type, text, time, has_media, data) " + + "(id, dialog_id, from_id, type, text, time, has_media, data, sticker) " + "VALUES " + - "(?, ?, ?, ?, ?, ?, ?, ?)"); + "(?, ?, ?, ?, ?, ?, ?, ?, ?)"); for (TLAbsMessage abs : all) { if (abs instanceof TLMessage) { TLMessage msg = (TLMessage) abs; @@ -150,6 +142,7 @@ class Database { } else if (msg.getMedia() instanceof TLMessageMediaPhoto) { text = ((TLMessageMediaPhoto)msg.getMedia()).getCaption(); } + System.out.println("" + msg.getId() + ": >" + text + "<"); } ps.setString(5, text); ps.setString(6, ""+msg.getDate()); @@ -157,6 +150,24 @@ class Database { ByteArrayOutputStream stream = new ByteArrayOutputStream(); msg.serializeBody(stream); ps.setBytes(8, stream.toByteArray()); + String sticker = null; + if (msg.getMedia()!=null && msg.getMedia() instanceof TLMessageMediaDocument) { + TLMessageMediaDocument md = (TLMessageMediaDocument)msg.getMedia(); + if (md.getDocument() instanceof TLDocument) { + for (TLAbsDocumentAttribute attr : ((TLDocument)md.getDocument()).getAttributes()) { + if (attr instanceof TLDocumentAttributeSticker) { + sticker = StickerConverter.makeFilename((TLDocumentAttributeSticker)attr); + break; + } + } + } + } + if (sticker != null) { + System.out.println("" + msg.getId() + ": >" + msg.getMessage() + "< " + sticker); + ps.setString(9, sticker); + } else { + ps.setNull(9, Types.VARCHAR); + } ps.addBatch(); } else if (abs instanceof TLMessageService) { // Ignore service messages. diff --git a/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java b/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java index 45617e0..bbaa6cd 100644 --- a/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java +++ b/src/main/java/de/fabianonline/telegram_backup/DownloadManager.java @@ -2,6 +2,7 @@ package de.fabianonline.telegram_backup; import de.fabianonline.telegram_backup.UserManager; import de.fabianonline.telegram_backup.Database; +import de.fabianonline.telegram_backup.StickerConverter; import com.github.badoualy.telegram.api.TelegramClient; import com.github.badoualy.telegram.tl.core.TLIntVector; @@ -10,6 +11,7 @@ import com.github.badoualy.telegram.tl.api.messages.TLAbsDialogs; import com.github.badoualy.telegram.tl.api.*; import com.github.badoualy.telegram.tl.api.upload.TLFile; import com.github.badoualy.telegram.tl.exception.RpcErrorException; +import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile; import java.io.IOException; import java.io.File; @@ -28,7 +30,7 @@ class DownloadManager { this.db = new Database(u); } - public void downloadMessages() throws RpcErrorException, IOException { + public void downloadMessages(Integer limit) throws RpcErrorException, IOException { System.out.print("Downloading dialogs... "); TLAbsDialogs dialogs = client.messagesGetDialogs( 0, @@ -44,6 +46,12 @@ class DownloadManager { int max_database_id = db.getTopMessageID(); System.out.println("Top message ID in database is " + max_database_id); + if (limit != null) { + System.out.println("Limit is set to " + limit); + max_database_id = Math.max(max_database_id, max_message_id-limit); + System.out.println("New top message id 'in database' is " + max_database_id); + } + int start_id = max_database_id + 1; int current_start_id = start_id; int end_id = max_message_id; @@ -75,29 +83,99 @@ class DownloadManager { TLAbsMessageMedia media = msg.getMedia(); if (media instanceof TLMessageMediaPhoto) { - TLMessageMediaPhoto p = (TLMessageMediaPhoto) media; - if (p.getPhoto() instanceof TLPhoto) { - TLPhoto photo = (TLPhoto) p.getPhoto(); - TLPhotoSize size = null; - for (TLAbsPhotoSize s : photo.getSizes()) { - if (s instanceof TLPhotoSize) { - TLPhotoSize s2 = (TLPhotoSize) s; - if (size == null || (s2.getW()>size.getW() && s2.getH()>size.getH())) { - size = s2; - } - } + this.downloadMessageMediaPhoto(msg, (TLMessageMediaPhoto)media); + } else if (media instanceof TLMessageMediaDocument) { + this.downloadMessageMediaDocument(msg, (TLMessageMediaDocument)media); + } else if (media instanceof TLMessageMediaVideo) { + this.downloadMessageMediaVideo(msg, (TLMessageMediaVideo)media); + } else if (media instanceof TLMessageMediaEmpty) { + // do nothing + } else if (media instanceof TLMessageMediaUnsupported) { + // do nothing + } else if (media instanceof TLMessageMediaGeo) { + // do nothing + } else if (media instanceof TLMessageMediaWebPage) { + // do nothing + } else if (media instanceof TLMessageMediaContact) { + // do nothing + } else if (media instanceof TLMessageMediaVenue) { + // do nothing + } else { + throw new RuntimeException("Unexpected " + media.getClass().getName()); + } + } + } + + private void downloadMessageMediaPhoto(TLMessage msg, TLMessageMediaPhoto p) throws RpcErrorException, IOException { + if (p.getPhoto() instanceof TLPhoto) { + TLPhoto photo = (TLPhoto) p.getPhoto(); + TLPhotoSize size = null; + for (TLAbsPhotoSize s : photo.getSizes()) { + if (s instanceof TLPhotoSize) { + TLPhotoSize s2 = (TLPhotoSize) s; + if (size == null || (s2.getW()>size.getW() && s2.getH()>size.getH())) { + size = s2; } - if (size==null) { - throw new RuntimeException("Could not find a size for a photo."); - } - if (size.getLocation() instanceof TLFileLocation) { - this.downloadPhoto(msg.getId(), (TLFileLocation)size.getLocation()); - return; - } - } else { - throw new RuntimeException("Got an unexpected " + p.getPhoto().getClass().getName()); } } + if (size==null) { + throw new RuntimeException("Could not find a size for a photo."); + } + if (size.getLocation() instanceof TLFileLocation) { + this.downloadPhoto(msg.getId(), (TLFileLocation)size.getLocation()); + } + } else { + throw new RuntimeException("Got an unexpected " + p.getPhoto().getClass().getName()); + } + } + + private void downloadMessageMediaDocument(TLMessage msg, TLMessageMediaDocument d) throws RpcErrorException, IOException { + if (d.getDocument() instanceof TLDocument) { + TLDocument doc = (TLDocument)d.getDocument(); + //Determine files extension + String original_filename = null; + TLDocumentAttributeSticker sticker = null; + + String ext = null; + for(TLAbsDocumentAttribute attr : doc.getAttributes()) { + if (attr instanceof TLDocumentAttributeFilename) { + original_filename = ((TLDocumentAttributeFilename)attr).getFileName(); + } else if (attr instanceof TLDocumentAttributeSticker) { + sticker = (TLDocumentAttributeSticker)attr; + } + } + if (original_filename != null) { + int i = original_filename.lastIndexOf('.'); + if (i>0) ext = original_filename.substring(i+1); + } + if (ext==null) { + int i = doc.getMimeType().lastIndexOf('/'); + String type = doc.getMimeType().substring(i+1).toLowerCase(); + if (type.equals("unknown")) { + ext = "dat"; + } else { + ext = type; + } + } + String filename; + if (sticker != null) { + filename = StickerConverter.makeFilenameWithPath(sticker); + } else { + filename = this.makeFilename(msg.getId(), ext); + } + + this.downloadDocument(filename, doc); + } else { + throw new RuntimeException("Got an unexpected " + d.getDocument().getClass().getName()); + } + } + + private void downloadMessageMediaVideo(TLMessage msg, TLMessageMediaVideo v) throws RpcErrorException, IOException { + if (v.getVideo() instanceof TLVideo) { + TLVideo vid = (TLVideo)v.getVideo(); + int i = vid.getMimeType().lastIndexOf('/'); + String ext = vid.getMimeType().substring(i+1).toLowerCase(); + this.downloadVideo(this.makeFilename(msg.getId(), ext), vid); } } @@ -117,26 +195,64 @@ class DownloadManager { this.downloadFile(this.makeFilename(msgId, "jpg"), loc); } + private void downloadDocument(String filename, TLDocument doc) throws RpcErrorException, IOException { + TLInputDocumentFileLocation loc = new TLInputDocumentFileLocation(); + loc.setId(doc.getId()); + loc.setAccessHash(doc.getAccessHash()); + this.downloadFileFromDc(filename, loc, doc.getDcId()); + } + + private void downloadVideo(String filename, TLVideo vid) throws RpcErrorException, IOException { + TLInputDocumentFileLocation loc = new TLInputDocumentFileLocation(); + loc.setId(vid.getId()); + loc.setAccessHash(vid.getAccessHash()); + this.downloadFileFromDc(filename, loc, vid.getDcId()); + } + private String makeFilename(int id, String ext) { - String path = Config.FILE_BASE + - File.separatorChar + + String path = this.user.getFileBase() + Config.FILE_FILES_BASE + File.separatorChar; new File(path).mkdirs(); if (ext!=null) return path + id + "." + ext; return path + id + ".dat"; } - - private void downloadFile(String target, TLInputFileLocation loc) throws RpcErrorException, IOException { - FileOutputStream fos = new FileOutputStream(target); - int offset = 0; - TLFile response; - do { - response = this.client.uploadGetFile(loc, offset, Config.FILE_DOWNLOAD_BLOCK_SIZE); - offset += Config.FILE_DOWNLOAD_BLOCK_SIZE; - fos.write(response.getBytes().getData()); - } while(response.getBytes().getLength() == Config.FILE_DOWNLOAD_BLOCK_SIZE); - fos.close(); + private void downloadFile(String target, TLAbsInputFileLocation loc) throws RpcErrorException, IOException { + downloadFileFromDc(target, loc, null); + } + + private void downloadFileFromDc(String target, TLAbsInputFileLocation loc, Integer dcID) throws RpcErrorException, IOException { + // Don't download already existing files. + if (new File(target).isFile()) return; + + FileOutputStream fos = null; + try { + fos = new FileOutputStream(target); + int offset = 0; + TLFile response; + do { + TLRequestUploadGetFile req = new TLRequestUploadGetFile(loc, offset, Config.FILE_DOWNLOAD_BLOCK_SIZE); + if (dcID==null) { + response = (TLFile)this.client.executeRpcQuery(req); + } else { + response = (TLFile)this.client.executeRpcQuery(req, dcID); + } + offset += Config.FILE_DOWNLOAD_BLOCK_SIZE; + fos.write(response.getBytes().getData()); + try { Thread.sleep(Config.DELAY_AFTER_GET_FILE); } catch(InterruptedException e) {} + } while(response.getBytes().getLength() == Config.FILE_DOWNLOAD_BLOCK_SIZE); + fos.close(); + } catch (java.io.IOException ex) { + if (fos!=null) fos.close(); + new File(target).delete(); + System.out.println("IOException happened while downloading " + target); + throw ex; + } catch (RpcErrorException ex) { + if (fos!=null) fos.close(); + new File(target).delete(); + System.out.println("RpcErrorException happened while downloading " + target); + throw ex; + } } } diff --git a/src/main/java/de/fabianonline/telegram_backup/StickerConverter.java b/src/main/java/de/fabianonline/telegram_backup/StickerConverter.java new file mode 100644 index 0000000..907e5c1 --- /dev/null +++ b/src/main/java/de/fabianonline/telegram_backup/StickerConverter.java @@ -0,0 +1,36 @@ +package de.fabianonline.telegram_backup; + +import com.github.badoualy.telegram.tl.api.*; +import java.lang.StringBuilder; +import java.io.File; + +class StickerConverter { + public static String makeFilenameWithPath(TLDocumentAttributeSticker attr) { + StringBuilder file = new StringBuilder(); + file.append(makePath()); + file.append(makeFilename(attr)); + return file.toString(); + } + + public static String makeFilename(TLDocumentAttributeSticker attr) { + StringBuilder file = new StringBuilder(); + if (attr.getStickerset() instanceof TLInputStickerSetShortName) { + file.append(((TLInputStickerSetShortName)attr.getStickerset()).getShortName()); + } else if (attr.getStickerset() instanceof TLInputStickerSetID) { + file.append(((TLInputStickerSetID)attr.getStickerset()).getId()); + } + file.append("_"); + file.append(attr.getAlt().hashCode()); + file.append(".webp"); + return file.toString(); + } + + public static String makePath() { + String path = Config.FILE_BASE + + File.separatorChar + + Config.FILE_STICKER_BASE + + File.separatorChar; + new File(path).mkdirs(); + return path; + } +} diff --git a/src/main/java/de/fabianonline/telegram_backup/UserManager.java b/src/main/java/de/fabianonline/telegram_backup/UserManager.java index 02d1b6e..be710e6 100644 --- a/src/main/java/de/fabianonline/telegram_backup/UserManager.java +++ b/src/main/java/de/fabianonline/telegram_backup/UserManager.java @@ -13,6 +13,7 @@ import com.github.badoualy.telegram.tl.core.TLBytes; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.io.IOException; +import java.io.File; class UserManager { public TLUser user = null; @@ -91,4 +92,8 @@ class UserManager { } public TLUser getUser() { return this.user; } + + public String getFileBase() { + return Config.FILE_BASE + File.separatorChar + "+" + this.user.getPhone() + File.separatorChar; + } }