mirror of
https://github.com/fabianonline/telegram_backup.git
synced 2024-11-22 08:46:15 +00:00
Implemented backing up of supergroups and channels added. Finally. Yaaaay. Be aware: There will possibly be a lot of bugs!
This commit is contained in:
parent
14d80a70fe
commit
7bf22f4692
@ -1,16 +1,16 @@
|
||||
/* Telegram_Backup
|
||||
* Copyright (C) 2016 Fabian Schlenz
|
||||
*
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. */
|
||||
|
||||
@ -39,11 +39,11 @@ public class CommandLineController {
|
||||
private static Logger logger = LoggerFactory.getLogger(CommandLineController.class);
|
||||
private ApiStorage storage;
|
||||
public TelegramApp app;
|
||||
|
||||
|
||||
public CommandLineController() {
|
||||
logger.info("CommandLineController started. App version {}", Config.APP_APPVER);
|
||||
this.printHeader();
|
||||
|
||||
|
||||
if (CommandLineOptions.cmd_version) {
|
||||
System.exit(0);
|
||||
} else if (CommandLineOptions.cmd_help) {
|
||||
@ -53,20 +53,20 @@ public class CommandLineController {
|
||||
this.show_license();
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
|
||||
this.setupFileBase();
|
||||
|
||||
|
||||
if (CommandLineOptions.cmd_list_accounts) {
|
||||
this.list_accounts();
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
|
||||
logger.debug("Initializing TelegramApp");
|
||||
app = new TelegramApp(Config.APP_ID, Config.APP_HASH, Config.APP_MODEL, Config.APP_SYSVER, Config.APP_APPVER, Config.APP_LANG);
|
||||
|
||||
logger.trace("Checking accounts");
|
||||
String account = this.selectAccount();
|
||||
|
||||
|
||||
logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login);
|
||||
|
||||
logger.info("Initializing ApiStorage");
|
||||
@ -75,12 +75,12 @@ public class CommandLineController {
|
||||
TelegramUpdateHandler handler = new TelegramUpdateHandler();
|
||||
logger.info("Creating Client");
|
||||
TelegramClient client = Kotlogram.getDefaultClient(app, storage, Kotlogram.PROD_DC4, handler);
|
||||
|
||||
|
||||
try {
|
||||
logger.info("Initializing UserManager");
|
||||
UserManager.init(client);
|
||||
Database.init(client);
|
||||
|
||||
|
||||
UserManager user = UserManager.getInstance();
|
||||
|
||||
if (!CommandLineOptions.cmd_login && !user.isLoggedIn()) {
|
||||
@ -93,12 +93,12 @@ public class CommandLineController {
|
||||
throw new RuntimeException("Account / User mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (CommandLineOptions.cmd_stats) {
|
||||
cmd_stats();
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
|
||||
if (CommandLineOptions.val_test != null) {
|
||||
if (CommandLineOptions.val_test == 1) {
|
||||
TestFeatures.test1();
|
||||
@ -109,7 +109,7 @@ public class CommandLineController {
|
||||
}
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
|
||||
logger.debug("CommandLineOptions.val_export: {}", CommandLineOptions.val_export);
|
||||
if (CommandLineOptions.val_export != null) {
|
||||
if (CommandLineOptions.val_export.toLowerCase().equals("html")) {
|
||||
@ -119,25 +119,25 @@ public class CommandLineController {
|
||||
show_error("Unknown export format.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login);
|
||||
if (CommandLineOptions.cmd_login) {
|
||||
cmd_login(account);
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
|
||||
if (user.isLoggedIn()) {
|
||||
System.out.println("You are logged in as " + Utils.anonymize(user.getUserString()));
|
||||
} else {
|
||||
System.out.println("You are not logged in.");
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
|
||||
logger.info("Initializing Download Manager");
|
||||
DownloadManager d = new DownloadManager(client, new CommandLineDownloadProgress());
|
||||
logger.debug("Calling DownloadManager.downloadMessages with limit {}", CommandLineOptions.val_limit_messages);
|
||||
d.downloadMessages(CommandLineOptions.val_limit_messages);
|
||||
|
||||
|
||||
logger.debug("CommandLineOptions.cmd_no_media: {}", CommandLineOptions.cmd_no_media);
|
||||
if (!CommandLineOptions.cmd_no_media) {
|
||||
logger.debug("Calling DownloadManager.downloadMedia");
|
||||
@ -160,7 +160,7 @@ public class CommandLineController {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void printHeader() {
|
||||
System.out.println("Telegram_Backup version " + Config.APP_APPVER + ", Copyright (C) 2016, 2017 Fabian Schlenz");
|
||||
System.out.println();
|
||||
@ -168,17 +168,17 @@ public class CommandLineController {
|
||||
System.out.println("welcome to redistribute it under certain conditions; run it with '--license' for details.");
|
||||
System.out.println();
|
||||
}
|
||||
|
||||
|
||||
private void setupFileBase() {
|
||||
logger.debug("Target dir at startup: {}", Utils.anonymize(Config.FILE_BASE));
|
||||
if (CommandLineOptions.val_target != null) {
|
||||
Config.FILE_BASE = CommandLineOptions.val_target;
|
||||
}
|
||||
logger.debug("Target dir after options: {}", Utils.anonymize(Config.FILE_BASE));
|
||||
|
||||
|
||||
System.out.println("Base directory for files: " + Utils.anonymize(Config.FILE_BASE));
|
||||
}
|
||||
|
||||
|
||||
private String selectAccount() {
|
||||
String account = null;
|
||||
Vector<String> accounts = Utils.getAccounts();
|
||||
@ -216,7 +216,7 @@ public class CommandLineController {
|
||||
logger.debug("account: {}", Utils.anonymize(account));
|
||||
return account;
|
||||
}
|
||||
|
||||
|
||||
private void cmd_stats() {
|
||||
System.out.println();
|
||||
System.out.println("Stats:");
|
||||
@ -226,20 +226,20 @@ public class CommandLineController {
|
||||
System.out.format(format, "Number of chats", Database.getInstance().getChatCount());
|
||||
System.out.format(format, "Number of users", Database.getInstance().getUserCount());
|
||||
System.out.format(format, "Top message ID", Database.getInstance().getTopMessageID());
|
||||
|
||||
|
||||
System.out.println();
|
||||
System.out.println("Media Types:");
|
||||
for(Map.Entry<String, Integer> pair : Database.getInstance().getMessageMediaTypesWithCount().entrySet()) {
|
||||
System.out.format(format, pair.getKey(), pair.getValue());
|
||||
}
|
||||
|
||||
|
||||
System.out.println();
|
||||
System.out.println("Api layers of messages:");
|
||||
for(Map.Entry<String, Integer> pair : Database.getInstance().getMessageApiLayerWithCount().entrySet()) {
|
||||
System.out.format(format, pair.getKey(), pair.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void cmd_login(String phone) throws RpcErrorException, IOException {
|
||||
UserManager user = UserManager.getInstance();
|
||||
if (phone==null) {
|
||||
@ -248,21 +248,21 @@ public class CommandLineController {
|
||||
phone = getLine();
|
||||
}
|
||||
user.sendCodeToPhoneNumber(phone);
|
||||
|
||||
|
||||
System.out.println("Telegram sent you a code. Please enter it here.");
|
||||
String code = getLine();
|
||||
user.verifyCode(code);
|
||||
|
||||
|
||||
if (user.isPasswordNeeded()) {
|
||||
System.out.println("We also need your account password. Please enter it now. It should not be printed, so it's okay if you see nothing while typing it.");
|
||||
String pw = getPassword();
|
||||
user.verifyPassword(pw);
|
||||
}
|
||||
storage.setPrefix("+" + user.getUser().getPhone());
|
||||
|
||||
|
||||
System.out.println("Everything seems fine. Please run this tool again with '--account +" + Utils.anonymize(user.getUser().getPhone()) + " to use this account.");
|
||||
}
|
||||
|
||||
|
||||
private String getLine() {
|
||||
if (System.console()!=null) {
|
||||
return System.console().readLine("> ");
|
||||
@ -271,7 +271,7 @@ public class CommandLineController {
|
||||
return new Scanner(System.in).nextLine();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private String getPassword() {
|
||||
if (System.console()!=null) {
|
||||
return String.valueOf(System.console().readPassword("> "));
|
||||
@ -279,7 +279,7 @@ public class CommandLineController {
|
||||
return getLine();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void show_help() {
|
||||
System.out.println("Valid options are:");
|
||||
System.out.println(" -h, --help Shows this help.");
|
||||
@ -298,8 +298,10 @@ public class CommandLineController {
|
||||
System.out.println(" -d, --daemon Keep running and automatically save new messages.");
|
||||
System.out.println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.");
|
||||
System.out.println(" --stats Print some usage statistics.");
|
||||
System.out.println(" --with-channels-and-supergroups");
|
||||
System.out.println(" Backup channels and supergroups as well.");
|
||||
}
|
||||
|
||||
|
||||
private void list_accounts() {
|
||||
System.out.println("List of available accounts:");
|
||||
List<String> accounts = Utils.getAccounts();
|
||||
@ -313,13 +315,13 @@ public class CommandLineController {
|
||||
System.out.println("Use '--login' to login to a telegram account.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void show_error(String error) {
|
||||
logger.error(error);
|
||||
System.out.println("ERROR: " + error);
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
|
||||
public static void show_license() {
|
||||
System.out.println(
|
||||
"GNU GENERAL PUBLIC LICENSE\n" +
|
||||
@ -942,7 +944,7 @@ public class CommandLineController {
|
||||
"Program, unless a warranty or assumption of liability accompanies a\n" +
|
||||
"copy of the Program in return for a fee.\n" +
|
||||
"\n" +
|
||||
" END OF TERMS AND CONDITIONS");
|
||||
" END OF TERMS AND CONDITIONS");
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
/* Telegram_Backup
|
||||
* Copyright (C) 2016 Fabian Schlenz
|
||||
*
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. */
|
||||
|
||||
@ -30,7 +30,8 @@ class CommandLineOptions {
|
||||
public static boolean cmd_no_media = false;
|
||||
public static boolean cmd_anonymize = false;
|
||||
public static boolean cmd_stats = false;
|
||||
|
||||
public static boolean cmd_channels_and_supergroups = false;
|
||||
|
||||
public static String val_account = null;
|
||||
public static Integer val_limit_messages = null;
|
||||
public static String val_target = null;
|
||||
@ -45,16 +46,16 @@ class CommandLineOptions {
|
||||
switch (last_cmd) {
|
||||
case "--account":
|
||||
val_account = arg; break;
|
||||
|
||||
|
||||
case "--limit-messages":
|
||||
val_limit_messages = Integer.parseInt(arg); break;
|
||||
|
||||
|
||||
case "--target":
|
||||
val_target = arg; break;
|
||||
|
||||
|
||||
case "--export":
|
||||
val_export = arg; break;
|
||||
|
||||
|
||||
case "--test":
|
||||
val_test = Integer.parseInt(arg); break;
|
||||
}
|
||||
@ -65,58 +66,61 @@ class CommandLineOptions {
|
||||
switch (arg) {
|
||||
case "-a": case "--account":
|
||||
last_cmd = "--account"; continue;
|
||||
|
||||
|
||||
case "-h": case "--help":
|
||||
cmd_help = true; break;
|
||||
|
||||
|
||||
case "-l": case "--login":
|
||||
cmd_login = true; break;
|
||||
|
||||
|
||||
case "--debug":
|
||||
cmd_debug = true; break;
|
||||
|
||||
|
||||
case "--trace":
|
||||
cmd_trace = true; break;
|
||||
|
||||
|
||||
case "--trace-telegram":
|
||||
cmd_trace_telegram = true; break;
|
||||
|
||||
|
||||
case "-A": case "--list-accounts":
|
||||
cmd_list_accounts = true; break;
|
||||
|
||||
|
||||
case "--limit-messages":
|
||||
last_cmd = arg; continue;
|
||||
|
||||
|
||||
case "--console":
|
||||
cmd_console = true; break;
|
||||
|
||||
|
||||
case "-t": case "--target":
|
||||
last_cmd = "--target"; continue;
|
||||
|
||||
|
||||
case "-V": case "--version":
|
||||
cmd_version = true; break;
|
||||
|
||||
|
||||
case "-e": case "--export":
|
||||
last_cmd = "--export"; continue;
|
||||
|
||||
|
||||
case "--license":
|
||||
cmd_license = true; break;
|
||||
|
||||
|
||||
case "-d": case "--daemon":
|
||||
cmd_daemon = true; break;
|
||||
|
||||
|
||||
case "--no-media":
|
||||
cmd_no_media = true; break;
|
||||
|
||||
|
||||
case "--test":
|
||||
last_cmd = "--test"; continue;
|
||||
|
||||
|
||||
case "--anonymize":
|
||||
cmd_anonymize = true; break;
|
||||
|
||||
|
||||
case "--stats":
|
||||
cmd_stats = true; break;
|
||||
|
||||
|
||||
case "--with-channels-and-supergroups":
|
||||
cmd_channels_and_supergroups = true; break;
|
||||
|
||||
default:
|
||||
throw new RuntimeException("Unknown command " + arg);
|
||||
}
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
/* Telegram_Backup
|
||||
* Copyright (C) 2016 Fabian Schlenz
|
||||
*
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. */
|
||||
|
||||
@ -54,7 +54,7 @@ public class Database {
|
||||
public TelegramClient client;
|
||||
private final static Logger logger = LoggerFactory.getLogger(Database.class);
|
||||
private static Database instance = null;
|
||||
|
||||
|
||||
private Database(TelegramClient client) {
|
||||
this.user_manager = UserManager.getInstance();
|
||||
this.client = client;
|
||||
@ -64,34 +64,34 @@ public class Database {
|
||||
} catch(ClassNotFoundException e) {
|
||||
CommandLineController.show_error("Could not load jdbc-sqlite class.");
|
||||
}
|
||||
|
||||
|
||||
String path = "jdbc:sqlite:" +
|
||||
user_manager.getFileBase() +
|
||||
Config.FILE_NAME_DB;
|
||||
|
||||
|
||||
try {
|
||||
conn = DriverManager.getConnection(path);
|
||||
stmt = conn.createStatement();
|
||||
} catch (SQLException e) {
|
||||
CommandLineController.show_error("Could not connect to SQLITE database.");
|
||||
}
|
||||
|
||||
|
||||
// Run updates
|
||||
DatabaseUpdates updates = new DatabaseUpdates(conn, this);
|
||||
updates.doUpdates();
|
||||
|
||||
|
||||
System.out.println("Database is ready.");
|
||||
}
|
||||
|
||||
|
||||
public static void init(TelegramClient c) {
|
||||
instance = new Database(c);
|
||||
}
|
||||
|
||||
|
||||
public static Database getInstance() {
|
||||
if (instance == null) throw new RuntimeException("Database is not initialized but getInstance() was called.");
|
||||
return instance;
|
||||
}
|
||||
|
||||
|
||||
public void backupDatabase(int currentVersion) {
|
||||
String filename = String.format(Config.FILE_NAME_DB_BACKUP, currentVersion);
|
||||
System.out.println(" Creating a backup of your database as " + filename);
|
||||
@ -109,17 +109,21 @@ public class Database {
|
||||
throw new RuntimeException("Could not create backup.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public int getTopMessageID() {
|
||||
try {
|
||||
ResultSet rs = stmt.executeQuery("SELECT MAX(id) FROM messages");
|
||||
ResultSet rs = stmt.executeQuery("SELECT MAX(message_id) FROM messages WHERE source_type IN ('group', 'dialog')");
|
||||
rs.next();
|
||||
return rs.getInt(1);
|
||||
} catch (SQLException e) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public int getTopMessageIDForChannel(int id) {
|
||||
return queryInt("SELECT MAX(message_id) FROM messages WHERE source_id=" + id + " AND source_type IN('channel', 'supergroup')");
|
||||
}
|
||||
|
||||
public void logRun(int start_id, int end_id, int count) {
|
||||
try {
|
||||
PreparedStatement ps = conn.prepareStatement("INSERT INTO runs "+
|
||||
@ -132,7 +136,7 @@ public class Database {
|
||||
ps.execute();
|
||||
} catch (SQLException e) {}
|
||||
}
|
||||
|
||||
|
||||
public int queryInt(String query) {
|
||||
try {
|
||||
ResultSet rs = stmt.executeQuery(query);
|
||||
@ -142,16 +146,16 @@ public class Database {
|
||||
throw new RuntimeException("Could not get count of messages.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public int getMessageCount() { return queryInt("SELECT COUNT(*) FROM messages"); }
|
||||
public int getChatCount() { return queryInt("SELECT COUNT(*) FROM chats"); }
|
||||
public int getUserCount() { return queryInt("SELECT COUNT(*) FROM users"); }
|
||||
|
||||
|
||||
public LinkedList<Integer> getMissingIDs() {
|
||||
try {
|
||||
LinkedList<Integer> missing = new LinkedList<Integer>();
|
||||
int max = getTopMessageID();
|
||||
ResultSet rs = stmt.executeQuery("SELECT id FROM messages ORDER BY id");
|
||||
ResultSet rs = stmt.executeQuery("SELECT message_id FROM messages WHERE source_type IN ('group', 'dialog') ORDER BY id");
|
||||
rs.next();
|
||||
int id=rs.getInt(1);
|
||||
for (int i=1; i<=max; i++) {
|
||||
@ -172,17 +176,17 @@ public class Database {
|
||||
throw new RuntimeException("Could not get list of ids.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public synchronized void saveMessages(TLVector<TLAbsMessage> all, Integer api_layer) {
|
||||
try {
|
||||
//"(id, dialog_id, from_id, from_type, text, time, has_media, data, sticker, type) " +
|
||||
//"VALUES " +
|
||||
//"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
|
||||
String columns =
|
||||
"(id, message_type, dialog_id, chat_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer) "+
|
||||
"(message_id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer) "+
|
||||
"VALUES " +
|
||||
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
|
||||
//1 2 3 4 5 6 7 8 9 10 11 12 13 14
|
||||
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
|
||||
//1 2 3 4 5 6 7 8 9 10 11 12 13 14
|
||||
PreparedStatement ps = conn.prepareStatement("INSERT OR REPLACE INTO messages " + columns);
|
||||
PreparedStatement ps_insert_or_ignore = conn.prepareStatement("INSERT OR IGNORE INTO messages " + columns);
|
||||
|
||||
@ -193,20 +197,29 @@ public class Database {
|
||||
ps.setString(2, "message");
|
||||
TLAbsPeer peer = msg.getToId();
|
||||
if (peer instanceof TLPeerChat) {
|
||||
ps.setNull(3, Types.INTEGER);
|
||||
ps.setString(3, "group");
|
||||
ps.setInt(4, ((TLPeerChat)peer).getChatId());
|
||||
} else if (peer instanceof TLPeerUser) {
|
||||
int id = ((TLPeerUser)peer).getUserId();
|
||||
if (id==this.user_manager.getUser().getId()) {
|
||||
id = msg.getFromId();
|
||||
}
|
||||
ps.setInt(3, id);
|
||||
ps.setNull(4, Types.INTEGER);
|
||||
ps.setString(3, "dialog");
|
||||
ps.setInt(4, id);
|
||||
} else if (peer instanceof TLPeerChannel) {
|
||||
ps.setString(3, "channel");
|
||||
ps.setInt(4, ((TLPeerChannel)peer).getChannelId());
|
||||
} else {
|
||||
throw new RuntimeException("Unexpected Peer type: " + peer.getClass().getName());
|
||||
}
|
||||
ps.setInt(5, msg.getFromId());
|
||||
|
||||
|
||||
if (peer instanceof TLPeerChannel) {
|
||||
// Message in a channel don't have a sender -> insert a null
|
||||
ps.setNull(5, Types.INTEGER);
|
||||
} else {
|
||||
ps.setInt(5, msg.getFromId());
|
||||
}
|
||||
|
||||
if (msg.getFwdFrom() != null && msg.getFwdFrom().getFromId() != null) {
|
||||
ps.setInt(6, msg.getFwdFrom().getFromId());
|
||||
} else {
|
||||
@ -388,7 +401,7 @@ public class Database {
|
||||
throw new RuntimeException("Exception shown above happened.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public LinkedList<TLMessage> getMessagesWithMedia() {
|
||||
try {
|
||||
LinkedList<TLMessage> list = new LinkedList<TLMessage>();
|
||||
@ -403,7 +416,7 @@ public class Database {
|
||||
throw new RuntimeException("Exception occured. See above.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public int getMessagesFromUserCount() {
|
||||
try {
|
||||
ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM messages WHERE sender_id=" + user_manager.getUser().getId());
|
||||
@ -413,7 +426,7 @@ public class Database {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public LinkedList<Integer> getIdsFromQuery(String query) {
|
||||
try {
|
||||
LinkedList<Integer> list = new LinkedList<Integer>();
|
||||
@ -423,31 +436,31 @@ public class Database {
|
||||
return list;
|
||||
} catch (SQLException e) { throw new RuntimeException(e); }
|
||||
}
|
||||
|
||||
|
||||
public HashMap<String, Integer> getMessageTypesWithCount() {
|
||||
return getMessageTypesWithCount(new GlobalChat());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public HashMap<String, Integer> getMessageTypesWithCount(AbstractChat c) {
|
||||
HashMap<String, Integer> map = new HashMap<String, Integer>();
|
||||
try {
|
||||
ResultSet rs = stmt.executeQuery("SELECT message_type, COUNT(id) FROM messages WHERE " + c.getQuery() + " GROUP BY message_type");
|
||||
ResultSet rs = stmt.executeQuery("SELECT message_type, COUNT(message_id) FROM messages WHERE " + c.getQuery() + " GROUP BY message_type");
|
||||
while (rs.next()) {
|
||||
map.put("count.messages.type." + rs.getString(1), rs.getInt(2));
|
||||
}
|
||||
return map;
|
||||
} catch (Exception e) { throw new RuntimeException(e); }
|
||||
}
|
||||
|
||||
|
||||
public HashMap<String, Integer> getMessageMediaTypesWithCount() {
|
||||
return getMessageMediaTypesWithCount(new GlobalChat());
|
||||
}
|
||||
|
||||
|
||||
public HashMap<String, Integer> getMessageMediaTypesWithCount(AbstractChat c) {
|
||||
HashMap<String, Integer> map = new HashMap<String, Integer>();
|
||||
try {
|
||||
int count = 0;
|
||||
ResultSet rs = stmt.executeQuery("SELECT media_type, COUNT(id) FROM messages WHERE " + c.getQuery() + " GROUP BY media_type");
|
||||
ResultSet rs = stmt.executeQuery("SELECT media_type, COUNT(message_id) FROM messages WHERE " + c.getQuery() + " GROUP BY media_type");
|
||||
while (rs.next()) {
|
||||
String s = rs.getString(1);
|
||||
if (s==null) {
|
||||
@ -461,7 +474,7 @@ public class Database {
|
||||
return map;
|
||||
} catch (Exception e) { throw new RuntimeException(e); }
|
||||
}
|
||||
|
||||
|
||||
public HashMap<String, Integer> getMessageApiLayerWithCount() {
|
||||
HashMap<String, Integer> map = new HashMap<String, Integer>();
|
||||
try {
|
||||
@ -475,11 +488,11 @@ public class Database {
|
||||
return map;
|
||||
} catch (Exception e) { throw new RuntimeException(e); }
|
||||
}
|
||||
|
||||
|
||||
public HashMap<String, Object> getMessageAuthorsWithCount() {
|
||||
return getMessageAuthorsWithCount(new GlobalChat());
|
||||
}
|
||||
|
||||
|
||||
public HashMap<String, Object> getMessageAuthorsWithCount(AbstractChat c) {
|
||||
HashMap<String, Object> map = new HashMap<String, Object>();
|
||||
HashMap<User, Integer> user_map = new HashMap<User, Integer>();
|
||||
@ -501,7 +514,7 @@ public class Database {
|
||||
return map;
|
||||
} catch (Exception e) { throw new RuntimeException(e); }
|
||||
}
|
||||
|
||||
|
||||
public int[][] getMessageTimesMatrix() {
|
||||
return getMessageTimesMatrix(new GlobalChat());
|
||||
}
|
||||
@ -530,13 +543,13 @@ public class Database {
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public LinkedList<Chat> getListOfChatsForExport() {
|
||||
LinkedList<Chat> list = new LinkedList<Chat>();
|
||||
try {
|
||||
ResultSet rs = stmt.executeQuery("SELECT chats.id, chats.name, COUNT(messages.id) as c "+
|
||||
"FROM chats, messages WHERE messages.chat_id IS NOT NULL AND messages.chat_id=chats.id "+
|
||||
"FROM chats, messages WHERE messages.source_type IN('group', 'supergroup', 'channel') AND messages.source_id=chats.id "+
|
||||
"GROUP BY chats.id ORDER BY c DESC");
|
||||
while (rs.next()) {
|
||||
list.add(new Chat(rs.getInt(1), rs.getString(2), rs.getInt(3)));
|
||||
@ -548,14 +561,14 @@ public class Database {
|
||||
throw new RuntimeException("Exception above!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public LinkedList<Dialog> getListOfDialogsForExport() {
|
||||
LinkedList<Dialog> list = new LinkedList<Dialog>();
|
||||
try {
|
||||
ResultSet rs = stmt.executeQuery(
|
||||
"SELECT users.id, first_name, last_name, username, COUNT(messages.id) as c " +
|
||||
"FROM users, messages WHERE messages.dialog_id IS NOT NULL AND messages.dialog_id=users.id " +
|
||||
"FROM users, messages WHERE messages.source_type='dialog' AND messages.source_id=users.id " +
|
||||
"GROUP BY users.id ORDER BY c DESC");
|
||||
while (rs.next()) {
|
||||
list.add(new Dialog(rs.getInt(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getInt(5)));
|
||||
@ -567,23 +580,25 @@ public class Database {
|
||||
throw new RuntimeException("Exception above!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public LinkedList<HashMap<String, Object>> getMessagesForExport(AbstractChat c) {
|
||||
try {
|
||||
|
||||
ResultSet rs = stmt.executeQuery("SELECT messages.id as message_id, text, time*1000 as time, has_media, " +
|
||||
|
||||
ResultSet rs = stmt.executeQuery("SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " +
|
||||
"media_type, media_file, media_size, users.first_name as user_first_name, users.last_name as user_last_name, " +
|
||||
"users.username as user_username, users.id as user_id, " +
|
||||
"users_fwd.first_name as user_fwd_first_name, users_fwd.last_name as user_fwd_last_name, users_fwd.username as user_fwd_username " +
|
||||
"FROM messages, users LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " +
|
||||
"users.id=messages.sender_id AND " + c.getQuery() + " " +
|
||||
"ORDER BY messages.id");
|
||||
"FROM messages " +
|
||||
"LEFT JOIN users ON users.id=messages.sender_id " +
|
||||
"LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " +
|
||||
c.getQuery() + " " +
|
||||
"ORDER BY messages.message_id");
|
||||
SimpleDateFormat format_time = new SimpleDateFormat("HH:mm:ss");
|
||||
SimpleDateFormat format_date = new SimpleDateFormat("d MMM yy");
|
||||
ResultSetMetaData meta = rs.getMetaData();
|
||||
int columns = meta.getColumnCount();
|
||||
LinkedList<HashMap<String, Object>> list = new LinkedList<HashMap<String, Object>>();
|
||||
|
||||
|
||||
Integer count=0;
|
||||
String old_date = null;
|
||||
Integer old_user = null;
|
||||
@ -606,7 +621,7 @@ public class Database {
|
||||
h.put("same_user", old_user!=null && rs.getInt("user_id")==old_user);
|
||||
old_user = rs.getInt("user_id");
|
||||
old_date = date;
|
||||
|
||||
|
||||
list.add(h);
|
||||
count++;
|
||||
}
|
||||
@ -617,7 +632,7 @@ public class Database {
|
||||
throw new RuntimeException("Exception above!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static TLMessage bytesToTLMessage(byte[] b) {
|
||||
try {
|
||||
if (b==null) return null;
|
||||
@ -630,22 +645,22 @@ public class Database {
|
||||
throw new RuntimeException("Could not deserialize message.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public abstract class AbstractChat {
|
||||
public abstract String getQuery();
|
||||
}
|
||||
|
||||
|
||||
public class Dialog extends AbstractChat{
|
||||
public int id;
|
||||
public String first_name;
|
||||
public String last_name;
|
||||
public String username;
|
||||
public int count;
|
||||
|
||||
|
||||
public Dialog (int id, String first_name, String last_name, String username, int count) {
|
||||
this.id = id;
|
||||
this.first_name = first_name;
|
||||
@ -653,28 +668,28 @@ public class Database {
|
||||
this.username = username;
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public String getQuery() { return "dialog_id=" + id; }
|
||||
|
||||
public String getQuery() { return "source_type='dialog' AND source_id=" + id; }
|
||||
}
|
||||
|
||||
|
||||
public class Chat extends AbstractChat {
|
||||
public int id;
|
||||
public String name;
|
||||
public int count;
|
||||
|
||||
|
||||
public Chat(int id, String name, int count) {
|
||||
this.id = id;
|
||||
this.name = name;
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public String getQuery() {return "chat_id=" + id; }
|
||||
|
||||
public String getQuery() {return "source_type IN('group', 'supergroup', 'channel') AND source_id=" + id; }
|
||||
}
|
||||
|
||||
|
||||
public class User {
|
||||
public String name;
|
||||
public boolean isMe;
|
||||
|
||||
|
||||
public User(int id, String first_name, String last_name, String username) {
|
||||
isMe = id==user_manager.getUser().getId();
|
||||
StringBuilder s = new StringBuilder();
|
||||
@ -683,7 +698,7 @@ public class Database {
|
||||
name = s.toString().trim();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public class GlobalChat extends AbstractChat {
|
||||
public String getQuery() { return "1=1"; }
|
||||
}
|
||||
|
||||
@ -20,7 +20,7 @@ public class DatabaseUpdates {
|
||||
protected Database db;
|
||||
private static final Logger logger = LoggerFactory.getLogger(DatabaseUpdates.class);
|
||||
private static LinkedList<DatabaseUpdate> updates = new LinkedList<DatabaseUpdate>();
|
||||
|
||||
|
||||
public DatabaseUpdates(Connection conn, Database db) {
|
||||
this.conn = conn;
|
||||
this.db = db;
|
||||
@ -32,8 +32,9 @@ public class DatabaseUpdates {
|
||||
register(new DB_Update_5(conn, db));
|
||||
register(new DB_Update_6(conn, db));
|
||||
register(new DB_Update_7(conn, db));
|
||||
register(new DB_Update_8(conn, db));
|
||||
}
|
||||
|
||||
|
||||
public void doUpdates() {
|
||||
try {
|
||||
Statement stmt = conn.createStatement();
|
||||
@ -59,7 +60,7 @@ public class DatabaseUpdates {
|
||||
logger.debug("version: {}", version);
|
||||
System.out.println("Database version: " + version);
|
||||
logger.debug("Max available database version is {}", getMaxPossibleVersion());
|
||||
|
||||
|
||||
if (version < getMaxPossibleVersion()) {
|
||||
logger.debug("Update is necessary. {} => {}.", version, getMaxPossibleVersion());
|
||||
boolean backup = false;
|
||||
@ -77,7 +78,7 @@ public class DatabaseUpdates {
|
||||
logger.debug("NOT performing a backup, because we are creating a fresh database and don't need a backup of that.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
logger.debug("Applying updates");
|
||||
try {
|
||||
for (int i=version+1; i<=getMaxPossibleVersion(); i++) {
|
||||
@ -87,18 +88,18 @@ public class DatabaseUpdates {
|
||||
} else {
|
||||
logger.debug("No update necessary.");
|
||||
}
|
||||
|
||||
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private DatabaseUpdate getUpdateToVersion(int i) { return updates.get(i-1); }
|
||||
|
||||
|
||||
private int getMaxPossibleVersion() {
|
||||
return updates.size();
|
||||
}
|
||||
|
||||
|
||||
private void register(DatabaseUpdate d) {
|
||||
logger.debug("Registering {} as update to version {}", d.getClass().getName(), d.getVersion());
|
||||
if (d.getVersion() != updates.size()+1) {
|
||||
@ -119,7 +120,7 @@ abstract class DatabaseUpdate {
|
||||
stmt = conn.createStatement();
|
||||
} catch (SQLException e) { throw new RuntimeException(e); }
|
||||
this.db = db;
|
||||
|
||||
|
||||
}
|
||||
public void doUpdate() throws SQLException {
|
||||
logger.debug("Applying update to version {}", getVersion());
|
||||
@ -128,16 +129,20 @@ abstract class DatabaseUpdate {
|
||||
logger.debug("Saving current database version to the db");
|
||||
stmt.executeUpdate("INSERT INTO database_versions (version) VALUES (" + getVersion() + ")");
|
||||
}
|
||||
|
||||
|
||||
protected abstract void _doUpdate() throws SQLException;
|
||||
public abstract int getVersion();
|
||||
public boolean needsBackup() { return false; }
|
||||
protected void execute(String sql) throws SQLException {
|
||||
logger.debug("Executing: {}", sql);
|
||||
stmt.executeUpdate(sql);
|
||||
}
|
||||
}
|
||||
|
||||
class DB_Update_1 extends DatabaseUpdate {
|
||||
public int getVersion() { return 1; }
|
||||
public DB_Update_1(Connection conn, Database db) { super(conn, db); }
|
||||
|
||||
|
||||
protected void _doUpdate() throws SQLException {
|
||||
stmt.executeUpdate("CREATE TABLE messages ("
|
||||
+ "id INTEGER PRIMARY KEY ASC, "
|
||||
@ -162,14 +167,14 @@ class DB_Update_1 extends DatabaseUpdate {
|
||||
+ "username TEXT, "
|
||||
+ "type TEXT)");
|
||||
stmt.executeUpdate("CREATE TABLE database_versions ("
|
||||
+ "version INTEGER)");
|
||||
+ "version INTEGER)");
|
||||
}
|
||||
}
|
||||
|
||||
class DB_Update_2 extends DatabaseUpdate {
|
||||
public int getVersion() { return 2; }
|
||||
public DB_Update_2(Connection conn, Database db) { super(conn, db); }
|
||||
|
||||
|
||||
protected void _doUpdate() throws SQLException {
|
||||
stmt.executeUpdate("ALTER TABLE people RENAME TO 'users'");
|
||||
stmt.executeUpdate("ALTER TABLE users ADD COLUMN phone TEXT");
|
||||
@ -179,7 +184,7 @@ class DB_Update_2 extends DatabaseUpdate {
|
||||
class DB_Update_3 extends DatabaseUpdate {
|
||||
public int getVersion() { return 3; }
|
||||
public DB_Update_3(Connection conn, Database db) { super(conn, db); }
|
||||
|
||||
|
||||
protected void _doUpdate() throws SQLException {
|
||||
stmt.executeUpdate("ALTER TABLE dialogs RENAME TO 'chats'");
|
||||
}
|
||||
@ -188,7 +193,7 @@ class DB_Update_3 extends DatabaseUpdate {
|
||||
class DB_Update_4 extends DatabaseUpdate {
|
||||
public int getVersion() { return 4; }
|
||||
public DB_Update_4(Connection conn, Database db) { super(conn, db); }
|
||||
|
||||
|
||||
protected void _doUpdate() throws SQLException {
|
||||
stmt.executeUpdate("CREATE TABLE messages_new (id INTEGER PRIMARY KEY ASC, dialog_id INTEGER, to_id INTEGER, from_id INTEGER, from_type TEXT, text TEXT, time INTEGER, has_media BOOLEAN, sticker TEXT, data BLOB, type TEXT);");
|
||||
stmt.executeUpdate("INSERT INTO messages_new SELECT * FROM messages");
|
||||
@ -210,7 +215,7 @@ class DB_Update_6 extends DatabaseUpdate {
|
||||
public int getVersion() { return 6; }
|
||||
public DB_Update_6(Connection conn, Database db) { super(conn, db); }
|
||||
public boolean needsBackup() { return true; }
|
||||
|
||||
|
||||
protected void _doUpdate() throws SQLException {
|
||||
stmt.executeUpdate(
|
||||
"CREATE TABLE messages_new (\n" +
|
||||
@ -256,7 +261,7 @@ class DB_Update_6 extends DatabaseUpdate {
|
||||
}
|
||||
query.append("\nFROM messages");
|
||||
stmt.executeUpdate(query.toString());
|
||||
|
||||
|
||||
System.out.println(" Updating the data (this might take some time)...");
|
||||
ResultSet rs = stmt.executeQuery("SELECT id, data FROM messages_new");
|
||||
PreparedStatement ps = conn.prepareStatement("UPDATE messages_new SET fwd_from_id=?, media_type=?, media_file=?, media_size=? WHERE id=?");
|
||||
@ -294,10 +299,51 @@ class DB_Update_7 extends DatabaseUpdate {
|
||||
public int getVersion() { return 7; }
|
||||
public boolean needsBackup() { return true; }
|
||||
public DB_Update_7(Connection conn, Database db) { super(conn, db); }
|
||||
|
||||
|
||||
protected void _doUpdate() throws SQLException {
|
||||
stmt.executeUpdate("ALTER TABLE messages ADD COLUMN api_layer INTEGER");
|
||||
|
||||
|
||||
stmt.executeUpdate("UPDATE messages SET api_layer=51");
|
||||
}
|
||||
}
|
||||
|
||||
class DB_Update_8 extends DatabaseUpdate {
|
||||
public int getVersion() { return 8; }
|
||||
public DB_Update_8(Connection conn, Database db) { super(conn, db); }
|
||||
public boolean needsBackup() { return true; }
|
||||
|
||||
protected void _doUpdate() throws SQLException {
|
||||
execute("ALTER TABLE messages ADD COLUMN source_type TEXT");
|
||||
execute("ALTER TABLE messages ADD COLUMN source_id INTEGER");
|
||||
execute("update messages set source_type='dialog', source_id=dialog_id where dialog_id is not null");
|
||||
execute("update messages set source_type='group', source_id=chat_id where chat_id is not null");
|
||||
|
||||
execute("CREATE TABLE messages_new (" +
|
||||
"id INTEGER PRIMARY KEY AUTOINCREMENT," +
|
||||
"message_id INTEGER," +
|
||||
"message_type TEXT," +
|
||||
"source_type TEXT," +
|
||||
"source_id INTEGER," +
|
||||
"sender_id INTEGER," +
|
||||
"fwd_from_id INTEGER," +
|
||||
"text TEXT," +
|
||||
"time INTEGER," +
|
||||
"has_media BOOLEAN," +
|
||||
"media_type TEXT," +
|
||||
"media_file TEXT," +
|
||||
"media_size INTEGER," +
|
||||
"media_json TEXT," +
|
||||
"markup_json TEXT," +
|
||||
"data BLOB," +
|
||||
"api_layer INTEGER)");
|
||||
execute("INSERT INTO messages_new" +
|
||||
"(message_id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type," +
|
||||
"media_file, media_size, media_json, markup_json, data, api_layer)" +
|
||||
"SELECT " +
|
||||
"id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type," +
|
||||
"media_file, media_size, media_json, markup_json, data, api_layer FROM messages");
|
||||
execute("DROP TABLE messages");
|
||||
execute("ALTER TABLE messages_new RENAME TO 'messages'");
|
||||
execute("CREATE UNIQUE INDEX unique_messages ON messages (source_type, source_id, message_id)");
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
/* Telegram_Backup
|
||||
* Copyright (C) 2016 Fabian Schlenz
|
||||
*
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. */
|
||||
|
||||
@ -42,6 +42,7 @@ import java.io.FileOutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.HashMap;
|
||||
import java.util.Random;
|
||||
import java.net.URL;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
@ -59,14 +60,15 @@ public class DownloadManager {
|
||||
static TelegramClient download_client;
|
||||
static boolean last_download_succeeded = true;
|
||||
static final Logger logger = LoggerFactory.getLogger(DownloadManager.class);
|
||||
|
||||
static public enum DownloadType { NORMAL, CHANNELS_AND_SUPERGROUPS }
|
||||
|
||||
public DownloadManager(TelegramClient c, DownloadProgressInterface p) {
|
||||
this.user = UserManager.getInstance();
|
||||
this.client = c;
|
||||
this.prog = p;
|
||||
this.db = Database.getInstance();
|
||||
}
|
||||
|
||||
|
||||
public void downloadMessages(Integer limit) throws RpcErrorException, IOException {
|
||||
boolean completed = true;
|
||||
do {
|
||||
@ -90,7 +92,7 @@ public class DownloadManager {
|
||||
}
|
||||
} while (!completed);
|
||||
}
|
||||
|
||||
|
||||
public void _downloadMessages(Integer limit) throws RpcErrorException, IOException, TimeoutException {
|
||||
logger.info("This is _downloadMessages with limit {}", limit);
|
||||
int dialog_limit = 100;
|
||||
@ -103,6 +105,7 @@ public class DownloadManager {
|
||||
new TLInputPeerEmpty(),
|
||||
dialog_limit);
|
||||
logger.debug("Got {} dialogs", dialogs.getDialogs().size());
|
||||
|
||||
for (TLDialog d : dialogs.getDialogs()) {
|
||||
if (d.getTopMessage() > max_message_id && ! (d.getPeer() instanceof TLPeerChannel)) {
|
||||
logger.trace("Updating top message id: {} => {}. Dialog type: {}", max_message_id, d.getTopMessage(), d.getPeer().getClass().getName());
|
||||
@ -131,11 +134,11 @@ public class DownloadManager {
|
||||
} else {
|
||||
int start_id = max_database_id + 1;
|
||||
int end_id = max_message_id;
|
||||
|
||||
|
||||
List<Integer> ids = makeIdList(start_id, end_id);
|
||||
downloadMessages(ids);
|
||||
downloadMessages(ids, null);
|
||||
}
|
||||
|
||||
|
||||
logger.info("Searching for missing messages in the db");
|
||||
int count_missing = 0;
|
||||
System.out.println("Checking message database for completeness...");
|
||||
@ -143,8 +146,8 @@ public class DownloadManager {
|
||||
int db_max = db.getTopMessageID();
|
||||
logger.debug("db_count: {}", db_count);
|
||||
logger.debug("db_max: {}", db_max);
|
||||
|
||||
if (db_count != db_max) {
|
||||
|
||||
/*if (db_count != db_max) {
|
||||
if (limit != null) {
|
||||
System.out.println("You are missing messages in your database. But since you're using '--limit-messages', I won't download these now.");
|
||||
} else {
|
||||
@ -156,19 +159,57 @@ public class DownloadManager {
|
||||
count_missing = all_missing_ids.size();
|
||||
System.out.println("" + all_missing_ids.size() + " messages are missing in your Database.");
|
||||
System.out.println("I can (and will) download " + downloadable_missing_ids.size() + " of them.");
|
||||
|
||||
downloadMessages(downloadable_missing_ids);
|
||||
|
||||
downloadMessages(downloadable_missing_ids, null);
|
||||
}
|
||||
|
||||
logger.info("Logging this run");
|
||||
db.logRun(Math.min(max_database_id + 1, max_message_id), max_message_id, count_missing);
|
||||
}
|
||||
*/
|
||||
|
||||
if (CommandLineOptions.cmd_channels_and_supergroups) {
|
||||
logger.info("Processing channels and supergroups...");
|
||||
|
||||
HashMap<Integer, Long> channel_access_hashes = new HashMap<Integer, Long>();
|
||||
|
||||
// TODO Add chat title (and other stuff?) to the database
|
||||
for (TLAbsChat c : dialogs.getChats()) {
|
||||
if (c instanceof TLChannel) {
|
||||
TLChannel ch = (TLChannel)c;
|
||||
channel_access_hashes.put(c.getId(), ch.getAccessHash());
|
||||
// Channel: TLChannel
|
||||
// Supergroup: getMegagroup()==true
|
||||
System.out.println("" + c.getId() + " - " + (ch.getMegagroup() ? "Supergroup" : "Channel") +": " + ch.getTitle());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
for (TLDialog d : dialogs.getDialogs()) {
|
||||
if (d.getPeer() instanceof TLPeerChannel) {
|
||||
int channel_id = ((TLPeerChannel)d.getPeer()).getChannelId();
|
||||
int max_known_id = db.getTopMessageIDForChannel(channel_id);
|
||||
if (d.getTopMessage() > max_known_id) {
|
||||
List<Integer> ids = makeIdList(max_known_id+1, d.getTopMessage());
|
||||
//messagesPerChannel.put(id, makeIdList(max_known_id+1, d.getTopMessage()));
|
||||
Long access_hash = channel_access_hashes.get(channel_id);
|
||||
if (access_hash==null) {
|
||||
throw new RuntimeException("AccessHash for Channel missing.");
|
||||
}
|
||||
TLInputChannel channel = new TLInputChannel(channel_id, access_hash);
|
||||
downloadMessages(ids, channel);
|
||||
}
|
||||
System.out.println("" + channel_id + " - Known: " + max_known_id + " Availalble: " + d.getTopMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("Logging this run");
|
||||
db.logRun(Math.min(max_database_id + 1, max_message_id), max_message_id, count_missing);
|
||||
}
|
||||
|
||||
private void downloadMessages(List<Integer> ids) throws RpcErrorException, IOException {
|
||||
|
||||
private void downloadMessages(List<Integer> ids, TLInputChannel channel) throws RpcErrorException, IOException {
|
||||
prog.onMessageDownloadStart(ids.size());
|
||||
boolean has_seen_flood_wait_message = false;
|
||||
|
||||
|
||||
logger.debug("Entering download loop");
|
||||
while (ids.size()>0) {
|
||||
logger.trace("Loop");
|
||||
@ -181,7 +222,7 @@ public class DownloadManager {
|
||||
}
|
||||
logger.trace("vector.size(): {}", vector.size());
|
||||
logger.trace("ids.size(): {}", ids.size());
|
||||
|
||||
|
||||
TLAbsMessages response;
|
||||
int tries = 0;
|
||||
while(true) {
|
||||
@ -191,7 +232,11 @@ public class DownloadManager {
|
||||
}
|
||||
tries++;
|
||||
try {
|
||||
response = client.messagesGetMessages(vector);
|
||||
if (channel == null) {
|
||||
response = client.messagesGetMessages(vector);
|
||||
} else {
|
||||
response = client.channelsGetMessages(channel, vector);
|
||||
}
|
||||
break;
|
||||
} catch (RpcErrorException e) {
|
||||
if (e.getCode()==420) { // FLOOD_WAIT
|
||||
@ -206,6 +251,7 @@ public class DownloadManager {
|
||||
if (response.getMessages().size() != vector.size()) {
|
||||
CommandLineController.show_error("Requested " + vector.size() + " messages, but got " + response.getMessages().size() + ". That is unexpected. Quitting.");
|
||||
}
|
||||
|
||||
prog.onMessageDownloaded(response.getMessages().size());
|
||||
db.saveMessages(response.getMessages(), Kotlogram.API_LAYER);
|
||||
db.saveChats(response.getChats());
|
||||
@ -216,10 +262,10 @@ public class DownloadManager {
|
||||
} catch (InterruptedException e) {}
|
||||
}
|
||||
logger.debug("Finished.");
|
||||
|
||||
|
||||
prog.onMessageDownloadFinished();
|
||||
}
|
||||
|
||||
|
||||
public void downloadMedia() throws RpcErrorException, IOException {
|
||||
download_client = client.getDownloaderClient();
|
||||
boolean completed = true;
|
||||
@ -245,7 +291,7 @@ public class DownloadManager {
|
||||
}*/
|
||||
} while (!completed);
|
||||
}
|
||||
|
||||
|
||||
private void _downloadMedia() throws RpcErrorException, IOException {
|
||||
logger.info("This is _downloadMedia");
|
||||
logger.info("Checking if there are messages in the DB with a too old API layer");
|
||||
@ -253,9 +299,9 @@ public class DownloadManager {
|
||||
if (ids.size()>0) {
|
||||
System.out.println("You have " + ids.size() + " messages in your db that need an update. Doing that now.");
|
||||
logger.debug("Found {} messages", ids.size());
|
||||
downloadMessages(ids);
|
||||
downloadMessages(ids, null);
|
||||
}
|
||||
|
||||
|
||||
LinkedList<TLMessage> messages = this.db.getMessagesWithMedia();
|
||||
logger.debug("Database returned {} messages with media", messages.size());
|
||||
prog.onMediaDownloadStart(messages.size());
|
||||
@ -283,30 +329,30 @@ public class DownloadManager {
|
||||
}
|
||||
prog.onMediaDownloadFinished();
|
||||
}
|
||||
|
||||
|
||||
private List<Integer> makeIdList(int start, int end) {
|
||||
LinkedList<Integer> a = new LinkedList<Integer>();
|
||||
for (int i=start; i<=end; i++) a.add(i);
|
||||
return a;
|
||||
return a;
|
||||
}
|
||||
|
||||
|
||||
public static void downloadFile(TelegramClient client, String targetFilename, int size, int dcId, long volumeId, int localId, long secret) throws RpcErrorException, IOException, TimeoutException {
|
||||
TLInputFileLocation loc = new TLInputFileLocation(volumeId, localId, secret);
|
||||
downloadFileFromDc(client, targetFilename, loc, dcId, size);
|
||||
}
|
||||
|
||||
|
||||
public static void downloadFile(TelegramClient client, String targetFilename, int size, int dcId, long id, long accessHash) throws RpcErrorException, IOException, TimeoutException {
|
||||
TLInputDocumentFileLocation loc = new TLInputDocumentFileLocation(id, accessHash);
|
||||
downloadFileFromDc(client, targetFilename, loc, dcId, size);
|
||||
}
|
||||
|
||||
|
||||
private static boolean downloadFileFromDc(TelegramClient client, String target, TLAbsInputFileLocation loc, Integer dcID, int size) throws RpcErrorException, IOException, TimeoutException {
|
||||
FileOutputStream fos = null;
|
||||
try {
|
||||
String temp_filename = target + ".downloading";
|
||||
logger.debug("Downloading file {}", target);
|
||||
logger.trace("Temporary filename: {}", temp_filename);
|
||||
|
||||
|
||||
int offset = 0;
|
||||
if (new File(temp_filename).isFile()) {
|
||||
logger.info("Temporary filename already exists; continuing this file");
|
||||
@ -340,10 +386,10 @@ public class DownloadManager {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
offset += response.getBytes().getData().length;
|
||||
logger.trace("response: {} total size: {}", response.getBytes().getData().length, offset);
|
||||
|
||||
|
||||
fos.write(response.getBytes().getData());
|
||||
fos.flush();
|
||||
try { TimeUnit.MILLISECONDS.sleep(Config.DELAY_AFTER_GET_FILE); } catch(InterruptedException e) {}
|
||||
@ -394,7 +440,7 @@ public class DownloadManager {
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static boolean downloadExternalFile(String target, String url) throws IOException {
|
||||
FileUtils.copyURLToFile(new URL(url), new File(target), 5000, 5000);
|
||||
return true;
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
/* Telegram_Backup
|
||||
* Copyright (C) 2016 Fabian Schlenz
|
||||
*
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. */
|
||||
|
||||
@ -44,60 +44,60 @@ import org.slf4j.LoggerFactory;
|
||||
|
||||
public class HTMLExporter {
|
||||
private static Logger logger = LoggerFactory.getLogger(HTMLExporter.class);
|
||||
|
||||
|
||||
public void export() throws IOException {
|
||||
try {
|
||||
UserManager user = UserManager.getInstance();
|
||||
Database db = Database.getInstance();
|
||||
|
||||
|
||||
// Create base dir
|
||||
logger.debug("Creating base dir");
|
||||
String base = user.getFileBase() + "files" + File.separatorChar;
|
||||
new File(base).mkdirs();
|
||||
new File(base + "dialogs").mkdirs();
|
||||
|
||||
|
||||
logger.debug("Fetching dialogs");
|
||||
LinkedList<Database.Dialog> dialogs = db.getListOfDialogsForExport();
|
||||
logger.trace("Got {} dialogs", dialogs.size());
|
||||
logger.debug("Fetching chats");
|
||||
LinkedList<Database.Chat> chats = db.getListOfChatsForExport();
|
||||
logger.trace("Got {} chats", chats.size());
|
||||
|
||||
|
||||
logger.debug("Generating index.html");
|
||||
HashMap<String, Object> scope = new HashMap<String, Object>();
|
||||
scope.put("user", user);
|
||||
scope.put("dialogs", dialogs);
|
||||
scope.put("chats", chats);
|
||||
|
||||
|
||||
// Collect stats data
|
||||
scope.put("count.chats", chats.size());
|
||||
scope.put("count.dialogs", dialogs.size());
|
||||
|
||||
|
||||
int count_messages_chats = 0;
|
||||
int count_messages_dialogs = 0;
|
||||
for (Database.Chat c : chats) count_messages_chats += c.count;
|
||||
for (Database.Dialog d : dialogs) count_messages_dialogs += d.count;
|
||||
|
||||
|
||||
scope.put("count.messages", count_messages_chats + count_messages_dialogs);
|
||||
scope.put("count.messages.chats", count_messages_chats);
|
||||
scope.put("count.messages.dialogs", count_messages_dialogs);
|
||||
|
||||
|
||||
scope.put("count.messages.from_me", db.getMessagesFromUserCount());
|
||||
|
||||
scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix()));
|
||||
|
||||
|
||||
scope.putAll(db.getMessageAuthorsWithCount());
|
||||
scope.putAll(db.getMessageTypesWithCount());
|
||||
scope.putAll(db.getMessageMediaTypesWithCount());
|
||||
|
||||
|
||||
MustacheFactory mf = new DefaultMustacheFactory();
|
||||
Mustache mustache = mf.compile("templates/html/index.mustache");
|
||||
OutputStreamWriter w = getWriter(base + "index.html");
|
||||
mustache.execute(w, scope);
|
||||
w.close();
|
||||
|
||||
|
||||
mustache = mf.compile("templates/html/chat.mustache");
|
||||
|
||||
|
||||
int i=0;
|
||||
logger.debug("Generating {} dialog pages", dialogs.size());
|
||||
for (Database.Dialog d : dialogs) {
|
||||
@ -108,17 +108,17 @@ public class HTMLExporter {
|
||||
scope.put("user", user);
|
||||
scope.put("dialog", d);
|
||||
scope.put("messages", messages);
|
||||
|
||||
|
||||
scope.putAll(db.getMessageAuthorsWithCount(d));
|
||||
scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(d)));
|
||||
scope.putAll(db.getMessageTypesWithCount(d));
|
||||
scope.putAll(db.getMessageMediaTypesWithCount(d));
|
||||
|
||||
|
||||
w = getWriter(base + "dialogs" + File.separatorChar + "user_" + d.id + ".html");
|
||||
mustache.execute(w, scope);
|
||||
w.close();
|
||||
}
|
||||
|
||||
|
||||
i=0;
|
||||
logger.debug("Generating {} chat pages", chats.size());
|
||||
for (Database.Chat c : chats) {
|
||||
@ -129,17 +129,17 @@ public class HTMLExporter {
|
||||
scope.put("user", user);
|
||||
scope.put("chat", c);
|
||||
scope.put("messages", messages);
|
||||
|
||||
|
||||
scope.putAll(db.getMessageAuthorsWithCount(c));
|
||||
scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(c)));
|
||||
scope.putAll(db.getMessageTypesWithCount(c));
|
||||
scope.putAll(db.getMessageMediaTypesWithCount(c));
|
||||
|
||||
|
||||
w = getWriter(base + "dialogs" + File.separatorChar + "chat_" + c.id + ".html");
|
||||
mustache.execute(w, scope);
|
||||
w.close();
|
||||
}
|
||||
|
||||
|
||||
logger.debug("Generating additional files");
|
||||
// Copy CSS
|
||||
URL cssFile = getClass().getResource("/templates/html/style.css");
|
||||
@ -152,7 +152,7 @@ public class HTMLExporter {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private OutputStreamWriter getWriter(String filename) throws FileNotFoundException {
|
||||
logger.trace("Creating writer for file {}", Utils.anonymize(filename));
|
||||
return new OutputStreamWriter(new FileOutputStream(filename), Charset.forName("UTF-8").newEncoder());
|
||||
@ -170,7 +170,7 @@ public class HTMLExporter {
|
||||
sb.append("]");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
private String mapToString(Map<String, Integer> map) {
|
||||
StringBuilder sb = new StringBuilder("[");
|
||||
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
/* Telegram_Backup
|
||||
* Copyright (C) 2016 Fabian Schlenz
|
||||
*
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. */
|
||||
|
||||
@ -48,7 +48,7 @@ public abstract class AbstractMediaFileManager {
|
||||
protected TLMessage message;
|
||||
protected TelegramClient client;
|
||||
protected boolean isEmpty = false;
|
||||
|
||||
|
||||
public AbstractMediaFileManager(TLMessage msg, UserManager user, TelegramClient client) {this.user = user; this.message = msg; this.client = client;};
|
||||
public abstract int getSize();
|
||||
public abstract String getExtension();
|
||||
@ -64,22 +64,27 @@ public abstract class AbstractMediaFileManager {
|
||||
new File(path).mkdirs();
|
||||
return path;
|
||||
}
|
||||
public String getTargetFilename() { return "" + message.getId() + "." + getExtension(); }
|
||||
public String getTargetFilename() {
|
||||
if (message.getToId() instanceof TLPeerChannel) {
|
||||
return "channel_" + ((TLPeerChannel)message.getToId()).getChannelId() + "_" + message.getId() + "." + getExtension();
|
||||
}
|
||||
return "" + message.getId() + "." + getExtension();
|
||||
}
|
||||
public String getTargetPathAndFilename() { return getTargetPath() + getTargetFilename(); }
|
||||
|
||||
|
||||
protected String extensionFromMimetype(String mime) {
|
||||
switch(mime) {
|
||||
case "text/plain": return "txt";
|
||||
}
|
||||
|
||||
|
||||
int i = mime.lastIndexOf('/');
|
||||
String ext = mime.substring(i+1).toLowerCase();
|
||||
|
||||
|
||||
if (ext=="unknown") return "dat";
|
||||
|
||||
|
||||
return ext;
|
||||
}
|
||||
|
||||
|
||||
public abstract String getLetter();
|
||||
public abstract String getName();
|
||||
public abstract String getDescription();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user