mirror of
https://github.com/fabianonline/telegram_backup.git
synced 2024-11-22 16:56:16 +00:00
Implemented backing up of supergroups and channels added. Finally. Yaaaay. Be aware: There will possibly be a lot of bugs!
This commit is contained in:
parent
14d80a70fe
commit
7bf22f4692
@ -298,6 +298,8 @@ public class CommandLineController {
|
||||
System.out.println(" -d, --daemon Keep running and automatically save new messages.");
|
||||
System.out.println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.");
|
||||
System.out.println(" --stats Print some usage statistics.");
|
||||
System.out.println(" --with-channels-and-supergroups");
|
||||
System.out.println(" Backup channels and supergroups as well.");
|
||||
}
|
||||
|
||||
private void list_accounts() {
|
||||
|
||||
@ -30,6 +30,7 @@ class CommandLineOptions {
|
||||
public static boolean cmd_no_media = false;
|
||||
public static boolean cmd_anonymize = false;
|
||||
public static boolean cmd_stats = false;
|
||||
public static boolean cmd_channels_and_supergroups = false;
|
||||
|
||||
public static String val_account = null;
|
||||
public static Integer val_limit_messages = null;
|
||||
@ -117,6 +118,9 @@ class CommandLineOptions {
|
||||
case "--stats":
|
||||
cmd_stats = true; break;
|
||||
|
||||
case "--with-channels-and-supergroups":
|
||||
cmd_channels_and_supergroups = true; break;
|
||||
|
||||
default:
|
||||
throw new RuntimeException("Unknown command " + arg);
|
||||
}
|
||||
|
||||
@ -112,7 +112,7 @@ public class Database {
|
||||
|
||||
public int getTopMessageID() {
|
||||
try {
|
||||
ResultSet rs = stmt.executeQuery("SELECT MAX(id) FROM messages");
|
||||
ResultSet rs = stmt.executeQuery("SELECT MAX(message_id) FROM messages WHERE source_type IN ('group', 'dialog')");
|
||||
rs.next();
|
||||
return rs.getInt(1);
|
||||
} catch (SQLException e) {
|
||||
@ -120,6 +120,10 @@ public class Database {
|
||||
}
|
||||
}
|
||||
|
||||
public int getTopMessageIDForChannel(int id) {
|
||||
return queryInt("SELECT MAX(message_id) FROM messages WHERE source_id=" + id + " AND source_type IN('channel', 'supergroup')");
|
||||
}
|
||||
|
||||
public void logRun(int start_id, int end_id, int count) {
|
||||
try {
|
||||
PreparedStatement ps = conn.prepareStatement("INSERT INTO runs "+
|
||||
@ -151,7 +155,7 @@ public class Database {
|
||||
try {
|
||||
LinkedList<Integer> missing = new LinkedList<Integer>();
|
||||
int max = getTopMessageID();
|
||||
ResultSet rs = stmt.executeQuery("SELECT id FROM messages ORDER BY id");
|
||||
ResultSet rs = stmt.executeQuery("SELECT message_id FROM messages WHERE source_type IN ('group', 'dialog') ORDER BY id");
|
||||
rs.next();
|
||||
int id=rs.getInt(1);
|
||||
for (int i=1; i<=max; i++) {
|
||||
@ -179,7 +183,7 @@ public class Database {
|
||||
//"VALUES " +
|
||||
//"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
|
||||
String columns =
|
||||
"(id, message_type, dialog_id, chat_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer) "+
|
||||
"(message_id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer) "+
|
||||
"VALUES " +
|
||||
"(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
|
||||
//1 2 3 4 5 6 7 8 9 10 11 12 13 14
|
||||
@ -193,19 +197,28 @@ public class Database {
|
||||
ps.setString(2, "message");
|
||||
TLAbsPeer peer = msg.getToId();
|
||||
if (peer instanceof TLPeerChat) {
|
||||
ps.setNull(3, Types.INTEGER);
|
||||
ps.setString(3, "group");
|
||||
ps.setInt(4, ((TLPeerChat)peer).getChatId());
|
||||
} else if (peer instanceof TLPeerUser) {
|
||||
int id = ((TLPeerUser)peer).getUserId();
|
||||
if (id==this.user_manager.getUser().getId()) {
|
||||
id = msg.getFromId();
|
||||
}
|
||||
ps.setInt(3, id);
|
||||
ps.setNull(4, Types.INTEGER);
|
||||
ps.setString(3, "dialog");
|
||||
ps.setInt(4, id);
|
||||
} else if (peer instanceof TLPeerChannel) {
|
||||
ps.setString(3, "channel");
|
||||
ps.setInt(4, ((TLPeerChannel)peer).getChannelId());
|
||||
} else {
|
||||
throw new RuntimeException("Unexpected Peer type: " + peer.getClass().getName());
|
||||
}
|
||||
|
||||
if (peer instanceof TLPeerChannel) {
|
||||
// Message in a channel don't have a sender -> insert a null
|
||||
ps.setNull(5, Types.INTEGER);
|
||||
} else {
|
||||
ps.setInt(5, msg.getFromId());
|
||||
}
|
||||
|
||||
if (msg.getFwdFrom() != null && msg.getFwdFrom().getFromId() != null) {
|
||||
ps.setInt(6, msg.getFwdFrom().getFromId());
|
||||
@ -431,7 +444,7 @@ public class Database {
|
||||
public HashMap<String, Integer> getMessageTypesWithCount(AbstractChat c) {
|
||||
HashMap<String, Integer> map = new HashMap<String, Integer>();
|
||||
try {
|
||||
ResultSet rs = stmt.executeQuery("SELECT message_type, COUNT(id) FROM messages WHERE " + c.getQuery() + " GROUP BY message_type");
|
||||
ResultSet rs = stmt.executeQuery("SELECT message_type, COUNT(message_id) FROM messages WHERE " + c.getQuery() + " GROUP BY message_type");
|
||||
while (rs.next()) {
|
||||
map.put("count.messages.type." + rs.getString(1), rs.getInt(2));
|
||||
}
|
||||
@ -447,7 +460,7 @@ public class Database {
|
||||
HashMap<String, Integer> map = new HashMap<String, Integer>();
|
||||
try {
|
||||
int count = 0;
|
||||
ResultSet rs = stmt.executeQuery("SELECT media_type, COUNT(id) FROM messages WHERE " + c.getQuery() + " GROUP BY media_type");
|
||||
ResultSet rs = stmt.executeQuery("SELECT media_type, COUNT(message_id) FROM messages WHERE " + c.getQuery() + " GROUP BY media_type");
|
||||
while (rs.next()) {
|
||||
String s = rs.getString(1);
|
||||
if (s==null) {
|
||||
@ -536,7 +549,7 @@ public class Database {
|
||||
LinkedList<Chat> list = new LinkedList<Chat>();
|
||||
try {
|
||||
ResultSet rs = stmt.executeQuery("SELECT chats.id, chats.name, COUNT(messages.id) as c "+
|
||||
"FROM chats, messages WHERE messages.chat_id IS NOT NULL AND messages.chat_id=chats.id "+
|
||||
"FROM chats, messages WHERE messages.source_type IN('group', 'supergroup', 'channel') AND messages.source_id=chats.id "+
|
||||
"GROUP BY chats.id ORDER BY c DESC");
|
||||
while (rs.next()) {
|
||||
list.add(new Chat(rs.getInt(1), rs.getString(2), rs.getInt(3)));
|
||||
@ -555,7 +568,7 @@ public class Database {
|
||||
try {
|
||||
ResultSet rs = stmt.executeQuery(
|
||||
"SELECT users.id, first_name, last_name, username, COUNT(messages.id) as c " +
|
||||
"FROM users, messages WHERE messages.dialog_id IS NOT NULL AND messages.dialog_id=users.id " +
|
||||
"FROM users, messages WHERE messages.source_type='dialog' AND messages.source_id=users.id " +
|
||||
"GROUP BY users.id ORDER BY c DESC");
|
||||
while (rs.next()) {
|
||||
list.add(new Dialog(rs.getInt(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getInt(5)));
|
||||
@ -571,13 +584,15 @@ public class Database {
|
||||
public LinkedList<HashMap<String, Object>> getMessagesForExport(AbstractChat c) {
|
||||
try {
|
||||
|
||||
ResultSet rs = stmt.executeQuery("SELECT messages.id as message_id, text, time*1000 as time, has_media, " +
|
||||
ResultSet rs = stmt.executeQuery("SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " +
|
||||
"media_type, media_file, media_size, users.first_name as user_first_name, users.last_name as user_last_name, " +
|
||||
"users.username as user_username, users.id as user_id, " +
|
||||
"users_fwd.first_name as user_fwd_first_name, users_fwd.last_name as user_fwd_last_name, users_fwd.username as user_fwd_username " +
|
||||
"FROM messages, users LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " +
|
||||
"users.id=messages.sender_id AND " + c.getQuery() + " " +
|
||||
"ORDER BY messages.id");
|
||||
"FROM messages " +
|
||||
"LEFT JOIN users ON users.id=messages.sender_id " +
|
||||
"LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " +
|
||||
c.getQuery() + " " +
|
||||
"ORDER BY messages.message_id");
|
||||
SimpleDateFormat format_time = new SimpleDateFormat("HH:mm:ss");
|
||||
SimpleDateFormat format_date = new SimpleDateFormat("d MMM yy");
|
||||
ResultSetMetaData meta = rs.getMetaData();
|
||||
@ -654,7 +669,7 @@ public class Database {
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public String getQuery() { return "dialog_id=" + id; }
|
||||
public String getQuery() { return "source_type='dialog' AND source_id=" + id; }
|
||||
}
|
||||
|
||||
public class Chat extends AbstractChat {
|
||||
@ -668,7 +683,7 @@ public class Database {
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public String getQuery() {return "chat_id=" + id; }
|
||||
public String getQuery() {return "source_type IN('group', 'supergroup', 'channel') AND source_id=" + id; }
|
||||
}
|
||||
|
||||
public class User {
|
||||
|
||||
@ -32,6 +32,7 @@ public class DatabaseUpdates {
|
||||
register(new DB_Update_5(conn, db));
|
||||
register(new DB_Update_6(conn, db));
|
||||
register(new DB_Update_7(conn, db));
|
||||
register(new DB_Update_8(conn, db));
|
||||
}
|
||||
|
||||
public void doUpdates() {
|
||||
@ -132,6 +133,10 @@ abstract class DatabaseUpdate {
|
||||
protected abstract void _doUpdate() throws SQLException;
|
||||
public abstract int getVersion();
|
||||
public boolean needsBackup() { return false; }
|
||||
protected void execute(String sql) throws SQLException {
|
||||
logger.debug("Executing: {}", sql);
|
||||
stmt.executeUpdate(sql);
|
||||
}
|
||||
}
|
||||
|
||||
class DB_Update_1 extends DatabaseUpdate {
|
||||
@ -301,3 +306,44 @@ class DB_Update_7 extends DatabaseUpdate {
|
||||
stmt.executeUpdate("UPDATE messages SET api_layer=51");
|
||||
}
|
||||
}
|
||||
|
||||
class DB_Update_8 extends DatabaseUpdate {
|
||||
public int getVersion() { return 8; }
|
||||
public DB_Update_8(Connection conn, Database db) { super(conn, db); }
|
||||
public boolean needsBackup() { return true; }
|
||||
|
||||
protected void _doUpdate() throws SQLException {
|
||||
execute("ALTER TABLE messages ADD COLUMN source_type TEXT");
|
||||
execute("ALTER TABLE messages ADD COLUMN source_id INTEGER");
|
||||
execute("update messages set source_type='dialog', source_id=dialog_id where dialog_id is not null");
|
||||
execute("update messages set source_type='group', source_id=chat_id where chat_id is not null");
|
||||
|
||||
execute("CREATE TABLE messages_new (" +
|
||||
"id INTEGER PRIMARY KEY AUTOINCREMENT," +
|
||||
"message_id INTEGER," +
|
||||
"message_type TEXT," +
|
||||
"source_type TEXT," +
|
||||
"source_id INTEGER," +
|
||||
"sender_id INTEGER," +
|
||||
"fwd_from_id INTEGER," +
|
||||
"text TEXT," +
|
||||
"time INTEGER," +
|
||||
"has_media BOOLEAN," +
|
||||
"media_type TEXT," +
|
||||
"media_file TEXT," +
|
||||
"media_size INTEGER," +
|
||||
"media_json TEXT," +
|
||||
"markup_json TEXT," +
|
||||
"data BLOB," +
|
||||
"api_layer INTEGER)");
|
||||
execute("INSERT INTO messages_new" +
|
||||
"(message_id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type," +
|
||||
"media_file, media_size, media_json, markup_json, data, api_layer)" +
|
||||
"SELECT " +
|
||||
"id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type," +
|
||||
"media_file, media_size, media_json, markup_json, data, api_layer FROM messages");
|
||||
execute("DROP TABLE messages");
|
||||
execute("ALTER TABLE messages_new RENAME TO 'messages'");
|
||||
execute("CREATE UNIQUE INDEX unique_messages ON messages (source_type, source_id, message_id)");
|
||||
}
|
||||
}
|
||||
|
||||
@ -42,6 +42,7 @@ import java.io.FileOutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.HashMap;
|
||||
import java.util.Random;
|
||||
import java.net.URL;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
@ -59,6 +60,7 @@ public class DownloadManager {
|
||||
static TelegramClient download_client;
|
||||
static boolean last_download_succeeded = true;
|
||||
static final Logger logger = LoggerFactory.getLogger(DownloadManager.class);
|
||||
static public enum DownloadType { NORMAL, CHANNELS_AND_SUPERGROUPS }
|
||||
|
||||
public DownloadManager(TelegramClient c, DownloadProgressInterface p) {
|
||||
this.user = UserManager.getInstance();
|
||||
@ -103,6 +105,7 @@ public class DownloadManager {
|
||||
new TLInputPeerEmpty(),
|
||||
dialog_limit);
|
||||
logger.debug("Got {} dialogs", dialogs.getDialogs().size());
|
||||
|
||||
for (TLDialog d : dialogs.getDialogs()) {
|
||||
if (d.getTopMessage() > max_message_id && ! (d.getPeer() instanceof TLPeerChannel)) {
|
||||
logger.trace("Updating top message id: {} => {}. Dialog type: {}", max_message_id, d.getTopMessage(), d.getPeer().getClass().getName());
|
||||
@ -133,7 +136,7 @@ public class DownloadManager {
|
||||
int end_id = max_message_id;
|
||||
|
||||
List<Integer> ids = makeIdList(start_id, end_id);
|
||||
downloadMessages(ids);
|
||||
downloadMessages(ids, null);
|
||||
}
|
||||
|
||||
logger.info("Searching for missing messages in the db");
|
||||
@ -144,7 +147,7 @@ public class DownloadManager {
|
||||
logger.debug("db_count: {}", db_count);
|
||||
logger.debug("db_max: {}", db_max);
|
||||
|
||||
if (db_count != db_max) {
|
||||
/*if (db_count != db_max) {
|
||||
if (limit != null) {
|
||||
System.out.println("You are missing messages in your database. But since you're using '--limit-messages', I won't download these now.");
|
||||
} else {
|
||||
@ -157,15 +160,53 @@ public class DownloadManager {
|
||||
System.out.println("" + all_missing_ids.size() + " messages are missing in your Database.");
|
||||
System.out.println("I can (and will) download " + downloadable_missing_ids.size() + " of them.");
|
||||
|
||||
downloadMessages(downloadable_missing_ids);
|
||||
}
|
||||
downloadMessages(downloadable_missing_ids, null);
|
||||
}
|
||||
|
||||
logger.info("Logging this run");
|
||||
db.logRun(Math.min(max_database_id + 1, max_message_id), max_message_id, count_missing);
|
||||
}
|
||||
*/
|
||||
|
||||
private void downloadMessages(List<Integer> ids) throws RpcErrorException, IOException {
|
||||
if (CommandLineOptions.cmd_channels_and_supergroups) {
|
||||
logger.info("Processing channels and supergroups...");
|
||||
|
||||
HashMap<Integer, Long> channel_access_hashes = new HashMap<Integer, Long>();
|
||||
|
||||
// TODO Add chat title (and other stuff?) to the database
|
||||
for (TLAbsChat c : dialogs.getChats()) {
|
||||
if (c instanceof TLChannel) {
|
||||
TLChannel ch = (TLChannel)c;
|
||||
channel_access_hashes.put(c.getId(), ch.getAccessHash());
|
||||
// Channel: TLChannel
|
||||
// Supergroup: getMegagroup()==true
|
||||
System.out.println("" + c.getId() + " - " + (ch.getMegagroup() ? "Supergroup" : "Channel") +": " + ch.getTitle());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
for (TLDialog d : dialogs.getDialogs()) {
|
||||
if (d.getPeer() instanceof TLPeerChannel) {
|
||||
int channel_id = ((TLPeerChannel)d.getPeer()).getChannelId();
|
||||
int max_known_id = db.getTopMessageIDForChannel(channel_id);
|
||||
if (d.getTopMessage() > max_known_id) {
|
||||
List<Integer> ids = makeIdList(max_known_id+1, d.getTopMessage());
|
||||
//messagesPerChannel.put(id, makeIdList(max_known_id+1, d.getTopMessage()));
|
||||
Long access_hash = channel_access_hashes.get(channel_id);
|
||||
if (access_hash==null) {
|
||||
throw new RuntimeException("AccessHash for Channel missing.");
|
||||
}
|
||||
TLInputChannel channel = new TLInputChannel(channel_id, access_hash);
|
||||
downloadMessages(ids, channel);
|
||||
}
|
||||
System.out.println("" + channel_id + " - Known: " + max_known_id + " Availalble: " + d.getTopMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void downloadMessages(List<Integer> ids, TLInputChannel channel) throws RpcErrorException, IOException {
|
||||
prog.onMessageDownloadStart(ids.size());
|
||||
boolean has_seen_flood_wait_message = false;
|
||||
|
||||
@ -191,7 +232,11 @@ public class DownloadManager {
|
||||
}
|
||||
tries++;
|
||||
try {
|
||||
if (channel == null) {
|
||||
response = client.messagesGetMessages(vector);
|
||||
} else {
|
||||
response = client.channelsGetMessages(channel, vector);
|
||||
}
|
||||
break;
|
||||
} catch (RpcErrorException e) {
|
||||
if (e.getCode()==420) { // FLOOD_WAIT
|
||||
@ -206,6 +251,7 @@ public class DownloadManager {
|
||||
if (response.getMessages().size() != vector.size()) {
|
||||
CommandLineController.show_error("Requested " + vector.size() + " messages, but got " + response.getMessages().size() + ". That is unexpected. Quitting.");
|
||||
}
|
||||
|
||||
prog.onMessageDownloaded(response.getMessages().size());
|
||||
db.saveMessages(response.getMessages(), Kotlogram.API_LAYER);
|
||||
db.saveChats(response.getChats());
|
||||
@ -253,7 +299,7 @@ public class DownloadManager {
|
||||
if (ids.size()>0) {
|
||||
System.out.println("You have " + ids.size() + " messages in your db that need an update. Doing that now.");
|
||||
logger.debug("Found {} messages", ids.size());
|
||||
downloadMessages(ids);
|
||||
downloadMessages(ids, null);
|
||||
}
|
||||
|
||||
LinkedList<TLMessage> messages = this.db.getMessagesWithMedia();
|
||||
|
||||
@ -64,7 +64,12 @@ public abstract class AbstractMediaFileManager {
|
||||
new File(path).mkdirs();
|
||||
return path;
|
||||
}
|
||||
public String getTargetFilename() { return "" + message.getId() + "." + getExtension(); }
|
||||
public String getTargetFilename() {
|
||||
if (message.getToId() instanceof TLPeerChannel) {
|
||||
return "channel_" + ((TLPeerChannel)message.getToId()).getChannelId() + "_" + message.getId() + "." + getExtension();
|
||||
}
|
||||
return "" + message.getId() + "." + getExtension();
|
||||
}
|
||||
public String getTargetPathAndFilename() { return getTargetPath() + getTargetFilename(); }
|
||||
|
||||
protected String extensionFromMimetype(String mime) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user