Merge branch 'feature-channels-and-supergroups'. Closes #21.

src/main/java/de/fabianonline/telegram_backup/CommandLineController.java

@@ -298,6 +298,8 @@ public class CommandLineController {
 		System.out.println("  -d, --daemon                 Keep running and automatically save new messages.");
 		System.out.println("      --anonymize              (Try to) Remove all sensitive information from output. Useful for requesting support.");
 		System.out.println("      --stats                  Print some usage statistics.");
+		System.out.println("      --with-channels          Backup channels as well.");
+		System.out.println("      --with-supergroups       Backup supergroups as well.");
 	}
 
 	private void list_accounts() {

src/main/java/de/fabianonline/telegram_backup/CommandLineDownloadProgress.java

@@ -23,7 +23,14 @@ class CommandLineDownloadProgress implements DownloadProgressInterface {
 	private int mediaCount = 0;
 	private int i = 0;
 
-	public void onMessageDownloadStart(int count) { i=0; System.out.println("Downloading " + count + " messages."); }
+	public void onMessageDownloadStart(int count, String source) {
+		i=0;
+		if (source==null) {
+			System.out.println("Downloading " + count + " messages.");
+		} else {
+			System.out.println("Downloading " + count + " messages from " + Utils.anonymize(source));
+		}
+	}
 	public void onMessageDownloaded(int number) { i+=number; System.out.print("..." + i); }
 	public void onMessageDownloadFinished() { System.out.println(" done."); }
 
@@ -55,4 +62,3 @@ class CommandLineDownloadProgress implements DownloadProgressInterface {
 	private void show(String letter) { System.out.print(letter); i++; if (i % 100 == 0) showNewLine();}
 	private void showNewLine() { System.out.println(" - " + i + "/" + mediaCount); }
 }
-	

src/main/java/de/fabianonline/telegram_backup/CommandLineOptions.java

@@ -30,6 +30,8 @@ class CommandLineOptions {
 	public static boolean cmd_no_media       = false;
 	public static boolean cmd_anonymize      = false;
 	public static boolean cmd_stats          = false;
+	public static boolean cmd_channels       = false;
+	public static boolean cmd_supergroups    = false;
 
 	public static String  val_account        = null;
 	public static Integer val_limit_messages = null;
@@ -117,6 +119,12 @@ class CommandLineOptions {
 				case "--stats":
 					cmd_stats = true;               break;
 
+				case "--with-channels":
+					cmd_channels = true;            break;
+
+				case "--with-supergroups":
+					cmd_supergroups = true;         break;
+
 				default:
 					throw new RuntimeException("Unknown command " + arg);
 			}

src/main/java/de/fabianonline/telegram_backup/Database.java

@@ -112,7 +112,7 @@ public class Database {
 
 	public int getTopMessageID() {
 		try {
-			ResultSet rs = stmt.executeQuery("SELECT MAX(id) FROM messages");
+			ResultSet rs = stmt.executeQuery("SELECT MAX(message_id) FROM messages WHERE source_type IN ('group', 'dialog')");
 			rs.next();
 			return rs.getInt(1);
 		} catch (SQLException e) {
@@ -120,6 +120,10 @@ public class Database {
 		}
 	}
 
+	public int getTopMessageIDForChannel(int id) {
+		return queryInt("SELECT MAX(message_id) FROM messages WHERE source_id=" + id + " AND source_type IN('channel', 'supergroup')");
+	}
+
 	public void logRun(int start_id, int end_id, int count) {
 		try {
 			PreparedStatement ps = conn.prepareStatement("INSERT INTO runs "+
@@ -151,7 +155,7 @@ public class Database {
 		try {
 			LinkedList<Integer> missing = new LinkedList<Integer>();
 			int max = getTopMessageID();
-			ResultSet rs = stmt.executeQuery("SELECT id FROM messages ORDER BY id");
+			ResultSet rs = stmt.executeQuery("SELECT message_id FROM messages WHERE source_type IN ('group', 'dialog') ORDER BY id");
 			rs.next();
 			int id=rs.getInt(1);
 			for (int i=1; i<=max; i++) {
@@ -179,7 +183,7 @@ public class Database {
 				//"VALUES " +
 				//"(?,  ?,         ?,       ?,         ?,    ?,    ?,         ?,    ?,       ?)");
 			String columns =
-				"(id, message_type, dialog_id, chat_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer) "+
+				"(message_id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type, media_file, media_size, data, api_layer) "+
 				"VALUES " +
 				"(?,          ?,            ?,           ?,         ?,         ?,           ?,    ?,    ?,         ?,          ?,          ?,          ?,    ?)";
 				//1           2             3            4          5          6            7     8     9          10          11          12          13    14
@@ -193,19 +197,28 @@ public class Database {
 					ps.setString(2, "message");
 					TLAbsPeer peer = msg.getToId();
 					if (peer instanceof TLPeerChat) {
-						ps.setNull(3, Types.INTEGER);
+						ps.setString(3, "group");
 						ps.setInt(4, ((TLPeerChat)peer).getChatId());
 					} else if (peer instanceof TLPeerUser) {
 						int id = ((TLPeerUser)peer).getUserId();
 						if (id==this.user_manager.getUser().getId()) {
 							id = msg.getFromId();
 						}
-						ps.setInt(3, id);
+						ps.setString(3, "dialog");
-						ps.setNull(4, Types.INTEGER);
+						ps.setInt(4, id);
+					} else if (peer instanceof TLPeerChannel) {
+						ps.setString(3, "channel");
+						ps.setInt(4, ((TLPeerChannel)peer).getChannelId());
 					} else {
 						throw new RuntimeException("Unexpected Peer type: " + peer.getClass().getName());
 					}
+
+					if (peer instanceof TLPeerChannel) {
+						// Message in a channel don't have a sender -> insert a null
+						ps.setNull(5, Types.INTEGER);
+					} else {
 						ps.setInt(5, msg.getFromId());
+					}
 
 					if (msg.getFwdFrom() != null && msg.getFwdFrom().getFromId() != null) {
 						ps.setInt(6, msg.getFwdFrom().getFromId());
@@ -431,7 +444,7 @@ public class Database {
 	public HashMap<String, Integer> getMessageTypesWithCount(AbstractChat c) {
 		HashMap<String, Integer> map = new HashMap<String, Integer>();
 		try {
-			ResultSet rs = stmt.executeQuery("SELECT message_type, COUNT(id) FROM messages WHERE " + c.getQuery() + " GROUP BY message_type");
+			ResultSet rs = stmt.executeQuery("SELECT message_type, COUNT(message_id) FROM messages WHERE " + c.getQuery() + " GROUP BY message_type");
 			while (rs.next()) {
 				map.put("count.messages.type." + rs.getString(1), rs.getInt(2));
 			}
@@ -447,7 +460,7 @@ public class Database {
 		HashMap<String, Integer> map = new HashMap<String, Integer>();
 		try {
 			int count = 0;
-			ResultSet rs = stmt.executeQuery("SELECT media_type, COUNT(id) FROM messages WHERE " + c.getQuery() + " GROUP BY media_type");
+			ResultSet rs = stmt.executeQuery("SELECT media_type, COUNT(message_id) FROM messages WHERE " + c.getQuery() + " GROUP BY media_type");
 			while (rs.next()) {
 				String s = rs.getString(1);
 				if (s==null) {
@@ -484,11 +497,21 @@ public class Database {
 		HashMap<String, Object> map = new HashMap<String, Object>();
 		HashMap<User, Integer> user_map = new HashMap<User, Integer>();
 		int count_others = 0;
+		// Set a default value for 'me' to fix the charts for channels - cause I
+		// possibly didn't send any messages there.
+		map.put("authors.count.me", 0);
 		try {
 			ResultSet rs = stmt.executeQuery("SELECT users.id, users.first_name, users.last_name, users.username, COUNT(messages.id) "+
-				"FROM messages, users WHERE users.id=messages.sender_id AND " + c.getQuery() + " GROUP BY sender_id");
+				"FROM messages " +
+				"LEFT JOIN users ON users.id=messages.sender_id " +
+				"WHERE " + c.getQuery() + " GROUP BY sender_id");
 			while (rs.next()) {
-				User u = new User(rs.getInt(1), rs.getString(2), rs.getString(3), rs.getString(4));
+				User u;
+				if (rs.getString(2)!=null || rs.getString(3)!=null || rs.getString(4)!=null) {
+					u = new User(rs.getInt(1), rs.getString(2), rs.getString(3), rs.getString(4));
+				} else {
+					u = new User(rs.getInt(1), "Unknown", "", "");
+				}
 				if (u.isMe) {
 					map.put("authors.count.me", rs.getInt(5));
 				} else {
@@ -536,7 +559,7 @@ public class Database {
 		LinkedList<Chat> list = new LinkedList<Chat>();
 		try {
 			ResultSet rs = stmt.executeQuery("SELECT chats.id, chats.name, COUNT(messages.id) as c "+
-				"FROM chats, messages WHERE messages.chat_id IS NOT NULL AND messages.chat_id=chats.id "+
+				"FROM chats, messages WHERE messages.source_type IN('group', 'supergroup', 'channel') AND messages.source_id=chats.id "+
 				"GROUP BY chats.id ORDER BY c DESC");
 			while (rs.next()) {
 				list.add(new Chat(rs.getInt(1), rs.getString(2), rs.getInt(3)));
@@ -555,7 +578,7 @@ public class Database {
 		try {
 			ResultSet rs = stmt.executeQuery(
 				"SELECT users.id, first_name, last_name, username, COUNT(messages.id) as c " +
-				"FROM users, messages WHERE messages.dialog_id IS NOT NULL AND messages.dialog_id=users.id " +
+				"FROM users, messages WHERE messages.source_type='dialog' AND messages.source_id=users.id " +
 				"GROUP BY users.id ORDER BY c DESC");
 			while (rs.next()) {
 				list.add(new Dialog(rs.getInt(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getInt(5)));
@@ -571,13 +594,15 @@ public class Database {
 	public LinkedList<HashMap<String, Object>> getMessagesForExport(AbstractChat c) {
 		try {
 
-			ResultSet rs = stmt.executeQuery("SELECT messages.id as message_id, text, time*1000 as time, has_media, " +
+			ResultSet rs = stmt.executeQuery("SELECT messages.message_id as message_id, text, time*1000 as time, has_media, " +
 				"media_type, media_file, media_size, users.first_name as user_first_name, users.last_name as user_last_name, " +
 				"users.username as user_username, users.id as user_id, " +
 				"users_fwd.first_name as user_fwd_first_name, users_fwd.last_name as user_fwd_last_name, users_fwd.username as user_fwd_username " +
-				"FROM messages, users LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " +
+				"FROM messages " +
-				"users.id=messages.sender_id AND " + c.getQuery() + " " +
+				"LEFT JOIN users ON users.id=messages.sender_id " +
-				"ORDER BY messages.id");
+				"LEFT JOIN users AS users_fwd ON users_fwd.id=fwd_from_id WHERE " +
+				c.getQuery() + " " +
+				"ORDER BY messages.message_id");
 			SimpleDateFormat format_time = new SimpleDateFormat("HH:mm:ss");
 			SimpleDateFormat format_date = new SimpleDateFormat("d MMM yy");
 			ResultSetMetaData meta = rs.getMetaData();
@@ -654,7 +679,7 @@ public class Database {
 			this.count = count;
 		}
 
-		public String getQuery() { return "dialog_id=" + id; }
+		public String getQuery() { return "source_type='dialog' AND source_id=" + id; }
 	}
 
 	public class Chat extends AbstractChat {
@@ -668,7 +693,7 @@ public class Database {
 			this.count = count;
 		}
 
-		public String getQuery() {return "chat_id=" + id; }
+		public String getQuery() {return "source_type IN('group', 'supergroup', 'channel') AND source_id=" + id; }
 	}
 
 	public class User {

src/main/java/de/fabianonline/telegram_backup/DatabaseUpdates.java

@@ -32,6 +32,7 @@ public class DatabaseUpdates {
 		register(new DB_Update_5(conn, db));
 		register(new DB_Update_6(conn, db));
 		register(new DB_Update_7(conn, db));
+		register(new DB_Update_8(conn, db));
 	}
 
 	public void doUpdates() {
@@ -132,6 +133,10 @@ abstract class DatabaseUpdate {
 	protected abstract void _doUpdate() throws SQLException;
 	public abstract int getVersion();
 	public boolean needsBackup() { return false; }
+	protected void execute(String sql) throws SQLException {
+		logger.debug("Executing: {}", sql);
+		stmt.executeUpdate(sql);
+	}
 }
 
 class DB_Update_1 extends DatabaseUpdate {
@@ -301,3 +306,44 @@ class DB_Update_7 extends DatabaseUpdate {
 		stmt.executeUpdate("UPDATE messages SET api_layer=51");
 	}
 }
+
+class DB_Update_8 extends DatabaseUpdate {
+	public int getVersion() { return 8; }
+	public DB_Update_8(Connection conn, Database db) { super(conn, db); }
+	public boolean needsBackup() { return true; }
+
+	protected void _doUpdate() throws SQLException {
+		execute("ALTER TABLE messages ADD COLUMN source_type TEXT");
+		execute("ALTER TABLE messages ADD COLUMN source_id INTEGER");
+		execute("update messages set source_type='dialog', source_id=dialog_id where dialog_id is not null");
+		execute("update messages set source_type='group', source_id=chat_id where chat_id is not null");
+
+		execute("CREATE TABLE messages_new (" +
+			"id INTEGER PRIMARY KEY AUTOINCREMENT," +
+			"message_id INTEGER," +
+		    "message_type TEXT," +
+			"source_type TEXT," +
+			"source_id INTEGER," +
+		    "sender_id INTEGER," +
+		    "fwd_from_id INTEGER," +
+		    "text TEXT," +
+		    "time INTEGER," +
+		    "has_media BOOLEAN," +
+		    "media_type TEXT," +
+		    "media_file TEXT," +
+		    "media_size INTEGER," +
+		    "media_json TEXT," +
+		    "markup_json TEXT," +
+		    "data BLOB," +
+			"api_layer INTEGER)");
+		execute("INSERT INTO messages_new" +
+			"(message_id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type," +
+			"media_file, media_size, media_json, markup_json, data, api_layer)" +
+			"SELECT " +
+			"id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type," +
+			"media_file, media_size, media_json, markup_json, data, api_layer FROM messages");
+		execute("DROP TABLE messages");
+		execute("ALTER TABLE messages_new RENAME TO 'messages'");
+		execute("CREATE UNIQUE INDEX unique_messages ON messages (source_type, source_id, message_id)");
+	}
+}

src/main/java/de/fabianonline/telegram_backup/DownloadManager.java

@@ -35,6 +35,7 @@ import com.github.badoualy.telegram.tl.exception.RpcErrorException;
 import com.github.badoualy.telegram.tl.api.request.TLRequestUploadGetFile;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Logger;
+import com.google.gson.Gson;
 
 import java.io.IOException;
 import java.io.File;
@@ -42,6 +43,7 @@ import java.io.FileOutputStream;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.HashMap;
 import java.util.Random;
 import java.net.URL;
 import java.util.concurrent.TimeoutException;
@@ -59,6 +61,7 @@ public class DownloadManager {
 	static TelegramClient download_client;
 	static boolean last_download_succeeded = true;
 	static final Logger logger = LoggerFactory.getLogger(DownloadManager.class);
+	boolean has_seen_flood_wait_message = false;
 
 	public DownloadManager(TelegramClient c, DownloadProgressInterface p) {
 		this.user = UserManager.getInstance();
@@ -103,6 +106,7 @@ public class DownloadManager {
 			new TLInputPeerEmpty(),
 			dialog_limit);
 		logger.debug("Got {} dialogs", dialogs.getDialogs().size());
+
 		for (TLDialog d : dialogs.getDialogs()) {
 			if (d.getTopMessage() > max_message_id && ! (d.getPeer() instanceof TLPeerChannel)) {
 				logger.trace("Updating top message id: {} => {}. Dialog type: {}", max_message_id, d.getTopMessage(), d.getPeer().getClass().getName());
@@ -133,7 +137,7 @@ public class DownloadManager {
 			int end_id = max_message_id;
 
 			List<Integer> ids = makeIdList(start_id, end_id);
-			downloadMessages(ids);
+			downloadMessages(ids, null, null);
 		}
 
 		logger.info("Searching for missing messages in the db");
@@ -144,7 +148,7 @@ public class DownloadManager {
 		logger.debug("db_count: {}", db_count);
 		logger.debug("db_max: {}", db_max);
 
-		if (db_count != db_max) {
+		/*if (db_count != db_max) {
 			if (limit != null) {
 				System.out.println("You are missing messages in your database. But since you're using '--limit-messages', I won't download these now.");
 			} else {
@@ -157,17 +161,73 @@ public class DownloadManager {
 				System.out.println("" + all_missing_ids.size() + " messages are missing in your Database.");
 				System.out.println("I can (and will) download " + downloadable_missing_ids.size() + " of them.");
 
-				downloadMessages(downloadable_missing_ids);
+				downloadMessages(downloadable_missing_ids, null);
-			}
 			}
 
 			logger.info("Logging this run");
 			db.logRun(Math.min(max_database_id + 1, max_message_id), max_message_id, count_missing);
 		}
+		*/
 
-	private void downloadMessages(List<Integer> ids) throws RpcErrorException, IOException {
+		if (CommandLineOptions.cmd_channels || CommandLineOptions.cmd_supergroups) {
-		prog.onMessageDownloadStart(ids.size());
+			System.out.println("Processing channels and/or supergroups...");
-		boolean has_seen_flood_wait_message = false;
+			System.out.println("Please note that only channels/supergroups in the last 100 active chats are processed.");
+
+			HashMap<Integer, Long> channel_access_hashes = new HashMap<Integer, Long>();
+			HashMap<Integer, String> channel_names = new HashMap<Integer, String>();
+			LinkedList<Integer> channels = new LinkedList<Integer>();
+			LinkedList<Integer> supergroups = new LinkedList<Integer>();
+
+			// TODO Add chat title (and other stuff?) to the database
+			for (TLAbsChat c : dialogs.getChats()) {
+				if (c instanceof TLChannel) {
+					TLChannel ch = (TLChannel)c;
+					channel_access_hashes.put(c.getId(), ch.getAccessHash());
+					channel_names.put(c.getId(), ch.getTitle());
+					if (ch.getMegagroup()) {
+						supergroups.add(c.getId());
+					} else {
+						channels.add(c.getId());
+					}
+					// Channel: TLChannel
+					// Supergroup: getMegagroup()==true
+				}
+			}
+
+
+
+			for (TLDialog d : dialogs.getDialogs()) {
+				if (d.getPeer() instanceof TLPeerChannel) {
+					int channel_id = ((TLPeerChannel)d.getPeer()).getChannelId();
+
+					// If this is a channel and we don't want to download channels OR
+					// it is a supergroups and we don't want to download supergroups, then
+					if ((channels.contains(channel_id) && !CommandLineOptions.cmd_channels) ||
+						(supergroups.contains(channel_id) && !CommandLineOptions.cmd_supergroups)) {
+						// Skip this chat.
+						continue;
+					}
+					int max_known_id = db.getTopMessageIDForChannel(channel_id);
+					if (d.getTopMessage() > max_known_id) {
+						List<Integer> ids = makeIdList(max_known_id+1, d.getTopMessage());
+						Long access_hash = channel_access_hashes.get(channel_id);
+						if (access_hash==null) {
+							throw new RuntimeException("AccessHash for Channel missing.");
+						}
+						String channel_name = channel_names.get(channel_id);
+						if (channel_name == null) {
+							channel_name = "?";
+						}
+						TLInputChannel channel = new TLInputChannel(channel_id, access_hash);
+						downloadMessages(ids, channel, "channel " + channel_name);
+					}
+				}
+			}
+		}
+	}
+
+	private void downloadMessages(List<Integer> ids, TLInputChannel channel, String source_string) throws RpcErrorException, IOException {
+		prog.onMessageDownloadStart(ids.size(), source_string);
 
 		logger.debug("Entering download loop");
 		while (ids.size()>0) {
@@ -191,7 +251,11 @@ public class DownloadManager {
 				}
 				tries++;
 				try {
+					if (channel == null) {
 						response = client.messagesGetMessages(vector);
+					} else {
+						response = client.channelsGetMessages(channel, vector);
+					}
 					break;
 				} catch (RpcErrorException e) {
 					if (e.getCode()==420) { // FLOOD_WAIT
@@ -206,6 +270,7 @@ public class DownloadManager {
 			if (response.getMessages().size() != vector.size()) {
 				CommandLineController.show_error("Requested " + vector.size() + " messages, but got " + response.getMessages().size() + ". That is unexpected. Quitting.");
 			}
+
 			prog.onMessageDownloaded(response.getMessages().size());
 			db.saveMessages(response.getMessages(), Kotlogram.API_LAYER);
 			db.saveChats(response.getChats());
@@ -253,7 +318,7 @@ public class DownloadManager {
 		if (ids.size()>0) {
 			System.out.println("You have " + ids.size() + " messages in your db that need an update. Doing that now.");
 			logger.debug("Found {} messages", ids.size());
-			downloadMessages(ids);
+			downloadMessages(ids, null, null);
 		}
 
 		LinkedList<TLMessage> messages = this.db.getMessagesWithMedia();

src/main/java/de/fabianonline/telegram_backup/DownloadProgressInterface.java

@@ -19,7 +19,7 @@ package de.fabianonline.telegram_backup;
 import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager;
 
 public interface DownloadProgressInterface {
-	public void onMessageDownloadStart(int count);
+	public void onMessageDownloadStart(int count, String source);
 	public void onMessageDownloaded(int number);
 	public void onMessageDownloadFinished();
 

src/main/java/de/fabianonline/telegram_backup/mediafilemanager/AbstractMediaFileManager.java

@@ -64,7 +64,12 @@ public abstract class AbstractMediaFileManager {
 		new File(path).mkdirs();
 		return path;
 	}
-	public String getTargetFilename() { return "" + message.getId() + "." + getExtension(); }
+	public String getTargetFilename() {
+		if (message.getToId() instanceof TLPeerChannel) {
+			return "channel_" + ((TLPeerChannel)message.getToId()).getChannelId() + "_" + message.getId() + "." + getExtension();
+		}
+		return "" + message.getId() + "." + getExtension();
+	}
 	public String getTargetPathAndFilename() { return getTargetPath() + getTargetFilename(); }
 
 	protected String extensionFromMimetype(String mime) {