Merging master into stable for version 1.1.3

.gitignore

@@ -10,3 +10,6 @@ src/main/main.iml
 cache4.*
 src/test/test.iml
 dev/
+todo
+deploy.secret.sh
+release_notes.txt

.travis.yml

@@ -11,13 +11,3 @@ cache:
   directories:
     - $HOME/.gradle/caches/
     - $HOME/.gradle/wrapper/
-
-deploy:
-  provider: pages
-  skip-cleanup: true
-  github-token: $github_token
-  keep-history: true
-  on:
-    branch: master
-  local-dir: build/libs
-  target-branch: gh-pages

DEPLOY.md

@@ -2,6 +2,7 @@
 
 * Update the version in the Dockerfile to the coming version.
 * Commit the new Dockerfile.
+* Merge into stable: `git checkout stable && git merge --no-ff master`
 * Create a new tag for the new version: `git tag -a <version>`.
 * Push everything to github: `git push --all && git push --tags`.
 * Build it: `gradle build`.

Dockerfile

@@ -1,10 +1,15 @@
 FROM openjdk:8
 
-ENV JAR_VERSION 1.1.2
+ENV JAR_VERSION 1.1.3
 ENV JAR_DOWNLOAD_URL https://github.com/fabianonline/telegram_backup/releases/download/${JAR_VERSION}/telegram_backup.jar
 
-RUN apt-get update -y && apt-get install -y curl && \
+RUN apt-get update -y && \
+	apt-get install --no-install-recommends -y curl && \
     curl -L "https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64" -o /bin/dumb-init && \
-    curl -L $JAR_DOWNLOAD_URL -o telegram_backup.jar && mkdir /data/ && chmod +x /bin/dumb-init
+    curl -L $JAR_DOWNLOAD_URL -o telegram_backup.jar && mkdir /data/ && \
+    chmod +x /bin/dumb-init && \
+    apt-get remove -y curl && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
 
 ENTRYPOINT ["/bin/dumb-init", "--", "java", "-jar", "telegram_backup.jar", "--target", "/data/"]

README.md

@@ -21,7 +21,7 @@ You can find the whole app packed into one fat jar file under
 
 ## Limitations
 This tool relies on Telegram's API. They started rate limiting the calls
-made by this tool some time ago. As of february 2017, downloading messages
+made by this tool some time ago. As of February 2017, downloading messages
 is limited to 400 messages every 30 seconds, resulting in 48,000 messages
 per hour. Media download is not throttled right now, so it should be a lot
 quicker.

deploy.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+error() {
+	echo "Error: $1"
+	exit 1
+}
+
+[ -z "$1" ] && error "Parameter's missing. Expecting version number like '1.2.3' as first and only parameter."
+
+if [ "$1" == "--help" ]; then
+	echo "Usage: `basename "$0"` 1.2.3"
+	exit 1
+fi
+
+release_notes="$(cat release_notes.txt 2>/dev/null)"
+[ -z "$release_notes" ] && error "release_notes.txt is empty"
+
+VERSION="$1"
+
+source "deploy.secret.sh"
+[ -z "$BOT_TOKEN" ] && error "BOT_TOKEN is not set or empty."
+[ -z "$CHAT_ID" ] && error "CHAT_ID is not set or empty."
+[ -z "$TOKEN" ] && error "TOKEN is not set or empty."
+
+CURL_OPTS="-u fabianonline:$TOKEN"
+
+git diff-files --quiet --ignore-submodules -- || error "You have changes in your working tree."
+
+git diff-index --cached --quiet HEAD --ignore-submodules -- || error "You have uncommited changes."
+
+branch_name=$(git symbolic-ref HEAD 2>/dev/null)
+branch_name=${branch_name##refs/heads/}
+[ "$branch_name" == "master" ] || error "Current branch is $branch_name, not master."
+
+echo "Updating the Dockerfile..."
+sed -i "s/ENV JAR_VERSION .\+/ENV JAR_VERSION $VERSION/g" Dockerfile || error "Couldn't modify Dockerfile."
+
+echo "Committing the new Dockerfile..."
+git commit -m "Bumping the version to $VERSION" Dockerfile || error "Couldn't commit the new Dockerfile."
+
+echo "Tagging the new version..."
+git tag -a "$VERSION" -m "Version $VERSION" || error
+
+echo "Checking out stable..."
+git checkout stable || error
+
+echo "Merging master into stable..."
+git merge --no-ff -m "Merging master into stable for version $VERSION" master || error
+
+echo "Pushing all to Github..."
+git push --all || error
+
+echo "Pushing tags to Github..."
+git push --tags || error
+
+echo "Building it..."
+gradle build || error "Build failed. What did you do?!"
+
+echo "Generating a release on Github..."
+json=$(ruby -e "require 'json'; puts({tag_name: '$VERSION', name: '$VERSION', draft: true, body: \$stdin.read}.to_json)" <<< "$release_notes") || error "Couldn't generate JSON for Github"
+
+json=$(curl $CURL_OPTS https://api.github.com/repos/fabianonline/telegram_backup/releases -XPOST -d "$json") || error "Github failure"
+
+echo "Uploading telegram_backup.jar to Github..."
+upload_url=$(jq -r ".upload_url" <<< "$json") || error "Could not parse JSON from Github"
+upload_url=$(sed 's/{.*}//' <<< "$upload_url")
+release_url=$(jq -r ".url" <<< "$json") || error "Could not parse JSON from Github"
+curl $CURL_OPTS --header "Content-Type: application/zip" "${upload_url}?name=telegram_backup.jar" --upload-file build/libs/telegram_backup.jar || error "Asset upload to github failed"
+
+echo "Building the docker image..."
+docker build -t fabianonline/telegram_backup:$VERSION -t fabianonline/telegram_backup:latest - < Dockerfile
+
+echo "Pushing the docker image..."
+docker push fabianonline/telegram_backup
+
+echo "Notifying the Telegram group..."
+release_notes=$(sed 's/\* /• /' <<< "$release_notes")
+message="Version $VERSION released"$'\n'$'\n'"$release_notes"$'\n'$'\n'"$release_url"
+
+curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" <<< "$message"
+
+echo "Cleaning release_notes.txt..."
+> release_notes.txt
+
+echo "Done."

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt

@@ -95,7 +95,7 @@ class CommandLineController {
 			}
 			logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login)
 			if (CommandLineOptions.cmd_login) {
-				cmd_login(account)
+				cmd_login(CommandLineOptions.val_account)
 				System.exit(0)
 			}
 			// If we reach this point, we can assume that there is an account and a database can be loaded / created.
@@ -140,7 +140,8 @@ class CommandLineController {
 			} else {
 				println("Skipping media download because --no-media is set.")
 			}
-		} catch (e: Exception) {
+		} catch (e: Throwable) {
+			println("An error occured!")
 			e.printStackTrace()
 			logger.error("Exception caught!", e)
 			// If we encountered an exception, we definitely don't want to start the daemon mode now.
@@ -282,6 +283,7 @@ class CommandLineController {
 		println(" --pagination <x>      Splits the HTML export into multiple HTML pages with <x> messages per page. Default is 5000.")
 		println(" --no-pagination       Disables pagination.")
 		println(" --license             Displays the license of this program.")
+		println(" -d, --daemon          Keep running after the backup and automatically save new messages.")
 		println(" --anonymize           (Try to) Remove all sensitive information from output. Useful for requesting support.")
 		println(" --stats               Print some usage statistics.")
 		println(" --with-channels       Backup channels as well.")

src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt

@@ -19,6 +19,7 @@ package de.fabianonline.telegram_backup
 import de.fabianonline.telegram_backup.CommandLineController
 import de.fabianonline.telegram_backup.Utils
 import de.fabianonline.telegram_backup.Version
+import java.util.concurrent.TimeUnit
 import org.slf4j.LoggerFactory
 import ch.qos.logback.classic.Logger
 import ch.qos.logback.classic.LoggerContext
@@ -78,14 +79,21 @@ object CommandLineRunner {
 	fun checkVersion(): Boolean {
 		val v = Utils.getNewestVersion()
 		if (v != null && v.isNewer) {
-			System.out.println("A newer version is vailable!")
-			System.out.println("You are using: " + Config.APP_APPVER)
-			System.out.println("Available:     " + v.version)
-			System.out.println("Get it here:   " + v.url)
-			System.out.println()
-			System.out.println("Changes in this version:")
-			System.out.println(v.body)
-			System.out.println()
+			println()
+			println()
+			println()
+			println("A newer version is vailable!")
+			println("You are using: " + Config.APP_APPVER)
+			println("Available:     " + v.version)
+			println("Get it here:   " + v.url)
+			println()
+			println()
+			println("Changes in this version:")
+			println(v.body)
+			println()
+			println()
+			println()
+			TimeUnit.SECONDS.sleep(5)
 			return false
 		}
 		return true

src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt

@@ -41,7 +41,7 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
 			logger.debug("DatabaseUpdate.doUpdates running")
 
 			logger.debug("Getting current database version")
-			val version: Int
+			var version: Int
 			logger.debug("Checking if table database_versions exists")
 			rs = stmt.executeQuery("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions'")
 			rs.next()
@@ -60,6 +60,26 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
 			System.out.println("Database version: " + version)
 			logger.debug("Max available database version is {}", maxPossibleVersion)
 
+			if (version == 0) {
+				logger.debug("Looking for DatabaseUpdate with create_query...")
+				// This is a fresh database - so we search for the latest available version with a create_query
+				// and use this as a shortcut.
+				var update: DatabaseUpdate? = null
+				for (i in maxPossibleVersion downTo 1) {
+					update = getUpdateToVersion(i)
+					logger.trace("Looking at DatabaseUpdate version {}", update.version)
+					if (update.create_query != null) break
+					update = null
+				}
+
+				if (update != null) {
+					logger.debug("Found DatabaseUpdate version {} with create_query.", update.version)
+					for (query in update.create_query!!) stmt.execute(query)
+					stmt.execute("INSERT INTO database_versions (version) VALUES (${update.version})")
+					version = update.version
+				}
+			}
+
 			if (version < maxPossibleVersion) {
 				logger.debug("Update is necessary. {} => {}.", version, maxPossibleVersion)
 				var backup = false
@@ -151,6 +171,8 @@ internal abstract class DatabaseUpdate(protected var conn: Connection, protected
 	companion object {
 		protected val logger = LoggerFactory.getLogger(DatabaseUpdate::class.java)
 	}
+
+	open val create_query: List<String>? = null
 }
 
 internal class DB_Update_1(conn: Connection, db: Database) : DatabaseUpdate(conn, db) {
@@ -308,6 +330,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn
 		rs.close()
 		conn.setAutoCommit(false)
 		ps.executeBatch()
+		ps.close()
 		conn.commit()
 		conn.setAutoCommit(true)
 		stmt.executeUpdate("DROP TABLE messages")
@@ -377,18 +400,28 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
 		get() = 9
 	override val needsBackup = true
 
+	override val create_query = listOf(
+		"CREATE TABLE \"chats\" (id INTEGER PRIMARY KEY ASC, name TEXT, type TEXT);",
+		"CREATE TABLE \"users\" (id INTEGER PRIMARY KEY ASC, first_name TEXT, last_name TEXT, username TEXT, type TEXT, phone TEXT);",
+		"CREATE TABLE database_versions (version INTEGER);",
+		"CREATE TABLE runs (id INTEGER PRIMARY KEY ASC, time INTEGER, start_id INTEGER, end_id INTEGER, count_missing INTEGER);",
+		"CREATE TABLE \"messages\" (id INTEGER PRIMARY KEY AUTOINCREMENT,message_id INTEGER,message_type TEXT,source_type TEXT,source_id INTEGER,sender_id INTEGER,fwd_from_id INTEGER,text TEXT,time INTEGER,has_media BOOLEAN,media_type TEXT,media_file TEXT,media_size INTEGER,media_json TEXT,markup_json TEXT,data BLOB,api_layer INTEGER);",
+		"CREATE UNIQUE INDEX unique_messages ON messages (source_type, source_id, message_id);"
+	)
+	
 	@Throws(SQLException::class)
 	override fun _doUpdate() {
 		val logger = LoggerFactory.getLogger(DB_Update_9::class.java)
 		println("    Updating supergroup channel message data (this might take some time)...")
+		print("    ")
 		val count = db.queryInt("SELECT COUNT(*) FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53")
 		logger.debug("Found $count candidates for conversion")
 		val limit = 5000
 		var offset = 0
 		var i = 0
-		while (offset + 1 < count) {
+		while (offset < count) {
 			logger.debug("Querying with limit $limit and offset $offset")
-			val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 LIMIT ${limit} OFFSET ${offset}")
+			val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 ORDER BY id LIMIT ${limit} OFFSET ${offset}")
 			val messages = TLVector<TLAbsMessage>()
 			val messages_to_delete = mutableListOf<Int>()
 			while (rs.next()) {
@@ -399,11 +432,14 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
 					messages_to_delete.add(rs.getInt(1))
 				}
 			}
+			rs.close()
 			db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP)
 			execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")")
+			print(".")
 			
 			offset += limit
 		}
+		println()
 		logger.info("Converted ${i} of ${count} messages.")
 		println("    Cleaning up the database (this might also take some time, sorry)...")
 		execute("VACUUM")

src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt

@@ -74,7 +74,8 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient)
 	@Throws(IOException::class)
 	override fun download(): Boolean {
 		val url = "https://maps.googleapis.com/maps/api/staticmap?" +
-			"center=" + geo.getLat() + "," + geo.getLong() + "&" +
+			"center=${geo.getLat()},${geo.getLong()}&" +
+			"markers=color:red|${geo.getLat()},${geo.getLong()}&" +
 			"zoom=14&size=300x150&scale=2&format=png&" +
 			"key=" + Config.SECRET_GMAPS
 		return DownloadManager.downloadExternalFile(targetPathAndFilename, url)

src/main/resources/templates/html/_messages.mustache

@@ -12,6 +12,7 @@
 			{{#media_sticker}}<span class="sticker"><img src="../stickers/{{media_file}}" /></span>{{/media_sticker}}
 			{{#media_photo}}<span class="photo"><img src="../{{media_file}}" /></span>{{/media_photo}}
 			{{#media_document}}<span class="document"><a href="../{{media_file}}">{{media_file}}</a></span>{{/media_document}}
+			{{#media_geo}}<span class="geo"><img src="../{{media_file}}" width="300" height="150"></span>{{/media_geo}}
 		</li>
 	{{/messages}}
 </ul>