diff --git a/.gitignore b/.gitignore index 6429296..953f7e4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ src/main/main.iml cache4.* src/test/test.iml dev/ +todo +deploy.secret.sh +release_notes.txt diff --git a/.travis.yml b/.travis.yml index a6f94ba..4112d3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,13 +11,3 @@ cache: directories: - $HOME/.gradle/caches/ - $HOME/.gradle/wrapper/ - -deploy: - provider: pages - skip-cleanup: true - github-token: $github_token - keep-history: true - on: - branch: master - local-dir: build/libs - target-branch: gh-pages diff --git a/DEPLOY.md b/DEPLOY.md index ff313d9..43c1fb5 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -2,6 +2,7 @@ * Update the version in the Dockerfile to the coming version. * Commit the new Dockerfile. +* Merge into stable: `git checkout stable && git merge --no-ff master` * Create a new tag for the new version: `git tag -a `. * Push everything to github: `git push --all && git push --tags`. * Build it: `gradle build`. diff --git a/Dockerfile b/Dockerfile index 54bf189..47d48cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,15 @@ FROM openjdk:8 -ENV JAR_VERSION 1.1.2 +ENV JAR_VERSION 1.1.3 ENV JAR_DOWNLOAD_URL https://github.com/fabianonline/telegram_backup/releases/download/${JAR_VERSION}/telegram_backup.jar -RUN apt-get update -y && apt-get install -y curl && \ +RUN apt-get update -y && \ + apt-get install --no-install-recommends -y curl && \ curl -L "https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64" -o /bin/dumb-init && \ - curl -L $JAR_DOWNLOAD_URL -o telegram_backup.jar && mkdir /data/ && chmod +x /bin/dumb-init + curl -L $JAR_DOWNLOAD_URL -o telegram_backup.jar && mkdir /data/ && \ + chmod +x /bin/dumb-init && \ + apt-get remove -y curl && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* ENTRYPOINT ["/bin/dumb-init", "--", "java", "-jar", "telegram_backup.jar", "--target", "/data/"] diff --git a/README.md b/README.md index 1015da6..4b20dda 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ You can find the whole app packed into one fat jar file under ## Limitations This tool relies on Telegram's API. They started rate limiting the calls -made by this tool some time ago. As of february 2017, downloading messages +made by this tool some time ago. As of February 2017, downloading messages is limited to 400 messages every 30 seconds, resulting in 48,000 messages per hour. Media download is not throttled right now, so it should be a lot quicker. diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..4ba05ea --- /dev/null +++ b/deploy.sh @@ -0,0 +1,84 @@ +#!/bin/bash +error() { + echo "Error: $1" + exit 1 +} + +[ -z "$1" ] && error "Parameter's missing. Expecting version number like '1.2.3' as first and only parameter." + +if [ "$1" == "--help" ]; then + echo "Usage: `basename "$0"` 1.2.3" + exit 1 +fi + +release_notes="$(cat release_notes.txt 2>/dev/null)" +[ -z "$release_notes" ] && error "release_notes.txt is empty" + +VERSION="$1" + +source "deploy.secret.sh" +[ -z "$BOT_TOKEN" ] && error "BOT_TOKEN is not set or empty." +[ -z "$CHAT_ID" ] && error "CHAT_ID is not set or empty." +[ -z "$TOKEN" ] && error "TOKEN is not set or empty." + +CURL_OPTS="-u fabianonline:$TOKEN" + +git diff-files --quiet --ignore-submodules -- || error "You have changes in your working tree." + +git diff-index --cached --quiet HEAD --ignore-submodules -- || error "You have uncommited changes." + +branch_name=$(git symbolic-ref HEAD 2>/dev/null) +branch_name=${branch_name##refs/heads/} +[ "$branch_name" == "master" ] || error "Current branch is $branch_name, not master." + +echo "Updating the Dockerfile..." +sed -i "s/ENV JAR_VERSION .\+/ENV JAR_VERSION $VERSION/g" Dockerfile || error "Couldn't modify Dockerfile." + +echo "Committing the new Dockerfile..." +git commit -m "Bumping the version to $VERSION" Dockerfile || error "Couldn't commit the new Dockerfile." + +echo "Tagging the new version..." +git tag -a "$VERSION" -m "Version $VERSION" || error + +echo "Checking out stable..." +git checkout stable || error + +echo "Merging master into stable..." +git merge --no-ff -m "Merging master into stable for version $VERSION" master || error + +echo "Pushing all to Github..." +git push --all || error + +echo "Pushing tags to Github..." +git push --tags || error + +echo "Building it..." +gradle build || error "Build failed. What did you do?!" + +echo "Generating a release on Github..." +json=$(ruby -e "require 'json'; puts({tag_name: '$VERSION', name: '$VERSION', draft: true, body: \$stdin.read}.to_json)" <<< "$release_notes") || error "Couldn't generate JSON for Github" + +json=$(curl $CURL_OPTS https://api.github.com/repos/fabianonline/telegram_backup/releases -XPOST -d "$json") || error "Github failure" + +echo "Uploading telegram_backup.jar to Github..." +upload_url=$(jq -r ".upload_url" <<< "$json") || error "Could not parse JSON from Github" +upload_url=$(sed 's/{.*}//' <<< "$upload_url") +release_url=$(jq -r ".url" <<< "$json") || error "Could not parse JSON from Github" +curl $CURL_OPTS --header "Content-Type: application/zip" "${upload_url}?name=telegram_backup.jar" --upload-file build/libs/telegram_backup.jar || error "Asset upload to github failed" + +echo "Building the docker image..." +docker build -t fabianonline/telegram_backup:$VERSION -t fabianonline/telegram_backup:latest - < Dockerfile + +echo "Pushing the docker image..." +docker push fabianonline/telegram_backup + +echo "Notifying the Telegram group..." +release_notes=$(sed 's/\* /• /' <<< "$release_notes") +message="Version $VERSION released"$'\n'$'\n'"$release_notes"$'\n'$'\n'"$release_url" + +curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" <<< "$message" + +echo "Cleaning release_notes.txt..." +> release_notes.txt + +echo "Done." diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt index 332f25a..70969bc 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt @@ -95,7 +95,7 @@ class CommandLineController { } logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login) if (CommandLineOptions.cmd_login) { - cmd_login(account) + cmd_login(CommandLineOptions.val_account) System.exit(0) } // If we reach this point, we can assume that there is an account and a database can be loaded / created. @@ -140,7 +140,8 @@ class CommandLineController { } else { println("Skipping media download because --no-media is set.") } - } catch (e: Exception) { + } catch (e: Throwable) { + println("An error occured!") e.printStackTrace() logger.error("Exception caught!", e) // If we encountered an exception, we definitely don't want to start the daemon mode now. @@ -282,6 +283,7 @@ class CommandLineController { println(" --pagination Splits the HTML export into multiple HTML pages with messages per page. Default is 5000.") println(" --no-pagination Disables pagination.") println(" --license Displays the license of this program.") + println(" -d, --daemon Keep running after the backup and automatically save new messages.") println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.") println(" --stats Print some usage statistics.") println(" --with-channels Backup channels as well.") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt index b8d6801..a922902 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt @@ -19,6 +19,7 @@ package de.fabianonline.telegram_backup import de.fabianonline.telegram_backup.CommandLineController import de.fabianonline.telegram_backup.Utils import de.fabianonline.telegram_backup.Version +import java.util.concurrent.TimeUnit import org.slf4j.LoggerFactory import ch.qos.logback.classic.Logger import ch.qos.logback.classic.LoggerContext @@ -78,14 +79,21 @@ object CommandLineRunner { fun checkVersion(): Boolean { val v = Utils.getNewestVersion() if (v != null && v.isNewer) { - System.out.println("A newer version is vailable!") - System.out.println("You are using: " + Config.APP_APPVER) - System.out.println("Available: " + v.version) - System.out.println("Get it here: " + v.url) - System.out.println() - System.out.println("Changes in this version:") - System.out.println(v.body) - System.out.println() + println() + println() + println() + println("A newer version is vailable!") + println("You are using: " + Config.APP_APPVER) + println("Available: " + v.version) + println("Get it here: " + v.url) + println() + println() + println("Changes in this version:") + println(v.body) + println() + println() + println() + TimeUnit.SECONDS.sleep(5) return false } return true diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt index 4ddfe59..0f8d12c 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt @@ -41,7 +41,7 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database logger.debug("DatabaseUpdate.doUpdates running") logger.debug("Getting current database version") - val version: Int + var version: Int logger.debug("Checking if table database_versions exists") rs = stmt.executeQuery("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions'") rs.next() @@ -60,6 +60,26 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database System.out.println("Database version: " + version) logger.debug("Max available database version is {}", maxPossibleVersion) + if (version == 0) { + logger.debug("Looking for DatabaseUpdate with create_query...") + // This is a fresh database - so we search for the latest available version with a create_query + // and use this as a shortcut. + var update: DatabaseUpdate? = null + for (i in maxPossibleVersion downTo 1) { + update = getUpdateToVersion(i) + logger.trace("Looking at DatabaseUpdate version {}", update.version) + if (update.create_query != null) break + update = null + } + + if (update != null) { + logger.debug("Found DatabaseUpdate version {} with create_query.", update.version) + for (query in update.create_query!!) stmt.execute(query) + stmt.execute("INSERT INTO database_versions (version) VALUES (${update.version})") + version = update.version + } + } + if (version < maxPossibleVersion) { logger.debug("Update is necessary. {} => {}.", version, maxPossibleVersion) var backup = false @@ -151,6 +171,8 @@ internal abstract class DatabaseUpdate(protected var conn: Connection, protected companion object { protected val logger = LoggerFactory.getLogger(DatabaseUpdate::class.java) } + + open val create_query: List? = null } internal class DB_Update_1(conn: Connection, db: Database) : DatabaseUpdate(conn, db) { @@ -308,6 +330,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn rs.close() conn.setAutoCommit(false) ps.executeBatch() + ps.close() conn.commit() conn.setAutoCommit(true) stmt.executeUpdate("DROP TABLE messages") @@ -376,19 +399,29 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn override val version: Int get() = 9 override val needsBackup = true + + override val create_query = listOf( + "CREATE TABLE \"chats\" (id INTEGER PRIMARY KEY ASC, name TEXT, type TEXT);", + "CREATE TABLE \"users\" (id INTEGER PRIMARY KEY ASC, first_name TEXT, last_name TEXT, username TEXT, type TEXT, phone TEXT);", + "CREATE TABLE database_versions (version INTEGER);", + "CREATE TABLE runs (id INTEGER PRIMARY KEY ASC, time INTEGER, start_id INTEGER, end_id INTEGER, count_missing INTEGER);", + "CREATE TABLE \"messages\" (id INTEGER PRIMARY KEY AUTOINCREMENT,message_id INTEGER,message_type TEXT,source_type TEXT,source_id INTEGER,sender_id INTEGER,fwd_from_id INTEGER,text TEXT,time INTEGER,has_media BOOLEAN,media_type TEXT,media_file TEXT,media_size INTEGER,media_json TEXT,markup_json TEXT,data BLOB,api_layer INTEGER);", + "CREATE UNIQUE INDEX unique_messages ON messages (source_type, source_id, message_id);" + ) @Throws(SQLException::class) override fun _doUpdate() { val logger = LoggerFactory.getLogger(DB_Update_9::class.java) println(" Updating supergroup channel message data (this might take some time)...") + print(" ") val count = db.queryInt("SELECT COUNT(*) FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53") logger.debug("Found $count candidates for conversion") val limit = 5000 var offset = 0 var i = 0 - while (offset + 1 < count) { + while (offset < count) { logger.debug("Querying with limit $limit and offset $offset") - val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 LIMIT ${limit} OFFSET ${offset}") + val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 ORDER BY id LIMIT ${limit} OFFSET ${offset}") val messages = TLVector() val messages_to_delete = mutableListOf() while (rs.next()) { @@ -399,11 +432,14 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn messages_to_delete.add(rs.getInt(1)) } } + rs.close() db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP) execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")") + print(".") offset += limit } + println() logger.info("Converted ${i} of ${count} messages.") println(" Cleaning up the database (this might also take some time, sorry)...") execute("VACUUM") diff --git a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt index 15726c8..41d1f1c 100644 --- a/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt +++ b/src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt @@ -74,7 +74,8 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient) @Throws(IOException::class) override fun download(): Boolean { val url = "https://maps.googleapis.com/maps/api/staticmap?" + - "center=" + geo.getLat() + "," + geo.getLong() + "&" + + "center=${geo.getLat()},${geo.getLong()}&" + + "markers=color:red|${geo.getLat()},${geo.getLong()}&" + "zoom=14&size=300x150&scale=2&format=png&" + "key=" + Config.SECRET_GMAPS return DownloadManager.downloadExternalFile(targetPathAndFilename, url) diff --git a/src/main/resources/templates/html/_messages.mustache b/src/main/resources/templates/html/_messages.mustache index d25ca2a..741e73b 100644 --- a/src/main/resources/templates/html/_messages.mustache +++ b/src/main/resources/templates/html/_messages.mustache @@ -12,6 +12,7 @@ {{#media_sticker}}{{/media_sticker}} {{#media_photo}}{{/media_photo}} {{#media_document}}{{media_file}}{{/media_document}} + {{#media_geo}}{{/media_geo}} {{/messages}}