fabian/telegram_backup
1
0
Fork 0
mirror of https://github.com/fabianonline/telegram_backup.git synced 2024-11-22 16:56:16 +00:00
Code Issues Releases Wiki Activity

Merging master into stable for version 1.1.3

Browse Source
This commit is contained in:
Fabian Schlenz 2018-03-08 22:21:11 +01:00
commit edfdb90073
11 changed files with 159 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -10,3 +10,6 @@ src/main/main.iml
cache4.*
src/test/test.iml
dev/
todo
deploy.secret.sh
release_notes.txt

10
.travis.yml
View File

@ -11,13 +11,3 @@ cache:
directories:
- $HOME/.gradle/caches/
- $HOME/.gradle/wrapper/
deploy:
provider: pages
skip-cleanup: true
github-token: $github_token
keep-history: true
on:
branch: master
local-dir: build/libs
target-branch: gh-pages

1
DEPLOY.md
View File

@ -2,6 +2,7 @@
* Update the version in the Dockerfile to the coming version.
* Commit the new Dockerfile.
* Merge into stable: `git checkout stable && git merge --no-ff master`
* Create a new tag for the new version: `git tag -a <version>`.
* Push everything to github: `git push --all && git push --tags`.
* Build it: `gradle build`.

11
Dockerfile
View File

@ -1,10 +1,15 @@
FROM openjdk:8
ENV JAR_VERSION 1.1.2
ENV JAR_VERSION 1.1.3
ENV JAR_DOWNLOAD_URL https://github.com/fabianonline/telegram_backup/releases/download/${JAR_VERSION}/telegram_backup.jar
RUN apt-get update -y && apt-get install -y curl && \
RUN apt-get update -y && \
apt-get install --no-install-recommends -y curl && \
curl -L "https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64" -o /bin/dumb-init && \
curl -L $JAR_DOWNLOAD_URL -o telegram_backup.jar && mkdir /data/ && chmod +x /bin/dumb-init
curl -L $JAR_DOWNLOAD_URL -o telegram_backup.jar && mkdir /data/ && \
chmod +x /bin/dumb-init && \
apt-get remove -y curl && \
apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/*
ENTRYPOINT ["/bin/dumb-init", "--", "java", "-jar", "telegram_backup.jar", "--target", "/data/"]

2
README.md
View File

@ -21,7 +21,7 @@ You can find the whole app packed into one fat jar file under
## Limitations
This tool relies on Telegram's API. They started rate limiting the calls
made by this tool some time ago. As of february 2017, downloading messages
made by this tool some time ago. As of February 2017, downloading messages
is limited to 400 messages every 30 seconds, resulting in 48,000 messages
per hour. Media download is not throttled right now, so it should be a lot
quicker.

84
deploy.sh Executable file
View File

@ -0,0 +1,84 @@
#!/bin/bash
error() {
echo "Error: $1"
exit 1
}
[ -z "$1" ] && error "Parameter's missing. Expecting version number like '1.2.3' as first and only parameter."
if [ "$1" == "--help" ]; then
echo "Usage: `basename "$0"` 1.2.3"
exit 1
fi
release_notes="$(cat release_notes.txt 2>/dev/null)"
[ -z "$release_notes" ] && error "release_notes.txt is empty"
VERSION="$1"
source "deploy.secret.sh"
[ -z "$BOT_TOKEN" ] && error "BOT_TOKEN is not set or empty."
[ -z "$CHAT_ID" ] && error "CHAT_ID is not set or empty."
[ -z "$TOKEN" ] && error "TOKEN is not set or empty."
CURL_OPTS="-u fabianonline:$TOKEN"
git diff-files --quiet --ignore-submodules -- || error "You have changes in your working tree."
git diff-index --cached --quiet HEAD --ignore-submodules -- || error "You have uncommited changes."
branch_name=$(git symbolic-ref HEAD 2>/dev/null)
branch_name=${branch_name##refs/heads/}
[ "$branch_name" == "master" ] || error "Current branch is $branch_name, not master."
echo "Updating the Dockerfile..."
sed -i "s/ENV JAR_VERSION .\+/ENV JAR_VERSION $VERSION/g" Dockerfile || error "Couldn't modify Dockerfile."
echo "Committing the new Dockerfile..."
git commit -m "Bumping the version to $VERSION" Dockerfile || error "Couldn't commit the new Dockerfile."
echo "Tagging the new version..."
git tag -a "$VERSION" -m "Version $VERSION" || error
echo "Checking out stable..."
git checkout stable || error
echo "Merging master into stable..."
git merge --no-ff -m "Merging master into stable for version $VERSION" master || error
echo "Pushing all to Github..."
git push --all || error
echo "Pushing tags to Github..."
git push --tags || error
echo "Building it..."
gradle build || error "Build failed. What did you do?!"
echo "Generating a release on Github..."
json=$(ruby -e "require 'json'; puts({tag_name: '$VERSION', name: '$VERSION', draft: true, body: \$stdin.read}.to_json)" <<< "$release_notes") || error "Couldn't generate JSON for Github"
json=$(curl $CURL_OPTS https://api.github.com/repos/fabianonline/telegram_backup/releases -XPOST -d "$json") || error "Github failure"
echo "Uploading telegram_backup.jar to Github..."
upload_url=$(jq -r ".upload_url" <<< "$json") || error "Could not parse JSON from Github"
upload_url=$(sed 's/{.*}//' <<< "$upload_url")
release_url=$(jq -r ".url" <<< "$json") || error "Could not parse JSON from Github"
curl $CURL_OPTS --header "Content-Type: application/zip" "${upload_url}?name=telegram_backup.jar" --upload-file build/libs/telegram_backup.jar || error "Asset upload to github failed"
echo "Building the docker image..."
docker build -t fabianonline/telegram_backup:$VERSION -t fabianonline/telegram_backup:latest - < Dockerfile
echo "Pushing the docker image..."
docker push fabianonline/telegram_backup
echo "Notifying the Telegram group..."
release_notes=$(sed 's/\* /• /' <<< "$release_notes")
message="Version $VERSION released"$'\n'$'\n'"$release_notes"$'\n'$'\n'"$release_url"
curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" <<< "$message"
echo "Cleaning release_notes.txt..."
> release_notes.txt
echo "Done."

6
src/main/kotlin/de/fabianonline/telegram_backup/CommandLineController.kt
View File

@ -95,7 +95,7 @@ class CommandLineController {
}
logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login)
if (CommandLineOptions.cmd_login) {
cmd_login(account)
cmd_login(CommandLineOptions.val_account)
System.exit(0)
}
// If we reach this point, we can assume that there is an account and a database can be loaded / created.
@ -140,7 +140,8 @@ class CommandLineController {
} else {
println("Skipping media download because --no-media is set.")
}
} catch (e: Exception) {
} catch (e: Throwable) {
println("An error occured!")
e.printStackTrace()
logger.error("Exception caught!", e)
// If we encountered an exception, we definitely don't want to start the daemon mode now.
@ -282,6 +283,7 @@ class CommandLineController {
println(" --pagination <x> Splits the HTML export into multiple HTML pages with <x> messages per page. Default is 5000.")
println(" --no-pagination Disables pagination.")
println(" --license Displays the license of this program.")
println(" -d, --daemon Keep running after the backup and automatically save new messages.")
println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.")
println(" --stats Print some usage statistics.")
println(" --with-channels Backup channels as well.")

24
src/main/kotlin/de/fabianonline/telegram_backup/CommandLineRunner.kt
View File

@ -19,6 +19,7 @@ package de.fabianonline.telegram_backup
import de.fabianonline.telegram_backup.CommandLineController
import de.fabianonline.telegram_backup.Utils
import de.fabianonline.telegram_backup.Version
import java.util.concurrent.TimeUnit
import org.slf4j.LoggerFactory
import ch.qos.logback.classic.Logger
import ch.qos.logback.classic.LoggerContext
@ -78,14 +79,21 @@ object CommandLineRunner {
fun checkVersion(): Boolean {
val v = Utils.getNewestVersion()
if (v != null && v.isNewer) {
System.out.println("A newer version is vailable!")
System.out.println("You are using: " + Config.APP_APPVER)
System.out.println("Available: " + v.version)
System.out.println("Get it here: " + v.url)
System.out.println()
System.out.println("Changes in this version:")
System.out.println(v.body)
System.out.println()
println()
println()
println()
println("A newer version is vailable!")
println("You are using: " + Config.APP_APPVER)
println("Available: " + v.version)
println("Get it here: " + v.url)
println()
println()
println("Changes in this version:")
println(v.body)
println()
println()
println()
TimeUnit.SECONDS.sleep(5)
return false
}
return true

42
src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt
View File

@ -41,7 +41,7 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
logger.debug("DatabaseUpdate.doUpdates running")
logger.debug("Getting current database version")
val version: Int
var version: Int
logger.debug("Checking if table database_versions exists")
rs = stmt.executeQuery("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions'")
rs.next()
@ -60,6 +60,26 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
System.out.println("Database version: " + version)
logger.debug("Max available database version is {}", maxPossibleVersion)
if (version == 0) {
logger.debug("Looking for DatabaseUpdate with create_query...")
// This is a fresh database - so we search for the latest available version with a create_query
// and use this as a shortcut.
var update: DatabaseUpdate? = null
for (i in maxPossibleVersion downTo 1) {
update = getUpdateToVersion(i)
logger.trace("Looking at DatabaseUpdate version {}", update.version)
if (update.create_query != null) break
update = null
}
if (update != null) {
logger.debug("Found DatabaseUpdate version {} with create_query.", update.version)
for (query in update.create_query!!) stmt.execute(query)
stmt.execute("INSERT INTO database_versions (version) VALUES (${update.version})")
version = update.version
}
}
if (version < maxPossibleVersion) {
logger.debug("Update is necessary. {} => {}.", version, maxPossibleVersion)
var backup = false
@ -151,6 +171,8 @@ internal abstract class DatabaseUpdate(protected var conn: Connection, protected
companion object {
protected val logger = LoggerFactory.getLogger(DatabaseUpdate::class.java)
}
open val create_query: List<String>? = null
}
internal class DB_Update_1(conn: Connection, db: Database) : DatabaseUpdate(conn, db) {
@ -308,6 +330,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn
rs.close()
conn.setAutoCommit(false)
ps.executeBatch()
ps.close()
conn.commit()
conn.setAutoCommit(true)
stmt.executeUpdate("DROP TABLE messages")
@ -376,19 +399,29 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
override val version: Int
get() = 9
override val needsBackup = true
override val create_query = listOf(
"CREATE TABLE \"chats\" (id INTEGER PRIMARY KEY ASC, name TEXT, type TEXT);",
"CREATE TABLE \"users\" (id INTEGER PRIMARY KEY ASC, first_name TEXT, last_name TEXT, username TEXT, type TEXT, phone TEXT);",
"CREATE TABLE database_versions (version INTEGER);",
"CREATE TABLE runs (id INTEGER PRIMARY KEY ASC, time INTEGER, start_id INTEGER, end_id INTEGER, count_missing INTEGER);",
"CREATE TABLE \"messages\" (id INTEGER PRIMARY KEY AUTOINCREMENT,message_id INTEGER,message_type TEXT,source_type TEXT,source_id INTEGER,sender_id INTEGER,fwd_from_id INTEGER,text TEXT,time INTEGER,has_media BOOLEAN,media_type TEXT,media_file TEXT,media_size INTEGER,media_json TEXT,markup_json TEXT,data BLOB,api_layer INTEGER);",
"CREATE UNIQUE INDEX unique_messages ON messages (source_type, source_id, message_id);"
)
@Throws(SQLException::class)
override fun _doUpdate() {
val logger = LoggerFactory.getLogger(DB_Update_9::class.java)
println(" Updating supergroup channel message data (this might take some time)...")
print(" ")
val count = db.queryInt("SELECT COUNT(*) FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53")
logger.debug("Found $count candidates for conversion")
val limit = 5000
var offset = 0
var i = 0
while (offset + 1 < count) {
while (offset < count) {
logger.debug("Querying with limit $limit and offset $offset")
val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 LIMIT ${limit} OFFSET ${offset}")
val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 ORDER BY id LIMIT ${limit} OFFSET ${offset}")
val messages = TLVector<TLAbsMessage>()
val messages_to_delete = mutableListOf<Int>()
while (rs.next()) {
@ -399,11 +432,14 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
messages_to_delete.add(rs.getInt(1))
}
}
rs.close()
db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP)
execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")")
print(".")
offset += limit
}
println()
logger.info("Converted ${i} of ${count} messages.")
println(" Cleaning up the database (this might also take some time, sorry)...")
execute("VACUUM")

3
src/main/kotlin/de/fabianonline/telegram_backup/mediafilemanager/GeoFileManager.kt
View File

@ -74,7 +74,8 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient)
@Throws(IOException::class)
override fun download(): Boolean {
val url = "https://maps.googleapis.com/maps/api/staticmap?" +
"center=" + geo.getLat() + "," + geo.getLong() + "&" +
"center=${geo.getLat()},${geo.getLong()}&" +
"markers=color:red|${geo.getLat()},${geo.getLong()}&" +
"zoom=14&size=300x150&scale=2&format=png&" +
"key=" + Config.SECRET_GMAPS
return DownloadManager.downloadExternalFile(targetPathAndFilename, url)

1
src/main/resources/templates/html/_messages.mustache
View File

@ -12,6 +12,7 @@
{{#media_sticker}}<span class="sticker"><img src="../stickers/{{media_file}}" /></span>{{/media_sticker}}
{{#media_photo}}<span class="photo"><img src="../{{media_file}}" /></span>{{/media_photo}}
{{#media_document}}<span class="document"><a href="../{{media_file}}">{{media_file}}</a></span>{{/media_document}}
{{#media_geo}}<span class="geo"><img src="../{{media_file}}" width="300" height="150"></span>{{/media_geo}}
</li>
{{/messages}}
</ul>