Merging master into stable for version 1.1.3
This commit is contained in:
commit
edfdb90073
|
|
@ -10,3 +10,6 @@ src/main/main.iml
|
|||
cache4.*
|
||||
src/test/test.iml
|
||||
dev/
|
||||
todo
|
||||
deploy.secret.sh
|
||||
release_notes.txt
|
||||
|
|
|
|||
10
.travis.yml
10
.travis.yml
|
|
@ -11,13 +11,3 @@ cache:
|
|||
directories:
|
||||
- $HOME/.gradle/caches/
|
||||
- $HOME/.gradle/wrapper/
|
||||
|
||||
deploy:
|
||||
provider: pages
|
||||
skip-cleanup: true
|
||||
github-token: $github_token
|
||||
keep-history: true
|
||||
on:
|
||||
branch: master
|
||||
local-dir: build/libs
|
||||
target-branch: gh-pages
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
* Update the version in the Dockerfile to the coming version.
|
||||
* Commit the new Dockerfile.
|
||||
* Merge into stable: `git checkout stable && git merge --no-ff master`
|
||||
* Create a new tag for the new version: `git tag -a <version>`.
|
||||
* Push everything to github: `git push --all && git push --tags`.
|
||||
* Build it: `gradle build`.
|
||||
|
|
|
|||
11
Dockerfile
11
Dockerfile
|
|
@ -1,10 +1,15 @@
|
|||
FROM openjdk:8
|
||||
|
||||
ENV JAR_VERSION 1.1.2
|
||||
ENV JAR_VERSION 1.1.3
|
||||
ENV JAR_DOWNLOAD_URL https://github.com/fabianonline/telegram_backup/releases/download/${JAR_VERSION}/telegram_backup.jar
|
||||
|
||||
RUN apt-get update -y && apt-get install -y curl && \
|
||||
RUN apt-get update -y && \
|
||||
apt-get install --no-install-recommends -y curl && \
|
||||
curl -L "https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64" -o /bin/dumb-init && \
|
||||
curl -L $JAR_DOWNLOAD_URL -o telegram_backup.jar && mkdir /data/ && chmod +x /bin/dumb-init
|
||||
curl -L $JAR_DOWNLOAD_URL -o telegram_backup.jar && mkdir /data/ && \
|
||||
chmod +x /bin/dumb-init && \
|
||||
apt-get remove -y curl && \
|
||||
apt-get autoremove -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENTRYPOINT ["/bin/dumb-init", "--", "java", "-jar", "telegram_backup.jar", "--target", "/data/"]
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ You can find the whole app packed into one fat jar file under
|
|||
|
||||
## Limitations
|
||||
This tool relies on Telegram's API. They started rate limiting the calls
|
||||
made by this tool some time ago. As of february 2017, downloading messages
|
||||
made by this tool some time ago. As of February 2017, downloading messages
|
||||
is limited to 400 messages every 30 seconds, resulting in 48,000 messages
|
||||
per hour. Media download is not throttled right now, so it should be a lot
|
||||
quicker.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
#!/bin/bash
|
||||
error() {
|
||||
echo "Error: $1"
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -z "$1" ] && error "Parameter's missing. Expecting version number like '1.2.3' as first and only parameter."
|
||||
|
||||
if [ "$1" == "--help" ]; then
|
||||
echo "Usage: `basename "$0"` 1.2.3"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
release_notes="$(cat release_notes.txt 2>/dev/null)"
|
||||
[ -z "$release_notes" ] && error "release_notes.txt is empty"
|
||||
|
||||
VERSION="$1"
|
||||
|
||||
source "deploy.secret.sh"
|
||||
[ -z "$BOT_TOKEN" ] && error "BOT_TOKEN is not set or empty."
|
||||
[ -z "$CHAT_ID" ] && error "CHAT_ID is not set or empty."
|
||||
[ -z "$TOKEN" ] && error "TOKEN is not set or empty."
|
||||
|
||||
CURL_OPTS="-u fabianonline:$TOKEN"
|
||||
|
||||
git diff-files --quiet --ignore-submodules -- || error "You have changes in your working tree."
|
||||
|
||||
git diff-index --cached --quiet HEAD --ignore-submodules -- || error "You have uncommited changes."
|
||||
|
||||
branch_name=$(git symbolic-ref HEAD 2>/dev/null)
|
||||
branch_name=${branch_name##refs/heads/}
|
||||
[ "$branch_name" == "master" ] || error "Current branch is $branch_name, not master."
|
||||
|
||||
echo "Updating the Dockerfile..."
|
||||
sed -i "s/ENV JAR_VERSION .\+/ENV JAR_VERSION $VERSION/g" Dockerfile || error "Couldn't modify Dockerfile."
|
||||
|
||||
echo "Committing the new Dockerfile..."
|
||||
git commit -m "Bumping the version to $VERSION" Dockerfile || error "Couldn't commit the new Dockerfile."
|
||||
|
||||
echo "Tagging the new version..."
|
||||
git tag -a "$VERSION" -m "Version $VERSION" || error
|
||||
|
||||
echo "Checking out stable..."
|
||||
git checkout stable || error
|
||||
|
||||
echo "Merging master into stable..."
|
||||
git merge --no-ff -m "Merging master into stable for version $VERSION" master || error
|
||||
|
||||
echo "Pushing all to Github..."
|
||||
git push --all || error
|
||||
|
||||
echo "Pushing tags to Github..."
|
||||
git push --tags || error
|
||||
|
||||
echo "Building it..."
|
||||
gradle build || error "Build failed. What did you do?!"
|
||||
|
||||
echo "Generating a release on Github..."
|
||||
json=$(ruby -e "require 'json'; puts({tag_name: '$VERSION', name: '$VERSION', draft: true, body: \$stdin.read}.to_json)" <<< "$release_notes") || error "Couldn't generate JSON for Github"
|
||||
|
||||
json=$(curl $CURL_OPTS https://api.github.com/repos/fabianonline/telegram_backup/releases -XPOST -d "$json") || error "Github failure"
|
||||
|
||||
echo "Uploading telegram_backup.jar to Github..."
|
||||
upload_url=$(jq -r ".upload_url" <<< "$json") || error "Could not parse JSON from Github"
|
||||
upload_url=$(sed 's/{.*}//' <<< "$upload_url")
|
||||
release_url=$(jq -r ".url" <<< "$json") || error "Could not parse JSON from Github"
|
||||
curl $CURL_OPTS --header "Content-Type: application/zip" "${upload_url}?name=telegram_backup.jar" --upload-file build/libs/telegram_backup.jar || error "Asset upload to github failed"
|
||||
|
||||
echo "Building the docker image..."
|
||||
docker build -t fabianonline/telegram_backup:$VERSION -t fabianonline/telegram_backup:latest - < Dockerfile
|
||||
|
||||
echo "Pushing the docker image..."
|
||||
docker push fabianonline/telegram_backup
|
||||
|
||||
echo "Notifying the Telegram group..."
|
||||
release_notes=$(sed 's/\* /• /' <<< "$release_notes")
|
||||
message="Version $VERSION released"$'\n'$'\n'"$release_notes"$'\n'$'\n'"$release_url"
|
||||
|
||||
curl https://api.telegram.org/bot${BOT_TOKEN}/sendMessage -XPOST --form "text=<-" --form-string "chat_id=${CHAT_ID}" <<< "$message"
|
||||
|
||||
echo "Cleaning release_notes.txt..."
|
||||
> release_notes.txt
|
||||
|
||||
echo "Done."
|
||||
|
|
@ -95,7 +95,7 @@ class CommandLineController {
|
|||
}
|
||||
logger.debug("CommandLineOptions.cmd_login: {}", CommandLineOptions.cmd_login)
|
||||
if (CommandLineOptions.cmd_login) {
|
||||
cmd_login(account)
|
||||
cmd_login(CommandLineOptions.val_account)
|
||||
System.exit(0)
|
||||
}
|
||||
// If we reach this point, we can assume that there is an account and a database can be loaded / created.
|
||||
|
|
@ -140,7 +140,8 @@ class CommandLineController {
|
|||
} else {
|
||||
println("Skipping media download because --no-media is set.")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
} catch (e: Throwable) {
|
||||
println("An error occured!")
|
||||
e.printStackTrace()
|
||||
logger.error("Exception caught!", e)
|
||||
// If we encountered an exception, we definitely don't want to start the daemon mode now.
|
||||
|
|
@ -282,6 +283,7 @@ class CommandLineController {
|
|||
println(" --pagination <x> Splits the HTML export into multiple HTML pages with <x> messages per page. Default is 5000.")
|
||||
println(" --no-pagination Disables pagination.")
|
||||
println(" --license Displays the license of this program.")
|
||||
println(" -d, --daemon Keep running after the backup and automatically save new messages.")
|
||||
println(" --anonymize (Try to) Remove all sensitive information from output. Useful for requesting support.")
|
||||
println(" --stats Print some usage statistics.")
|
||||
println(" --with-channels Backup channels as well.")
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package de.fabianonline.telegram_backup
|
|||
import de.fabianonline.telegram_backup.CommandLineController
|
||||
import de.fabianonline.telegram_backup.Utils
|
||||
import de.fabianonline.telegram_backup.Version
|
||||
import java.util.concurrent.TimeUnit
|
||||
import org.slf4j.LoggerFactory
|
||||
import ch.qos.logback.classic.Logger
|
||||
import ch.qos.logback.classic.LoggerContext
|
||||
|
|
@ -78,14 +79,21 @@ object CommandLineRunner {
|
|||
fun checkVersion(): Boolean {
|
||||
val v = Utils.getNewestVersion()
|
||||
if (v != null && v.isNewer) {
|
||||
System.out.println("A newer version is vailable!")
|
||||
System.out.println("You are using: " + Config.APP_APPVER)
|
||||
System.out.println("Available: " + v.version)
|
||||
System.out.println("Get it here: " + v.url)
|
||||
System.out.println()
|
||||
System.out.println("Changes in this version:")
|
||||
System.out.println(v.body)
|
||||
System.out.println()
|
||||
println()
|
||||
println()
|
||||
println()
|
||||
println("A newer version is vailable!")
|
||||
println("You are using: " + Config.APP_APPVER)
|
||||
println("Available: " + v.version)
|
||||
println("Get it here: " + v.url)
|
||||
println()
|
||||
println()
|
||||
println("Changes in this version:")
|
||||
println(v.body)
|
||||
println()
|
||||
println()
|
||||
println()
|
||||
TimeUnit.SECONDS.sleep(5)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
|
|||
logger.debug("DatabaseUpdate.doUpdates running")
|
||||
|
||||
logger.debug("Getting current database version")
|
||||
val version: Int
|
||||
var version: Int
|
||||
logger.debug("Checking if table database_versions exists")
|
||||
rs = stmt.executeQuery("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions'")
|
||||
rs.next()
|
||||
|
|
@ -60,6 +60,26 @@ class DatabaseUpdates(protected var conn: Connection, protected var db: Database
|
|||
System.out.println("Database version: " + version)
|
||||
logger.debug("Max available database version is {}", maxPossibleVersion)
|
||||
|
||||
if (version == 0) {
|
||||
logger.debug("Looking for DatabaseUpdate with create_query...")
|
||||
// This is a fresh database - so we search for the latest available version with a create_query
|
||||
// and use this as a shortcut.
|
||||
var update: DatabaseUpdate? = null
|
||||
for (i in maxPossibleVersion downTo 1) {
|
||||
update = getUpdateToVersion(i)
|
||||
logger.trace("Looking at DatabaseUpdate version {}", update.version)
|
||||
if (update.create_query != null) break
|
||||
update = null
|
||||
}
|
||||
|
||||
if (update != null) {
|
||||
logger.debug("Found DatabaseUpdate version {} with create_query.", update.version)
|
||||
for (query in update.create_query!!) stmt.execute(query)
|
||||
stmt.execute("INSERT INTO database_versions (version) VALUES (${update.version})")
|
||||
version = update.version
|
||||
}
|
||||
}
|
||||
|
||||
if (version < maxPossibleVersion) {
|
||||
logger.debug("Update is necessary. {} => {}.", version, maxPossibleVersion)
|
||||
var backup = false
|
||||
|
|
@ -151,6 +171,8 @@ internal abstract class DatabaseUpdate(protected var conn: Connection, protected
|
|||
companion object {
|
||||
protected val logger = LoggerFactory.getLogger(DatabaseUpdate::class.java)
|
||||
}
|
||||
|
||||
open val create_query: List<String>? = null
|
||||
}
|
||||
|
||||
internal class DB_Update_1(conn: Connection, db: Database) : DatabaseUpdate(conn, db) {
|
||||
|
|
@ -308,6 +330,7 @@ internal class DB_Update_6(conn: Connection, db: Database) : DatabaseUpdate(conn
|
|||
rs.close()
|
||||
conn.setAutoCommit(false)
|
||||
ps.executeBatch()
|
||||
ps.close()
|
||||
conn.commit()
|
||||
conn.setAutoCommit(true)
|
||||
stmt.executeUpdate("DROP TABLE messages")
|
||||
|
|
@ -376,19 +399,29 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
|
|||
override val version: Int
|
||||
get() = 9
|
||||
override val needsBackup = true
|
||||
|
||||
override val create_query = listOf(
|
||||
"CREATE TABLE \"chats\" (id INTEGER PRIMARY KEY ASC, name TEXT, type TEXT);",
|
||||
"CREATE TABLE \"users\" (id INTEGER PRIMARY KEY ASC, first_name TEXT, last_name TEXT, username TEXT, type TEXT, phone TEXT);",
|
||||
"CREATE TABLE database_versions (version INTEGER);",
|
||||
"CREATE TABLE runs (id INTEGER PRIMARY KEY ASC, time INTEGER, start_id INTEGER, end_id INTEGER, count_missing INTEGER);",
|
||||
"CREATE TABLE \"messages\" (id INTEGER PRIMARY KEY AUTOINCREMENT,message_id INTEGER,message_type TEXT,source_type TEXT,source_id INTEGER,sender_id INTEGER,fwd_from_id INTEGER,text TEXT,time INTEGER,has_media BOOLEAN,media_type TEXT,media_file TEXT,media_size INTEGER,media_json TEXT,markup_json TEXT,data BLOB,api_layer INTEGER);",
|
||||
"CREATE UNIQUE INDEX unique_messages ON messages (source_type, source_id, message_id);"
|
||||
)
|
||||
|
||||
@Throws(SQLException::class)
|
||||
override fun _doUpdate() {
|
||||
val logger = LoggerFactory.getLogger(DB_Update_9::class.java)
|
||||
println(" Updating supergroup channel message data (this might take some time)...")
|
||||
print(" ")
|
||||
val count = db.queryInt("SELECT COUNT(*) FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53")
|
||||
logger.debug("Found $count candidates for conversion")
|
||||
val limit = 5000
|
||||
var offset = 0
|
||||
var i = 0
|
||||
while (offset + 1 < count) {
|
||||
while (offset < count) {
|
||||
logger.debug("Querying with limit $limit and offset $offset")
|
||||
val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 LIMIT ${limit} OFFSET ${offset}")
|
||||
val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 ORDER BY id LIMIT ${limit} OFFSET ${offset}")
|
||||
val messages = TLVector<TLAbsMessage>()
|
||||
val messages_to_delete = mutableListOf<Int>()
|
||||
while (rs.next()) {
|
||||
|
|
@ -399,11 +432,14 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
|
|||
messages_to_delete.add(rs.getInt(1))
|
||||
}
|
||||
}
|
||||
rs.close()
|
||||
db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP)
|
||||
execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")")
|
||||
print(".")
|
||||
|
||||
offset += limit
|
||||
}
|
||||
println()
|
||||
logger.info("Converted ${i} of ${count} messages.")
|
||||
println(" Cleaning up the database (this might also take some time, sorry)...")
|
||||
execute("VACUUM")
|
||||
|
|
|
|||
|
|
@ -74,7 +74,8 @@ class GeoFileManager(msg: TLMessage, user: UserManager, client: TelegramClient)
|
|||
@Throws(IOException::class)
|
||||
override fun download(): Boolean {
|
||||
val url = "https://maps.googleapis.com/maps/api/staticmap?" +
|
||||
"center=" + geo.getLat() + "," + geo.getLong() + "&" +
|
||||
"center=${geo.getLat()},${geo.getLong()}&" +
|
||||
"markers=color:red|${geo.getLat()},${geo.getLong()}&" +
|
||||
"zoom=14&size=300x150&scale=2&format=png&" +
|
||||
"key=" + Config.SECRET_GMAPS
|
||||
return DownloadManager.downloadExternalFile(targetPathAndFilename, url)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
{{#media_sticker}}<span class="sticker"><img src="../stickers/{{media_file}}" /></span>{{/media_sticker}}
|
||||
{{#media_photo}}<span class="photo"><img src="../{{media_file}}" /></span>{{/media_photo}}
|
||||
{{#media_document}}<span class="document"><a href="../{{media_file}}">{{media_file}}</a></span>{{/media_document}}
|
||||
{{#media_geo}}<span class="geo"><img src="../{{media_file}}" width="300" height="150"></span>{{/media_geo}}
|
||||
</li>
|
||||
{{/messages}}
|
||||
</ul>
|
||||
|
|
|
|||
Loading…
Reference in New Issue