2017-12-07 07:00:18 +00:00
package de.fabianonline.telegram_backup
import java.util.HashMap
import java.util.LinkedHashMap
import java.util.LinkedList
import java.sql.Connection
import java.sql.SQLException
import java.sql.Statement
import java.sql.Types
import java.sql.ResultSet
import java.sql.PreparedStatement
import com.github.badoualy.telegram.tl.api.*
2018-02-21 05:31:39 +00:00
import com.github.badoualy.telegram.tl.core.TLVector
2017-12-07 07:00:18 +00:00
import org.slf4j.LoggerFactory
import org.slf4j.Logger
import de.fabianonline.telegram_backup.mediafilemanager.FileManagerFactory
import de.fabianonline.telegram_backup.mediafilemanager.AbstractMediaFileManager
2018-04-12 04:54:14 +00:00
import com.github.salomonbrys.kotson.*
import com.google.gson.*
2017-12-07 07:00:18 +00:00
class DatabaseUpdates ( protected var conn : Connection , protected var db : Database ) {
2017-12-12 21:04:20 +00:00
private val maxPossibleVersion : Int
get ( ) = updates . size
init {
logger . debug ( " Registering Database Updates... " )
register ( DB _Update _1 ( conn , db ) )
register ( DB _Update _2 ( conn , db ) )
register ( DB _Update _3 ( conn , db ) )
register ( DB _Update _4 ( conn , db ) )
register ( DB _Update _5 ( conn , db ) )
register ( DB _Update _6 ( conn , db ) )
register ( DB _Update _7 ( conn , db ) )
register ( DB _Update _8 ( conn , db ) )
2018-02-21 05:31:39 +00:00
register ( DB _Update _9 ( conn , db ) )
2018-02-22 06:55:53 +00:00
register ( DB _Update _10 ( conn , db ) )
2018-04-12 04:54:14 +00:00
register ( DB _Update _11 ( conn , db ) )
2017-12-12 21:04:20 +00:00
}
fun doUpdates ( ) {
try {
val stmt = conn . createStatement ( )
logger . debug ( " DatabaseUpdate.doUpdates running " )
logger . debug ( " Getting current database version " )
2018-03-08 21:15:49 +00:00
var version : Int
2017-12-12 21:04:20 +00:00
logger . debug ( " Checking if table database_versions exists " )
2018-03-12 21:01:47 +00:00
val table _count = db . queryInt ( " SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='database_versions' " )
if ( table _count == 0 ) {
2017-12-12 21:04:20 +00:00
logger . debug ( " Table does not exist " )
version = 0
} else {
logger . debug ( " Table exists. Checking max version " )
2018-03-12 21:01:47 +00:00
version = db . queryInt ( " SELECT MAX(version) FROM database_versions " )
2017-12-12 21:04:20 +00:00
}
logger . debug ( " version: {} " , version )
System . out . println ( " Database version: " + version )
logger . debug ( " Max available database version is {} " , maxPossibleVersion )
2018-03-08 21:15:49 +00:00
if ( version == 0 ) {
logger . debug ( " Looking for DatabaseUpdate with create_query... " )
// This is a fresh database - so we search for the latest available version with a create_query
// and use this as a shortcut.
var update : DatabaseUpdate ? = null
for ( i in maxPossibleVersion downTo 1 ) {
update = getUpdateToVersion ( i )
logger . trace ( " Looking at DatabaseUpdate version {} " , update . version )
if ( update . create _query != null ) break
update = null
}
if ( update != null ) {
logger . debug ( " Found DatabaseUpdate version {} with create_query. " , update . version )
for ( query in update . create _query !! ) stmt . execute ( query )
stmt . execute ( " INSERT INTO database_versions (version) VALUES ( ${update.version} ) " )
version = update . version
}
}
2017-12-12 21:04:20 +00:00
if ( version < maxPossibleVersion ) {
logger . debug ( " Update is necessary. {} => {}. " , version , maxPossibleVersion )
var backup = false
for ( i in version + 1. . maxPossibleVersion ) {
if ( getUpdateToVersion ( i ) . needsBackup ) {
logger . debug ( " Update to version {} needs a backup " , i )
backup = true
}
}
if ( backup ) {
if ( version > 0 ) {
logger . debug ( " Performing backup " )
db . backupDatabase ( version )
} else {
logger . debug ( " NOT performing a backup, because we are creating a fresh database and don't need a backup of that. " )
}
}
logger . debug ( " Applying updates " )
try {
for ( i in version + 1. . maxPossibleVersion ) {
getUpdateToVersion ( i ) . doUpdate ( )
}
} catch ( e : SQLException ) {
throw RuntimeException ( e )
}
2018-03-12 21:01:47 +00:00
println ( " Cleaning up the database (this might take some time)... " )
try { stmt . executeUpdate ( " VACUUM " ) } catch ( t : Throwable ) { logger . debug ( " Exception during VACUUMing: {} " , t ) }
2017-12-12 21:04:20 +00:00
} else {
logger . debug ( " No update necessary. " )
}
} catch ( e : SQLException ) {
throw RuntimeException ( e )
}
}
private fun getUpdateToVersion ( i : Int ) : DatabaseUpdate {
return updates . get ( i - 1 )
}
private fun register ( d : DatabaseUpdate ) {
logger . debug ( " Registering {} as update to version {} " , d . javaClass , d . version )
if ( d . version != updates . size + 1 ) {
throw RuntimeException ( " Tried to register DB update to version ${d.version} , but would need update to version ${updates.size + 1} " )
}
updates . add ( d )
}
companion object {
private val logger = LoggerFactory . getLogger ( DatabaseUpdates :: class . java )
private val updates = LinkedList < DatabaseUpdate > ( )
}
2017-12-07 07:00:18 +00:00
}
internal abstract class DatabaseUpdate ( protected var conn : Connection , protected var db : Database ) {
2017-12-12 21:04:20 +00:00
protected var stmt : Statement
abstract val version : Int
init {
try {
stmt = conn . createStatement ( )
} catch ( e : SQLException ) {
throw RuntimeException ( e )
}
}
@Throws ( SQLException :: class )
fun doUpdate ( ) {
logger . debug ( " Applying update to version {} " , version )
System . out . println ( " Updating to version $version ... " )
_doUpdate ( )
logger . debug ( " Saving current database version to the db " )
stmt . executeUpdate ( " INSERT INTO database_versions (version) VALUES ( $version ) " )
}
@Throws ( SQLException :: class )
protected abstract fun _doUpdate ( )
open val needsBackup = false
@Throws ( SQLException :: class )
protected fun execute ( sql : String ) {
logger . debug ( " Executing: {} " , sql )
stmt . executeUpdate ( sql )
}
companion object {
protected val logger = LoggerFactory . getLogger ( DatabaseUpdate :: class . java )
}
2018-03-08 21:15:49 +00:00
open val create _query : List < String > ? = null
2017-12-07 07:00:18 +00:00
}
internal class DB _Update _1 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
2017-12-12 21:04:20 +00:00
override val version : Int
get ( ) = 1
@Throws ( SQLException :: class )
override fun _doUpdate ( ) {
stmt . executeUpdate ( " CREATE TABLE messages ( "
+ " id INTEGER PRIMARY KEY ASC, "
+ " dialog_id INTEGER, "
+ " to_id INTEGER, "
+ " from_id INTEGER, "
+ " from_type TEXT, "
+ " text TEXT, "
+ " time TEXT, "
+ " has_media BOOLEAN, "
+ " sticker TEXT, "
+ " data BLOB, "
+ " type TEXT) " )
stmt . executeUpdate ( " CREATE TABLE dialogs ( "
+ " id INTEGER PRIMARY KEY ASC, "
+ " name TEXT, "
+ " type TEXT) " )
stmt . executeUpdate ( " CREATE TABLE people ( "
+ " id INTEGER PRIMARY KEY ASC, "
+ " first_name TEXT, "
+ " last_name TEXT, "
+ " username TEXT, "
+ " type TEXT) " )
stmt . executeUpdate ( " CREATE TABLE database_versions ( " + " version INTEGER) " )
}
2017-12-07 07:00:18 +00:00
}
internal class DB _Update _2 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
2017-12-12 21:04:20 +00:00
override val version : Int
get ( ) = 2
@Throws ( SQLException :: class )
override fun _doUpdate ( ) {
stmt . executeUpdate ( " ALTER TABLE people RENAME TO 'users' " )
stmt . executeUpdate ( " ALTER TABLE users ADD COLUMN phone TEXT " )
}
2017-12-07 07:00:18 +00:00
}
internal class DB _Update _3 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
2017-12-12 21:04:20 +00:00
override val version : Int
get ( ) = 3
2017-12-07 07:00:18 +00:00
2017-12-12 21:04:20 +00:00
@Throws ( SQLException :: class )
override fun _doUpdate ( ) {
stmt . executeUpdate ( " ALTER TABLE dialogs RENAME TO 'chats' " )
}
2017-12-07 07:00:18 +00:00
}
internal class DB _Update _4 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
2017-12-12 21:04:20 +00:00
override val version : Int
get ( ) = 4
@Throws ( SQLException :: class )
override fun _doUpdate ( ) {
stmt . executeUpdate ( " CREATE TABLE messages_new (id INTEGER PRIMARY KEY ASC, dialog_id INTEGER, to_id INTEGER, from_id INTEGER, from_type TEXT, text TEXT, time INTEGER, has_media BOOLEAN, sticker TEXT, data BLOB, type TEXT); " )
stmt . executeUpdate ( " INSERT INTO messages_new SELECT * FROM messages " )
stmt . executeUpdate ( " DROP TABLE messages " )
stmt . executeUpdate ( " ALTER TABLE messages_new RENAME TO 'messages' " )
}
2017-12-07 07:00:18 +00:00
}
internal class DB _Update _5 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
2017-12-12 21:04:20 +00:00
override val version : Int
get ( ) = 5
2017-12-07 07:00:18 +00:00
2017-12-12 21:04:20 +00:00
@Throws ( SQLException :: class )
override fun _doUpdate ( ) {
stmt . executeUpdate ( " CREATE TABLE runs (id INTEGER PRIMARY KEY ASC, time INTEGER, start_id INTEGER, end_id INTEGER, count_missing INTEGER) " )
}
2017-12-07 07:00:18 +00:00
}
internal class DB _Update _6 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
2017-12-12 21:04:20 +00:00
override val version : Int
get ( ) = 6
override val needsBackup = true
@Throws ( SQLException :: class )
override fun _doUpdate ( ) {
stmt . executeUpdate (
" CREATE TABLE messages_new ( \n " +
" id INTEGER PRIMARY KEY ASC, \n " +
" message_type TEXT, \n " +
" dialog_id INTEGER, \n " +
" chat_id INTEGER, \n " +
" sender_id INTEGER, \n " +
" fwd_from_id INTEGER, \n " +
" text TEXT, \n " +
" time INTEGER, \n " +
" has_media BOOLEAN, \n " +
" media_type TEXT, \n " +
" media_file TEXT, \n " +
" media_size INTEGER, \n " +
" media_json TEXT, \n " +
" markup_json TEXT, \n " +
" data BLOB) " )
val mappings = LinkedHashMap < String , String > ( )
mappings . put ( " id " , " id " )
mappings . put ( " message_type " , " type " )
mappings . put ( " dialog_id " , " CASE from_type WHEN 'user' THEN dialog_id ELSE NULL END " )
mappings . put ( " chat_id " , " CASE from_type WHEN 'chat' THEN dialog_id ELSE NULL END " )
mappings . put ( " sender_id " , " from_id " )
mappings . put ( " text " , " text " )
mappings . put ( " time " , " time " )
mappings . put ( " has_media " , " has_media " )
mappings . put ( " data " , " data " )
val query = StringBuilder ( " INSERT INTO messages_new \n ( " )
var first : Boolean
first = true
for ( s in mappings . keys ) {
if ( ! first ) query . append ( " , " )
query . append ( s )
first = false
}
query . append ( " ) \n SELECT \n " )
first = true
for ( s in mappings . values ) {
if ( ! first ) query . append ( " , " )
query . append ( s )
first = false
}
query . append ( " \n FROM messages " )
stmt . executeUpdate ( query . toString ( ) )
System . out . println ( " Updating the data (this might take some time)... " )
val rs = stmt . executeQuery ( " SELECT id, data FROM messages_new " )
val ps = conn . prepareStatement ( " UPDATE messages_new SET fwd_from_id=?, media_type=?, media_file=?, media_size=? WHERE id=? " )
while ( rs . next ( ) ) {
ps . setInt ( 5 , rs . getInt ( 1 ) )
val msg = Database . bytesToTLMessage ( rs . getBytes ( 2 ) )
if ( msg == null || msg . getFwdFrom ( ) == null ) {
ps . setNull ( 1 , Types . INTEGER )
} else {
ps . setInt ( 1 , msg . getFwdFrom ( ) . getFromId ( ) )
}
2018-04-09 04:07:53 +00:00
val f = FileManagerFactory . getFileManager ( msg , db . user _manager , db . file _base , settings = null )
2017-12-12 21:04:20 +00:00
if ( f == null ) {
ps . setNull ( 2 , Types . VARCHAR )
ps . setNull ( 3 , Types . VARCHAR )
ps . setNull ( 4 , Types . INTEGER )
} else {
ps . setString ( 2 , f . name )
ps . setString ( 3 , f . targetFilename )
ps . setInt ( 4 , f . size )
}
ps . addBatch ( )
}
rs . close ( )
conn . setAutoCommit ( false )
ps . executeBatch ( )
2018-03-08 21:17:01 +00:00
ps . close ( )
2017-12-12 21:04:20 +00:00
conn . commit ( )
conn . setAutoCommit ( true )
stmt . executeUpdate ( " DROP TABLE messages " )
stmt . executeUpdate ( " ALTER TABLE messages_new RENAME TO messages " )
}
2017-12-07 07:00:18 +00:00
}
internal class DB _Update _7 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
2017-12-12 21:04:20 +00:00
override val version : Int
get ( ) = 7
2017-12-07 07:00:18 +00:00
2017-12-12 21:04:20 +00:00
override val needsBackup = true
2017-12-07 07:00:18 +00:00
2017-12-12 21:04:20 +00:00
@Throws ( SQLException :: class )
override fun _doUpdate ( ) {
stmt . executeUpdate ( " ALTER TABLE messages ADD COLUMN api_layer INTEGER " )
2017-12-07 07:00:18 +00:00
2017-12-12 21:04:20 +00:00
stmt . executeUpdate ( " UPDATE messages SET api_layer=51 " )
}
2017-12-07 07:00:18 +00:00
}
internal class DB _Update _8 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
2017-12-12 21:04:20 +00:00
override val version : Int
get ( ) = 8
override val needsBackup = true
@Throws ( SQLException :: class )
override fun _doUpdate ( ) {
execute ( " ALTER TABLE messages ADD COLUMN source_type TEXT " )
execute ( " ALTER TABLE messages ADD COLUMN source_id INTEGER " )
execute ( " update messages set source_type='dialog', source_id=dialog_id where dialog_id is not null " )
execute ( " update messages set source_type='group', source_id=chat_id where chat_id is not null " )
execute ( " CREATE TABLE messages_new ( " +
" id INTEGER PRIMARY KEY AUTOINCREMENT, " +
" message_id INTEGER, " +
" message_type TEXT, " +
" source_type TEXT, " +
" source_id INTEGER, " +
" sender_id INTEGER, " +
" fwd_from_id INTEGER, " +
" text TEXT, " +
" time INTEGER, " +
" has_media BOOLEAN, " +
" media_type TEXT, " +
" media_file TEXT, " +
" media_size INTEGER, " +
" media_json TEXT, " +
" markup_json TEXT, " +
" data BLOB, " +
" api_layer INTEGER) " )
execute ( " INSERT INTO messages_new " +
" (message_id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type, " +
" media_file, media_size, media_json, markup_json, data, api_layer) " +
" SELECT " +
" id, message_type, source_type, source_id, sender_id, fwd_from_id, text, time, has_media, media_type, " +
" media_file, media_size, media_json, markup_json, data, api_layer FROM messages " )
execute ( " DROP TABLE messages " )
execute ( " ALTER TABLE messages_new RENAME TO 'messages' " )
execute ( " CREATE UNIQUE INDEX unique_messages ON messages (source_type, source_id, message_id) " )
}
2017-12-07 07:00:18 +00:00
}
2018-02-21 05:31:39 +00:00
internal class DB _Update _9 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
override val version : Int
get ( ) = 9
override val needsBackup = true
2018-03-08 21:15:49 +00:00
override val create _query = listOf (
" CREATE TABLE \" chats \" (id INTEGER PRIMARY KEY ASC, name TEXT, type TEXT); " ,
" CREATE TABLE \" users \" (id INTEGER PRIMARY KEY ASC, first_name TEXT, last_name TEXT, username TEXT, type TEXT, phone TEXT); " ,
" CREATE TABLE database_versions (version INTEGER); " ,
" CREATE TABLE runs (id INTEGER PRIMARY KEY ASC, time INTEGER, start_id INTEGER, end_id INTEGER, count_missing INTEGER); " ,
" CREATE TABLE \" messages \" (id INTEGER PRIMARY KEY AUTOINCREMENT,message_id INTEGER,message_type TEXT,source_type TEXT,source_id INTEGER,sender_id INTEGER,fwd_from_id INTEGER,text TEXT,time INTEGER,has_media BOOLEAN,media_type TEXT,media_file TEXT,media_size INTEGER,media_json TEXT,markup_json TEXT,data BLOB,api_layer INTEGER); " ,
" CREATE UNIQUE INDEX unique_messages ON messages (source_type, source_id, message_id); "
)
2018-02-21 05:31:39 +00:00
@Throws ( SQLException :: class )
override fun _doUpdate ( ) {
val logger = LoggerFactory . getLogger ( DB _Update _9 :: class . java )
println ( " Updating supergroup channel message data (this might take some time)... " )
2018-03-06 05:20:02 +00:00
print ( " " )
2018-03-05 05:39:30 +00:00
val count = db . queryInt ( " SELECT COUNT(*) FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 " )
logger . debug ( " Found $count candidates for conversion " )
val limit = 5000
var offset = 0
2018-02-21 05:31:39 +00:00
var i = 0
2018-03-06 05:20:02 +00:00
while ( offset < count ) {
2018-03-05 05:39:30 +00:00
logger . debug ( " Querying with limit $limit and offset $offset " )
2018-03-06 05:20:02 +00:00
val rs = stmt . executeQuery ( " SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 ORDER BY id LIMIT ${limit} OFFSET ${offset} " )
2018-03-05 05:39:30 +00:00
val messages = TLVector < TLAbsMessage > ( )
val messages _to _delete = mutableListOf < Int > ( )
while ( rs . next ( ) ) {
val msg = Database . bytesToTLMessage ( rs . getBytes ( 2 ) )
if ( msg !! . getFromId ( ) != null ) {
i ++
messages . add ( msg )
messages _to _delete . add ( rs . getInt ( 1 ) )
}
2018-02-21 05:31:39 +00:00
}
2018-03-06 05:20:02 +00:00
rs . close ( )
2018-04-09 04:07:53 +00:00
db . saveMessages ( messages , api _layer = 53 , source _type = MessageSource . SUPERGROUP , settings = null )
2018-03-05 05:39:30 +00:00
execute ( " DELETE FROM messages WHERE id IN ( " + messages _to _delete . joinToString ( ) + " ) " )
2018-03-06 05:20:02 +00:00
print ( " . " )
2018-03-05 05:39:30 +00:00
offset += limit
2018-02-21 05:31:39 +00:00
}
2018-03-06 05:20:02 +00:00
println ( )
2018-03-05 05:39:30 +00:00
logger . info ( " Converted ${i} of ${count} messages. " )
2018-02-21 05:31:39 +00:00
}
}
2018-02-22 06:55:53 +00:00
internal class DB _Update _10 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
override val version : Int
get ( ) = 10
@Throws ( SQLException :: class )
override fun _doUpdate ( ) {
execute ( " CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT) " )
}
}
2018-04-12 04:54:14 +00:00
internal class DB _Update _11 ( conn : Connection , db : Database ) : DatabaseUpdate ( conn , db ) {
override val version = 11
val logger = LoggerFactory . getLogger ( DB _Update _11 :: class . java )
override fun _doUpdate ( ) {
execute ( " ALTER TABLE messages ADD COLUMN json TEXT NULL " )
val limit = 5000
var offset = 0
var i = 0
val ps = conn . prepareStatement ( " UPDATE messages SET json=? WHERE id=? " )
do {
i = 0
logger . debug ( " Querying with limit $limit and offset $offset " )
val rs = db . executeQuery ( " SELECT id, data FROM messages WHERE json IS NULL AND api_layer=53 LIMIT $limit " )
while ( rs . next ( ) ) {
i ++
val id = rs . getInt ( 1 )
val msg = Database . bytesToTLMessage ( rs . getBytes ( 2 ) )
if ( msg == null ) continue
val json = msg . toJson ( )
ps . setString ( 1 , json )
ps . setInt ( 2 , id )
ps . addBatch ( )
}
rs . close ( )
conn . setAutoCommit ( false )
ps . executeBatch ( )
ps . clearBatch ( )
conn . commit ( )
conn . setAutoCommit ( true )
print ( " . " )
} while ( i >= limit )
ps . close ( )
}
}