fabian/telegram_backup
1
0
Fork 0
mirror of https://github.com/fabianonline/telegram_backup.git synced 2024-11-22 16:56:16 +00:00
Code Issues Releases Wiki Activity

DB_Update_9 now uses chunks of messages to (massively) reduce the amount of heap space needed.

Browse Source
This commit is contained in:
Fabian Schlenz 2018-03-05 06:39:30 +01:00
parent f5431e805e
commit eb1f731b9d
1 changed files with 21 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt
View File

@ -381,21 +381,30 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
override fun _doUpdate() { override fun _doUpdate() {
val logger = LoggerFactory.getLogger(DB_Update_9::class.java) val logger = LoggerFactory.getLogger(DB_Update_9::class.java)
println(" Updating supergroup channel message data (this might take some time)...") println(" Updating supergroup channel message data (this might take some time)...")
val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53") val count = db.queryInt("SELECT COUNT(*) FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53")
val messages = TLVector<TLAbsMessage>() logger.debug("Found $count candidates for conversion")
val messages_to_delete = mutableListOf<Int>() val limit = 5000
var offset = 0
var i = 0 var i = 0
while (rs.next()) { while (offset + 1 < count) {
i++ logger.debug("Querying with limit $limit and offset $offset")
val msg = Database.bytesToTLMessage(rs.getBytes(2)) val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 LIMIT ${limit} OFFSET ${offset}")
if (msg!!.getFromId() != null) { val messages = TLVector<TLAbsMessage>()
messages.add(msg) val messages_to_delete = mutableListOf<Int>()
messages_to_delete.add(rs.getInt(1)) while (rs.next()) {
val msg = Database.bytesToTLMessage(rs.getBytes(2))
if (msg!!.getFromId() != null) {
i++
messages.add(msg)
messages_to_delete.add(rs.getInt(1))
}
} }
db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP)
execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")")
offset += limit
} }
logger.info("Converted ${messages.size} of ${i} messages.") logger.info("Converted ${i} of ${count} messages.")
db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP)
execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")")
println(" Cleaning up the database (this might also take some time, sorry)...") println(" Cleaning up the database (this might also take some time, sorry)...")
execute("VACUUM") execute("VACUUM")
} }