fabian/telegram_backup
1
0
Fork 0
mirror of https://github.com/fabianonline/telegram_backup.git synced 2024-11-23 01:06:17 +00:00
Code Issues Releases Wiki Activity

DB_Update_9 now uses chunks of messages to (massively) reduce the amount of heap space needed.

Browse Source
This commit is contained in:
Fabian Schlenz 2018-03-05 06:39:30 +01:00
parent f5431e805e
commit eb1f731b9d
1 changed files with 21 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/main/kotlin/de/fabianonline/telegram_backup/DatabaseUpdates.kt
View File

@ -381,21 +381,30 @@ internal class DB_Update_9(conn: Connection, db: Database) : DatabaseUpdate(conn
override fun _doUpdate() { override fun _doUpdate() {
val logger = LoggerFactory.getLogger(DB_Update_9::class.java) val logger = LoggerFactory.getLogger(DB_Update_9::class.java)
println(" Updating supergroup channel message data (this might take some time)...") println(" Updating supergroup channel message data (this might take some time)...")
val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53") val count = db.queryInt("SELECT COUNT(*) FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53")
logger.debug("Found $count candidates for conversion")
val limit = 5000
var offset = 0
var i = 0
while (offset + 1 < count) {
logger.debug("Querying with limit $limit and offset $offset")
val rs = stmt.executeQuery("SELECT id, data, source_id FROM messages WHERE source_type='channel' and sender_id IS NULL and api_layer=53 LIMIT ${limit} OFFSET ${offset}")
val messages = TLVector<TLAbsMessage>() val messages = TLVector<TLAbsMessage>()
val messages_to_delete = mutableListOf<Int>() val messages_to_delete = mutableListOf<Int>()
var i = 0
while (rs.next()) { while (rs.next()) {
i++
val msg = Database.bytesToTLMessage(rs.getBytes(2)) val msg = Database.bytesToTLMessage(rs.getBytes(2))
if (msg!!.getFromId() != null) { if (msg!!.getFromId() != null) {
i++
messages.add(msg) messages.add(msg)
messages_to_delete.add(rs.getInt(1)) messages_to_delete.add(rs.getInt(1))
} }
} }
logger.info("Converted ${messages.size} of ${i} messages.")
db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP) db.saveMessages(messages, api_layer=53, source_type=MessageSource.SUPERGROUP)
execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")") execute("DELETE FROM messages WHERE id IN (" + messages_to_delete.joinToString() + ")")
offset += limit
}
logger.info("Converted ${i} of ${count} messages.")
println(" Cleaning up the database (this might also take some time, sorry)...") println(" Cleaning up the database (this might also take some time, sorry)...")
execute("VACUUM") execute("VACUUM")
} }