fabian/telegram_backup
1
0
Fork 0
mirror of https://github.com/fabianonline/telegram_backup.git synced 2024-11-22 16:56:16 +00:00
Code Issues Releases Wiki Activity

Added limit and offset parameters to Database#getMessagesWithMedia in order to prevent high memory usage in downloadMedia.

Browse Source
This commit is contained in:
Fabian Schlenz 2018-03-15 20:49:15 +01:00
parent 968ee831f0
commit f4d563226c
2 changed files with 43 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/kotlin/de/fabianonline/telegram_backup/Database.kt
View File

@ -98,10 +98,14 @@ class Database private constructor(var client: TelegramClient) {
} }
fun getMessagesWithMedia(): LinkedList<TLMessage?> { fun getMessagesWithMediaCount() = queryInt("SELECT COUNT(*) FROM messages WHERE has_media=1")
fun getMessagesWithMedia(limit: Int = 0, offset: Int = 0): LinkedList<TLMessage?> {
try { try {
val list = LinkedList<TLMessage?>() val list = LinkedList<TLMessage?>()
val rs = stmt!!.executeQuery("SELECT data FROM messages WHERE has_media=1") var query = "SELECT data FROM messages WHERE has_media=1 ORDER BY id"
if (limit > 0) query += " LIMIT ${limit} OFFSET ${offset}"
val rs = stmt!!.executeQuery(query)
while (rs.next()) { while (rs.next()) {
list.add(bytesToTLMessage(rs.getBytes(1))) list.add(bytesToTLMessage(rs.getBytes(1)))
} }

64
src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt
View File

@ -310,37 +310,45 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
downloadMessages(ids, null, source_type=MessageSource.NORMAL) downloadMessages(ids, null, source_type=MessageSource.NORMAL)
} }
val messages = this.db!!.getMessagesWithMedia() var offset = 0
logger.debug("Database returned {} messages with media", messages.size) val limit = 1000
prog!!.onMediaDownloadStart(messages.size) val message_count = this.db!!.getMessagesWithMediaCount()
for (msg in messages) { prog!!.onMediaDownloadStart(message_count)
if (msg == null) continue while (true) {
val m = FileManagerFactory.getFileManager(msg, user!!, client!!) logger.debug("Querying messages with media, limit={}, offset={}", limit, offset)
logger.trace("message {}, {}, {}, {}, {}", val messages = this.db!!.getMessagesWithMedia(limit=limit, offset=offset)
msg.getId(), if (messages.size == 0) break
msg.getMedia().javaClass.getSimpleName().replace("TLMessageMedia", "…"), offset += limit
m!!.javaClass.getSimpleName(), logger.debug("Database returned {} messages with media", messages.size)
if (m.isEmpty) "empty" else "non-empty",
if (m.downloaded) "downloaded" else "not downloaded") for (msg in messages) {
if (m.isEmpty) { if (msg == null) continue
prog!!.onMediaDownloadedEmpty() val m = FileManagerFactory.getFileManager(msg, user!!, client!!)
} else if (m.downloaded) { logger.trace("message {}, {}, {}, {}, {}",
prog!!.onMediaAlreadyPresent(m) msg.getId(),
} else if (IniSettings.max_file_age!=null && (System.currentTimeMillis() / 1000) - msg.date > IniSettings.max_file_age * 24 * 60 * 60) { msg.getMedia().javaClass.getSimpleName().replace("TLMessageMedia", "…"),
prog!!.onMediaTooOld() m!!.javaClass.getSimpleName(),
} else { if (m.isEmpty) "empty" else "non-empty",
try { if (m.downloaded) "downloaded" else "not downloaded")
val result = m.download() if (m.isEmpty) {
if (result) { prog!!.onMediaDownloadedEmpty()
prog!!.onMediaDownloaded(m) } else if (m.downloaded) {
} else { prog!!.onMediaAlreadyPresent(m)
} else if (IniSettings.max_file_age!=null && (System.currentTimeMillis() / 1000) - msg.date > IniSettings.max_file_age * 24 * 60 * 60) {
prog!!.onMediaTooOld()
} else {
try {
val result = m.download()
if (result) {
prog!!.onMediaDownloaded(m)
} else {
prog!!.onMediaSkipped()
}
} catch (e: TimeoutException) {
// do nothing - skip this file
prog!!.onMediaSkipped() prog!!.onMediaSkipped()
} }
} catch (e: TimeoutException) {
// do nothing - skip this file
prog!!.onMediaSkipped()
} }
} }
} }
prog!!.onMediaDownloadFinished() prog!!.onMediaDownloadFinished()