fabian
/
telegram_backup
mirror of https://github.com/fabianonline/telegram_backup.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Added limit and offset parameters to Database#getMessagesWithMedia in order to prevent high memory usage in downloadMedia.

This commit is contained in:
Fabian Schlenz 2018-03-15 20:49:15 +01:00
parent 968ee831f0
commit f4d563226c
2 changed files with 43 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/kotlin/de/fabianonline/telegram_backup/Database.kt
View File

@ -98,10 +98,14 @@ class Database private constructor(var client: TelegramClient) {
}
fun getMessagesWithMedia(): LinkedList<TLMessage?> {
fun getMessagesWithMediaCount() = queryInt("SELECT COUNT(*) FROM messages WHERE has_media=1")
fun getMessagesWithMedia(limit: Int = 0, offset: Int = 0): LinkedList<TLMessage?> {
try {
val list = LinkedList<TLMessage?>()
val rs = stmt!!.executeQuery("SELECT data FROM messages WHERE has_media=1")
var query = "SELECT data FROM messages WHERE has_media=1 ORDER BY id"
if (limit > 0) query += " LIMIT ${limit} OFFSET ${offset}"
val rs = stmt!!.executeQuery(query)
while (rs.next()) {
list.add(bytesToTLMessage(rs.getBytes(1)))
}

66
src/main/kotlin/de/fabianonline/telegram_backup/DownloadManager.kt
View File

@ -309,38 +309,46 @@ class DownloadManager(internal var client: TelegramClient?, p: DownloadProgressI
logger.debug("Found {} messages", ids.size)
downloadMessages(ids, null, source_type=MessageSource.NORMAL)
}
val messages = this.db!!.getMessagesWithMedia()
logger.debug("Database returned {} messages with media", messages.size)
prog!!.onMediaDownloadStart(messages.size)
for (msg in messages) {
if (msg == null) continue
val m = FileManagerFactory.getFileManager(msg, user!!, client!!)
logger.trace("message {}, {}, {}, {}, {}",
msg.getId(),
msg.getMedia().javaClass.getSimpleName().replace("TLMessageMedia", "…"),
m!!.javaClass.getSimpleName(),
if (m.isEmpty) "empty" else "non-empty",
if (m.downloaded) "downloaded" else "not downloaded")
if (m.isEmpty) {
prog!!.onMediaDownloadedEmpty()
} else if (m.downloaded) {
prog!!.onMediaAlreadyPresent(m)
} else if (IniSettings.max_file_age!=null && (System.currentTimeMillis() / 1000) - msg.date > IniSettings.max_file_age * 24 * 60 * 60) {
prog!!.onMediaTooOld()
} else {
try {
val result = m.download()
if (result) {
prog!!.onMediaDownloaded(m)
} else {
var offset = 0
val limit = 1000
val message_count = this.db!!.getMessagesWithMediaCount()
prog!!.onMediaDownloadStart(message_count)
while (true) {
logger.debug("Querying messages with media, limit={}, offset={}", limit, offset)
val messages = this.db!!.getMessagesWithMedia(limit=limit, offset=offset)
if (messages.size == 0) break
offset += limit
logger.debug("Database returned {} messages with media", messages.size)
for (msg in messages) {
if (msg == null) continue
val m = FileManagerFactory.getFileManager(msg, user!!, client!!)
logger.trace("message {}, {}, {}, {}, {}",
msg.getId(),
msg.getMedia().javaClass.getSimpleName().replace("TLMessageMedia", "…"),
m!!.javaClass.getSimpleName(),
if (m.isEmpty) "empty" else "non-empty",
if (m.downloaded) "downloaded" else "not downloaded")
if (m.isEmpty) {
prog!!.onMediaDownloadedEmpty()
} else if (m.downloaded) {
prog!!.onMediaAlreadyPresent(m)
} else if (IniSettings.max_file_age!=null && (System.currentTimeMillis() / 1000) - msg.date > IniSettings.max_file_age * 24 * 60 * 60) {
prog!!.onMediaTooOld()
} else {
try {
val result = m.download()
if (result) {
prog!!.onMediaDownloaded(m)
} else {
prog!!.onMediaSkipped()
}
} catch (e: TimeoutException) {
// do nothing - skip this file
prog!!.onMediaSkipped()
}
} catch (e: TimeoutException) {
// do nothing - skip this file
prog!!.onMediaSkipped()
}
}
}
prog!!.onMediaDownloadFinished()