mirror/mihon
1
0
Fork 0
mirror of https://github.com/mihonapp/mihon.git synced 2024-12-24 01:48:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

Implement a download cache

Browse Source
This commit is contained in:
inorichi 2017-11-28 00:32:51 +01:00
parent 604929d002
commit bff329a329
10 changed files with 327 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

253
app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt Normal file
View File

@ -0,0 +1,253 @@
package eu.kanade.tachiyomi.data.download
import android.content.Context
import android.net.Uri
import com.hippo.unifile.UniFile
import eu.kanade.tachiyomi.data.database.models.Chapter
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.source.SourceManager
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.util.concurrent.TimeUnit
/**
* Cache where we dump the downloads directory from the filesystem. This class is needed because
* directory checking is expensive and it slowdowns the app. The cache is invalidated by the time
* defined in [renewInterval] as we don't have any control over the filesystem and the user can
* delete the folders at any time without the app noticing.
*
* @param context the application context.
* @param provider the downloads directories provider.
* @param sourceManager the source manager.
* @param preferences the preferences of the app.
*/
class DownloadCache(private val context: Context,
private val provider: DownloadProvider,
private val sourceManager: SourceManager = Injekt.get(),
preferences: PreferencesHelper = Injekt.get()) {
/**
* The interval after which this cache should be invalidated. 1 hour shouldn't cause major
* issues, as the cache is only used for UI feedback.
*/
private val renewInterval = TimeUnit.HOURS.toMillis(1)
/**
* The last time the cache was refreshed.
*/
private var lastRenew = 0L
/**
* The root directory for downloads.
*/
private var rootDir = setRootDir(preferences.downloadsDirectory().getOrDefault())
init {
setRootDir(preferences.downloadsDirectory().getOrDefault())
preferences.downloadsDirectory().asObservable()
.skip(1)
.subscribe { setRootDir(it) }
}
/**
* Sets the root downloads directory and invalidates the cache.
*
* @param directory the downloads directory in [Uri] format.
*/
private fun setRootDir(directory: String): RootDirectory {
rootDir = RootDirectory(UniFile.fromUri(context, Uri.parse(directory)))
lastRenew = 0L
return rootDir
}
/**
* Returns true if the chapter is downloaded.
*
* @param chapter the chapter to check.
* @param manga the manga of the chapter.
* @param skipCache whether to skip the directory cache and check in the filesystem.
*/
fun isChapterDownloaded(chapter: Chapter, manga: Manga, skipCache: Boolean): Boolean {
if (skipCache) {
val source = sourceManager.get(manga.source) ?: return false
return provider.findChapterDir(chapter, manga, source) != null
}
checkRenew()
val sourceDir = rootDir.files[manga.source]
if (sourceDir != null) {
val mangaDir = sourceDir.files[provider.getMangaDirName(manga)]
if (mangaDir != null) {
return provider.getChapterDirName(chapter) in mangaDir.files
}
}
return false
}
/**
* Returns the amount of downloaded chapters for a manga.
*
* @param manga the manga to check.
*/
fun getDownloadCount(manga: Manga): Int {
checkRenew()
val sourceDir = rootDir.files[manga.source]
if (sourceDir != null) {
val mangaDir = sourceDir.files[provider.getMangaDirName(manga)]
if (mangaDir != null) {
return mangaDir.files.size
}
}
return 0
}
/**
* Checks if the cache needs a renewal and performs it if needed.
*/
@Synchronized
private fun checkRenew() {
if (lastRenew + renewInterval < System.currentTimeMillis()) {
renew()
lastRenew = System.currentTimeMillis()
}
}
/**
* Renews the downloads cache.
*/
private fun renew() {
val onlineSources = sourceManager.getOnlineSources()
val sourceDirs = rootDir.dir.listFiles()
.orEmpty()
.associate { it.name to SourceDirectory(it) }
.mapNotNullKeys { entry ->
onlineSources.find { provider.getSourceDirName(it) == entry.key }?.id
}
rootDir.files = sourceDirs
sourceDirs.values.forEach { sourceDir ->
val mangaDirs = sourceDir.dir.listFiles()
.orEmpty()
.associateNotNullKeys { it.name to MangaDirectory(it) }
sourceDir.files = mangaDirs
mangaDirs.values.forEach { mangaDir ->
val chapterDirs = mangaDir.dir.listFiles()
.orEmpty()
.mapNotNull { it.name }
.toHashSet()
mangaDir.files = chapterDirs
}
}
}
/**
* Adds a chapter that has just been download to this cache.
*
* @param chapterDirName the downloaded chapter's directory name.
* @param mangaUniFile the directory of the manga.
* @param manga the manga of the chapter.
*/
@Synchronized
fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) {
// Retrieve the cached source directory or cache a new one
var sourceDir = rootDir.files[manga.source]
if (sourceDir == null) {
val source = sourceManager.get(manga.source) ?: return
val sourceUniFile = provider.findSourceDir(source) ?: return
sourceDir = SourceDirectory(sourceUniFile)
rootDir.files += manga.source to sourceDir
}
// Retrieve the cached manga directory or cache a new one
val mangaDirName = provider.getMangaDirName(manga)
var mangaDir = sourceDir.files[mangaDirName]
if (mangaDir == null) {
mangaDir = MangaDirectory(mangaUniFile)
sourceDir.files += mangaDirName to mangaDir
}
// Save the chapter directory
mangaDir.files += chapterDirName
}
/**
* Removes a chapter that has been deleted from this cache.
*
* @param chapter the chapter to remove.
* @param manga the manga of the chapter.
*/
@Synchronized
fun removeChapter(chapter: Chapter, manga: Manga) {
val sourceDir = rootDir.files[manga.source] ?: return
val mangaDir = sourceDir.files[provider.getMangaDirName(manga)] ?: return
val chapterDirName = provider.getChapterDirName(chapter)
if (chapterDirName in mangaDir.files) {
mangaDir.files -= chapterDirName
}
}
/**
* Removes a manga that has been deleted from this cache.
*
* @param manga the manga to remove.
*/
@Synchronized
fun removeManga(manga: Manga) {
val sourceDir = rootDir.files[manga.source] ?: return
val mangaDirName = provider.getMangaDirName(manga)
if (mangaDirName in sourceDir.files) {
sourceDir.files -= mangaDirName
}
}
/**
* Class to store the files under the root downloads directory.
*/
private class RootDirectory(val dir: UniFile,
var files: Map<Long, SourceDirectory> = hashMapOf())
/**
* Class to store the files under a source directory.
*/
private class SourceDirectory(val dir: UniFile,
var files: Map<String, MangaDirectory> = hashMapOf())
/**
* Class to store the files under a manga directory.
*/
private class MangaDirectory(val dir: UniFile,
var files: Set<String> = hashSetOf())
/**
* Returns a new map containing only the key entries of [transform] that are not null.
*/
private inline fun <K, V, R> Map<out K, V>.mapNotNullKeys(transform: (Map.Entry<K?, V>) -> R?): Map<R, V> {
val destination = LinkedHashMap<R, V>()
forEach { element -> transform(element)?.let { destination.put(it, element.value) } }
return destination
}
/**
* Returns a map from a list containing only the key entries of [transform] that are not null.
*/
private inline fun <T, K, V> Array<T>.associateNotNullKeys(transform: (T) -> Pair<K?, V>): Map<K, V> {
val destination = LinkedHashMap<K, V>()
for (element in this) {
val (key, value) = transform(element)
if (key != null) {
destination.put(key, value)
}
}
return destination
}
}

85
app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt
View File

@ -24,10 +24,15 @@ class DownloadManager(context: Context) {
*/
private val provider = DownloadProvider(context)
/**
* Cache of downloaded chapters.
*/
private val cache = DownloadCache(context, provider)
/**
* Downloader whose only task is to download chapters.
*/
private val downloader = Downloader(context, provider)
private val downloader = Downloader(context, provider, cache)
/**
* Downloads queue, where the pending chapters are stored.
@ -94,7 +99,7 @@ class DownloadManager(context: Context) {
* @return an observable containing the list of pages from the chapter.
*/
fun buildPageList(source: Source, manga: Manga, chapter: Chapter): Observable<List<Page>> {
return buildPageList(provider.findChapterDir(source, manga, chapter))
return buildPageList(provider.findChapterDir(chapter, manga, source))
}
/**
@ -120,61 +125,45 @@ class DownloadManager(context: Context) {
}
/**
* Returns the directory name for a manga.
* Returns true if the chapter is downloaded.
*
* @param manga the manga to query.
*/
fun getMangaDirName(manga: Manga): String {
return provider.getMangaDirName(manga)
}
/**
* Returns the directory name for the given chapter.
*
* @param chapter the chapter to query.
*/
fun getChapterDirName(chapter: Chapter): String {
return provider.getChapterDirName(chapter)
}
/**
* Returns the download directory for a source if it exists.
*
* @param source the source to query.
*/
fun findSourceDir(source: Source): UniFile? {
return provider.findSourceDir(source)
}
/**
* Returns the directory for the given manga, if it exists.
*
* @param source the source of the manga.
* @param manga the manga to query.
*/
fun findMangaDir(source: Source, manga: Manga): UniFile? {
return provider.findMangaDir(source, manga)
}
/**
* Returns the directory for the given chapter, if it exists.
*
* @param source the source of the chapter.
* @param chapter the chapter to check.
* @param manga the manga of the chapter.
* @param chapter the chapter to query.
* @param skipCache whether to skip the directory cache and check in the filesystem.
*/
fun findChapterDir(source: Source, manga: Manga, chapter: Chapter): UniFile? {
return provider.findChapterDir(source, manga, chapter)
fun isChapterDownloaded(chapter: Chapter, manga: Manga, skipCache: Boolean = false): Boolean {
return cache.isChapterDownloaded(chapter, manga, skipCache)
}
/**
* Returns the amount of downloaded chapters for a manga.
*
* @param manga the manga to check.
*/
fun getDownloadCount(manga: Manga): Int {
return cache.getDownloadCount(manga)
}
/**
* Deletes the directory of a downloaded chapter.
*
* @param source the source of the chapter.
* @param manga the manga of the chapter.
* @param chapter the chapter to delete.
* @param manga the manga of the chapter.
* @param source the source of the chapter.
*/
fun deleteChapter(source: Source, manga: Manga, chapter: Chapter) {
provider.findChapterDir(source, manga, chapter)?.delete()
fun deleteChapter(chapter: Chapter, manga: Manga, source: Source) {
provider.findChapterDir(chapter, manga, source)?.delete()
cache.removeChapter(chapter, manga)
}
/**
* Deletes the directory of a downloaded manga.
*
* @param manga the manga to delete.
* @param source the source of the manga.
*/
fun deleteManga(manga: Manga, source: Source) {
provider.findMangaDir(manga, source)?.delete()
cache.removeManga(manga)
}
}

16
app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadProvider.kt
View File

@ -40,10 +40,10 @@ class DownloadProvider(private val context: Context) {
/**
* Returns the download directory for a manga. For internal use only.
*
* @param source the source of the manga.
* @param manga the manga to query.
* @param source the source of the manga.
*/
internal fun getMangaDir(source: Source, manga: Manga): UniFile {
internal fun getMangaDir(manga: Manga, source: Source): UniFile {
return downloadsDir
.createDirectory(getSourceDirName(source))
.createDirectory(getMangaDirName(manga))
@ -61,10 +61,10 @@ class DownloadProvider(private val context: Context) {
/**
* Returns the download directory for a manga if it exists.
*
* @param source the source of the manga.
* @param manga the manga to query.
* @param source the source of the manga.
*/
fun findMangaDir(source: Source, manga: Manga): UniFile? {
fun findMangaDir(manga: Manga, source: Source): UniFile? {
val sourceDir = findSourceDir(source)
return sourceDir?.findFile(getMangaDirName(manga))
}
@ -72,12 +72,12 @@ class DownloadProvider(private val context: Context) {
/**
* Returns the download directory for a chapter if it exists.
*
* @param source the source of the chapter.
* @param manga the manga of the chapter.
* @param chapter the chapter to query.
* @param manga the manga of the chapter.
* @param source the source of the chapter.
*/
fun findChapterDir(source: Source, manga: Manga, chapter: Chapter): UniFile? {
val mangaDir = findMangaDir(source, manga)
fun findChapterDir(chapter: Chapter, manga: Manga, source: Source): UniFile? {
val mangaDir = findMangaDir(manga, source)
return mangaDir?.findFile(getChapterDirName(chapter))
}

17
app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt
View File

@ -37,8 +37,11 @@ import uy.kohesive.injekt.injectLazy
*
* @param context the application context.
* @param provider the downloads directory provider.
* @param cache the downloads cache, used to add the downloads to the cache after their completion.
*/
class Downloader(private val context: Context, private val provider: DownloadProvider) {
class Downloader(private val context: Context,
private val provider: DownloadProvider,
private val cache: DownloadCache) {
/**
* Store for persisting downloads across restarts.
@ -222,7 +225,7 @@ class Downloader(private val context: Context, private val provider: DownloadPro
// Called in background thread, the operation can be slow with SAF.
val chaptersWithoutDir = async {
val mangaDir = provider.findMangaDir(source, manga)
val mangaDir = provider.findMangaDir(manga, source)
chapters
// Avoid downloading chapters with the same name.
@ -269,7 +272,7 @@ class Downloader(private val context: Context, private val provider: DownloadPro
*/
private fun downloadChapter(download: Download): Observable<Download> {
val chapterDirname = provider.getChapterDirName(download.chapter)
val mangaDir = provider.getMangaDir(download.source, download.manga)
val mangaDir = provider.getMangaDir(download.manga, download.source)
val tmpDir = mangaDir.createDirectory("${chapterDirname}_tmp")
val pageListObservable = if (download.pages == null) {
@ -305,7 +308,7 @@ class Downloader(private val context: Context, private val provider: DownloadPro
.toList()
.map { _ -> download }
// Do after download completes
.doOnNext { ensureSuccessfulDownload(download, tmpDir, chapterDirname) }
.doOnNext { ensureSuccessfulDownload(download, mangaDir, tmpDir, chapterDirname) }
// If the page list threw, it will resume here
.onErrorReturn { error ->
download.status = Download.ERROR
@ -411,10 +414,13 @@ class Downloader(private val context: Context, private val provider: DownloadPro
* Checks if the download was successful.
*
* @param download the download to check.
* @param mangaDir the manga directory of the download.
* @param tmpDir the directory where the download is currently stored.
* @param dirname the real (non temporary) directory name of the download.
*/
private fun ensureSuccessfulDownload(download: Download, tmpDir: UniFile, dirname: String) {
private fun ensureSuccessfulDownload(download: Download, mangaDir: UniFile,
tmpDir: UniFile, dirname: String) {
// Ensure that the chapter folder has all the images.
val downloadedImages = tmpDir.listFiles().orEmpty().filterNot { it.name!!.endsWith(".tmp") }
@ -427,6 +433,7 @@ class Downloader(private val context: Context, private val provider: DownloadPro
// Only rename the directory if it's downloaded.
if (download.status == Download.DOWNLOADED) {
tmpDir.renameTo(dirname)
cache.addChapter(dirname, mangaDir, download.manga)
}
}

53
app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryPresenter.kt
View File

@ -1,7 +1,6 @@
package eu.kanade.tachiyomi.ui.library
import android.os.Bundle
import com.hippo.unifile.UniFile
import com.jakewharton.rxrelay.BehaviorRelay
import eu.kanade.tachiyomi.data.cache.CoverCache
import eu.kanade.tachiyomi.data.database.DatabaseHelper
@ -107,12 +106,6 @@ class LibraryPresenter(
* @param map the map to filter.
*/
private fun applyFilters(map: LibraryMap): LibraryMap {
// Cached list of downloaded manga directories given a source id.
val mangaDirsForSource = mutableMapOf<Long, Map<String?, UniFile>>()
// Cached list of downloaded chapter directories for a manga.
val chapterDirectories = mutableMapOf<Long, Boolean>()
val filterDownloaded = preferences.filterDownloaded().getOrDefault()
val filterUnread = preferences.filterUnread().getOrDefault()
@ -121,7 +114,7 @@ class LibraryPresenter(
val filterFn: (LibraryItem) -> Boolean = f@ { item ->
// Filter out manga without source.
val source = sourceManager.get(item.manga.source) ?: return@f false
sourceManager.get(item.manga.source) ?: return@f false
// Filter when there isn't unread chapters.
if (filterUnread && item.manga.unread == 0) {
@ -132,28 +125,14 @@ class LibraryPresenter(
return@f false
}
// Filter when the download directory doesn't exist or is null.
// Filter when there are no downloads.
if (filterDownloaded) {
// Don't bother with directory checking if download count has been set.
if (item.downloadCount != -1) {
return@f item.downloadCount > 0
}
// Get the directories for the source of the manga.
val dirsForSource = mangaDirsForSource.getOrPut(source.id) {
val sourceDir = downloadManager.findSourceDir(source)
sourceDir?.listFiles()?.associateBy { it.name }.orEmpty()
}
val mangaDirName = downloadManager.getMangaDirName(item.manga)
val mangaDir = dirsForSource[mangaDirName] ?: return@f false
val hasDirs = chapterDirectories.getOrPut(item.manga.id!!) {
mangaDir.listFiles()?.isNotEmpty() ?: false
}
if (!hasDirs) {
return@f false
}
return@f downloadManager.getDownloadCount(item.manga) > 0
}
true
}
@ -177,31 +156,9 @@ class LibraryPresenter(
return
}
// Cached list of downloaded manga directories given a source id.
val mangaDirsForSource = mutableMapOf<Long, Map<String?, UniFile>>()
// Cached list of downloaded chapter directories for a manga.
val chapterDirectories = mutableMapOf<Long, Int>()
val downloadCountFn: (LibraryItem) -> Int = f@ { item ->
val source = sourceManager.get(item.manga.source) ?: return@f 0
// Get the directories for the source of the manga.
val dirsForSource = mangaDirsForSource.getOrPut(source.id) {
val sourceDir = downloadManager.findSourceDir(source)
sourceDir?.listFiles()?.associateBy { it.name }.orEmpty()
}
val mangaDirName = downloadManager.getMangaDirName(item.manga)
val mangaDir = dirsForSource[mangaDirName] ?: return@f 0
chapterDirectories.getOrPut(item.manga.id!!) {
mangaDir.listFiles()?.size ?: 0
}
}
for ((_, itemList) in map) {
for (item in itemList) {
item.downloadCount = downloadCountFn(item)
item.downloadCount = downloadManager.getDownloadCount(item.manga)
}
}
}
@ -360,7 +317,7 @@ class LibraryPresenter(
if (deleteChapters) {
val source = sourceManager.get(manga.source) as? HttpSource
if (source != null) {
downloadManager.findMangaDir(source, manga)?.delete()
downloadManager.deleteManga(manga, source)
}
}
}

14
app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt
View File

@ -128,13 +128,11 @@ class ChaptersPresenter(
* @param chapters the list of chapter from the database.
*/
private fun setDownloadedChapters(chapters: List<ChapterItem>) {
val files = downloadManager.findMangaDir(source, manga)?.listFiles() ?: return
val cached = mutableMapOf<Chapter, String>()
files.mapNotNull { it.name }
.mapNotNull { name -> chapters.find {
name == cached.getOrPut(it) { downloadManager.getChapterDirName(it) }
} }
.forEach { it.status = Download.DOWNLOADED }
for (chapter in chapters) {
if (downloadManager.isChapterDownloaded(chapter, manga)) {
chapter.status = Download.DOWNLOADED
}
}
}
/**
@ -283,7 +281,7 @@ class ChaptersPresenter(
*/
private fun deleteChapter(chapter: ChapterItem) {
downloadManager.queue.remove(chapter)
downloadManager.deleteChapter(source, manga, chapter)
downloadManager.deleteChapter(chapter, manga, source)
chapter.status = Download.NOT_DOWNLOADED
chapter.download = null
}

4
app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoPresenter.kt
View File

@ -115,14 +115,14 @@ class MangaInfoPresenter(
* Returns true if the manga has any downloads.
*/
fun hasDownloads(): Boolean {
return downloadManager.findMangaDir(source, manga) != null
return downloadManager.getDownloadCount(manga) > 0
}
/**
* Deletes all the downloads for the manga.
*/
fun deleteDownloads() {
downloadManager.findMangaDir(source, manga)?.delete()
downloadManager.deleteManga(manga, source)
}
/**

2
app/src/main/java/eu/kanade/tachiyomi/ui/reader/ChapterLoader.kt
View File

@ -79,7 +79,7 @@ class ChapterLoader(
private fun retrievePageList(chapter: ReaderChapter) = Observable.just(chapter)
.flatMap {
// Check if the chapter is downloaded.
chapter.isDownloaded = downloadManager.findChapterDir(source, manga, chapter) != null
chapter.isDownloaded = downloadManager.isChapterDownloaded(chapter, manga, true)
if (chapter.isDownloaded) {
// Fetch the page list from disk.

2
app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt
View File

@ -411,7 +411,7 @@ class ReaderPresenter(
fun deleteChapter(chapter: ReaderChapter, manga: Manga) {
chapter.isDownloaded = false
chapter.pages?.forEach { it.status == Page.QUEUE }
downloadManager.deleteChapter(source, manga, chapter)
downloadManager.deleteChapter(chapter, manga, source)
}
/**

30
app/src/main/java/eu/kanade/tachiyomi/ui/recent_updates/RecentChaptersPresenter.kt
View File

@ -1,7 +1,6 @@
package eu.kanade.tachiyomi.ui.recent_updates
import android.os.Bundle
import com.hippo.unifile.UniFile
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.MangaChapter
import eu.kanade.tachiyomi.data.download.DownloadManager
@ -114,36 +113,11 @@ class RecentChaptersPresenter(
* @param items the list of chapter from the database.
*/
private fun setDownloadedChapters(items: List<RecentChapterItem>) {
// Cached list of downloaded manga directories. Directory name is also cached because
// it's slow when using SAF.
val mangaDirsForSource = mutableMapOf<Long, Map<String?, UniFile>>()
// Cached list of downloaded chapter directories for a manga.
val chapterDirsForManga = mutableMapOf<Long, Map<String?, UniFile>>()
for (item in items) {
val manga = item.manga
val chapter = item.chapter
val source = sourceManager.get(manga.source) ?: continue
// Get the directories for the source of the manga.
val dirsForSource = mangaDirsForSource.getOrPut(source.id) {
val sourceDir = downloadManager.findSourceDir(source)
sourceDir?.listFiles()?.associateBy { it.name }.orEmpty()
}
// Get the manga directory in the source or continue.
val mangaDirName = downloadManager.getMangaDirName(manga)
val mangaDir = dirsForSource[mangaDirName] ?: continue
// Get the directories for the manga.
val chapterDirs = chapterDirsForManga.getOrPut(manga.id!!) {
mangaDir.listFiles()?.associateBy { it.name }.orEmpty()
}
// Assign the download if the directory exists.
val chapterDirName = downloadManager.getChapterDirName(chapter)
if (chapterDirName in chapterDirs) {
if (downloadManager.isChapterDownloaded(chapter, manga)) {
item.status = Download.DOWNLOADED
}
}
@ -216,7 +190,7 @@ class RecentChaptersPresenter(
private fun deleteChapter(item: RecentChapterItem) {
val source = sourceManager.get(item.manga.source) ?: return
downloadManager.queue.remove(item.chapter)
downloadManager.deleteChapter(source, item.manga, item.chapter)
downloadManager.deleteChapter(item.chapter, item.manga, source)
item.status = Download.NOT_DOWNLOADED
item.download = null
}