From 94a90c6ceceeed5c0ed807b810525300bdc43b0f Mon Sep 17 00:00:00 2001 From: Jays2Kings Date: Mon, 19 Apr 2021 00:14:24 -0400 Subject: [PATCH] Minor updates to downloader from upstream Such as available space check and using io thread instead of ui Co-Authored-By: arkon <4098258+arkon@users.noreply.github.com> --- .../tachiyomi/data/download/Downloader.kt | 117 +++++++++++------- .../kanade/tachiyomi/util/storage/DiskUtil.kt | 13 ++ .../util/system/CoroutinesExtensions.kt | 3 + app/src/main/res/values/strings.xml | 1 + 4 files changed, 86 insertions(+), 48 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt index 5f93b4d004..b119272f7d 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt @@ -5,6 +5,7 @@ import android.webkit.MimeTypeMap import com.hippo.unifile.UniFile import com.jakewharton.rxrelay.BehaviorRelay import com.jakewharton.rxrelay.PublishRelay +import eu.kanade.tachiyomi.R import eu.kanade.tachiyomi.data.cache.ChapterCache import eu.kanade.tachiyomi.data.database.models.Chapter import eu.kanade.tachiyomi.data.database.models.Manga @@ -20,8 +21,8 @@ import eu.kanade.tachiyomi.util.lang.plusAssign import eu.kanade.tachiyomi.util.storage.DiskUtil import eu.kanade.tachiyomi.util.storage.saveTo import eu.kanade.tachiyomi.util.system.ImageUtil +import eu.kanade.tachiyomi.util.system.launchIO import eu.kanade.tachiyomi.util.system.launchNow -import eu.kanade.tachiyomi.util.system.launchUI import kotlinx.coroutines.async import okhttp3.Response import rx.Observable @@ -29,6 +30,7 @@ import rx.android.schedulers.AndroidSchedulers import rx.schedulers.Schedulers import rx.subscriptions.CompositeSubscription import timber.log.Timber +import uy.kohesive.injekt.injectLazy import java.io.File /** @@ -52,6 +54,8 @@ class Downloader( private val sourceManager: SourceManager ) { + private val chapterCache: ChapterCache by injectLazy() + /** * Store for persisting downloads across restarts. */ @@ -119,7 +123,9 @@ class Downloader( */ fun stop(reason: String? = null) { destroySubscriptions() - queue.filter { it.status == Download.DOWNLOADING }.forEach { it.status = Download.ERROR } + queue + .filter { it.status == Download.DOWNLOADING } + .forEach { it.status = Download.ERROR } if (reason != null) { notifier.onWarning(reason) @@ -142,7 +148,9 @@ class Downloader( */ fun pause() { destroySubscriptions() - queue.filter { it.status == Download.DOWNLOADING }.forEach { it.status = Download.QUEUE } + queue + .filter { it.status == Download.DOWNLOADING } + .forEach { it.status = Download.QUEUE } notifier.paused = true } @@ -161,7 +169,8 @@ class Downloader( // Needed to update the chapter view if (isNotification) { - queue.filter { it.status == Download.QUEUE } + queue + .filter { it.status == Download.QUEUE } .forEach { it.status = Download.NOT_DOWNLOADED } } queue.clear() @@ -207,7 +216,7 @@ class Downloader( }, 5 ) - .onBackpressureBuffer() + .onBackpressureLatest() .observeOn(AndroidSchedulers.mainThread()) .subscribe( { @@ -239,8 +248,8 @@ class Downloader( * @param chapters the list of chapters to download. * @param autoStart whether to start the downloader after enqueing the chapters. */ - fun queueChapters(manga: Manga, chapters: List, autoStart: Boolean) = launchUI { - val source = sourceManager.get(manga.source) as? HttpSource ?: return@launchUI + fun queueChapters(manga: Manga, chapters: List, autoStart: Boolean) = launchIO { + val source = sourceManager.get(manga.source) as? HttpSource ?: return@launchIO val wasEmpty = queue.isEmpty() // Called in background thread, the operation can be slow with SAF. val chaptersWithoutDir = async { @@ -281,15 +290,23 @@ class Downloader( * @param download the chapter to be downloaded. */ private fun downloadChapter(download: Download): Observable = Observable.defer { - val chapterDirname = provider.getChapterDirName(download.chapter) val mangaDir = provider.getMangaDir(download.manga, download.source) + + val availSpace = DiskUtil.getAvailableStorageSpace(mangaDir) + if (availSpace != -1L && availSpace < MIN_DISK_SPACE) { + download.status = Download.ERROR + notifier.onError(context.getString(R.string.couldnt_download_low_space), download.chapter.name) + return@defer Observable.just(download) + } + + val chapterDirname = provider.getChapterDirName(download.chapter) val tmpDir = mangaDir.createDirectory(chapterDirname + TMP_DIR_SUFFIX) val pageListObservable = if (download.pages == null) { // Pull page list from network and add them to download object download.source.fetchPageList(download.chapter).doOnNext { pages -> if (pages.isEmpty()) { - throw Exception("Page list is empty") + throw Exception(context.getString(R.string.no_pages_found)) } download.pages = pages } @@ -298,20 +315,25 @@ class Downloader( Observable.just(download.pages!!) } - pageListObservable.doOnNext { _ -> - // Delete all temporary (unfinished) files - tmpDir.listFiles()?.filter { it.name!!.endsWith(".tmp") }?.forEach { it.delete() } + pageListObservable + .doOnNext { _ -> + // Delete all temporary (unfinished) files + tmpDir.listFiles() + ?.filter { it.name!!.endsWith(".tmp") } + ?.forEach { it.delete() } - download.downloadedImages = 0 - download.status = Download.DOWNLOADING - } + download.downloadedImages = 0 + download.status = Download.DOWNLOADING + } // Get all the URLs to the source images, fetch pages if necessary .flatMap { download.source.fetchAllImageUrlsFromPageList(it) } // Start downloading images, consider we can have downloaded images already // Concurrently do 5 pages at a time .flatMap({ page -> getOrDownloadImage(page, download, tmpDir) }, 5) // Do when page is downloaded. - .doOnNext { notifier.onProgressChange(download) }.toList().map { _ -> download } + .doOnNext { notifier.onProgressChange(download) } + .toList() + .map { download } // Do after download completes .doOnNext { ensureSuccessfulDownload(download, mangaDir, tmpDir, chapterDirname) } // If the page list threw, it will resume here @@ -336,7 +358,9 @@ class Downloader( tmpDir: UniFile ): Observable { // If the image URL is empty, do nothing - if (page.imageUrl == null) return Observable.just(page) + if (page.imageUrl == null) { + return Observable.just(page) + } val filename = String.format("%03d", page.number) val tmpFile = tmpDir.findFile("$filename.tmp") @@ -346,16 +370,11 @@ class Downloader( // Try to find the image file. val imageFile = tmpDir.listFiles()!!.find { it.name!!.startsWith("$filename.") } - val cache = ChapterCache(context) + // If the image is already downloaded, do nothing. Otherwise download from network val pageObservable = when { imageFile != null -> Observable.just(imageFile) - cache.isImageInCache(page.imageUrl!!) -> moveFromCache( - page, - cache.getImageFile(page.imageUrl!!), - tmpDir, - filename - ) + chapterCache.isImageInCache(page.imageUrl!!) -> moveImageFromCache(chapterCache.getImageFile(page.imageUrl!!), tmpDir, filename) else -> downloadImage(page, download.source, tmpDir, filename) } @@ -366,7 +385,8 @@ class Downloader( page.progress = 100 download.downloadedImages++ page.status = Page.READY - }.map { page } + } + .map { page } // Mark this page as error and allow to download the remaining .onErrorReturn { page.progress = 0 @@ -376,30 +396,27 @@ class Downloader( } /** - * Returns the observable which takes from the downloaded image from cache + * Return the observable which copies the image from cache. * - * @param page the page to download. - * @param file the file from cache + * @param cacheFile the file from cache. * @param tmpDir the temporary directory of the download. * @param filename the filename of the image. */ - private fun moveFromCache( - page: Page, - file: File, + private fun moveImageFromCache( + cacheFile: File, tmpDir: UniFile, filename: String ): Observable { - return Observable.just(file).map { + return Observable.just(cacheFile).map { val tmpFile = tmpDir.createFile("$filename.tmp") - val inputStream = file.inputStream() - inputStream.use { input -> + cacheFile.inputStream().use { input -> tmpFile.openOutputStream().use { output -> input.copyTo(output) } } - val extension = ImageUtil.findImageType(file.inputStream()) ?: return@map tmpFile + val extension = ImageUtil.findImageType(cacheFile.inputStream()) ?: return@map tmpFile tmpFile.renameTo("$filename.${extension.extension}") - file.delete() + cacheFile.delete() tmpFile } } @@ -420,19 +437,20 @@ class Downloader( ): Observable { page.status = Page.DOWNLOAD_IMAGE page.progress = 0 - return source.fetchImage(page).map { response -> - val file = tmpDir.createFile("$filename.tmp") - try { - response.body!!.source().saveTo(file.openOutputStream()) - val extension = getImageExtension(response, file) - file.renameTo("$filename.$extension") - } catch (e: Exception) { - response.close() - file.delete() - throw e + return source.fetchImage(page) + .map { response -> + val file = tmpDir.createFile("$filename.tmp") + try { + response.body!!.source().saveTo(file.openOutputStream()) + val extension = getImageExtension(response, file) + file.renameTo("$filename.$extension") + } catch (e: Exception) { + response.close() + file.delete() + throw e + } + file } - file - } // Retry 3 times, waiting 2, 4 and 8 seconds between attempts. .retryWhen(RetryWithDelay(3, { (2 shl it - 1) * 1000 }, Schedulers.trampoline())) } @@ -514,5 +532,8 @@ class Downloader( companion object { const val TMP_DIR_SUFFIX = "_tmp" + + // Arbitrary minimum required space to start a download: 50 MB + const val MIN_DISK_SPACE = 50 * 1024 * 1024 } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/util/storage/DiskUtil.kt b/app/src/main/java/eu/kanade/tachiyomi/util/storage/DiskUtil.kt index d44a63b5d7..e19d5c0c36 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/util/storage/DiskUtil.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/util/storage/DiskUtil.kt @@ -4,6 +4,7 @@ import android.content.Context import android.content.Intent import android.net.Uri import android.os.Environment +import android.os.StatFs import androidx.core.content.ContextCompat import androidx.core.os.EnvironmentCompat import com.hippo.unifile.UniFile @@ -28,6 +29,18 @@ object DiskUtil { return size } + /** + * Gets the available space for the disk that a file path points to, in bytes. + */ + fun getAvailableStorageSpace(f: UniFile): Long { + return try { + val stat = StatFs(f.uri.path) + stat.availableBlocksLong * stat.blockSizeLong + } catch (_: Exception) { + -1L + } + } + /** * Returns the root folders of all the available external storages. */ diff --git a/app/src/main/java/eu/kanade/tachiyomi/util/system/CoroutinesExtensions.kt b/app/src/main/java/eu/kanade/tachiyomi/util/system/CoroutinesExtensions.kt index 295fb88e5e..a77e169e5c 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/util/system/CoroutinesExtensions.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/util/system/CoroutinesExtensions.kt @@ -11,6 +11,9 @@ import kotlinx.coroutines.withContext fun launchUI(block: suspend CoroutineScope.() -> Unit): Job = GlobalScope.launch(Dispatchers.Main, CoroutineStart.DEFAULT, block) +fun launchIO(block: suspend CoroutineScope.() -> Unit): Job = + GlobalScope.launch(Dispatchers.IO, CoroutineStart.DEFAULT, block) + fun launchNow(block: suspend CoroutineScope.() -> Unit): Job = GlobalScope.launch(Dispatchers.Main, CoroutineStart.UNDISPATCHED, block) diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 8a4b722fb3..7a72c7fb37 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -760,6 +760,7 @@ Manage what\'s downloading Visit the recents tab to access the download queue. You can also double tap or press and hold for quicker access + Couldn\'t download chapters due to low disk space Could not download chapter due to unexpected error