Manga cover updates (#3101)

* cover caching overhaul

* add ui for removing custom cover

* skip some loading work

* minor cleanup

* allow refresh library metadata to refresh local manga

* rename metadata_date to cover_last_modified

* rearrange removeMangaFromLibrary

* change custom cover directory
add setting for updating cover when refreshing library

* remove toggle and explicit action for updating covers
This commit is contained in:
MCAxiaz
2020-05-10 08:15:25 -07:00
committed by GitHub
parent 436253dd63
commit dc54299e24
30 changed files with 441 additions and 208 deletions

View File

@@ -1,6 +1,7 @@
package eu.kanade.tachiyomi.data.cache
import android.content.Context
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.util.storage.DiskUtil
import java.io.File
import java.io.IOException
@@ -17,51 +18,89 @@ import java.io.InputStream
*/
class CoverCache(private val context: Context) {
companion object {
private const val COVERS_DIR = "covers"
private const val CUSTOM_COVERS_DIR = "covers/custom"
}
/**
* Cache directory used for cache management.
*/
private val cacheDir = context.getExternalFilesDir("covers")
?: File(context.filesDir, "covers").also { it.mkdirs() }
private val cacheDir = getCacheDir(COVERS_DIR)
private val customCoverCacheDir = getCacheDir(CUSTOM_COVERS_DIR)
/**
* Returns the cover from cache.
*
* @param thumbnailUrl the thumbnail url.
* @param manga the manga.
* @return cover image.
*/
fun getCoverFile(thumbnailUrl: String): File {
return File(cacheDir, DiskUtil.hashKeyForDisk(thumbnailUrl))
fun getCoverFile(manga: Manga): File? {
return manga.thumbnail_url?.let {
File(cacheDir, DiskUtil.hashKeyForDisk(it))
}
}
/**
* Copy the given stream to this cache.
* Returns the custom cover from cache.
*
* @param thumbnailUrl url of the thumbnail.
* @param manga the manga.
* @return cover image.
*/
fun getCustomCoverFile(manga: Manga): File {
return File(customCoverCacheDir, DiskUtil.hashKeyForDisk(manga.id.toString()))
}
/**
* Saves the given stream as the manga's custom cover to cache.
*
* @param manga the manga.
* @param inputStream the stream to copy.
* @throws IOException if there's any error.
*/
@Throws(IOException::class)
fun copyToCache(thumbnailUrl: String, inputStream: InputStream) {
// Get destination file.
val destFile = getCoverFile(thumbnailUrl)
destFile.outputStream().use { inputStream.copyTo(it) }
fun setCustomCoverToCache(manga: Manga, inputStream: InputStream) {
getCustomCoverFile(manga).outputStream().use {
inputStream.copyTo(it)
}
}
/**
* Delete the cover file from the cache.
* Delete the cover files of the manga from the cache.
*
* @param thumbnailUrl the thumbnail url.
* @return status of deletion.
* @param manga the manga.
* @param deleteCustomCover whether the custom cover should be deleted.
* @return number of files that were deleted.
*/
fun deleteFromCache(thumbnailUrl: String?): Boolean {
// Check if url is empty.
if (thumbnailUrl.isNullOrEmpty()) {
return false
fun deleteFromCache(manga: Manga, deleteCustomCover: Boolean = false): Int {
var deleted = 0
getCoverFile(manga)?.let {
if (it.exists() && it.delete()) ++deleted
}
// Remove file.
val file = getCoverFile(thumbnailUrl)
return file.exists() && file.delete()
if (deleteCustomCover) {
if (deleteCustomCover(manga)) ++deleted
}
return deleted
}
/**
* Delete custom cover of the manga from the cache
*
* @param manga the manga.
* @return whether the cover was deleted.
*/
fun deleteCustomCover(manga: Manga): Boolean {
return getCustomCoverFile(manga).let {
it.exists() && it.delete()
}
}
private fun getCacheDir(dir: String): File {
return context.getExternalFilesDir(dir)
?: File(context.filesDir, dir).also { it.mkdirs() }
}
}

View File

@@ -20,7 +20,7 @@ class DbOpenCallback : SupportSQLiteOpenHelper.Callback(DATABASE_VERSION) {
/**
* Version of the database.
*/
const val DATABASE_VERSION = 9
const val DATABASE_VERSION = 10
}
override fun onCreate(db: SupportSQLiteDatabase) = with(db) {
@@ -75,6 +75,9 @@ class DbOpenCallback : SupportSQLiteOpenHelper.Callback(DATABASE_VERSION) {
db.execSQL(TrackTable.addStartDate)
db.execSQL(TrackTable.addFinishDate)
}
if (oldVersion < 10) {
db.execSQL(MangaTable.addCoverLastModified)
}
}
override fun onConfigure(db: SupportSQLiteDatabase) {

View File

@@ -14,6 +14,7 @@ import eu.kanade.tachiyomi.data.database.models.MangaImpl
import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_ARTIST
import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_AUTHOR
import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_CHAPTER_FLAGS
import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_COVER_LAST_MODIFIED
import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_DESCRIPTION
import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_FAVORITE
import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_GENRE
@@ -62,6 +63,7 @@ class MangaPutResolver : DefaultPutResolver<Manga>() {
put(COL_INITIALIZED, obj.initialized)
put(COL_VIEWER, obj.viewer)
put(COL_CHAPTER_FLAGS, obj.chapter_flags)
put(COL_COVER_LAST_MODIFIED, obj.cover_last_modified)
}
}
@@ -82,6 +84,7 @@ interface BaseMangaGetResolver {
initialized = cursor.getInt(cursor.getColumnIndex(COL_INITIALIZED)) == 1
viewer = cursor.getInt(cursor.getColumnIndex(COL_VIEWER))
chapter_flags = cursor.getInt(cursor.getColumnIndex(COL_CHAPTER_FLAGS))
cover_last_modified = cursor.getLong(cursor.getColumnIndex(COL_COVER_LAST_MODIFIED))
}
}

View File

@@ -16,6 +16,8 @@ interface Manga : SManga {
var chapter_flags: Int
var cover_last_modified: Long
fun setChapterOrder(order: Int) {
setFlags(order, SORT_MASK)
}

View File

@@ -32,6 +32,8 @@ open class MangaImpl : Manga {
override var chapter_flags: Int = 0
override var cover_last_modified: Long = 0
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false

View File

@@ -7,6 +7,7 @@ import eu.kanade.tachiyomi.data.database.DbProvider
import eu.kanade.tachiyomi.data.database.models.LibraryManga
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.data.database.resolvers.LibraryMangaGetResolver
import eu.kanade.tachiyomi.data.database.resolvers.MangaCoverLastModifiedPutResolver
import eu.kanade.tachiyomi.data.database.resolvers.MangaFavoritePutResolver
import eu.kanade.tachiyomi.data.database.resolvers.MangaFlagsPutResolver
import eu.kanade.tachiyomi.data.database.resolvers.MangaLastUpdatedPutResolver
@@ -102,6 +103,11 @@ interface MangaQueries : DbProvider {
.withPutResolver(MangaTitlePutResolver())
.prepare()
fun updateMangaCoverLastModified(manga: Manga) = db.put()
.`object`(manga)
.withPutResolver(MangaCoverLastModifiedPutResolver())
.prepare()
fun deleteManga(manga: Manga) = db.delete().`object`(manga).prepare()
fun deleteMangas(mangas: List<Manga>) = db.delete().objects(mangas).prepare()

View File

@@ -0,0 +1,31 @@
package eu.kanade.tachiyomi.data.database.resolvers
import android.content.ContentValues
import com.pushtorefresh.storio.sqlite.StorIOSQLite
import com.pushtorefresh.storio.sqlite.operations.put.PutResolver
import com.pushtorefresh.storio.sqlite.operations.put.PutResult
import com.pushtorefresh.storio.sqlite.queries.UpdateQuery
import eu.kanade.tachiyomi.data.database.inTransactionReturn
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.data.database.tables.MangaTable
class MangaCoverLastModifiedPutResolver : PutResolver<Manga>() {
override fun performPut(db: StorIOSQLite, manga: Manga) = db.inTransactionReturn {
val updateQuery = mapToUpdateQuery(manga)
val contentValues = mapToContentValues(manga)
val numberOfRowsUpdated = db.lowLevel().update(updateQuery, contentValues)
PutResult.newUpdateResult(numberOfRowsUpdated, updateQuery.table())
}
fun mapToUpdateQuery(manga: Manga) = UpdateQuery.builder()
.table(MangaTable.TABLE)
.where("${MangaTable.COL_ID} = ?")
.whereArgs(manga.id)
.build()
fun mapToContentValues(manga: Manga) = ContentValues(1).apply {
put(MangaTable.COL_COVER_LAST_MODIFIED, manga.cover_last_modified)
}
}

View File

@@ -38,6 +38,8 @@ object MangaTable {
const val COL_CATEGORY = "category"
const val COL_COVER_LAST_MODIFIED = "cover_last_modified"
val createTableQuery: String
get() =
"""CREATE TABLE $TABLE(
@@ -55,7 +57,8 @@ object MangaTable {
$COL_LAST_UPDATE LONG,
$COL_INITIALIZED BOOLEAN NOT NULL,
$COL_VIEWER INTEGER NOT NULL,
$COL_CHAPTER_FLAGS INTEGER NOT NULL
$COL_CHAPTER_FLAGS INTEGER NOT NULL,
$COL_COVER_LAST_MODIFIED LONG NOT NULL
)"""
val createUrlIndexQuery: String
@@ -64,4 +67,7 @@ object MangaTable {
val createLibraryIndexQuery: String
get() = "CREATE INDEX library_${COL_FAVORITE}_index ON $TABLE($COL_FAVORITE) " +
"WHERE $COL_FAVORITE = 1"
val addCoverLastModified: String
get() = "ALTER TABLE $TABLE ADD COLUMN $COL_COVER_LAST_MODIFIED LONG NOT NULL DEFAULT 0"
}

View File

@@ -12,7 +12,7 @@ import java.io.IOException
import java.io.InputStream
import timber.log.Timber
open class FileFetcher(private val file: File) : DataFetcher<InputStream> {
open class FileFetcher(private val filePath: String = "") : DataFetcher<InputStream> {
private var data: InputStream? = null
@@ -20,7 +20,11 @@ open class FileFetcher(private val file: File) : DataFetcher<InputStream> {
loadFromFile(callback)
}
protected fun loadFromFile(callback: DataFetcher.DataCallback<in InputStream>) {
private fun loadFromFile(callback: DataFetcher.DataCallback<in InputStream>) {
loadFromFile(File(filePath), callback)
}
protected fun loadFromFile(file: File, callback: DataFetcher.DataCallback<in InputStream>) {
try {
data = FileInputStream(file)
} catch (e: FileNotFoundException) {

View File

@@ -0,0 +1,25 @@
package eu.kanade.tachiyomi.data.glide
import com.bumptech.glide.Priority
import com.bumptech.glide.load.data.DataFetcher
import eu.kanade.tachiyomi.data.cache.CoverCache
import eu.kanade.tachiyomi.data.database.models.Manga
import java.io.File
import java.io.InputStream
import java.lang.Exception
open class LibraryMangaCustomCoverFetcher(
private val manga: Manga,
private val coverCache: CoverCache
) : FileFetcher() {
override fun loadData(priority: Priority, callback: DataFetcher.DataCallback<in InputStream>) {
getCustomCoverFile()?.let {
loadFromFile(it, callback)
} ?: callback.onLoadFailed(Exception("Custom cover file not found"))
}
protected fun getCustomCoverFile(): File? {
return coverCache.getCustomCoverFile(manga).takeIf { it.exists() }
}
}

View File

@@ -2,6 +2,7 @@ package eu.kanade.tachiyomi.data.glide
import com.bumptech.glide.Priority
import com.bumptech.glide.load.data.DataFetcher
import eu.kanade.tachiyomi.data.cache.CoverCache
import eu.kanade.tachiyomi.data.database.models.Manga
import java.io.File
import java.io.FileNotFoundException
@@ -19,31 +20,41 @@ import java.io.InputStream
class LibraryMangaUrlFetcher(
private val networkFetcher: DataFetcher<InputStream>,
private val manga: Manga,
private val file: File
) :
FileFetcher(file) {
private val coverCache: CoverCache
) : LibraryMangaCustomCoverFetcher(manga, coverCache) {
override fun loadData(priority: Priority, callback: DataFetcher.DataCallback<in InputStream>) {
if (!file.exists()) {
getCustomCoverFile()?.let {
loadFromFile(it, callback)
return
}
val cover = coverCache.getCoverFile(manga)
if (cover == null) {
callback.onLoadFailed(Exception("Null thumbnail url"))
return
}
if (!cover.exists()) {
networkFetcher.loadData(
priority,
object : DataFetcher.DataCallback<InputStream> {
override fun onDataReady(data: InputStream?) {
if (data != null) {
val tmpFile = File(file.path + ".tmp")
val tmpFile = File(cover.path + ".tmp")
try {
// Retrieve destination stream, create parent folders if needed.
val output = try {
tmpFile.outputStream()
} catch (e: FileNotFoundException) {
tmpFile.parentFile.mkdirs()
tmpFile.parentFile!!.mkdirs()
tmpFile.outputStream()
}
// Copy the file and rename to the original.
data.use { output.use { data.copyTo(output) } }
tmpFile.renameTo(file)
loadFromFile(callback)
tmpFile.renameTo(cover)
loadFromFile(cover, callback)
} catch (e: Exception) {
tmpFile.delete()
callback.onLoadFailed(e)
@@ -59,7 +70,7 @@ class LibraryMangaUrlFetcher(
}
)
} else {
loadFromFile(callback)
loadFromFile(cover, callback)
}
}

View File

@@ -1,27 +0,0 @@
package eu.kanade.tachiyomi.data.glide
import com.bumptech.glide.load.Key
import eu.kanade.tachiyomi.data.database.models.Manga
import java.io.File
import java.security.MessageDigest
class MangaSignature(manga: Manga, file: File) : Key {
private val key = manga.thumbnail_url + file.lastModified()
override fun equals(other: Any?): Boolean {
return if (other is MangaSignature) {
key == other.key
} else {
false
}
}
override fun hashCode(): Int {
return key.hashCode()
}
override fun updateDiskCacheKey(md: MessageDigest) {
md.update(key.toByteArray(Key.CHARSET))
}
}

View File

@@ -1,7 +1,15 @@
package eu.kanade.tachiyomi.data.glide
import com.bumptech.glide.load.Key
import eu.kanade.tachiyomi.data.database.models.Manga
import java.security.MessageDigest
data class MangaThumbnail(val manga: Manga, val url: String?)
data class MangaThumbnail(val manga: Manga, val coverLastModified: Long) : Key {
val key = manga.url + coverLastModified
fun Manga.toMangaThumbnail() = MangaThumbnail(this, this.thumbnail_url)
override fun updateDiskCacheKey(messageDigest: MessageDigest) {
messageDigest.update(key.toByteArray(Key.CHARSET))
}
}
fun Manga.toMangaThumbnail() = MangaThumbnail(this, cover_last_modified)

View File

@@ -14,7 +14,7 @@ import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.online.HttpSource
import java.io.File
import eu.kanade.tachiyomi.util.isLocal
import java.io.InputStream
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
@@ -48,12 +48,6 @@ class MangaThumbnailModelLoader : ModelLoader<MangaThumbnail, InputStream> {
*/
private val defaultClient = Injekt.get<NetworkHelper>().client
/**
* LRU cache whose key is the thumbnail url of the manga, and the value contains the request url
* and the file where it should be stored in case the manga is a favorite.
*/
private val lruCache = LruCache<GlideUrl, File>(100)
/**
* Map where request headers are stored for a source.
*/
@@ -78,7 +72,7 @@ class MangaThumbnailModelLoader : ModelLoader<MangaThumbnail, InputStream> {
/**
* Returns a fetcher for the given manga or null if the url is empty.
*
* @param manga the model.
* @param mangaThumbnail the model.
* @param width the width of the view where the resource will be loaded.
* @param height the height of the view where the resource will be loaded.
*/
@@ -88,13 +82,16 @@ class MangaThumbnailModelLoader : ModelLoader<MangaThumbnail, InputStream> {
height: Int,
options: Options
): ModelLoader.LoadData<InputStream>? {
// Check thumbnail is not null or empty
val url = mangaThumbnail.url
if (url == null || url.isEmpty()) {
return null
}
val manga = mangaThumbnail.manga
val url = manga.thumbnail_url
if (url.isNullOrEmpty()) {
return if (!manga.favorite || manga.isLocal()) {
null
} else {
ModelLoader.LoadData(mangaThumbnail, LibraryMangaCustomCoverFetcher(manga, coverCache))
}
}
if (url.startsWith("http", true)) {
val source = sourceManager.get(manga.source) as? HttpSource
@@ -107,19 +104,13 @@ class MangaThumbnailModelLoader : ModelLoader<MangaThumbnail, InputStream> {
return ModelLoader.LoadData(glideUrl, networkFetcher)
}
// Obtain the file for this url from the LRU cache, or retrieve and add it to the cache.
val file = lruCache.getOrPut(glideUrl) { coverCache.getCoverFile(url) }
val libraryFetcher = LibraryMangaUrlFetcher(networkFetcher, manga, file)
val libraryFetcher = LibraryMangaUrlFetcher(networkFetcher, manga, coverCache)
// Return an instance of the fetcher providing the needed elements.
return ModelLoader.LoadData(MangaSignature(manga, file), libraryFetcher)
return ModelLoader.LoadData(mangaThumbnail, libraryFetcher)
} else {
// Get the file from the url, removing the scheme if present.
val file = File(url.substringAfter("file://"))
// Return an instance of the fetcher providing the needed elements.
return ModelLoader.LoadData(MangaSignature(manga, file), FileFetcher(file))
return ModelLoader.LoadData(mangaThumbnail, FileFetcher(url.removePrefix("file://")))
}
}

View File

@@ -15,6 +15,7 @@ import androidx.core.app.NotificationCompat.GROUP_ALERT_SUMMARY
import androidx.core.app.NotificationManagerCompat
import com.bumptech.glide.Glide
import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.data.cache.CoverCache
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Category
import eu.kanade.tachiyomi.data.database.models.Chapter
@@ -31,10 +32,10 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.track.TrackManager
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.ui.main.MainActivity
import eu.kanade.tachiyomi.util.chapter.syncChaptersWithSource
import eu.kanade.tachiyomi.util.lang.chop
import eu.kanade.tachiyomi.util.prepUpdateCover
import eu.kanade.tachiyomi.util.system.acquireWakeLock
import eu.kanade.tachiyomi.util.system.isServiceRunning
import eu.kanade.tachiyomi.util.system.notification
@@ -64,7 +65,8 @@ class LibraryUpdateService(
val sourceManager: SourceManager = Injekt.get(),
val preferences: PreferencesHelper = Injekt.get(),
val downloadManager: DownloadManager = Injekt.get(),
val trackManager: TrackManager = Injekt.get()
val trackManager: TrackManager = Injekt.get(),
val coverCache: CoverCache = Injekt.get()
) : Service() {
/**
@@ -110,6 +112,7 @@ class LibraryUpdateService(
*/
enum class Target {
CHAPTERS, // Manga chapters
COVERS, // Manga covers
TRACKING // Tracking metadata
}
@@ -233,6 +236,7 @@ class LibraryUpdateService(
// Update either chapter list or manga details.
when (target) {
Target.CHAPTERS -> updateChapterList(mangaList)
Target.COVERS -> updateCovers(mangaList)
Target.TRACKING -> updateTrackings(mangaList)
}
}
@@ -387,11 +391,14 @@ class LibraryUpdateService(
* @return a pair of the inserted and removed chapters.
*/
fun updateManga(manga: Manga): Observable<Pair<List<Chapter>, List<Chapter>>> {
val source = sourceManager.get(manga.source) as? HttpSource ?: return Observable.empty()
val source = sourceManager.get(manga.source) ?: return Observable.empty()
// Update manga details metadata in the background
source.fetchMangaDetails(manga)
.map { networkManga ->
if (manga.thumbnail_url != networkManga.thumbnail_url) {
manga.prepUpdateCover(coverCache)
}
manga.copyFrom(networkManga)
db.insertManga(manga).executeAsBlocking()
manga
@@ -404,6 +411,21 @@ class LibraryUpdateService(
.map { syncChaptersWithSource(db, it, manga, source) }
}
private fun updateCovers(mangaToUpdate: List<LibraryManga>): Observable<LibraryManga> {
var count = 0
return Observable.from(mangaToUpdate)
.doOnNext { showProgressNotification(it, count++, mangaToUpdate.size) }
.map { manga ->
manga.prepUpdateCover(coverCache)
db.insertManga(manga).executeAsBlocking()
manga
}
.doOnCompleted {
cancelProgressNotification()
}
}
/**
* Method that updates the metadata of the connected tracking services. It's called in a
* background thread, so it's safe to do heavy operations or network calls here.