Improve handling of downloads for chapters with same metadata and optionally for OSes that don't support Unicode in filename (#2305)

Co-authored-by: jkim <jhskim@hotmail.com>
Co-authored-by: fatotak <111342761+fatotak@users.noreply.github.com>
Co-authored-by: MajorTanya <39014446+MajorTanya@users.noreply.github.com>
Co-authored-by: AntsyLich <59261191+AntsyLich@users.noreply.github.com>

app/src/main/java/eu/kanade/domain/chapter/interactor/SyncChaptersWithSource.kt

@@ -114,6 +114,7 @@ class SyncChaptersWithSource(
                         downloadManager.isChapterDownloaded(
                             dbChapter.name,
                             dbChapter.scanlator,
+                            dbChapter.url,
                             manga.title,
                             manga.source,
                         )
@@ -121,12 +122,14 @@ class SyncChaptersWithSource(
                     if (shouldRenameChapter) {
                         downloadManager.renameChapter(source, manga, dbChapter, chapter)
                     }
+
                     var toChangeChapter = dbChapter.copy(
                         name = chapter.name,
                         chapterNumber = chapter.chapterNumber,
                         scanlator = chapter.scanlator,
                         sourceOrder = chapter.sourceOrder,
                     )
+
                     if (chapter.dateUpload != 0L) {
                         toChangeChapter = toChangeChapter.copy(dateUpload = chapter.dateUpload)
                     }

app/src/main/java/eu/kanade/domain/chapter/model/ChapterFilter.kt

@@ -26,6 +26,7 @@ fun List<Chapter>.applyFilters(manga: Manga, downloadManager: DownloadManager):
                 val downloaded = downloadManager.isChapterDownloaded(
                     chapter.name,
                     chapter.scanlator,
+                    chapter.url,
                     manga.title,
                     manga.source,
                 )

app/src/main/java/eu/kanade/presentation/more/settings/screen/SettingsAdvancedScreen.kt

@@ -323,6 +323,11 @@ object SettingsAdvancedScreen : SearchableSettings {
                     title = stringResource(MR.strings.pref_update_library_manga_titles),
                     subtitle = stringResource(MR.strings.pref_update_library_manga_titles_summary),
                 ),
+                Preference.PreferenceItem.SwitchPreference(
+                    preference = libraryPreferences.disallowNonAsciiFilenames(),
+                    title = stringResource(MR.strings.pref_disallow_non_ascii_filenames),
+                    subtitle = stringResource(MR.strings.pref_disallow_non_ascii_filenames_details),
+                ),
             ),
         )
     }

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt

@@ -128,6 +128,7 @@ class DownloadCache(
      *
      * @param chapterName the name of the chapter to query.
      * @param chapterScanlator scanlator of the chapter to query
+     * @param chapterUrl the url of the chapter to query
      * @param mangaTitle the title of the manga to query.
      * @param sourceId the id of the source of the chapter.
      * @param skipCache whether to skip the directory cache and check in the filesystem.
@@ -135,13 +136,14 @@ class DownloadCache(
     fun isChapterDownloaded(
         chapterName: String,
         chapterScanlator: String?,
+        chapterUrl: String,
         mangaTitle: String,
         sourceId: Long,
         skipCache: Boolean,
     ): Boolean {
         if (skipCache) {
             val source = sourceManager.getOrStub(sourceId)
-            return provider.findChapterDir(chapterName, chapterScanlator, mangaTitle, source) != null
+            return provider.findChapterDir(chapterName, chapterScanlator, chapterUrl, mangaTitle, source) != null
         }
 
         renewCache()
@@ -153,6 +155,7 @@ class DownloadCache(
                 return provider.getValidChapterDirNames(
                     chapterName,
                     chapterScanlator,
+                    chapterUrl,
                 ).any { it in mangaDir.chapterDirs }
             }
         }
@@ -233,7 +236,7 @@ class DownloadCache(
         rootDownloadsDirMutex.withLock {
             val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
             val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
-            provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
+            provider.getValidChapterDirNames(chapter.name, chapter.scanlator, chapter.url).forEach {
                 if (it in mangaDir.chapterDirs) {
                     mangaDir.chapterDirs -= it
                 }
@@ -254,7 +257,7 @@ class DownloadCache(
             val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
             val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
             chapters.forEach { chapter ->
-                provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
+                provider.getValidChapterDirNames(chapter.name, chapter.scanlator, chapter.url).forEach {
                     if (it in mangaDir.chapterDirs) {
                         mangaDir.chapterDirs -= it
                     }

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt

@@ -159,7 +159,7 @@ class DownloadManager(
      * @return the list of pages from the chapter.
      */
     fun buildPageList(source: Source, manga: Manga, chapter: Chapter): List<Page> {
-        val chapterDir = provider.findChapterDir(chapter.name, chapter.scanlator, manga.title, source)
+        val chapterDir = provider.findChapterDir(chapter.name, chapter.scanlator, chapter.url, manga.title, source)
         val files = chapterDir?.listFiles().orEmpty()
             .filter { it.isFile && ImageUtil.isImage(it.name) { it.openInputStream() } }
 
@@ -185,11 +185,12 @@ class DownloadManager(
     fun isChapterDownloaded(
         chapterName: String,
         chapterScanlator: String?,
+        chapterUrl: String,
         mangaTitle: String,
         sourceId: Long,
         skipCache: Boolean = false,
     ): Boolean {
-        return cache.isChapterDownloaded(chapterName, chapterScanlator, mangaTitle, sourceId, skipCache)
+        return cache.isChapterDownloaded(chapterName, chapterScanlator, chapterUrl, mangaTitle, sourceId, skipCache)
     }
 
     /**
@@ -368,7 +369,7 @@ class DownloadManager(
      * @param newChapter the target chapter with the new name.
      */
     suspend fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
-        val oldNames = provider.getValidChapterDirNames(oldChapter.name, oldChapter.scanlator)
+        val oldNames = provider.getValidChapterDirNames(oldChapter.name, oldChapter.scanlator, oldChapter.url)
         val mangaDir = provider.getMangaDir(manga.title, source).getOrElse { e ->
             logcat(LogPriority.ERROR, e) { "Manga download folder doesn't exist. Skipping renaming after source sync" }
             return
@@ -379,7 +380,7 @@ class DownloadManager(
             .mapNotNull { mangaDir.findFile(it) }
             .firstOrNull() ?: return
 
-        var newName = provider.getChapterDirName(newChapter.name, newChapter.scanlator)
+        var newName = provider.getChapterDirName(newChapter.name, newChapter.scanlator, newChapter.url)
         if (oldDownload.isFile && oldDownload.extension == "cbz") {
             newName += ".cbz"
         }

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadProvider.kt

@@ -3,12 +3,14 @@ package eu.kanade.tachiyomi.data.download
 import android.content.Context
 import com.hippo.unifile.UniFile
 import eu.kanade.tachiyomi.source.Source
+import eu.kanade.tachiyomi.util.lang.Hash.md5
 import eu.kanade.tachiyomi.util.storage.DiskUtil
 import logcat.LogPriority
 import tachiyomi.core.common.i18n.stringResource
 import tachiyomi.core.common.storage.displayablePath
 import tachiyomi.core.common.util.system.logcat
 import tachiyomi.domain.chapter.model.Chapter
+import tachiyomi.domain.library.service.LibraryPreferences
 import tachiyomi.domain.manga.model.Manga
 import tachiyomi.domain.storage.service.StorageManager
 import tachiyomi.i18n.MR
@@ -25,6 +27,7 @@ import java.io.IOException
 class DownloadProvider(
     private val context: Context,
     private val storageManager: StorageManager = Injekt.get(),
+    private val libraryPreferences: LibraryPreferences = Injekt.get(),
 ) {
 
     private val downloadsDir: UniFile?
@@ -96,9 +99,15 @@ class DownloadProvider(
      * @param mangaTitle the title of the manga to query.
      * @param source the source of the chapter.
      */
-    fun findChapterDir(chapterName: String, chapterScanlator: String?, mangaTitle: String, source: Source): UniFile? {
+    fun findChapterDir(
+        chapterName: String,
+        chapterScanlator: String?,
+        chapterUrl: String,
+        mangaTitle: String,
+        source: Source,
+    ): UniFile? {
         val mangaDir = findMangaDir(mangaTitle, source)
-        return getValidChapterDirNames(chapterName, chapterScanlator).asSequence()
+        return getValidChapterDirNames(chapterName, chapterScanlator, chapterUrl).asSequence()
             .mapNotNull { mangaDir?.findFile(it) }
             .firstOrNull()
     }
@@ -113,7 +122,7 @@ class DownloadProvider(
     fun findChapterDirs(chapters: List<Chapter>, manga: Manga, source: Source): Pair<UniFile?, List<UniFile>> {
         val mangaDir = findMangaDir(manga.title, source) ?: return null to emptyList()
         return mangaDir to chapters.mapNotNull { chapter ->
-            getValidChapterDirNames(chapter.name, chapter.scanlator).asSequence()
+            getValidChapterDirNames(chapter.name, chapter.scanlator, chapter.url).asSequence()
                 .mapNotNull { mangaDir.findFile(it) }
                 .firstOrNull()
         }
@@ -125,7 +134,10 @@ class DownloadProvider(
      * @param source the source to query.
      */
     fun getSourceDirName(source: Source): String {
-        return DiskUtil.buildValidFilename(source.toString())
+        return DiskUtil.buildValidFilename(
+            source.toString(),
+            disallowNonAscii = libraryPreferences.disallowNonAsciiFilenames().get(),
+        )
     }
 
     /**
@@ -134,23 +146,75 @@ class DownloadProvider(
      * @param mangaTitle the title of the manga to query.
      */
     fun getMangaDirName(mangaTitle: String): String {
-        return DiskUtil.buildValidFilename(mangaTitle)
+        return DiskUtil.buildValidFilename(
+            mangaTitle,
+            disallowNonAscii = libraryPreferences.disallowNonAsciiFilenames().get(),
+        )
     }
 
     /**
      * Returns the chapter directory name for a chapter.
      *
      * @param chapterName the name of the chapter to query.
-     * @param chapterScanlator scanlator of the chapter to query
+     * @param chapterScanlator scanlator of the chapter to query.
+     * @param chapterUrl url of the chapter to query.
      */
-    fun getChapterDirName(chapterName: String, chapterScanlator: String?): String {
-        val newChapterName = sanitizeChapterName(chapterName)
-        return DiskUtil.buildValidFilename(
+    fun getChapterDirName(
+        chapterName: String,
+        chapterScanlator: String?,
+        chapterUrl: String,
+        disallowNonAsciiFilenames: Boolean = libraryPreferences.disallowNonAsciiFilenames().get(),
+    ): String {
+        var dirName = sanitizeChapterName(chapterName)
+        if (!chapterScanlator.isNullOrBlank()) {
+            dirName = chapterScanlator + "_" + dirName
+        }
+        // Subtract 7 bytes for hash and underscore, 4 bytes for .cbz
+        dirName = DiskUtil.buildValidFilename(dirName, DiskUtil.MAX_FILE_NAME_BYTES - 11, disallowNonAsciiFilenames)
+        dirName += "_" + md5(chapterUrl).take(6)
+        return dirName
+    }
+
+    /**
+     * Returns list of names that might have been previously used as
+     * the directory name for a chapter.
+     * Add to this list if naming pattern ever changes.
+     *
+     * @param chapterName the name of the chapter to query.
+     * @param chapterScanlator scanlator of the chapter to query.
+     * @param chapterUrl url of the chapter to query.
+     */
+    private fun getLegacyChapterDirNames(
+        chapterName: String,
+        chapterScanlator: String?,
+        chapterUrl: String,
+    ): List<String> {
+        val sanitizedChapterName = sanitizeChapterName(chapterName)
+        val chapterNameV1 = DiskUtil.buildValidFilename(
             when {
-                !chapterScanlator.isNullOrBlank() -> "${chapterScanlator}_$newChapterName"
-                else -> newChapterName
+                !chapterScanlator.isNullOrBlank() -> "${chapterScanlator}_$sanitizedChapterName"
+                else -> sanitizedChapterName
             },
         )
+
+        // Get the filename that would be generated if the user were
+        // using the other value for the disallow non-ASCII
+        // filenames setting. This ensures that chapters downloaded
+        // before the user changed the setting can still be found.
+        val otherChapterDirName =
+            getChapterDirName(
+                chapterName,
+                chapterScanlator,
+                chapterUrl,
+                !libraryPreferences.disallowNonAsciiFilenames().get(),
+            )
+
+        return buildList(2) {
+            // Chapter name without hash (unable to handle duplicate
+            // chapter names)
+            add(chapterNameV1)
+            add(otherChapterDirName)
+        }
     }
 
     /**
@@ -165,24 +229,30 @@ class DownloadProvider(
     }
 
     fun isChapterDirNameChanged(oldChapter: Chapter, newChapter: Chapter): Boolean {
-        return oldChapter.name != newChapter.name ||
-            oldChapter.scanlator?.takeIf { it.isNotBlank() } != newChapter.scanlator?.takeIf { it.isNotBlank() }
+        return getChapterDirName(oldChapter.name, oldChapter.scanlator, oldChapter.url) !=
+            getChapterDirName(newChapter.name, newChapter.scanlator, newChapter.url)
     }
 
     /**
      * Returns valid downloaded chapter directory names.
      *
-     * @param chapterName the name of the chapter to query.
-     * @param chapterScanlator scanlator of the chapter to query
+     * @param chapter the domain chapter object.
      */
-    fun getValidChapterDirNames(chapterName: String, chapterScanlator: String?): List<String> {
-        val chapterDirName = getChapterDirName(chapterName, chapterScanlator)
-        return buildList(2) {
+    fun getValidChapterDirNames(chapterName: String, chapterScanlator: String?, chapterUrl: String): List<String> {
+        val chapterDirName = getChapterDirName(chapterName, chapterScanlator, chapterUrl)
+        val legacyChapterDirNames = getLegacyChapterDirNames(chapterName, chapterScanlator, chapterUrl)
+
+        return buildList {
             // Folder of images
             add(chapterDirName)
-
             // Archived chapters
             add("$chapterDirName.cbz")
+
+            // any legacy names
+            legacyChapterDirNames.forEach {
+                add(it)
+                add("$it.cbz")
+            }
         }
     }
 }

app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt

@@ -274,7 +274,7 @@ class Downloader(
         val wasEmpty = queueState.value.isEmpty()
         val chaptersToQueue = chapters.asSequence()
             // Filter out those already downloaded.
-            .filter { provider.findChapterDir(it.name, it.scanlator, manga.title, source) == null }
+            .filter { provider.findChapterDir(it.name, it.scanlator, it.url, manga.title, source) == null }
             // Add chapters to queue from the start.
             .sortedByDescending { it.sourceOrder }
             // Filter out those already enqueued.
@@ -336,7 +336,11 @@ class Downloader(
             return
         }
 
-        val chapterDirname = provider.getChapterDirName(download.chapter.name, download.chapter.scanlator)
+        val chapterDirname = provider.getChapterDirName(
+            download.chapter.name,
+            download.chapter.scanlator,
+            download.chapter.url,
+        )
         val tmpDir = mangaDir.createDirectory(chapterDirname + TMP_DIR_SUFFIX)!!
 
         try {

app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryScreenModel.kt

@@ -484,6 +484,7 @@ class LibraryScreenModel(
                             downloadManager.isChapterDownloaded(
                                 chapter.name,
                                 chapter.scanlator,
+                                chapter.url,
                                 manga.title,
                                 manga.source,
                             )

app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaScreenModel.kt

@@ -527,7 +527,13 @@ class MangaScreenModel(
             val downloaded = if (isLocal) {
                 true
             } else {
-                downloadManager.isChapterDownloaded(chapter.name, chapter.scanlator, manga.title, manga.source)
+                downloadManager.isChapterDownloaded(
+                    chapter.name,
+                    chapter.scanlator,
+                    chapter.url,
+                    manga.title,
+                    manga.source,
+                )
             }
             val downloadState = when {
                 activeDownload != null -> activeDownload.status

app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderViewModel.kt

@@ -38,7 +38,6 @@ import eu.kanade.tachiyomi.util.chapter.filterDownloaded
 import eu.kanade.tachiyomi.util.chapter.removeDuplicates
 import eu.kanade.tachiyomi.util.editCover
 import eu.kanade.tachiyomi.util.lang.byteSize
-import eu.kanade.tachiyomi.util.lang.takeBytes
 import eu.kanade.tachiyomi.util.storage.DiskUtil
 import eu.kanade.tachiyomi.util.storage.cacheImageDir
 import kotlinx.coroutines.CancellationException
@@ -175,6 +174,7 @@ class ReaderViewModel @JvmOverloads constructor(
                                         !downloadManager.isChapterDownloaded(
                                             it.name,
                                             it.scanlator,
+                                            it.url,
                                             manga.title,
                                             manga.source,
                                         )
@@ -184,6 +184,7 @@ class ReaderViewModel @JvmOverloads constructor(
                                         downloadManager.isChapterDownloaded(
                                             it.name,
                                             it.scanlator,
+                                            it.url,
                                             manga.title,
                                             manga.source,
                                         )
@@ -397,6 +398,7 @@ class ReaderViewModel @JvmOverloads constructor(
             val isDownloaded = downloadManager.isChapterDownloaded(
                 dbChapter.name,
                 dbChapter.scanlator,
+                dbChapter.url,
                 manga.title,
                 manga.source,
                 skipCache = true,
@@ -473,6 +475,7 @@ class ReaderViewModel @JvmOverloads constructor(
             val isNextChapterDownloaded = downloadManager.isChapterDownloaded(
                 nextChapter.name,
                 nextChapter.scanlator,
+                nextChapter.url,
                 manga.title,
                 manga.source,
             )
@@ -757,7 +760,8 @@ class ReaderViewModel @JvmOverloads constructor(
         val chapter = page.chapter.chapter
         val filenameSuffix = " - ${page.number}"
         return DiskUtil.buildValidFilename(
-            "${manga.title} - ${chapter.name}".takeBytes(DiskUtil.MAX_FILE_NAME_BYTES - filenameSuffix.byteSize()),
+            "${manga.title} - ${chapter.name}",
+            DiskUtil.MAX_FILE_NAME_BYTES - filenameSuffix.byteSize(),
         ) + filenameSuffix
     }
 

app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/ChapterLoader.kt

@@ -80,6 +80,7 @@ class ChapterLoader(
         val isDownloaded = downloadManager.isChapterDownloaded(
             dbChapter.name,
             dbChapter.scanlator,
+            dbChapter.url,
             manga.title,
             manga.source,
             skipCache = true,

app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/DownloadPageLoader.kt

@@ -33,7 +33,13 @@ internal class DownloadPageLoader(
 
     override suspend fun getPages(): List<ReaderPage> {
         val dbChapter = chapter.chapter
-        val chapterPath = downloadProvider.findChapterDir(dbChapter.name, dbChapter.scanlator, manga.title, source)
+        val chapterPath = downloadProvider.findChapterDir(
+            dbChapter.name,
+            dbChapter.scanlator,
+            dbChapter.url,
+            manga.title,
+            source,
+        )
         return if (chapterPath?.isFile == true) {
             getPagesFromArchive(chapterPath)
         } else {

app/src/main/java/eu/kanade/tachiyomi/ui/reader/viewer/ReaderTransitionView.kt

@@ -37,6 +37,7 @@ class ReaderTransitionView @JvmOverloads constructor(context: Context, attrs: At
                         downloadManager.isChapterDownloaded(
                             chapterName = goingToChapter.name,
                             chapterScanlator = goingToChapter.scanlator,
+                            chapterUrl = goingToChapter.url,
                             mangaTitle = manga.title,
                             sourceId = manga.source,
                             skipCache = true,

app/src/main/java/eu/kanade/tachiyomi/ui/updates/UpdatesScreenModel.kt

@@ -107,6 +107,7 @@ class UpdatesScreenModel(
                 val downloaded = downloadManager.isChapterDownloaded(
                     update.chapterName,
                     update.scanlator,
+                    update.chapterUrl,
                     update.mangaTitle,
                     update.sourceId,
                 )

app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterFilterDownloaded.kt

@@ -15,5 +15,5 @@ fun List<Chapter>.filterDownloaded(manga: Manga): List<Chapter> {
 
     val downloadCache: DownloadCache = Injekt.get()
 
-    return filter { downloadCache.isChapterDownloaded(it.name, it.scanlator, manga.title, manga.source, false) }
+    return filter { downloadCache.isChapterDownloaded(it.name, it.scanlator, it.url, manga.title, manga.source, false) }
 }