Move Local Source to separate module (#9152)

* Move Local Source to separate module

* Review changes

source-local/.gitignore

@@ -0,0 +1 @@
+/build

source-local/build.gradle.kts

@@ -0,0 +1,29 @@
+plugins {
+    id("com.android.library")
+    kotlin("android")
+}
+
+android {
+    namespace = "tachiyomi.source.local"
+
+    defaultConfig {
+
+        testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+        consumerProguardFiles("consumer-rules.pro")
+    }
+}
+
+dependencies {
+
+    implementation(project(":source-api"))
+    implementation(project(":core"))
+    implementation(project(":core-metadata"))
+
+    // Move ChapterRecognition to separate module?
+    implementation(project(":domain"))
+
+    implementation(kotlinx.bundles.serialization)
+
+    implementation(libs.unifile)
+    implementation(libs.junrar)
+}

source-local/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

source-local/src/main/AndroidManifest.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest />

source-local/src/main/java/tachiyomi/source/local/LocalSource.kt

@@ -0,0 +1,351 @@
+package tachiyomi.source.local
+
+import android.content.Context
+import eu.kanade.domain.manga.model.COMIC_INFO_FILE
+import eu.kanade.domain.manga.model.ComicInfo
+import eu.kanade.domain.manga.model.copyFromComicInfo
+import eu.kanade.tachiyomi.source.CatalogueSource
+import eu.kanade.tachiyomi.source.UnmeteredSource
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
+import eu.kanade.tachiyomi.util.storage.EpubFile
+import kotlinx.coroutines.runBlocking
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.decodeFromStream
+import logcat.LogPriority
+import nl.adaptivity.xmlutil.AndroidXmlReader
+import nl.adaptivity.xmlutil.serialization.XML
+import rx.Observable
+import tachiyomi.core.metadata.tachiyomi.MangaDetails
+import tachiyomi.core.util.lang.withIOContext
+import tachiyomi.core.util.system.ImageUtil
+import tachiyomi.core.util.system.logcat
+import tachiyomi.domain.chapter.service.ChapterRecognition
+import tachiyomi.source.local.filter.OrderBy
+import tachiyomi.source.local.image.LocalCoverManager
+import tachiyomi.source.local.io.Archive
+import tachiyomi.source.local.io.Format
+import tachiyomi.source.local.io.LocalSourceFileSystem
+import tachiyomi.source.local.metadata.fillChapterMetadata
+import tachiyomi.source.local.metadata.fillMangaMetadata
+import uy.kohesive.injekt.injectLazy
+import java.io.File
+import java.io.FileInputStream
+import java.io.InputStream
+import java.nio.charset.StandardCharsets
+import java.util.concurrent.TimeUnit
+import java.util.zip.ZipFile
+import com.github.junrar.Archive as JunrarArchive
+
+class LocalSource(
+    private val context: Context,
+    private val fileSystem: LocalSourceFileSystem,
+    private val coverManager: LocalCoverManager,
+) : CatalogueSource, UnmeteredSource {
+
+    private val json: Json by injectLazy()
+    private val xml: XML by injectLazy()
+
+    private val POPULAR_FILTERS = FilterList(OrderBy.Popular(context))
+    private val LATEST_FILTERS = FilterList(OrderBy.Latest(context))
+
+    override val name: String = context.getString(R.string.local_source)
+
+    override val id: Long = ID
+
+    override val lang: String = "other"
+
+    override fun toString() = name
+
+    override val supportsLatest: Boolean = true
+
+    // Browse related
+    override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", POPULAR_FILTERS)
+
+    override fun fetchLatestUpdates(page: Int) = fetchSearchManga(page, "", LATEST_FILTERS)
+
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+        val baseDirsFiles = fileSystem.getFilesInBaseDirectories()
+        val lastModifiedLimit by lazy { if (filters === LATEST_FILTERS) System.currentTimeMillis() - LATEST_THRESHOLD else 0L }
+        var mangaDirs = baseDirsFiles
+            // Filter out files that are hidden and is not a folder
+            .filter { it.isDirectory && !it.name.startsWith('.') }
+            .distinctBy { it.name }
+            .filter { // Filter by query or last modified
+                if (lastModifiedLimit == 0L) {
+                    it.name.contains(query, ignoreCase = true)
+                } else {
+                    it.lastModified() >= lastModifiedLimit
+                }
+            }
+
+        filters.forEach { filter ->
+            when (filter) {
+                is OrderBy.Popular -> {
+                    mangaDirs = if (filter.state!!.ascending) {
+                        mangaDirs.sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.name })
+                    } else {
+                        mangaDirs.sortedWith(compareByDescending(String.CASE_INSENSITIVE_ORDER) { it.name })
+                    }
+                }
+                is OrderBy.Latest -> {
+                    mangaDirs = if (filter.state!!.ascending) {
+                        mangaDirs.sortedBy(File::lastModified)
+                    } else {
+                        mangaDirs.sortedByDescending(File::lastModified)
+                    }
+                }
+
+                else -> {
+                    /* Do nothing */
+                }
+            }
+        }
+
+        // Transform mangaDirs to list of SManga
+        val mangas = mangaDirs.map { mangaDir ->
+            SManga.create().apply {
+                title = mangaDir.name
+                url = mangaDir.name
+
+                // Try to find the cover
+                coverManager.find(mangaDir.name)
+                    ?.takeIf(File::exists)
+                    ?.let { thumbnail_url = it.absolutePath }
+            }
+        }
+
+        // Fetch chapters of all the manga
+        mangas.forEach { manga ->
+            runBlocking {
+                val chapters = getChapterList(manga)
+                if (chapters.isNotEmpty()) {
+                    val chapter = chapters.last()
+                    val format = getFormat(chapter)
+
+                    if (format is Format.Epub) {
+                        EpubFile(format.file).use { epub ->
+                            epub.fillMangaMetadata(manga)
+                        }
+                    }
+
+                    // Copy the cover from the first chapter found if not available
+                    if (manga.thumbnail_url == null) {
+                        updateCover(chapter, manga)
+                    }
+                }
+            }
+        }
+
+        return Observable.just(MangasPage(mangas.toList(), false))
+    }
+
+    // Manga details related
+    override suspend fun getMangaDetails(manga: SManga): SManga = withIOContext {
+        coverManager.find(manga.url)?.let {
+            manga.thumbnail_url = it.absolutePath
+        }
+
+        // Augment manga details based on metadata files
+        try {
+            val mangaDirFiles = fileSystem.getFilesInMangaDirectory(manga.url).toList()
+
+            val comicInfoFile = mangaDirFiles
+                .firstOrNull { it.name == COMIC_INFO_FILE }
+            val noXmlFile = mangaDirFiles
+                .firstOrNull { it.name == ".noxml" }
+            val legacyJsonDetailsFile = mangaDirFiles
+                .firstOrNull { it.extension == "json" }
+
+            when {
+                // Top level ComicInfo.xml
+                comicInfoFile != null -> {
+                    noXmlFile?.delete()
+                    setMangaDetailsFromComicInfoFile(comicInfoFile.inputStream(), manga)
+                }
+
+                // TODO: automatically convert these to ComicInfo.xml
+                legacyJsonDetailsFile != null -> {
+                    json.decodeFromStream<MangaDetails>(legacyJsonDetailsFile.inputStream()).run {
+                        title?.let { manga.title = it }
+                        author?.let { manga.author = it }
+                        artist?.let { manga.artist = it }
+                        description?.let { manga.description = it }
+                        genre?.let { manga.genre = it.joinToString() }
+                        status?.let { manga.status = it }
+                    }
+                }
+
+                // Copy ComicInfo.xml from chapter archive to top level if found
+                noXmlFile == null -> {
+                    val chapterArchives = mangaDirFiles
+                        .filter(Archive::isSupported)
+                        .toList()
+
+                    val mangaDir = fileSystem.getMangaDirectory(manga.url)
+                    val folderPath = mangaDir?.absolutePath
+
+                    val copiedFile = copyComicInfoFileFromArchive(chapterArchives, folderPath)
+                    if (copiedFile != null) {
+                        setMangaDetailsFromComicInfoFile(copiedFile.inputStream(), manga)
+                    } else {
+                        // Avoid re-scanning
+                        File("$folderPath/.noxml").createNewFile()
+                    }
+                }
+            }
+        } catch (e: Throwable) {
+            logcat(LogPriority.ERROR, e) { "Error setting manga details from local metadata for ${manga.title}" }
+        }
+
+        return@withIOContext manga
+    }
+
+    private fun copyComicInfoFileFromArchive(chapterArchives: List<File>, folderPath: String?): File? {
+        for (chapter in chapterArchives) {
+            when (Format.valueOf(chapter)) {
+                is Format.Zip -> {
+                    ZipFile(chapter).use { zip: ZipFile ->
+                        zip.getEntry(COMIC_INFO_FILE)?.let { comicInfoFile ->
+                            zip.getInputStream(comicInfoFile).buffered().use { stream ->
+                                return copyComicInfoFile(stream, folderPath)
+                            }
+                        }
+                    }
+                }
+                is Format.Rar -> {
+                    JunrarArchive(chapter).use { rar ->
+                        rar.fileHeaders.firstOrNull { it.fileName == COMIC_INFO_FILE }?.let { comicInfoFile ->
+                            rar.getInputStream(comicInfoFile).buffered().use { stream ->
+                                return copyComicInfoFile(stream, folderPath)
+                            }
+                        }
+                    }
+                }
+                else -> {}
+            }
+        }
+        return null
+    }
+
+    private fun copyComicInfoFile(comicInfoFileStream: InputStream, folderPath: String?): File {
+        return File("$folderPath/$COMIC_INFO_FILE").apply {
+            outputStream().use { outputStream ->
+                comicInfoFileStream.use { it.copyTo(outputStream) }
+            }
+        }
+    }
+
+    private fun setMangaDetailsFromComicInfoFile(stream: InputStream, manga: SManga) {
+        val comicInfo = AndroidXmlReader(stream, StandardCharsets.UTF_8.name()).use {
+            xml.decodeFromReader<ComicInfo>(it)
+        }
+
+        manga.copyFromComicInfo(comicInfo)
+    }
+
+    // Chapters
+    override suspend fun getChapterList(manga: SManga): List<SChapter> {
+        return fileSystem.getFilesInMangaDirectory(manga.url)
+            // Only keep supported formats
+            .filter { it.isDirectory || Archive.isSupported(it) }
+            .map { chapterFile ->
+                SChapter.create().apply {
+                    url = "${manga.url}/${chapterFile.name}"
+                    name = if (chapterFile.isDirectory) {
+                        chapterFile.name
+                    } else {
+                        chapterFile.nameWithoutExtension
+                    }
+                    date_upload = chapterFile.lastModified()
+                    chapter_number = ChapterRecognition.parseChapterNumber(manga.title, this.name, this.chapter_number)
+
+                    val format = Format.valueOf(chapterFile)
+                    if (format is Format.Epub) {
+                        EpubFile(format.file).use { epub ->
+                            epub.fillChapterMetadata(this)
+                        }
+                    }
+                }
+            }
+            .sortedWith { c1, c2 ->
+                val c = c2.chapter_number.compareTo(c1.chapter_number)
+                if (c == 0) c2.name.compareToCaseInsensitiveNaturalOrder(c1.name) else c
+            }
+            .toList()
+    }
+
+    // Filters
+    override fun getFilterList() = FilterList(OrderBy.Popular(context))
+
+    // Unused stuff
+    override suspend fun getPageList(chapter: SChapter) = throw UnsupportedOperationException("Unused")
+
+    fun getFormat(chapter: SChapter): Format {
+        try {
+            return fileSystem.getBaseDirectories()
+                .map { directory -> File(directory, chapter.url) }
+                .find { chapterFile -> chapterFile.exists() }
+                ?.let(Format.Companion::valueOf)
+                ?: throw Exception(context.getString(R.string.chapter_not_found))
+        } catch (e: Format.UnknownFormatException) {
+            throw Exception(context.getString(R.string.local_invalid_format))
+        } catch (e: Exception) {
+            throw e
+        }
+    }
+
+    private fun updateCover(chapter: SChapter, manga: SManga): File? {
+        return try {
+            when (val format = getFormat(chapter)) {
+                is Format.Directory -> {
+                    val entry = format.file.listFiles()
+                        ?.sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
+                        ?.find { !it.isDirectory && ImageUtil.isImage(it.name) { FileInputStream(it) } }
+
+                    entry?.let { coverManager.update(manga, it.inputStream()) }
+                }
+                is Format.Zip -> {
+                    ZipFile(format.file).use { zip ->
+                        val entry = zip.entries().toList()
+                            .sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
+                            .find { !it.isDirectory && ImageUtil.isImage(it.name) { zip.getInputStream(it) } }
+
+                        entry?.let { coverManager.update(manga, zip.getInputStream(it)) }
+                    }
+                }
+                is Format.Rar -> {
+                    JunrarArchive(format.file).use { archive ->
+                        val entry = archive.fileHeaders
+                            .sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) }
+                            .find { !it.isDirectory && ImageUtil.isImage(it.fileName) { archive.getInputStream(it) } }
+
+                        entry?.let { coverManager.update(manga, archive.getInputStream(it)) }
+                    }
+                }
+                is Format.Epub -> {
+                    EpubFile(format.file).use { epub ->
+                        val entry = epub.getImagesFromPages()
+                            .firstOrNull()
+                            ?.let { epub.getEntry(it) }
+
+                        entry?.let { coverManager.update(manga, epub.getInputStream(it)) }
+                    }
+                }
+            }
+        } catch (e: Throwable) {
+            logcat(LogPriority.ERROR, e) { "Error updating cover for ${manga.title}" }
+            null
+        }
+    }
+
+    companion object {
+        const val ID = 0L
+        const val HELP_URL = "https://tachiyomi.org/help/guides/local-manga/"
+
+        private val LATEST_THRESHOLD = TimeUnit.MILLISECONDS.convert(7, TimeUnit.DAYS)
+    }
+}

source-local/src/main/java/tachiyomi/source/local/filter/OrderBy.kt

@@ -0,0 +1,14 @@
+package tachiyomi.source.local.filter
+
+import android.content.Context
+import eu.kanade.tachiyomi.source.model.Filter
+import tachiyomi.source.local.R
+
+sealed class OrderBy(context: Context, selection: Selection) : Filter.Sort(
+    context.getString(R.string.local_filter_order_by),
+    arrayOf(context.getString(R.string.title), context.getString(R.string.date)),
+    selection,
+) {
+    class Popular(context: Context) : OrderBy(context, Selection(0, true))
+    class Latest(context: Context) : OrderBy(context, Selection(1, false))
+}

source-local/src/main/java/tachiyomi/source/local/image/AndroidLocalCoverManager.kt

@@ -0,0 +1,55 @@
+package tachiyomi.source.local.image
+
+import android.content.Context
+import com.hippo.unifile.UniFile
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.storage.DiskUtil
+import tachiyomi.core.util.system.ImageUtil
+import tachiyomi.source.local.io.LocalSourceFileSystem
+import java.io.File
+import java.io.InputStream
+
+private const val DEFAULT_COVER_NAME = "cover.jpg"
+
+class AndroidLocalCoverManager(
+    private val context: Context,
+    private val fileSystem: LocalSourceFileSystem,
+) : LocalCoverManager {
+
+    override fun find(mangaUrl: String): File? {
+        return fileSystem.getFilesInMangaDirectory(mangaUrl)
+            // Get all file whose names start with 'cover'
+            .filter { it.isFile && it.nameWithoutExtension.equals("cover", ignoreCase = true) }
+            // Get the first actual image
+            .firstOrNull {
+                ImageUtil.isImage(it.name) { it.inputStream() }
+            }
+    }
+
+    override fun update(manga: SManga, inputStream: InputStream): File? {
+        val directory = fileSystem.getMangaDirectory(manga.url)
+        if (directory == null) {
+            inputStream.close()
+            return null
+        }
+
+        var targetFile = find(manga.url)
+        if (targetFile == null) {
+            targetFile = File(directory.absolutePath, DEFAULT_COVER_NAME)
+            targetFile.createNewFile()
+        }
+
+        // It might not exist at this point
+        targetFile.parentFile?.mkdirs()
+        inputStream.use { input ->
+            targetFile.outputStream().use { output ->
+                input.copyTo(output)
+            }
+        }
+
+        DiskUtil.createNoMediaFile(UniFile.fromFile(directory), context)
+
+        manga.thumbnail_url = targetFile.absolutePath
+        return targetFile
+    }
+}

source-local/src/main/java/tachiyomi/source/local/image/LocalCoverManager.kt

@@ -0,0 +1,12 @@
+package tachiyomi.source.local.image
+
+import eu.kanade.tachiyomi.source.model.SManga
+import java.io.File
+import java.io.InputStream
+
+interface LocalCoverManager {
+
+    fun find(mangaUrl: String): File?
+
+    fun update(manga: SManga, inputStream: InputStream): File?
+}

source-local/src/main/java/tachiyomi/source/local/io/AndroidLocalSourceFileSystem.kt

@@ -0,0 +1,39 @@
+package tachiyomi.source.local.io
+
+import android.content.Context
+import eu.kanade.tachiyomi.util.storage.DiskUtil
+import tachiyomi.source.local.R
+import java.io.File
+
+class AndroidLocalSourceFileSystem(
+    private val context: Context,
+) : LocalSourceFileSystem {
+
+    private val baseFolderLocation = "${context.getString(R.string.app_name)}${File.separator}local"
+
+    override fun getBaseDirectories(): Sequence<File> {
+        return DiskUtil.getExternalStorages(context)
+            .map { File(it.absolutePath, baseFolderLocation) }
+            .asSequence()
+    }
+
+    override fun getFilesInBaseDirectories(): Sequence<File> {
+        return getBaseDirectories()
+            // Get all the files inside all baseDir
+            .flatMap { it.listFiles().orEmpty().toList() }
+    }
+
+    override fun getMangaDirectory(name: String): File? {
+        return getFilesInBaseDirectories()
+            // Get the first mangaDir or null
+            .firstOrNull { it.isDirectory && it.name == name }
+    }
+
+    override fun getFilesInMangaDirectory(name: String): Sequence<File> {
+        return getFilesInBaseDirectories()
+            // Filter out ones that are not related to the manga and is not a directory
+            .filter { it.isDirectory && it.name == name }
+            // Get all the files inside the filtered folders
+            .flatMap { it.listFiles().orEmpty().toList() }
+    }
+}

source-local/src/main/java/tachiyomi/source/local/io/Archive.kt

@@ -0,0 +1,12 @@
+package tachiyomi.source.local.io
+
+import java.io.File
+
+object Archive {
+
+    private val SUPPORTED_ARCHIVE_TYPES = listOf("zip", "cbz", "rar", "cbr", "epub")
+
+    fun isSupported(file: File): Boolean = with(file) {
+        return extension.lowercase() in SUPPORTED_ARCHIVE_TYPES
+    }
+}

source-local/src/main/java/tachiyomi/source/local/io/Format.kt

@@ -0,0 +1,25 @@
+package tachiyomi.source.local.io
+
+import java.io.File
+
+sealed class Format {
+    data class Directory(val file: File) : Format()
+    data class Zip(val file: File) : Format()
+    data class Rar(val file: File) : Format()
+    data class Epub(val file: File) : Format()
+
+    class UnknownFormatException : Exception()
+
+    companion object {
+
+        fun valueOf(file: File) = with(file) {
+            when {
+                isDirectory -> Directory(this)
+                extension.equals("zip", true) || extension.equals("cbz", true) -> Zip(this)
+                extension.equals("rar", true) || extension.equals("cbr", true) -> Rar(this)
+                extension.equals("epub", true) -> Epub(this)
+                else -> throw UnknownFormatException()
+            }
+        }
+    }
+}

source-local/src/main/java/tachiyomi/source/local/io/LocalSourceFileSystem.kt

@@ -0,0 +1,14 @@
+package tachiyomi.source.local.io
+
+import java.io.File
+
+interface LocalSourceFileSystem {
+
+    fun getBaseDirectories(): Sequence<File>
+
+    fun getFilesInBaseDirectories(): Sequence<File>
+
+    fun getMangaDirectory(name: String): File?
+
+    fun getFilesInMangaDirectory(name: String): Sequence<File>
+}

source-local/src/main/java/tachiyomi/source/local/metadata/EpubFile.kt

@@ -0,0 +1,60 @@
+package tachiyomi.source.local.metadata
+
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.storage.EpubFile
+import java.text.ParseException
+import java.text.SimpleDateFormat
+import java.util.Locale
+
+/**
+ * Fills manga metadata using this epub file's metadata.
+ */
+fun EpubFile.fillMangaMetadata(manga: SManga) {
+    val ref = getPackageHref()
+    val doc = getPackageDocument(ref)
+
+    val creator = doc.getElementsByTag("dc:creator").first()
+    val description = doc.getElementsByTag("dc:description").first()
+
+    manga.author = creator?.text()
+    manga.description = description?.text()
+}
+
+/**
+ * Fills chapter metadata using this epub file's metadata.
+ */
+fun EpubFile.fillChapterMetadata(chapter: SChapter) {
+    val ref = getPackageHref()
+    val doc = getPackageDocument(ref)
+
+    val title = doc.getElementsByTag("dc:title").first()
+    val publisher = doc.getElementsByTag("dc:publisher").first()
+    val creator = doc.getElementsByTag("dc:creator").first()
+    var date = doc.getElementsByTag("dc:date").first()
+    if (date == null) {
+        date = doc.select("meta[property=dcterms:modified]").first()
+    }
+
+    if (title != null) {
+        chapter.name = title.text()
+    }
+
+    if (publisher != null) {
+        chapter.scanlator = publisher.text()
+    } else if (creator != null) {
+        chapter.scanlator = creator.text()
+    }
+
+    if (date != null) {
+        val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.getDefault())
+        try {
+            val parsedDate = dateFormat.parse(date.text())
+            if (parsedDate != null) {
+                chapter.date_upload = parsedDate.time
+            }
+        } catch (e: ParseException) {
+            // Empty
+        }
+    }
+}