Local manga in zip/cbz/folder format (#648)

* add local source

* small fixes

* change Chapter to SChapter and and Manga to SManga in ChapterRecognition.
Use ChapterRecognition.parseChapterNumber() to recognize chapter numbers.

* use thread poll

* update isImage()

* add isImage() function to DiskUtil

* improve cover handling

* Support external SD cards

* use R.string.app_name as root folder name

app/build.gradle

@@ -179,6 +179,9 @@ dependencies {
     // Crash reports
     compile 'ch.acra:acra:4.9.2'
 
+    // Sort
+    compile 'com.github.gpanther:java-nat-sort:natural-comparator-1.1'
+
     // UI
     compile 'com.dmitrymalkovich.android:material-design-dimens:1.4'
     compile 'com.github.dmytrodanylyk.android-process-button:library:1.0.4'

app/src/main/AndroidManifest.xml

@@ -76,6 +76,11 @@
                 android:resource="@xml/provider_paths" />
         </provider>
 
+        <provider
+            android:name="eu.kanade.tachiyomi.util.ZipContentProvider"
+            android:authorities="${applicationId}.zip-provider"
+            android:exported="false"></provider>
+
         <receiver
             android:name=".data.notification.NotificationReceiver"
             android:exported="false" />

app/src/main/java/eu/kanade/tachiyomi/data/glide/MangaModelLoader.kt

@@ -1,8 +1,10 @@
 package eu.kanade.tachiyomi.data.glide
 
 import android.content.Context
+import android.net.Uri
 import android.util.LruCache
 import com.bumptech.glide.Glide
+import com.bumptech.glide.Priority
 import com.bumptech.glide.load.data.DataFetcher
 import com.bumptech.glide.load.model.*
 import com.bumptech.glide.load.model.stream.StreamModelLoader
@@ -43,6 +45,12 @@ class MangaModelLoader(context: Context) : StreamModelLoader<Manga> {
     private val baseLoader = Glide.buildModelLoader(GlideUrl::class.java,
             InputStream::class.java, context)
 
+    /**
+     * Base file loader.
+     */
+    private val baseFileLoader = Glide.buildModelLoader(Uri::class.java,
+            InputStream::class.java, context)
+
     /**
      * LRU cache whose key is the thumbnail url of the manga, and the value contains the request url
      * and the file where it should be stored in case the manga is a favorite.
@@ -82,6 +90,18 @@ class MangaModelLoader(context: Context) : StreamModelLoader<Manga> {
             return null
         }
 
+        if (url!!.startsWith("file://")) {
+            val cover = File(url.substring(7))
+            val id = url + File.separator + cover.lastModified()
+            val rf = baseFileLoader.getResourceFetcher(Uri.fromFile(cover), width, height)
+            return object : DataFetcher<InputStream> {
+                override fun cleanup() = rf.cleanup()
+                override fun loadData(priority: Priority?): InputStream = rf.loadData(priority)
+                override fun cancel() = rf.cancel()
+                override fun getId() = id
+            }
+        }
+
         // Obtain the request url and the file for this url from the LRU cache, or calculate it
         // and add them to the cache.
         val (glideUrl, file) = lruCache.get(url) ?:

app/src/main/java/eu/kanade/tachiyomi/source/LocalSource.kt

@@ -0,0 +1,178 @@
+package eu.kanade.tachiyomi.source
+
+import android.content.Context
+import android.net.Uri
+import eu.kanade.tachiyomi.R
+import eu.kanade.tachiyomi.source.model.*
+import eu.kanade.tachiyomi.util.ChapterRecognition
+import eu.kanade.tachiyomi.util.DiskUtil
+import eu.kanade.tachiyomi.util.ZipContentProvider
+import net.greypanther.natsort.CaseInsensitiveSimpleNaturalComparator
+import rx.Observable
+import timber.log.Timber
+import java.io.File
+import java.io.FileInputStream
+import java.io.InputStream
+import java.util.*
+import java.util.concurrent.TimeUnit
+import java.util.zip.ZipEntry
+import java.util.zip.ZipFile
+
+class LocalSource(private val context: Context) : CatalogueSource {
+    companion object {
+        private val FILE_PROTOCOL = "file://"
+        private val COVER_NAME = "cover.jpg"
+        private val POPULAR_FILTERS = FilterList(OrderBy())
+        private val LATEST_FILTERS = FilterList(OrderBy().apply { state = Filter.Sort.Selection(1, false) })
+        private val LATEST_THRESHOLD = TimeUnit.MILLISECONDS.convert(7, TimeUnit.DAYS)
+        val ID = 0L
+
+        fun updateCover(context: Context, manga: SManga, input: InputStream): File? {
+            val dir = getBaseDirectories(context).firstOrNull()
+            if (dir == null) {
+                input.close()
+                return null
+            }
+            val cover = File("${dir.absolutePath}/${manga.url}", COVER_NAME)
+
+            // It might not exist if using the external SD card
+            cover.parentFile.mkdirs()
+            input.use {
+                cover.outputStream().use {
+                    input.copyTo(it)
+                }
+            }
+            return cover
+        }
+
+        private fun getBaseDirectories(context: Context): List<File> {
+            val c = File.separator + context.getString(R.string.app_name) + File.separator + "local"
+            return DiskUtil.getExternalStorages(context).map { File(it.absolutePath + c) }
+        }
+    }
+
+    override val id = ID
+    override val name = "LocalSource"
+    override val lang = "en"
+    override val supportsLatest = true
+
+    override fun toString() = context.getString(R.string.local_source)
+
+    override fun fetchMangaDetails(manga: SManga) = Observable.just(manga)
+
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        val chapters = getBaseDirectories(context)
+                .mapNotNull { File(it, manga.url).listFiles()?.toList() }
+                .flatten()
+                .filter { it.isDirectory || isSupportedFormat(it.extension) }
+                .map { chapterFile ->
+                    SChapter.create().apply {
+                        url = chapterFile.absolutePath
+                        val chapName = if (chapterFile.isDirectory) {
+                            chapterFile.name
+                        } else {
+                            chapterFile.nameWithoutExtension
+                        }
+                        val chapNameCut = chapName.replace(manga.title, "", true)
+                        name = if (chapNameCut.isEmpty()) chapName else chapNameCut
+                        date_upload = chapterFile.lastModified()
+                        ChapterRecognition.parseChapterNumber(this, manga)
+                    }
+                }
+
+        return Observable.just(chapters.sortedByDescending { it.chapter_number })
+    }
+
+    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
+        val chapFile = File(chapter.url)
+        if (chapFile.isDirectory) {
+            return Observable.just(chapFile.listFiles()
+                    .filter { !it.isDirectory && DiskUtil.isImage(it.name, { FileInputStream(it) }) }
+                    .sortedWith(Comparator<File> { t1, t2 -> CaseInsensitiveSimpleNaturalComparator.getInstance<String>().compare(t1.name, t2.name) })
+                    .mapIndexed { i, v -> Page(i, FILE_PROTOCOL + v.absolutePath, FILE_PROTOCOL + v.absolutePath, Uri.fromFile(v)).apply { status = Page.READY } })
+        } else {
+            val zip = ZipFile(chapFile)
+            return Observable.just(ZipFile(chapFile).entries().toList()
+                    .filter { !it.isDirectory && DiskUtil.isImage(it.name, { zip.getInputStream(it) }) }
+                    .sortedWith(Comparator<ZipEntry> { t1, t2 -> CaseInsensitiveSimpleNaturalComparator.getInstance<String>().compare(t1.name, t2.name) })
+                    .mapIndexed { i, v ->
+                        val path = "content://${ZipContentProvider.PROVIDER}${chapFile.absolutePath}!/${v.name}"
+                        Page(i, path, path, Uri.parse(path)).apply { status = Page.READY }
+                    })
+        }
+    }
+
+    override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", POPULAR_FILTERS)
+
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+        val baseDirs = getBaseDirectories(context)
+
+        val time = if (filters === LATEST_FILTERS) System.currentTimeMillis() - LATEST_THRESHOLD else 0L
+        var mangaDirs = baseDirs.mapNotNull { it.listFiles()?.toList() }
+                .flatten()
+                .filter { it.isDirectory && if (time == 0L) it.name.contains(query, ignoreCase = true) else it.lastModified() >= time }
+                .distinctBy { it.name }
+
+        val state = ((if (filters.isEmpty()) POPULAR_FILTERS else filters)[0] as OrderBy).state
+        when (state?.index) {
+            0 -> {
+                if (state!!.ascending)
+                    mangaDirs = mangaDirs.sortedBy { it.name.toLowerCase(Locale.ENGLISH) }
+                else
+                    mangaDirs = mangaDirs.sortedByDescending { it.name.toLowerCase(Locale.ENGLISH) }
+            }
+            1 -> {
+                if (state!!.ascending)
+                    mangaDirs = mangaDirs.sortedBy(File::lastModified)
+                else
+                    mangaDirs = mangaDirs.sortedByDescending(File::lastModified)
+            }
+        }
+
+        val mangas = mangaDirs.map { mangaDir ->
+            SManga.create().apply {
+                title = mangaDir.name
+                url = mangaDir.name
+
+                // Try to find the cover
+                for (dir in baseDirs) {
+                    val cover = File("${dir.absolutePath}/$url", COVER_NAME)
+                    if (cover.exists()) {
+                        thumbnail_url = FILE_PROTOCOL + cover.absolutePath
+                        break
+                    }
+                }
+
+                // Copy the cover from the first chapter found.
+                if (thumbnail_url == null) {
+                    val chapters = fetchChapterList(this).toBlocking().first()
+                    if (chapters.isNotEmpty()) {
+                        val url = fetchPageList(chapters.last()).toBlocking().first().firstOrNull()?.url
+                        if (url != null) {
+                            val input = context.contentResolver.openInputStream(Uri.parse(url))
+                            try {
+                                val dest = updateCover(context, this, input)
+                                thumbnail_url = dest?.let { FILE_PROTOCOL + it.absolutePath }
+                            } catch (e: Exception) {
+                                Timber.e(e)
+                            }
+                        }
+                    }
+                }
+
+                initialized = true
+            }
+        }
+        return Observable.just(MangasPage(mangas, false))
+    }
+
+    override fun fetchLatestUpdates(page: Int) = fetchSearchManga(page, "", LATEST_FILTERS)
+
+    private fun isSupportedFormat(extension: String): Boolean {
+        return extension.equals("zip", true) || extension.equals("cbz", true)
+    }
+
+    private class OrderBy : Filter.Sort("Order by", arrayOf("Title", "Date"), Filter.Sort.Selection(0, true))
+
+    override fun getFilterList() = FilterList(OrderBy())
+}

app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt

@@ -48,6 +48,7 @@ open class SourceManager(private val context: Context) {
     }
 
     private fun createInternalSources(): List<Source> = listOf(
+            LocalSource(context),
             Batoto(),
             Mangahere(),
             Mangafox(),

app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryPresenter.kt

@@ -13,6 +13,7 @@ import eu.kanade.tachiyomi.data.database.models.MangaCategory
 import eu.kanade.tachiyomi.data.download.DownloadManager
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.preference.getOrDefault
+import eu.kanade.tachiyomi.source.LocalSource
 import eu.kanade.tachiyomi.source.SourceManager
 import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter
 import eu.kanade.tachiyomi.util.combineLatest
@@ -345,6 +346,11 @@ class LibraryPresenter : BasePresenter<LibraryFragment>() {
      */
     @Throws(IOException::class)
     fun editCoverWithStream(inputStream: InputStream, manga: Manga): Boolean {
+        if (manga.source == LocalSource.ID) {
+            LocalSource.updateCover(context, manga, inputStream)
+            return true
+        }
+
         if (manga.thumbnail_url != null && manga.favorite) {
             coverCache.copyToCache(manga.thumbnail_url!!, inputStream)
             return true

app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt

@@ -15,6 +15,7 @@ import eu.kanade.tachiyomi.data.download.DownloadManager
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.track.TrackManager
 import eu.kanade.tachiyomi.data.track.TrackUpdateService
+import eu.kanade.tachiyomi.source.LocalSource
 import eu.kanade.tachiyomi.source.SourceManager
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.online.HttpSource
@@ -539,6 +540,13 @@ class ReaderPresenter : BasePresenter<ReaderActivity>() {
      */
     internal fun setImageAsCover(page: Page) {
         try {
+            if (manga.source == LocalSource.ID) {
+                val input = context.contentResolver.openInputStream(page.uri)
+                LocalSource.updateCover(context, manga, input)
+                context.toast(R.string.cover_updated)
+                return
+            }
+
             val thumbUrl = manga.thumbnail_url ?: throw Exception("Image url not found")
             if (manga.favorite) {
                 val input = context.contentResolver.openInputStream(page.uri)

app/src/main/java/eu/kanade/tachiyomi/ui/recently_read/RecentlyReadHolder.kt

@@ -50,7 +50,7 @@ class RecentlyReadHolder(view: View, private val adapter: RecentlyReadAdapter)
         // Set source + chapter title
         val formattedNumber = decimalFormat.format(chapter.chapter_number.toDouble())
         itemView.manga_source.text = itemView.context.getString(R.string.recent_manga_source)
-                .format(adapter.sourceManager.get(manga.source)?.name, formattedNumber)
+                .format(adapter.sourceManager.get(manga.source)?.toString(), formattedNumber)
 
         // Set last read timestamp title
         itemView.last_read.text = df.format(Date(history.last_read))

app/src/main/java/eu/kanade/tachiyomi/util/ChapterRecognition.kt

@@ -1,7 +1,7 @@
 package eu.kanade.tachiyomi.util
 
-import eu.kanade.tachiyomi.data.database.models.Chapter
+import eu.kanade.tachiyomi.source.model.SChapter
-import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.source.model.SManga
 
 /**
  * -R> = regex conversion.
@@ -37,7 +37,7 @@ object ChapterRecognition {
      */
     private val unwantedWhiteSpace = Regex("""(\s)(extra|special|omake)""")
 
-    fun parseChapterNumber(chapter: Chapter, manga: Manga) {
+    fun parseChapterNumber(chapter: SChapter, manga: SManga) {
         // If chapter number is known return.
         if (chapter.chapter_number == -2f || chapter.chapter_number > -1f)
             return
@@ -91,7 +91,7 @@ object ChapterRecognition {
      * @param chapter chapter object
      * @return true if volume is found
      */
-    fun updateChapter(match: MatchResult?, chapter: Chapter): Boolean {
+    fun updateChapter(match: MatchResult?, chapter: SChapter): Boolean {
         match?.let {
             val initial = it.groups[1]?.value?.toFloat()!!
             val subChapterDecimal = it.groups[2]?.value

app/src/main/java/eu/kanade/tachiyomi/util/DiskUtil.kt

@@ -1,11 +1,53 @@
 package eu.kanade.tachiyomi.util
 
+import android.content.Context
+import android.os.Environment
+import android.support.v4.content.ContextCompat
+import android.support.v4.os.EnvironmentCompat
 import java.io.File
+import java.io.InputStream
+import java.net.URLConnection
 import java.security.MessageDigest
 import java.security.NoSuchAlgorithmException
 
 object DiskUtil {
 
+    fun isImage(name: String, openStream: (() -> InputStream)? = null): Boolean {
+        val contentType = URLConnection.guessContentTypeFromName(name)
+        if (contentType != null)
+            return contentType.startsWith("image/")
+
+        if (openStream != null) try {
+            openStream.invoke().buffered().use {
+                var bytes = ByteArray(11)
+                it.mark(bytes.size)
+                var length = it.read(bytes, 0, bytes.size)
+                it.reset()
+                if (length == -1)
+                    return false
+                if (bytes[0] == 'G'.toByte() && bytes[1] == 'I'.toByte() && bytes[2] == 'F'.toByte() && bytes[3] == '8'.toByte()) {
+                    return true // image/gif
+                } else if (bytes[0] == 0x89.toByte() && bytes[1] == 0x50.toByte() && bytes[2] == 0x4E.toByte()
+                        && bytes[3] == 0x47.toByte() && bytes[4] == 0x0D.toByte() && bytes[5] == 0x0A.toByte()
+                        && bytes[6] == 0x1A.toByte() && bytes[7] == 0x0A.toByte()) {
+                    return true // image/png
+                } else if (bytes[0] == 0xFF.toByte() && bytes[1] == 0xD8.toByte() && bytes[2] == 0xFF.toByte()) {
+                    if (bytes[3] == 0xE0.toByte() || bytes[3] == 0xE1.toByte() && bytes[6] == 'E'.toByte()
+                            && bytes[7] == 'x'.toByte() && bytes[8] == 'i'.toByte()
+                            && bytes[9] == 'f'.toByte() && bytes[10] == 0.toByte()) {
+                        return true // image/jpeg
+                    } else if (bytes[3] == 0xEE.toByte()) {
+                        return true // image/jpg
+                    }
+                } else if (bytes[0] == 'W'.toByte() && bytes[1] == 'E'.toByte() && bytes[2] == 'B'.toByte() && bytes[3] == 'P'.toByte()) {
+                    return true // image/webp
+                }
+            }
+        } catch(e: Exception) {
+        }
+        return false
+    }
+
     fun hashKeyForDisk(key: String): String {
         return try {
             val bytes = MessageDigest.getInstance("MD5").digest(key.toByteArray())
@@ -31,9 +73,26 @@ object DiskUtil {
         return size
     }
 
+    /**
+     * Returns the root folders of all the available external storages.
+     */
+    fun getExternalStorages(context: Context): List<File> {
+        return ContextCompat.getExternalFilesDirs(context, null)
+                .filterNotNull()
+                .mapNotNull {
+                    val file = File(it.absolutePath.substringBefore("/Android/"))
+                    val state = EnvironmentCompat.getStorageState(file)
+                    if (state == Environment.MEDIA_MOUNTED || state == Environment.MEDIA_MOUNTED_READ_ONLY) {
+                        file
+                    } else {
+                        null
+                    }
+                }
+    }
+
     /**
      * Mutate the given filename to make it valid for a FAT filesystem,
-     * replacing any invalid characters with "_". This method doesn't allow private files (starting
+     * replacing any invalid characters with "_". This method doesn't allow hidden files (starting
      * with a dot), but you can manually add it later.
      */
     fun buildValidFilename(origName: String): String {

app/src/main/java/eu/kanade/tachiyomi/util/ZipContentProvider.kt

@@ -0,0 +1,71 @@
+package eu.kanade.tachiyomi.util
+
+import android.content.ContentProvider
+import android.content.ContentValues
+import android.content.res.AssetFileDescriptor
+import android.database.Cursor
+import android.net.Uri
+import android.os.ParcelFileDescriptor
+import eu.kanade.tachiyomi.BuildConfig
+import timber.log.Timber
+import java.io.IOException
+import java.net.URL
+import java.net.URLConnection
+import java.util.concurrent.Executors
+
+class ZipContentProvider : ContentProvider() {
+
+    private val pool by lazy { Executors.newCachedThreadPool() }
+
+    companion object {
+        const val PROVIDER = "${BuildConfig.APPLICATION_ID}.zip-provider"
+    }
+
+    override fun onCreate(): Boolean {
+        return true
+    }
+
+    override fun getType(uri: Uri): String? {
+        return URLConnection.guessContentTypeFromName(uri.toString())
+    }
+
+    override fun openAssetFile(uri: Uri, mode: String): AssetFileDescriptor? {
+        try {
+            val url = "jar:file://" + uri.toString().substringAfter("content://$PROVIDER")
+            val input = URL(url).openStream()
+            val pipe = ParcelFileDescriptor.createPipe()
+            pool.execute {
+                try {
+                    val output = ParcelFileDescriptor.AutoCloseOutputStream(pipe[1])
+                    input.use {
+                        output.use {
+                            input.copyTo(output)
+                            output.flush()
+                        }
+                    }
+                } catch (e: IOException) {
+                    Timber.e(e)
+                }
+            }
+            return AssetFileDescriptor(pipe[0], 0, -1)
+        } catch (e: IOException) {
+            return null
+        }
+    }
+
+    override fun query(p0: Uri?, p1: Array<out String>?, p2: String?, p3: Array<out String>?, p4: String?): Cursor? {
+        return null
+    }
+
+    override fun insert(p0: Uri?, p1: ContentValues?): Uri {
+        throw UnsupportedOperationException("not implemented")
+    }
+
+    override fun update(p0: Uri?, p1: ContentValues?, p2: String?, p3: Array<out String>?): Int {
+        throw UnsupportedOperationException("not implemented")
+    }
+
+    override fun delete(p0: Uri?, p1: String?, p2: Array<out String>?): Int {
+        throw UnsupportedOperationException("not implemented")
+    }
+}

app/src/main/res/values-bg/strings.xml

@@ -1,6 +1,4 @@
 <resources>
-    <string name="app_name">Tachiyomi</string>
-
     <string name="name">Име</string>
 
     <!-- Activities and fragments labels (toolbar title) -->

app/src/main/res/values-es/strings.xml

@@ -1,6 +1,4 @@
 <resources>
-    <string name="app_name">Tachiyomi</string>
-
     <string name="name">Nombre</string>
 
     <!-- Activities and fragments labels (toolbar title) -->

app/src/main/res/values-fr/strings.xml

@@ -1,6 +1,4 @@
 <resources>
-    <string name="app_name">Tachiyomi</string>
-
     <string name="name">Nom</string>
 
     <!-- Activities and fragments labels (toolbar title) -->

app/src/main/res/values-it/strings.xml

@@ -1,6 +1,4 @@
 <resources>
-    <string name="app_name">Tachiyomi</string>
-
     <string name="name">Nome</string>
 
     <!-- Activities and fragments labels (toolbar title) -->

app/src/main/res/values-pt/strings.xml

@@ -1,7 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <resources>
-    <string name="app_name">Tachiyomi</string>
-
     <string name="name">Nome</string>
 
     <!-- Activities and fragments labels (toolbar title) -->

app/src/main/res/values-ru/strings.xml

@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <resources>
-    <string name="app_name">Tachiyomi</string>
     <string name="action_add">Добавить</string>
     <string name="action_add_category">Добавить категорию</string>
     <string name="action_add_to_home_screen">Добавить на домашний экран</string>

app/src/main/res/values/strings.xml

@@ -1,5 +1,5 @@
 <resources>
-    <string name="app_name">Tachiyomi</string>
+    <string name="app_name" translatable="false">Tachiyomi</string>
 
     <string name="name">Name</string>
 
@@ -224,6 +224,7 @@
     <string name="select_source">Select a source</string>
     <string name="no_valid_sources">Please enable at least one valid source</string>
     <string name="no_more_results">No more results</string>
+    <string name="local_source">Local manga</string>
 
     <!-- Manga activity -->
     <string name="manga_not_in_db">This manga was removed from the database!</string>