Initial work on SmartSearch

2025-10-31 14:27:57 +01:00 · 2019-07-29 02:12:30 -04:00
parent b5263a6968
commit 8934d251d9
12 changed files with 361 additions and 15 deletions
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -253,9 +253,13 @@ dependencies {
    implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
    implementation "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"

-    final coroutines_version = '1.2.0'
+    final coroutines_version = '1.3.0-RC'
    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutines_version"
    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$coroutines_version"
+    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-reactive:$coroutines_version"
+
+    // Text distance (EH)
+    implementation 'info.debatty:java-string-similarity:1.2.1'

    // Pin lock view (EH)
    implementation 'com.andrognito.pinlockview:pinlockview:2.1.0'
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/CatalogueController.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/CatalogueController.kt
@@ -1,6 +1,8 @@
 package eu.kanade.tachiyomi.ui.catalogue

 import android.Manifest.permission.WRITE_EXTERNAL_STORAGE
+import android.os.Bundle
+import android.os.Parcelable
 import android.support.v7.widget.LinearLayoutManager
 import android.support.v7.widget.SearchView
 import android.view.*
@@ -23,6 +25,8 @@ import eu.kanade.tachiyomi.ui.catalogue.global_search.CatalogueSearchController
 import eu.kanade.tachiyomi.ui.catalogue.latest.LatestUpdatesController
 import eu.kanade.tachiyomi.ui.setting.SettingsSourcesController
 import eu.kanade.tachiyomi.widget.preference.SourceLoginDialog
+import exh.ui.smartsearch.SmartSearchController
+import kotlinx.android.parcel.Parcelize
 import kotlinx.android.synthetic.main.catalogue_main_controller.*
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
@@ -34,7 +38,7 @@ import uy.kohesive.injekt.api.get
 * [CatalogueAdapter.OnBrowseClickListener] call function data on browse item click.
 * [CatalogueAdapter.OnLatestClickListener] call function data on latest item click
 */
-class CatalogueController : NucleusController<CataloguePresenter>(),
+class CatalogueController(bundle: Bundle? = null) : NucleusController<CataloguePresenter>(bundle),
        SourceLoginDialog.Listener,
        FlexibleAdapter.OnItemClickListener,
        CatalogueAdapter.OnBrowseClickListener,
@@ -50,12 +54,18 @@ class CatalogueController : NucleusController<CataloguePresenter>(),
     */
    private var adapter: CatalogueAdapter? = null

+    private val smartSearchConfig: SmartSearchConfig? = args.getParcelable(SMART_SEARCH_CONFIG)
+
+    // EXH -->
+    private val mode = if(smartSearchConfig == null) Mode.CATALOGUE else Mode.SMART_SEARCH
+    // EXH <--
+
    /**
     * Called when controller is initialized.
     */
    init {
        // Enable the option menu
-        setHasOptionsMenu(true)
+        setHasOptionsMenu(mode == Mode.CATALOGUE)
    }

    /**
@@ -64,7 +74,10 @@ class CatalogueController : NucleusController<CataloguePresenter>(),
     * @return title.
     */
    override fun getTitle(): String? {
-        return applicationContext?.getString(R.string.label_catalogues)
+        return when(mode) {
+            Mode.CATALOGUE -> applicationContext?.getString(R.string.label_catalogues)
+            Mode.SMART_SEARCH -> "Find in another source"
+        }
    }

    /**
@@ -73,7 +86,7 @@ class CatalogueController : NucleusController<CataloguePresenter>(),
     * @return instance of [CataloguePresenter]
     */
    override fun createPresenter(): CataloguePresenter {
-        return CataloguePresenter()
+        return CataloguePresenter(controllerMode = mode)
    }

    /**
@@ -140,8 +153,16 @@ class CatalogueController : NucleusController<CataloguePresenter>(),
            dialog.targetController = this
            dialog.showDialog(router)
        } else {
-            // Open the catalogue view.
-            openCatalogue(source, BrowseCatalogueController(source))
+            when(mode) {
+                Mode.CATALOGUE -> {
+                    // Open the catalogue view.
+                    openCatalogue(source, BrowseCatalogueController(source))
+                }
+                Mode.SMART_SEARCH -> router.pushController(SmartSearchController(Bundle().apply {
+                    putLong(SmartSearchController.ARG_SOURCE_ID, source.id)
+                    putParcelable(SmartSearchController.ARG_SMART_SEARCH_CONFIG, smartSearchConfig)
+                }).withFadeTransaction())
+            }
        }
        return false
    }
@@ -233,4 +254,18 @@ class CatalogueController : NucleusController<CataloguePresenter>(),
    }

    class SettingsSourcesFadeChangeHandler : FadeChangeHandler()
+
+    // EXH -->
+    @Parcelize
+    data class SmartSearchConfig(val title: String) : Parcelable
+    // EXH <--
+
+    enum class Mode {
+        CATALOGUE,
+        SMART_SEARCH
+    }
+
+    companion object {
+        const val SMART_SEARCH_CONFIG = "SMART_SEARCH_CONFIG"
+    }
 }
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/CataloguePresenter.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/CataloguePresenter.kt
@@ -24,7 +24,8 @@ import java.util.concurrent.TimeUnit
 */
 class CataloguePresenter(
        val sourceManager: SourceManager = Injekt.get(),
-        private val preferences: PreferencesHelper = Injekt.get()
+        private val preferences: PreferencesHelper = Injekt.get(),
+        private val controllerMode: CatalogueController.Mode
 ) : BasePresenter<CatalogueController>() {

    /**
@@ -62,7 +63,7 @@ class CataloguePresenter(
        val byLang = sources.groupByTo(map, { it.lang })
        val sourceItems = byLang.flatMap {
            val langItem = LangItem(it.key)
-            it.value.map { source -> SourceItem(source, langItem) }
+            it.value.map { source -> SourceItem(source, langItem, controllerMode == CatalogueController.Mode.CATALOGUE) }
        }

        sourceSubscription = Observable.just(sourceItems)
@@ -77,7 +78,7 @@ class CataloguePresenter(
                sharedObs.take(1),
                sharedObs.skip(1).delay(500, TimeUnit.MILLISECONDS, AndroidSchedulers.mainThread()))
                .distinctUntilChanged()
-                .map { (sourceManager.get(it) as? CatalogueSource)?.let { SourceItem(it) } }
+                .map { (sourceManager.get(it) as? CatalogueSource)?.let { SourceItem(it, showButtons = controllerMode == CatalogueController.Mode.CATALOGUE) } }
                .subscribeLatestCache(CatalogueController::setLastUsedSource)
    }

--- a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/SourceHolder.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/SourceHolder.kt
@@ -11,7 +11,7 @@ import eu.kanade.tachiyomi.util.visible
 import io.github.mthli.slice.Slice
 import kotlinx.android.synthetic.main.catalogue_main_controller_card_item.*

-class SourceHolder(view: View, override val adapter: CatalogueAdapter) :
+class SourceHolder(view: View, override val adapter: CatalogueAdapter, val showButtons: Boolean) :
        BaseFlexibleViewHolder(view, adapter),
        SlicedHolder {

@@ -30,6 +30,11 @@ class SourceHolder(view: View, override val adapter: CatalogueAdapter) :
        source_latest.setOnClickListener {
            adapter.latestClickListener.onLatestClick(adapterPosition)
        }
+
+        if(!showButtons) {
+            source_browse.gone()
+            source_latest.gone()
+        }
    }

    fun bind(item: SourceItem) {
@@ -50,7 +55,7 @@ class SourceHolder(view: View, override val adapter: CatalogueAdapter) :
            source_latest.gone()
        } else {
            source_browse.setText(R.string.browse)
-            if (source.supportsLatest) {
+            if (source.supportsLatest && showButtons) {
                source_latest.visible()
            } else {
                source_latest.gone()
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/SourceItem.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/SourceItem.kt
@@ -14,7 +14,7 @@ import eu.kanade.tachiyomi.source.CatalogueSource
 * @param source Instance of [CatalogueSource] containing source information.
 * @param header The header for this item.
 */
-data class SourceItem(val source: CatalogueSource, val header: LangItem? = null) :
+data class SourceItem(val source: CatalogueSource, val header: LangItem? = null, val showButtons: Boolean) :
        AbstractSectionableItem<SourceHolder, LangItem>(header) {

    /**
@@ -28,7 +28,7 @@ data class SourceItem(val source: CatalogueSource, val header: LangItem? = null)
     * Creates a new view holder for this item.
     */
    override fun createViewHolder(view: View, adapter: FlexibleAdapter<IFlexible<RecyclerView.ViewHolder>>): SourceHolder {
-        return SourceHolder(view, adapter as CatalogueAdapter)
+        return SourceHolder(view, adapter as CatalogueAdapter, showButtons)
    }

    /**
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt
@@ -38,6 +38,7 @@ import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.ui.base.controller.DialogController
 import eu.kanade.tachiyomi.ui.base.controller.NucleusController
 import eu.kanade.tachiyomi.ui.base.controller.withFadeTransaction
+import eu.kanade.tachiyomi.ui.catalogue.CatalogueController
 import eu.kanade.tachiyomi.ui.catalogue.global_search.CatalogueSearchController
 import eu.kanade.tachiyomi.ui.library.ChangeMangaCategoriesDialog
 import eu.kanade.tachiyomi.ui.main.MainActivity
@@ -160,6 +161,9 @@ class MangaInfoController : NucleusController<MangaInfoPresenter>(),

    override fun onOptionsItemSelected(item: MenuItem): Boolean {
        when (item.itemId) {
+            // EXH -->
+            R.id.action_smart_search -> openSmartSearch()
+            // EXH <--
            R.id.action_open_in_browser -> openInBrowser()
            R.id.action_open_in_web_view -> openInWebView()
            R.id.action_share -> shareManga()
@@ -169,6 +173,17 @@ class MangaInfoController : NucleusController<MangaInfoPresenter>(),
        return true
    }

+
+    // EXH -->
+    private fun openSmartSearch() {
+        val smartSearchConfig = CatalogueController.SmartSearchConfig(presenter.manga.title)
+
+        parentController?.router?.pushController(CatalogueController(Bundle().apply {
+            putParcelable(CatalogueController.SMART_SEARCH_CONFIG, smartSearchConfig)
+        }).withFadeTransaction())
+    }
+    // EXH <--
+
    /**
     * Check if manga is initialized.
     * If true update view with manga information,
--- a/app/src/main/java/exh/ui/smartsearch/SmartSearchController.kt
+++ b/app/src/main/java/exh/ui/smartsearch/SmartSearchController.kt
@@ -0,0 +1,214 @@
+package exh.ui.smartsearch
+
+import android.os.Bundle
+import android.view.LayoutInflater
+import android.view.View
+import android.view.ViewGroup
+import eu.kanade.tachiyomi.R
+import eu.kanade.tachiyomi.data.database.DatabaseHelper
+import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.source.CatalogueSource
+import eu.kanade.tachiyomi.source.SourceManager
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.ui.base.controller.NucleusController
+import eu.kanade.tachiyomi.ui.base.controller.withFadeTransaction
+import eu.kanade.tachiyomi.ui.catalogue.CatalogueController
+import eu.kanade.tachiyomi.ui.manga.MangaController
+import eu.kanade.tachiyomi.util.toast
+import exh.util.await
+import info.debatty.java.stringsimilarity.NormalizedLevenshtein
+import kotlinx.android.synthetic.main.eh_smart_search.*
+import kotlinx.coroutines.*
+import rx.schedulers.Schedulers
+import uy.kohesive.injekt.injectLazy
+import kotlin.text.StringBuilder
+
+class SmartSearchController(bundle: Bundle? = null) : NucleusController<SmartSearchPresenter>() {
+    private val sourceManager: SourceManager by injectLazy()
+    private val db: DatabaseHelper by injectLazy()
+
+    private val source = sourceManager.get(bundle?.getLong(ARG_SOURCE_ID, -1) ?: -1) as? CatalogueSource
+    private val smartSearchConfig: CatalogueController.SmartSearchConfig? = bundle?.getParcelable(ARG_SMART_SEARCH_CONFIG)
+
+    override fun inflateView(inflater: LayoutInflater, container: ViewGroup) =
+            inflater.inflate(R.layout.eh_smart_search, container, false)!!
+
+    override fun getTitle() = source?.name ?: ""
+
+    override fun createPresenter() = SmartSearchPresenter()
+
+    override fun onViewCreated(view: View) {
+        super.onViewCreated(view)
+
+        appbar.bringToFront()
+
+        if(source == null || smartSearchConfig == null) {
+            router.popCurrentController()
+            applicationContext?.toast("Missing data!")
+            return
+        }
+
+        // TODO Error handling
+
+        // TODO Use activity scope
+        GlobalScope.launch(Dispatchers.IO) {
+            val resultManga = initiateSmartSearch(source, smartSearchConfig)
+            if(resultManga != null) {
+                val localManga = networkToLocalManga(resultManga, source.id)
+                val transaction = MangaController(localManga, true).withFadeTransaction()
+                withContext(Dispatchers.Main) {
+                    router.replaceTopController(transaction)
+                }
+            } else {
+                // TODO Open search
+                router.popCurrentController()
+            }
+            println(resultManga)
+        }
+    }
+
+    suspend fun initiateSmartSearch(source: CatalogueSource, config: CatalogueController.SmartSearchConfig): SManga? {
+        val cleanedTitle = cleanSmartSearchTitle(config.title)
+
+        val queries = getSmartSearchQueries(cleanedTitle)
+
+        val eligibleManga = supervisorScope {
+            queries.map { query ->
+                async(Dispatchers.IO) {
+                    val searchResults = source.fetchSearchManga(1, query, FilterList()).toSingle().await(Schedulers.io())
+
+                    searchResults.mangas.map {
+                        val cleanedMangaTitle = cleanSmartSearchTitle(it.title)
+                        val normalizedDistance = NormalizedLevenshtein().similarity(cleanedTitle, cleanedMangaTitle)
+                        SearchEntry(it, normalizedDistance)
+                    }.filter { (_, normalizedDistance) ->
+                        normalizedDistance >= MIN_ELIGIBLE_THRESHOLD
+                    }
+                }
+            }.flatMap { it.await() }
+        }
+
+        return eligibleManga.maxBy { it.dist }?.manga
+    }
+
+    fun getSmartSearchQueries(cleanedTitle: String): List<String> {
+        val splitCleanedTitle = cleanedTitle.split(" ")
+        val splitSortedByLargest = splitCleanedTitle.sortedByDescending { it.length }
+
+        if(splitCleanedTitle.isEmpty()) {
+            return emptyList()
+        }
+
+        // Search cleaned title
+        // Search two largest words
+        // Search largest word
+        // Search first two words
+        // Search first word
+
+        val searchQueries = listOf(
+                listOf(cleanedTitle),
+                splitSortedByLargest.take(2),
+                splitSortedByLargest.take(1),
+                splitCleanedTitle.take(2),
+                splitCleanedTitle.take(1)
+        )
+
+        return searchQueries.map {
+            it.joinToString().trim()
+        }.distinct()
+    }
+
+    fun cleanSmartSearchTitle(title: String): String {
+        val preTitle = title.toLowerCase()
+
+        // Remove text in brackets
+        var cleanedTitle = removeTextInBrackets(preTitle, true)
+        if(cleanedTitle.length <= 5) { // Title is suspiciously short, try parsing it backwards
+            cleanedTitle = removeTextInBrackets(preTitle, false)
+        }
+
+        // Strip non-special characters
+        cleanedTitle = cleanedTitle.replace(titleRegex, " ")
+
+        // Strip splitters and consecutive spaces
+        cleanedTitle = cleanedTitle.trim().replace(" - ", " ").replace(consecutiveSpacesRegex, " ").trim()
+
+        return cleanedTitle
+    }
+
+    private fun removeTextInBrackets(text: String, readForward: Boolean): String {
+        val bracketPairs = listOf(
+                '(' to ')',
+                '[' to ']',
+                '<' to '>',
+                '{' to '}'
+        )
+        var openingBracketPairs = bracketPairs.mapIndexed { index, (opening, _) ->
+            opening to index
+        }.toMap()
+        var closingBracketPairs = bracketPairs.mapIndexed { index, (_, closing) ->
+            closing to index
+        }.toMap()
+
+        // Reverse pairs if reading backwards
+        if(!readForward) {
+            val tmp = openingBracketPairs
+            openingBracketPairs = closingBracketPairs
+            closingBracketPairs = tmp
+        }
+
+        val depthPairs = bracketPairs.map { 0 }.toMutableList()
+
+        val result = StringBuilder()
+        for(c in if(readForward) text else text.reversed()) {
+            val openingBracketDepthIndex = openingBracketPairs[c]
+            if(openingBracketDepthIndex != null) {
+                depthPairs[openingBracketDepthIndex]++
+            } else {
+                val closingBracketDepthIndex = closingBracketPairs[c]
+                if(closingBracketDepthIndex != null) {
+                    depthPairs[closingBracketDepthIndex]--
+                } else {
+                    if(depthPairs.all { it <= 0 }) {
+                        result.append(c)
+                    } else {
+                        // In brackets, do not append to result
+                    }
+                }
+            }
+        }
+
+        return result.toString()
+    }
+
+    /**
+     * Returns a manga from the database for the given manga from network. It creates a new entry
+     * if the manga is not yet in the database.
+     *
+     * @param sManga the manga from the source.
+     * @return a manga from the database.
+     */
+    private fun networkToLocalManga(sManga: SManga, sourceId: Long): Manga {
+        var localManga = db.getManga(sManga.url, sourceId).executeAsBlocking()
+        if (localManga == null) {
+            val newManga = Manga.create(sManga.url, sManga.title, sourceId)
+            newManga.copyFrom(sManga)
+            val result = db.insertManga(newManga).executeAsBlocking()
+            newManga.id = result.insertedId()
+            localManga = newManga
+        }
+        return localManga
+    }
+
+    data class SearchEntry(val manga: SManga, val dist: Double)
+
+    companion object {
+        const val ARG_SOURCE_ID = "SOURCE_ID"
+        const val ARG_SMART_SEARCH_CONFIG = "SMART_SEARCH_CONFIG"
+        const val MIN_ELIGIBLE_THRESHOLD = 0.7
+
+        private val titleRegex = Regex("[^a-zA-Z0-9- ]")
+        private val consecutiveSpacesRegex = Regex(" +")
+    }
+}
--- a/app/src/main/java/exh/ui/smartsearch/SmartSearchPresenter.kt
+++ b/app/src/main/java/exh/ui/smartsearch/SmartSearchPresenter.kt
@@ -0,0 +1,6 @@
+package exh.ui.smartsearch
+
+import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter
+
+class SmartSearchPresenter: BasePresenter<SmartSearchPresenter>() {
+}
--- a/app/src/main/res/drawable/eh_ic_find_replace_white_24dp.xml
+++ b/app/src/main/res/drawable/eh_ic_find_replace_white_24dp.xml
@@ -0,0 +1,5 @@
+<vector android:height="24dp" android:tint="#FFFFFF"
+    android:viewportHeight="24.0" android:viewportWidth="24.0"
+    android:width="24dp" xmlns:android="http://schemas.android.com/apk/res/android">
+    <path android:fillColor="#FF000000" android:pathData="M11,6c1.38,0 2.63,0.56 3.54,1.46L12,10h6L18,4l-2.05,2.05C14.68,4.78 12.93,4 11,4c-3.53,0 -6.43,2.61 -6.92,6L6.1,10c0.46,-2.28 2.48,-4 4.9,-4zM16.64,15.14c0.66,-0.9 1.12,-1.97 1.28,-3.14L15.9,12c-0.46,2.28 -2.48,4 -4.9,4 -1.38,0 -2.63,-0.56 -3.54,-1.46L10,12L4,12v6l2.05,-2.05C7.32,17.22 9.07,18 11,18c1.55,0 2.98,-0.51 4.14,-1.36L20,21.49 21.49,20l-4.85,-4.86z"/>
+</vector>
--- a/app/src/main/res/layout/eh_activity_intercept.xml
+++ b/app/src/main/res/layout/eh_activity_intercept.xml
@@ -20,7 +20,6 @@

            <android.support.v7.widget.Toolbar
                android:id="@+id/toolbar"
-                xmlns:android="http://schemas.android.com/apk/res/android"
                android:layout_width="match_parent"
                android:layout_height="?attr/actionBarSize"
                android:background="?attr/colorPrimary"
--- a/app/src/main/res/layout/eh_smart_search.xml
+++ b/app/src/main/res/layout/eh_smart_search.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="utf-8"?>
+<android.support.design.widget.CoordinatorLayout
+    xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:fitsSystemWindows="true">
+
+    <LinearLayout
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:orientation="vertical">
+
+        <android.support.design.widget.AppBarLayout
+            android:id="@+id/appbar"
+            android:layout_width="match_parent"
+            android:layout_height="wrap_content"
+            app:elevation="0dp">
+
+            <android.support.v7.widget.Toolbar
+                android:id="@+id/toolbar"
+                android:layout_width="match_parent"
+                android:layout_height="?attr/actionBarSize"
+                android:background="?attr/colorPrimary"
+                android:theme="?attr/actionBarTheme" />
+
+        </android.support.design.widget.AppBarLayout>
+
+        <LinearLayout
+            android:layout_width="match_parent"
+            android:layout_height="match_parent"
+            android:layout_gravity="center"
+            android:background="?attr/colorPrimary"
+            android:gravity="center"
+            android:orientation="vertical">
+
+            <TextView
+                android:id="@+id/intercept_status"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:layout_marginBottom="8dp"
+                android:text="Searching source..."
+                android:textAppearance="@style/TextAppearance.Medium.Title"
+                android:textColor="@color/white" />
+
+            <ProgressBar
+                android:id="@+id/intercept_progress"
+                style="?android:attr/progressBarStyleLarge"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:layout_gravity="center"
+                android:indeterminateTint="@color/white" />
+        </LinearLayout>
+    </LinearLayout>
+
+</android.support.design.widget.CoordinatorLayout>
--- a/app/src/main/res/menu/manga_info.xml
+++ b/app/src/main/res/menu/manga_info.xml
@@ -8,6 +8,11 @@
        android:title="@string/action_share"
        app:showAsAction="ifRoom" />

+    <item android:id="@+id/action_smart_search"
+        android:icon="@drawable/eh_ic_find_replace_white_24dp"
+        android:title="Find in another source"
+        app:showAsAction="ifRoom" />
+
    <item android:id="@+id/action_open_in_browser"
        android:title="@string/action_open_in_browser"
        app:showAsAction="never"/>