Significantly improve browsing speed (near instantaneous) (#1946)

CHANGELOG.md

@@ -16,6 +16,9 @@ The format is a modified version of [Keep a Changelog](https://keepachangelog.co
 - Add user manga notes ([@imkunet](https://github.com/imkunet), [@AntsyLich](https://github.com/AntsyLich)) ([#428](https://github.com/mihonapp/mihon/pull/428))
   - Fix user notes not restoring when manga doesn't exist in DB ([@AntsyLich](https://github.com/AntsyLich)) ([#1945](https://github.com/mihonapp/mihon/pull/1945))
 
+### Improved
+- Significantly improve browsing speed (near instantaneous) ([@AntsyLich](https://github.com/AntsyLich)) ([#1946](https://github.com/mihonapp/mihon/pull/1946))
+
 ### Fixes
 - Fix Bangumi search results including novels ([@MajorTanya](https://github.com/MajorTanya)) ([#1885](https://github.com/mihonapp/mihon/pull/1885))
 - Fix next chapter button occasionally jumping to the last page of the current chapter ([@perokhe](https://github.com/perokhe)) ([#1920](https://github.com/mihonapp/mihon/pull/1920))

app/src/main/java/eu/kanade/domain/manga/model/Manga.kt

@@ -69,22 +69,6 @@ fun Manga.copyFrom(other: SManga): Manga {
     )
 }
 
-fun SManga.toDomainManga(sourceId: Long): Manga {
-    return Manga.create().copy(
-        url = url,
-        title = title,
-        artist = artist,
-        author = author,
-        description = description,
-        genre = getGenres(),
-        status = status.toLong(),
-        thumbnailUrl = thumbnail_url,
-        updateStrategy = update_strategy,
-        initialized = initialized,
-        source = sourceId,
-    )
-}
-
 fun Manga.hasCustomCover(coverCache: CoverCache = Injekt.get()): Boolean {
     return coverCache.getCustomCoverFile(id).exists()
 }

app/src/main/java/eu/kanade/tachiyomi/ui/browse/source/browse/BrowseSourceScreenModel.kt

@@ -15,7 +15,6 @@ import cafe.adriel.voyager.core.model.StateScreenModel
 import cafe.adriel.voyager.core.model.screenModelScope
 import eu.kanade.core.preference.asState
 import eu.kanade.domain.manga.interactor.UpdateManga
-import eu.kanade.domain.manga.model.toDomainManga
 import eu.kanade.domain.source.interactor.GetIncognitoState
 import eu.kanade.domain.source.service.SourcePreferences
 import eu.kanade.domain.track.interactor.AddTracks
@@ -29,7 +28,6 @@ import kotlinx.collections.immutable.toImmutableList
 import kotlinx.coroutines.flow.SharingStarted
 import kotlinx.coroutines.flow.distinctUntilChanged
 import kotlinx.coroutines.flow.emptyFlow
-import kotlinx.coroutines.flow.filterNotNull
 import kotlinx.coroutines.flow.firstOrNull
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.stateIn
@@ -45,7 +43,6 @@ import tachiyomi.domain.chapter.interactor.SetMangaDefaultChapterFlags
 import tachiyomi.domain.library.service.LibraryPreferences
 import tachiyomi.domain.manga.interactor.GetDuplicateLibraryManga
 import tachiyomi.domain.manga.interactor.GetManga
-import tachiyomi.domain.manga.interactor.NetworkToLocalManga
 import tachiyomi.domain.manga.model.Manga
 import tachiyomi.domain.manga.model.toMangaUpdate
 import tachiyomi.domain.source.interactor.GetRemoteManga
@@ -68,7 +65,6 @@ class BrowseSourceScreenModel(
     private val setMangaCategories: SetMangaCategories = Injekt.get(),
     private val setMangaDefaultChapterFlags: SetMangaDefaultChapterFlags = Injekt.get(),
     private val getManga: GetManga = Injekt.get(),
-    private val networkToLocalManga: NetworkToLocalManga = Injekt.get(),
     private val updateManga: UpdateManga = Injekt.get(),
     private val addTracks: AddTracks = Injekt.get(),
     private val getIncognitoState: GetIncognitoState = Injekt.get(),
@@ -110,12 +106,11 @@ class BrowseSourceScreenModel(
         .distinctUntilChanged()
         .map { listing ->
             Pager(PagingConfig(pageSize = 25)) {
-                getRemoteManga.subscribe(sourceId, listing.query ?: "", listing.filters)
+                getRemoteManga(sourceId, listing.query ?: "", listing.filters)
             }.flow.map { pagingData ->
-                pagingData.map {
+                pagingData.map { manga ->
-                    networkToLocalManga.await(it.toDomainManga(sourceId))
+                    getManga.subscribe(manga.url, manga.source)
-                        .let { localManga -> getManga.subscribe(localManga.url, localManga.source) }
+                        .map { it ?: manga }
-                        .filterNotNull()
                         .stateIn(ioCoroutineScope)
                 }
                     .filter { !hideInLibraryItems || !it.value.favorite }

app/src/main/java/eu/kanade/tachiyomi/ui/browse/source/globalsearch/SearchScreenModel.kt

@@ -5,7 +5,6 @@ import androidx.compose.runtime.Immutable
 import androidx.compose.runtime.produceState
 import cafe.adriel.voyager.core.model.StateScreenModel
 import cafe.adriel.voyager.core.model.screenModelScope
-import eu.kanade.domain.manga.model.toDomainManga
 import eu.kanade.domain.source.service.SourcePreferences
 import eu.kanade.presentation.util.ioCoroutineScope
 import eu.kanade.tachiyomi.extension.ExtensionManager
@@ -24,6 +23,7 @@ import kotlinx.coroutines.flow.update
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
+import mihon.domain.manga.model.toDomainManga
 import tachiyomi.core.common.preference.toggle
 import tachiyomi.domain.manga.interactor.GetManga
 import tachiyomi.domain.manga.interactor.NetworkToLocalManga
@@ -165,9 +165,8 @@ abstract class SearchScreenModel(
                             source.getSearchManga(1, query, source.getFilterList())
                         }
 
-                        val titles = page.mangas.map {
+                        val titles = page.mangas.map { it.toDomainManga(source.id) }
-                            networkToLocalManga.await(it.toDomainManga(source.id))
+                            .let { networkToLocalManga(it) }
-                        }
 
                         if (isActive) {
                             updateItem(source, SearchItemResult.Success(titles))

app/src/main/java/eu/kanade/tachiyomi/ui/deeplink/DeepLinkScreenModel.kt

@@ -4,18 +4,16 @@ import androidx.compose.runtime.Immutable
 import cafe.adriel.voyager.core.model.StateScreenModel
 import cafe.adriel.voyager.core.model.screenModelScope
 import eu.kanade.domain.chapter.interactor.SyncChaptersWithSource
-import eu.kanade.domain.manga.model.toDomainManga
 import eu.kanade.domain.manga.model.toSManga
 import eu.kanade.tachiyomi.source.Source
 import eu.kanade.tachiyomi.source.model.SChapter
-import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.ResolvableSource
 import eu.kanade.tachiyomi.source.online.UriType
 import kotlinx.coroutines.flow.update
+import mihon.domain.manga.model.toDomainManga
 import tachiyomi.core.common.util.lang.launchIO
 import tachiyomi.domain.chapter.interactor.GetChapterByUrlAndMangaId
 import tachiyomi.domain.chapter.model.Chapter
-import tachiyomi.domain.manga.interactor.GetMangaByUrlAndSourceId
 import tachiyomi.domain.manga.interactor.NetworkToLocalManga
 import tachiyomi.domain.manga.model.Manga
 import tachiyomi.domain.source.service.SourceManager
@@ -27,7 +25,6 @@ class DeepLinkScreenModel(
     private val sourceManager: SourceManager = Injekt.get(),
     private val networkToLocalManga: NetworkToLocalManga = Injekt.get(),
     private val getChapterByUrlAndMangaId: GetChapterByUrlAndMangaId = Injekt.get(),
-    private val getMangaByUrlAndSourceId: GetMangaByUrlAndSourceId = Injekt.get(),
     private val syncChaptersWithSource: SyncChaptersWithSource = Injekt.get(),
 ) : StateScreenModel<DeepLinkScreenModel.State>(State.Loading) {
 
@@ -38,7 +35,7 @@ class DeepLinkScreenModel(
                 .firstOrNull { it.getUriType(query) != UriType.Unknown }
 
             val manga = source?.getManga(query)?.let {
-                getMangaFromSManga(it, source.id)
+                networkToLocalManga(it.toDomainManga(source.id))
             }
 
             val chapter = if (source?.getUriType(query) == UriType.Chapter && manga != null) {
@@ -73,11 +70,6 @@ class DeepLinkScreenModel(
         }
     }
 
-    private suspend fun getMangaFromSManga(sManga: SManga, sourceId: Long): Manga {
-        return getMangaByUrlAndSourceId.await(sManga.url, sourceId)
-            ?: networkToLocalManga.await(sManga.toDomainManga(sourceId))
-    }
-
     sealed interface State {
         @Immutable
         data object Loading : State

data/src/main/java/tachiyomi/data/manga/MangaRepositoryImpl.kt

@@ -97,35 +97,6 @@ class MangaRepositoryImpl(
         }
     }
 
-    override suspend fun insert(manga: Manga): Long? {
-        return handler.awaitOneOrNullExecutable(inTransaction = true) {
-            mangasQueries.insert(
-                source = manga.source,
-                url = manga.url,
-                artist = manga.artist,
-                author = manga.author,
-                description = manga.description,
-                genre = manga.genre,
-                title = manga.title,
-                status = manga.status,
-                thumbnailUrl = manga.thumbnailUrl,
-                favorite = manga.favorite,
-                lastUpdate = manga.lastUpdate,
-                nextUpdate = manga.nextUpdate,
-                calculateInterval = manga.fetchInterval.toLong(),
-                initialized = manga.initialized,
-                viewerFlags = manga.viewerFlags,
-                chapterFlags = manga.chapterFlags,
-                coverLastModified = manga.coverLastModified,
-                dateAdded = manga.dateAdded,
-                updateStrategy = manga.updateStrategy,
-                version = manga.version,
-                notes = manga.notes,
-            )
-            mangasQueries.selectLastInsertedRowId()
-        }
-    }
-
     override suspend fun update(update: MangaUpdate): Boolean {
         return try {
             partialUpdate(update)
@@ -146,6 +117,37 @@ class MangaRepositoryImpl(
         }
     }
 
+    override suspend fun insertNetworkManga(manga: List<Manga>): List<Manga> {
+        return handler.await(inTransaction = true) {
+            manga.map {
+                mangasQueries.insertNetworkManga(
+                    source = it.source,
+                    url = it.url,
+                    artist = it.artist,
+                    author = it.author,
+                    description = it.description,
+                    genre = it.genre,
+                    title = it.title,
+                    status = it.status,
+                    thumbnailUrl = it.thumbnailUrl,
+                    favorite = it.favorite,
+                    lastUpdate = it.lastUpdate,
+                    nextUpdate = it.nextUpdate,
+                    calculateInterval = it.fetchInterval.toLong(),
+                    initialized = it.initialized,
+                    viewerFlags = it.viewerFlags,
+                    chapterFlags = it.chapterFlags,
+                    coverLastModified = it.coverLastModified,
+                    dateAdded = it.dateAdded,
+                    updateStrategy = it.updateStrategy,
+                    version = it.version,
+                    mapper = MangaMapper::mapManga,
+                )
+                    .executeAsOne()
+            }
+        }
+    }
+
     private suspend fun partialUpdate(vararg mangaUpdates: MangaUpdate) {
         handler.await(inTransaction = true) {
             mangaUpdates.forEach { value ->

data/src/main/java/tachiyomi/data/source/SourcePagingSource.kt

@@ -4,56 +4,67 @@ import androidx.paging.PagingState
 import eu.kanade.tachiyomi.source.CatalogueSource
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
-import eu.kanade.tachiyomi.source.model.SManga
+import mihon.domain.manga.model.toDomainManga
 import tachiyomi.core.common.util.lang.withIOContext
-import tachiyomi.domain.source.repository.SourcePagingSourceType
+import tachiyomi.domain.manga.interactor.NetworkToLocalManga
+import tachiyomi.domain.manga.model.Manga
+import tachiyomi.domain.source.repository.SourcePagingSource
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
 
-class SourceSearchPagingSource(source: CatalogueSource, val query: String, val filters: FilterList) :
+class SourceSearchPagingSource(
-    SourcePagingSource(source) {
+    source: CatalogueSource,
+    private val query: String,
+    private val filters: FilterList,
+) : BaseSourcePagingSource(source) {
     override suspend fun requestNextPage(currentPage: Int): MangasPage {
         return source.getSearchManga(currentPage, query, filters)
     }
 }
 
-class SourcePopularPagingSource(source: CatalogueSource) : SourcePagingSource(source) {
+class SourcePopularPagingSource(source: CatalogueSource) : BaseSourcePagingSource(source) {
     override suspend fun requestNextPage(currentPage: Int): MangasPage {
         return source.getPopularManga(currentPage)
     }
 }
 
-class SourceLatestPagingSource(source: CatalogueSource) : SourcePagingSource(source) {
+class SourceLatestPagingSource(source: CatalogueSource) : BaseSourcePagingSource(source) {
     override suspend fun requestNextPage(currentPage: Int): MangasPage {
         return source.getLatestUpdates(currentPage)
     }
 }
 
-abstract class SourcePagingSource(
+abstract class BaseSourcePagingSource(
     protected val source: CatalogueSource,
-) : SourcePagingSourceType() {
+    private val networkToLocalManga: NetworkToLocalManga = Injekt.get(),
+) : SourcePagingSource() {
 
     abstract suspend fun requestNextPage(currentPage: Int): MangasPage
 
-    override suspend fun load(params: LoadParams<Long>): LoadResult<Long, SManga> {
+    override suspend fun load(params: LoadParams<Long>): LoadResult<Long, Manga> {
         val page = params.key ?: 1
 
-        val mangasPage = try {
+        return try {
-            withIOContext {
+            val mangasPage = withIOContext {
                 requestNextPage(page.toInt())
                     .takeIf { it.mangas.isNotEmpty() }
                     ?: throw NoResultsException()
             }
-        } catch (e: Exception) {
-            return LoadResult.Error(e)
-        }
 
-        return LoadResult.Page(
+            val manga = mangasPage.mangas.map { it.toDomainManga(source.id) }
-            data = mangasPage.mangas,
+                .let { networkToLocalManga(it) }
+
+            LoadResult.Page(
+                data = manga,
                 prevKey = null,
                 nextKey = if (mangasPage.hasNextPage) page + 1 else null,
             )
+        } catch (e: Exception) {
+            LoadResult.Error(e)
+        }
     }
 
-    override fun getRefreshKey(state: PagingState<Long, SManga>): Long? {
+    override fun getRefreshKey(state: PagingState<Long, Manga>): Long? {
         return state.anchorPosition?.let { anchorPosition ->
             val anchorPage = state.closestPageToPosition(anchorPosition)
             anchorPage?.prevKey ?: anchorPage?.nextKey

data/src/main/java/tachiyomi/data/source/SourceRepositoryImpl.kt

@@ -10,7 +10,7 @@ import kotlinx.coroutines.flow.map
 import tachiyomi.data.DatabaseHandler
 import tachiyomi.domain.source.model.SourceWithCount
 import tachiyomi.domain.source.model.StubSource
-import tachiyomi.domain.source.repository.SourcePagingSourceType
+import tachiyomi.domain.source.repository.SourcePagingSource
 import tachiyomi.domain.source.repository.SourceRepository
 import tachiyomi.domain.source.service.SourceManager
 import tachiyomi.domain.source.model.Source as DomainSource
@@ -72,17 +72,17 @@ class SourceRepositoryImpl(
         sourceId: Long,
         query: String,
         filterList: FilterList,
-    ): SourcePagingSourceType {
+    ): SourcePagingSource {
         val source = sourceManager.get(sourceId) as CatalogueSource
         return SourceSearchPagingSource(source, query, filterList)
     }
 
-    override fun getPopular(sourceId: Long): SourcePagingSourceType {
+    override fun getPopular(sourceId: Long): SourcePagingSource {
         val source = sourceManager.get(sourceId) as CatalogueSource
         return SourcePopularPagingSource(source)
     }
 
-    override fun getLatest(sourceId: Long): SourcePagingSourceType {
+    override fun getLatest(sourceId: Long): SourcePagingSource {
         val source = sourceManager.get(sourceId) as CatalogueSource
         return SourceLatestPagingSource(source)
     }

data/src/main/sqldelight/tachiyomi/data/mangas.sq

@@ -182,3 +182,31 @@ WHERE _id = :mangaId;
 
 selectLastInsertedRowId:
 SELECT last_insert_rowid();
+
+insertNetworkManga {
+    -- Insert the manga if it doesn't exist already
+    INSERT INTO mangas(
+        source, url, artist, author, description, genre, title, status, thumbnail_url, favorite,
+        last_update, next_update, initialized, viewer, chapter_flags, cover_last_modified, date_added,
+        update_strategy, calculate_interval, last_modified_at, version
+    )
+    SELECT
+        :source, :url, :artist, :author, :description, :genre, :title, :status, :thumbnailUrl, :favorite,
+        :lastUpdate, :nextUpdate, :initialized, :viewerFlags, :chapterFlags, :coverLastModified, :dateAdded,
+        :updateStrategy, :calculateInterval, 0, :version
+    WHERE NOT EXISTS(SELECT 0 FROM mangas WHERE source = :source AND url = :url);
+
+    -- Update the title if it is not favorite
+    UPDATE mangas
+    SET title = :title
+    WHERE source = :source
+    AND url = :url
+    AND favorite = 0;
+
+    -- Finally return the manga
+    SELECT *
+    FROM mangas
+    WHERE source = :source
+    AND url = :url
+    LIMIT 1;
+}

domain/src/main/java/mihon/domain/manga/model/SManga.kt

@@ -0,0 +1,20 @@
+package mihon.domain.manga.model
+
+import eu.kanade.tachiyomi.source.model.SManga
+import tachiyomi.domain.manga.model.Manga
+
+fun SManga.toDomainManga(sourceId: Long): Manga {
+    return Manga.create().copy(
+        url = url,
+        title = title,
+        artist = artist,
+        author = author,
+        description = description,
+        genre = getGenres(),
+        status = status.toLong(),
+        thumbnailUrl = thumbnail_url,
+        updateStrategy = update_strategy,
+        initialized = initialized,
+        source = sourceId,
+    )
+}

domain/src/main/java/tachiyomi/domain/manga/interactor/NetworkToLocalManga.kt

@@ -7,29 +7,11 @@ class NetworkToLocalManga(
     private val mangaRepository: MangaRepository,
 ) {
 
-    suspend fun await(manga: Manga): Manga {
+    suspend operator fun invoke(manga: Manga): Manga {
-        val localManga = getManga(manga.url, manga.source)
+        return mangaRepository.insertNetworkManga(listOf(manga)).single()
-        return when {
-            localManga == null -> {
-                val id = insertManga(manga)
-                manga.copy(id = id!!)
-            }
-            !localManga.favorite -> {
-                // if the manga isn't a favorite, set its display title from source
-                // if it later becomes a favorite, updated title will go to db
-                localManga.copy(title = manga.title)
-            }
-            else -> {
-                localManga
-            }
-        }
     }
 
-    private suspend fun getManga(url: String, sourceId: Long): Manga? {
+    suspend operator fun invoke(manga: List<Manga>): List<Manga> {
-        return mangaRepository.getMangaByUrlAndSourceId(url, sourceId)
+        return mangaRepository.insertNetworkManga(manga)
-    }
-
-    private suspend fun insertManga(manga: Manga): Long? {
-        return mangaRepository.insert(manga)
     }
 }

domain/src/main/java/tachiyomi/domain/manga/repository/MangaRepository.kt

@@ -33,9 +33,9 @@ interface MangaRepository {
 
     suspend fun setMangaCategories(mangaId: Long, categoryIds: List<Long>)
 
-    suspend fun insert(manga: Manga): Long?
-
     suspend fun update(update: MangaUpdate): Boolean
 
     suspend fun updateAll(mangaUpdates: List<MangaUpdate>): Boolean
+
+    suspend fun insertNetworkManga(manga: List<Manga>): List<Manga>
 }

domain/src/main/java/tachiyomi/domain/source/interactor/GetRemoteManga.kt

@@ -1,14 +1,14 @@
 package tachiyomi.domain.source.interactor
 
 import eu.kanade.tachiyomi.source.model.FilterList
-import tachiyomi.domain.source.repository.SourcePagingSourceType
+import tachiyomi.domain.source.repository.SourcePagingSource
 import tachiyomi.domain.source.repository.SourceRepository
 
 class GetRemoteManga(
     private val repository: SourceRepository,
 ) {
 
-    fun subscribe(sourceId: Long, query: String, filterList: FilterList): SourcePagingSourceType {
+    operator fun invoke(sourceId: Long, query: String, filterList: FilterList): SourcePagingSource {
         return when (query) {
             QUERY_POPULAR -> repository.getPopular(sourceId)
             QUERY_LATEST -> repository.getLatest(sourceId)

domain/src/main/java/tachiyomi/domain/source/repository/SourceRepository.kt

@@ -2,12 +2,12 @@ package tachiyomi.domain.source.repository
 
 import androidx.paging.PagingSource
 import eu.kanade.tachiyomi.source.model.FilterList
-import eu.kanade.tachiyomi.source.model.SManga
 import kotlinx.coroutines.flow.Flow
+import tachiyomi.domain.manga.model.Manga
 import tachiyomi.domain.source.model.Source
 import tachiyomi.domain.source.model.SourceWithCount
 
-typealias SourcePagingSourceType = PagingSource<Long, SManga>
+typealias SourcePagingSource = PagingSource<Long, Manga>
 
 interface SourceRepository {
 
@@ -19,9 +19,9 @@ interface SourceRepository {
 
     fun getSourcesWithNonLibraryManga(): Flow<List<SourceWithCount>>
 
-    fun search(sourceId: Long, query: String, filterList: FilterList): SourcePagingSourceType
+    fun search(sourceId: Long, query: String, filterList: FilterList): SourcePagingSource
 
-    fun getPopular(sourceId: Long): SourcePagingSourceType
+    fun getPopular(sourceId: Long): SourcePagingSource
 
-    fun getLatest(sourceId: Long): SourcePagingSourceType
+    fun getLatest(sourceId: Long): SourcePagingSource
 }