feat: Komga page-based sync

feat: chapter tracker overhaul: add switch, replace existing tracker logic if enabled

fix: sync page progress regardless of chapter index

chore: change log level

feat: Komga page-based sync

app/src/main/java/eu/kanade/domain/track/interactor/SyncChapterProgressWithTrack.kt

@@ -1,8 +1,17 @@
 package eu.kanade.domain.track.interactor
 
+import android.app.Application
+import com.google.common.annotations.VisibleForTesting
+import eu.kanade.domain.chapter.model.toDbChapter
 import eu.kanade.domain.track.model.toDbTrack
+import eu.kanade.domain.track.service.TrackPreferences
+import eu.kanade.tachiyomi.BuildConfig
+import eu.kanade.tachiyomi.data.database.models.Chapter
+import eu.kanade.tachiyomi.data.database.models.toDomainChapter
 import eu.kanade.tachiyomi.data.track.EnhancedTracker
+import eu.kanade.tachiyomi.data.track.PageTracker
 import eu.kanade.tachiyomi.data.track.Tracker
+import eu.kanade.tachiyomi.util.system.toast
 import logcat.LogPriority
 import tachiyomi.core.common.util.system.logcat
 import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId
@@ -10,6 +19,9 @@ import tachiyomi.domain.chapter.interactor.UpdateChapter
 import tachiyomi.domain.chapter.model.toChapterUpdate
 import tachiyomi.domain.track.interactor.InsertTrack
 import tachiyomi.domain.track.model.Track
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+import uy.kohesive.injekt.injectLazy
 
 class SyncChapterProgressWithTrack(
     private val updateChapter: UpdateChapter,
@@ -17,6 +29,57 @@ class SyncChapterProgressWithTrack(
     private val getChaptersByMangaId: GetChaptersByMangaId,
 ) {
 
+    companion object {
+        //Equal compare
+        private const val SYNC_STRATEGY_DEFAULT = 1
+        private fun syncStrategyDefault(local: PageTracker.ChapterReadProgress, remote: PageTracker.ChapterReadProgress): RemoteProgressResolution {
+            return when {
+                local > remote -> RemoteProgressResolution.REJECT
+                local < remote -> RemoteProgressResolution.ACCEPT
+                else -> RemoteProgressResolution.SAME
+            }
+        }
+
+        //Flush local with remote
+        private const val SYNC_STRATEGY_ACCEPT_ALL = 2
+        private fun syncStrategyAcceptAll(local: PageTracker.ChapterReadProgress, remote: PageTracker.ChapterReadProgress): RemoteProgressResolution {
+            return if (local.completed && remote.completed || local.page == remote.page) RemoteProgressResolution.SAME else RemoteProgressResolution.ACCEPT
+        }
+
+        //Update remote only when both local and remote are not completed and local page index gt remote
+        private const val SYNC_STRATEGY_ALLOW_REREAD = 3
+
+        private fun syncStrategyAllowReread(local: PageTracker.ChapterReadProgress, remote: PageTracker.ChapterReadProgress): RemoteProgressResolution {
+            return if (local.completed && !remote.completed && remote.page > 1) RemoteProgressResolution.ACCEPT else syncStrategyDefault(local, remote)
+        }
+
+        @VisibleForTesting
+        internal var syncStrategy = SYNC_STRATEGY_ALLOW_REREAD
+
+        @VisibleForTesting
+        internal fun resolveRemoteProgress(chapter: eu.kanade.tachiyomi.data.database.models.Chapter, remote: PageTracker.ChapterReadProgress): RemoteProgressResolution {
+            val local = PageTracker.ChapterReadProgress(chapter.read, chapter.last_page_read)
+            return when(syncStrategy) {
+                SYNC_STRATEGY_ACCEPT_ALL -> syncStrategyAcceptAll(local, remote)
+                SYNC_STRATEGY_ALLOW_REREAD -> syncStrategyAllowReread(local, remote)
+                else -> syncStrategyDefault(local, remote)
+            }
+        }
+
+        @VisibleForTesting
+        internal val Chapter.debugString:String
+            get() = "$name(id = $id, read = $read, page = $last_page_read, url = $url)"
+    }
+
+    @VisibleForTesting
+    internal enum class RemoteProgressResolution {
+        ACCEPT,
+        REJECT,
+        SAME
+    }
+
+    private val trackPreferences: TrackPreferences by injectLazy()
+
     suspend fun await(
         mangaId: Long,
         remoteTrack: Track,
@@ -33,14 +96,37 @@ class SyncChapterProgressWithTrack(
         val chapterUpdates = sortedChapters
             .filter { chapter -> chapter.chapterNumber <= remoteTrack.lastChapterRead && !chapter.read }
             .map { it.copy(read = true).toChapterUpdate() }
-
         // only take into account continuous reading
         val localLastRead = sortedChapters.takeWhile { it.read }.lastOrNull()?.chapterNumber ?: 0F
         val updatedTrack = remoteTrack.copy(lastChapterRead = localLastRead.toDouble())
 
         try {
+            if (tracker is PageTracker && trackPreferences.chapterBasedTracking().get()) {
+                val remoteUpdatesMapping = sortedChapters.map { it.toDbChapter() }
+                    .let { tracker.batchGetChapterProgress(it) }
+                    .entries.groupBy { resolveRemoteProgress(it.key, it.value) }
+                val updatesToLocal = remoteUpdatesMapping[RemoteProgressResolution.ACCEPT]?.mapNotNull {  (chapter, remote) ->
+                    if (remote.page > 1 && chapter.last_page_read != remote.page - 1 )
+                        //In komga page starts from 1
+                        chapter.toDomainChapter()?.copy(lastPageRead = remote.page.toLong() - 1, read = remote.completed)?.toChapterUpdate()
+                    else null
+                } ?: listOf()
+                val updatesToRemote = remoteUpdatesMapping[RemoteProgressResolution.REJECT]?.map { it.key } ?: listOf()
+
+                updateChapter.awaitAll(updatesToLocal)
+                (tracker as PageTracker).batchUpdateRemoteProgress(updatesToRemote)
+                logcat(LogPriority.INFO) {
+                    "Tracker $tracker updated page progress" +
+                    "\nwrite-local: " + updatesToLocal +
+                    "\nwrite-remote " + updatesToRemote.map { it.debugString }
+                }
+                if (BuildConfig.APPLICATION_ID == "app.mihon.debug") {
+                    Injekt.get<Application>().toast("Finished syncing PageTracker ${tracker.javaClass.simpleName}")
+                }
+            } else {
                 tracker.update(updatedTrack.toDbTrack())
                 updateChapter.awaitAll(chapterUpdates)
+            }
             insertTrack.await(updatedTrack)
         } catch (e: Throwable) {
             logcat(LogPriority.WARN, e)

app/src/main/java/eu/kanade/domain/track/interactor/TrackChapter.kt

@@ -5,6 +5,7 @@ import eu.kanade.domain.track.model.toDbTrack
 import eu.kanade.domain.track.model.toDomainTrack
 import eu.kanade.domain.track.service.DelayedTrackingUpdateJob
 import eu.kanade.domain.track.store.DelayedTrackingStore
+import eu.kanade.tachiyomi.data.track.PageTracker
 import eu.kanade.tachiyomi.data.track.TrackerManager
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
@@ -56,4 +57,27 @@ class TrackChapter(
                 .forEach { logcat(LogPriority.WARN, it) }
         }
     }
+
+    suspend fun reportPageProgress(mangaId: Long, chapterUrl: String, pageIndex: Int) {
+        withNonCancellableContext {
+            val tracks = getTracks.await(mangaId)
+            if (tracks.isEmpty()) return@withNonCancellableContext
+
+            tracks.mapNotNull { track ->
+                val service = trackerManager.get(track.trackerId)
+                if (service == null || !service.isLoggedIn || service !is PageTracker) {
+                    return@mapNotNull null
+                }
+                async {
+                    runCatching {
+                        (service as PageTracker).updatePageProgress(track, pageIndex)
+                        (service as PageTracker).updatePageProgressWithUrl(chapterUrl, pageIndex)
+                    }
+                }
+            }
+                .awaitAll()
+                .mapNotNull { it.exceptionOrNull() }
+                .forEach { logcat(LogPriority.WARN, it) }
+        }
+    }
 }

app/src/main/java/eu/kanade/domain/track/service/TrackPreferences.kt

@@ -35,4 +35,6 @@ class TrackPreferences(
     fun anilistScoreType() = preferenceStore.getString("anilist_score_type", Anilist.POINT_10)
 
     fun autoUpdateTrack() = preferenceStore.getBoolean("pref_auto_update_manga_sync_key", true)
+
+    fun chapterBasedTracking() = preferenceStore.getBoolean("pref_tracking_granularity_chapter", false)
 }

app/src/main/java/eu/kanade/presentation/more/settings/screen/SettingsTrackingScreen.kt

@@ -181,6 +181,11 @@ object SettingsTrackingScreen : SearchableSettings {
                         } + listOf(Preference.PreferenceItem.InfoPreference(enhancedTrackerInfo))
                     ).toImmutableList(),
             ),
+            Preference.PreferenceItem.SwitchPreference(
+                pref = trackPreferences.chapterBasedTracking(),
+                title = stringResource(MR.strings.pref_chapter_level_tracking_title),
+                subtitle = stringResource(MR.strings.pref_chapter_level_tracking_desc),
+            ),
         )
     }
 

app/src/main/java/eu/kanade/tachiyomi/data/track/PageTracker.kt

@@ -0,0 +1,30 @@
+package eu.kanade.tachiyomi.data.track
+
+import eu.kanade.tachiyomi.data.database.models.Chapter
+
+
+/**
+ *
+ */
+interface PageTracker {
+
+    data class ChapterReadProgress(
+        val completed: Boolean,
+        val page: Int
+    ) {
+        operator fun compareTo(b: ChapterReadProgress): Int =
+            if (completed == b.completed)  page.coerceAtLeast(0) - b.page.coerceAtLeast(0)
+            else completed.compareTo(b.completed)
+
+    }
+
+    suspend fun updatePageProgress(track: tachiyomi.domain.track.model.Track, page: Int) {}
+    suspend fun updatePageProgressWithUrl(chapterUrl:String, page: Int) {}
+
+    suspend fun batchUpdateRemoteProgress(chapters: List<Chapter>)
+
+    suspend fun getChapterProgress(chapter: Chapter): ChapterReadProgress
+    suspend fun batchGetChapterProgress(chapters: List<Chapter>): Map<Chapter, ChapterReadProgress> {
+        return chapters.associateWith { getChapterProgress(it) }
+    }
+}

app/src/main/java/eu/kanade/tachiyomi/data/track/komga/Komga.kt

@@ -3,9 +3,11 @@ package eu.kanade.tachiyomi.data.track.komga
 import android.graphics.Color
 import dev.icerock.moko.resources.StringResource
 import eu.kanade.tachiyomi.R
+import eu.kanade.tachiyomi.data.database.models.Chapter
 import eu.kanade.tachiyomi.data.database.models.Track
 import eu.kanade.tachiyomi.data.track.BaseTracker
 import eu.kanade.tachiyomi.data.track.EnhancedTracker
+import eu.kanade.tachiyomi.data.track.PageTracker
 import eu.kanade.tachiyomi.data.track.model.TrackSearch
 import eu.kanade.tachiyomi.source.Source
 import kotlinx.collections.immutable.ImmutableList
@@ -16,7 +18,7 @@ import tachiyomi.domain.manga.model.Manga
 import tachiyomi.i18n.MR
 import tachiyomi.domain.track.model.Track as DomainTrack
 
-class Komga(id: Long) : BaseTracker(id, "Komga"), EnhancedTracker {
+class Komga(id: Long) : BaseTracker(id, "Komga"), EnhancedTracker, PageTracker {
 
     companion object {
         const val UNREAD = 1L
@@ -64,8 +66,7 @@ class Komga(id: Long) : BaseTracker(id, "Komga"), EnhancedTracker {
                 }
             }
         }
-
+        return if (trackPreferences.chapterBasedTracking().get()) track else api.updateProgress(track)
-        return api.updateProgress(track)
     }
 
     override suspend fun bind(track: Track, hasReadChapters: Boolean): Track {
@@ -111,4 +112,34 @@ class Komga(id: Long) : BaseTracker(id, "Komga"), EnhancedTracker {
         } else {
             null
         }
+
+    override suspend fun updatePageProgressWithUrl(chapterUrl: String, page: Int) {
+        api.updateBookProgress(chapterUrl, page)
+    }
+
+    override suspend fun batchUpdateRemoteProgress(chapters: List<Chapter>) {
+        chapters.forEach {
+            api.updateBookProgress(it.url, it.last_page_read, it.read)
+        }
+    }
+
+    override suspend fun getChapterProgress(chapter: Chapter): PageTracker.ChapterReadProgress {
+        val book = api.getBookInfo(chapter)
+        return PageTracker.ChapterReadProgress(book.readProgress?.completed ?: false, book.readProgress?.page ?: 0)
+    }
+
+    override suspend fun batchGetChapterProgress(chapters: List<Chapter>): Map<Chapter, PageTracker.ChapterReadProgress> {
+        if (chapters.isEmpty()) return mapOf()
+        val seriesId = api.getBookInfo(chapters[0]).seriesId
+        val urlBase = chapters[0].url.split("/books")[0]
+        val books = api.getAllBooksOfSeries(urlBase, seriesId)
+        return chapters.associateWith { chapter ->
+            val book = books.find { chapter.url.toBookId() == it.id }
+            return@associateWith PageTracker.ChapterReadProgress(book?.readProgress?.completed ?: false, book?.readProgress?.page ?: 0)
+        }
+    }
+
+    private fun String.toBookId():String? {
+        return Regex("/api/v1/books/(\\S+)").find(this)?.destructured?.let { (id) -> id }
+    }
 }

app/src/main/java/eu/kanade/tachiyomi/data/track/komga/KomgaApi.kt

@@ -6,6 +6,7 @@ import eu.kanade.tachiyomi.data.track.model.TrackSearch
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.awaitSuccess
 import eu.kanade.tachiyomi.network.parseAs
+import eu.kanade.tachiyomi.source.model.SChapter
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import logcat.LogPriority
@@ -107,4 +108,32 @@ class KomgaApi(
     private fun ReadListDto.toTrack(): TrackSearch = TrackSearch.create(trackId).also {
         it.title = name
     }
+
+    internal suspend fun getBookInfo(chapter: SChapter):BookDtoPartial {
+        with(json){
+            return client.newCall(GET(chapter.url, headers)).awaitSuccess().parseAs<BookDtoPartial>()
+        }
+    }
+
+    internal suspend fun getAllBooksOfSeries(v1UrlBase: String, seriesId: String): List<BookDtoPartial> {
+        with(json) {
+            return client.newCall(GET("$v1UrlBase/series/$seriesId/books?unpaged=true", headers)).awaitSuccess().parseAs<SeriesBookListDtoPartial>().content ?: listOf()
+        }
+    }
+
+    /**
+     * Komga book progress starts from 1.
+     *
+     * Komga API spec: page can be omitted if completed is set to true. completed can be omitted, and will be set accordingly depending on the page passed and the total number of pages in the book.
+     */
+    internal suspend fun updateBookProgress(bookUrl: String, pageIndex: Int = 0, complete: Boolean = false) {
+        //TODO: rate limit
+        val resp = client.newCall(
+            Request.Builder()
+                .url("${bookUrl}/read-progress")
+                .patch("{\"page\": ${pageIndex + 1}, \"completed\": $complete }".toRequestBody("Application/json".toMediaType()))
+                .build()
+        ).awaitSuccess()
+        logcat(LogPriority.DEBUG) { "update progress to ${pageIndex + 1} and complete status $complete with $resp"  }
+    }
 }

app/src/main/java/eu/kanade/tachiyomi/data/track/komga/KomgaModels.kt

@@ -105,3 +105,34 @@ data class ReadProgressV2Dto(
     val lastReadContinuousNumberSort: Double,
     val maxNumberSort: Float,
 )
+
+@Serializable
+data class BookReadProgressDto(
+    val page: Int,
+    val completed: Boolean,
+    val readDate: String?,
+    val created: String?,
+    val lastModified: String?,
+    val deviceId: String?,
+    val deviceName: String?
+)
+
+@Serializable
+data class BookDtoPartial(
+    val id: String,
+    val seriesId: String,
+    val seriesTitle: String,
+    val name: String,
+    val url: String,
+    val readProgress: BookReadProgressDto?,
+    val fileHash: String
+)
+
+@Serializable
+data class SeriesBookListDtoPartial(
+    val totalElements: Long?,
+    val totalPages: Int?,
+    val size: Int?,
+    val content: List<BookDtoPartial>?,
+    val empty: Boolean?
+)

app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderViewModel.kt

@@ -540,6 +540,7 @@ class ReaderViewModel @JvmOverloads constructor(
                 ),
             )
         }
+        updatePageReadProgress(readerChapter)
     }
 
     fun restartReadTimer() {
@@ -887,6 +888,17 @@ class ReaderViewModel @JvmOverloads constructor(
         }
     }
 
+    private fun updatePageReadProgress(readerChapter: ReaderChapter) {
+        if (incognitoMode) return
+        if (!trackPreferences.autoUpdateTrack().get()) return
+        if (!trackPreferences.chapterBasedTracking().get()) return
+
+        val manga = manga ?: return
+        viewModelScope.launchNonCancellable {
+            trackChapter.reportPageProgress(manga.id, readerChapter.chapter.url, chapterPageIndex)
+        }
+    }
+
     /**
      * Enqueues this [chapter] to be deleted when [deletePendingChapters] is called. The download
      * manager handles persisting it across process deaths.

app/src/test/java/eu/kanade/domain/track/PageTrackerTest.kt

@@ -0,0 +1,69 @@
+package eu.kanade.domain.track
+
+import eu.kanade.domain.track.interactor.SyncChapterProgressWithTrack
+import eu.kanade.tachiyomi.data.database.models.Chapter
+import eu.kanade.tachiyomi.data.database.models.ChapterImpl
+import eu.kanade.tachiyomi.data.track.PageTracker
+import org.junit.jupiter.api.Test
+
+
+class PageTrackerTest {
+    companion object {
+        private object SampleSeries {
+            val chaptersWithRemoteProgress: Map<Chapter, PageTracker.ChapterReadProgress> = mapOf(
+                createTestChapterEntry(1, true, 100, false, 5),  //reread finished
+                createTestChapterEntry(2, true, 100, true, 100),
+                createTestChapterEntry(3, false, 3, false, 3),
+                createTestChapterEntry(4, false, 5, false, 10),
+                createTestChapterEntry(5, false, 10, false, 15),
+                createTestChapterEntry(6, false, 10, false, 5),
+                createTestChapterEntry(7, false, 0, false, 0),
+                createTestChapterEntry(8, true, 100, false, 0), //local read, remote has not started reread
+                createTestChapterEntry(9, false, 0, false, -1),
+            )
+        }
+        
+        private val Chapter.progress: PageTracker.ChapterReadProgress
+            get() = PageTracker.ChapterReadProgress(read, last_page_read)
+
+        private fun PageTracker.ChapterReadProgress.compareWith(b: PageTracker.ChapterReadProgress): String {
+            return StringBuilder("Update(").apply {
+                if (completed != b.completed) append("completed ${b.completed} -> $completed; ")
+                if (page != b.page) append("page: ${b.page} ->  $page")
+                append(")")
+            }.toString()
+        }
+
+        private fun createTestChapterEntry(localId: Int,  localRead: Boolean, localPage: Int, remoteRead: Boolean, remotePage: Int) =
+            ChapterImpl().apply {
+                id = localId.toLong()
+                read = localRead
+                last_page_read = localPage
+                name = "Chapter $localId"
+                url = "sample.site/series/114514/books/$localId"
+            } to PageTracker.ChapterReadProgress(remoteRead, remotePage)
+    }
+
+    @Test
+    fun testSyncStrategies() {
+        testSampleWithStrategy(1)
+        testSampleWithStrategy(2)
+        testSampleWithStrategy(3)
+    }
+
+    private fun testSampleWithStrategy(strategy: Int) {
+        SyncChapterProgressWithTrack.Companion.syncStrategy = strategy
+        val result = SampleSeries.chaptersWithRemoteProgress.entries.groupBy {
+            SyncChapterProgressWithTrack.Companion.resolveRemoteProgress(it.key, it.value)
+        }
+
+        println("\nStrategy: $strategy, split: ${result.entries.associate { it.key to it.value.size }}")
+
+
+        println("Update to local : ${result[SyncChapterProgressWithTrack.RemoteProgressResolution.ACCEPT]?.map { "${it.key.name}  ${it.value.compareWith(it.key.progress)}" }}")
+        println("Update to remote : ${result[SyncChapterProgressWithTrack.RemoteProgressResolution.REJECT]?.map {  "${it.key.name}  ${it.key.progress.compareWith(it.value)}"  }}")
+        println("No change : ${result[SyncChapterProgressWithTrack.RemoteProgressResolution.SAME]?.map { it.key.name }}")
+    }
+
+
+}

i18n/src/commonMain/resources/MR/base/strings.xml

@@ -494,6 +494,8 @@
     <string name="enhanced_tracking_info">Provides enhanced features for specific sources. Entries are automatically tracked when added to your library.</string>
     <string name="action_track">Track</string>
     <string name="track_activity_name">Tracker login</string>
+    <string name="pref_chapter_level_tracking_title">Enable detailed tracking</string>
+    <string name="pref_chapter_level_tracking_desc">Try to sync reading progress of each chapter (e.g. page last read, complete status) with enhanced trackers. Currently only supports Komga. </string>
 
       <!-- Browse section -->
     <string name="pref_hide_in_library_items">Hide entries already in library</string>