Database changes to support library syncing (#9683)

* feat: added migrations.

* feat: create triggers, account for new installs.

* feat: update mappers to include the new field.

* feat: update backupManga and backupChapter.

Include the new fields to be backed up as well.

* feat: add sql query to fetch all manga with `last_favorited_at` field.

* feat: version bump.

* chore: revert and refactor.

* chore: forgot to lower case the field name.

* chore: added getAllManga query as well renamed `fetchMangaWithLastFavorite` to `getMangasWithFavoriteTimestamp`

* chore: oops that's not meant to be there.

* feat: back fill and set last_modified_at to not null.

* chore: remove redundant triggers.

* fix: build error, accidentally removed insert.

* fix: build error, accidentally removed insert.

* refactor: review pointer, make fields not null.

app/build.gradle.kts

@@ -23,7 +23,8 @@ android {
     defaultConfig {
         applicationId = "eu.kanade.tachiyomi"
 	
-        versionCode = 103
+        versionCode = 104
+      
         versionName = "0.14.6"
 
         buildConfigField("String", "COMMIT_COUNT", "\"${getCommitCount()}\"")

app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupChapter.kt

@@ -20,6 +20,7 @@ data class BackupChapter(
     // chapterNumber is called number is 1.x
     @ProtoNumber(9) var chapterNumber: Float = 0F,
     @ProtoNumber(10) var sourceOrder: Long = 0,
+    @ProtoNumber(11) var lastModifiedAt: Long = 0,
 ) {
     fun toChapterImpl(): Chapter {
         return Chapter.create().copy(
@@ -33,11 +34,12 @@ data class BackupChapter(
             dateFetch = this@BackupChapter.dateFetch,
             dateUpload = this@BackupChapter.dateUpload,
             sourceOrder = this@BackupChapter.sourceOrder,
+            lastModifiedAt = this@BackupChapter.lastModifiedAt,
         )
     }
 }
 
-val backupChapterMapper = { _: Long, _: Long, url: String, name: String, scanlator: String?, read: Boolean, bookmark: Boolean, lastPageRead: Long, chapterNumber: Float, source_order: Long, dateFetch: Long, dateUpload: Long ->
+val backupChapterMapper = { _: Long, _: Long, url: String, name: String, scanlator: String?, read: Boolean, bookmark: Boolean, lastPageRead: Long, chapterNumber: Float, source_order: Long, dateFetch: Long, dateUpload: Long, lastModifiedAt: Long ->
     BackupChapter(
         url = url,
         name = name,
@@ -49,5 +51,6 @@ val backupChapterMapper = { _: Long, _: Long, url: String, name: String, scanlat
         dateFetch = dateFetch,
         dateUpload = dateUpload,
         sourceOrder = source_order,
+        lastModifiedAt = lastModifiedAt,
     )
 }

app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupManga.kt

@@ -39,6 +39,8 @@ data class BackupManga(
     @ProtoNumber(103) var viewer_flags: Int? = null,
     @ProtoNumber(104) var history: List<BackupHistory> = emptyList(),
     @ProtoNumber(105) var updateStrategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE,
+    @ProtoNumber(106) var lastModifiedAt: Long = 0,
+    @ProtoNumber(107) var favoriteModifiedAt: Long? = 0,
 ) {
     fun getMangaImpl(): Manga {
         return Manga.create().copy(
@@ -56,6 +58,8 @@ data class BackupManga(
             viewerFlags = (this@BackupManga.viewer_flags ?: this@BackupManga.viewer).toLong(),
             chapterFlags = this@BackupManga.chapterFlags.toLong(),
             updateStrategy = this@BackupManga.updateStrategy,
+            lastModifiedAt = this@BackupManga.lastModifiedAt,
+            favoriteModifiedAt = this@BackupManga.favoriteModifiedAt,
         )
     }
 
@@ -89,6 +93,8 @@ data class BackupManga(
                 viewer_flags = manga.viewerFlags.toInt(),
                 chapterFlags = manga.chapterFlags.toInt(),
                 updateStrategy = manga.updateStrategy,
+                lastModifiedAt = manga.lastModifiedAt,
+                favoriteModifiedAt = manga.favoriteModifiedAt,
             )
         }
     }

app/src/main/java/eu/kanade/tachiyomi/data/database/models/Chapter.kt

@@ -19,6 +19,8 @@ interface Chapter : SChapter, Serializable {
     var date_fetch: Long
 
     var source_order: Int
+
+    var last_modified: Long
 }
 
 fun Chapter.toDomainChapter(): DomainChapter? {
@@ -36,5 +38,6 @@ fun Chapter.toDomainChapter(): DomainChapter? {
         dateUpload = date_upload,
         chapterNumber = chapter_number,
         scanlator = scanlator,
+        lastModifiedAt = last_modified,
     )
 }

app/src/main/java/eu/kanade/tachiyomi/data/database/models/ChapterImpl.kt

@@ -26,6 +26,8 @@ class ChapterImpl : Chapter {
 
     override var source_order: Int = 0
 
+    override var last_modified: Long = 0
+
     override fun equals(other: Any?): Boolean {
         if (this === other) return true
         if (other == null || javaClass != other.javaClass) return false

data/src/main/java/tachiyomi/data/chapter/ChapterMapper.kt

@@ -2,8 +2,8 @@ package tachiyomi.data.chapter
 
 import tachiyomi.domain.chapter.model.Chapter
 
-val chapterMapper: (Long, Long, String, String, String?, Boolean, Boolean, Long, Float, Long, Long, Long) -> Chapter =
-    { id, mangaId, url, name, scanlator, read, bookmark, lastPageRead, chapterNumber, sourceOrder, dateFetch, dateUpload ->
+val chapterMapper: (Long, Long, String, String, String?, Boolean, Boolean, Long, Float, Long, Long, Long, Long) -> Chapter =
+    { id, mangaId, url, name, scanlator, read, bookmark, lastPageRead, chapterNumber, sourceOrder, dateFetch, dateUpload, lastModifiedAt ->
         Chapter(
             id = id,
             mangaId = mangaId,
@@ -17,5 +17,6 @@ val chapterMapper: (Long, Long, String, String, String?, Boolean, Boolean, Long,
             dateUpload = dateUpload,
             chapterNumber = chapterNumber,
             scanlator = scanlator,
+            lastModifiedAt = lastModifiedAt,
         )
     }

data/src/main/java/tachiyomi/data/manga/MangaMapper.kt

@@ -4,8 +4,8 @@ import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import tachiyomi.domain.library.model.LibraryManga
 import tachiyomi.domain.manga.model.Manga
 
-val mangaMapper: (Long, Long, String, String?, String?, String?, List<String>?, String, Long, String?, Boolean, Long?, Long?, Boolean, Long, Long, Long, Long, UpdateStrategy, Long) -> Manga =
-    { id, source, url, artist, author, description, genre, title, status, thumbnailUrl, favorite, lastUpdate, nextUpdate, initialized, viewerFlags, chapterFlags, coverLastModified, dateAdded, updateStrategy, calculateInterval ->
+val mangaMapper: (Long, Long, String, String?, String?, String?, List<String>?, String, Long, String?, Boolean, Long?, Long?, Boolean, Long, Long, Long, Long, UpdateStrategy, Long, Long, Long?) -> Manga =
+    { id, source, url, artist, author, description, genre, title, status, thumbnailUrl, favorite, lastUpdate, nextUpdate, initialized, viewerFlags, chapterFlags, coverLastModified, dateAdded, updateStrategy, calculateInterval, lastModifiedAt, favoriteModifiedAt ->
         Manga(
             id = id,
             source = source,
@@ -27,11 +27,13 @@ val mangaMapper: (Long, Long, String, String?, String?, String?, List<String>?,
             thumbnailUrl = thumbnailUrl,
             updateStrategy = updateStrategy,
             initialized = initialized,
+            lastModifiedAt = lastModifiedAt,
+            favoriteModifiedAt = favoriteModifiedAt,
         )
     }
 
-val libraryManga: (Long, Long, String, String?, String?, String?, List<String>?, String, Long, String?, Boolean, Long?, Long?, Boolean, Long, Long, Long, Long, UpdateStrategy, Long, Long, Long, Long, Long, Long, Long, Long) -> LibraryManga =
-    { id, source, url, artist, author, description, genre, title, status, thumbnailUrl, favorite, lastUpdate, nextUpdate, initialized, viewerFlags, chapterFlags, coverLastModified, dateAdded, updateStrategy, calculateInterval, totalCount, readCount, latestUpload, chapterFetchedAt, lastRead, bookmarkCount, category ->
+val libraryManga: (Long, Long, String, String?, String?, String?, List<String>?, String, Long, String?, Boolean, Long?, Long?, Boolean, Long, Long, Long, Long, UpdateStrategy, Long, Long, Long?, Long, Long, Long, Long, Long, Long, Long) -> LibraryManga =
+    { id, source, url, artist, author, description, genre, title, status, thumbnailUrl, favorite, lastUpdate, nextUpdate, initialized, viewerFlags, chapterFlags, coverLastModified, dateAdded, updateStrategy, calculateInterval, lastModifiedAt, favoriteModifiedAt, totalCount, readCount, latestUpload, chapterFetchedAt, lastRead, bookmarkCount, category ->
         LibraryManga(
             manga = mangaMapper(
                 id,
@@ -54,6 +56,8 @@ val libraryManga: (Long, Long, String, String?, String?, String?, List<String>?,
                 dateAdded,
                 updateStrategy,
                 calculateInterval,
+                lastModifiedAt,
+                favoriteModifiedAt,
             ),
             category = category,
             totalChapters = totalCount,

data/src/main/sqldelight/tachiyomi/data/chapters.sq

@@ -11,6 +11,7 @@ CREATE TABLE chapters(
     source_order INTEGER NOT NULL,
     date_fetch INTEGER AS Long NOT NULL,
     date_upload INTEGER AS Long NOT NULL,
+    last_modified_at INTEGER AS Long NOT NULL,
     FOREIGN KEY(manga_id) REFERENCES mangas (_id)
     ON DELETE CASCADE
 );
@@ -18,6 +19,15 @@ CREATE TABLE chapters(
 CREATE INDEX chapters_manga_id_index ON chapters(manga_id);
 CREATE INDEX chapters_unread_by_manga_index ON chapters(manga_id, read) WHERE read = 0;
 
+CREATE TRIGGER update_last_modified_at_chapters
+AFTER UPDATE ON chapters
+FOR EACH ROW
+BEGIN
+  UPDATE chapters
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
 getChapterById:
 SELECT *
 FROM chapters
@@ -50,8 +60,8 @@ DELETE FROM chapters
 WHERE _id IN :chapterIds;
 
 insert:
-INSERT INTO chapters(manga_id,url,name,scanlator,read,bookmark,last_page_read,chapter_number,source_order,date_fetch,date_upload)
-VALUES (:mangaId,:url,:name,:scanlator,:read,:bookmark,:lastPageRead,:chapterNumber,:sourceOrder,:dateFetch,:dateUpload);
+INSERT INTO chapters(manga_id, url, name, scanlator, read, bookmark, last_page_read, chapter_number, source_order, date_fetch, date_upload, last_modified_at)
+VALUES (:mangaId, :url, :name, :scanlator, :read, :bookmark, :lastPageRead, :chapterNumber, :sourceOrder, :dateFetch, :dateUpload, strftime('%s', 'now'));
 
 update:
 UPDATE chapters

data/src/main/sqldelight/tachiyomi/data/mangas.sq

@@ -22,12 +22,31 @@ CREATE TABLE mangas(
     cover_last_modified INTEGER AS Long NOT NULL,
     date_added INTEGER AS Long NOT NULL,
     update_strategy INTEGER AS UpdateStrategy NOT NULL DEFAULT 0,
-    calculate_interval INTEGER DEFAULT 0 NOT NULL
+    calculate_interval INTEGER DEFAULT 0 NOT NULL,
+    last_modified_at INTEGER AS Long NOT NULL,
+    favorite_modified_at INTEGER AS Long
 );
 
 CREATE INDEX library_favorite_index ON mangas(favorite) WHERE favorite = 1;
 CREATE INDEX mangas_url_index ON mangas(url);
 
+CREATE TRIGGER update_favorite_modified_at_mangas
+AFTER UPDATE OF favorite ON mangas
+BEGIN
+  UPDATE mangas
+  SET favorite_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
+CREATE TRIGGER update_last_modified_at_mangas
+AFTER UPDATE ON mangas
+FOR EACH ROW
+BEGIN
+  UPDATE mangas
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
 getMangaById:
 SELECT *
 FROM mangas
@@ -45,6 +64,15 @@ SELECT *
 FROM mangas
 WHERE favorite = 1;
 
+getAllManga:
+SELECT *
+FROM mangas;
+
+getMangasWithFavoriteTimestamp:
+SELECT *
+FROM mangas
+WHERE favorite_modified_at IS NOT NULL;
+
 getSourceIdWithFavoriteCount:
 SELECT
 source,
@@ -81,8 +109,8 @@ DELETE FROM mangas
 WHERE favorite = 0 AND source IN :sourceIds;
 
 insert:
-INSERT INTO mangas(source,url,artist,author,description,genre,title,status,thumbnail_url,favorite,last_update,next_update,initialized,viewer,chapter_flags,cover_last_modified,date_added,update_strategy,calculate_interval)
-VALUES (:source,:url,:artist,:author,:description,:genre,:title,:status,:thumbnailUrl,:favorite,:lastUpdate,:nextUpdate,:initialized,:viewerFlags,:chapterFlags,:coverLastModified,:dateAdded,:updateStrategy,:calculateInterval);
+INSERT INTO mangas(source, url, artist, author, description, genre, title, status, thumbnail_url, favorite, last_update, next_update, initialized, viewer, chapter_flags, cover_last_modified, date_added, update_strategy, calculate_interval, last_modified_at)
+VALUES (:source, :url, :artist, :author, :description, :genre, :title, :status, :thumbnailUrl, :favorite, :lastUpdate, :nextUpdate, :initialized, :viewerFlags, :chapterFlags, :coverLastModified, :dateAdded, :updateStrategy, :calculateInterval, strftime('%s', 'now'));
 
 update:
 UPDATE mangas SET

data/src/main/sqldelight/tachiyomi/data/mangas_categories.sq

@@ -2,15 +2,25 @@ CREATE TABLE mangas_categories(
     _id INTEGER NOT NULL PRIMARY KEY,
     manga_id INTEGER NOT NULL,
     category_id INTEGER NOT NULL,
+    last_modified_at INTEGER AS Long NOT NULL,
     FOREIGN KEY(category_id) REFERENCES categories (_id)
     ON DELETE CASCADE,
     FOREIGN KEY(manga_id) REFERENCES mangas (_id)
     ON DELETE CASCADE
 );
 
+CREATE TRIGGER update_last_modified_at_mangas_categories
+AFTER UPDATE ON mangas_categories
+FOR EACH ROW
+BEGIN
+  UPDATE mangas_categories
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
 insert:
-INSERT INTO mangas_categories(manga_id, category_id)
-VALUES (:mangaId, :categoryId);
+INSERT INTO mangas_categories(manga_id, category_id, last_modified_at)
+VALUES (:mangaId, :categoryId, strftime('%s', 'now'));
 
 deleteMangaCategoryByMangaId:
 DELETE FROM mangas_categories

data/src/main/sqldelight/tachiyomi/migrations/25.sqm

@@ -0,0 +1,49 @@
+ALTER TABLE mangas ADD COLUMN last_modified_at INTEGER AS Long NOT NULL;
+ALTER TABLE mangas ADD COLUMN favorite_modified_at INTEGER AS Long;
+ALTER TABLE mangas_categories ADD COLUMN last_modified_at INTEGER AS Long NOT NULL;
+ALTER TABLE chapters ADD COLUMN last_modified_at INTEGER AS Long NOT NULL;
+
+UPDATE mangas SET last_modified_at = strftime('%s', 'now');
+UPDATE mangas SET favorite_modified_at = strftime('%s', 'now') WHERE favorite = 1;
+UPDATE mangas_categories SET last_modified_at = strftime('%s', 'now');
+UPDATE chapters SET last_modified_at = strftime('%s', 'now');
+
+-- Create triggers
+DROP TRIGGER IF EXISTS update_last_modified_at_mangas;
+CREATE TRIGGER update_last_modified_at_mangas
+AFTER UPDATE ON mangas
+FOR EACH ROW
+BEGIN
+  UPDATE mangas
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
+DROP TRIGGER IF EXISTS update_favorite_modified_at_mangas;
+CREATE TRIGGER update_last_favorited_at_mangas
+AFTER UPDATE OF favorite ON mangas
+BEGIN
+  UPDATE mangas
+  SET favorite_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
+DROP TRIGGER IF EXISTS update_last_modified_at_chapters;
+CREATE TRIGGER update_last_modified_at_chapters
+AFTER UPDATE ON chapters
+FOR EACH ROW
+BEGIN
+  UPDATE chapters
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
+DROP TRIGGER IF EXISTS update_last_modified_at_mangas_categories;
+CREATE TRIGGER update_last_modified_at_mangas_categories
+AFTER UPDATE ON mangas_categories
+FOR EACH ROW
+BEGIN
+  UPDATE mangas_categories
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;

domain/src/main/java/tachiyomi/domain/chapter/model/Chapter.kt

@@ -13,6 +13,7 @@ data class Chapter(
     val dateUpload: Long,
     val chapterNumber: Float,
     val scanlator: String?,
+    val lastModifiedAt: Long,
 ) {
     val isRecognizedNumber: Boolean
         get() = chapterNumber >= 0f
@@ -31,6 +32,7 @@ data class Chapter(
             dateUpload = -1,
             chapterNumber = -1f,
             scanlator = null,
+            lastModifiedAt = 0,
         )
     }
 }

domain/src/main/java/tachiyomi/domain/manga/model/Manga.kt

@@ -24,6 +24,8 @@ data class Manga(
     val thumbnailUrl: String?,
     val updateStrategy: UpdateStrategy,
     val initialized: Boolean,
+    val lastModifiedAt: Long,
+    val favoriteModifiedAt: Long?,
 ) : Serializable {
 
     val sorting: Long
@@ -109,6 +111,8 @@ data class Manga(
             thumbnailUrl = null,
             updateStrategy = UpdateStrategy.ALWAYS_UPDATE,
             initialized = false,
+            lastModifiedAt = 0L,
+            favoriteModifiedAt = 0L,
         )
     }
 }