feat: db changes to accommodate new cross device syncing logic. (#450)

* feat: db changes to accommodate new syncing logic.

Using timestamp to sync is a bit skewed due to system clock etc and therefore there was a lot of issues with it such as removing a manga that shouldn't have been removed. Marking chapters as unread even though it was marked as a read. Hopefully by using versioning system it should eliminate those issues.

* chore: add new line.

* chore: remove isSyncing from Chapter/Manga model.

* chore: remove isSyncing leftover.

* chore: remove isSyncing.

* refactor: remove isSync guard.

Just use it directly to 1 now since we don't have the isSyncing field in Manga or Chapter.

* Lint and stuff

* Add missing ,

---------

Co-authored-by: AntsyLich <59261191+AntsyLich@users.noreply.github.com>
This commit is contained in:
KaiserBh 2024-03-10 06:45:41 +11:00 committed by GitHub
parent 402e579a69
commit 4ae9dbe524
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 161 additions and 20 deletions

View File

@ -22,7 +22,7 @@ android {
defaultConfig {
applicationId = "app.mihon"
versionCode = 5
versionCode = 6
versionName = "0.16.4"
buildConfigField("String", "COMMIT_COUNT", "\"${getCommitCount()}\"")

View File

@ -98,4 +98,5 @@ private fun Manga.toBackupManga() =
updateStrategy = this.updateStrategy,
lastModifiedAt = this.lastModifiedAt,
favoriteModifiedAt = this.favoriteModifiedAt,
version = this.version,
)

View File

@ -4,6 +4,7 @@ import kotlinx.serialization.Serializable
import kotlinx.serialization.protobuf.ProtoNumber
import tachiyomi.domain.chapter.model.Chapter
@Suppress("MagicNumber")
@Serializable
data class BackupChapter(
// in 1.x some of these values have different names
@ -21,6 +22,7 @@ data class BackupChapter(
@ProtoNumber(9) var chapterNumber: Float = 0F,
@ProtoNumber(10) var sourceOrder: Long = 0,
@ProtoNumber(11) var lastModifiedAt: Long = 0,
@ProtoNumber(12) var version: Long = 0,
) {
fun toChapterImpl(): Chapter {
return Chapter.create().copy(
@ -35,6 +37,7 @@ data class BackupChapter(
dateUpload = this@BackupChapter.dateUpload,
sourceOrder = this@BackupChapter.sourceOrder,
lastModifiedAt = this@BackupChapter.lastModifiedAt,
version = this@BackupChapter.version,
)
}
}
@ -53,6 +56,8 @@ val backupChapterMapper = {
dateFetch: Long,
dateUpload: Long,
lastModifiedAt: Long,
version: Long,
_: Long,
->
BackupChapter(
url = url,
@ -66,5 +71,6 @@ val backupChapterMapper = {
dateUpload = dateUpload,
sourceOrder = sourceOrder,
lastModifiedAt = lastModifiedAt,
version = version,
)
}

View File

@ -5,7 +5,10 @@ import kotlinx.serialization.Serializable
import kotlinx.serialization.protobuf.ProtoNumber
import tachiyomi.domain.manga.model.Manga
@Suppress("DEPRECATION")
@Suppress(
"DEPRECATION",
"MagicNumber",
)
@Serializable
data class BackupManga(
// in 1.x some of these values have different names
@ -39,6 +42,7 @@ data class BackupManga(
@ProtoNumber(106) var lastModifiedAt: Long = 0,
@ProtoNumber(107) var favoriteModifiedAt: Long? = null,
@ProtoNumber(108) var excludedScanlators: List<String> = emptyList(),
@ProtoNumber(109) var version: Long = 0,
) {
fun getMangaImpl(): Manga {
return Manga.create().copy(
@ -58,6 +62,7 @@ data class BackupManga(
updateStrategy = this@BackupManga.updateStrategy,
lastModifiedAt = this@BackupManga.lastModifiedAt,
favoriteModifiedAt = this@BackupManga.favoriteModifiedAt,
version = this@BackupManga.version,
)
}
}

View File

@ -83,7 +83,7 @@ class MangaRestorer(
}
private suspend fun restoreExistingManga(manga: Manga, dbManga: Manga): Manga {
return if (manga.lastModifiedAt > dbManga.lastModifiedAt) {
return if (manga.version > dbManga.version) {
updateManga(dbManga.copyFrom(manga).copy(id = dbManga.id))
} else {
updateManga(manga.copyFrom(dbManga).copy(id = dbManga.id))
@ -100,6 +100,7 @@ class MangaRestorer(
thumbnailUrl = newer.thumbnailUrl,
status = newer.status,
initialized = this.initialized || newer.initialized,
version = newer.version,
)
}
@ -126,6 +127,8 @@ class MangaRestorer(
dateAdded = manga.dateAdded,
mangaId = manga.id,
updateStrategy = manga.updateStrategy.let(UpdateStrategyColumnAdapter::encode),
version = manga.version,
isSyncing = 1,
)
}
return manga
@ -137,6 +140,7 @@ class MangaRestorer(
return manga.copy(
initialized = manga.description != null,
id = insertManga(manga),
version = manga.version,
)
}
@ -183,7 +187,7 @@ class MangaRestorer(
}
private fun Chapter.forComparison() =
this.copy(id = 0L, mangaId = 0L, dateFetch = 0L, dateUpload = 0L, lastModifiedAt = 0L)
this.copy(id = 0L, mangaId = 0L, dateFetch = 0L, dateUpload = 0L, lastModifiedAt = 0L, version = 0L)
private suspend fun insertNewChapters(chapters: List<Chapter>) {
handler.await(true) {
@ -200,6 +204,7 @@ class MangaRestorer(
chapter.sourceOrder,
chapter.dateFetch,
chapter.dateUpload,
chapter.version,
)
}
}
@ -221,6 +226,8 @@ class MangaRestorer(
dateFetch = null,
dateUpload = null,
chapterId = chapter.id,
version = chapter.version,
isSyncing = 0,
)
}
}
@ -253,6 +260,7 @@ class MangaRestorer(
coverLastModified = manga.coverLastModified,
dateAdded = manga.dateAdded,
updateStrategy = manga.updateStrategy,
version = manga.version,
)
mangasQueries.selectLastInsertedRowId()
}

View File

@ -21,6 +21,8 @@ interface Chapter : SChapter, Serializable {
var source_order: Int
var last_modified: Long
var version: Long
}
fun Chapter.toDomainChapter(): DomainChapter? {
@ -39,5 +41,6 @@ fun Chapter.toDomainChapter(): DomainChapter? {
chapterNumber = chapter_number.toDouble(),
scanlator = scanlator,
lastModifiedAt = last_modified,
version = version,
)
}

View File

@ -28,6 +28,8 @@ class ChapterImpl : Chapter {
override var last_modified: Long = 0
override var version: Long = 0
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false

View File

@ -29,6 +29,7 @@ class ChapterRepositoryImpl(
chapter.sourceOrder,
chapter.dateFetch,
chapter.dateUpload,
chapter.version,
)
val lastInsertId = chaptersQueries.selectLastInsertedRowId().executeAsOne()
chapter.copy(id = lastInsertId)
@ -64,6 +65,8 @@ class ChapterRepositoryImpl(
dateFetch = chapterUpdate.dateFetch,
dateUpload = chapterUpdate.dateUpload,
chapterId = chapterUpdate.id,
version = chapterUpdate.version,
isSyncing = 0,
)
}
}
@ -124,6 +127,7 @@ class ChapterRepositoryImpl(
}
}
@Suppress("LongParameterList")
private fun mapChapter(
id: Long,
mangaId: Long,
@ -138,6 +142,9 @@ class ChapterRepositoryImpl(
dateFetch: Long,
dateUpload: Long,
lastModifiedAt: Long,
version: Long,
@Suppress("UNUSED_PARAMETER")
isSyncing: Long,
): Chapter = Chapter(
id = id,
mangaId = mangaId,
@ -152,5 +159,6 @@ class ChapterRepositoryImpl(
chapterNumber = chapterNumber,
scanlator = scanlator,
lastModifiedAt = lastModifiedAt,
version = version,
)
}

View File

@ -5,6 +5,7 @@ import tachiyomi.domain.library.model.LibraryManga
import tachiyomi.domain.manga.model.Manga
object MangaMapper {
@Suppress("LongParameterList")
fun mapManga(
id: Long,
source: Long,
@ -28,6 +29,9 @@ object MangaMapper {
calculateInterval: Long,
lastModifiedAt: Long,
favoriteModifiedAt: Long?,
version: Long,
@Suppress("UNUSED_PARAMETER")
isSyncing: Long,
): Manga = Manga(
id = id,
source = source,
@ -51,8 +55,10 @@ object MangaMapper {
initialized = initialized,
lastModifiedAt = lastModifiedAt,
favoriteModifiedAt = favoriteModifiedAt,
version = version,
)
@Suppress("LongParameterList")
fun mapLibraryManga(
id: Long,
source: Long,
@ -76,6 +82,8 @@ object MangaMapper {
calculateInterval: Long,
lastModifiedAt: Long,
favoriteModifiedAt: Long?,
version: Long,
isSyncing: Long,
totalCount: Long,
readCount: Double,
latestUpload: Long,
@ -107,6 +115,8 @@ object MangaMapper {
calculateInterval,
lastModifiedAt,
favoriteModifiedAt,
version,
isSyncing,
),
category = category,
totalChapters = totalCount,

View File

@ -106,6 +106,7 @@ class MangaRepositoryImpl(
coverLastModified = manga.coverLastModified,
dateAdded = manga.dateAdded,
updateStrategy = manga.updateStrategy,
version = manga.version,
)
mangasQueries.selectLastInsertedRowId()
}
@ -155,6 +156,8 @@ class MangaRepositoryImpl(
dateAdded = value.dateAdded,
mangaId = value.id,
updateStrategy = value.updateStrategy?.let(UpdateStrategyColumnAdapter::encode),
version = value.version,
isSyncing = 0,
)
}
}

View File

@ -14,6 +14,8 @@ CREATE TABLE chapters(
date_fetch INTEGER NOT NULL,
date_upload INTEGER NOT NULL,
last_modified_at INTEGER NOT NULL DEFAULT 0,
version INTEGER NOT NULL DEFAULT 0,
is_syncing INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY(manga_id) REFERENCES mangas (_id)
ON DELETE CASCADE
);
@ -30,6 +32,22 @@ BEGIN
WHERE _id = new._id;
END;
CREATE TRIGGER update_chapter_and_manga_version AFTER UPDATE ON chapters
WHEN new.is_syncing = 0 AND (
new.read != old.read OR
new.bookmark != old.bookmark OR
new.last_page_read != old.last_page_read
)
BEGIN
-- Update the chapter version
UPDATE chapters SET version = version + 1
WHERE _id = new._id;
-- Update the manga version
UPDATE mangas SET version = version + 1
WHERE _id = new.manga_id AND (SELECT is_syncing FROM mangas WHERE _id = new.manga_id) = 0;
END;
getChapterById:
SELECT *
FROM chapters
@ -73,9 +91,14 @@ removeChaptersWithIds:
DELETE FROM chapters
WHERE _id IN :chapterIds;
resetIsSyncing:
UPDATE chapters
SET is_syncing = 0
WHERE is_syncing = 1;
insert:
INSERT INTO chapters(manga_id, url, name, scanlator, read, bookmark, last_page_read, chapter_number, source_order, date_fetch, date_upload, last_modified_at)
VALUES (:mangaId, :url, :name, :scanlator, :read, :bookmark, :lastPageRead, :chapterNumber, :sourceOrder, :dateFetch, :dateUpload, 0);
INSERT INTO chapters(manga_id, url, name, scanlator, read, bookmark, last_page_read, chapter_number, source_order, date_fetch, date_upload, last_modified_at, version, is_syncing)
VALUES (:mangaId, :url, :name, :scanlator, :read, :bookmark, :lastPageRead, :chapterNumber, :sourceOrder, :dateFetch, :dateUpload, 0, :version, 0);
update:
UPDATE chapters
@ -89,7 +112,9 @@ SET manga_id = coalesce(:mangaId, manga_id),
chapter_number = coalesce(:chapterNumber, chapter_number),
source_order = coalesce(:sourceOrder, source_order),
date_fetch = coalesce(:dateFetch, date_fetch),
date_upload = coalesce(:dateUpload, date_upload)
date_upload = coalesce(:dateUpload, date_upload),
version = coalesce(:version, version),
is_syncing = coalesce(:isSyncing, is_syncing)
WHERE _id = :chapterId;
selectLastInsertedRowId:

View File

@ -25,7 +25,9 @@ CREATE TABLE mangas(
update_strategy INTEGER AS UpdateStrategy NOT NULL DEFAULT 0,
calculate_interval INTEGER DEFAULT 0 NOT NULL,
last_modified_at INTEGER NOT NULL DEFAULT 0,
favorite_modified_at INTEGER
favorite_modified_at INTEGER,
version INTEGER NOT NULL DEFAULT 0,
is_syncing INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX library_favorite_index ON mangas(favorite) WHERE favorite = 1;
@ -48,6 +50,16 @@ BEGIN
WHERE _id = new._id;
END;
CREATE TRIGGER update_manga_version AFTER UPDATE ON mangas
BEGIN
UPDATE mangas SET version = version + 1
WHERE _id = new._id AND new.is_syncing = 0 AND (
new.url != old.url OR
new.description != old.description OR
new.favorite != old.favorite
);
END;
getMangaById:
SELECT *
FROM mangas
@ -104,6 +116,11 @@ resetViewerFlags:
UPDATE mangas
SET viewer = 0;
resetIsSyncing:
UPDATE mangas
SET is_syncing = 0
WHERE is_syncing = 1;
getSourceIdsWithNonLibraryManga:
SELECT source, COUNT(*) AS manga_count
FROM mangas
@ -116,8 +133,8 @@ WHERE favorite = 0
AND source IN :sourceIds;
insert:
INSERT INTO mangas(source, url, artist, author, description, genre, title, status, thumbnail_url, favorite, last_update, next_update, initialized, viewer, chapter_flags, cover_last_modified, date_added, update_strategy, calculate_interval, last_modified_at)
VALUES (:source, :url, :artist, :author, :description, :genre, :title, :status, :thumbnailUrl, :favorite, :lastUpdate, :nextUpdate, :initialized, :viewerFlags, :chapterFlags, :coverLastModified, :dateAdded, :updateStrategy, :calculateInterval, 0);
INSERT INTO mangas(source, url, artist, author, description, genre, title, status, thumbnail_url, favorite, last_update, next_update, initialized, viewer, chapter_flags, cover_last_modified, date_added, update_strategy, calculate_interval, last_modified_at, version)
VALUES (:source, :url, :artist, :author, :description, :genre, :title, :status, :thumbnailUrl, :favorite, :lastUpdate, :nextUpdate, :initialized, :viewerFlags, :chapterFlags, :coverLastModified, :dateAdded, :updateStrategy, :calculateInterval, 0, :version);
update:
UPDATE mangas SET
@ -139,7 +156,9 @@ UPDATE mangas SET
cover_last_modified = coalesce(:coverLastModified, cover_last_modified),
date_added = coalesce(:dateAdded, date_added),
update_strategy = coalesce(:updateStrategy, update_strategy),
calculate_interval = coalesce(:calculateInterval, calculate_interval)
calculate_interval = coalesce(:calculateInterval, calculate_interval),
version = coalesce(:version, version),
is_syncing = coalesce(:isSyncing, is_syncing)
WHERE _id = :mangaId;
selectLastInsertedRowId:

View File

@ -2,25 +2,22 @@ CREATE TABLE mangas_categories(
_id INTEGER NOT NULL PRIMARY KEY,
manga_id INTEGER NOT NULL,
category_id INTEGER NOT NULL,
last_modified_at INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY(category_id) REFERENCES categories (_id)
ON DELETE CASCADE,
FOREIGN KEY(manga_id) REFERENCES mangas (_id)
ON DELETE CASCADE
);
CREATE TRIGGER update_last_modified_at_mangas_categories
AFTER UPDATE ON mangas_categories
FOR EACH ROW
CREATE TRIGGER insert_manga_category_update_version AFTER INSERT ON mangas_categories
BEGIN
UPDATE mangas_categories
SET last_modified_at = strftime('%s', 'now')
WHERE _id = new._id;
UPDATE mangas
SET version = version + 1
WHERE _id = new.manga_id AND (SELECT is_syncing FROM mangas WHERE _id = new.manga_id) = 0;
END;
insert:
INSERT INTO mangas_categories(manga_id, category_id, last_modified_at)
VALUES (:mangaId, :categoryId, 0);
INSERT INTO mangas_categories(manga_id, category_id)
VALUES (:mangaId, :categoryId);
deleteMangaCategoryByMangaId:
DELETE FROM mangas_categories

View File

@ -0,0 +1,46 @@
-- Mangas table
ALTER TABLE mangas ADD COLUMN version INTEGER NOT NULL DEFAULT 0;
ALTER TABLE mangas ADD COLUMN is_syncing INTEGER NOT NULL DEFAULT 0;
-- Chapters table
ALTER TABLE chapters ADD COLUMN version INTEGER NOT NULL DEFAULT 0;
ALTER TABLE chapters ADD COLUMN is_syncing INTEGER NOT NULL DEFAULT 0;
-- Mangas triggers
DROP TRIGGER IF EXISTS update_manga_version;
CREATE TRIGGER update_manga_version AFTER UPDATE ON mangas
BEGIN
UPDATE mangas SET version = version + 1
WHERE _id = new._id AND new.is_syncing = 0 AND (
new.url != old.url OR
new.description != old.description OR
new.favorite != old.favorite
);
END;
-- Chapters triggers
DROP TRIGGER IF EXISTS update_chapter_and_manga_version;
CREATE TRIGGER update_chapter_and_manga_version AFTER UPDATE ON chapters
WHEN new.is_syncing = 0 AND (
new.read != old.read OR
new.bookmark != old.bookmark OR
new.last_page_read != old.last_page_read
)
BEGIN
-- Update the chapter version
UPDATE chapters SET version = version + 1
WHERE _id = new._id;
-- Update the manga version
UPDATE mangas SET version = version + 1
WHERE _id = new.manga_id AND (SELECT is_syncing FROM mangas WHERE _id = new.manga_id) = 0;
END;
-- manga_categories table
DROP TRIGGER IF EXISTS insert_manga_category_update_version;
CREATE TRIGGER insert_manga_category_update_version AFTER INSERT ON mangas_categories
BEGIN
UPDATE mangas
SET version = version + 1
WHERE _id = new.manga_id AND (SELECT is_syncing FROM mangas WHERE _id = new.manga_id) = 0;
END;

View File

@ -14,6 +14,7 @@ data class Chapter(
val chapterNumber: Double,
val scanlator: String?,
val lastModifiedAt: Long,
val version: Long,
) {
val isRecognizedNumber: Boolean
get() = chapterNumber >= 0f
@ -43,6 +44,7 @@ data class Chapter(
chapterNumber = -1.0,
scanlator = null,
lastModifiedAt = 0,
version = 1,
)
}
}

View File

@ -13,6 +13,7 @@ data class ChapterUpdate(
val dateUpload: Long? = null,
val chapterNumber: Double? = null,
val scanlator: String? = null,
val version: Long? = null,
)
fun Chapter.toChapterUpdate(): ChapterUpdate {
@ -29,5 +30,6 @@ fun Chapter.toChapterUpdate(): ChapterUpdate {
dateUpload,
chapterNumber,
scanlator,
version,
)
}

View File

@ -29,6 +29,7 @@ data class Manga(
val initialized: Boolean,
val lastModifiedAt: Long,
val favoriteModifiedAt: Long?,
val version: Long,
) : Serializable {
val expectedNextUpdate: Instant?
@ -122,6 +123,7 @@ data class Manga(
initialized = false,
lastModifiedAt = 0L,
favoriteModifiedAt = null,
version = 0L,
)
}
}

View File

@ -23,6 +23,7 @@ data class MangaUpdate(
val thumbnailUrl: String? = null,
val updateStrategy: UpdateStrategy? = null,
val initialized: Boolean? = null,
val version: Long? = null,
)
fun Manga.toMangaUpdate(): MangaUpdate {
@ -47,5 +48,6 @@ fun Manga.toMangaUpdate(): MangaUpdate {
thumbnailUrl = thumbnailUrl,
updateStrategy = updateStrategy,
initialized = initialized,
version = version,
)
}