From b23c100fab1568706a4d94a06fb822cd50fa0df8 Mon Sep 17 00:00:00 2001 From: KaiserBh Date: Wed, 10 Jan 2024 09:29:54 +1100 Subject: [PATCH 1/8] refactor(GoogleDrive): Use gson to encode the syncData. So on a really big data we will run into OOM issue. With this change we implement streaming approach, rather than loading the entire JSON string into memory at once. Signed-off-by: KaiserBh --- .../sync/service/GoogleDriveSyncService.kt | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt index d23e04065..ab3cdb81b 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt @@ -19,6 +19,8 @@ import com.google.api.services.drive.Drive import com.google.api.services.drive.DriveScopes import com.google.api.services.drive.model.File import eu.kanade.tachiyomi.R +import com.google.gson.Gson +import com.google.gson.stream.JsonWriter import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.withContext @@ -128,17 +130,21 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync } override suspend fun pushSyncData(syncData: SyncData) { - val jsonData = json.encodeToString(syncData) val drive = googleDriveService.driveService - ?: throw Exception(context.getString(R.string.google_drive_not_signed_in)) + ?: throw Exception(context.stringResource(MR.strings.google_drive_not_signed_in)) val fileList = getFileList(drive) val byteArrayOutputStream = ByteArrayOutputStream() + withContext(Dispatchers.IO) { val gzipOutputStream = GZIPOutputStream(byteArrayOutputStream) - gzipOutputStream.write(jsonData.toByteArray(Charsets.UTF_8)) - gzipOutputStream.close() + val jsonWriter = JsonWriter(OutputStreamWriter(gzipOutputStream, Charsets.UTF_8)) + val gson = Gson() + + jsonWriter.use { jWriter -> + gson.toJson(syncData, SyncData::class.java, jWriter) + } } val byteArrayContent = ByteArrayContent("application/octet-stream", byteArrayOutputStream.toByteArray()) @@ -154,6 +160,7 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync val fileMetadata = File().apply { name = remoteFileName mimeType = "application/gzip" + parents = listOf("appDataFolder") } val uploadedFile = drive.files().create(fileMetadata, byteArrayContent) .setFields("id") @@ -168,19 +175,20 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync deleteLockFile(drive) } catch (e: Exception) { logcat(LogPriority.ERROR) { "Failed to push or update sync data: ${e.message}" } + throw Exception(context.stringResource(MR.strings.error_uploading_sync_data) + ": ${e.message}") } } - private fun getFileList(drive: Drive): MutableList { + private fun getAppDataFileList(drive: Drive): MutableList { try { - // Search for the existing file by name - val query = "mimeType='application/gzip' and trashed = false and name = '$remoteFileName'" - val fileList = drive.files().list().setQ(query).execute().files - Log.d("GoogleDrive", "File list: $fileList") + // Search for the existing file by name in the appData folder + val query = "mimeType='application/gzip' and name = '$remoteFileName'" + val fileList = drive.files().list().setSpaces("appDataFolder").setQ(query).setFields("files(id, name, createdTime)").execute().files + Log.d("GoogleDrive", "AppData folder file list: $fileList") return fileList } catch (e: Exception) { - Log.e("GoogleDrive", "Error no sync data found: ${e.message}") + Log.e("GoogleDrive", "Error no sync data found in appData folder: ${e.message}") return mutableListOf() } } @@ -318,7 +326,7 @@ class GoogleDriveService(private val context: Context) { NetHttpTransport(), jsonFactory, secrets, - listOf(DriveScopes.DRIVE_FILE), + listOf(DriveScopes.DRIVE_FILE, DriveScopes.DRIVE_APPDATA), ).setAccessType("offline").build() return flow.newAuthorizationUrl() From 5b2bbb11236a6167f4089b4c4f8e643b7cedee28 Mon Sep 17 00:00:00 2001 From: KaiserBh Date: Wed, 10 Jan 2024 09:50:06 +1100 Subject: [PATCH 2/8] refactor(GoogleDrive): Use gson to encode the syncData. Same as before. OOM (Out of Memory) issue. Signed-off-by: KaiserBh --- .../data/sync/service/GoogleDriveSyncService.kt | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt index ab3cdb81b..07e3cbfec 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt @@ -106,14 +106,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync override suspend fun pullSyncData(): SyncData? { val drive = googleDriveService.driveService - // Check if the Google Drive service is initialized if (drive == null) { logcat(LogPriority.ERROR) { "Google Drive service not initialized" } - throw Exception(context.getString(R.string.google_drive_not_signed_in)) + throw Exception(context.stringResource(MR.strings.google_drive_not_signed_in)) } - val fileList = getFileList(drive) - + val fileList = getAppDataFileList(drive) if (fileList.isEmpty()) { return null } @@ -121,12 +119,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync val outputStream = ByteArrayOutputStream() drive.files().get(gdriveFileId).executeMediaAndDownloadTo(outputStream) - val jsonString = withContext(Dispatchers.IO) { - val gzipInputStream = GZIPInputStream(ByteArrayInputStream(outputStream.toByteArray())) - gzipInputStream.bufferedReader(Charsets.UTF_8).use { it.readText() } - } - return json.decodeFromString(SyncData.serializer(), jsonString) + return withContext(Dispatchers.IO) { + val gzipInputStream = GZIPInputStream(ByteArrayInputStream(outputStream.toByteArray())) + val gson = Gson() + gson.fromJson(gzipInputStream.reader(Charsets.UTF_8), SyncData::class.java) + } } override suspend fun pushSyncData(syncData: SyncData) { From 14e01514cb211c99a5c8a7e670746e36c737e9a5 Mon Sep 17 00:00:00 2001 From: KaiserBh Date: Wed, 10 Jan 2024 11:11:01 +1100 Subject: [PATCH 3/8] refactor(GoogleDrive): clean up some stuff. Signed-off-by: KaiserBh --- .../sync/service/GoogleDriveSyncService.kt | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt index 07e3cbfec..1d498d875 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt @@ -18,23 +18,24 @@ import com.google.api.client.json.jackson2.JacksonFactory import com.google.api.services.drive.Drive import com.google.api.services.drive.DriveScopes import com.google.api.services.drive.model.File -import eu.kanade.tachiyomi.R import com.google.gson.Gson import com.google.gson.stream.JsonWriter import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.withContext -import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import logcat.LogPriority -import tachiyomi.core.util.system.logcat +import logcat.logcat +import tachiyomi.core.i18n.stringResource import tachiyomi.domain.sync.SyncPreferences +import tachiyomi.i18n.MR import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.IOException import java.io.InputStreamReader +import java.io.OutputStreamWriter import java.time.Instant import java.util.zip.GZIPInputStream import java.util.zip.GZIPOutputStream @@ -131,14 +132,13 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync val drive = googleDriveService.driveService ?: throw Exception(context.stringResource(MR.strings.google_drive_not_signed_in)) - val fileList = getFileList(drive) - + val fileList = getAppDataFileList(drive) val byteArrayOutputStream = ByteArrayOutputStream() withContext(Dispatchers.IO) { val gzipOutputStream = GZIPOutputStream(byteArrayOutputStream) val jsonWriter = JsonWriter(OutputStreamWriter(gzipOutputStream, Charsets.UTF_8)) - val gson = Gson() + val gson = Gson().newBuilder().serializeNulls().create() jsonWriter.use { jWriter -> gson.toJson(syncData, SyncData::class.java, jWriter) @@ -181,7 +181,13 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync try { // Search for the existing file by name in the appData folder val query = "mimeType='application/gzip' and name = '$remoteFileName'" - val fileList = drive.files().list().setSpaces("appDataFolder").setQ(query).setFields("files(id, name, createdTime)").execute().files + val fileList = drive.files() + .list() + .setSpaces("appDataFolder") + .setQ(query) + .setFields("files(id, name, createdTime)") + .execute() + .files Log.d("GoogleDrive", "AppData folder file list: $fileList") return fileList @@ -212,8 +218,13 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync private fun findLockFile(drive: Drive): MutableList { try { - val query = "mimeType='text/plain' and trashed = false and name = '$lockFileName'" - val fileList = drive.files().list().setQ(query).setFields("files(id, name, createdTime)").execute().files + val query = "mimeType='text/plain' and name = '$lockFileName'" + val fileList = drive.files() + .list() + .setSpaces("appDataFolder") + .setQ(query) + .setFields("files(id, name, createdTime)") + .execute().files Log.d("GoogleDrive", "Lock file search result: $fileList") return fileList } catch (e: Exception) { @@ -250,16 +261,16 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync googleDriveService.refreshToken() return withContext(Dispatchers.IO) { - val query = "mimeType='application/gzip' and trashed = false and name = '$remoteFileName'" - val fileList = drive.files().list().setQ(query).execute().files + val appDataFileList = getAppDataFileList(drive) - if (fileList.isNullOrEmpty()) { - logcat(LogPriority.DEBUG) { "No sync data file found in Google Drive" } + if (appDataFileList.isEmpty()) { + logcat(LogPriority.DEBUG) { "No sync data file found in appData folder of Google Drive" } DeleteSyncDataStatus.NO_FILES } else { - val fileId = fileList[0].id - drive.files().delete(fileId).execute() - logcat(LogPriority.DEBUG) { "Deleted sync data file in Google Drive with file ID: $fileId" } + for (file in appDataFileList) { + drive.files().delete(file.id).execute() + logcat(LogPriority.DEBUG) { "Deleted sync data file in appData folder of Google Drive with file ID: ${file.id}" } + } DeleteSyncDataStatus.SUCCESS } } @@ -349,7 +360,7 @@ class GoogleDriveService(private val context: Context) { .build() if (refreshToken == "") { - throw Exception(context.getString(R.string.google_drive_not_signed_in)) + throw Exception(context.stringResource(MR.strings.google_drive_not_signed_in)) } credential.refreshToken = refreshToken From 0366de2604d744ca3c117d0f5468ea92a9dee94c Mon Sep 17 00:00:00 2001 From: KaiserBh Date: Wed, 10 Jan 2024 11:12:29 +1100 Subject: [PATCH 4/8] refactor(GoogleDrive): use context.stringResource Signed-off-by: KaiserBh --- .../tachiyomi/data/sync/service/GoogleDriveSyncService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt index 1d498d875..b2e8dfc36 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt @@ -247,7 +247,7 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync } } catch (e: Exception) { Log.e("GoogleDrive", "Error deleting lock file: ${e.message}") - throw Exception(context.getString(R.string.error_deleting_google_drive_lock_file)) + throw Exception(context.stringResource(MR.strings.error_deleting_google_drive_lock_file)) } } From 9a53f4c0abcb7017b91a091d5cb641a912fe416b Mon Sep 17 00:00:00 2001 From: KaiserBh Date: Wed, 10 Jan 2024 11:13:43 +1100 Subject: [PATCH 5/8] refactor(GoogleDrive): update strings.xml Signed-off-by: KaiserBh --- i18n/src/commonMain/resources/MR/base/strings.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/i18n/src/commonMain/resources/MR/base/strings.xml b/i18n/src/commonMain/resources/MR/base/strings.xml index 4280f360a..e0ba8bec1 100644 --- a/i18n/src/commonMain/resources/MR/base/strings.xml +++ b/i18n/src/commonMain/resources/MR/base/strings.xml @@ -575,6 +575,7 @@ Logged in to Google Drive Failed to log in to Google Drive: %s Not signed in to Google Drive + Error uploading sync data to Google Drive Error Deleting Google Drive Lock File Purge confirmation Purging sync data will delete all your sync data from Google Drive. Are you sure you want to continue? From e04d191dfbe5c3dd73a63be9e130d3c5024d987a Mon Sep 17 00:00:00 2001 From: KaiserBh Date: Wed, 10 Jan 2024 11:23:27 +1100 Subject: [PATCH 6/8] refactor(GoogleDrive): add more logging, also use appdata folder. Signed-off-by: KaiserBh --- .../sync/service/GoogleDriveSyncService.kt | 68 +++++++++++-------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt index b2e8dfc36..2078699ad 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt @@ -67,40 +67,46 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync private val googleDriveService = GoogleDriveService(context) override suspend fun beforeSync() { - googleDriveService.refreshToken() - val drive = googleDriveService.driveService ?: throw Exception("Google Drive service not initialized") + try { + googleDriveService.refreshToken() + val drive = googleDriveService.driveService ?: throw Exception("Google Drive service not initialized") - var backoff = 2000L // Start with 2 seconds + var backoff = 2000L - while (true) { - val lockFiles = findLockFile(drive) // Fetch the current list of lock files + while (true) { + val lockFiles = findLockFile(drive) + logcat(LogPriority.DEBUG) { "Found ${lockFiles.size} lock file(s)" } - when { - lockFiles.isEmpty() -> { - // No lock file exists, try to create a new one - createLockFile(drive) - } - lockFiles.size == 1 -> { - // Exactly one lock file exists - val lockFile = lockFiles.first() - val createdTime = Instant.parse(lockFile.createdTime.toString()) - val ageMinutes = java.time.Duration.between(createdTime, Instant.now()).toMinutes() - if (ageMinutes <= 3) { - // Lock file is new and presumably held by this process, break the loop to proceed - break - } else { - // Lock file is old, delete and attempt to create a new one - deleteLockFile(drive) + when { + lockFiles.isEmpty() -> { + logcat(LogPriority.DEBUG) { "No lock file found, creating a new one" } createLockFile(drive) } + lockFiles.size == 1 -> { + val lockFile = lockFiles.first() + val createdTime = Instant.parse(lockFile.createdTime.toString()) + val ageMinutes = java.time.Duration.between(createdTime, Instant.now()).toMinutes() + logcat(LogPriority.DEBUG) { "Lock file age: $ageMinutes minutes" } + if (ageMinutes <= 3) { + logcat(LogPriority.DEBUG) { "Lock file is new, proceeding with sync" } + break + } else { + logcat(LogPriority.DEBUG) { "Lock file is old, deleting and creating a new one" } + deleteLockFile(drive) + createLockFile(drive) + } + } + else -> { + logcat(LogPriority.DEBUG) { "Multiple lock files found, applying backoff" } + delay(backoff) // Apply backoff strategy + backoff = (backoff * 2).coerceAtMost(32000L) + logcat(LogPriority.DEBUG) { "Backoff increased to $backoff milliseconds" } + } } - else -> { - // More than one lock file exists, likely due to a race condition - delay(backoff) // Apply backoff strategy - backoff = (backoff * 2).coerceAtMost(32000L) // Max backoff of 32 seconds - } + logcat(LogPriority.DEBUG) { "Loop iteration complete, backoff time: $backoff" } } - // The loop continues until it can confirm that there's exactly one new lock file. + } catch (e: Exception) { + logcat(LogPriority.ERROR) { "Error in GoogleDrive beforeSync: ${e.message}" } } } @@ -199,9 +205,11 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync private fun createLockFile(drive: Drive) { try { - val fileMetadata = File() - fileMetadata.name = lockFileName - fileMetadata.mimeType = "text/plain" + val fileMetadata = File().apply { + name = lockFileName + mimeType = "text/plain" + parents = listOf("appDataFolder") + } // Create an empty content to upload as the lock file val emptyContent = ByteArrayContent.fromString("text/plain", "") From a13e731d1c94e9543524783b1c1c6e536e58e09e Mon Sep 17 00:00:00 2001 From: KaiserBh Date: Wed, 10 Jan 2024 11:30:15 +1100 Subject: [PATCH 7/8] refactor(GoogleDrive): update backoff delay. Signed-off-by: KaiserBh --- .../tachiyomi/data/sync/service/GoogleDriveSyncService.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt index 2078699ad..e468764e1 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt @@ -71,7 +71,7 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync googleDriveService.refreshToken() val drive = googleDriveService.driveService ?: throw Exception("Google Drive service not initialized") - var backoff = 2000L + var backoff = 1000L while (true) { val lockFiles = findLockFile(drive) @@ -99,7 +99,7 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync else -> { logcat(LogPriority.DEBUG) { "Multiple lock files found, applying backoff" } delay(backoff) // Apply backoff strategy - backoff = (backoff * 2).coerceAtMost(32000L) + backoff = (backoff * 2).coerceAtMost(16000L) logcat(LogPriority.DEBUG) { "Backoff increased to $backoff milliseconds" } } } From d1a55ed7fefccebd148990bf3f4e6a433a0b26ae Mon Sep 17 00:00:00 2001 From: KaiserBh Date: Wed, 10 Jan 2024 11:36:43 +1100 Subject: [PATCH 8/8] chore: Ktlint Signed-off-by: KaiserBh --- .../data/sync/service/GoogleDriveSyncService.kt | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt index e468764e1..af757463a 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt @@ -188,12 +188,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync // Search for the existing file by name in the appData folder val query = "mimeType='application/gzip' and name = '$remoteFileName'" val fileList = drive.files() - .list() - .setSpaces("appDataFolder") - .setQ(query) - .setFields("files(id, name, createdTime)") - .execute() - .files + .list() + .setSpaces("appDataFolder") + .setQ(query) + .setFields("files(id, name, createdTime)") + .execute() + .files Log.d("GoogleDrive", "AppData folder file list: $fileList") return fileList @@ -277,7 +277,9 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync } else { for (file in appDataFileList) { drive.files().delete(file.id).execute() - logcat(LogPriority.DEBUG) { "Deleted sync data file in appData folder of Google Drive with file ID: ${file.id}" } + logcat( + LogPriority.DEBUG, + ) { "Deleted sync data file in appData folder of Google Drive with file ID: ${file.id}" } } DeleteSyncDataStatus.SUCCESS }