mirror/mihon
1
0
Fork 0
mirror of https://github.com/mihonapp/mihon.git synced 2024-11-15 15:02:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'sync-part-final' into feat/add-sync-triggers-experimental

Browse Source
This commit is contained in:
KaiserBh 2024-01-10 11:36:58 +11:00
commit 8f6fa1f500
No known key found for this signature in database
GPG Key ID: 14D73B142042BBA9
2 changed files with 94 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt
View File

@ -18,21 +18,24 @@ import com.google.api.client.json.jackson2.JacksonFactory
import com.google.api.services.drive.Drive import com.google.api.services.drive.Drive
import com.google.api.services.drive.DriveScopes import com.google.api.services.drive.DriveScopes
import com.google.api.services.drive.model.File import com.google.api.services.drive.model.File
import eu.kanade.tachiyomi.R import com.google.gson.Gson
import com.google.gson.stream.JsonWriter
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json import kotlinx.serialization.json.Json
import logcat.LogPriority import logcat.LogPriority
import tachiyomi.core.util.system.logcat import logcat.logcat
import tachiyomi.core.i18n.stringResource
import tachiyomi.domain.sync.SyncPreferences import tachiyomi.domain.sync.SyncPreferences
import tachiyomi.i18n.MR
import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get import uy.kohesive.injekt.api.get
import java.io.ByteArrayInputStream import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import java.io.IOException import java.io.IOException
import java.io.InputStreamReader import java.io.InputStreamReader
import java.io.OutputStreamWriter
import java.time.Instant import java.time.Instant
import java.util.zip.GZIPInputStream import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream import java.util.zip.GZIPOutputStream
@ -64,54 +67,58 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
private val googleDriveService = GoogleDriveService(context) private val googleDriveService = GoogleDriveService(context)
override suspend fun beforeSync() { override suspend fun beforeSync() {
try {
googleDriveService.refreshToken() googleDriveService.refreshToken()
val drive = googleDriveService.driveService ?: throw Exception("Google Drive service not initialized") val drive = googleDriveService.driveService ?: throw Exception("Google Drive service not initialized")
var backoff = 2000L // Start with 2 seconds var backoff = 1000L
while (true) { while (true) {
val lockFiles = findLockFile(drive) // Fetch the current list of lock files val lockFiles = findLockFile(drive)
logcat(LogPriority.DEBUG) { "Found ${lockFiles.size} lock file(s)" }
when { when {
lockFiles.isEmpty() -> { lockFiles.isEmpty() -> {
// No lock file exists, try to create a new one logcat(LogPriority.DEBUG) { "No lock file found, creating a new one" }
createLockFile(drive) createLockFile(drive)
} }
lockFiles.size == 1 -> { lockFiles.size == 1 -> {
// Exactly one lock file exists
val lockFile = lockFiles.first() val lockFile = lockFiles.first()
val createdTime = Instant.parse(lockFile.createdTime.toString()) val createdTime = Instant.parse(lockFile.createdTime.toString())
val ageMinutes = java.time.Duration.between(createdTime, Instant.now()).toMinutes() val ageMinutes = java.time.Duration.between(createdTime, Instant.now()).toMinutes()
logcat(LogPriority.DEBUG) { "Lock file age: $ageMinutes minutes" }
if (ageMinutes <= 3) { if (ageMinutes <= 3) {
// Lock file is new and presumably held by this process, break the loop to proceed logcat(LogPriority.DEBUG) { "Lock file is new, proceeding with sync" }
break break
} else { } else {
// Lock file is old, delete and attempt to create a new one logcat(LogPriority.DEBUG) { "Lock file is old, deleting and creating a new one" }
deleteLockFile(drive) deleteLockFile(drive)
createLockFile(drive) createLockFile(drive)
} }
} }
else -> { else -> {
// More than one lock file exists, likely due to a race condition logcat(LogPriority.DEBUG) { "Multiple lock files found, applying backoff" }
delay(backoff) // Apply backoff strategy delay(backoff) // Apply backoff strategy
backoff = (backoff * 2).coerceAtMost(32000L) // Max backoff of 32 seconds backoff = (backoff * 2).coerceAtMost(16000L)
logcat(LogPriority.DEBUG) { "Backoff increased to $backoff milliseconds" }
} }
} }
// The loop continues until it can confirm that there's exactly one new lock file. logcat(LogPriority.DEBUG) { "Loop iteration complete, backoff time: $backoff" }
}
} catch (e: Exception) {
logcat(LogPriority.ERROR) { "Error in GoogleDrive beforeSync: ${e.message}" }
} }
} }
override suspend fun pullSyncData(): SyncData? { override suspend fun pullSyncData(): SyncData? {
val drive = googleDriveService.driveService val drive = googleDriveService.driveService
// Check if the Google Drive service is initialized
if (drive == null) { if (drive == null) {
logcat(LogPriority.ERROR) { "Google Drive service not initialized" } logcat(LogPriority.ERROR) { "Google Drive service not initialized" }
throw Exception(context.getString(R.string.google_drive_not_signed_in)) throw Exception(context.stringResource(MR.strings.google_drive_not_signed_in))
} }
val fileList = getFileList(drive) val fileList = getAppDataFileList(drive)
if (fileList.isEmpty()) { if (fileList.isEmpty()) {
return null return null
} }
@ -119,26 +126,29 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
val outputStream = ByteArrayOutputStream() val outputStream = ByteArrayOutputStream()
drive.files().get(gdriveFileId).executeMediaAndDownloadTo(outputStream) drive.files().get(gdriveFileId).executeMediaAndDownloadTo(outputStream)
val jsonString = withContext(Dispatchers.IO) {
val gzipInputStream = GZIPInputStream(ByteArrayInputStream(outputStream.toByteArray()))
gzipInputStream.bufferedReader(Charsets.UTF_8).use { it.readText() }
}
return json.decodeFromString(SyncData.serializer(), jsonString) return withContext(Dispatchers.IO) {
val gzipInputStream = GZIPInputStream(ByteArrayInputStream(outputStream.toByteArray()))
val gson = Gson()
gson.fromJson(gzipInputStream.reader(Charsets.UTF_8), SyncData::class.java)
}
} }
override suspend fun pushSyncData(syncData: SyncData) { override suspend fun pushSyncData(syncData: SyncData) {
val jsonData = json.encodeToString(syncData)
val drive = googleDriveService.driveService val drive = googleDriveService.driveService
?: throw Exception(context.getString(R.string.google_drive_not_signed_in)) ?: throw Exception(context.stringResource(MR.strings.google_drive_not_signed_in))
val fileList = getFileList(drive)
val fileList = getAppDataFileList(drive)
val byteArrayOutputStream = ByteArrayOutputStream() val byteArrayOutputStream = ByteArrayOutputStream()
withContext(Dispatchers.IO) { withContext(Dispatchers.IO) {
val gzipOutputStream = GZIPOutputStream(byteArrayOutputStream) val gzipOutputStream = GZIPOutputStream(byteArrayOutputStream)
gzipOutputStream.write(jsonData.toByteArray(Charsets.UTF_8)) val jsonWriter = JsonWriter(OutputStreamWriter(gzipOutputStream, Charsets.UTF_8))
gzipOutputStream.close() val gson = Gson().newBuilder().serializeNulls().create()
jsonWriter.use { jWriter ->
gson.toJson(syncData, SyncData::class.java, jWriter)
}
} }
val byteArrayContent = ByteArrayContent("application/octet-stream", byteArrayOutputStream.toByteArray()) val byteArrayContent = ByteArrayContent("application/octet-stream", byteArrayOutputStream.toByteArray())
@ -154,6 +164,7 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
val fileMetadata = File().apply { val fileMetadata = File().apply {
name = remoteFileName name = remoteFileName
mimeType = "application/gzip" mimeType = "application/gzip"
parents = listOf("appDataFolder")
} }
val uploadedFile = drive.files().create(fileMetadata, byteArrayContent) val uploadedFile = drive.files().create(fileMetadata, byteArrayContent)
.setFields("id") .setFields("id")
@ -168,28 +179,37 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
deleteLockFile(drive) deleteLockFile(drive)
} catch (e: Exception) { } catch (e: Exception) {
logcat(LogPriority.ERROR) { "Failed to push or update sync data: ${e.message}" } logcat(LogPriority.ERROR) { "Failed to push or update sync data: ${e.message}" }
throw Exception(context.stringResource(MR.strings.error_uploading_sync_data) + ": ${e.message}")
} }
} }
private fun getFileList(drive: Drive): MutableList<File> { private fun getAppDataFileList(drive: Drive): MutableList<File> {
try { try {
// Search for the existing file by name // Search for the existing file by name in the appData folder
val query = "mimeType='application/gzip' and trashed = false and name = '$remoteFileName'" val query = "mimeType='application/gzip' and name = '$remoteFileName'"
val fileList = drive.files().list().setQ(query).execute().files val fileList = drive.files()
Log.d("GoogleDrive", "File list: $fileList") .list()
.setSpaces("appDataFolder")
.setQ(query)
.setFields("files(id, name, createdTime)")
.execute()
.files
Log.d("GoogleDrive", "AppData folder file list: $fileList")
return fileList return fileList
} catch (e: Exception) { } catch (e: Exception) {
Log.e("GoogleDrive", "Error no sync data found: ${e.message}") Log.e("GoogleDrive", "Error no sync data found in appData folder: ${e.message}")
return mutableListOf() return mutableListOf()
} }
} }
private fun createLockFile(drive: Drive) { private fun createLockFile(drive: Drive) {
try { try {
val fileMetadata = File() val fileMetadata = File().apply {
fileMetadata.name = lockFileName name = lockFileName
fileMetadata.mimeType = "text/plain" mimeType = "text/plain"
parents = listOf("appDataFolder")
}
// Create an empty content to upload as the lock file // Create an empty content to upload as the lock file
val emptyContent = ByteArrayContent.fromString("text/plain", "") val emptyContent = ByteArrayContent.fromString("text/plain", "")
@ -206,8 +226,13 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
private fun findLockFile(drive: Drive): MutableList<File> { private fun findLockFile(drive: Drive): MutableList<File> {
try { try {
val query = "mimeType='text/plain' and trashed = false and name = '$lockFileName'" val query = "mimeType='text/plain' and name = '$lockFileName'"
val fileList = drive.files().list().setQ(query).setFields("files(id, name, createdTime)").execute().files val fileList = drive.files()
.list()
.setSpaces("appDataFolder")
.setQ(query)
.setFields("files(id, name, createdTime)")
.execute().files
Log.d("GoogleDrive", "Lock file search result: $fileList") Log.d("GoogleDrive", "Lock file search result: $fileList")
return fileList return fileList
} catch (e: Exception) { } catch (e: Exception) {
@ -230,7 +255,7 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
} }
} catch (e: Exception) { } catch (e: Exception) {
Log.e("GoogleDrive", "Error deleting lock file: ${e.message}") Log.e("GoogleDrive", "Error deleting lock file: ${e.message}")
throw Exception(context.getString(R.string.error_deleting_google_drive_lock_file)) throw Exception(context.stringResource(MR.strings.error_deleting_google_drive_lock_file))
} }
} }
@ -244,16 +269,18 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
googleDriveService.refreshToken() googleDriveService.refreshToken()
return withContext(Dispatchers.IO) { return withContext(Dispatchers.IO) {
val query = "mimeType='application/gzip' and trashed = false and name = '$remoteFileName'" val appDataFileList = getAppDataFileList(drive)
val fileList = drive.files().list().setQ(query).execute().files
if (fileList.isNullOrEmpty()) { if (appDataFileList.isEmpty()) {
logcat(LogPriority.DEBUG) { "No sync data file found in Google Drive" } logcat(LogPriority.DEBUG) { "No sync data file found in appData folder of Google Drive" }
DeleteSyncDataStatus.NO_FILES DeleteSyncDataStatus.NO_FILES
} else { } else {
val fileId = fileList[0].id for (file in appDataFileList) {
drive.files().delete(fileId).execute() drive.files().delete(file.id).execute()
logcat(LogPriority.DEBUG) { "Deleted sync data file in Google Drive with file ID: $fileId" } logcat(
LogPriority.DEBUG,
) { "Deleted sync data file in appData folder of Google Drive with file ID: ${file.id}" }
}
DeleteSyncDataStatus.SUCCESS DeleteSyncDataStatus.SUCCESS
} }
} }
@ -318,7 +345,7 @@ class GoogleDriveService(private val context: Context) {
NetHttpTransport(), NetHttpTransport(),
jsonFactory, jsonFactory,
secrets, secrets,
listOf(DriveScopes.DRIVE_FILE), listOf(DriveScopes.DRIVE_FILE, DriveScopes.DRIVE_APPDATA),
).setAccessType("offline").build() ).setAccessType("offline").build()
return flow.newAuthorizationUrl() return flow.newAuthorizationUrl()
@ -343,7 +370,7 @@ class GoogleDriveService(private val context: Context) {
.build() .build()
if (refreshToken == "") { if (refreshToken == "") {
throw Exception(context.getString(R.string.google_drive_not_signed_in)) throw Exception(context.stringResource(MR.strings.google_drive_not_signed_in))
} }
credential.refreshToken = refreshToken credential.refreshToken = refreshToken

1
i18n/src/commonMain/resources/MR/base/strings.xml
View File

@ -575,6 +575,7 @@
<string name="google_drive_login_success">Logged in to Google Drive</string> <string name="google_drive_login_success">Logged in to Google Drive</string>
<string name="google_drive_login_failed">Failed to log in to Google Drive: %s</string> <string name="google_drive_login_failed">Failed to log in to Google Drive: %s</string>
<string name="google_drive_not_signed_in">Not signed in to Google Drive</string> <string name="google_drive_not_signed_in">Not signed in to Google Drive</string>
<string name="error_uploading_sync_data">Error uploading sync data to Google Drive</string>
<string name="error_deleting_google_drive_lock_file">Error Deleting Google Drive Lock File</string> <string name="error_deleting_google_drive_lock_file">Error Deleting Google Drive Lock File</string>
<string name="pref_purge_confirmation_title">Purge confirmation</string> <string name="pref_purge_confirmation_title">Purge confirmation</string>
<string name="pref_purge_confirmation_message">Purging sync data will delete all your sync data from Google Drive. Are you sure you want to continue?</string> <string name="pref_purge_confirmation_message">Purging sync data will delete all your sync data from Google Drive. Are you sure you want to continue?</string>