Initial implementation of favorites syncing

General code cleanup
Fix some cases of duplicate galleries (not completely fixed)

app/src/main/java/exh/FavoritesSyncHelper.kt

@@ -1,137 +0,0 @@
-package exh
-
-import android.app.Activity
-import android.support.v7.app.AlertDialog
-import com.afollestad.materialdialogs.MaterialDialog
-import eu.kanade.tachiyomi.data.database.DatabaseHelper
-import eu.kanade.tachiyomi.data.database.models.Category
-import eu.kanade.tachiyomi.data.database.models.Manga
-import eu.kanade.tachiyomi.data.database.models.MangaCategory
-import eu.kanade.tachiyomi.data.preference.PreferencesHelper
-import eu.kanade.tachiyomi.data.preference.getOrDefault
-import eu.kanade.tachiyomi.source.SourceManager
-import eu.kanade.tachiyomi.source.online.all.EHentai
-import eu.kanade.tachiyomi.util.syncChaptersWithSource
-import timber.log.Timber
-import uy.kohesive.injekt.injectLazy
-import kotlin.concurrent.thread
-
-class FavoritesSyncHelper(val activity: Activity) {
-
-    val db: DatabaseHelper by injectLazy()
-
-    val sourceManager: SourceManager by injectLazy()
-
-    val prefs: PreferencesHelper by injectLazy()
-
-    fun guiSyncFavorites(onComplete: () -> Unit) {
-        //ExHentai must be enabled/user must be logged in
-        if (!prefs.enableExhentai().getOrDefault()) {
-            AlertDialog.Builder(activity).setTitle("Error")
-                    .setMessage("You are not logged in! Please log in and try again!")
-                    .setPositiveButton("Ok") { dialog, _ -> dialog.dismiss() }.show()
-            return
-        }
-        val dialog = MaterialDialog.Builder(activity)
-                .progress(true, 0)
-                .title("Downloading favorites")
-                .content("Please wait...")
-                .cancelable(false)
-                .show()
-        thread {
-            var error = false
-            try {
-                syncFavorites()
-            } catch (e: Exception) {
-                error = true
-                Timber.e(e, "Could not sync favorites!")
-            }
-
-            dialog.dismiss()
-
-            activity.runOnUiThread {
-                if (error)
-                    MaterialDialog.Builder(activity)
-                            .title("Error")
-                            .content("There was an error downloading your favorites, please try again later!")
-                            .positiveText("Ok")
-                            .show()
-                onComplete()
-            }
-        }
-    }
-
-    fun syncFavorites() {
-        val onlineSources = sourceManager.getOnlineSources()
-        var ehSource: EHentai? = null
-        var exSource: EHentai? = null
-        onlineSources.forEach {
-            if(it.id == EH_SOURCE_ID)
-                ehSource = it as EHentai
-            else if(it.id == EXH_SOURCE_ID)
-                exSource = it as EHentai
-        }
-
-        (exSource ?: ehSource)?.let { source ->
-            val favResponse = source.fetchFavorites()
-            val ourCategories = db.getCategories().executeAsBlocking().toMutableList()
-            val ourMangas = db.getMangas().executeAsBlocking().filter {
-                it.source == EH_SOURCE_ID || it.source == EXH_SOURCE_ID
-            }.toMutableList()
-            //Add required categories (categories do not sync upwards)
-            favResponse.second.filter { theirCategory ->
-                ourCategories.find {
-                    it.name.endsWith(theirCategory)
-                } == null
-            }.map {
-                Category.create(it)
-            }.let {
-                db.inTransaction {
-                    //Insert new categories
-                    db.insertCategories(it).executeAsBlocking().results().entries.filter {
-                        it.value.wasInserted()
-                    }.forEach { it.key.id = it.value.insertedId()!!.toInt() }
-
-                    val categoryMap = (it + ourCategories).associateBy { it.name }
-
-                    //Insert new mangas
-                    val mangaToInsert = mutableListOf<Manga>()
-                    favResponse.first.map {
-                        val category = categoryMap[it.fav]!!
-                        var manga = it.manga
-                        val alreadyHaveManga = ourMangas.find {
-                            it.url == manga.url
-                        }?.apply {
-                            manga = this
-                        } != null
-                        if (!alreadyHaveManga) {
-                            ourMangas.add(manga)
-                            mangaToInsert.add(manga)
-                        }
-                        manga.favorite = true
-                        Pair(manga, category)
-                    }.apply {
-                        //Insert mangas
-                        db.insertMangas(mangaToInsert).executeAsBlocking().results().entries.filter {
-                            it.value.wasInserted()
-                        }.forEach { manga ->
-                            manga.key.id = manga.value.insertedId()
-                            try {
-                                source.fetchChapterList(manga.key).map {
-                                    syncChaptersWithSource(db, it, manga.key, source)
-                                }.toBlocking().first()
-                            } catch (e: Exception) {
-                                Timber.w(e, "Failed to update chapters for gallery: ${manga.key.title}!")
-                            }
-                        }
-
-                        //Set categories
-                        val categories = map { MangaCategory.create(it.first, it.second) }
-                        val mangas = map { it.first }
-                        db.setMangaCategories(categories, mangas)
-                    }
-                }
-            }
-        }
-    }
-}

app/src/main/java/exh/FavoritesSyncManager.java

@@ -1,192 +0,0 @@
-package exh;
-
-import android.app.ProgressDialog;
-import android.content.Context;
-import android.content.DialogInterface;
-import android.os.Handler;
-import android.os.Looper;
-import android.support.v7.app.AlertDialog;
-
-import com.pushtorefresh.storio.sqlite.operations.put.PutResult;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import eu.kanade.tachiyomi.data.database.DatabaseHelper;
-import eu.kanade.tachiyomi.data.database.models.Category;
-import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.database.models.MangaCategory;
-import eu.kanade.tachiyomi.source.online.all.EHentai;
-import kotlin.Pair;
-//import eu.kanade.tachiyomi.data.source.online.english.EHentai;
-
-public class FavoritesSyncManager {
-    /*Context context;
-    DatabaseHelper db;
-
-    public FavoritesSyncManager(Context context, DatabaseHelper db) {
-        this.context = context;
-        this.db = db;
-    }
-
-    public void guiSyncFavorites(final Runnable onComplete) {
-        if(!DialogLogin.isLoggedIn(context, false)) {
-            new AlertDialog.Builder(context).setTitle("Error")
-                    .setMessage("You are not logged in! Please log in and try again!")
-                    .setPositiveButton("Ok", new DialogInterface.OnClickListener() {
-                        @Override
-                        public void onClick(DialogInterface dialog, int which) {
-                            dialog.dismiss();
-                        }
-                    }).show();
-            return;
-        }
-        final ProgressDialog dialog = ProgressDialog.show(context, "Downloading Favorites", "Please wait...", true, false);
-        new Thread(new Runnable() {
-            @Override
-            public void run() {
-                Handler mainLooper = new Handler(Looper.getMainLooper());
-                try {
-                    syncFavorites();
-                } catch (Exception e) {
-                    mainLooper.post(new Runnable() {
-                        @Override
-                        public void run() {
-                            new AlertDialog.Builder(context)
-                                    .setTitle("Error")
-                                    .setMessage("There was an error downloading your favorites, please try again later!")
-                                    .setPositiveButton("Ok", new DialogInterface.OnClickListener() {
-                                        @Override
-                                        public void onClick(DialogInterface dialog, int which) {
-                                            dialog.dismiss();
-                                        }
-                                    }).show();
-                        }
-                    });
-                    e.printStackTrace();
-                }
-                dialog.dismiss();
-                mainLooper.post(onComplete);
-            }
-        }).start();
-    }*/
-/*
-    public void syncFavorites() throws IOException {
-        Pair favResponse = EHentai.fetchFavorites(context);
-        Map<String, List<Manga>> favorites = favResponse.favs;
-        List<Category> ourCategories = new ArrayList<>(db.getCategories().executeAsBlocking());
-        List<Manga> ourMangas = new ArrayList<>(db.getMangas().executeAsBlocking());
-        //Add required categories (categories do not sync upwards)
-        List<Category> categoriesToInsert = new ArrayList<>();
-        for (String theirCategory : favorites.keySet()) {
-            boolean haveCategory = false;
-            for (Category category : ourCategories) {
-                if (category.getName().endsWith(theirCategory)) {
-                    haveCategory = true;
-                }
-            }
-            if (!haveCategory) {
-                Category category = Category.Companion.create(theirCategory);
-                ourCategories.add(category);
-                categoriesToInsert.add(category);
-            }
-        }
-        if (!categoriesToInsert.isEmpty()) {
-            for(Map.Entry<Category, PutResult> result : db.insertCategories(categoriesToInsert).executeAsBlocking().results().entrySet()) {
-                if(result.getValue().wasInserted()) {
-                    result.getKey().setId(result.getValue().insertedId().intValue());
-                }
-            }
-        }
-        //Build category map
-        Map<String, Category> categoryMap = new HashMap<>();
-        for (Category category : ourCategories) {
-            categoryMap.put(category.getName(), category);
-        }
-        //Insert new mangas
-        List<Manga> mangaToInsert = new ArrayList<>();
-        Map<Manga, Category> mangaToSetCategories = new HashMap<>();
-        for (Map.Entry<String, List<Manga>> entry : favorites.entrySet()) {
-            Category category = categoryMap.get(entry.getKey());
-            for (Manga manga : entry.getValue()) {
-                boolean alreadyHaveManga = false;
-                for (Manga ourManga : ourMangas) {
-                    if (ourManga.getUrl().equals(manga.getUrl())) {
-                        alreadyHaveManga = true;
-                        manga = ourManga;
-                        break;
-                    }
-                }
-                if (!alreadyHaveManga) {
-                    ourMangas.add(manga);
-                    mangaToInsert.add(manga);
-                }
-                mangaToSetCategories.put(manga, category);
-                manga.setFavorite(true);
-            }
-        }
-        for (Map.Entry<Manga, PutResult> results : db.insertMangas(mangaToInsert).executeAsBlocking().results().entrySet()) {
-            if(results.getValue().wasInserted()) {
-                results.getKey().setId(results.getValue().insertedId());
-            }
-        }
-        for(Map.Entry<Manga, Category> entry : mangaToSetCategories.entrySet()) {
-            db.setMangaCategories(Collections.singletonList(MangaCategory.Companion.create(entry.getKey(), entry.getValue())),
-                    Collections.singletonList(entry.getKey()));
-        }*/
-        //Determines what
-        /*Map<Integer, List<Manga>> toUpload = new HashMap<>();
-        for (Manga manga : ourMangas) {
-            if(manga.getFavorite()) {
-                boolean remoteHasManga = false;
-                for (List<Manga> remoteMangas : favorites.values()) {
-                    for (Manga remoteManga : remoteMangas) {
-                        if (remoteManga.getUrl().equals(manga.getUrl())) {
-                            remoteHasManga = true;
-                            break;
-                        }
-                    }
-                }
-                if (!remoteHasManga) {
-                    List<Category> mangaCategories = db.getCategoriesForManga(manga).executeAsBlocking();
-                    for (Category category : mangaCategories) {
-                        int categoryIndex = favResponse.favCategories.indexOf(category.getName());
-                        if (categoryIndex >= 0) {
-                            List<Manga> uploadMangas = toUpload.get(categoryIndex);
-                            if (uploadMangas == null) {
-                                uploadMangas = new ArrayList<>();
-                                toUpload.put(categoryIndex, uploadMangas);
-                            }
-                            uploadMangas.add(manga);
-                        }
-                    }
-                }
-            }
-        }*/
-        /********** NON-FUNCTIONAL, modifygids[] CANNOT ADD NEW FAVORITES! (or as of my testing it can't, maybe I'll do more testing)**/
-        /*PreferencesHelper helper = new PreferencesHelper(context);
-        for(Map.Entry<Integer, List<Manga>> entry : toUpload.entrySet()) {
-            FormBody.Builder formBody = new FormBody.Builder()
-                    .add("ddact", "fav" + entry.getKey());
-            for(Manga manga : entry.getValue()) {
-                List<String> splitUrl = new ArrayList<>(Arrays.asList(manga.getUrl().split("/")));
-                splitUrl.removeAll(Collections.singleton(""));
-                if(splitUrl.size() < 2) {
-                    continue;
-                }
-                formBody.add("modifygids[]", splitUrl.get(1).trim());
-            }
-            formBody.add("apply", "Apply");
-            Request request = RequestsKt.POST(EHentai.buildFavoritesBase(context, helper.getPrefs()).favoritesBase,
-                    EHentai.getHeadersBuilder(helper).build(),
-                    formBody.build(),
-                    RequestsKt.getDEFAULT_CACHE_CONTROL());
-            Response response = NetworkManager.getInstance().getClient().newCall(request).execute();
-            Util.d("EHentai", response.body().string());
-        }*/
-//    }
-}

app/src/main/java/exh/GalleryAdder.kt

@@ -10,8 +10,7 @@ import eu.kanade.tachiyomi.data.database.models.Manga
 import eu.kanade.tachiyomi.network.NetworkHelper
 import eu.kanade.tachiyomi.source.SourceManager
 import eu.kanade.tachiyomi.util.syncChaptersWithSource
-import exh.metadata.models.*
-import exh.util.defRealm
+import exh.metadata.models.ExGalleryMetadata
 import okhttp3.MediaType
 import okhttp3.Request
 import okhttp3.RequestBody
@@ -131,7 +130,7 @@ class GalleryAdder {
                     ?: return GalleryAddEvent.Fail.Error(url, "Could not find EH source!")
 
             val cleanedUrl = when(source) {
-                EH_SOURCE_ID, EXH_SOURCE_ID -> getUrlWithoutDomain(realUrl)
+                EH_SOURCE_ID, EXH_SOURCE_ID -> ExGalleryMetadata.normalizeUrl(getUrlWithoutDomain(realUrl))
                 NHENTAI_SOURCE_ID -> realUrl //nhentai uses URLs directly (oops, my bad when implementing this source)
                 PERV_EDEN_EN_SOURCE_ID,
                 PERV_EDEN_IT_SOURCE_ID -> getUrlWithoutDomain(realUrl)
@@ -152,19 +151,6 @@ class GalleryAdder {
             manga.copyFrom(newManga)
             manga.title = newManga.title //Forcibly copy title as copyFrom does not copy title
 
-            //Apply metadata
-            defRealm { realm ->
-                when (source) {
-                    EH_SOURCE_ID, EXH_SOURCE_ID -> ExGalleryMetadata.UrlQuery(realUrl, isExSource(source))
-                    NHENTAI_SOURCE_ID -> NHentaiMetadata.UrlQuery(realUrl)
-                    PERV_EDEN_EN_SOURCE_ID,
-                    PERV_EDEN_IT_SOURCE_ID -> PervEdenGalleryMetadata.UrlQuery(realUrl, PervEdenLang.source(source))
-                    HENTAI_CAFE_SOURCE_ID -> HentaiCafeMetadata.UrlQuery(realUrl)
-                    TSUMINO_SOURCE_ID -> TsuminoMetadata.UrlQuery(realUrl)
-                    else -> return GalleryAddEvent.Fail.UnknownType(url)
-                }.query(realm).findFirst()
-            }
-
             if (fav) manga.favorite = true
 
             db.insertManga(manga).executeAsBlocking().insertedId()?.let {

app/src/main/java/exh/favorites/FavoriteEntry.kt

@@ -0,0 +1,23 @@
+package exh.favorites
+
+import exh.metadata.models.ExGalleryMetadata
+import io.realm.RealmObject
+import io.realm.annotations.Index
+import io.realm.annotations.PrimaryKey
+import io.realm.annotations.RealmClass
+import java.util.*
+
+@RealmClass
+open class FavoriteEntry : RealmObject() {
+    @PrimaryKey var id: String = UUID.randomUUID().toString()
+
+    var title: String? = null
+
+    @Index lateinit var gid: String
+
+    @Index lateinit var token: String
+
+    @Index var category: Int = -1
+
+    fun getUrl() = ExGalleryMetadata.normalizeUrl(gid, token)
+}

app/src/main/java/exh/favorites/FavoritesSyncHelper.kt

@@ -0,0 +1,274 @@
+package exh.favorites
+
+import android.content.Context
+import eu.kanade.tachiyomi.data.database.DatabaseHelper
+import eu.kanade.tachiyomi.data.database.models.Category
+import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.data.database.models.MangaCategory
+import eu.kanade.tachiyomi.data.preference.PreferencesHelper
+import eu.kanade.tachiyomi.data.preference.getOrDefault
+import eu.kanade.tachiyomi.source.SourceManager
+import eu.kanade.tachiyomi.source.online.all.EHentai
+import exh.EH_METADATA_SOURCE_ID
+import exh.EXH_SOURCE_ID
+import exh.GalleryAddEvent
+import exh.GalleryAdder
+import okhttp3.FormBody
+import okhttp3.Request
+import rx.subjects.BehaviorSubject
+import timber.log.Timber
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+import uy.kohesive.injekt.injectLazy
+import kotlin.concurrent.thread
+
+class FavoritesSyncHelper(context: Context) {
+    private val db: DatabaseHelper by injectLazy()
+
+    private val prefs: PreferencesHelper by injectLazy()
+
+    private val exh by lazy {
+        Injekt.get<SourceManager>().get(EXH_SOURCE_ID) as? EHentai
+                ?: EHentai(0, true, context)
+    }
+
+    private val storage = LocalFavoritesStorage()
+
+    private val galleryAdder = GalleryAdder()
+
+    val status = BehaviorSubject.create<FavoritesSyncStatus>(FavoritesSyncStatus.Idle())
+
+    @Synchronized
+    fun runSync() {
+        if(status.value !is FavoritesSyncStatus.Idle) {
+            return
+        }
+
+        status.onNext(FavoritesSyncStatus.Initializing())
+
+        thread { beginSync() }
+    }
+
+    private fun beginSync() {
+        //Check if logged in
+        if(!prefs.enableExhentai().getOrDefault()) {
+            status.onNext(FavoritesSyncStatus.Error("Please log in!"))
+            return
+        }
+
+        //Download remote favorites
+        val favorites = try {
+            status.onNext(FavoritesSyncStatus.Processing("Downloading favorites from remote server"))
+            exh.fetchFavorites()
+        } catch(e: Exception) {
+            status.onNext(FavoritesSyncStatus.Error("Failed to fetch favorites from remote server!"))
+            Timber.e(e, "Could not fetch favorites!")
+            return
+        }
+
+        val errors = mutableListOf<String>()
+
+        try {
+            db.inTransaction {
+                val remoteChanges = storage.getChangedRemoteEntries(favorites.first)
+                val localChanges = storage.getChangedDbEntries()
+
+                //Apply remote categories
+                status.onNext(FavoritesSyncStatus.Processing("Updating category names"))
+                applyRemoteCategories(favorites.second)
+
+                //Apply ChangeSets
+                applyChangeSetToLocal(remoteChanges, errors)
+                applyChangeSetToRemote(localChanges, errors)
+
+                status.onNext(FavoritesSyncStatus.Processing("Cleaning up"))
+                storage.snapshotEntries()
+            }
+        } catch(e: IgnoredException) {
+            //Do not display error as this error has already been reported
+            Timber.w(e, "Ignoring exception!")
+            return
+        } catch (e: Exception) {
+            status.onNext(FavoritesSyncStatus.Error("Unknown error: ${e.message}"))
+            Timber.e(e, "Sync error!")
+            return
+        }
+
+        status.onNext(FavoritesSyncStatus.Complete(errors))
+    }
+
+    private fun applyRemoteCategories(categories: List<String>) {
+        val localCategories = db.getCategories().executeAsBlocking()
+
+        val newLocalCategories = localCategories.toMutableList()
+
+        var changed = false
+
+        categories.forEachIndexed { index, remote ->
+            val local = localCategories.getOrElse(index) {
+                changed = true
+
+                Category.create(remote).apply {
+                    order = index
+
+                    //Going through categories list from front to back
+                    //If category does not exist, list size <= category index
+                    //Thus, we can just add it here and not worry about indexing
+                    newLocalCategories += this
+                }
+            }
+
+            if(local.name != remote) {
+                changed = true
+
+                local.name = remote
+            }
+        }
+
+        //Ensure consistent ordering
+        newLocalCategories.forEachIndexed { index, category ->
+            if(category.order != index) {
+                changed = true
+
+                category.order = index
+            }
+        }
+
+        //Only insert categories if changed
+        if(changed)
+            db.insertCategories(newLocalCategories).executeAsBlocking()
+    }
+
+    private fun addGalleryRemote(gallery: FavoriteEntry, errors: MutableList<String>) {
+        val url = "${exh.baseUrl}/gallerypopups.php?gid=${gallery.gid}&t=${gallery.token}&act=addfav"
+
+        val request = Request.Builder()
+                .url(url)
+                .post(FormBody.Builder()
+                        .add("favcat", gallery.category.toString())
+                        .add("favnote", "")
+                        .add("apply", "Add to Favorites")
+                        .add("update", "1")
+                        .build())
+                .build()
+
+        if(!explicitlyRetryExhRequest(10, request)) {
+            errors += "Unable to add gallery to remote server: '${gallery.title}' (GID: ${gallery.gid})!"
+        }
+    }
+
+    private fun explicitlyRetryExhRequest(retryCount: Int, request: Request): Boolean {
+        var success = false
+
+        for(i in 1 .. retryCount) {
+            try {
+                val resp = exh.client.newCall(request).execute()
+
+                if (resp.isSuccessful) {
+                    success = true
+                    break
+                }
+            } catch (e: Exception) {
+                Timber.e(e, "Sync network error!")
+            }
+        }
+
+        return success
+    }
+
+    private fun applyChangeSetToRemote(changeSet: ChangeSet, errors: MutableList<String>) {
+        //Apply removals
+        if(changeSet.removed.isNotEmpty()) {
+            status.onNext(FavoritesSyncStatus.Processing("Removing ${changeSet.removed.size} galleries from remote server"))
+
+            val formBody = FormBody.Builder()
+                    .add("ddact", "delete")
+                    .add("apply", "Apply")
+
+            //Add change set to form
+            changeSet.removed.forEach {
+                formBody.add("modifygids[]", it.gid)
+            }
+
+            val request = Request.Builder()
+                    .url("https://exhentai.org/favorites.php")
+                    .post(formBody.build())
+                    .build()
+
+            if(!explicitlyRetryExhRequest(10, request)) {
+                status.onNext(FavoritesSyncStatus.Error("Unable to delete galleries from the remote servers!"))
+
+                //It is still safe to stop here so crash
+                throw IgnoredException()
+            }
+        }
+
+        //Apply additions
+        changeSet.added.forEachIndexed { index, it ->
+            status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to remote server"))
+
+            addGalleryRemote(it, errors)
+        }
+    }
+
+    private fun applyChangeSetToLocal(changeSet: ChangeSet, errors: MutableList<String>) {
+        val removedManga = mutableListOf<Manga>()
+
+        //Apply removals
+        changeSet.removed.forEachIndexed { index, it ->
+            status.onNext(FavoritesSyncStatus.Processing("Removing gallery ${index + 1} of ${changeSet.removed.size} from local library"))
+            val url = it.getUrl()
+
+            //Consider both EX and EH sources
+            listOf(db.getManga(url, EXH_SOURCE_ID),
+                    db.getManga(url, EH_METADATA_SOURCE_ID)).forEach {
+                val manga = it.executeAsBlocking()
+
+                if(manga?.favorite == true) {
+                    manga.favorite = false
+                    db.updateMangaFavorite(manga).executeAsBlocking()
+                    removedManga += manga
+                }
+            }
+        }
+
+        db.deleteOldMangasCategories(removedManga).executeAsBlocking()
+
+        val insertedMangaCategories = mutableListOf<MangaCategory>()
+        val insertedMangaCategoriesMangas = mutableListOf<Manga>()
+        val categories = db.getCategories().executeAsBlocking()
+
+        //Apply additions
+        changeSet.added.forEachIndexed { index, it ->
+            status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to local library"))
+
+            //Import using gallery adder
+            val result = galleryAdder.addGallery("${exh.baseUrl}${it.getUrl()}",
+                    true,
+                    EXH_SOURCE_ID)
+
+            if(result is GalleryAddEvent.Fail) {
+                errors += "Failed to add gallery to local database: " + when (result) {
+                    is GalleryAddEvent.Fail.Error -> "'${it.title}' ${result.logMessage}"
+                    is GalleryAddEvent.Fail.UnknownType -> "'${it.title}' (${result.galleryUrl}) is not a valid gallery!"
+                }
+            } else if(result is GalleryAddEvent.Success) {
+                insertedMangaCategories += MangaCategory.create(result.manga,
+                        categories[it.category])
+                insertedMangaCategoriesMangas += result.manga
+            }
+        }
+
+        db.setMangaCategories(insertedMangaCategories, insertedMangaCategoriesMangas)
+    }
+
+    class IgnoredException : RuntimeException()
+}
+
+sealed class FavoritesSyncStatus(val message: String) {
+    class Error(message: String) : FavoritesSyncStatus(message)
+    class Idle : FavoritesSyncStatus("Waiting for sync to start")
+    class Initializing : FavoritesSyncStatus("Initializing sync")
+    class Processing(message: String) : FavoritesSyncStatus(message)
+    class Complete(val errors: List<String>) : FavoritesSyncStatus("Sync complete!")
+}

app/src/main/java/exh/favorites/LocalFavoritesStorage.kt

@@ -0,0 +1,132 @@
+package exh.favorites
+
+import eu.kanade.tachiyomi.data.database.DatabaseHelper
+import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.source.online.all.EHentai
+import exh.EH_SOURCE_ID
+import exh.EXH_SOURCE_ID
+import exh.metadata.models.ExGalleryMetadata
+import exh.util.trans
+import io.realm.Realm
+import io.realm.RealmConfiguration
+import uy.kohesive.injekt.injectLazy
+
+class LocalFavoritesStorage {
+    private val db: DatabaseHelper by injectLazy()
+
+    private val realmConfig = RealmConfiguration.Builder()
+            .name("fav-sync")
+            .deleteRealmIfMigrationNeeded()
+            .build()
+
+    private val realm
+        get() = Realm.getInstance(realmConfig)
+
+    fun getChangedDbEntries()
+            = getChangedEntries(
+            parseToFavoriteEntries(
+                    loadDbCategories(
+                            db.getFavoriteMangas()
+                                    .executeAsBlocking()
+                                    .asSequence()
+                    )
+            )
+    )
+
+    fun getChangedRemoteEntries(entries: List<EHentai.ParsedManga>)
+            = getChangedEntries(
+            parseToFavoriteEntries(
+                    entries.asSequence().map {
+                        Pair(it.fav, it.manga.apply {
+                            favorite = true
+                        })
+                    }
+            )
+    )
+
+    fun snapshotEntries() {
+        val dbMangas = parseToFavoriteEntries(
+                loadDbCategories(
+                        db.getFavoriteMangas()
+                                .executeAsBlocking()
+                                .asSequence()
+                )
+        )
+
+        realm.use { realm ->
+            realm.trans {
+                //Delete old snapshot
+                realm.delete(FavoriteEntry::class.java)
+
+                //Insert new snapshots
+                realm.copyToRealm(dbMangas.toList())
+            }
+        }
+    }
+
+    private fun getChangedEntries(entries: Sequence<FavoriteEntry>): ChangeSet {
+        return realm.use { realm ->
+            val terminated = entries.toList()
+
+            val added = terminated.filter {
+                realm.queryRealmForEntry(it) == null
+            }
+
+            val removed = realm.where(FavoriteEntry::class.java)
+                    .findAll()
+                    .filter {
+                        queryListForEntry(terminated, it) == null
+                    }.map {
+                        realm.copyFromRealm(it)
+                    }
+
+            ChangeSet(added, removed)
+        }
+    }
+
+    private fun Realm.queryRealmForEntry(entry: FavoriteEntry)
+            = where(FavoriteEntry::class.java)
+            .equalTo(FavoriteEntry::gid.name, entry.gid)
+            .equalTo(FavoriteEntry::token.name, entry.token)
+            .equalTo(FavoriteEntry::category.name, entry.category)
+            .findFirst()
+
+    private fun queryListForEntry(list: List<FavoriteEntry>, entry: FavoriteEntry)
+        = list.find {
+        it.gid == entry.gid
+                && it.token == entry.token
+                && it.category == entry.category
+    }
+
+    private fun loadDbCategories(manga: Sequence<Manga>): Sequence<Pair<Int, Manga>> {
+        val dbCategories = db.getCategories().executeAsBlocking()
+
+        return manga.filter(this::validateDbManga).mapNotNull {
+            val category = db.getCategoriesForManga(it).executeAsBlocking()
+
+            Pair(dbCategories.indexOf(category.firstOrNull()
+                    ?: return@mapNotNull null), it)
+        }
+    }
+
+    private fun parseToFavoriteEntries(manga: Sequence<Pair<Int, Manga>>)
+            = manga.filter {
+        validateDbManga(it.second)
+    }.mapNotNull {
+                FavoriteEntry().apply {
+                    title = it.second.title
+                    gid = ExGalleryMetadata.galleryId(it.second.url)
+                    token = ExGalleryMetadata.galleryToken(it.second.url)
+                    category = it.first
+
+                    if(this.category > 9)
+                        return@mapNotNull null
+                }
+            }
+
+    private fun validateDbManga(manga: Manga)
+            = manga.favorite && (manga.source == EH_SOURCE_ID || manga.source == EXH_SOURCE_ID)
+}
+
+data class ChangeSet(val added: List<FavoriteEntry>,
+                     val removed: List<FavoriteEntry>)

app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt

@@ -26,6 +26,10 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
     override var uuid: String = UUID.randomUUID().toString()
 
     var url: String? = null
+        set(value) {
+            //Ensure that URLs are always formatted in the same way to reduce duplicate galleries
+            field = value?.let { normalizeUrl(it) }
+        }
 
     @Index
     var gId: String? = null
@@ -60,7 +64,7 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
 
     override var tags: RealmList<Tag> = RealmList()
 
-    override fun getTitles() = listOf(title, altTitle).filterNotNull()
+    override fun getTitles() = listOfNotNull(title, altTitle)
 
     @Ignore
     override val titleFields = TITLE_FIELDS
@@ -93,7 +97,7 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
     }
 
     override fun copyTo(manga: SManga) {
-        url?.let { manga.url = it }
+        url?.let { manga.url = normalizeUrl(it) }
         thumbnailUrl?.let { manga.thumbnail_url = it }
 
         //No title bug?
@@ -118,8 +122,8 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
             ONGOING_SUFFIX.find {
                 t.endsWith(it, ignoreCase = true)
             }?.let {
-                manga.status = SManga.ONGOING
-            }
+                        manga.status = SManga.ONGOING
+                    }
         }
 
         //Build a nice looking description out of what we know
@@ -165,6 +169,12 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
         fun galleryToken(url: String) =
                 splitGalleryUrl(url).last()
 
+        fun normalizeUrl(id: String, token: String)
+                = "/g/$id/$token/?nw=always"
+
+        fun normalizeUrl(url: String)
+                = normalizeUrl(galleryId(url), galleryToken(url))
+
         val TITLE_FIELDS = listOf(
                 ExGalleryMetadata::title.name,
                 ExGalleryMetadata::altTitle.name

app/src/main/java/exh/metadata/models/GalleryQuery.kt

@@ -1,6 +1,8 @@
 package exh.metadata.models
 
-import io.realm.*
+import io.realm.Case
+import io.realm.Realm
+import io.realm.RealmQuery
 import java.util.*
 import kotlin.reflect.KClass
 import kotlin.reflect.KProperty
@@ -58,7 +60,7 @@ abstract class GalleryQuery<T : SearchableGalleryMetadata>(val clazz: KClass<T>)
                     is Long -> newMeta.equalTo(n, v)
                     is Short -> newMeta.equalTo(n, v)
                     is String -> newMeta.equalTo(n, v, Case.INSENSITIVE)
-                    else -> throw IllegalArgumentException("Unknown type: ${v::class.qualifiedName}!")
+                    else -> throw IllegalArgumentException("Unknown type: ${v::class.java.name}!")
                 }
             }
         }