Migrate to realm for metadata

This commit is contained in:
NerdNumber9 2017-08-25 17:31:38 -04:00
parent bb6b88a703
commit cd291f0a27
31 changed files with 1394 additions and 588 deletions

View File

@ -3,6 +3,8 @@ import java.text.SimpleDateFormat
apply plugin: 'com.android.application' apply plugin: 'com.android.application'
apply plugin: 'kotlin-android' apply plugin: 'kotlin-android'
apply plugin: 'kotlin-android-extensions' apply plugin: 'kotlin-android-extensions'
//Realm (EH)
apply plugin: 'realm-android'
if (file("custom.gradle").exists()) { if (file("custom.gradle").exists()) {
apply from: "custom.gradle" apply from: "custom.gradle"
@ -207,18 +209,6 @@ dependencies {
compile "com.jakewharton.rxbinding:rxbinding-support-v4-kotlin:$rxbindings_version" compile "com.jakewharton.rxbinding:rxbinding-support-v4-kotlin:$rxbindings_version"
compile "com.jakewharton.rxbinding:rxbinding-recyclerview-v7-kotlin:$rxbindings_version" compile "com.jakewharton.rxbinding:rxbinding-recyclerview-v7-kotlin:$rxbindings_version"
//Firebase (EH)
final firebase_version = '10.0.1'
releaseCompile "com.google.firebase:firebase-core:$firebase_version"
releaseCompile "com.google.firebase:firebase-messaging:$firebase_version"
releaseCompile "com.google.firebase:firebase-crash:$firebase_version"
//SnappyDB (EH)
compile 'io.paperdb:paperdb:2.1'
//JVE (Regex) (EH)
compile 'ru.lanwen.verbalregex:java-verbal-expressions:1.4'
//Pin lock view (EXH) //Pin lock view (EXH)
compile 'com.andrognito.pinlockview:pinlockview:1.0.1' compile 'com.andrognito.pinlockview:pinlockview:1.0.1'
@ -303,5 +293,3 @@ afterEvaluate {
} }
} }
} }
//Firebase (EH)
apply plugin: 'com.google.gms.google-services'

View File

@ -89,6 +89,16 @@
# [EH] # [EH]
-keep class exh.** { *; } -keep class exh.** { *; }
# Realm
-dontnote rx.internal.util.PlatformDependent
-keep * public class * extends io.realm.RealmObject
-keep * public class * implements io.realm.RealmModel
-keep class io.realm.annotations.RealmModule
-keep @io.realm.annotations.RealmModule class *
-keep class io.realm.internal.Keep
-keep @io.realm.internal.Keep class *
-dontwarn io.realm.**
# Keep google stuff # Keep google stuff
-dontwarn com.google.android.gms.** -dontwarn com.google.android.gms.**
-dontwarn com.google.firebase.** -dontwarn com.google.firebase.**

View File

@ -139,6 +139,14 @@
android:host="exhentai.org" android:host="exhentai.org"
android:pathPrefix="/g/" android:pathPrefix="/g/"
android:scheme="https"/> android:scheme="https"/>
<data
android:host="nhentai.net"
android:pathPrefix="/g/"
android:scheme="http"/>
<data
android:host="nhentai.net"
android:pathPrefix="/g/"
android:scheme="https"/>
</intent-filter> </intent-filter>
</activity> </activity>

View File

@ -10,11 +10,14 @@ import eu.kanade.tachiyomi.data.backup.BackupCreatorJob
import eu.kanade.tachiyomi.data.library.LibraryUpdateJob import eu.kanade.tachiyomi.data.library.LibraryUpdateJob
import eu.kanade.tachiyomi.data.updater.UpdateCheckerJob import eu.kanade.tachiyomi.data.updater.UpdateCheckerJob
import eu.kanade.tachiyomi.util.LocaleHelper import eu.kanade.tachiyomi.util.LocaleHelper
import io.paperdb.Paper import io.realm.Realm
import io.realm.RealmConfiguration
import timber.log.Timber import timber.log.Timber
import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.InjektScope import uy.kohesive.injekt.api.InjektScope
import uy.kohesive.injekt.registry.default.DefaultRegistrar import uy.kohesive.injekt.registry.default.DefaultRegistrar
import java.io.File
import kotlin.concurrent.thread
open class App : Application() { open class App : Application() {
@ -26,7 +29,7 @@ open class App : Application() {
if (BuildConfig.DEBUG) Timber.plant(Timber.DebugTree()) if (BuildConfig.DEBUG) Timber.plant(Timber.DebugTree())
setupJobManager() setupJobManager()
Paper.init(this) //Setup metadata DB (EH) setupRealm() //Setup metadata DB (EH)
Reprint.initialize(this) //Setup fingerprint (EH) Reprint.initialize(this) //Setup fingerprint (EH)
LocaleHelper.updateConfiguration(this, resources.configuration) LocaleHelper.updateConfiguration(this, resources.configuration)
@ -55,4 +58,25 @@ open class App : Application() {
} }
} }
private fun setupRealm() {
Realm.init(this)
val config = RealmConfiguration.Builder()
.name("gallery-metadata.realm")
.schemaVersion(1)
.build()
Realm.setDefaultConfiguration(config)
//Delete old paper db files
listOf(
File(filesDir, "gallery-ex"),
File(filesDir, "gallery-perveden"),
File(filesDir, "gallery-nhentai")
).forEach {
if(it.exists()) {
thread {
it.deleteRecursively()
}
}
}
}
} }

View File

@ -180,14 +180,12 @@ class PreferencesHelper(val context: Context) {
fun thumbnailRows() = rxPrefs.getString("ex_thumb_rows", "tr_2") fun thumbnailRows() = rxPrefs.getString("ex_thumb_rows", "tr_2")
fun migrateLibraryAsked() = rxPrefs.getBoolean("ex_migrate_library", false) fun migrateLibraryAsked2() = rxPrefs.getBoolean("ex_migrate_library2", false)
fun migrationStatus() = rxPrefs.getInteger("migration_status", MigrationStatus.NOT_INITIALIZED) fun migrationStatus() = rxPrefs.getInteger("migration_status", MigrationStatus.NOT_INITIALIZED)
fun hasPerformedURLMigration() = rxPrefs.getBoolean("performed_url_migration", false) fun hasPerformedURLMigration() = rxPrefs.getBoolean("performed_url_migration", false)
fun hasPerformedSourceMigration() = rxPrefs.getBoolean("performed_source_migration", false)
//EH Cookies //EH Cookies
fun memberIdVal() = rxPrefs.getString("eh_ipb_member_id", null) fun memberIdVal() = rxPrefs.getString("eh_ipb_member_id", null)
fun passHashVal() = rxPrefs.getString("eh_ipb_pass_hash", null) fun passHashVal() = rxPrefs.getString("eh_ipb_pass_hash", null)

View File

@ -10,7 +10,6 @@ import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.source.online.all.EHentai import eu.kanade.tachiyomi.source.online.all.EHentai
import eu.kanade.tachiyomi.source.online.all.EHentaiMetadata
import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.source.online.YamlHttpSource import eu.kanade.tachiyomi.source.online.YamlHttpSource
import eu.kanade.tachiyomi.source.online.all.NHentai import eu.kanade.tachiyomi.source.online.all.NHentai
@ -88,13 +87,11 @@ open class SourceManager(private val context: Context) {
) )
private fun createEHSources(): List<Source> { private fun createEHSources(): List<Source> {
val exSrcs = mutableListOf( val exSrcs = mutableListOf<HttpSource>(
EHentai(EH_SOURCE_ID, false, context), EHentai(EH_SOURCE_ID, false, context)
EHentaiMetadata(EH_METADATA_SOURCE_ID, false, context)
) )
if(prefs.enableExhentai().getOrDefault()) { if(prefs.enableExhentai().getOrDefault()) {
exSrcs += EHentai(EXH_SOURCE_ID, true, context) exSrcs += EHentai(EXH_SOURCE_ID, true, context)
exSrcs += EHentaiMetadata(EXH_METADATA_SOURCE_ID, true, context)
} }
exSrcs += PervEden(PERV_EDEN_EN_SOURCE_ID, "en") exSrcs += PervEden(PERV_EDEN_EN_SOURCE_ID, "en")
exSrcs += PervEden(PERV_EDEN_IT_SOURCE_ID, "it") exSrcs += PervEden(PERV_EDEN_IT_SOURCE_ID, "it")

View File

@ -20,14 +20,13 @@ import uy.kohesive.injekt.injectLazy
import java.net.URLEncoder import java.net.URLEncoder
import java.util.* import java.util.*
import exh.ui.login.LoginController import exh.ui.login.LoginController
import exh.util.UriFilter
import exh.util.UriGroup
import okhttp3.CacheControl import okhttp3.CacheControl
import okhttp3.Headers import okhttp3.Headers
import okhttp3.Request import okhttp3.Request
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import exh.GalleryAdder import exh.GalleryAdder
import exh.util.urlImportFetchSearchManga import exh.util.*
import io.realm.Realm
class EHentai(override val id: Long, class EHentai(override val id: Long,
val exh: Boolean, val exh: Boolean,
@ -50,8 +49,6 @@ class EHentai(override val id: Long,
val prefs: PreferencesHelper by injectLazy() val prefs: PreferencesHelper by injectLazy()
val metadataHelper = MetadataHelper()
val galleryAdder = GalleryAdder() val galleryAdder = GalleryAdder()
/** /**
@ -168,10 +165,10 @@ class EHentai(override val id: Long,
override fun latestUpdatesParse(response: Response) = genericMangaParse(response) override fun latestUpdatesParse(response: Response) = genericMangaParse(response)
fun exGet(url: String, page: Int? = null, additionalHeaders: Headers? = null, cache: Boolean = true) fun exGet(url: String, page: Int? = null, additionalHeaders: Headers? = null, cache: Boolean = true)
= GET(page?.let { = GET(page?.let {
addParam(url, "page", Integer.toString(page - 1)) addParam(url, "page", Integer.toString(page - 1))
} ?: url, additionalHeaders?.let { } ?: url, additionalHeaders?.let {
val headers = headers.newBuilder() val headers = headers.newBuilder()
it.toMultimap().forEach { (t, u) -> it.toMultimap().forEach { (t, u) ->
u.forEach { u.forEach {
headers.add(t, it) headers.add(t, it)
@ -188,86 +185,90 @@ class EHentai(override val id: Long,
/** /**
* Parse gallery page to metadata model * Parse gallery page to metadata model
*/ */
override fun mangaDetailsParse(response: Response) = with(response.asJsoup()) { override fun mangaDetailsParse(response: Response)
val metdata = ExGalleryMetadata() = with(response.asJsoup()) {
with(metdata) { realmTrans { realm ->
url = response.request().url().encodedPath() val url = response.request().url().encodedPath()!!
exh = this@EHentai.exh val gId = ExGalleryMetadata.galleryId(url)
title = select("#gn").text().nullIfBlank()?.trim() val gToken = ExGalleryMetadata.galleryToken(url)
altTitle = select("#gj").text().nullIfBlank()?.trim() val metdata = (realm.loadEh(gId, gToken, exh)
?: realm.createUUIDObj(ExGalleryMetadata::class.java))
with(metdata) {
this.url = url
this.gId = gId
this.gToken = gToken
thumbnailUrl = select("#gd1 div").attr("style").nullIfBlank()?.let { exh = this@EHentai.exh
it.substring(it.indexOf('(') + 1 until it.lastIndexOf(')')) title = select("#gn").text().nullIfBlank()?.trim()
}
genre = select(".ic").parents().attr("href").nullIfBlank()?.trim()?.substringAfterLast('/') altTitle = select("#gj").text().nullIfBlank()?.trim()
uploader = select("#gdn").text().nullIfBlank()?.trim() thumbnailUrl = select("#gd1 div").attr("style").nullIfBlank()?.let {
it.substring(it.indexOf('(') + 1 until it.lastIndexOf(')'))
}
genre = select(".ic").parents().attr("href").nullIfBlank()?.trim()?.substringAfterLast('/')
//Parse the table uploader = select("#gdn").text().nullIfBlank()?.trim()
select("#gdd tr").forEach {
it.select(".gdt1") //Parse the table
.text() select("#gdd tr").forEach {
.nullIfBlank() it.select(".gdt1")
?.trim() .text()
?.let { left -> .nullIfBlank()
it.select(".gdt2") ?.trim()
.text() ?.let { left ->
.nullIfBlank() it.select(".gdt2")
?.trim() .text()
?.let { right -> .nullIfBlank()
ignore { ?.trim()
when (left.removeSuffix(":") ?.let { right ->
.toLowerCase()) { ignore {
"posted" -> datePosted = EX_DATE_FORMAT.parse(right).time when (left.removeSuffix(":")
"visible" -> visible = right.nullIfBlank() .toLowerCase()) {
"language" -> { "posted" -> datePosted = EX_DATE_FORMAT.parse(right).time
language = right.removeSuffix(TR_SUFFIX).trim().nullIfBlank() "visible" -> visible = right.nullIfBlank()
translated = right.endsWith(TR_SUFFIX, true) "language" -> {
language = right.removeSuffix(TR_SUFFIX).trim().nullIfBlank()
translated = right.endsWith(TR_SUFFIX, true)
}
"file size" -> size = parseHumanReadableByteCount(right)?.toLong()
"length" -> length = right.removeSuffix("pages").trim().nullIfBlank()?.toInt()
"favorited" -> favorites = right.removeSuffix("times").trim().nullIfBlank()?.toInt()
} }
"file size" -> size = parseHumanReadableByteCount(right)?.toLong()
"length" -> length = right.removeSuffix("pages").trim().nullIfBlank()?.toInt()
"favorited" -> favorites = right.removeSuffix("times").trim().nullIfBlank()?.toInt()
} }
} }
} }
}
}
//Parse ratings
ignore {
averageRating = select("#rating_label")
.text()
.removePrefix("Average:")
.trim()
.nullIfBlank()
?.toDouble()
ratingCount = select("#rating_count")
.text()
.trim()
.nullIfBlank()
?.toInt()
}
//Parse tags
tags.clear()
select("#taglist tr").forEach {
val namespace = it.select(".tc").text().removeSuffix(":")
val currentTags = it.select("div").map {
Tag(it.text().trim(),
it.hasClass("gtl"))
} }
tags.put(namespace, ArrayList(currentTags))
}
//Save metadata //Parse ratings
metadataHelper.writeGallery(this, id) ignore {
averageRating = select("#rating_label")
.text()
.removePrefix("Average:")
.trim()
.nullIfBlank()
?.toDouble()
ratingCount = select("#rating_count")
.text()
.trim()
.nullIfBlank()
?.toInt()
}
//Copy metadata to manga //Parse tags
SManga.create().let { tags.clear()
copyTo(it) select("#taglist tr").forEach {
it val namespace = it.select(".tc").text().removeSuffix(":")
tags.addAll(it.select("div").map {
Tag(namespace, it.text().trim(), it.hasClass("gtl"))
})
}
//Copy metadata to manga
SManga.create().apply {
copyTo(this)
}
} }
} }
} }

View File

@ -1,127 +0,0 @@
package eu.kanade.tachiyomi.source.online.all
import android.content.Context
import eu.kanade.tachiyomi.data.database.models.Chapter
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.source.online.HttpSource
import exh.metadata.MetadataHelper
import exh.metadata.copyTo
import exh.metadata.models.ExGalleryMetadata
import exh.search.SearchEngine
import okhttp3.Response
import rx.Observable
/**
* Offline metadata store source
*
* TODO This no longer fakes an online source because of technical reasons.
* If we still want offline search, we must find out a way to rearchitecture the source system so it supports
* online source faking again.
*/
class EHentaiMetadata(override val id: Long,
val exh: Boolean,
val context: Context) : HttpSource() {
override fun popularMangaRequest(page: Int)
= throw UnsupportedOperationException("Unused method called!")
override fun popularMangaParse(response: Response)
= throw UnsupportedOperationException("Unused method called!")
override fun searchMangaRequest(page: Int, query: String, filters: FilterList)
= throw UnsupportedOperationException("Unused method called!")
override fun searchMangaParse(response: Response)
= throw UnsupportedOperationException("Unused method called!")
override fun latestUpdatesRequest(page: Int)
= throw UnsupportedOperationException("Unused method called!")
override fun latestUpdatesParse(response: Response)
= throw UnsupportedOperationException("Unused method called!")
override fun mangaDetailsParse(response: Response)
= throw UnsupportedOperationException("Unused method called!")
override fun chapterListParse(response: Response)
= throw UnsupportedOperationException("Unused method called!")
override fun pageListParse(response: Response)
= throw UnsupportedOperationException("Unused method called!")
override fun imageUrlParse(response: Response)
= throw UnsupportedOperationException("Unused method called!")
val metadataHelper = MetadataHelper()
val internalEx = EHentai(id - 2, exh, context)
val searchEngine = SearchEngine()
override val baseUrl: String
get() = throw UnsupportedOperationException()
override val lang: String
get() = "advanced"
override val supportsLatest: Boolean
get() = true
override fun fetchChapterList(manga: SManga): Observable<List<SChapter>>
= Observable.just(listOf(Chapter.create().apply {
url = manga.url
name = "ONLINE - Chapter"
chapter_number = 1f
}))
override fun fetchPageList(chapter: SChapter) = internalEx.fetchPageList(chapter)
override fun fetchImageUrl(page: Page) = internalEx.fetchImageUrl(page)
fun List<ExGalleryMetadata>.mapToManga() = filter { it.exh == exh }
.map {
Manga.create(id).apply {
it.copyTo(this)
source = this@EHentaiMetadata.id
}
}
fun sortedByTimeGalleries() = metadataHelper.getAllGalleries().sortedByDescending {
it.datePosted ?: 0
}
override fun fetchPopularManga(page: Int)
= Observable.fromCallable {
MangasPage(metadataHelper.getAllGalleries().sortedByDescending {
it.ratingCount ?: 0
}.mapToManga(), false)
}!!
override fun fetchSearchManga(page: Int, query: String, filters: FilterList)
= Observable.fromCallable {
val genreGroup = filters.find {
it is EHentai.GenreGroup
}!! as EHentai.GenreGroup
val disableGenreFilter = genreGroup.state.find(EHentai.GenreOption::state) == null
val parsed = searchEngine.parseQuery(query)
MangasPage(sortedByTimeGalleries().filter { manga ->
disableGenreFilter || genreGroup.state.find {
it.state && it.genreId == manga.genre
} != null
}.filter {
searchEngine.matches(it, parsed)
}.mapToManga(), false)
}!!
override fun fetchLatestUpdates(page: Int)
= Observable.fromCallable {
MangasPage(sortedByTimeGalleries().mapToManga(), false)
}!!
override fun fetchMangaDetails(manga: SManga) = Observable.fromCallable {
//Hack to convert the gallery into an online gallery when favoriting it or reading it
metadataHelper.fetchEhMetadata(manga.url, exh)?.copyTo(manga)
manga
}!!
override fun getFilterList() = FilterList(EHentai.GenreGroup())
override val name: String
get() = if(exh) {
"ExHentai"
} else {
"E-Hentai"
} + " - METADATA"
}

View File

@ -17,10 +17,15 @@ import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.model.* import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.source.online.HttpSource
import exh.NHENTAI_SOURCE_ID import exh.NHENTAI_SOURCE_ID
import exh.metadata.MetadataHelper
import exh.metadata.copyTo import exh.metadata.copyTo
import exh.metadata.loadNhentai
import exh.metadata.loadNhentaiAsync
import exh.metadata.models.NHentaiMetadata import exh.metadata.models.NHentaiMetadata
import exh.metadata.models.PageImageType
import exh.metadata.models.Tag import exh.metadata.models.Tag
import exh.util.createUUIDObj
import exh.util.defRealm
import exh.util.realmTrans
import exh.util.urlImportFetchSearchManga import exh.util.urlImportFetchSearchManga
import okhttp3.Request import okhttp3.Request
import okhttp3.Response import okhttp3.Response
@ -108,62 +113,73 @@ class NHentai(context: Context) : HttpSource() {
return MangasPage(emptyList(), false) return MangasPage(emptyList(), false)
} }
fun rawParseGallery(obj: JsonObject) = NHentaiMetadata().apply { fun rawParseGallery(obj: JsonObject) = realmTrans { realm ->
uploadDate = obj.get("upload_date")?.notNull()?.long val nhId = obj.get("id").asLong
favoritesCount = obj.get("num_favorites")?.notNull()?.long (realm.loadNhentai(nhId)
?: realm.createUUIDObj(NHentaiMetadata::class.java)).apply {
this.nhId = nhId
mediaId = obj.get("media_id")?.notNull()?.string uploadDate = obj.get("upload_date")?.notNull()?.long
obj.get("title")?.asJsonObject?.let { favoritesCount = obj.get("num_favorites")?.notNull()?.long
japaneseTitle = it.get("japanese")?.notNull()?.string
shortTitle = it.get("pretty")?.notNull()?.string
englishTitle = it.get("english")?.notNull()?.string
}
obj.get("images")?.asJsonObject?.let { mediaId = obj.get("media_id")?.notNull()?.string
coverImageType = it.get("cover")?.get("t")?.notNull()?.asString
it.get("pages")?.asJsonArray?.map { obj.get("title")?.asJsonObject?.let {
it?.asJsonObject?.get("t")?.notNull()?.asString japaneseTitle = it.get("japanese")?.notNull()?.string
}?.filterNotNull()?.let { shortTitle = it.get("pretty")?.notNull()?.string
pageImageTypes.clear() englishTitle = it.get("english")?.notNull()?.string
pageImageTypes.addAll(it)
} }
thumbnailImageType = it.get("thumbnail")?.get("t")?.notNull()?.asString
}
scanlator = obj.get("scanlator")?.notNull()?.asString obj.get("images")?.asJsonObject?.let {
coverImageType = it.get("cover")?.get("t")?.notNull()?.asString
it.get("pages")?.asJsonArray?.map {
it?.asJsonObject?.get("t")?.notNull()?.asString
}?.filterNotNull()?.map {
PageImageType(it)
}?.let {
pageImageTypes.clear()
pageImageTypes.addAll(it)
}
thumbnailImageType = it.get("thumbnail")?.get("t")?.notNull()?.asString
}
id = obj.get("id")?.asLong scanlator = obj.get("scanlator")?.notNull()?.asString
obj.get("tags")?.asJsonArray?.map { obj.get("tags")?.asJsonArray?.map {
val asObj = it.asJsonObject val asObj = it.asJsonObject
Pair(asObj.get("type")?.string, asObj.get("name")?.string) Pair(asObj.get("type")?.string, asObj.get("name")?.string)
}?.apply { }?.apply {
tags.clear() tags.clear()
}?.forEach { }?.forEach {
if(it.first != null && it.second != null) if(it.first != null && it.second != null)
tags.getOrPut(it.first!!, { ArrayList() }).add(Tag(it.second!!, false)) tags.add(Tag(it.first!!, it.second!!, false))
}
} }
} }
fun parseGallery(obj: JsonObject) = rawParseGallery(obj).let { fun parseGallery(obj: JsonObject) = rawParseGallery(obj).let {
metadataHelper.writeGallery(it, id)
SManga.create().apply { SManga.create().apply {
it.copyTo(this) it.copyTo(this)
} }
} }
fun lazyLoadMetadata(url: String) = fun lazyLoadMetadata(url: String) =
Observable.fromCallable { defRealm { realm ->
metadataHelper.fetchNhentaiMetadata(url) realm.loadNhentaiAsync(NHentaiMetadata.nhIdFromUrl(url))
?: client.newCall(urlToDetailsRequest(url)) .flatMap {
.asObservableSuccess() if(it == null)
.map { client.newCall(urlToDetailsRequest(url))
rawParseGallery(jsonParser.parse(it.body()!!.string()).asJsonObject) .asObservableSuccess()
}.toBlocking().first() .map {
}!! rawParseGallery(jsonParser.parse(it.body()!!.string())
.asJsonObject)
}.first()
else
Observable.just(it)
}.map { realm.copyFromRealm(it) }
}
override fun fetchChapterList(manga: SManga) override fun fetchChapterList(manga: SManga)
= lazyLoadMetadata(manga.url).map { = lazyLoadMetadata(manga.url).map {
@ -181,7 +197,7 @@ class NHentai(context: Context) : HttpSource() {
if(metadata.mediaId == null) emptyList() if(metadata.mediaId == null) emptyList()
else else
metadata.pageImageTypes.mapIndexed { index, s -> metadata.pageImageTypes.mapIndexed { index, s ->
val imageUrl = imageUrlFromType(metadata.mediaId!!, index + 1, s) val imageUrl = imageUrlFromType(metadata.mediaId!!, index + 1, s.type!!)
Page(index, imageUrl!!, imageUrl) Page(index, imageUrl!!, imageUrl)
} }
}!! }!!
@ -231,14 +247,10 @@ class NHentai(context: Context) : HttpSource() {
val jsonParser by lazy { val jsonParser by lazy {
JsonParser() JsonParser()
} }
val metadataHelper by lazy {
MetadataHelper()
}
} }
fun JsonElement.notNull() = fun JsonElement.notNull() =
if(this is JsonNull) if(this is JsonNull)
null null
else this else this
} }

View File

@ -6,12 +6,15 @@ import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.source.online.ParsedHttpSource import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import eu.kanade.tachiyomi.util.ChapterRecognition import eu.kanade.tachiyomi.util.ChapterRecognition
import eu.kanade.tachiyomi.util.asJsoup import eu.kanade.tachiyomi.util.asJsoup
import exh.metadata.MetadataHelper
import exh.metadata.copyTo import exh.metadata.copyTo
import exh.metadata.loadPervEden
import exh.metadata.models.PervEdenGalleryMetadata import exh.metadata.models.PervEdenGalleryMetadata
import exh.metadata.models.PervEdenTitle
import exh.metadata.models.Tag import exh.metadata.models.Tag
import exh.util.UriFilter import exh.util.UriFilter
import exh.util.UriGroup import exh.util.UriGroup
import exh.util.createUUIDObj
import exh.util.realmTrans
import okhttp3.Request import okhttp3.Request
import okhttp3.Response import okhttp3.Response
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
@ -27,8 +30,6 @@ class PervEden(override val id: Long, override val lang: String) : ParsedHttpSou
override val name = "Perv Eden" override val name = "Perv Eden"
override val baseUrl = "http://www.perveden.com" override val baseUrl = "http://www.perveden.com"
val metadataHelper by lazy { MetadataHelper() }
override fun popularMangaSelector() = "#topManga > ul > li" override fun popularMangaSelector() = "#topManga > ul > li"
override fun popularMangaFromElement(element: Element): SManga { override fun popularMangaFromElement(element: Element): SManga {
@ -99,72 +100,68 @@ class PervEden(override val id: Long, override val lang: String) : ParsedHttpSou
} }
override fun mangaDetailsParse(document: Document): SManga { override fun mangaDetailsParse(document: Document): SManga {
val metadata = PervEdenGalleryMetadata() realmTrans { realm ->
with(metadata) { val url = document.location()
url = document.location() val metadata = (realm.loadPervEden(PervEdenGalleryMetadata.pvIdFromUrl(url), id)
?: realm.createUUIDObj(PervEdenGalleryMetadata::class.java))
with(metadata) {
this.url = url
lang = this@PervEden.lang lang = this@PervEden.lang
title = document.getElementsByClass("manga-title").first()?.text() title = document.getElementsByClass("manga-title").first()?.text()
thumbnailUrl = "http:" + document.getElementsByClass("mangaImage2").first()?.child(0)?.attr("src") thumbnailUrl = "http:" + document.getElementsByClass("mangaImage2").first()?.child(0)?.attr("src")
val rightBoxElement = document.select(".rightBox:not(.info)").first() val rightBoxElement = document.select(".rightBox:not(.info)").first()
tags.clear() tags.clear()
var inStatus: String? = null var inStatus: String? = null
rightBoxElement.childNodes().forEach { rightBoxElement.childNodes().forEach {
if(it is Element && it.tagName().toLowerCase() == "h4") { if(it is Element && it.tagName().toLowerCase() == "h4") {
inStatus = it.text().trim() inStatus = it.text().trim()
} else { } else {
when(inStatus) { when(inStatus) {
"Alternative name(s)" -> { "Alternative name(s)" -> {
if(it is TextNode) { if(it is TextNode) {
val text = it.text().trim() val text = it.text().trim()
if(!text.isBlank()) if(!text.isBlank())
altTitles.add(text) altTitles.add(PervEdenTitle(this, text))
}
} }
} "Artist" -> {
"Artist" -> { if(it is Element && it.tagName() == "a") {
if(it is Element && it.tagName() == "a") { artist = it.text()
artist = it.text() tags.add(Tag("artist", it.text().toLowerCase(), false))
tags.getOrPut("artist", { }
ArrayList()
}).add(Tag(it.text().toLowerCase(), false))
} }
} "Genres" -> {
"Genres" -> { if(it is Element && it.tagName() == "a")
if(it is Element && it.tagName() == "a") tags.add(Tag("genre", it.text().toLowerCase(), false))
tags.getOrPut("genre", {
ArrayList()
}).add(Tag(it.text().toLowerCase(), false))
}
"Type" -> {
if(it is TextNode) {
val text = it.text().trim()
if(!text.isBlank())
type = text
} }
} "Type" -> {
"Status" -> { if(it is TextNode) {
if(it is TextNode) { val text = it.text().trim()
val text = it.text().trim() if(!text.isBlank())
if(!text.isBlank()) type = text
status = text }
}
"Status" -> {
if(it is TextNode) {
val text = it.text().trim()
if(!text.isBlank())
status = text
}
} }
} }
} }
} }
}
rating = document.getElementById("rating-score")?.attr("value")?.toFloat() rating = document.getElementById("rating-score")?.attr("value")?.toFloat()
//Save metadata return SManga.create().apply {
Timber.d("LNG: " + metadata.lang) copyTo(this)
metadataHelper.writeGallery(this, id) }
return SManga.create().apply {
copyTo(this)
} }
} }
} }
@ -197,12 +194,12 @@ class PervEden(override val id: Long, override val lang: String) : ParsedHttpSou
} }
override fun pageListParse(document: Document) override fun pageListParse(document: Document)
= document.getElementById("pageSelect").getElementsByTag("option").map { = document.getElementById("pageSelect").getElementsByTag("option").map {
Page(it.attr("data-page").toInt() - 1, baseUrl + it.attr("value")) Page(it.attr("data-page").toInt() - 1, baseUrl + it.attr("value"))
} }
override fun imageUrlParse(document: Document) override fun imageUrlParse(document: Document)
= "http:" + document.getElementById("mainImg").attr("src")!! = "http:" + document.getElementById("mainImg").attr("src")!!
override fun getFilterList() = FilterList ( override fun getFilterList() = FilterList (
AuthorFilter(), AuthorFilter(),

View File

@ -2,6 +2,18 @@ package eu.kanade.tachiyomi.ui.library
import eu.davidea.flexibleadapter.FlexibleAdapter import eu.davidea.flexibleadapter.FlexibleAdapter
import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.data.database.models.Manga
import exh.*
import exh.metadata.loadAllMetadata
import exh.metadata.models.ExGalleryMetadata
import exh.metadata.models.NHentaiMetadata
import exh.metadata.models.PervEdenGalleryMetadata
import exh.metadata.queryMetadataFromManga
import exh.search.SearchEngine
import exh.util.defRealm
import rx.Observable
import rx.android.schedulers.AndroidSchedulers
import rx.schedulers.Schedulers
import kotlin.concurrent.thread
/** /**
* Adapter storing a list of manga in a certain category. * Adapter storing a list of manga in a certain category.
@ -16,7 +28,9 @@ class LibraryCategoryAdapter(view: LibraryCategoryView) :
*/ */
private var mangas: List<LibraryItem> = emptyList() private var mangas: List<LibraryItem> = emptyList()
var asyncSearchText: String? = null // --> EH
private val searchEngine = SearchEngine()
// <-- EH
/** /**
* Sets a list of manga in the adapter. * Sets a list of manga in the adapter.
@ -40,9 +54,42 @@ class LibraryCategoryAdapter(view: LibraryCategoryView) :
} }
fun performFilter() { fun performFilter() {
updateDataSet(mangas.filter { Observable.fromCallable {
it.filter(searchText) defRealm { realm ->
}) val parsedQuery = searchEngine.parseQuery(searchText)
val metadata = realm.loadAllMetadata().map {
Pair(it.key, searchEngine.filterResults(it.value, parsedQuery))
}
mangas.filter { manga ->
// --> EH
if (isLewdSource(manga.manga.source)) {
metadata.any {
when (manga.manga.source) {
EH_SOURCE_ID,
EXH_SOURCE_ID ->
if (it.first != ExGalleryMetadata::class)
return@any false
PERV_EDEN_IT_SOURCE_ID,
PERV_EDEN_EN_SOURCE_ID ->
if (it.first != PervEdenGalleryMetadata::class)
return@any false
NHENTAI_SOURCE_ID ->
if (it.first != NHentaiMetadata::class)
return@any false
}
realm.queryMetadataFromManga(manga.manga, it.second.where()).count() > 0
}
} else {
manga.filter(searchText)
}
// <-- EH
}
}
}.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe {
updateDataSet(it)
}
} }
} }

View File

@ -39,10 +39,12 @@ import eu.kanade.tachiyomi.widget.DrawerSwipeCloseListener
import kotlinx.android.synthetic.main.main_activity.* import kotlinx.android.synthetic.main.main_activity.*
import kotlinx.android.synthetic.main.library_controller.view.* import kotlinx.android.synthetic.main.library_controller.view.*
import rx.Subscription import rx.Subscription
import rx.android.schedulers.AndroidSchedulers
import timber.log.Timber import timber.log.Timber
import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get import uy.kohesive.injekt.api.get
import java.io.IOException import java.io.IOException
import java.util.concurrent.TimeUnit
class LibraryController( class LibraryController(
@ -342,7 +344,11 @@ class LibraryController(
// Mutate the filter icon because it needs to be tinted and the resource is shared. // Mutate the filter icon because it needs to be tinted and the resource is shared.
menu.findItem(R.id.action_filter).icon.mutate() menu.findItem(R.id.action_filter).icon.mutate()
searchView.queryTextChanges().subscribeUntilDestroy { // Debounce search (EH)
searchView.queryTextChanges()
.debounce(200, TimeUnit.MILLISECONDS)
.observeOn(AndroidSchedulers.mainThread())
.subscribeUntilDestroy {
query = it.toString() query = it.toString()
searchRelay.call(query) searchRelay.call(query)
} }

View File

@ -12,18 +12,10 @@ import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.util.inflate import eu.kanade.tachiyomi.util.inflate
import eu.kanade.tachiyomi.widget.AutofitRecyclerView import eu.kanade.tachiyomi.widget.AutofitRecyclerView
import exh.isLewdSource
import exh.metadata.MetadataHelper
import exh.search.SearchEngine
import kotlinx.android.synthetic.main.catalogue_grid_item.view.* import kotlinx.android.synthetic.main.catalogue_grid_item.view.*
class LibraryItem(val manga: Manga) : AbstractFlexibleItem<LibraryHolder>(), IFilterable { class LibraryItem(val manga: Manga) : AbstractFlexibleItem<LibraryHolder>(), IFilterable {
// --> EH
private val searchEngine = SearchEngine()
private val metadataHelper = MetadataHelper()
// <-- EH
override fun getLayoutRes(): Int { override fun getLayoutRes(): Int {
return R.layout.catalogue_grid_item return R.layout.catalogue_grid_item
} }
@ -61,15 +53,6 @@ class LibraryItem(val manga: Manga) : AbstractFlexibleItem<LibraryHolder>(), IFi
* @return true if the manga should be included, false otherwise. * @return true if the manga should be included, false otherwise.
*/ */
override fun filter(constraint: String): Boolean { override fun filter(constraint: String): Boolean {
// --> EH
if(!isLewdSource(manga.source)) {
//Use gallery search engine for EH manga
metadataHelper.fetchMetadata(manga.url, manga.source)?.let {
return searchEngine.matches(it, searchEngine.parseQuery(constraint))
}
}
// <-- EH
return manga.title.contains(constraint, true) || return manga.title.contains(constraint, true) ||
(manga.author?.contains(constraint, true) ?: false) (manga.author?.contains(constraint, true) ?: false)
} }

View File

@ -17,6 +17,7 @@ import com.bluelinelabs.conductor.changehandler.FadeChangeHandler
import eu.kanade.tachiyomi.Migrations import eu.kanade.tachiyomi.Migrations
import eu.kanade.tachiyomi.R import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.ui.base.activity.BaseActivity import eu.kanade.tachiyomi.ui.base.activity.BaseActivity
import eu.kanade.tachiyomi.ui.base.controller.DialogController import eu.kanade.tachiyomi.ui.base.controller.DialogController
import eu.kanade.tachiyomi.ui.base.controller.NoToolbarElevationController import eu.kanade.tachiyomi.ui.base.controller.NoToolbarElevationController
@ -35,6 +36,7 @@ import exh.ui.lock.LockChangeHandler
import exh.ui.lock.LockController import exh.ui.lock.LockController
import exh.ui.lock.lockEnabled import exh.ui.lock.lockEnabled
import exh.ui.lock.notifyLockSecurity import exh.ui.lock.notifyLockSecurity
import exh.ui.migration.MetadataFetchDialog
import kotlinx.android.synthetic.main.main_activity.* import kotlinx.android.synthetic.main.main_activity.*
import uy.kohesive.injekt.injectLazy import uy.kohesive.injekt.injectLazy
@ -165,6 +167,10 @@ class MainActivity : BaseActivity() {
if (Migrations.upgrade(preferences)) { if (Migrations.upgrade(preferences)) {
ChangelogDialogController().showDialog(router) ChangelogDialogController().showDialog(router)
} }
// Migrate metadata to Realm (EH)
if(!preferences.migrateLibraryAsked2().getOrDefault())
MetadataFetchDialog().askMigration(this)
} }
} }

View File

@ -10,8 +10,13 @@ import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.network.NetworkHelper import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.source.SourceManager import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.util.syncChaptersWithSource import eu.kanade.tachiyomi.util.syncChaptersWithSource
import exh.metadata.MetadataHelper
import exh.metadata.copyTo import exh.metadata.copyTo
import exh.metadata.loadEh
import exh.metadata.loadNhentai
import exh.metadata.models.ExGalleryMetadata
import exh.metadata.models.NHentaiMetadata
import exh.util.defRealm
import io.realm.Realm
import okhttp3.MediaType import okhttp3.MediaType
import okhttp3.Request import okhttp3.Request
import okhttp3.RequestBody import okhttp3.RequestBody
@ -27,8 +32,6 @@ class GalleryAdder {
private val sourceManager: SourceManager by injectLazy() private val sourceManager: SourceManager by injectLazy()
private val metadataHelper = MetadataHelper()
private val networkHelper: NetworkHelper by injectLazy() private val networkHelper: NetworkHelper by injectLazy()
companion object { companion object {
@ -119,11 +122,17 @@ class GalleryAdder {
manga.copyFrom(sourceObj.fetchMangaDetails(manga).toBlocking().first()) manga.copyFrom(sourceObj.fetchMangaDetails(manga).toBlocking().first())
//Apply metadata //Apply metadata
when(source) { defRealm { realm ->
EH_SOURCE_ID, EXH_SOURCE_ID -> when (source) {
metadataHelper.fetchEhMetadata(realUrl, isExSource(source))?.copyTo(manga) EH_SOURCE_ID, EXH_SOURCE_ID ->
NHENTAI_SOURCE_ID -> realm.loadEh(ExGalleryMetadata.galleryId(realUrl),
metadataHelper.fetchNhentaiMetadata(realUrl)?.copyTo(manga) ExGalleryMetadata.galleryToken(realUrl),
isExSource(source))?.copyTo(manga)
NHENTAI_SOURCE_ID ->
realm.loadNhentai(NHentaiMetadata.nhIdFromUrl(realUrl))
?.copyTo(manga)
else -> return GalleryAddEvent.Fail.UnknownType(url)
}
} }
if (fav) manga.favorite = true if (fav) manga.favorite = true

View File

@ -1,5 +0,0 @@
package exh
import ru.lanwen.verbalregex.VerbalExpression
fun VerbalExpression.Builder.anyChar() = add(".")!!

View File

@ -1,62 +1,114 @@
package exh.metadata package exh.metadata
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.source.model.SManga
import exh.* import exh.*
import exh.metadata.models.ExGalleryMetadata import exh.metadata.models.ExGalleryMetadata
import exh.metadata.models.NHentaiMetadata import exh.metadata.models.NHentaiMetadata
import exh.metadata.models.PervEdenGalleryMetadata import exh.metadata.models.PervEdenGalleryMetadata
import exh.metadata.models.SearchableGalleryMetadata import exh.metadata.models.SearchableGalleryMetadata
import io.paperdb.Paper import io.realm.Realm
import io.realm.RealmQuery
import io.realm.RealmResults
import rx.Observable
import kotlin.reflect.KClass
class MetadataHelper { fun Realm.ehMetaQueryFromUrl(url: String,
exh: Boolean,
meta: RealmQuery<ExGalleryMetadata>? = null) =
ehMetadataQuery(
ExGalleryMetadata.galleryId(url),
ExGalleryMetadata.galleryToken(url),
exh,
meta
)
fun writeGallery(galleryMetadata: SearchableGalleryMetadata, source: Long) fun Realm.ehMetadataQuery(gId: String,
= (if(isExSource(source) || isEhSource(source)) exGalleryBook() gToken: String,
else if(isPervEdenSource(source)) pervEdenGalleryBook() exh: Boolean,
else if(isNhentaiSource(source)) nhentaiGalleryBook() meta: RealmQuery<ExGalleryMetadata>? = null)
else null)?.write(galleryMetadata.galleryUniqueIdentifier(), galleryMetadata)!! = (meta ?: where(ExGalleryMetadata::class.java))
.equalTo(ExGalleryMetadata::gId.name, gId)
.equalTo(ExGalleryMetadata::gToken.name, gToken)
.equalTo(ExGalleryMetadata::exh.name, exh)
fun fetchEhMetadata(url: String, exh: Boolean): ExGalleryMetadata? fun Realm.loadEh(gId: String, gToken: String, exh: Boolean): ExGalleryMetadata?
= ExGalleryMetadata().let { = ehMetadataQuery(gId, gToken, exh)
it.url = url .findFirst()
it.exh = exh
return exGalleryBook().read<ExGalleryMetadata>(it.galleryUniqueIdentifier()) fun Realm.loadEhAsync(gId: String, gToken: String, exh: Boolean): Observable<ExGalleryMetadata?>
= ehMetadataQuery(gId, gToken, exh)
.findFirstAsync()
.asObservable()
private fun pervEdenSourceToLang(source: Long)
= when (source) {
PERV_EDEN_EN_SOURCE_ID -> "en"
PERV_EDEN_IT_SOURCE_ID -> "it"
else -> throw IllegalArgumentException()
}
fun Realm.pervEdenMetaQueryFromUrl(url: String,
source: Long,
meta: RealmQuery<PervEdenGalleryMetadata>?) =
pervEdenMetadataQuery(
PervEdenGalleryMetadata.pvIdFromUrl(url),
source,
meta
)
fun Realm.pervEdenMetadataQuery(pvId: String,
source: Long,
meta: RealmQuery<PervEdenGalleryMetadata>? = null)
= (meta ?: where(PervEdenGalleryMetadata::class.java))
.equalTo(PervEdenGalleryMetadata::lang.name, pervEdenSourceToLang(source))
.equalTo(PervEdenGalleryMetadata::pvId.name, pvId)
fun Realm.loadPervEden(pvId: String, source: Long): PervEdenGalleryMetadata?
= pervEdenMetadataQuery(pvId, source)
.findFirst()
fun Realm.loadPervEdenAsync(pvId: String, source: Long): Observable<PervEdenGalleryMetadata?>
= pervEdenMetadataQuery(pvId, source)
.findFirstAsync()
.asObservable()
fun Realm.nhentaiMetaQueryFromUrl(url: String,
meta: RealmQuery<NHentaiMetadata>?) =
nhentaiMetadataQuery(
NHentaiMetadata.nhIdFromUrl(url),
meta
)
fun Realm.nhentaiMetadataQuery(nhId: Long,
meta: RealmQuery<NHentaiMetadata>? = null)
= (meta ?: where(NHentaiMetadata::class.java))
.equalTo(NHentaiMetadata::nhId.name, nhId)
fun Realm.loadNhentai(nhId: Long): NHentaiMetadata?
= nhentaiMetadataQuery(nhId)
.findFirst()
fun Realm.loadNhentaiAsync(nhId: Long): Observable<NHentaiMetadata?>
= nhentaiMetadataQuery(nhId)
.findFirstAsync()
.asObservable()
fun Realm.loadAllMetadata(): Map<KClass<out SearchableGalleryMetadata>, RealmResults<out SearchableGalleryMetadata>> =
mapOf(
Pair(ExGalleryMetadata::class, where(ExGalleryMetadata::class.java).findAll()),
Pair(NHentaiMetadata::class, where(NHentaiMetadata::class.java).findAll()),
Pair(PervEdenGalleryMetadata::class, where(PervEdenGalleryMetadata::class.java).findAll())
)
fun Realm.queryMetadataFromManga(manga: Manga,
meta: RealmQuery<out SearchableGalleryMetadata>? = null): RealmQuery<out SearchableGalleryMetadata> =
when(manga.source) {
EH_SOURCE_ID -> ehMetaQueryFromUrl(manga.url, false, meta as? RealmQuery<ExGalleryMetadata>)
EXH_SOURCE_ID -> ehMetaQueryFromUrl(manga.url, true, meta as? RealmQuery<ExGalleryMetadata>)
PERV_EDEN_EN_SOURCE_ID,
PERV_EDEN_IT_SOURCE_ID ->
pervEdenMetaQueryFromUrl(manga.url, manga.source, meta as? RealmQuery<PervEdenGalleryMetadata>)
NHENTAI_SOURCE_ID -> nhentaiMetaQueryFromUrl(manga.url, meta as? RealmQuery<NHentaiMetadata>)
else -> throw IllegalArgumentException("Unknown source type!")
} }
fun fetchPervEdenMetadata(url: String, source: Long): PervEdenGalleryMetadata?
= PervEdenGalleryMetadata().let {
it.url = url
if(source == PERV_EDEN_EN_SOURCE_ID)
it.lang = "en"
else if(source == PERV_EDEN_IT_SOURCE_ID)
it.lang = "it"
else throw IllegalArgumentException("Invalid source id!")
return pervEdenGalleryBook().read<PervEdenGalleryMetadata>(it.galleryUniqueIdentifier())
}
fun fetchNhentaiMetadata(url: String) = NHentaiMetadata().let {
it.url = url
nhentaiGalleryBook().read<NHentaiMetadata>(it.galleryUniqueIdentifier())
}
fun fetchMetadata(url: String, source: Long): SearchableGalleryMetadata? {
if(isExSource(source) || isEhSource(source)) {
return fetchEhMetadata(url, isExSource(source))
} else if(isPervEdenSource(source)) {
return fetchPervEdenMetadata(url, source)
} else if(isNhentaiSource(source)) {
return fetchNhentaiMetadata(url)
} else {
return null
}
}
fun getAllGalleries() = exGalleryBook().allKeys.map {
exGalleryBook().read<ExGalleryMetadata>(it)
}
fun exGalleryBook() = Paper.book("gallery-ex")!!
fun pervEdenGalleryBook() = Paper.book("gallery-perveden")!!
fun nhentaiGalleryBook() = Paper.book("gallery-nhentai")!!
}

View File

@ -4,7 +4,6 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.all.EHentai import eu.kanade.tachiyomi.source.online.all.EHentai
import eu.kanade.tachiyomi.source.online.all.EHentaiMetadata
import eu.kanade.tachiyomi.source.online.all.PervEden import eu.kanade.tachiyomi.source.online.all.PervEden
import exh.metadata.models.* import exh.metadata.models.*
import exh.plusAssign import exh.plusAssign
@ -51,12 +50,12 @@ fun ExGalleryMetadata.copyTo(manga: SManga) {
titleObj?.let { manga.title = it } titleObj?.let { manga.title = it }
//Set artist (if we can find one) //Set artist (if we can find one)
tags[EH_ARTIST_NAMESPACE]?.let { tags.filter { it.namespace == EH_ARTIST_NAMESPACE }.let {
if(it.isNotEmpty()) manga.artist = it.joinToString(transform = Tag::name) if(it.isNotEmpty()) manga.artist = it.joinToString(transform = { it.name!! })
} }
//Set author (if we can find one) //Set author (if we can find one)
tags[EH_AUTHOR_NAMESPACE]?.let { tags.filter { it.namespace == EH_AUTHOR_NAMESPACE }.let {
if(it.isNotEmpty()) manga.author = it.joinToString(transform = Tag::name) if(it.isNotEmpty()) manga.author = it.joinToString(transform = { it.name!! })
} }
//Set genre //Set genre
genre?.let { manga.genre = it } genre?.let { manga.genre = it }
@ -159,12 +158,12 @@ fun NHentaiMetadata.copyTo(manga: SManga) {
manga.title = englishTitle ?: japaneseTitle ?: shortTitle!! manga.title = englishTitle ?: japaneseTitle ?: shortTitle!!
//Set artist (if we can find one) //Set artist (if we can find one)
tags[NHENTAI_ARTIST_NAMESPACE]?.let { tags.filter { it.namespace == NHENTAI_ARTIST_NAMESPACE }.let {
if(it.isNotEmpty()) manga.artist = it.joinToString(transform = Tag::name) if(it.isNotEmpty()) manga.artist = it.joinToString(transform = { it.name!! })
} }
tags[NHENTAI_CATEGORIES_NAMESPACE]?.let { tags.filter { it.namespace == NHENTAI_CATEGORIES_NAMESPACE }.let {
if(it.isNotEmpty()) manga.genre = it.joinToString(transform = Tag::name) if(it.isNotEmpty()) manga.genre = it.joinToString(transform = { it.name!! })
} }
//Try to automatically identify if it is ongoing, we try not to be too lenient here to avoid making mistakes //Try to automatically identify if it is ongoing, we try not to be too lenient here to avoid making mistakes
@ -209,7 +208,9 @@ fun SearchableGalleryMetadata.genericCopyTo(manga: SManga): Boolean {
private fun buildTagsDescription(metadata: SearchableGalleryMetadata) private fun buildTagsDescription(metadata: SearchableGalleryMetadata)
= StringBuilder("Tags:\n").apply { = StringBuilder("Tags:\n").apply {
//BiConsumer only available in Java 8, don't bother calling forEach directly on 'tags' //BiConsumer only available in Java 8, don't bother calling forEach directly on 'tags'
metadata.tags.entries.forEach { namespace, tags -> metadata.tags.groupBy {
it.namespace
}.entries.forEach { namespace, tags ->
if (tags.isNotEmpty()) { if (tags.isNotEmpty()) {
val joinedTags = tags.joinToString(separator = " ", transform = { "<${it.name}>" }) val joinedTags = tags.joinToString(separator = " ", transform = { "<${it.name}>" })
this += "$namespace: $joinedTags\n" this += "$namespace: $joinedTags\n"

View File

@ -1,22 +1,43 @@
package exh.metadata.models package exh.metadata.models
import android.net.Uri import android.net.Uri
import io.realm.RealmList
import io.realm.RealmObject
import io.realm.annotations.Ignore
import io.realm.annotations.Index
import io.realm.annotations.PrimaryKey
import io.realm.annotations.RealmClass
import java.util.* import java.util.*
/** /**
* Gallery metadata storage model * Gallery metadata storage model
*/ */
class ExGalleryMetadata : SearchableGalleryMetadata() { @RealmClass
open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
@PrimaryKey
override var uuid: String = UUID.randomUUID().toString()
var url: String? = null var url: String? = null
@Index
var gId: String? = null
@Index
var gToken: String? = null
@Index
var exh: Boolean? = null var exh: Boolean? = null
var thumbnailUrl: String? = null var thumbnailUrl: String? = null
@Index
var title: String? = null var title: String? = null
@Index
var altTitle: String? = null var altTitle: String? = null
@Index
override var uploader: String? = null
var genre: String? = null var genre: String? = null
var datePosted: Long? = null var datePosted: Long? = null
@ -30,22 +51,26 @@ class ExGalleryMetadata : SearchableGalleryMetadata() {
var ratingCount: Int? = null var ratingCount: Int? = null
var averageRating: Double? = null var averageRating: Double? = null
override var tags: RealmList<Tag> = RealmList()
override fun getTitles() = listOf(title, altTitle).filterNotNull() override fun getTitles() = listOf(title, altTitle).filterNotNull()
private fun splitGalleryUrl() @Ignore
= url?.let { override val titleFields = listOf(
Uri.parse(it).pathSegments.filterNot(String::isNullOrBlank) ExGalleryMetadata::title.name,
} ExGalleryMetadata::altTitle.name
)
fun galleryId() = splitGalleryUrl()?.let { it[it.size - 2] } companion object {
private fun splitGalleryUrl(url: String)
= url.let {
Uri.parse(it).pathSegments
.filterNot(String::isNullOrBlank)
}
fun galleryToken() = fun galleryId(url: String) = splitGalleryUrl(url).let { it[it.size - 2] }
splitGalleryUrl()?.last()
override fun galleryUniqueIdentifier() = exh?.let { exh -> fun galleryToken(url: String) =
url?.let { splitGalleryUrl(url).last()
//Fuck, this should be EXH and EH but it's too late to change it now...
"${if(exh) "EXH" else "EX"}-${galleryId()}-${galleryToken()}"
}
} }
} }

View File

@ -1,40 +1,64 @@
package exh.metadata.models package exh.metadata.models
import io.realm.RealmList
import io.realm.RealmObject
import io.realm.annotations.Ignore
import io.realm.annotations.Index
import io.realm.annotations.PrimaryKey
import io.realm.annotations.RealmClass
import java.util.*
/** /**
* NHentai metadata * NHentai metadata
*/ */
class NHentaiMetadata : SearchableGalleryMetadata() { @RealmClass
open class NHentaiMetadata : RealmObject(), SearchableGalleryMetadata {
@PrimaryKey
override var uuid: String = UUID.randomUUID().toString()
var id: Long? = null var nhId: Long? = null
var url get() = id?.let { "$BASE_URL/g/$it" } var url get() = nhId?.let { "$BASE_URL/g/$it" }
set(a) { set(a) {
a?.let { a?.let {
id = a.split("/").last { it.isNotBlank() }.toLong() nhId = nhIdFromUrl(a)
} }
} }
@Index
override var uploader: String? = null
var uploadDate: Long? = null var uploadDate: Long? = null
var favoritesCount: Long? = null var favoritesCount: Long? = null
var mediaId: String? = null var mediaId: String? = null
@Index
var japaneseTitle: String? = null var japaneseTitle: String? = null
@Index
var englishTitle: String? = null var englishTitle: String? = null
@Index
var shortTitle: String? = null var shortTitle: String? = null
var coverImageType: String? = null var coverImageType: String? = null
var pageImageTypes: MutableList<String> = mutableListOf() var pageImageTypes: RealmList<PageImageType> = RealmList()
var thumbnailImageType: String? = null var thumbnailImageType: String? = null
var scanlator: String? = null var scanlator: String? = null
override fun galleryUniqueIdentifier(): String? = "NHENTAI-$id" override var tags: RealmList<Tag> = RealmList()
override fun getTitles() = listOf(japaneseTitle, englishTitle, shortTitle).filterNotNull() override fun getTitles() = listOf(japaneseTitle, englishTitle, shortTitle).filterNotNull()
@Ignore
override val titleFields = listOf(
NHentaiMetadata::japaneseTitle.name,
NHentaiMetadata::englishTitle.name,
NHentaiMetadata::shortTitle.name
)
companion object { companion object {
val BASE_URL = "https://nhentai.net" val BASE_URL = "https://nhentai.net"
@ -44,5 +68,27 @@ class NHentaiMetadata : SearchableGalleryMetadata() {
"j" -> "jpg" "j" -> "jpg"
else -> null else -> null
} }
fun nhIdFromUrl(url: String)
= url.split("/").last { it.isNotBlank() }.toLong()
} }
} }
@RealmClass
open class PageImageType(var type: String? = null): RealmObject() {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as PageImageType
if (type != other.type) return false
return true
}
override fun hashCode() = type?.hashCode() ?: 0
override fun toString() = "PageImageType(type=$type)"
}

View File

@ -1,14 +1,33 @@
package exh.metadata.models package exh.metadata.models
import android.net.Uri import android.net.Uri
import io.realm.RealmList
import io.realm.RealmObject
import io.realm.annotations.Ignore
import io.realm.annotations.Index
import io.realm.annotations.PrimaryKey
import io.realm.annotations.RealmClass
import java.util.*
@RealmClass
open class PervEdenGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
@PrimaryKey
override var uuid: String = UUID.randomUUID().toString()
@Index
var pvId: String? = null
class PervEdenGalleryMetadata : SearchableGalleryMetadata() {
var url: String? = null var url: String? = null
var thumbnailUrl: String? = null var thumbnailUrl: String? = null
@Index
var title: String? = null var title: String? = null
var altTitles: MutableList<String> = mutableListOf() var altTitles: RealmList<PervEdenTitle> = RealmList()
@Index
override var uploader: String? = null
@Index
var artist: String? = null var artist: String? = null
var type: String? = null var type: String? = null
@ -19,14 +38,48 @@ class PervEdenGalleryMetadata : SearchableGalleryMetadata() {
var lang: String? = null var lang: String? = null
override fun getTitles() = listOf(title).plus(altTitles).filterNotNull() override var tags: RealmList<Tag> = RealmList()
private fun splitGalleryUrl() override fun getTitles() = listOf(title).plus(altTitles.map {
= url?.let { it.title
Uri.parse(it).pathSegments.filterNot(String::isNullOrBlank) }).filterNotNull()
}
override fun galleryUniqueIdentifier() = splitGalleryUrl()?.let { @Ignore
"PERVEDEN-${lang?.toUpperCase()}-${it.last()}" override val titleFields = listOf(
//TODO Somehow include altTitles
PervEdenGalleryMetadata::title.name
)
companion object {
private fun splitGalleryUrl(url: String)
= url.let {
Uri.parse(it).pathSegments.filterNot(String::isNullOrBlank)
}
fun pvIdFromUrl(url: String) = splitGalleryUrl(url).last()
} }
} }
@RealmClass
open class PervEdenTitle(var metadata: PervEdenGalleryMetadata? = null,
@Index var title: String? = null): RealmObject() {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as PervEdenTitle
if (metadata != other.metadata) return false
if (title != other.title) return false
return true
}
override fun hashCode(): Int {
var result = metadata?.hashCode() ?: 0
result = 31 * result + (title?.hashCode() ?: 0)
return result
}
override fun toString() = "PervEdenTitle(metadata=$metadata, title=$title)"
}

View File

@ -1,18 +1,24 @@
package exh.metadata.models package exh.metadata.models
import io.realm.RealmList
import io.realm.RealmModel
import io.realm.annotations.Index
import java.util.ArrayList import java.util.ArrayList
import java.util.HashMap import java.util.HashMap
import kotlin.reflect.KCallable
/** /**
* A gallery that can be searched using the EH search engine * A gallery that can be searched using the EH search engine
*/ */
abstract class SearchableGalleryMetadata { interface SearchableGalleryMetadata: RealmModel {
var uploader: String? = null var uuid: String
var uploader: String?
//Being specific about which classes are used in generics to make deserialization easier //Being specific about which classes are used in generics to make deserialization easier
val tags: HashMap<String, ArrayList<Tag>> = HashMap() var tags: RealmList<Tag>
abstract fun galleryUniqueIdentifier(): String? fun getTitles(): List<String>
abstract fun getTitles(): List<String> val titleFields: List<String>
} }

View File

@ -1,7 +1,36 @@
package exh.metadata.models package exh.metadata.models
import io.realm.RealmObject
import io.realm.annotations.Index
import io.realm.annotations.RealmClass
/** /**
* Simple tag model * Simple tag model
*/ */
data class Tag(var name: String, var light: Boolean) @RealmClass
open class Tag(@Index var namespace: String? = null,
@Index var name: String? = null,
var light: Boolean? = null): RealmObject() {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as Tag
if (namespace != other.namespace) return false
if (name != other.name) return false
if (light != other.light) return false
return true
}
override fun hashCode(): Int {
var result = namespace?.hashCode() ?: 0
result = 31 * result + (name?.hashCode() ?: 0)
result = 31 * result + (light?.hashCode() ?: 0)
return result
}
override fun toString() = "Tag(namespace=$namespace, name=$name, light=$light)"
}

View File

@ -2,74 +2,94 @@ package exh.search
import exh.metadata.models.SearchableGalleryMetadata import exh.metadata.models.SearchableGalleryMetadata
import exh.metadata.models.Tag import exh.metadata.models.Tag
import exh.util.beginLog
import io.realm.Case
import io.realm.RealmResults
class SearchEngine { class SearchEngine {
private val queryCache = mutableMapOf<String, List<QueryComponent>>() private val queryCache = mutableMapOf<String, List<QueryComponent>>()
fun matches(metadata: SearchableGalleryMetadata, query: List<QueryComponent>): Boolean { fun filterResults(metadata: RealmResults<out SearchableGalleryMetadata>, query: List<QueryComponent>):
RealmResults<out SearchableGalleryMetadata> {
val first = metadata.firstOrNull() ?: return metadata
val rQuery = metadata.where()//.beginLog(SearchableGalleryMetadata::class.java)
var queryEmpty = true
fun matchTagList(tags: Sequence<Tag>, fun matchTagList(namespace: String?,
component: Text): Boolean { component: Text?,
//Match tags excluded: Boolean) {
val tagMatcher = if(!component.exact) if(excluded)
component.asLenientRegex() rQuery.not()
else if (queryEmpty)
queryEmpty = false
else else
component.asRegex() rQuery.or()
//Match beginning of tag
if (tags.find {
tagMatcher.testExact(it.name)
} != null) {
if(component.excluded) return false
} else {
//No tag matched for this component
return false
}
return true
}
val cachedTitles = metadata.getTitles().map(String::toLowerCase) rQuery.beginGroup()
//Match namespace if specified
namespace?.let {
rQuery.equalTo("${SearchableGalleryMetadata::tags.name}.${Tag::namespace.name}",
it,
Case.INSENSITIVE)
}
//Match tag name if specified
component?.let {
rQuery.beginGroup()
val q = if (!it.exact)
it.asLenientTagQueries()
else
listOf(it.asQuery())
q.forEachIndexed { index, s ->
if(index > 0)
rQuery.or()
rQuery.like("${SearchableGalleryMetadata::tags.name}.${Tag::name.name}", s, Case.INSENSITIVE)
}
rQuery.endGroup()
}
rQuery.endGroup()
}
for(component in query) { for(component in query) {
if(component is Text) { if(component is Text) {
if(component.excluded)
rQuery.not()
rQuery.beginGroup()
//Match title //Match title
if (cachedTitles.find { component.asRegex().test(it) } != null) { first.titleFields
continue .forEachIndexed { index, s ->
} queryEmpty = false
if(index > 0)
rQuery.or()
rQuery.like(s, component.asLenientTitleQuery(), Case.INSENSITIVE)
}
//Match tags //Match tags
if(!matchTagList(metadata.tags.entries.asSequence().flatMap { it.value.asSequence() }, matchTagList(null, component, false) //We already deal with exclusions here
component)) return false rQuery.endGroup()
} else if(component is Namespace) { } else if(component is Namespace) {
if(component.namespace == "uploader") { if(component.namespace == "uploader") {
queryEmpty = false
//Match uploader //Match uploader
if(!component.tag?.rawTextOnly().equals(metadata.uploader, rQuery.equalTo(SearchableGalleryMetadata::uploader.name,
ignoreCase = true)) { component.tag!!.rawTextOnly(),
return false Case.INSENSITIVE)
}
} else { } else {
if(component.tag!!.components.size > 0) { if(component.tag!!.components.size > 0) {
//Match namespace //Match namespace + tags
val ns = metadata.tags.entries.asSequence().filter { matchTagList(component.namespace, component.tag!!, component.tag!!.excluded)
it.key == component.namespace
}.flatMap { it.value.asSequence() }
//Match tags
if (!matchTagList(ns, component.tag!!))
return false
} else { } else {
//Perform namespace search //Perform namespace search
val hasNs = metadata.tags.entries.find { matchTagList(component.namespace, null, component.excluded)
it.key == component.namespace
} != null
if(hasNs && component.excluded)
return false
else if(!hasNs && !component.excluded)
return false
} }
} }
} }
} }
return true return rQuery.findAll()
} }
fun parseQuery(query: String) = queryCache.getOrPut(query, { fun parseQuery(query: String) = queryCache.getOrPut(query, {

View File

@ -1,36 +1,51 @@
package exh.search package exh.search
import exh.anyChar import exh.plusAssign
import ru.lanwen.verbalregex.VerbalExpression
class Text: QueryComponent() { class Text: QueryComponent() {
val components = mutableListOf<TextComponent>() val components = mutableListOf<TextComponent>()
private var regex: VerbalExpression? = null private var query: String? = null
private var lenientRegex: VerbalExpression? = null private var lenientTitleQuery: String? = null
private var lenientTagQueries: List<String>? = null
private var rawText: String? = null private var rawText: String? = null
fun asRegex(): VerbalExpression { fun asQuery(): String {
if(regex == null) { if(query == null) {
regex = baseBuilder().build() query = rBaseBuilder().toString()
} }
return regex!! return query!!
} }
fun asLenientRegex(): VerbalExpression { fun asLenientTitleQuery(): String {
if(lenientRegex == null) { if(lenientTitleQuery == null) {
lenientRegex = baseBuilder().anything().build() lenientTitleQuery = StringBuilder("*").append(rBaseBuilder()).append("*").toString()
} }
return lenientRegex!! return lenientTitleQuery!!
} }
fun baseBuilder(): VerbalExpression.Builder { fun asLenientTagQueries(): List<String> {
val builder = VerbalExpression.regex() if(lenientTagQueries == null) {
lenientTagQueries = listOf(
//Match beginning of tag
rBaseBuilder().append("*").toString(),
//Tag word matcher (that matches multiple words)
//Can't make it match a single word in Realm :(
StringBuilder(" ").append(rBaseBuilder()).append(" ").toString(),
StringBuilder(" ").append(rBaseBuilder()).toString(),
rBaseBuilder().append(" ").toString()
)
}
return lenientTagQueries!!
}
fun rBaseBuilder(): StringBuilder {
val builder = StringBuilder()
for(component in components) { for(component in components) {
when(component) { when(component) {
is StringTextComponent -> builder.then(component.value) is StringTextComponent -> builder += component.value
is SingleWildcard -> builder.anyChar() is SingleWildcard -> builder += "?"
is MultiWildcard -> builder.anything() is MultiWildcard -> builder += "*"
} }
} }
return builder return builder

View File

@ -9,20 +9,18 @@ import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.source.SourceManager import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.online.all.EHentai
import exh.isExSource import exh.isExSource
import exh.isLewdSource import exh.isLewdSource
import exh.metadata.MetadataHelper
import exh.metadata.copyTo
import exh.metadata.genericCopyTo import exh.metadata.genericCopyTo
import exh.metadata.queryMetadataFromManga
import exh.util.defRealm
import exh.util.realmTrans
import timber.log.Timber import timber.log.Timber
import uy.kohesive.injekt.injectLazy import uy.kohesive.injekt.injectLazy
import kotlin.concurrent.thread import kotlin.concurrent.thread
class MetadataFetchDialog { class MetadataFetchDialog {
val metadataHelper by lazy { MetadataHelper() }
val db: DatabaseHelper by injectLazy() val db: DatabaseHelper by injectLazy()
val sourceManager: SourceManager by injectLazy() val sourceManager: SourceManager by injectLazy()
@ -42,43 +40,45 @@ class MetadataFetchDialog {
.show() .show()
thread { thread {
db.deleteMangasNotInLibrary().executeAsBlocking() defRealm { realm ->
db.deleteMangasNotInLibrary().executeAsBlocking()
val libraryMangas = db.getLibraryMangas() val libraryMangas = db.getLibraryMangas()
.executeAsBlocking() .executeAsBlocking()
.filter { .filter {
isLewdSource(it.source) isLewdSource(it.source)
&& metadataHelper.fetchMetadata(it.url, it.source) == null && realm.queryMetadataFromManga(it).findFirst() == null
} }
context.runOnUiThread {
progressDialog.maxProgress = libraryMangas.size
}
//Actual metadata fetch code
libraryMangas.forEachIndexed { i, manga ->
context.runOnUiThread { context.runOnUiThread {
progressDialog.setContent("Processing: ${manga.title}") progressDialog.maxProgress = libraryMangas.size
progressDialog.setProgress(i + 1)
} }
try {
val source = sourceManager.get(manga.source) //Actual metadata fetch code
source?.let { libraryMangas.forEachIndexed { i, manga ->
manga.copyFrom(it.fetchMangaDetails(manga).toBlocking().first()) context.runOnUiThread {
metadataHelper.fetchMetadata(manga.url, manga.source)?.genericCopyTo(manga) progressDialog.setContent("Processing: ${manga.title}")
progressDialog.setProgress(i + 1)
}
try {
val source = sourceManager.get(manga.source)
source?.let {
manga.copyFrom(it.fetchMangaDetails(manga).toBlocking().first())
realm.queryMetadataFromManga(manga).findFirst()?.genericCopyTo(manga)
}
} catch (t: Throwable) {
Timber.e(t, "Could not migrate manga!")
} }
} catch(t: Throwable) {
Timber.e(t, "Could not migrate manga!")
} }
}
context.runOnUiThread { context.runOnUiThread {
progressDialog.dismiss() progressDialog.dismiss()
//Enable orientation changes again //Enable orientation changes again
context.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_NOSENSOR context.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_NOSENSOR
displayMigrationComplete(context) displayMigrationComplete(context)
}
} }
} }
} }
@ -106,7 +106,7 @@ class MetadataFetchDialog {
.cancelable(false) .cancelable(false)
.canceledOnTouchOutside(false) .canceledOnTouchOutside(false)
.dismissListener { .dismissListener {
preferenceHelper.migrateLibraryAsked().set(true) preferenceHelper.migrateLibraryAsked2().set(true)
}.show() }.show()
} }
} }

View File

@ -6,7 +6,8 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.data.preference.getOrDefault
import exh.isExSource import exh.isExSource
import exh.isLewdSource import exh.isLewdSource
import exh.metadata.MetadataHelper import exh.metadata.ehMetaQueryFromUrl
import exh.util.realmTrans
import uy.kohesive.injekt.injectLazy import uy.kohesive.injekt.injectLazy
class UrlMigrator { class UrlMigrator {
@ -14,8 +15,6 @@ class UrlMigrator {
private val prefs: PreferencesHelper by injectLazy() private val prefs: PreferencesHelper by injectLazy()
private val metadataHelper: MetadataHelper by lazy { MetadataHelper() }
fun perform() { fun perform() {
db.inTransaction { db.inTransaction {
val dbMangas = db.getMangas() val dbMangas = db.getMangas()
@ -39,33 +38,34 @@ class UrlMigrator {
//Sort possible dups so we can use binary search on it //Sort possible dups so we can use binary search on it
possibleDups.sortBy { it.url } possibleDups.sortBy { it.url }
badMangas.forEach { manga -> realmTrans { realm ->
//Build fixed URL badMangas.forEach { manga ->
val urlWithSlash = "/" + manga.url //Build fixed URL
//Fix metadata if required val urlWithSlash = "/" + manga.url
val metadata = metadataHelper.fetchEhMetadata(manga.url, isExSource(manga.source)) //Fix metadata if required
metadata?.url?.let { val metadata = realm.ehMetaQueryFromUrl(manga.url, isExSource(manga.source)).findFirst()
if(it.startsWith("g/")) { //Check if metadata URL has no slash metadata?.url?.let {
metadata.url = urlWithSlash //Fix it if (it.startsWith("g/")) { //Check if metadata URL has no slash
metadataHelper.writeGallery(metadata, manga.source) //Write new metadata to disk metadata.url = urlWithSlash //Fix it
}
} }
} //If we have a dup (with the fixed url), use the dup instead
//If we have a dup (with the fixed url), use the dup instead val possibleDup = possibleDups.binarySearchBy(urlWithSlash, selector = { it.url })
val possibleDup = possibleDups.binarySearchBy(urlWithSlash, selector = { it.url }) if (possibleDup >= 0) {
if(possibleDup >= 0) { //Make sure it is favorited if we are
//Make sure it is favorited if we are if (manga.favorite) {
if(manga.favorite) { val dup = possibleDups[possibleDup]
val dup = possibleDups[possibleDup] dup.favorite = true
dup.favorite = true db.insertManga(dup).executeAsBlocking() //Update DB with changes
db.insertManga(dup).executeAsBlocking() //Update DB with changes }
//Delete ourself (but the dup is still there)
db.deleteManga(manga).executeAsBlocking()
return@forEach
} }
//Delete ourself (but the dup is still there) //No dup, correct URL and reinsert ourselves
db.deleteManga(manga).executeAsBlocking() manga.url = urlWithSlash
return@forEach db.insertManga(manga).executeAsBlocking()
} }
//No dup, correct URL and reinsert ourselves
manga.url = urlWithSlash
db.insertManga(manga).executeAsBlocking()
} }
} }
} }

View File

@ -0,0 +1,550 @@
package exh.util
import io.realm.*
import java.util.*
/**
* Realm query with logging
*
* @author nulldev
*/
inline fun <reified E : RealmModel> RealmQuery<out E>.beginLog(clazz: Class<out E>? =
E::class.java): LoggingRealmQuery<out E>
= LoggingRealmQuery.fromQuery(this, clazz)
class LoggingRealmQuery<E : RealmModel>(val query: RealmQuery<E>) {
companion object {
fun <E : RealmModel> fromQuery(q: RealmQuery<out E>, clazz: Class<out E>?)
= LoggingRealmQuery(q).apply {
log += "SELECT * FROM ${clazz?.name ?: "???"} WHERE"
}
}
private val log = mutableListOf<String>()
private fun sec(section: String) = "{$section}"
fun log() = log.joinToString(separator = " ")
fun isValid(): Boolean {
return query.isValid
}
fun isNull(fieldName: String): RealmQuery<E> {
log += sec("\"$fieldName\" IS NULL")
return query.isNull(fieldName)
}
fun isNotNull(fieldName: String): RealmQuery<E> {
log += sec("\"$fieldName\" IS NOT NULL")
return query.isNotNull(fieldName)
}
private fun appendEqualTo(fieldName: String, value: String, casing: Case? = null) {
log += sec("\"$fieldName\" == \"$value\"" + (casing?.let {
" CASE ${casing.name}"
} ?: ""))
}
fun equalTo(fieldName: String, value: String): RealmQuery<E> {
appendEqualTo(fieldName, value)
return query.equalTo(fieldName, value)
}
fun equalTo(fieldName: String, value: String, casing: Case): RealmQuery<E> {
appendEqualTo(fieldName, value, casing)
return query.equalTo(fieldName, value, casing)
}
fun equalTo(fieldName: String, value: Byte?): RealmQuery<E> {
appendEqualTo(fieldName, value.toString())
return query.equalTo(fieldName, value)
}
fun equalTo(fieldName: String, value: ByteArray): RealmQuery<E> {
appendEqualTo(fieldName, value.toString())
return query.equalTo(fieldName, value)
}
fun equalTo(fieldName: String, value: Short?): RealmQuery<E> {
appendEqualTo(fieldName, value.toString())
return query.equalTo(fieldName, value)
}
fun equalTo(fieldName: String, value: Int?): RealmQuery<E> {
appendEqualTo(fieldName, value.toString())
return query.equalTo(fieldName, value)
}
fun equalTo(fieldName: String, value: Long?): RealmQuery<E> {
appendEqualTo(fieldName, value.toString())
return query.equalTo(fieldName, value)
}
fun equalTo(fieldName: String, value: Double?): RealmQuery<E> {
appendEqualTo(fieldName, value.toString())
return query.equalTo(fieldName, value)
}
fun equalTo(fieldName: String, value: Float?): RealmQuery<E> {
appendEqualTo(fieldName, value.toString())
return query.equalTo(fieldName, value)
}
fun equalTo(fieldName: String, value: Boolean?): RealmQuery<E> {
appendEqualTo(fieldName, value.toString())
return query.equalTo(fieldName, value)
}
fun equalTo(fieldName: String, value: Date): RealmQuery<E> {
appendEqualTo(fieldName, value.toString())
return query.equalTo(fieldName, value)
}
fun appendIn(fieldName: String, values: Array<out Any?>, casing: Case? = null) {
log += sec("[${values.joinToString(separator = ", ", transform = {
"\"$it\""
})}] IN \"$fieldName\"" + (casing?.let {
" CASE ${casing.name}"
} ?: ""))
}
fun `in`(fieldName: String, values: Array<String>): RealmQuery<E> {
appendIn(fieldName, values)
return query.`in`(fieldName, values)
}
fun `in`(fieldName: String, values: Array<String>, casing: Case): RealmQuery<E> {
appendIn(fieldName, values, casing)
return query.`in`(fieldName, values, casing)
}
fun `in`(fieldName: String, values: Array<Byte>): RealmQuery<E> {
appendIn(fieldName, values)
return query.`in`(fieldName, values)
}
fun `in`(fieldName: String, values: Array<Short>): RealmQuery<E> {
appendIn(fieldName, values)
return query.`in`(fieldName, values)
}
fun `in`(fieldName: String, values: Array<Int>): RealmQuery<E> {
appendIn(fieldName, values)
return query.`in`(fieldName, values)
}
fun `in`(fieldName: String, values: Array<Long>): RealmQuery<E> {
appendIn(fieldName, values)
return query.`in`(fieldName, values)
}
fun `in`(fieldName: String, values: Array<Double>): RealmQuery<E> {
appendIn(fieldName, values)
return query.`in`(fieldName, values)
}
fun `in`(fieldName: String, values: Array<Float>): RealmQuery<E> {
appendIn(fieldName, values)
return query.`in`(fieldName, values)
}
fun `in`(fieldName: String, values: Array<Boolean>): RealmQuery<E> {
appendIn(fieldName, values)
return query.`in`(fieldName, values)
}
fun `in`(fieldName: String, values: Array<Date>): RealmQuery<E> {
appendIn(fieldName, values)
return query.`in`(fieldName, values)
}
private fun appendNotEqualTo(fieldName: String, value: Any?, casing: Case? = null) {
log += sec("\"$fieldName\" != \"$value\"" + (casing?.let {
" CASE ${casing.name}"
} ?: ""))
}
fun notEqualTo(fieldName: String, value: String): RealmQuery<E> {
appendNotEqualTo(fieldName, value)
return query.notEqualTo(fieldName, value)
}
fun notEqualTo(fieldName: String, value: String, casing: Case): RealmQuery<E> {
appendNotEqualTo(fieldName, value, casing)
return query.notEqualTo(fieldName, value, casing)
}
fun notEqualTo(fieldName: String, value: Byte?): RealmQuery<E> {
appendNotEqualTo(fieldName, value)
return query.notEqualTo(fieldName, value)
}
fun notEqualTo(fieldName: String, value: ByteArray): RealmQuery<E> {
appendNotEqualTo(fieldName, value)
return query.notEqualTo(fieldName, value)
}
fun notEqualTo(fieldName: String, value: Short?): RealmQuery<E> {
appendNotEqualTo(fieldName, value)
return query.notEqualTo(fieldName, value)
}
fun notEqualTo(fieldName: String, value: Int?): RealmQuery<E> {
appendNotEqualTo(fieldName, value)
return query.notEqualTo(fieldName, value)
}
fun notEqualTo(fieldName: String, value: Long?): RealmQuery<E> {
appendNotEqualTo(fieldName, value)
return query.notEqualTo(fieldName, value)
}
fun notEqualTo(fieldName: String, value: Double?): RealmQuery<E> {
appendNotEqualTo(fieldName, value)
return query.notEqualTo(fieldName, value)
}
fun notEqualTo(fieldName: String, value: Float?): RealmQuery<E> {
appendNotEqualTo(fieldName, value)
return query.notEqualTo(fieldName, value)
}
fun notEqualTo(fieldName: String, value: Boolean?): RealmQuery<E> {
appendNotEqualTo(fieldName, value)
return query.notEqualTo(fieldName, value)
}
fun notEqualTo(fieldName: String, value: Date): RealmQuery<E> {
appendNotEqualTo(fieldName, value)
return query.notEqualTo(fieldName, value)
}
private fun appendGreaterThan(fieldName: String, value: Any?) {
log += sec("\"$fieldName\" > $value")
}
fun greaterThan(fieldName: String, value: Int): RealmQuery<E> {
appendGreaterThan(fieldName, value)
return query.greaterThan(fieldName, value)
}
fun greaterThan(fieldName: String, value: Long): RealmQuery<E> {
appendGreaterThan(fieldName, value)
return query.greaterThan(fieldName, value)
}
fun greaterThan(fieldName: String, value: Double): RealmQuery<E> {
appendGreaterThan(fieldName, value)
return query.greaterThan(fieldName, value)
}
fun greaterThan(fieldName: String, value: Float): RealmQuery<E> {
appendGreaterThan(fieldName, value)
return query.greaterThan(fieldName, value)
}
fun greaterThan(fieldName: String, value: Date): RealmQuery<E> {
appendGreaterThan(fieldName, value)
return query.greaterThan(fieldName, value)
}
private fun appendGreaterThanOrEqualTo(fieldName: String, value: Any?) {
log += sec("\"$fieldName\" >= $value")
}
fun greaterThanOrEqualTo(fieldName: String, value: Int): RealmQuery<E> {
appendGreaterThanOrEqualTo(fieldName, value)
return query.greaterThanOrEqualTo(fieldName, value)
}
fun greaterThanOrEqualTo(fieldName: String, value: Long): RealmQuery<E> {
appendGreaterThanOrEqualTo(fieldName, value)
return query.greaterThanOrEqualTo(fieldName, value)
}
fun greaterThanOrEqualTo(fieldName: String, value: Double): RealmQuery<E> {
appendGreaterThanOrEqualTo(fieldName, value)
return query.greaterThanOrEqualTo(fieldName, value)
}
fun greaterThanOrEqualTo(fieldName: String, value: Float): RealmQuery<E> {
appendGreaterThanOrEqualTo(fieldName, value)
return query.greaterThanOrEqualTo(fieldName, value)
}
fun greaterThanOrEqualTo(fieldName: String, value: Date): RealmQuery<E> {
appendGreaterThanOrEqualTo(fieldName, value)
return query.greaterThanOrEqualTo(fieldName, value)
}
private fun appendLessThan(fieldName: String, value: Any?) {
log += sec("\"$fieldName\" < $value")
}
fun lessThan(fieldName: String, value: Int): RealmQuery<E> {
appendLessThan(fieldName, value)
return query.lessThan(fieldName, value)
}
fun lessThan(fieldName: String, value: Long): RealmQuery<E> {
appendLessThan(fieldName, value)
return query.lessThan(fieldName, value)
}
fun lessThan(fieldName: String, value: Double): RealmQuery<E> {
appendLessThan(fieldName, value)
return query.lessThan(fieldName, value)
}
fun lessThan(fieldName: String, value: Float): RealmQuery<E> {
appendLessThan(fieldName, value)
return query.lessThan(fieldName, value)
}
fun lessThan(fieldName: String, value: Date): RealmQuery<E> {
appendLessThan(fieldName, value)
return query.lessThan(fieldName, value)
}
private fun appendLessThanOrEqualTo(fieldName: String, value: Any?) {
log += sec("\"$fieldName\" <= $value")
}
fun lessThanOrEqualTo(fieldName: String, value: Int): RealmQuery<E> {
appendLessThanOrEqualTo(fieldName, value)
return query.lessThanOrEqualTo(fieldName, value)
}
fun lessThanOrEqualTo(fieldName: String, value: Long): RealmQuery<E> {
appendLessThanOrEqualTo(fieldName, value)
return query.lessThanOrEqualTo(fieldName, value)
}
fun lessThanOrEqualTo(fieldName: String, value: Double): RealmQuery<E> {
appendLessThanOrEqualTo(fieldName, value)
return query.lessThanOrEqualTo(fieldName, value)
}
fun lessThanOrEqualTo(fieldName: String, value: Float): RealmQuery<E> {
appendLessThanOrEqualTo(fieldName, value)
return query.lessThanOrEqualTo(fieldName, value)
}
fun lessThanOrEqualTo(fieldName: String, value: Date): RealmQuery<E> {
appendLessThanOrEqualTo(fieldName, value)
return query.lessThanOrEqualTo(fieldName, value)
}
private fun appendBetween(fieldName: String, from: Any?, to: Any?) {
log += sec("\"$fieldName\" BETWEEN $from - $to")
}
fun between(fieldName: String, from: Int, to: Int): RealmQuery<E> {
appendBetween(fieldName, from, to)
return query.between(fieldName, from, to)
}
fun between(fieldName: String, from: Long, to: Long): RealmQuery<E> {
appendBetween(fieldName, from, to)
return query.between(fieldName, from, to)
}
fun between(fieldName: String, from: Double, to: Double): RealmQuery<E> {
appendBetween(fieldName, from, to)
return query.between(fieldName, from, to)
}
fun between(fieldName: String, from: Float, to: Float): RealmQuery<E> {
appendBetween(fieldName, from, to)
return query.between(fieldName, from, to)
}
fun between(fieldName: String, from: Date, to: Date): RealmQuery<E> {
appendBetween(fieldName, from, to)
return query.between(fieldName, from, to)
}
private fun appendContains(fieldName: String, value: Any?, casing: Case? = null) {
log += sec("\"$fieldName\" CONTAINS \"$value\"" + (casing?.let {
" CASE ${casing.name}"
} ?: ""))
}
fun contains(fieldName: String, value: String): RealmQuery<E> {
appendContains(fieldName, value)
return query.contains(fieldName, value)
}
fun contains(fieldName: String, value: String, casing: Case): RealmQuery<E> {
appendContains(fieldName, value, casing)
return query.contains(fieldName, value, casing)
}
private fun appendBeginsWith(fieldName: String, value: Any?, casing: Case? = null) {
log += sec("\"$fieldName\" BEGINS WITH \"$value\"" + (casing?.let {
" CASE ${casing.name}"
} ?: ""))
}
fun beginsWith(fieldName: String, value: String): RealmQuery<E> {
appendBeginsWith(fieldName, value)
return query.beginsWith(fieldName, value)
}
fun beginsWith(fieldName: String, value: String, casing: Case): RealmQuery<E> {
appendBeginsWith(fieldName, value, casing)
return query.beginsWith(fieldName, value, casing)
}
private fun appendEndsWith(fieldName: String, value: Any?, casing: Case? = null) {
log += sec("\"$fieldName\" ENDS WITH \"$value\"" + (casing?.let {
" CASE ${casing.name}"
} ?: ""))
}
fun endsWith(fieldName: String, value: String): RealmQuery<E> {
appendEndsWith(fieldName, value)
return query.endsWith(fieldName, value)
}
fun endsWith(fieldName: String, value: String, casing: Case): RealmQuery<E> {
appendEndsWith(fieldName, value, casing)
return query.endsWith(fieldName, value, casing)
}
private fun appendLike(fieldName: String, value: Any?, casing: Case? = null) {
log += sec("\"$fieldName\" LIKE \"$value\"" + (casing?.let {
" CASE ${casing.name}"
} ?: ""))
}
fun like(fieldName: String, value: String): RealmQuery<E> {
appendLike(fieldName, value)
return query.like(fieldName, value)
}
fun like(fieldName: String, value: String, casing: Case): RealmQuery<E> {
appendLike(fieldName, value, casing)
return query.like(fieldName, value, casing)
}
fun beginGroup(): RealmQuery<E> {
log += "("
return query.beginGroup()
}
fun endGroup(): RealmQuery<E> {
log += ")"
return query.endGroup()
}
fun or(): RealmQuery<E> {
log += "OR"
return query.or()
}
operator fun not(): RealmQuery<E> {
log += "NOT"
return query.not()
}
fun isEmpty(fieldName: String): RealmQuery<E> {
log += "\"$fieldName\" IS EMPTY"
return query.isEmpty(fieldName)
}
fun isNotEmpty(fieldName: String): RealmQuery<E> {
log += "\"$fieldName\" IS NOT EMPTY"
return query.isNotEmpty(fieldName)
}
fun distinct(fieldName: String): RealmResults<E> {
return query.distinct(fieldName)
}
fun distinctAsync(fieldName: String): RealmResults<E> {
return query.distinctAsync(fieldName)
}
fun distinct(firstFieldName: String, vararg remainingFieldNames: String): RealmResults<E> {
return query.distinct(firstFieldName, *remainingFieldNames)
}
fun sum(fieldName: String): Number {
return query.sum(fieldName)
}
fun average(fieldName: String): Double {
return query.average(fieldName)
}
fun min(fieldName: String): Number {
return query.min(fieldName)
}
fun minimumDate(fieldName: String): Date {
return query.minimumDate(fieldName)
}
fun max(fieldName: String): Number {
return query.max(fieldName)
}
fun maximumDate(fieldName: String): Date {
return query.maximumDate(fieldName)
}
fun count(): Long {
return query.count()
}
fun findAll(): RealmResults<E> {
return query.findAll()
}
fun findAllAsync(): RealmResults<E> {
return query.findAllAsync()
}
fun findAllSorted(fieldName: String, sortOrder: Sort): RealmResults<E> {
return query.findAllSorted(fieldName, sortOrder)
}
fun findAllSortedAsync(fieldName: String, sortOrder: Sort): RealmResults<E> {
return query.findAllSortedAsync(fieldName, sortOrder)
}
fun findAllSorted(fieldName: String): RealmResults<E> {
return query.findAllSorted(fieldName)
}
fun findAllSortedAsync(fieldName: String): RealmResults<E> {
return query.findAllSortedAsync(fieldName)
}
fun findAllSorted(fieldNames: Array<String>, sortOrders: Array<Sort>): RealmResults<E> {
return query.findAllSorted(fieldNames, sortOrders)
}
fun findAllSortedAsync(fieldNames: Array<String>, sortOrders: Array<Sort>): RealmResults<E> {
return query.findAllSortedAsync(fieldNames, sortOrders)
}
fun findAllSorted(fieldName1: String, sortOrder1: Sort, fieldName2: String, sortOrder2: Sort): RealmResults<E> {
return query.findAllSorted(fieldName1, sortOrder1, fieldName2, sortOrder2)
}
fun findAllSortedAsync(fieldName1: String, sortOrder1: Sort, fieldName2: String, sortOrder2: Sort): RealmResults<E> {
return query.findAllSortedAsync(fieldName1, sortOrder1, fieldName2, sortOrder2)
}
fun findFirst(): E {
return query.findFirst()
}
fun findFirstAsync(): E {
return query.findFirstAsync()
}
}

View File

@ -0,0 +1,39 @@
package exh.util
import io.realm.Realm
import io.realm.RealmModel
import io.realm.log.RealmLog
import java.util.*
inline fun <T> realmTrans(block: (Realm) -> T): T {
return defRealm {
it.beginTransaction()
try {
val res = block(it)
it.commitTransaction()
res
} catch(t: Throwable) {
if (it.isInTransaction) {
it.cancelTransaction()
} else {
RealmLog.warn("Could not cancel transaction, not currently in a transaction.")
}
throw t
} finally {
//Just in case
if (it.isInTransaction) {
it.cancelTransaction()
}
}
}
}
inline fun <T> defRealm(block: (Realm) -> T): T {
return Realm.getDefaultInstance().use {
block(it)
}
}
fun <T : RealmModel> Realm.createUUIDObj(clazz: Class<T>)
= createObject(clazz, UUID.randomUUID().toString())

View File

@ -1,6 +1,20 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<changelog bulletedList="true"> <changelog bulletedList="true">
<changelogversion versionName="v6.1.1-EH" changeDate="">
<changelogtext>EH - Rewrite batch add screen</changelogtext>
<changelogtext>EH - Add nhentai link import support</changelogtext>
<changelogtext>EH - Add the ability to import links by searching them in the catalogues</changelogtext>
<changelogtext>EH - Increase library tag search speed</changelogtext>
<changelogtext>EH - Rewrite app lock UI</changelogtext>
<changelogtext>EH - Add fingerprint support to app lock</changelogtext>
</changelogversion>
<changelogversion versionName="v0.6.1" changeDate=""> <changelogversion versionName="v0.6.1" changeDate="">
<changelogtext>Bugfix release.</changelogtext> <changelogtext>Bugfix release.</changelogtext>
</changelogversion> </changelogversion>

View File

@ -13,6 +13,8 @@ buildscript {
//Firebase (EH) //Firebase (EH)
classpath 'com.google.gms:google-services:3.0.0' classpath 'com.google.gms:google-services:3.0.0'
//Realm (EH)
classpath "io.realm:realm-gradle-plugin:3.5.0"
} }
} }