Rewrite tag searching to use SQL

Fix EHentai/ExHentai
Fix hitomi.la
Fix hitomi.la crashing application
Rewrite hitomi.la search engine to be faster, use less CPU and require no preloading
Fix nhentai
Add additional filters to nhentai
Fix PervEden
Introduce delegated sources
Rewrite HentaiCafe to be a delegated source
Introduce ability to save/load search presets
Temporarily disable misbehaving native Tachiyomi migrations
Fix tap-to-search-tag breaking on aliased tags
Add debug menu
Add experimental automatic captcha solver
Add app name to wakelock names
Add ability to interrupt metadata migrator
Fix incognito open-in-browser being zoomed in immediately when it's opened

app/src/main/java/exh/ui/captcha/AutoSolvingWebViewClient.kt

@@ -0,0 +1,36 @@
+package exh.ui.captcha
+
+import android.os.Build
+import android.support.annotation.RequiresApi
+import android.webkit.WebResourceRequest
+import android.webkit.WebResourceResponse
+import android.webkit.WebView
+import eu.kanade.tachiyomi.util.asJsoup
+import exh.ui.captcha.SolveCaptchaActivity.Companion.CROSS_WINDOW_SCRIPT_INNER
+import org.jsoup.nodes.DataNode
+import org.jsoup.nodes.Element
+import java.nio.charset.Charset
+
+@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+class AutoSolvingWebViewClient(activity: SolveCaptchaActivity,
+                               source: CaptchaCompletionVerifier,
+                               injectScript: String?)
+    : BasicWebViewClient(activity, source, injectScript) {
+
+    override fun shouldInterceptRequest(view: WebView, request: WebResourceRequest): WebResourceResponse? {
+        // Inject our custom script into the recaptcha iframes
+        val lastPathSegment = request.url.pathSegments.lastOrNull()
+        if(lastPathSegment == "anchor" ||  lastPathSegment == "bframe") {
+            val oReq = request.toOkHttpRequest()
+            val response = activity.httpClient.newCall(oReq).execute()
+            val doc = response.asJsoup()
+            doc.body().appendChild(Element("script").appendChild(DataNode(CROSS_WINDOW_SCRIPT_INNER)))
+            return WebResourceResponse(
+                    "text/html",
+                    "UTF-8",
+                    doc.toString().byteInputStream(Charset.forName("UTF-8")).buffered()
+            )
+        }
+        return super.shouldInterceptRequest(view, request)
+    }
+}

app/src/main/java/exh/ui/captcha/BasicWebViewClient.kt

@@ -0,0 +1,18 @@
+package exh.ui.captcha
+
+import android.webkit.WebView
+import android.webkit.WebViewClient
+
+open class BasicWebViewClient(protected val activity: SolveCaptchaActivity,
+                              protected val source: CaptchaCompletionVerifier,
+                              private val injectScript: String?) : WebViewClient() {
+    override fun onPageFinished(view: WebView, url: String) {
+        super.onPageFinished(view, url)
+
+        if(source.verifyNoCaptcha(url)) {
+            activity.finish()
+        } else {
+            if(injectScript != null) view.loadUrl("javascript:(function() {$injectScript})();")
+        }
+    }
+}

app/src/main/java/exh/ui/captcha/SolveCaptchaActivity.kt

@@ -4,25 +4,48 @@ import android.content.Context
 import android.content.Intent
 import android.os.Build
 import android.os.Bundle
+import android.support.annotation.RequiresApi
 import android.support.v7.app.AppCompatActivity
-import android.webkit.CookieManager
-import android.webkit.CookieSyncManager
-import android.webkit.WebView
-import android.webkit.WebViewClient
-import eu.kanade.tachiyomi.R
+import android.webkit.*
+import com.github.salomonbrys.kotson.get
+import com.github.salomonbrys.kotson.string
+import com.google.gson.JsonParser
+import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.source.Source
 import eu.kanade.tachiyomi.source.SourceManager
 import kotlinx.android.synthetic.main.eh_activity_captcha.*
+import okhttp3.*
+import rx.Single
+import rx.schedulers.Schedulers
+import timber.log.Timber
 import uy.kohesive.injekt.injectLazy
 import java.net.URL
+import java.util.*
+import android.view.MotionEvent
+import android.os.SystemClock
+import com.afollestad.materialdialogs.MaterialDialog
+import eu.kanade.tachiyomi.data.preference.PreferencesHelper
+import eu.kanade.tachiyomi.data.preference.getOrDefault
+import exh.util.melt
+import rx.Observable
 
 class SolveCaptchaActivity : AppCompatActivity() {
     private val sourceManager: SourceManager by injectLazy()
+    private val preferencesHelper: PreferencesHelper by injectLazy()
+
+    val httpClient = OkHttpClient()
+    private val jsonParser = JsonParser()
+
+    private var currentLoopId: String? = null
+    private var validateCurrentLoopId: String? = null
+    private var strictValidationStartTime: Long? = null
+
+    lateinit var credentialsObservable: Observable<String>
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
 
-        setContentView(R.layout.eh_activity_captcha)
+        setContentView(eu.kanade.tachiyomi.R.layout.eh_activity_captcha)
 
         val sourceId = intent.getLongExtra(SOURCE_ID_EXTRA, -1)
         val source = if(sourceId != -1L)
@@ -59,18 +82,56 @@ class SolveCaptchaActivity : AppCompatActivity() {
             webview.settings.javaScriptEnabled = true
             webview.settings.domStorageEnabled = true
 
-            webview.webViewClient = object : WebViewClient() {
-                override fun onPageFinished(view: WebView, url: String) {
-                    super.onPageFinished(view, url)
+            var loadedInners = 0
 
-                    if(source.verify(url)) {
-                        finish()
-                    } else {
-                        view.loadUrl("javascript:(function() {$script})();")
+            webview.webChromeClient = object : WebChromeClient() {
+                override fun onJsAlert(view: WebView?, url: String?, message: String, result: JsResult): Boolean {
+                    if(message.startsWith("exh-")) {
+                        loadedInners++
+                        // Wait for both inner scripts to be loaded
+                        if(loadedInners >= 2) {
+                            // Attempt to autosolve captcha
+                            if(preferencesHelper.eh_autoSolveCaptchas().getOrDefault()
+                                    && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+                                webview.post {
+                                    // 10 seconds to auto-solve captcha
+                                    strictValidationStartTime = System.currentTimeMillis() + 1000 * 10
+                                    beginSolveLoop()
+                                    beginValidateCaptchaLoop()
+                                    webview.evaluateJavascript(SOLVE_UI_SCRIPT_HIDE) {
+                                        webview.evaluateJavascript(SOLVE_UI_SCRIPT_SHOW, null)
+                                    }
+                                }
+                            }
+                        }
+                        result.confirm()
+                        return true
                     }
+                    return false
                 }
             }
 
+            webview.webViewClient = if (preferencesHelper.eh_autoSolveCaptchas().getOrDefault()
+                    && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+                // Fetch auto-solve credentials early for speed
+                credentialsObservable = httpClient.newCall(Request.Builder()
+                        // Rob demo credentials
+                        .url("https://speech-to-text-demo.ng.bluemix.net/api/v1/credentials")
+                        .build())
+                        .asObservableSuccess()
+                        .subscribeOn(Schedulers.io())
+                        .map {
+                            val json = jsonParser.parse(it.body()!!.string())
+                            it.close()
+                            json["token"].string
+                        }.melt()
+
+                webview.addJavascriptInterface(this@SolveCaptchaActivity, "exh")
+                AutoSolvingWebViewClient(this, source, script)
+            } else {
+                BasicWebViewClient(this, source, script)
+            }
+
             webview.loadUrl(url)
         }
 
@@ -91,22 +152,458 @@ class SolveCaptchaActivity : AppCompatActivity() {
         return true
     }
 
+    @RequiresApi(Build.VERSION_CODES.KITKAT)
+    fun captchaSolveFail() {
+        currentLoopId = null
+        validateCurrentLoopId = null
+        Timber.e(IllegalStateException("Captcha solve failure!"))
+        runOnUiThread {
+            webview.evaluateJavascript(SOLVE_UI_SCRIPT_HIDE, null)
+            MaterialDialog.Builder(this)
+                    .title("Captcha solve failure")
+                    .content("Failed to auto-solve the captcha!")
+                    .cancelable(true)
+                    .canceledOnTouchOutside(true)
+                    .positiveText("Ok")
+                    .show()
+        }
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    @JavascriptInterface
+    fun callback(result: String?, loopId: String, stage: Int) {
+        if(loopId != currentLoopId) return
+
+        when(stage) {
+            STAGE_CHECKBOX -> {
+                if(result!!.toBoolean()) {
+                    webview.postDelayed({
+                        getAudioButtonLocation(loopId)
+                    }, 250)
+                } else {
+                    webview.postDelayed({
+                        doStageCheckbox(loopId)
+                    }, 250)
+                }
+            }
+            STAGE_GET_AUDIO_BTN_LOCATION -> {
+                if(result != null) {
+                    val splitResult = result.split(" ").map { it.toFloat() }
+                    val origX = splitResult[0]
+                    val origY = splitResult[1]
+                    val iw = splitResult[2]
+                    val ih = splitResult[3]
+                    val x = webview.x + origX / iw * webview.width
+                    val y = webview.y + origY / ih * webview.height
+                    Timber.d("Found audio button coords: %f %f", x, y)
+                    simulateClick(x + 50, y + 50)
+                    webview.post {
+                        doStageDownloadAudio(loopId)
+                    }
+                } else {
+                    webview.postDelayed({
+                        getAudioButtonLocation(loopId)
+                    }, 250)
+                }
+            }
+            STAGE_DOWNLOAD_AUDIO -> {
+                if(result != null) {
+                    Timber.d("Got audio URL: $result")
+                    performRecognize(result)
+                            .observeOn(Schedulers.io())
+                            .subscribe ({
+                                Timber.d("Got audio transcript: $it")
+                                webview.post {
+                                    typeResult(loopId, it!!
+                                            .replace(TRANSCRIPT_CLEANER_REGEX, "")
+                                            .replace(SPACE_DEDUPE_REGEX, " ")
+                                            .trim())
+                                }
+                            }, {
+                                captchaSolveFail()
+                            })
+                } else {
+                    webview.postDelayed({
+                        doStageDownloadAudio(loopId)
+                    }, 250)
+                }
+            }
+            STAGE_TYPE_RESULT -> {
+                if(result!!.toBoolean()) {
+                    // Fail if captcha still not solved after 1.5s
+                    strictValidationStartTime = System.currentTimeMillis() + 1500
+                } else {
+                    captchaSolveFail()
+                }
+            }
+        }
+    }
+
+    fun performRecognize(url: String): Single<String> {
+        return credentialsObservable.flatMap { token ->
+            httpClient.newCall(Request.Builder()
+                    .url(url)
+                    .build()).asObservableSuccess().map {
+                token to it
+            }
+        }.flatMap { (token, response) ->
+            val audioFile = response.body()!!.bytes()
+
+            httpClient.newCall(Request.Builder()
+                    .url(HttpUrl.parse("https://stream.watsonplatform.net/speech-to-text/api/v1/recognize")!!
+                            .newBuilder()
+                            .addQueryParameter("watson-token", token)
+                            .build())
+                    .post(MultipartBody.Builder()
+                            .setType(MultipartBody.FORM)
+                            .addFormDataPart("jsonDescription", RECOGNIZE_JSON)
+                            .addFormDataPart("audio.mp3",
+                                    "audio.mp3",
+                                    RequestBody.create(MediaType.parse("audio/mp3"), audioFile))
+                            .build())
+                    .build()).asObservableSuccess()
+        }.map { response ->
+            jsonParser.parse(response.body()!!.string())["results"][0]["alternatives"][0]["transcript"].string.trim()
+        }.toSingle()
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    fun doStageCheckbox(loopId: String) {
+        if(loopId != currentLoopId) return
+
+        webview.evaluateJavascript("""
+            (function() {
+                $CROSS_WINDOW_SCRIPT_OUTER
+
+                let exh_cframe = document.querySelector('iframe[role=presentation][name|=a]');
+
+                if(exh_cframe != null) {
+                    cwmExec(exh_cframe, `
+                        let exh_cb = document.getElementsByClassName('recaptcha-checkbox-checkmark')[0];
+                        if(exh_cb != null) {
+                            exh_cb.click();
+                            return "true";
+                        } else {
+                            return "false";
+                        }
+                    `, function(result) {
+                        exh.callback(result, '$loopId', $STAGE_CHECKBOX);
+                    });
+                } else {
+                    exh.callback("false", '$loopId', $STAGE_CHECKBOX);
+                }
+            })();
+        """.trimIndent().replace("\n", ""), null)
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    fun getAudioButtonLocation(loopId: String) {
+        webview.evaluateJavascript("""
+            (function() {
+                $CROSS_WINDOW_SCRIPT_OUTER
+
+                let exh_bframe = document.querySelector("iframe[title='recaptcha challenge'][name|=c]");
+
+                if(exh_bframe != null) {
+                    let bfb = exh_bframe.getBoundingClientRect();
+                    let iw = window.innerWidth;
+                    let ih = window.innerHeight;
+                    if(bfb.left < 0 || bfb.top < 0) {
+                        exh.callback(null, '$loopId', $STAGE_GET_AUDIO_BTN_LOCATION);
+                    } else {
+                        cwmExec(exh_bframe, ` let exh_ab = document.getElementById("recaptcha-audio-button");
+                            if(exh_ab != null) {
+                                let bounds = exh_ab.getBoundingClientRect();
+                                return (${'$'}{bfb.left} + bounds.left) + " " + (${'$'}{bfb.top} + bounds.top) + " " + ${'$'}{iw} + " " + ${'$'}{ih};
+                            } else {
+                                return null;
+                            }
+                        `, function(result) {
+                            exh.callback(result, '$loopId', $STAGE_GET_AUDIO_BTN_LOCATION);
+                        });
+                    }
+                } else {
+                    exh.callback(null, '$loopId', $STAGE_GET_AUDIO_BTN_LOCATION);
+                }
+            })();
+        """.trimIndent().replace("\n", ""), null)
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    fun doStageDownloadAudio(loopId: String) {
+        webview.evaluateJavascript("""
+            (function() {
+                $CROSS_WINDOW_SCRIPT_OUTER
+
+                let exh_bframe = document.querySelector("iframe[title='recaptcha challenge'][name|=c]");
+
+                if(exh_bframe != null) {
+                    cwmExec(exh_bframe, `
+                        let exh_as = document.getElementById("audio-source");
+                        if(exh_as != null) {
+                            return exh_as.src;
+                        } else {
+                            return null;
+                        }
+                    `, function(result) {
+                        exh.callback(result, '$loopId', $STAGE_DOWNLOAD_AUDIO);
+                    });
+                } else {
+                    exh.callback(null, '$loopId', $STAGE_DOWNLOAD_AUDIO);
+                }
+            })();
+        """.trimIndent().replace("\n", ""), null)
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    fun typeResult(loopId: String, result: String) {
+        webview.evaluateJavascript("""
+            (function() {
+                $CROSS_WINDOW_SCRIPT_OUTER
+
+                let exh_bframe = document.querySelector("iframe[title='recaptcha challenge'][name|=c]");
+
+                if(exh_bframe != null) {
+                    cwmExec(exh_bframe, `
+                        let exh_as = document.getElementById("audio-response");
+                        let exh_vb = document.getElementById("recaptcha-verify-button");
+                        if(exh_as != null && exh_vb != null) {
+                            exh_as.value = "$result";
+                            exh_vb.click();
+                            return "true";
+                        } else {
+                            return "false";
+                        }
+                    `, function(result) {
+                        exh.callback(result, '$loopId', $STAGE_TYPE_RESULT);
+                    });
+                } else {
+                    exh.callback("false", '$loopId', $STAGE_TYPE_RESULT);
+                }
+            })();
+        """.trimIndent().replace("\n", ""), null)
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    fun beginSolveLoop() {
+        val loopId = UUID.randomUUID().toString()
+        currentLoopId = loopId
+        doStageCheckbox(loopId)
+    }
+
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    @JavascriptInterface
+    fun validateCaptchaCallback(result: Boolean, loopId: String) {
+        if(loopId != validateCurrentLoopId) return
+
+        if(result) {
+            Timber.d("Captcha solved!")
+            webview.post {
+                webview.evaluateJavascript(SOLVE_UI_SCRIPT_HIDE, null)
+            }
+            val asbtn = intent.getStringExtra(ASBTN_EXTRA)
+            if(asbtn != null) {
+                webview.post {
+                    webview.evaluateJavascript("(function() {document.querySelector('$asbtn').click();})();", null)
+                }
+            }
+        } else {
+            val savedStrictValidationStartTime = strictValidationStartTime
+            if(savedStrictValidationStartTime != null
+                    && System.currentTimeMillis() > savedStrictValidationStartTime) {
+                captchaSolveFail()
+            } else {
+                webview.postDelayed({
+                    runValidateCaptcha(loopId)
+                }, 250)
+            }
+        }
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    fun runValidateCaptcha(loopId: String) {
+        if(loopId != validateCurrentLoopId) return
+
+        webview.evaluateJavascript("""
+            (function() {
+                $CROSS_WINDOW_SCRIPT_OUTER
+
+                let exh_cframe = document.querySelector('iframe[role=presentation][name|=a]');
+
+                if(exh_cframe != null) {
+                    cwmExec(exh_cframe, `
+                        let exh_cb = document.querySelector(".recaptcha-checkbox[aria-checked=true]");
+                        if(exh_cb != null) {
+                            return true;
+                        } else {
+                            return false;
+                        }
+                    `, function(result) {
+                        exh.validateCaptchaCallback(result, '$loopId');
+                    });
+                } else {
+                    exh.validateCaptchaCallback(false, '$loopId');
+                }
+            })();
+        """.trimIndent().replace("\n", ""), null)
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    fun beginValidateCaptchaLoop() {
+        val loopId = UUID.randomUUID().toString()
+        validateCurrentLoopId = loopId
+        runValidateCaptcha(loopId)
+    }
+
+    private fun simulateClick(x: Float, y: Float) {
+        val downTime = SystemClock.uptimeMillis()
+        val eventTime = SystemClock.uptimeMillis()
+        val properties = arrayOfNulls<MotionEvent.PointerProperties>(1)
+        val pp1 = MotionEvent.PointerProperties().apply {
+            id = 0
+            toolType = MotionEvent.TOOL_TYPE_FINGER
+        }
+        properties[0] = pp1
+        val pointerCoords = arrayOfNulls<MotionEvent.PointerCoords>(1)
+        val pc1 = MotionEvent.PointerCoords().apply {
+            this.x = x
+            this.y = y
+            pressure = 1f
+            size = 1f
+        }
+        pointerCoords[0] = pc1
+        var motionEvent = MotionEvent.obtain(downTime, eventTime, MotionEvent.ACTION_DOWN, 1, properties, pointerCoords, 0, 0, 1f, 1f, 0, 0, 0, 0)
+        dispatchTouchEvent(motionEvent)
+        motionEvent.recycle()
+        motionEvent = MotionEvent.obtain(downTime, eventTime, MotionEvent.ACTION_UP, 1, properties, pointerCoords, 0, 0, 1f, 1f, 0, 0, 0, 0)
+        dispatchTouchEvent(motionEvent)
+        motionEvent.recycle()
+    }
+
     companion object {
         const val SOURCE_ID_EXTRA = "source_id_extra"
         const val COOKIES_EXTRA = "cookies_extra"
         const val SCRIPT_EXTRA = "script_extra"
         const val URL_EXTRA = "url_extra"
+        const val ASBTN_EXTRA = "asbtn_extra"
+
+        const val STAGE_CHECKBOX = 0
+        const val STAGE_GET_AUDIO_BTN_LOCATION = 1
+        const val STAGE_DOWNLOAD_AUDIO = 2
+        const val STAGE_TYPE_RESULT = 3
+
+        val CROSS_WINDOW_SCRIPT_OUTER = """
+            function cwmExec(element, code, cb) {
+                console.log(">>> [CWM-Outer] Running: " + code);
+                let runId = Math.random();
+                if(cb != null) {
+                    let listener;
+                    listener = function(event) {
+                        if(typeof event.data === "string" && event.data.startsWith("exh-")) {
+                            let response = JSON.parse(event.data.substring(4));
+                            if(response.id === runId) {
+                                cb(response.result);
+                                window.removeEventListener('message', listener);
+                                console.log(">>> [CWM-Outer] Finished: " + response.id + " ==> " + response.result);
+                            }
+                        }
+                    };
+                    window.addEventListener('message', listener, false);
+                }
+                let runRequest = { id: runId, code: code };
+                element.contentWindow.postMessage("exh-" + JSON.stringify(runRequest), "*");
+            }
+        """.trimIndent().replace("\n", "")
+
+        val CROSS_WINDOW_SCRIPT_INNER = """
+            window.addEventListener('message', function(event) {
+                if(typeof event.data === "string" && event.data.startsWith("exh-")) {
+                    let request = JSON.parse(event.data.substring(4));
+                    console.log(">>> [CWM-Inner] Incoming: " + request.id);
+                    let result = eval("(function() {" + request.code + "})();");
+                    let response = { id: request.id, result: result };
+                    console.log(">>> [CWM-Inner] Outgoing: " + response.id + " ==> " + response.result);
+                    event.source.postMessage("exh-" + JSON.stringify(response), event.origin);
+                }
+            }, false);
+            console.log(">>> [CWM-Inner] Loaded!");
+            alert("exh-");
+        """.trimIndent()
+
+        val SOLVE_UI_SCRIPT_SHOW = """
+            (function() {
+                let exh_overlay = document.createElement("div");
+                exh_overlay.id = "exh_overlay";
+                exh_overlay.style.zIndex = 2000000001;
+                exh_overlay.style.backgroundColor = "rgba(0, 0, 0, 0.8)";
+                exh_overlay.style.position = "fixed";
+                exh_overlay.style.top = 0;
+                exh_overlay.style.left = 0;
+                exh_overlay.style.width = "100%";
+                exh_overlay.style.height = "100%";
+                exh_overlay.style.pointerEvents = "none";
+                document.body.appendChild(exh_overlay);
+                let exh_otext = document.createElement("div");
+                exh_otext.id = "exh_otext";
+                exh_otext.style.zIndex = 2000000002;
+                exh_otext.style.position = "fixed";
+                exh_otext.style.top = "50%";
+                exh_otext.style.left = 0;
+                exh_otext.style.transform = "translateY(-50%)";
+                exh_otext.style.color = "white";
+                exh_otext.style.fontSize = "25pt";
+                exh_otext.style.pointerEvents = "none";
+                exh_otext.style.width = "100%";
+                exh_otext.style.textAlign = "center";
+                exh_otext.textContent = "Solving captcha..."
+                document.body.appendChild(exh_otext);
+            })();
+        """.trimIndent()
+
+        val SOLVE_UI_SCRIPT_HIDE = """
+            (function() {
+                let exh_overlay = document.getElementById("exh_overlay");
+                let exh_otext = document.getElementById("exh_otext");
+                if(exh_overlay != null) exh_overlay.remove();
+                if(exh_otext != null) exh_otext.remove();
+            })();
+        """.trimIndent()
+
+        val RECOGNIZE_JSON = """
+            {
+               "part_content_type": "audio/mp3",
+               "keywords": [],
+               "profanity_filter": false,
+               "max_alternatives": 1,
+               "speaker_labels": false,
+               "firstReadyInSession": false,
+               "preserveAdaptation": false,
+               "timestamps": false,
+               "inactivity_timeout": 30,
+               "word_confidence": false,
+               "audioMetrics": false,
+               "latticeGeneration": true,
+               "customGrammarWords": [],
+               "action": "recognize"
+            }
+        """.trimIndent()
+
+        val TRANSCRIPT_CLEANER_REGEX = Regex("[^0-9a-zA-Z_ -]")
+        val SPACE_DEDUPE_REGEX = Regex(" +")
 
         fun launch(context: Context,
                    source: CaptchaCompletionVerifier,
                    cookies: Map<String, String>,
                    script: String,
-                   url: String) {
+                   url: String,
+                   autoSolveSubmitBtnSelector: String? = null) {
             val intent = Intent(context, SolveCaptchaActivity::class.java).apply {
                 putExtra(SOURCE_ID_EXTRA, source.id)
                 putExtra(COOKIES_EXTRA, HashMap(cookies))
                 putExtra(SCRIPT_EXTRA, script)
                 putExtra(URL_EXTRA, url)
+                putExtra(ASBTN_EXTRA, autoSolveSubmitBtnSelector)
             }
 
             context.startActivity(intent)
@@ -115,6 +612,6 @@ class SolveCaptchaActivity : AppCompatActivity() {
 }
 
 interface CaptchaCompletionVerifier : Source {
-    fun verify(url: String): Boolean
+    fun verifyNoCaptcha(url: String): Boolean
 }
 

app/src/main/java/exh/ui/captcha/WebViewUtil.kt

@@ -0,0 +1,19 @@
+package exh.ui.captcha
+
+import android.os.Build
+import android.support.annotation.RequiresApi
+import android.webkit.WebResourceRequest
+import okhttp3.Request
+
+@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+fun WebResourceRequest.toOkHttpRequest(): Request {
+    val request = Request.Builder()
+            .url(url.toString())
+            .method(method, null)
+
+    requestHeaders.entries.forEach { (t, u) ->
+        request.addHeader(t, u)
+    }
+
+    return request.build()
+}

app/src/main/java/exh/ui/migration/MetadataFetchDialog.kt

@@ -11,8 +11,6 @@ import eu.kanade.tachiyomi.data.preference.getOrDefault
 import eu.kanade.tachiyomi.source.SourceManager
 import exh.isExSource
 import exh.isLewdSource
-import exh.metadata.queryMetadataFromManga
-import exh.util.defRealm
 import timber.log.Timber
 import uy.kohesive.injekt.injectLazy
 import kotlin.concurrent.thread
@@ -29,54 +27,71 @@ class MetadataFetchDialog {
         //Too lazy to actually deal with orientation changes
         context.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_NOSENSOR
 
+        var running = true
+
         val progressDialog = MaterialDialog.Builder(context)
                 .title("Fetching library metadata")
                 .content("Preparing library")
                 .progress(false, 0, true)
+                .negativeText("Stop")
+                .onNegative { dialog, which ->
+                    running = false
+                    dialog.dismiss()
+                    notifyMigrationStopped(context)
+                }
                 .cancelable(false)
                 .canceledOnTouchOutside(false)
                 .show()
 
         thread {
-            defRealm { realm ->
-                db.deleteMangasNotInLibrary().executeAsBlocking()
+            val libraryMangas = db.getLibraryMangas().executeAsBlocking()
+                    .filter { isLewdSource(it.source) }
+                    .distinctBy { it.id }
 
-                val libraryMangas = db.getLibraryMangas()
-                        .executeAsBlocking()
-                        .filter {
-                            isLewdSource(it.source)
-                                    && realm.queryMetadataFromManga(it).findFirst() == null
+            context.runOnUiThread {
+                progressDialog.maxProgress = libraryMangas.size
+            }
+
+            val mangaWithMissingMetadata = libraryMangas
+                    .filterIndexed { index, libraryManga ->
+                        if(index % 100 == 0) {
+                            context.runOnUiThread {
+                                progressDialog.setContent("[Stage 1/2] Scanning for missing metadata...")
+                                progressDialog.setProgress(index + 1)
+                            }
                         }
-
-                context.runOnUiThread {
-                    progressDialog.maxProgress = libraryMangas.size
-                }
-
-                //Actual metadata fetch code
-                libraryMangas.forEachIndexed { i, manga ->
-                    context.runOnUiThread {
-                        progressDialog.setContent("Processing: ${manga.title}")
-                        progressDialog.setProgress(i + 1)
+                        db.getSearchMetadataForManga(libraryManga.id!!).executeAsBlocking() == null
                     }
-                    try {
-                        val source = sourceManager.get(manga.source)
-                        source?.let {
-                            manga.copyFrom(it.fetchMangaDetails(manga).toBlocking().first())
-                            realm.queryMetadataFromManga(manga).findFirst()?.copyTo(manga)
-                        }
-                    } catch (t: Throwable) {
-                        Timber.e(t, "Could not migrate manga!")
-                    }
-                }
+                    .toList()
 
+            context.runOnUiThread {
+                progressDialog.maxProgress = mangaWithMissingMetadata.size
+            }
+
+            //Actual metadata fetch code
+            for((i, manga) in mangaWithMissingMetadata.withIndex()) {
+                if(!running) break
                 context.runOnUiThread {
-                    progressDialog.dismiss()
-
-                    //Enable orientation changes again
-                    context.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_NOSENSOR
-
-                    displayMigrationComplete(context)
+                    progressDialog.setContent("[Stage 2/2] Processing: ${manga.title}")
+                    progressDialog.setProgress(i + 1)
                 }
+                try {
+                    val source = sourceManager.get(manga.source)
+                    source?.let {
+                        manga.copyFrom(it.fetchMangaDetails(manga).toBlocking().first())
+                    }
+                } catch (t: Throwable) {
+                    Timber.e(t, "Could not migrate manga!")
+                }
+            }
+
+            context.runOnUiThread {
+                progressDialog.dismiss()
+
+                //Enable orientation changes again
+                context.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_NOSENSOR
+
+                if(running) displayMigrationComplete(context)
             }
         }
     }
@@ -85,7 +100,9 @@ class MetadataFetchDialog {
         var extra = ""
         db.getLibraryMangas().asRxSingle().subscribe {
             if(!explicit && it.none { isLewdSource(it.source) }) {
-                //Do not open dialog on startup if no manga
+                // Do not open dialog on startup if no manga
+                // Also do not check again
+                preferenceHelper.migrateLibraryAsked().set(true)
             } else {
                 //Not logged in but have ExHentai galleries
                 if (!preferenceHelper.enableExhentai().getOrDefault()) {
@@ -97,13 +114,14 @@ class MetadataFetchDialog {
                     MaterialDialog.Builder(activity)
                             .title("Fetch library metadata")
                             .content(Html.fromHtml("You need to fetch your library's metadata before tag searching in the library will function.<br><br>" +
-                                    "This process may take a long time depending on your library size and will also use up a significant amount of internet bandwidth.<br><br>" +
+                                    "This process may take a long time depending on your library size and will also use up a significant amount of internet bandwidth but can be stopped and started whenever you wish.<br><br>" +
                                     extra +
                                     "This process can be done later if required."))
                             .positiveText("Migrate")
                             .negativeText("Later")
                             .onPositive { _, _ -> show(activity) }
-                            .onNegative({ _, _ -> adviseMigrationLater(activity) })
+                            .onNegative { _, _ -> adviseMigrationLater(activity) }
+                            .onAny { _, _ -> preferenceHelper.migrateLibraryAsked().set(true) }
                             .cancelable(false)
                             .canceledOnTouchOutside(false)
                             .show()
@@ -124,6 +142,17 @@ class MetadataFetchDialog {
                 .show()
     }
 
+    fun notifyMigrationStopped(activity: Activity) {
+        MaterialDialog.Builder(activity)
+                .title("Metadata fetch stopped")
+                .content("Library metadata fetch has been stopped.\n\n" +
+                        "You can continue this operation later by going to: Settings > Advanced > Migrate library metadata")
+                .positiveText("Ok")
+                .cancelable(true)
+                .canceledOnTouchOutside(true)
+                .show()
+    }
+
     fun displayMigrationComplete(activity: Activity) {
         MaterialDialog.Builder(activity)
                 .title("Migration complete")

app/src/main/java/exh/ui/migration/UrlMigrator.kt

@@ -1,81 +0,0 @@
-package exh.ui.migration
-
-import eu.kanade.tachiyomi.data.database.DatabaseHelper
-import eu.kanade.tachiyomi.data.database.models.Manga
-import eu.kanade.tachiyomi.data.preference.PreferencesHelper
-import eu.kanade.tachiyomi.data.preference.getOrDefault
-import exh.isExSource
-import exh.isLewdSource
-import exh.metadata.models.ExGalleryMetadata
-import exh.util.realmTrans
-import uy.kohesive.injekt.injectLazy
-
-class UrlMigrator {
-    private val db: DatabaseHelper by injectLazy()
-
-    private val prefs: PreferencesHelper by injectLazy()
-
-    fun perform() {
-        db.inTransaction {
-            val dbMangas = db.getMangas()
-                    .executeAsBlocking()
-
-            //Find all EX mangas
-            val qualifyingMangas = dbMangas.asSequence().filter {
-                isLewdSource(it.source)
-            }
-
-            val possibleDups = mutableListOf<Manga>()
-            val badMangas = mutableListOf<Manga>()
-
-            qualifyingMangas.forEach {
-                if(it.url.startsWith("g/")) //Missing slash at front so we are bad
-                    badMangas.add(it)
-                else
-                    possibleDups.add(it)
-            }
-
-            //Sort possible dups so we can use binary search on it
-            possibleDups.sortBy { it.url }
-
-            realmTrans { realm ->
-                badMangas.forEach { manga ->
-                    //Build fixed URL
-                    val urlWithSlash = "/" + manga.url
-                    //Fix metadata if required
-                    val metadata = ExGalleryMetadata.UrlQuery(manga.url, isExSource(manga.source))
-                            .query(realm)
-                            .findFirst()
-                    metadata?.url?.let {
-                        if (it.startsWith("g/")) { //Check if metadata URL has no slash
-                            metadata.url = urlWithSlash //Fix it
-                        }
-                    }
-                    //If we have a dup (with the fixed url), use the dup instead
-                    val possibleDup = possibleDups.binarySearchBy(urlWithSlash, selector = { it.url })
-                    if (possibleDup >= 0) {
-                        //Make sure it is favorited if we are
-                        if (manga.favorite) {
-                            val dup = possibleDups[possibleDup]
-                            dup.favorite = true
-                            db.insertManga(dup).executeAsBlocking() //Update DB with changes
-                        }
-                        //Delete ourself (but the dup is still there)
-                        db.deleteManga(manga).executeAsBlocking()
-                        return@forEach
-                    }
-                    //No dup, correct URL and reinsert ourselves
-                    manga.url = urlWithSlash
-                    db.insertManga(manga).executeAsBlocking()
-                }
-            }
-        }
-    }
-
-    fun tryMigration() {
-        if(!prefs.hasPerformedURLMigration().getOrDefault()) {
-            perform()
-            prefs.hasPerformedURLMigration().set(true)
-        }
-    }
-}

app/src/main/java/exh/ui/webview/NestedWebView.java

@@ -1,126 +0,0 @@
-package exh.ui.webview;
-
-import android.content.Context;
-import android.support.v4.view.MotionEventCompat;
-import android.support.v4.view.NestedScrollingChild;
-import android.support.v4.view.NestedScrollingChildHelper;
-import android.support.v4.view.ViewCompat;
-import android.util.AttributeSet;
-import android.view.MotionEvent;
-import android.webkit.WebView;
-
-public class NestedWebView extends WebView implements NestedScrollingChild {
-    private int mLastY;
-    private final int[] mScrollOffset = new int[2];
-    private final int[] mScrollConsumed = new int[2];
-    private int mNestedOffsetY;
-    private NestedScrollingChildHelper mChildHelper;
-
-    public NestedWebView(Context context) {
-        this(context, null);
-    }
-
-    public NestedWebView(Context context, AttributeSet attrs) {
-        this(context, attrs, android.R.attr.webViewStyle);
-    }
-
-    public NestedWebView(Context context, AttributeSet attrs, int defStyleAttr) {
-        super(context, attrs, defStyleAttr);
-        mChildHelper = new NestedScrollingChildHelper(this);
-        setNestedScrollingEnabled(true);
-    }
-
-    @Override
-    public boolean onTouchEvent(MotionEvent ev) {
-        boolean returnValue = false;
-
-        MotionEvent event = MotionEvent.obtain(ev);
-        final int action = MotionEventCompat.getActionMasked(event);
-        if (action == MotionEvent.ACTION_DOWN) {
-            mNestedOffsetY = 0;
-        }
-        int eventY = (int) event.getY();
-        event.offsetLocation(0, mNestedOffsetY);
-        switch (action) {
-            case MotionEvent.ACTION_MOVE:
-                int deltaY = mLastY - eventY;
-                // NestedPreScroll
-                if (dispatchNestedPreScroll(0, deltaY, mScrollConsumed, mScrollOffset)) {
-                    deltaY -= mScrollConsumed[1];
-                    mLastY = eventY - mScrollOffset[1];
-                    event.offsetLocation(0, -mScrollOffset[1]);
-                    mNestedOffsetY += mScrollOffset[1];
-                }
-                returnValue = super.onTouchEvent(event);
-
-                // NestedScroll
-                if (dispatchNestedScroll(0, mScrollOffset[1], 0, deltaY, mScrollOffset)) {
-                    event.offsetLocation(0, mScrollOffset[1]);
-                    mNestedOffsetY += mScrollOffset[1];
-                    mLastY -= mScrollOffset[1];
-                }
-                break;
-            case MotionEvent.ACTION_DOWN:
-                returnValue = super.onTouchEvent(event);
-                mLastY = eventY;
-                // start NestedScroll
-                startNestedScroll(ViewCompat.SCROLL_AXIS_VERTICAL);
-                break;
-            case MotionEvent.ACTION_UP:
-            case MotionEvent.ACTION_CANCEL:
-                returnValue = super.onTouchEvent(event);
-                // end NestedScroll
-                stopNestedScroll();
-                break;
-        }
-        return returnValue;
-    }
-
-    // Nested Scroll implements
-    @Override
-    public void setNestedScrollingEnabled(boolean enabled) {
-        mChildHelper.setNestedScrollingEnabled(enabled);
-    }
-
-    @Override
-    public boolean isNestedScrollingEnabled() {
-        return mChildHelper.isNestedScrollingEnabled();
-    }
-
-    @Override
-    public boolean startNestedScroll(int axes) {
-        return mChildHelper.startNestedScroll(axes);
-    }
-
-    @Override
-    public void stopNestedScroll() {
-        mChildHelper.stopNestedScroll();
-    }
-
-    @Override
-    public boolean hasNestedScrollingParent() {
-        return mChildHelper.hasNestedScrollingParent();
-    }
-
-    @Override
-    public boolean dispatchNestedScroll(int dxConsumed, int dyConsumed, int dxUnconsumed, int dyUnconsumed,
-                                        int[] offsetInWindow) {
-        return mChildHelper.dispatchNestedScroll(dxConsumed, dyConsumed, dxUnconsumed, dyUnconsumed, offsetInWindow);
-    }
-
-    @Override
-    public boolean dispatchNestedPreScroll(int dx, int dy, int[] consumed, int[] offsetInWindow) {
-        return mChildHelper.dispatchNestedPreScroll(dx, dy, consumed, offsetInWindow);
-    }
-
-    @Override
-    public boolean dispatchNestedFling(float velocityX, float velocityY, boolean consumed) {
-        return mChildHelper.dispatchNestedFling(velocityX, velocityY, consumed);
-    }
-
-    @Override
-    public boolean dispatchNestedPreFling(float velocityX, float velocityY) {
-        return mChildHelper.dispatchNestedPreFling(velocityX, velocityY);
-    }
-
-}

app/src/main/java/exh/ui/webview/WebViewActivity.kt

@@ -57,6 +57,10 @@ class WebViewActivity : BaseActivity() {
         webview.settings.javaScriptEnabled = true
         webview.settings.domStorageEnabled = true
         webview.settings.databaseEnabled = true
+        webview.settings.useWideViewPort = true
+        webview.settings.loadWithOverviewMode = true
+        webview.settings.builtInZoomControls = true
+        webview.settings.displayZoomControls = false
         webview.webViewClient = object : WebViewClient() {
             override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) {
                 super.onPageStarted(view, url, favicon)
@@ -134,7 +138,6 @@ class WebViewActivity : BaseActivity() {
 
     override fun onPrepareOptionsMenu(menu: Menu?): Boolean {
         menu?.findItem(R.id.action_forward)?.isEnabled = webview.canGoForward()
-        menu?.findItem(R.id.action_desktop_site)?.isChecked = isDesktop
 
         return super.onPrepareOptionsMenu(menu)
     }
@@ -156,26 +159,6 @@ class WebViewActivity : BaseActivity() {
             android.R.id.home -> finish()
             R.id.action_refresh -> webview.reload()
             R.id.action_forward -> webview.goForward()
-            R.id.action_desktop_site -> {
-                isDesktop = !item.isChecked
-                item.isChecked = isDesktop
-
-                (if(isDesktop) {
-                    mobileUserAgent?.replace("\\([^(]*(Mobile|Android)[^)]*\\)"
-                            .toRegex(RegexOption.IGNORE_CASE), "")
-                            ?.replace("Mobile", "", true)
-                            ?.replace("Android", "", true)
-                } else {
-                    mobileUserAgent
-                })?.let {
-                    webview.settings.userAgentString = it
-                }
-
-                webview.settings.useWideViewPort = isDesktop
-                webview.settings.loadWithOverviewMode = isDesktop
-
-                webview.reload()
-            }
             R.id.action_open_in_browser ->
                 startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(webview.url)))
         }