Extract source api from app module (#8014)

* Extract source api from app module

* Extract source online api from app module
This commit is contained in:
Andreas
2022-09-16 00:12:27 +02:00
committed by GitHub
parent 30ac94181b
commit 86fe850794
53 changed files with 219 additions and 106 deletions

View File

@ -0,0 +1,54 @@
package eu.kanade.tachiyomi.network
import android.webkit.CookieManager
import okhttp3.Cookie
import okhttp3.CookieJar
import okhttp3.HttpUrl
class AndroidCookieJar : CookieJar {
private val manager = CookieManager.getInstance()
override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
val urlString = url.toString()
cookies.forEach { manager.setCookie(urlString, it.toString()) }
}
override fun loadForRequest(url: HttpUrl): List<Cookie> {
return get(url)
}
fun get(url: HttpUrl): List<Cookie> {
val cookies = manager.getCookie(url.toString())
return if (cookies != null && cookies.isNotEmpty()) {
cookies.split(";").mapNotNull { Cookie.parse(url, it) }
} else {
emptyList()
}
}
fun remove(url: HttpUrl, cookieNames: List<String>? = null, maxAge: Int = -1): Int {
val urlString = url.toString()
val cookies = manager.getCookie(urlString) ?: return 0
fun List<String>.filterNames(): List<String> {
return if (cookieNames != null) {
this.filter { it in cookieNames }
} else {
this
}
}
return cookies.split(";")
.map { it.substringBefore("=") }
.filterNames()
.onEach { manager.setCookie(urlString, "$it=;Max-Age=$maxAge") }
.count()
}
fun removeAll() {
manager.removeAllCookies {}
}
}

View File

@ -0,0 +1,174 @@
package eu.kanade.tachiyomi.network
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import okhttp3.dnsoverhttps.DnsOverHttps
import java.net.InetAddress
/**
* Based on https://github.com/square/okhttp/blob/ef5d0c83f7bbd3a0c0534e7ca23cbc4ee7550f3b/okhttp-dnsoverhttps/src/test/java/okhttp3/dnsoverhttps/DohProviders.java
*/
const val PREF_DOH_CLOUDFLARE = 1
const val PREF_DOH_GOOGLE = 2
const val PREF_DOH_ADGUARD = 3
const val PREF_DOH_QUAD9 = 4
const val PREF_DOH_ALIDNS = 5
const val PREF_DOH_DNSPOD = 6
const val PREF_DOH_360 = 7
const val PREF_DOH_QUAD101 = 8
const val PREF_DOH_MULLVAD = 9
const val PREF_DOH_CONTROLD = 10
const val PREF_DOH_NJALLA = 11
fun OkHttpClient.Builder.dohCloudflare() = dns(
DnsOverHttps.Builder().client(build())
.url("https://cloudflare-dns.com/dns-query".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("162.159.36.1"),
InetAddress.getByName("162.159.46.1"),
InetAddress.getByName("1.1.1.1"),
InetAddress.getByName("1.0.0.1"),
InetAddress.getByName("162.159.132.53"),
InetAddress.getByName("2606:4700:4700::1111"),
InetAddress.getByName("2606:4700:4700::1001"),
InetAddress.getByName("2606:4700:4700::0064"),
InetAddress.getByName("2606:4700:4700::6400"),
)
.build(),
)
fun OkHttpClient.Builder.dohGoogle() = dns(
DnsOverHttps.Builder().client(build())
.url("https://dns.google/dns-query".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("8.8.4.4"),
InetAddress.getByName("8.8.8.8"),
InetAddress.getByName("2001:4860:4860::8888"),
InetAddress.getByName("2001:4860:4860::8844"),
)
.build(),
)
// AdGuard "Default" DNS works too but for the sake of making sure no site is blacklisted,
// we use "Unfiltered"
fun OkHttpClient.Builder.dohAdGuard() = dns(
DnsOverHttps.Builder().client(build())
.url("https://dns-unfiltered.adguard.com/dns-query".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("94.140.14.140"),
InetAddress.getByName("94.140.14.141"),
InetAddress.getByName("2a10:50c0::1:ff"),
InetAddress.getByName("2a10:50c0::2:ff"),
)
.build(),
)
fun OkHttpClient.Builder.dohQuad9() = dns(
DnsOverHttps.Builder().client(build())
.url("https://dns.quad9.net/dns-query".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("9.9.9.9"),
InetAddress.getByName("149.112.112.112"),
InetAddress.getByName("2620:fe::fe"),
InetAddress.getByName("2620:fe::9"),
)
.build(),
)
fun OkHttpClient.Builder.dohAliDNS() = dns(
DnsOverHttps.Builder().client(build())
.url("https://dns.alidns.com/dns-query".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("223.5.5.5"),
InetAddress.getByName("223.6.6.6"),
InetAddress.getByName("2400:3200::1"),
InetAddress.getByName("2400:3200:baba::1"),
)
.build(),
)
fun OkHttpClient.Builder.dohDNSPod() = dns(
DnsOverHttps.Builder().client(build())
.url("https://doh.pub/dns-query".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("1.12.12.12"),
InetAddress.getByName("120.53.53.53"),
)
.build(),
)
fun OkHttpClient.Builder.doh360() = dns(
DnsOverHttps.Builder().client(build())
.url("https://doh.360.cn/dns-query".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("101.226.4.6"),
InetAddress.getByName("218.30.118.6"),
InetAddress.getByName("123.125.81.6"),
InetAddress.getByName("140.207.198.6"),
InetAddress.getByName("180.163.249.75"),
InetAddress.getByName("101.199.113.208"),
InetAddress.getByName("36.99.170.86"),
)
.build(),
)
fun OkHttpClient.Builder.dohQuad101() = dns(
DnsOverHttps.Builder().client(build())
.url("https://dns.twnic.tw/dns-query".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("101.101.101.101"),
InetAddress.getByName("2001:de4::101"),
InetAddress.getByName("2001:de4::102"),
)
.build(),
)
/*
* Mullvad DoH
* without ad blocking option
* Source : https://mullvad.net/en/help/dns-over-https-and-dns-over-tls/
*/
fun OkHttpClient.Builder.dohMullvad() = dns(
DnsOverHttps.Builder().client(build())
.url("https://doh.mullvad.net/dns-query".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("194.242.2.2"),
InetAddress.getByName("193.19.108.2"),
InetAddress.getByName("2a07:e340::2"),
)
.build(),
)
/*
* Control D
* unfiltered option
* Source : https://controld.com/free-dns/?
*/
fun OkHttpClient.Builder.dohControlD() = dns(
DnsOverHttps.Builder().client(build())
.url("https://freedns.controld.com/p0".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("76.76.2.0"),
InetAddress.getByName("76.76.10.0"),
InetAddress.getByName("2606:1a40::"),
InetAddress.getByName("2606:1a40:1::"),
)
.build(),
)
/*
* Njalla
*
* Non logging and uncensored
*/
fun OkHttpClient.Builder.dohNajalla() = dns(
DnsOverHttps.Builder().client(build())
.url("https://dns.njal.la/dns-query".toHttpUrl())
.bootstrapDnsHosts(
InetAddress.getByName("95.215.19.53"),
InetAddress.getByName("2001:67c:2354:2::53"),
)
.build(),
)

View File

@ -0,0 +1,75 @@
package eu.kanade.tachiyomi.network
import android.content.Context
import androidx.preference.PreferenceManager
import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor
import eu.kanade.tachiyomi.network.interceptor.Http103Interceptor
import eu.kanade.tachiyomi.network.interceptor.UserAgentInterceptor
import okhttp3.Cache
import okhttp3.OkHttpClient
import okhttp3.logging.HttpLoggingInterceptor
import java.io.File
import java.util.concurrent.TimeUnit
class NetworkHelper(context: Context) {
// TODO: Abstract preferences similar to 1.x
private val preferences = PreferenceManager.getDefaultSharedPreferences(context)
private val cacheDir = File(context.cacheDir, "network_cache")
private val cacheSize = 5L * 1024 * 1024 // 5 MiB
val cookieManager = AndroidCookieJar()
private val userAgentInterceptor by lazy { UserAgentInterceptor() }
private val http103Interceptor by lazy { Http103Interceptor(context) }
private val cloudflareInterceptor by lazy { CloudflareInterceptor(context) }
private val baseClientBuilder: OkHttpClient.Builder
get() {
val builder = OkHttpClient.Builder()
.cookieJar(cookieManager)
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(30, TimeUnit.SECONDS)
.callTimeout(2, TimeUnit.MINUTES)
.fastFallback(true)
.addInterceptor(userAgentInterceptor)
.addNetworkInterceptor(http103Interceptor)
if (preferences.getBoolean("verbose_logging", false)) {
val httpLoggingInterceptor = HttpLoggingInterceptor().apply {
level = HttpLoggingInterceptor.Level.HEADERS
}
builder.addNetworkInterceptor(httpLoggingInterceptor)
}
when (preferences.getInt("doh_provider", -1)) {
PREF_DOH_CLOUDFLARE -> builder.dohCloudflare()
PREF_DOH_GOOGLE -> builder.dohGoogle()
PREF_DOH_ADGUARD -> builder.dohAdGuard()
PREF_DOH_QUAD9 -> builder.dohQuad9()
PREF_DOH_ALIDNS -> builder.dohAliDNS()
PREF_DOH_DNSPOD -> builder.dohDNSPod()
PREF_DOH_360 -> builder.doh360()
PREF_DOH_QUAD101 -> builder.dohQuad101()
PREF_DOH_MULLVAD -> builder.dohMullvad()
PREF_DOH_CONTROLD -> builder.dohControlD()
PREF_DOH_NJALLA -> builder.dohNajalla()
}
return builder
}
val client by lazy { baseClientBuilder.cache(Cache(cacheDir, cacheSize)).build() }
@Suppress("UNUSED")
val cloudflareClient by lazy {
client.newBuilder()
.addInterceptor(cloudflareInterceptor)
.build()
}
val defaultUserAgent by lazy {
preferences.getString("default_user_agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:104.0) Gecko/20100101 Firefox/104.0")!!
}
}

View File

@ -0,0 +1,127 @@
package eu.kanade.tachiyomi.network
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import okhttp3.Call
import okhttp3.Callback
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import okhttp3.internal.closeQuietly
import rx.Observable
import rx.Producer
import rx.Subscription
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.fullType
import java.io.IOException
import java.util.concurrent.atomic.AtomicBoolean
import kotlin.coroutines.resumeWithException
val jsonMime = "application/json; charset=utf-8".toMediaType()
fun Call.asObservable(): Observable<Response> {
return Observable.unsafeCreate { subscriber ->
// Since Call is a one-shot type, clone it for each new subscriber.
val call = clone()
// Wrap the call in a helper which handles both unsubscription and backpressure.
val requestArbiter = object : AtomicBoolean(), Producer, Subscription {
override fun request(n: Long) {
if (n == 0L || !compareAndSet(false, true)) return
try {
val response = call.execute()
if (!subscriber.isUnsubscribed) {
subscriber.onNext(response)
subscriber.onCompleted()
}
} catch (error: Exception) {
if (!subscriber.isUnsubscribed) {
subscriber.onError(error)
}
}
}
override fun unsubscribe() {
call.cancel()
}
override fun isUnsubscribed(): Boolean {
return call.isCanceled()
}
}
subscriber.add(requestArbiter)
subscriber.setProducer(requestArbiter)
}
}
// Based on https://github.com/gildor/kotlin-coroutines-okhttp
suspend fun Call.await(): Response {
return suspendCancellableCoroutine { continuation ->
enqueue(
object : Callback {
override fun onResponse(call: Call, response: Response) {
if (!response.isSuccessful) {
continuation.resumeWithException(HttpException(response.code))
return
}
continuation.resume(response) {
response.body.closeQuietly()
}
}
override fun onFailure(call: Call, e: IOException) {
// Don't bother with resuming the continuation if it is already cancelled.
if (continuation.isCancelled) return
continuation.resumeWithException(e)
}
},
)
continuation.invokeOnCancellation {
try {
cancel()
} catch (ex: Throwable) {
// Ignore cancel exception
}
}
}
}
fun Call.asObservableSuccess(): Observable<Response> {
return asObservable().doOnNext { response ->
if (!response.isSuccessful) {
response.close()
throw HttpException(response.code)
}
}
}
fun OkHttpClient.newCallWithProgress(request: Request, listener: ProgressListener): Call {
val progressClient = newBuilder()
.cache(null)
.addNetworkInterceptor { chain ->
val originalResponse = chain.proceed(chain.request())
originalResponse.newBuilder()
.body(ProgressResponseBody(originalResponse.body, listener))
.build()
}
.build()
return progressClient.newCall(request)
}
inline fun <reified T> Response.parseAs(): T {
// Avoiding Injekt.get<Json>() due to compiler issues
val json = Injekt.getInstance<Json>(fullType<Json>().type)
this.use {
val responseBody = it.body.string()
return json.decodeFromString(responseBody)
}
}
class HttpException(val code: Int) : IllegalStateException("HTTP error $code")

View File

@ -0,0 +1,5 @@
package eu.kanade.tachiyomi.network
interface ProgressListener {
fun update(bytesRead: Long, contentLength: Long, done: Boolean)
}

View File

@ -0,0 +1,44 @@
package eu.kanade.tachiyomi.network
import okhttp3.MediaType
import okhttp3.ResponseBody
import okio.Buffer
import okio.BufferedSource
import okio.ForwardingSource
import okio.Source
import okio.buffer
import java.io.IOException
class ProgressResponseBody(private val responseBody: ResponseBody, private val progressListener: ProgressListener) : ResponseBody() {
private val bufferedSource: BufferedSource by lazy {
source(responseBody.source()).buffer()
}
override fun contentType(): MediaType? {
return responseBody.contentType()
}
override fun contentLength(): Long {
return responseBody.contentLength()
}
override fun source(): BufferedSource {
return bufferedSource
}
private fun source(source: Source): Source {
return object : ForwardingSource(source) {
var totalBytesRead = 0L
@Throws(IOException::class)
override fun read(sink: Buffer, byteCount: Long): Long {
val bytesRead = super.read(sink, byteCount)
// read() returns the number of bytes read, or -1 if this source is exhausted.
totalBytesRead += if (bytesRead != -1L) bytesRead else 0
progressListener.update(totalBytesRead, responseBody.contentLength(), bytesRead == -1L)
return bytesRead
}
}
}
}

View File

@ -0,0 +1,67 @@
package eu.kanade.tachiyomi.network
import okhttp3.CacheControl
import okhttp3.FormBody
import okhttp3.Headers
import okhttp3.Request
import okhttp3.RequestBody
import java.util.concurrent.TimeUnit.MINUTES
private val DEFAULT_CACHE_CONTROL = CacheControl.Builder().maxAge(10, MINUTES).build()
private val DEFAULT_HEADERS = Headers.Builder().build()
private val DEFAULT_BODY: RequestBody = FormBody.Builder().build()
val CACHE_CONTROL_NO_STORE = CacheControl.Builder().noStore().build()
fun GET(
url: String,
headers: Headers = DEFAULT_HEADERS,
cache: CacheControl = DEFAULT_CACHE_CONTROL,
): Request {
return Request.Builder()
.url(url)
.headers(headers)
.cacheControl(cache)
.build()
}
fun POST(
url: String,
headers: Headers = DEFAULT_HEADERS,
body: RequestBody = DEFAULT_BODY,
cache: CacheControl = DEFAULT_CACHE_CONTROL,
): Request {
return Request.Builder()
.url(url)
.post(body)
.headers(headers)
.cacheControl(cache)
.build()
}
fun PUT(
url: String,
headers: Headers = DEFAULT_HEADERS,
body: RequestBody = DEFAULT_BODY,
cache: CacheControl = DEFAULT_CACHE_CONTROL,
): Request {
return Request.Builder()
.url(url)
.put(body)
.headers(headers)
.cacheControl(cache)
.build()
}
fun DELETE(
url: String,
headers: Headers = DEFAULT_HEADERS,
body: RequestBody = DEFAULT_BODY,
cache: CacheControl = DEFAULT_CACHE_CONTROL,
): Request {
return Request.Builder()
.url(url)
.delete(body)
.headers(headers)
.cacheControl(cache)
.build()
}

View File

@ -0,0 +1,149 @@
package eu.kanade.tachiyomi.network.interceptor
import android.annotation.SuppressLint
import android.content.Context
import android.webkit.WebView
import android.widget.Toast
import androidx.core.content.ContextCompat
import eu.kanade.tachiyomi.core.R
import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.util.system.WebViewClientCompat
import eu.kanade.tachiyomi.util.system.isOutdated
import eu.kanade.tachiyomi.util.system.setDefaultSettings
import eu.kanade.tachiyomi.util.system.toast
import okhttp3.Cookie
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
import uy.kohesive.injekt.injectLazy
import java.io.IOException
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
class CloudflareInterceptor(private val context: Context) : WebViewInterceptor(context) {
private val executor = ContextCompat.getMainExecutor(context)
private val networkHelper: NetworkHelper by injectLazy()
override fun shouldIntercept(response: Response): Boolean {
// Check if Cloudflare anti-bot is on
return response.code in ERROR_CODES && response.header("Server") in SERVER_CHECK
}
override fun intercept(chain: Interceptor.Chain, request: Request, response: Response): Response {
try {
response.close()
networkHelper.cookieManager.remove(request.url, COOKIE_NAMES, 0)
val oldCookie = networkHelper.cookieManager.get(request.url)
.firstOrNull { it.name == "cf_clearance" }
resolveWithWebView(request, oldCookie)
return chain.proceed(request)
}
// Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
// we don't crash the entire app
catch (e: CloudflareBypassException) {
throw IOException(context.getString(R.string.information_cloudflare_bypass_failure))
} catch (e: Exception) {
throw IOException(e)
}
}
@SuppressLint("SetJavaScriptEnabled")
private fun resolveWithWebView(originalRequest: Request, oldCookie: Cookie?) {
// We need to lock this thread until the WebView finds the challenge solution url, because
// OkHttp doesn't support asynchronous interceptors.
val latch = CountDownLatch(1)
var webView: WebView? = null
var challengeFound = false
var cloudflareBypassed = false
var isWebViewOutdated = false
val origRequestUrl = originalRequest.url.toString()
val headers = originalRequest.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap()
executor.execute {
val webview = WebView(context)
webView = webview
webview.setDefaultSettings()
// Avoid sending empty User-Agent, Chromium WebView will reset to default if empty
webview.settings.userAgentString = originalRequest.header("User-Agent")
?: networkHelper.defaultUserAgent
webview.webViewClient = object : WebViewClientCompat() {
override fun onPageFinished(view: WebView, url: String) {
fun isCloudFlareBypassed(): Boolean {
return networkHelper.cookieManager.get(origRequestUrl.toHttpUrl())
.firstOrNull { it.name == "cf_clearance" }
.let { it != null && it != oldCookie }
}
if (isCloudFlareBypassed()) {
cloudflareBypassed = true
latch.countDown()
}
if (url == origRequestUrl && !challengeFound) {
// The first request didn't return the challenge, abort.
latch.countDown()
}
}
override fun onReceivedErrorCompat(
view: WebView,
errorCode: Int,
description: String?,
failingUrl: String,
isMainFrame: Boolean,
) {
if (isMainFrame) {
if (errorCode in ERROR_CODES) {
// Found the Cloudflare challenge page.
challengeFound = true
} else {
// Unlock thread, the challenge wasn't found.
latch.countDown()
}
}
}
}
webView?.loadUrl(origRequestUrl, headers)
}
// Wait a reasonable amount of time to retrieve the solution. The minimum should be
// around 4 seconds but it can take more due to slow networks or server issues.
latch.await(12, TimeUnit.SECONDS)
executor.execute {
if (!cloudflareBypassed) {
isWebViewOutdated = webView?.isOutdated() == true
}
webView?.stopLoading()
webView?.destroy()
webView = null
}
// Throw exception if we failed to bypass Cloudflare
if (!cloudflareBypassed) {
// Prompt user to update WebView if it seems too outdated
if (isWebViewOutdated) {
context.toast(R.string.information_webview_outdated, Toast.LENGTH_LONG)
}
throw CloudflareBypassException()
}
}
}
private val ERROR_CODES = listOf(403, 503)
private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
private val COOKIE_NAMES = listOf("cf_clearance")
private class CloudflareBypassException : Exception()

View File

@ -0,0 +1,112 @@
package eu.kanade.tachiyomi.network.interceptor
import android.annotation.SuppressLint
import android.content.Context
import android.webkit.JavascriptInterface
import android.webkit.WebView
import androidx.core.content.ContextCompat
import eu.kanade.tachiyomi.util.system.WebViewClientCompat
import eu.kanade.tachiyomi.util.system.logcat
import okhttp3.Interceptor
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.Protocol
import okhttp3.Request
import okhttp3.Response
import okhttp3.ResponseBody.Companion.toResponseBody
import java.io.IOException
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
// TODO: Remove when OkHttp can handle HTTP 103 responses
class Http103Interceptor(context: Context) : WebViewInterceptor(context) {
private val executor = ContextCompat.getMainExecutor(context)
override fun shouldIntercept(response: Response): Boolean {
return response.code == 103
}
override fun intercept(chain: Interceptor.Chain, request: Request, response: Response): Response {
logcat { "Proceeding with WebView for request $request" }
try {
return proceedWithWebView(request, response)
} catch (e: Exception) {
throw IOException(e)
}
}
@SuppressLint("SetJavaScriptEnabled", "AddJavascriptInterface")
private fun proceedWithWebView(originalRequest: Request, originalResponse: Response): Response {
// We need to lock this thread until the WebView loads the page, because
// OkHttp doesn't support asynchronous interceptors.
val latch = CountDownLatch(1)
val jsInterface = JsInterface(latch)
var outerWebView: WebView? = null
var exception: Exception? = null
val requestUrl = originalRequest.url.toString()
val headers = originalRequest.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap()
executor.execute {
val webview = createWebView(originalRequest).also { outerWebView = it }
webview.addJavascriptInterface(jsInterface, "android")
webview.webViewClient = object : WebViewClientCompat() {
override fun onPageFinished(view: WebView, url: String) {
view.evaluateJavascript(jsScript) {}
}
override fun onReceivedErrorCompat(
view: WebView,
errorCode: Int,
description: String?,
failingUrl: String,
isMainFrame: Boolean,
) {
if (isMainFrame) {
exception = Exception("Error $errorCode - $description")
latch.countDown()
}
}
}
webview.loadUrl(requestUrl, headers)
}
latch.await(10, TimeUnit.SECONDS)
executor.execute {
outerWebView?.run {
stopLoading()
destroy()
}
outerWebView = null
}
exception?.let { throw it }
val responseHtml = jsInterface.responseHtml ?: throw Exception("Couldn't fetch site through webview")
return originalResponse.newBuilder()
.code(200)
.protocol(Protocol.HTTP_1_1)
.message("OK")
.body(responseHtml.toResponseBody(htmlMediaType))
.build()
}
}
internal class JsInterface(private val latch: CountDownLatch, var responseHtml: String? = null) {
@Suppress("UNUSED")
@JavascriptInterface
fun passPayload(passedPayload: String) {
responseHtml = passedPayload
latch.countDown()
}
}
private const val jsScript = "window.android.passPayload(document.querySelector('html').outerHTML)"
private val htmlMediaType = "text/html".toMediaType()

View File

@ -0,0 +1,105 @@
package eu.kanade.tachiyomi.network.interceptor
import android.os.SystemClock
import okhttp3.Interceptor
import okhttp3.OkHttpClient
import okhttp3.Response
import java.io.IOException
import java.util.ArrayDeque
import java.util.concurrent.Semaphore
import java.util.concurrent.TimeUnit
/**
* An OkHttp interceptor that handles rate limiting.
*
* Examples:
*
* permits = 5, period = 1, unit = seconds => 5 requests per second
* permits = 10, period = 2, unit = minutes => 10 requests per 2 minutes
*
* @since extension-lib 1.3
*
* @param permits {Int} Number of requests allowed within a period of units.
* @param period {Long} The limiting duration. Defaults to 1.
* @param unit {TimeUnit} The unit of time for the period. Defaults to seconds.
*/
fun OkHttpClient.Builder.rateLimit(
permits: Int,
period: Long = 1,
unit: TimeUnit = TimeUnit.SECONDS,
) = addInterceptor(RateLimitInterceptor(null, permits, period, unit))
/** We can probably accept domains or wildcards by comparing with [endsWith], etc. */
@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
internal class RateLimitInterceptor(
private val host: String?,
private val permits: Int,
period: Long,
unit: TimeUnit,
) : Interceptor {
private val requestQueue = ArrayDeque<Long>(permits)
private val rateLimitMillis = unit.toMillis(period)
private val fairLock = Semaphore(1, true)
override fun intercept(chain: Interceptor.Chain): Response {
val call = chain.call()
if (call.isCanceled()) throw IOException("Canceled")
val request = chain.request()
when (host) {
null, request.url.host -> {} // need rate limit
else -> return chain.proceed(request)
}
try {
fairLock.acquire()
} catch (e: InterruptedException) {
throw IOException(e)
}
val requestQueue = this.requestQueue
val timestamp: Long
try {
synchronized(requestQueue) {
while (requestQueue.size >= permits) { // queue is full, remove expired entries
val periodStart = SystemClock.elapsedRealtime() - rateLimitMillis
var hasRemovedExpired = false
while (requestQueue.isEmpty().not() && requestQueue.first <= periodStart) {
requestQueue.removeFirst()
hasRemovedExpired = true
}
if (call.isCanceled()) {
throw IOException("Canceled")
} else if (hasRemovedExpired) {
break
} else {
try { // wait for the first entry to expire, or notified by cached response
(requestQueue as Object).wait(requestQueue.first - periodStart)
} catch (_: InterruptedException) {
continue
}
}
}
// add request to queue
timestamp = SystemClock.elapsedRealtime()
requestQueue.addLast(timestamp)
}
} finally {
fairLock.release()
}
val response = chain.proceed(request)
if (response.networkResponse == null) { // response is cached, remove it from queue
synchronized(requestQueue) {
if (requestQueue.isEmpty() || timestamp < requestQueue.first) return@synchronized
requestQueue.removeFirstOccurrence(timestamp)
(requestQueue as Object).notifyAll()
}
}
return response
}
}

View File

@ -0,0 +1,27 @@
package eu.kanade.tachiyomi.network.interceptor
import okhttp3.HttpUrl
import okhttp3.OkHttpClient
import java.util.concurrent.TimeUnit
/**
* An OkHttp interceptor that handles given url host's rate limiting.
*
* Examples:
*
* httpUrl = "api.manga.com".toHttpUrlOrNull(), permits = 5, period = 1, unit = seconds => 5 requests per second to api.manga.com
* httpUrl = "imagecdn.manga.com".toHttpUrlOrNull(), permits = 10, period = 2, unit = minutes => 10 requests per 2 minutes to imagecdn.manga.com
*
* @since extension-lib 1.3
*
* @param httpUrl {HttpUrl} The url host that this interceptor should handle. Will get url's host by using HttpUrl.host()
* @param permits {Int} Number of requests allowed within a period of units.
* @param period {Long} The limiting duration. Defaults to 1.
* @param unit {TimeUnit} The unit of time for the period. Defaults to seconds.
*/
fun OkHttpClient.Builder.rateLimitHost(
httpUrl: HttpUrl,
permits: Int,
period: Long = 1,
unit: TimeUnit = TimeUnit.SECONDS,
) = addInterceptor(RateLimitInterceptor(httpUrl.host, permits, period, unit))

View File

@ -0,0 +1,26 @@
package eu.kanade.tachiyomi.network.interceptor
import eu.kanade.tachiyomi.network.NetworkHelper
import okhttp3.Interceptor
import okhttp3.Response
import uy.kohesive.injekt.injectLazy
class UserAgentInterceptor : Interceptor {
private val networkHelper: NetworkHelper by injectLazy()
override fun intercept(chain: Interceptor.Chain): Response {
val originalRequest = chain.request()
return if (originalRequest.header("User-Agent").isNullOrEmpty()) {
val newRequest = originalRequest
.newBuilder()
.removeHeader("User-Agent")
.addHeader("User-Agent", networkHelper.defaultUserAgent)
.build()
chain.proceed(newRequest)
} else {
chain.proceed(originalRequest)
}
}
}

View File

@ -0,0 +1,68 @@
package eu.kanade.tachiyomi.network.interceptor
import android.content.Context
import android.os.Build
import android.webkit.WebSettings
import android.webkit.WebView
import android.widget.Toast
import eu.kanade.tachiyomi.core.R
import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.util.lang.launchUI
import eu.kanade.tachiyomi.util.system.DeviceUtil
import eu.kanade.tachiyomi.util.system.WebViewUtil
import eu.kanade.tachiyomi.util.system.setDefaultSettings
import eu.kanade.tachiyomi.util.system.toast
import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
import uy.kohesive.injekt.injectLazy
abstract class WebViewInterceptor(private val context: Context) : Interceptor {
private val networkHelper: NetworkHelper by injectLazy()
/**
* When this is called, it initializes the WebView if it wasn't already. We use this to avoid
* blocking the main thread too much. If used too often we could consider moving it to the
* Application class.
*/
private val initWebView by lazy {
// Crashes on some devices. We skip this in some cases since the only impact is slower
// WebView init in those rare cases.
// See https://bugs.chromium.org/p/chromium/issues/detail?id=1279562
if (DeviceUtil.isMiui || Build.VERSION.SDK_INT == Build.VERSION_CODES.S && DeviceUtil.isSamsung) {
return@lazy
}
WebSettings.getDefaultUserAgent(context)
}
abstract fun shouldIntercept(response: Response): Boolean
abstract fun intercept(chain: Interceptor.Chain, request: Request, response: Response): Response
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)
if (!shouldIntercept(response)) {
return response
}
if (!WebViewUtil.supportsWebView(context)) {
launchUI {
context.toast(R.string.information_webview_required, Toast.LENGTH_LONG)
}
return response
}
initWebView
return intercept(chain, request, response)
}
fun createWebView(request: Request): WebView {
val webview = WebView(context)
webview.setDefaultSettings()
webview.settings.userAgentString = request.header("User-Agent") ?: networkHelper.defaultUserAgent
return webview
}
}

View File

@ -0,0 +1,57 @@
package eu.kanade.tachiyomi.util.lang
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.DelicateCoroutinesApi
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
/**
* Think twice before using this. This is a delicate API. It is easy to accidentally create resource or memory leaks when GlobalScope is used.
*
* **Possible replacements**
* - suspend function
* - custom scope like view or presenter scope
*/
@DelicateCoroutinesApi
fun launchUI(block: suspend CoroutineScope.() -> Unit): Job =
GlobalScope.launch(Dispatchers.Main, CoroutineStart.DEFAULT, block)
/**
* Think twice before using this. This is a delicate API. It is easy to accidentally create resource or memory leaks when GlobalScope is used.
*
* **Possible replacements**
* - suspend function
* - custom scope like view or presenter scope
*/
@DelicateCoroutinesApi
fun launchIO(block: suspend CoroutineScope.() -> Unit): Job =
GlobalScope.launch(Dispatchers.IO, CoroutineStart.DEFAULT, block)
/**
* Think twice before using this. This is a delicate API. It is easy to accidentally create resource or memory leaks when GlobalScope is used.
*
* **Possible replacements**
* - suspend function
* - custom scope like view or presenter scope
*/
@DelicateCoroutinesApi
fun launchNow(block: suspend CoroutineScope.() -> Unit): Job =
GlobalScope.launch(Dispatchers.Main, CoroutineStart.UNDISPATCHED, block)
fun CoroutineScope.launchUI(block: suspend CoroutineScope.() -> Unit): Job =
launch(Dispatchers.Main, block = block)
fun CoroutineScope.launchIO(block: suspend CoroutineScope.() -> Unit): Job =
launch(Dispatchers.IO, block = block)
fun CoroutineScope.launchNonCancellableIO(block: suspend CoroutineScope.() -> Unit): Job =
launchIO { withContext(NonCancellable, block) }
suspend fun <T> withUIContext(block: suspend CoroutineScope.() -> T) = withContext(Dispatchers.Main, block)
suspend fun <T> withIOContext(block: suspend CoroutineScope.() -> T) = withContext(Dispatchers.IO, block)

View File

@ -0,0 +1,89 @@
package eu.kanade.tachiyomi.util.lang
import kotlinx.coroutines.CancellableContinuation
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.InternalCoroutinesApi
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import rx.Emitter
import rx.Observable
import rx.Subscriber
import rx.Subscription
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
/*
* Util functions for bridging RxJava and coroutines. Taken from TachiyomiEH/SY.
*/
suspend fun <T> Observable<T>.awaitSingle(): T = single().awaitOne()
@OptIn(InternalCoroutinesApi::class)
private suspend fun <T> Observable<T>.awaitOne(): T = suspendCancellableCoroutine { cont ->
cont.unsubscribeOnCancellation(
subscribe(
object : Subscriber<T>() {
override fun onStart() {
request(1)
}
override fun onNext(t: T) {
cont.resume(t)
}
override fun onCompleted() {
if (cont.isActive) {
cont.resumeWithException(
IllegalStateException(
"Should have invoked onNext",
),
)
}
}
override fun onError(e: Throwable) {
/*
* Rx1 observable throws NoSuchElementException if cancellation happened before
* element emission. To mitigate this we try to atomically resume continuation with exception:
* if resume failed, then we know that continuation successfully cancelled itself
*/
val token = cont.tryResumeWithException(e)
if (token != null) {
cont.completeResume(token)
}
}
},
),
)
}
internal fun <T> CancellableContinuation<T>.unsubscribeOnCancellation(sub: Subscription) =
invokeOnCancellation { sub.unsubscribe() }
fun <T> runAsObservable(
backpressureMode: Emitter.BackpressureMode = Emitter.BackpressureMode.NONE,
block: suspend () -> T,
): Observable<T> {
return Observable.create(
{ emitter ->
val job = GlobalScope.launch(Dispatchers.Unconfined, start = CoroutineStart.ATOMIC) {
try {
emitter.onNext(block())
emitter.onCompleted()
} catch (e: Throwable) {
// Ignore `CancellationException` as error, since it indicates "normal cancellation"
if (e !is CancellationException) {
emitter.onError(e)
} else {
emitter.onCompleted()
}
}
}
emitter.setCancellation { job.cancel() }
},
backpressureMode,
)
}

View File

@ -0,0 +1,46 @@
package eu.kanade.tachiyomi.util.system
import android.annotation.SuppressLint
import android.os.Build
import logcat.LogPriority
object DeviceUtil {
val isMiui by lazy {
getSystemProperty("ro.miui.ui.version.name")?.isNotEmpty() ?: false
}
@SuppressLint("PrivateApi")
fun isMiuiOptimizationDisabled(): Boolean {
val sysProp = getSystemProperty("persist.sys.miui_optimization")
if (sysProp == "0" || sysProp == "false") {
return true
}
return try {
Class.forName("android.miui.AppOpsUtils")
.getDeclaredMethod("isXOptMode")
.invoke(null) as Boolean
} catch (e: Exception) {
false
}
}
val isSamsung by lazy {
Build.MANUFACTURER.equals("samsung", ignoreCase = true)
}
val invalidDefaultBrowsers = listOf("android", "com.huawei.android.internal.app")
@SuppressLint("PrivateApi")
private fun getSystemProperty(key: String?): String? {
return try {
Class.forName("android.os.SystemProperties")
.getDeclaredMethod("get", String::class.java)
.invoke(null, key) as String
} catch (e: Exception) {
logcat(LogPriority.WARN, e) { "Unable to use SystemProperties.get()" }
null
}
}
}

View File

@ -0,0 +1,18 @@
package eu.kanade.tachiyomi.util.system
import logcat.LogPriority
import logcat.asLog
import logcat.logcat
inline fun Any.logcat(
priority: LogPriority = LogPriority.DEBUG,
throwable: Throwable? = null,
message: () -> String = { "" },
) = logcat(priority = priority) {
var msg = message()
if (throwable != null) {
if (msg.isNotBlank()) msg += "\n"
msg += throwable.asLog()
}
msg
}

View File

@ -0,0 +1,28 @@
package eu.kanade.tachiyomi.util.system
import android.content.Context
import android.widget.Toast
import androidx.annotation.StringRes
/**
* Display a toast in this context.
*
* @param resource the text resource.
* @param duration the duration of the toast. Defaults to short.
*/
fun Context.toast(@StringRes resource: Int, duration: Int = Toast.LENGTH_SHORT, block: (Toast) -> Unit = {}): Toast {
return toast(getString(resource), duration, block)
}
/**
* Display a toast in this context.
*
* @param text the text to display.
* @param duration the duration of the toast. Defaults to short.
*/
fun Context.toast(text: String?, duration: Int = Toast.LENGTH_SHORT, block: (Toast) -> Unit = {}): Toast {
return Toast.makeText(applicationContext, text.orEmpty(), duration).also {
block(it)
it.show()
}
}

View File

@ -0,0 +1,94 @@
package eu.kanade.tachiyomi.util.system
import android.annotation.TargetApi
import android.os.Build
import android.webkit.WebResourceError
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebView
import android.webkit.WebViewClient
@Suppress("OverridingDeprecatedMember")
abstract class WebViewClientCompat : WebViewClient() {
open fun shouldOverrideUrlCompat(view: WebView, url: String): Boolean {
return false
}
open fun shouldInterceptRequestCompat(view: WebView, url: String): WebResourceResponse? {
return null
}
open fun onReceivedErrorCompat(
view: WebView,
errorCode: Int,
description: String?,
failingUrl: String,
isMainFrame: Boolean,
) {
}
@TargetApi(Build.VERSION_CODES.N)
final override fun shouldOverrideUrlLoading(
view: WebView,
request: WebResourceRequest,
): Boolean {
return shouldOverrideUrlCompat(view, request.url.toString())
}
final override fun shouldOverrideUrlLoading(view: WebView, url: String): Boolean {
return shouldOverrideUrlCompat(view, url)
}
final override fun shouldInterceptRequest(
view: WebView,
request: WebResourceRequest,
): WebResourceResponse? {
return shouldInterceptRequestCompat(view, request.url.toString())
}
final override fun shouldInterceptRequest(
view: WebView,
url: String,
): WebResourceResponse? {
return shouldInterceptRequestCompat(view, url)
}
final override fun onReceivedError(
view: WebView,
request: WebResourceRequest,
error: WebResourceError,
) {
onReceivedErrorCompat(
view,
error.errorCode,
error.description?.toString(),
request.url.toString(),
request.isForMainFrame,
)
}
final override fun onReceivedError(
view: WebView,
errorCode: Int,
description: String?,
failingUrl: String,
) {
onReceivedErrorCompat(view, errorCode, description, failingUrl, failingUrl == view.url)
}
final override fun onReceivedHttpError(
view: WebView,
request: WebResourceRequest,
error: WebResourceResponse,
) {
onReceivedErrorCompat(
view,
error.statusCode,
error.reasonPhrase,
request.url
.toString(),
request.isForMainFrame,
)
}
}

View File

@ -0,0 +1,67 @@
package eu.kanade.tachiyomi.util.system
import android.annotation.SuppressLint
import android.content.Context
import android.content.pm.PackageManager
import android.webkit.CookieManager
import android.webkit.WebSettings
import android.webkit.WebView
import logcat.LogPriority
object WebViewUtil {
const val SPOOF_PACKAGE_NAME = "org.chromium.chrome"
const val MINIMUM_WEBVIEW_VERSION = 100
fun supportsWebView(context: Context): Boolean {
try {
// May throw android.webkit.WebViewFactory$MissingWebViewPackageException if WebView
// is not installed
CookieManager.getInstance()
} catch (e: Throwable) {
logcat(LogPriority.ERROR, e)
return false
}
return context.packageManager.hasSystemFeature(PackageManager.FEATURE_WEBVIEW)
}
}
fun WebView.isOutdated(): Boolean {
return getWebViewMajorVersion() < WebViewUtil.MINIMUM_WEBVIEW_VERSION
}
@SuppressLint("SetJavaScriptEnabled")
fun WebView.setDefaultSettings() {
with(settings) {
javaScriptEnabled = true
domStorageEnabled = true
databaseEnabled = true
useWideViewPort = true
loadWithOverviewMode = true
cacheMode = WebSettings.LOAD_DEFAULT
}
}
private fun WebView.getWebViewMajorVersion(): Int {
val uaRegexMatch = """.*Chrome/(\d+)\..*""".toRegex().matchEntire(getDefaultUserAgentString())
return if (uaRegexMatch != null && uaRegexMatch.groupValues.size > 1) {
uaRegexMatch.groupValues[1].toInt()
} else {
0
}
}
// Based on https://stackoverflow.com/a/29218966
private fun WebView.getDefaultUserAgentString(): String {
val originalUA: String = settings.userAgentString
// Next call to getUserAgentString() will get us the default
settings.userAgentString = null
val defaultUserAgentString = settings.userAgentString
// Revert to original UA string
settings.userAgentString = originalUA
return defaultUserAgentString
}