From 63fcca3c7ddd0d0c59697fc3f47b9d30b34b5293 Mon Sep 17 00:00:00 2001 From: FooIbar <118464521+FooIbar@users.noreply.github.com> Date: Sat, 1 Jul 2023 13:16:28 +0800 Subject: [PATCH] Bypass Cloudflare with webview --- .../java/com/hippo/ehviewer/EhApplication.kt | 3 + .../hippo/ehviewer/client/EhCookieStore.kt | 4 + .../ehviewer/ui/login/WebviewSignInScreen.kt | 9 +- .../ehviewer/ui/settings/MyTagsScreen.kt | 9 +- .../ehviewer/ui/settings/UConfigScreen.kt | 9 +- .../hippo/ehviewer/util/WebViewExtensions.kt | 11 ++ .../interceptor/CloudflareInterceptor.kt | 105 ++++++++++++++++++ .../network/interceptor/WebViewInterceptor.kt | 86 ++++++++++++++ 8 files changed, 215 insertions(+), 21 deletions(-) create mode 100644 app/src/main/java/com/hippo/ehviewer/util/WebViewExtensions.kt create mode 100644 app/src/main/java/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt create mode 100644 app/src/main/java/eu/kanade/tachiyomi/network/interceptor/WebViewInterceptor.kt diff --git a/app/src/main/java/com/hippo/ehviewer/EhApplication.kt b/app/src/main/java/com/hippo/ehviewer/EhApplication.kt index e53d8e1c0..ac85efda1 100644 --- a/app/src/main/java/com/hippo/ehviewer/EhApplication.kt +++ b/app/src/main/java/com/hippo/ehviewer/EhApplication.kt @@ -45,6 +45,7 @@ import com.hippo.ehviewer.ui.keepNoMediaFileStatus import com.hippo.ehviewer.ui.lockObserver import com.hippo.ehviewer.util.ReadableTime import com.hippo.ehviewer.yorozuya.FileUtils +import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor import eu.kanade.tachiyomi.util.lang.launchIO import eu.kanade.tachiyomi.util.lang.withUIContext import kotlinx.coroutines.DelicateCoroutinesApi @@ -158,6 +159,8 @@ class EhApplication : Application(), ImageLoaderFactory { }.build(), ) + addInterceptor(CloudflareInterceptor(appCtx)) + // TODO: Rewrite CronetInterceptor to use android.net.http.HttpEngine and make it Android 14 only when released addInterceptor { chain -> val request = chain.request() diff --git a/app/src/main/java/com/hippo/ehviewer/client/EhCookieStore.kt b/app/src/main/java/com/hippo/ehviewer/client/EhCookieStore.kt index c0005b0fb..4f437fd97 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/EhCookieStore.kt +++ b/app/src/main/java/com/hippo/ehviewer/client/EhCookieStore.kt @@ -62,6 +62,10 @@ object EhCookieStore : CookieJar { flush() } + fun deleteCookie(url: HttpUrl, name: String) { + manager.setCookie(url.toString(), "$name=;Max-Age=0") + } + fun addCookie(cookie: Cookie) { if (EhUrl.DOMAIN_E in cookie.domain) manager.setCookie(EhUrl.HOST_E, cookie.toString()) else manager.setCookie(EhUrl.DOMAIN_EX, cookie.toString()) } diff --git a/app/src/main/java/com/hippo/ehviewer/ui/login/WebviewSignInScreen.kt b/app/src/main/java/com/hippo/ehviewer/ui/login/WebviewSignInScreen.kt index 1005f612b..de1f5ffa4 100644 --- a/app/src/main/java/com/hippo/ehviewer/ui/login/WebviewSignInScreen.kt +++ b/app/src/main/java/com/hippo/ehviewer/ui/login/WebviewSignInScreen.kt @@ -1,6 +1,5 @@ package com.hippo.ehviewer.ui.login -import android.annotation.SuppressLint import android.webkit.CookieManager import android.webkit.WebView import androidx.compose.foundation.layout.fillMaxSize @@ -18,12 +17,12 @@ import com.hippo.ehviewer.client.EhUtils import com.hippo.ehviewer.ui.FINISH_ROUTE_NAME import com.hippo.ehviewer.ui.LocalNavController import com.hippo.ehviewer.ui.SELECT_SITE_ROUTE_NAME +import com.hippo.ehviewer.util.setDefaultSettings import eu.kanade.tachiyomi.util.lang.launchIO import eu.kanade.tachiyomi.util.lang.withNonCancellableContext import eu.kanade.tachiyomi.util.lang.withUIContext import okhttp3.HttpUrl.Companion.toHttpUrl -@SuppressLint("SetJavaScriptEnabled") @Composable fun WebviewSignInScreen() { val navController = LocalNavController.current @@ -68,11 +67,7 @@ fun WebviewSignInScreen() { state = state, modifier = Modifier.fillMaxSize(), onCreated = { - it.settings.run { - builtInZoomControls = true - displayZoomControls = true - javaScriptEnabled = true - } + it.setDefaultSettings() }, client = client, ) diff --git a/app/src/main/java/com/hippo/ehviewer/ui/settings/MyTagsScreen.kt b/app/src/main/java/com/hippo/ehviewer/ui/settings/MyTagsScreen.kt index 8017f41bb..363774031 100644 --- a/app/src/main/java/com/hippo/ehviewer/ui/settings/MyTagsScreen.kt +++ b/app/src/main/java/com/hippo/ehviewer/ui/settings/MyTagsScreen.kt @@ -1,6 +1,5 @@ package com.hippo.ehviewer.ui.settings -import android.annotation.SuppressLint import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding import androidx.compose.material.icons.Icons @@ -18,8 +17,8 @@ import com.google.accompanist.web.rememberWebViewState import com.hippo.ehviewer.R import com.hippo.ehviewer.client.EhUrl import com.hippo.ehviewer.ui.LocalNavController +import com.hippo.ehviewer.util.setDefaultSettings -@SuppressLint("SetJavaScriptEnabled") @Composable fun MyTagsScreen() { val navController = LocalNavController.current @@ -41,11 +40,7 @@ fun MyTagsScreen() { state = state, modifier = Modifier.padding(top = paddingValues.calculateTopPadding()).fillMaxSize(), onCreated = { - it.settings.run { - builtInZoomControls = true - displayZoomControls = false - javaScriptEnabled = true - } + it.setDefaultSettings() }, ) } diff --git a/app/src/main/java/com/hippo/ehviewer/ui/settings/UConfigScreen.kt b/app/src/main/java/com/hippo/ehviewer/ui/settings/UConfigScreen.kt index a267919b6..d21fb8a14 100644 --- a/app/src/main/java/com/hippo/ehviewer/ui/settings/UConfigScreen.kt +++ b/app/src/main/java/com/hippo/ehviewer/ui/settings/UConfigScreen.kt @@ -1,6 +1,5 @@ package com.hippo.ehviewer.ui.settings -import android.annotation.SuppressLint import android.webkit.WebView import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding @@ -27,10 +26,10 @@ import com.hippo.ehviewer.R import com.hippo.ehviewer.client.EhCookieStore import com.hippo.ehviewer.client.EhUrl import com.hippo.ehviewer.ui.LocalNavController +import com.hippo.ehviewer.util.setDefaultSettings private const val applyJs = "javascript:(function(){var apply = document.getElementById(\"apply\").children[0];apply.click();})();" -@SuppressLint("SetJavaScriptEnabled") @Composable fun UConfigScreen() { val navController = LocalNavController.current @@ -65,11 +64,7 @@ fun UConfigScreen() { state = state, modifier = Modifier.padding(top = paddingValues.calculateTopPadding()).fillMaxSize(), onCreated = { - it.settings.run { - builtInZoomControls = true - displayZoomControls = false - javaScriptEnabled = true - } + it.setDefaultSettings() }, factory = { WebView(it).apply { webview.set(this) } }, ) diff --git a/app/src/main/java/com/hippo/ehviewer/util/WebViewExtensions.kt b/app/src/main/java/com/hippo/ehviewer/util/WebViewExtensions.kt new file mode 100644 index 000000000..d68fb01b5 --- /dev/null +++ b/app/src/main/java/com/hippo/ehviewer/util/WebViewExtensions.kt @@ -0,0 +1,11 @@ +package com.hippo.ehviewer.util + +import android.annotation.SuppressLint +import android.webkit.WebView + +@SuppressLint("SetJavaScriptEnabled") +fun WebView.setDefaultSettings() = settings.run { + builtInZoomControls = true + displayZoomControls = false + javaScriptEnabled = true +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt b/app/src/main/java/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt new file mode 100644 index 000000000..f783eb92e --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt @@ -0,0 +1,105 @@ +package eu.kanade.tachiyomi.network.interceptor + +import android.content.Context +import android.webkit.WebResourceError +import android.webkit.WebResourceRequest +import android.webkit.WebView +import android.webkit.WebViewClient +import androidx.core.content.ContextCompat +import com.hippo.ehviewer.client.EhCookieStore +import com.hippo.ehviewer.client.exception.EhException +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Request +import okhttp3.Response +import java.util.concurrent.CountDownLatch + +class CloudflareInterceptor(context: Context) : WebViewInterceptor(context) { + private val executor = ContextCompat.getMainExecutor(context) + + override fun shouldIntercept(response: Response): Boolean { + // Check if Cloudflare anti-bot is on + return response.code in ERROR_CODES && response.header("Server") in SERVER_CHECK + } + + override fun intercept( + chain: Interceptor.Chain, + request: Request, + response: Response, + ): Response { + response.close() + EhCookieStore.deleteCookie(request.url, COOKIE_NAME) + resolveWithWebView(request) + + return chain.proceed(request) + } + + private fun resolveWithWebView(originalRequest: Request) { + // We need to lock this thread until the WebView finds the challenge solution url, because + // OkHttp doesn't support asynchronous interceptors. + val latch = CountDownLatch(1) + + var webview: WebView? = null + + var challengeFound = false + var cloudflareBypassed = false + + val origRequestUrl = originalRequest.url.toString() + val headers = parseHeaders(originalRequest.headers) + + executor.execute { + webview = createWebView(originalRequest) + + webview?.webViewClient = object : WebViewClient() { + override fun onPageFinished(view: WebView, url: String) { + fun isCloudFlareBypassed(): Boolean { + return EhCookieStore.get(origRequestUrl.toHttpUrl()) + .any { it.name == COOKIE_NAME } + } + + if (isCloudFlareBypassed()) { + cloudflareBypassed = true + latch.countDown() + } + + if (url == origRequestUrl && !challengeFound) { + // The first request didn't return the challenge, abort. + latch.countDown() + } + } + + override fun onReceivedError( + view: WebView, + request: WebResourceRequest, + error: WebResourceError, + ) { + if (request.isForMainFrame && error.errorCode in ERROR_CODES) { + // Found the Cloudflare challenge page. + challengeFound = true + } else { + // Unlock thread, the challenge wasn't found. + latch.countDown() + } + } + } + + webview?.loadUrl(origRequestUrl, headers) + } + + latch.awaitFor30Seconds() + + executor.execute { + webview?.run { + stopLoading() + destroy() + } + } + + // Throw exception if we failed to bypass Cloudflare + if (!cloudflareBypassed) throw EhException("Failed to bypass Cloudflare") + } +} + +private val ERROR_CODES = intArrayOf(403, 503) +private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare") +private const val COOKIE_NAME = "cf_clearance" diff --git a/app/src/main/java/eu/kanade/tachiyomi/network/interceptor/WebViewInterceptor.kt b/app/src/main/java/eu/kanade/tachiyomi/network/interceptor/WebViewInterceptor.kt new file mode 100644 index 000000000..5eef8000d --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/network/interceptor/WebViewInterceptor.kt @@ -0,0 +1,86 @@ +package eu.kanade.tachiyomi.network.interceptor + +import android.content.Context +import android.webkit.WebSettings +import android.webkit.WebView +import com.hippo.ehviewer.util.setDefaultSettings +import okhttp3.Headers +import okhttp3.Interceptor +import okhttp3.Request +import okhttp3.Response +import java.util.Locale +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +abstract class WebViewInterceptor(private val context: Context) : Interceptor { + /** + * When this is called, it initializes the WebView if it wasn't already. We use this to avoid + * blocking the main thread too much. If used too often we could consider moving it to the + * Application class. + */ + private val initWebView by lazy { + try { + WebSettings.getDefaultUserAgent(context) + } catch (_: Exception) { + // Avoid some crashes like when Chrome/WebView is being updated. + } + } + + abstract fun shouldIntercept(response: Response): Boolean + + abstract fun intercept(chain: Interceptor.Chain, request: Request, response: Response): Response + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + if (!shouldIntercept(response)) { + return response + } + + initWebView + + return intercept(chain, request, response) + } + + fun parseHeaders(headers: Headers): Map { + return headers + // Keeping unsafe header makes webview throw [net::ERR_INVALID_ARGUMENT] + .filter { (name, value) -> + isRequestHeaderSafe(name, value) + } + .groupBy(keySelector = { (name, _) -> name }) { (_, value) -> value } + .mapValues { it.value.getOrNull(0).orEmpty() } + } + + fun CountDownLatch.awaitFor30Seconds() { + await(30, TimeUnit.SECONDS) + } + + fun createWebView(request: Request): WebView { + return WebView(context).apply { + setDefaultSettings() + // Avoid sending empty User-Agent, Chromium WebView will reset to default if empty + settings.userAgentString = request.header("User-Agent") + } + } +} + +// Based on [IsRequestHeaderSafe] in https://source.chromium.org/chromium/chromium/src/+/main:services/network/public/cpp/header_util.cc +private fun isRequestHeaderSafe(_name: String, _value: String): Boolean { + val name = _name.lowercase(Locale.ENGLISH) + val value = _value.lowercase(Locale.ENGLISH) + if (name in unsafeHeaderNames || name.startsWith("proxy-")) return false + return !(name == "connection" && value == "upgrade") +} + +private val unsafeHeaderNames = arrayOf( + "content-length", + "host", + "trailer", + "te", + "upgrade", + "cookie2", + "keep-alive", + "transfer-encoding", + "set-cookie", +)