Bypass Cloudflare with webview

app/src/main/java/com/hippo/ehviewer/EhApplication.kt

@@ -45,6 +45,7 @@ import com.hippo.ehviewer.ui.keepNoMediaFileStatus
 import com.hippo.ehviewer.ui.lockObserver
 import com.hippo.ehviewer.util.ReadableTime
 import com.hippo.ehviewer.yorozuya.FileUtils
+import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor
 import eu.kanade.tachiyomi.util.lang.launchIO
 import eu.kanade.tachiyomi.util.lang.withUIContext
 import kotlinx.coroutines.DelicateCoroutinesApi
@@ -158,6 +159,8 @@ class EhApplication : Application(), ImageLoaderFactory {
                     }.build(),
                 )
 
+                addInterceptor(CloudflareInterceptor(appCtx))
+
                 // TODO: Rewrite CronetInterceptor to use android.net.http.HttpEngine and make it Android 14 only when released
                 addInterceptor { chain ->
                     val request = chain.request()

app/src/main/java/com/hippo/ehviewer/client/EhCookieStore.kt

@@ -62,6 +62,10 @@ object EhCookieStore : CookieJar {
         flush()
     }
 
+    fun deleteCookie(url: HttpUrl, name: String) {
+        manager.setCookie(url.toString(), "$name=;Max-Age=0")
+    }
+
     fun addCookie(cookie: Cookie) {
         if (EhUrl.DOMAIN_E in cookie.domain) manager.setCookie(EhUrl.HOST_E, cookie.toString()) else manager.setCookie(EhUrl.DOMAIN_EX, cookie.toString())
     }

app/src/main/java/com/hippo/ehviewer/ui/login/WebviewSignInScreen.kt

@@ -1,6 +1,5 @@
 package com.hippo.ehviewer.ui.login
 
-import android.annotation.SuppressLint
 import android.webkit.CookieManager
 import android.webkit.WebView
 import androidx.compose.foundation.layout.fillMaxSize
@@ -18,12 +17,12 @@ import com.hippo.ehviewer.client.EhUtils
 import com.hippo.ehviewer.ui.FINISH_ROUTE_NAME
 import com.hippo.ehviewer.ui.LocalNavController
 import com.hippo.ehviewer.ui.SELECT_SITE_ROUTE_NAME
+import com.hippo.ehviewer.util.setDefaultSettings
 import eu.kanade.tachiyomi.util.lang.launchIO
 import eu.kanade.tachiyomi.util.lang.withNonCancellableContext
 import eu.kanade.tachiyomi.util.lang.withUIContext
 import okhttp3.HttpUrl.Companion.toHttpUrl
 
-@SuppressLint("SetJavaScriptEnabled")
 @Composable
 fun WebviewSignInScreen() {
     val navController = LocalNavController.current
@@ -68,11 +67,7 @@ fun WebviewSignInScreen() {
         state = state,
         modifier = Modifier.fillMaxSize(),
         onCreated = {
-            it.settings.run {
-                builtInZoomControls = true
-                displayZoomControls = true
-                javaScriptEnabled = true
-            }
+            it.setDefaultSettings()
         },
         client = client,
     )

app/src/main/java/com/hippo/ehviewer/ui/settings/MyTagsScreen.kt

@@ -1,6 +1,5 @@
 package com.hippo.ehviewer.ui.settings
 
-import android.annotation.SuppressLint
 import androidx.compose.foundation.layout.fillMaxSize
 import androidx.compose.foundation.layout.padding
 import androidx.compose.material.icons.Icons
@@ -18,8 +17,8 @@ import com.google.accompanist.web.rememberWebViewState
 import com.hippo.ehviewer.R
 import com.hippo.ehviewer.client.EhUrl
 import com.hippo.ehviewer.ui.LocalNavController
+import com.hippo.ehviewer.util.setDefaultSettings
 
-@SuppressLint("SetJavaScriptEnabled")
 @Composable
 fun MyTagsScreen() {
     val navController = LocalNavController.current
@@ -41,11 +40,7 @@ fun MyTagsScreen() {
             state = state,
             modifier = Modifier.padding(top = paddingValues.calculateTopPadding()).fillMaxSize(),
             onCreated = {
-                it.settings.run {
-                    builtInZoomControls = true
-                    displayZoomControls = false
-                    javaScriptEnabled = true
-                }
+                it.setDefaultSettings()
             },
         )
     }

app/src/main/java/com/hippo/ehviewer/ui/settings/UConfigScreen.kt

@@ -1,6 +1,5 @@
 package com.hippo.ehviewer.ui.settings
 
-import android.annotation.SuppressLint
 import android.webkit.WebView
 import androidx.compose.foundation.layout.fillMaxSize
 import androidx.compose.foundation.layout.padding
@@ -27,10 +26,10 @@ import com.hippo.ehviewer.R
 import com.hippo.ehviewer.client.EhCookieStore
 import com.hippo.ehviewer.client.EhUrl
 import com.hippo.ehviewer.ui.LocalNavController
+import com.hippo.ehviewer.util.setDefaultSettings
 
 private const val applyJs = "javascript:(function(){var apply = document.getElementById(\"apply\").children[0];apply.click();})();"
 
-@SuppressLint("SetJavaScriptEnabled")
 @Composable
 fun UConfigScreen() {
     val navController = LocalNavController.current
@@ -65,11 +64,7 @@ fun UConfigScreen() {
             state = state,
             modifier = Modifier.padding(top = paddingValues.calculateTopPadding()).fillMaxSize(),
             onCreated = {
-                it.settings.run {
-                    builtInZoomControls = true
-                    displayZoomControls = false
-                    javaScriptEnabled = true
-                }
+                it.setDefaultSettings()
             },
             factory = { WebView(it).apply { webview.set(this) } },
         )

app/src/main/java/com/hippo/ehviewer/util/WebViewExtensions.kt

@@ -0,0 +1,11 @@
+package com.hippo.ehviewer.util
+
+import android.annotation.SuppressLint
+import android.webkit.WebView
+
+@SuppressLint("SetJavaScriptEnabled")
+fun WebView.setDefaultSettings() = settings.run {
+    builtInZoomControls = true
+    displayZoomControls = false
+    javaScriptEnabled = true
+}

app/src/main/java/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt

@@ -0,0 +1,105 @@
+package eu.kanade.tachiyomi.network.interceptor
+
+import android.content.Context
+import android.webkit.WebResourceError
+import android.webkit.WebResourceRequest
+import android.webkit.WebView
+import android.webkit.WebViewClient
+import androidx.core.content.ContextCompat
+import com.hippo.ehviewer.client.EhCookieStore
+import com.hippo.ehviewer.client.exception.EhException
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Interceptor
+import okhttp3.Request
+import okhttp3.Response
+import java.util.concurrent.CountDownLatch
+
+class CloudflareInterceptor(context: Context) : WebViewInterceptor(context) {
+    private val executor = ContextCompat.getMainExecutor(context)
+
+    override fun shouldIntercept(response: Response): Boolean {
+        // Check if Cloudflare anti-bot is on
+        return response.code in ERROR_CODES && response.header("Server") in SERVER_CHECK
+    }
+
+    override fun intercept(
+        chain: Interceptor.Chain,
+        request: Request,
+        response: Response,
+    ): Response {
+        response.close()
+        EhCookieStore.deleteCookie(request.url, COOKIE_NAME)
+        resolveWithWebView(request)
+
+        return chain.proceed(request)
+    }
+
+    private fun resolveWithWebView(originalRequest: Request) {
+        // We need to lock this thread until the WebView finds the challenge solution url, because
+        // OkHttp doesn't support asynchronous interceptors.
+        val latch = CountDownLatch(1)
+
+        var webview: WebView? = null
+
+        var challengeFound = false
+        var cloudflareBypassed = false
+
+        val origRequestUrl = originalRequest.url.toString()
+        val headers = parseHeaders(originalRequest.headers)
+
+        executor.execute {
+            webview = createWebView(originalRequest)
+
+            webview?.webViewClient = object : WebViewClient() {
+                override fun onPageFinished(view: WebView, url: String) {
+                    fun isCloudFlareBypassed(): Boolean {
+                        return EhCookieStore.get(origRequestUrl.toHttpUrl())
+                            .any { it.name == COOKIE_NAME }
+                    }
+
+                    if (isCloudFlareBypassed()) {
+                        cloudflareBypassed = true
+                        latch.countDown()
+                    }
+
+                    if (url == origRequestUrl && !challengeFound) {
+                        // The first request didn't return the challenge, abort.
+                        latch.countDown()
+                    }
+                }
+
+                override fun onReceivedError(
+                    view: WebView,
+                    request: WebResourceRequest,
+                    error: WebResourceError,
+                ) {
+                    if (request.isForMainFrame && error.errorCode in ERROR_CODES) {
+                        // Found the Cloudflare challenge page.
+                        challengeFound = true
+                    } else {
+                        // Unlock thread, the challenge wasn't found.
+                        latch.countDown()
+                    }
+                }
+            }
+
+            webview?.loadUrl(origRequestUrl, headers)
+        }
+
+        latch.awaitFor30Seconds()
+
+        executor.execute {
+            webview?.run {
+                stopLoading()
+                destroy()
+            }
+        }
+
+        // Throw exception if we failed to bypass Cloudflare
+        if (!cloudflareBypassed) throw EhException("Failed to bypass Cloudflare")
+    }
+}
+
+private val ERROR_CODES = intArrayOf(403, 503)
+private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
+private const val COOKIE_NAME = "cf_clearance"

app/src/main/java/eu/kanade/tachiyomi/network/interceptor/WebViewInterceptor.kt

@@ -0,0 +1,86 @@
+package eu.kanade.tachiyomi.network.interceptor
+
+import android.content.Context
+import android.webkit.WebSettings
+import android.webkit.WebView
+import com.hippo.ehviewer.util.setDefaultSettings
+import okhttp3.Headers
+import okhttp3.Interceptor
+import okhttp3.Request
+import okhttp3.Response
+import java.util.Locale
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+
+abstract class WebViewInterceptor(private val context: Context) : Interceptor {
+    /**
+     * When this is called, it initializes the WebView if it wasn't already. We use this to avoid
+     * blocking the main thread too much. If used too often we could consider moving it to the
+     * Application class.
+     */
+    private val initWebView by lazy {
+        try {
+            WebSettings.getDefaultUserAgent(context)
+        } catch (_: Exception) {
+            // Avoid some crashes like when Chrome/WebView is being updated.
+        }
+    }
+
+    abstract fun shouldIntercept(response: Response): Boolean
+
+    abstract fun intercept(chain: Interceptor.Chain, request: Request, response: Response): Response
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val request = chain.request()
+        val response = chain.proceed(request)
+        if (!shouldIntercept(response)) {
+            return response
+        }
+
+        initWebView
+
+        return intercept(chain, request, response)
+    }
+
+    fun parseHeaders(headers: Headers): Map<String, String> {
+        return headers
+            // Keeping unsafe header makes webview throw [net::ERR_INVALID_ARGUMENT]
+            .filter { (name, value) ->
+                isRequestHeaderSafe(name, value)
+            }
+            .groupBy(keySelector = { (name, _) -> name }) { (_, value) -> value }
+            .mapValues { it.value.getOrNull(0).orEmpty() }
+    }
+
+    fun CountDownLatch.awaitFor30Seconds() {
+        await(30, TimeUnit.SECONDS)
+    }
+
+    fun createWebView(request: Request): WebView {
+        return WebView(context).apply {
+            setDefaultSettings()
+            // Avoid sending empty User-Agent, Chromium WebView will reset to default if empty
+            settings.userAgentString = request.header("User-Agent")
+        }
+    }
+}
+
+// Based on [IsRequestHeaderSafe] in https://source.chromium.org/chromium/chromium/src/+/main:services/network/public/cpp/header_util.cc
+private fun isRequestHeaderSafe(_name: String, _value: String): Boolean {
+    val name = _name.lowercase(Locale.ENGLISH)
+    val value = _value.lowercase(Locale.ENGLISH)
+    if (name in unsafeHeaderNames || name.startsWith("proxy-")) return false
+    return !(name == "connection" && value == "upgrade")
+}
+
+private val unsafeHeaderNames = arrayOf(
+    "content-length",
+    "host",
+    "trailer",
+    "te",
+    "upgrade",
+    "cookie2",
+    "keep-alive",
+    "transfer-encoding",
+    "set-cookie",
+)