H@H blacklist (#2058)

* Move SpeedTracker to proper place

* SpeedTracker use Instant to avoid value class in generic

* Initial HAtHBlacklist

* H@H blacklist switch

* Allow to trim H@H blacklist

* Don't blacklist if that H@H returned twice

---------

Co-authored-by: Dude so hot <djohn@fbi.gov>

app/src/main/kotlin/com/hippo/ehviewer/EhApplication.kt

@@ -56,6 +56,7 @@ import com.hippo.ehviewer.ktbuilder.imageLoader
 import com.hippo.ehviewer.ktor.Cronet
 import com.hippo.ehviewer.ktor.configureClient
 import com.hippo.ehviewer.ktor.configureCommon
+import com.hippo.ehviewer.spider.trimHAtHBlacklist
 import com.hippo.ehviewer.ui.keepNoMediaFileStatus
 import com.hippo.ehviewer.ui.lockObserver
 import com.hippo.ehviewer.ui.screen.detailCache
@@ -77,8 +78,10 @@ import eu.kanade.tachiyomi.util.lang.withUIContext
 import eu.kanade.tachiyomi.util.system.logcat
 import io.ktor.client.HttpClient
 import io.ktor.client.engine.okhttp.OkHttp
+import kotlin.time.Duration.Companion.days
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
+import kotlinx.datetime.Clock
 import logcat.AndroidLogcatLogger
 import logcat.LogPriority
 import logcat.LogcatLogger
@@ -112,6 +115,7 @@ class EhApplication :
         super.onCreate()
         System.loadLibrary("ehviewer")
         lifecycleScope.launchIO {
+            launch { trimHAtHBlacklist(Clock.System.now() - 7.days) }
             launchUI { FavouriteStatusRouter.collect { (gid, slot) -> detailCache[gid]?.favoriteSlot = slot } }
             EhTagDatabase.launchUpdate()
             launch { EhDB }

app/src/main/kotlin/com/hippo/ehviewer/Settings.kt

@@ -148,6 +148,7 @@ object Settings : DataStorePreferences(null) {
     var launchPage by intPref("launch_page_2", 0)
     var commentThreshold by intPref("comment_threshold", -101)
     var hardwareBitmapThreshold by intPref("hardware_bitmap_threshold", 16384)
+    var hAtHBlacklist by boolPref("H@H_blacklist", false).observed { updateWhenHAtHBlacklistChanges() }
     var showComments by boolPref("show_gallery_comments", true)
     var requestNews by boolPref("request_news", false).observed { updateWhenRequestNewsChanges() }
     var hideHvEvents by boolPref("hide_hv_events", false)

app/src/main/kotlin/com/hippo/ehviewer/SettingsCollector.kt

@@ -5,6 +5,7 @@ import androidx.appcompat.app.AppCompatDelegate
 import com.hippo.ehviewer.client.EhEngine
 import com.hippo.ehviewer.client.EhTagDatabase
 import com.hippo.ehviewer.dailycheck.updateDailyCheckWork
+import com.hippo.ehviewer.spider.trimHAtHBlacklist
 import com.hippo.ehviewer.ui.keepNoMediaFileStatus
 import com.hippo.ehviewer.util.isAtLeastS
 import eu.kanade.tachiyomi.util.lang.launchIO
@@ -67,3 +68,9 @@ fun updateWhenGallerySiteChanges() {
 fun updateWhenTagTranslationChanges() {
     EhTagDatabase.launchUpdate()
 }
+
+fun updateWhenHAtHBlacklistChanges() {
+    collectScope.launchIO {
+        trimHAtHBlacklist()
+    }
+}

app/src/main/kotlin/com/hippo/ehviewer/download/DownloadManager.kt

@@ -33,6 +33,7 @@ import com.hippo.ehviewer.dao.DownloadArtist
 import com.hippo.ehviewer.dao.DownloadInfo
 import com.hippo.ehviewer.dao.DownloadLabel
 import com.hippo.ehviewer.spider.COMIC_INFO_FILE
+import com.hippo.ehviewer.spider.SpeedTracker
 import com.hippo.ehviewer.spider.SpiderQueen
 import com.hippo.ehviewer.spider.SpiderQueen.Companion.SPIDER_INFO_FILENAME
 import com.hippo.ehviewer.spider.SpiderQueen.OnSpiderListener
@@ -40,7 +41,6 @@ import com.hippo.ehviewer.spider.downloadDirname
 import com.hippo.ehviewer.spider.readComicInfo
 import com.hippo.ehviewer.spider.readCompatFromPath
 import com.hippo.ehviewer.spider.toSimpleTags
-import com.hippo.ehviewer.ui.tools.SpeedTracker
 import com.hippo.ehviewer.util.AppConfig
 import com.hippo.ehviewer.util.insertWith
 import com.hippo.ehviewer.util.mapNotNull

app/src/main/kotlin/com/hippo/ehviewer/spider/HAtHBlacklist.kt

@@ -0,0 +1,37 @@
+package com.hippo.ehviewer.spider
+
+import com.hippo.files.read
+import com.hippo.files.write
+import io.ktor.utils.io.core.writeFully
+import kotlinx.datetime.Clock
+import kotlinx.datetime.Instant
+import kotlinx.io.readByteArray
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.cbor.Cbor
+import kotlinx.serialization.decodeFromByteArray
+import kotlinx.serialization.encodeToByteArray
+import okio.Path.Companion.toOkioPath
+import splitties.init.appCtx
+
+@Serializable
+class HAtHBlacklist(val blacklist: MutableMap<String, Instant> = mutableMapOf())
+
+private val cache = appCtx.cacheDir.toOkioPath() / "H@H_blacklist"
+
+val blacklist by lazy {
+    runCatching {
+        cache.read { Cbor.decodeFromByteArray<HAtHBlacklist>(readByteArray()) }
+    }.getOrElse {
+        cache.write { HAtHBlacklist().also { writeFully(Cbor.encodeToByteArray(it)) } }
+    }
+}
+
+fun appendHAtHBlacklist(host: String) = synchronized(blacklist) {
+    blacklist.blacklist[host] = Clock.System.now()
+    cache.write { writeFully(Cbor.encodeToByteArray(blacklist)) }
+}
+
+fun trimHAtHBlacklist(time: Instant = Clock.System.now()) = synchronized(blacklist) {
+    blacklist.blacklist.values.removeIf { i -> i < time }
+    cache.write { writeFully(Cbor.encodeToByteArray(blacklist)) }
+}

app/src/main/kotlin/com/hippo/ehviewer/spider/SpeedTracker.kt

@@ -1,4 +1,4 @@
-package com.hippo.ehviewer.ui.tools
+package com.hippo.ehviewer.spider
 
 import androidx.collection.MutableObjectIntMap
 import androidx.collection.mutableObjectIntMapOf
@@ -17,25 +17,26 @@ import io.ktor.client.statement.HttpStatement
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.milliseconds
 import kotlin.time.Duration.Companion.seconds
-import kotlin.time.TimeSource
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
+import kotlinx.datetime.Clock
+import kotlinx.datetime.Instant
 
 class SpeedTracker(val window: Duration = 1.seconds) {
     private val mutex = Mutex()
-    private val received = mutableObjectIntMapOf<TimeSource.Monotonic.ValueTimeMark>()
-    private var start: TimeSource.Monotonic.ValueTimeMark? = null
+    private val received = mutableObjectIntMapOf<Instant>()
+    private var start: Instant? = null
 
     suspend fun reset() = mutex.withLock {
         received.clear()
         start = null
     }
 
-    suspend fun track(bytes: Int) = TimeSource.Monotonic.markNow().let { now ->
+    suspend fun track(bytes: Int) = Clock.System.now().let { now ->
         mutex.withLock {
             start = start ?: now
             received[now] = bytes
@@ -45,7 +46,7 @@ class SpeedTracker(val window: Duration = 1.seconds) {
     fun speedFlow(sample: Duration = 1.seconds) = fixedRate(sample).map {
         mutex.withLock {
             start?.let { start ->
-                val now = TimeSource.Monotonic.markNow()
+                val now = Clock.System.now()
                 val passed = now - start
                 val window = passed.coerceIn(10.milliseconds, window)
                 val cutoff = now - window

app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt

@@ -37,7 +37,6 @@ import com.hippo.ehviewer.download.downloadLocation
 import com.hippo.ehviewer.download.tempDownloadDir
 import com.hippo.ehviewer.image.PathSource
 import com.hippo.ehviewer.jni.archiveFdBatch
-import com.hippo.ehviewer.ui.tools.timeoutBySpeed
 import com.hippo.ehviewer.util.FileUtils
 import com.hippo.ehviewer.util.copyTo
 import com.hippo.ehviewer.util.sendTo

app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt

@@ -38,8 +38,10 @@ import com.hippo.ehviewer.client.parser.GalleryDetailParser.parsePreviewPages
 import com.hippo.ehviewer.client.parser.GalleryMultiPageViewerPTokenParser
 import com.hippo.ehviewer.client.parser.GalleryPageUrlParser
 import com.hippo.ehviewer.util.displayString
+import com.hippo.ehviewer.util.isHAtHTimeoutException
 import com.hippo.files.find
 import eu.kanade.tachiyomi.util.system.logcat
+import io.ktor.http.Url
 import java.util.concurrent.atomic.AtomicInteger
 import kotlin.time.Duration.Companion.milliseconds
 import kotlin.time.TimeSource
@@ -576,6 +578,7 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS
             var forceHtml = false
             val original = Settings.downloadOriginImage || orgImg
             runSuspendCatching {
+                var lasthost: String? = null
                 repeat(3) { retries ->
                     var imageUrl: String? = null
                     var localShowKey: String?
@@ -635,6 +638,20 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS
                     }
                     checkNotNull(targetImageUrl)
 
+                    val host = Url(targetImageUrl).host
+                    synchronized(blacklist) {
+                        if (lasthost == host) {
+                            blacklist.blacklist.remove(host)
+                            logcat(WORKER_DEBUG_TAG) { "Drop $host from H@H blacklist as its resource is unique for now!" }
+                        }
+
+                        if (Settings.hAtHBlacklist && blacklist.blacklist.containsKey(host)) {
+                            logcat(WORKER_DEBUG_TAG) { "$host hits H@H blacklist!" }
+                            lasthost = host
+                            return@repeat
+                        }
+                    }
+
                     runCatching {
                         logcat(WORKER_DEBUG_TAG) { "Start download image $index" }
                         spiderDen.makeHttpCallAndSaveImage(
@@ -648,7 +665,10 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS
                         return
                     }.onFailure {
                         spiderDen.removeIntermediateFiles(index)
-                        logcat(WORKER_DEBUG_TAG) { "Download image $index attempt" }
+                        logcat(WORKER_DEBUG_TAG) { "Download image $index failed" }
+                        if (Settings.hAtHBlacklist && it.isHAtHTimeoutException()) {
+                            appendHAtHBlacklist(host)
+                        }
                         when (it) {
                             is CancellationException, is FileNotFoundException -> throw it
                         }

app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/EnStrings.kt

@@ -310,6 +310,7 @@ object EnStrings : Strings {
     override val settingsDownloadDownloadDelaySummary = { a: Int -> "Delay $a ms per download" }
     override val settingsDownloadConnectionTimeout = "Connection timeout (in seconds)"
     override val settingsDownloadTimeoutSpeed = "Minimum response speed (in KB/s)"
+    override val settingsDownloadHAtHBlacklist = "Enable H@H blacklist"
     override val settingsDownloadPreloadImage = "Preload image"
     override val settingsDownloadPreloadImageSummary = { a: Int -> "Preload next $a image" }
     override val settingsDownloadDownloadOriginImage = "Download original image"

app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/Strings.kt

@@ -315,6 +315,7 @@ interface Strings {
     val settingsDownloadDownloadDelaySummary: (Int) -> String
     val settingsDownloadConnectionTimeout: String
     val settingsDownloadTimeoutSpeed: String
+    val settingsDownloadHAtHBlacklist: String
     val settingsDownloadPreloadImage: String
     val settingsDownloadPreloadImageSummary: (Int) -> String
     val settingsDownloadDownloadOriginImage: String

app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt

@@ -408,6 +408,7 @@ object ZhCnStrings : Strings by EnStrings {
     override val settingsDownloadDownloadDelaySummary = { a: Int -> "每次下载延时 $a 毫秒" }
     override val settingsDownloadConnectionTimeout = "连接超时（秒）"
     override val settingsDownloadTimeoutSpeed = "最低响应速度（KB/s）"
+    override val settingsDownloadHAtHBlacklist = "启用H@H黑名单"
     override val noBrowserInstalled = "请安装一个浏览器。"
     override val toplistAlltime = "从始至终"
     override val toplistPastyear = "过去一年"

app/src/main/kotlin/com/hippo/ehviewer/ui/settings/DownloadScreen.kt

@@ -193,6 +193,10 @@ fun AnimatedVisibilityScope.DownloadScreen(navigator: DestinationsNavigator) = c
                 title = settingsDownloadTimeoutSpeed,
                 value = Settings::timeoutSpeed,
             )
+            SwitchPreference(
+                title = settingsDownloadHAtHBlacklist,
+                value = Settings::hAtHBlacklist,
+            )
             val preloadImage = Settings::preloadImage.observed
             SimpleMenuPreferenceInt(
                 title = settingsDownloadPreloadImage,

app/src/main/kotlin/com/hippo/ehviewer/updater/AppUpdater.kt

@@ -4,7 +4,7 @@ import com.hippo.ehviewer.BuildConfig
 import com.hippo.ehviewer.EhApplication.Companion.ktorClient
 import com.hippo.ehviewer.Settings
 import com.hippo.ehviewer.client.executeAndParseAs
-import com.hippo.ehviewer.ui.tools.timeoutBySpeed
+import com.hippo.ehviewer.spider.timeoutBySpeed
 import com.hippo.ehviewer.util.copyTo
 import com.hippo.files.write
 import io.ktor.client.request.HttpRequestBuilder

app/src/main/kotlin/com/hippo/ehviewer/util/ExceptionUtils.kt

@@ -20,9 +20,15 @@ import eu.kanade.tachiyomi.util.system.logcat
 import io.ktor.client.network.sockets.ConnectTimeoutException
 import io.ktor.client.network.sockets.SocketTimeoutException
 import io.ktor.client.plugins.HttpRequestTimeoutException
+import javax.net.ssl.SSLHandshakeException
 import kotlinx.io.IOException
 import splitties.init.appCtx
 
+fun Throwable.isHAtHTimeoutException() = when (this) {
+    is HttpRequestTimeoutException, is ConnectTimeoutException, is SocketTimeoutException, is LowSpeedException, is SSLHandshakeException -> true
+    else -> false
+}
+
 class LowSpeedException(
     url: String,
     speed: Long,