From 0561a9bcf66ce39d060cbf73a37781085e495861 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <99822064+revonateB0T@users.noreply.github.com> Date: Tue, 31 Dec 2024 19:46:28 +0800 Subject: [PATCH] H@H blacklist (#2058) * Move SpeedTracker to proper place * SpeedTracker use Instant to avoid value class in generic * Initial HAtHBlacklist * H@H blacklist switch * Allow to trim H@H blacklist * Don't blacklist if that H@H returned twice --------- Co-authored-by: Dude so hot --- .../com/hippo/ehviewer/EhApplication.kt | 4 ++ .../kotlin/com/hippo/ehviewer/Settings.kt | 1 + .../com/hippo/ehviewer/SettingsCollector.kt | 7 ++++ .../ehviewer/download/DownloadManager.kt | 2 +- .../hippo/ehviewer/spider/HAtHBlacklist.kt | 37 +++++++++++++++++++ .../{ui/tools => spider}/SpeedTracker.kt | 13 ++++--- .../com/hippo/ehviewer/spider/SpiderDen.kt | 1 - .../com/hippo/ehviewer/spider/SpiderQueen.kt | 22 ++++++++++- .../com/hippo/ehviewer/ui/i18n/EnStrings.kt | 1 + .../com/hippo/ehviewer/ui/i18n/Strings.kt | 1 + .../com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt | 1 + .../ehviewer/ui/settings/DownloadScreen.kt | 4 ++ .../com/hippo/ehviewer/updater/AppUpdater.kt | 2 +- .../com/hippo/ehviewer/util/ExceptionUtils.kt | 6 +++ 14 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 app/src/main/kotlin/com/hippo/ehviewer/spider/HAtHBlacklist.kt rename app/src/main/kotlin/com/hippo/ehviewer/{ui/tools => spider}/SpeedTracker.kt (90%) diff --git a/app/src/main/kotlin/com/hippo/ehviewer/EhApplication.kt b/app/src/main/kotlin/com/hippo/ehviewer/EhApplication.kt index 01dc40b93..2daea7060 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/EhApplication.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/EhApplication.kt @@ -56,6 +56,7 @@ import com.hippo.ehviewer.ktbuilder.imageLoader import com.hippo.ehviewer.ktor.Cronet import com.hippo.ehviewer.ktor.configureClient import com.hippo.ehviewer.ktor.configureCommon +import com.hippo.ehviewer.spider.trimHAtHBlacklist import com.hippo.ehviewer.ui.keepNoMediaFileStatus import com.hippo.ehviewer.ui.lockObserver import com.hippo.ehviewer.ui.screen.detailCache @@ -77,8 +78,10 @@ import eu.kanade.tachiyomi.util.lang.withUIContext import eu.kanade.tachiyomi.util.system.logcat import io.ktor.client.HttpClient import io.ktor.client.engine.okhttp.OkHttp +import kotlin.time.Duration.Companion.days import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch +import kotlinx.datetime.Clock import logcat.AndroidLogcatLogger import logcat.LogPriority import logcat.LogcatLogger @@ -112,6 +115,7 @@ class EhApplication : super.onCreate() System.loadLibrary("ehviewer") lifecycleScope.launchIO { + launch { trimHAtHBlacklist(Clock.System.now() - 7.days) } launchUI { FavouriteStatusRouter.collect { (gid, slot) -> detailCache[gid]?.favoriteSlot = slot } } EhTagDatabase.launchUpdate() launch { EhDB } diff --git a/app/src/main/kotlin/com/hippo/ehviewer/Settings.kt b/app/src/main/kotlin/com/hippo/ehviewer/Settings.kt index 3b5ed071d..ab8348e0c 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/Settings.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/Settings.kt @@ -148,6 +148,7 @@ object Settings : DataStorePreferences(null) { var launchPage by intPref("launch_page_2", 0) var commentThreshold by intPref("comment_threshold", -101) var hardwareBitmapThreshold by intPref("hardware_bitmap_threshold", 16384) + var hAtHBlacklist by boolPref("H@H_blacklist", false).observed { updateWhenHAtHBlacklistChanges() } var showComments by boolPref("show_gallery_comments", true) var requestNews by boolPref("request_news", false).observed { updateWhenRequestNewsChanges() } var hideHvEvents by boolPref("hide_hv_events", false) diff --git a/app/src/main/kotlin/com/hippo/ehviewer/SettingsCollector.kt b/app/src/main/kotlin/com/hippo/ehviewer/SettingsCollector.kt index 9e77a92ff..6543a2f4c 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/SettingsCollector.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/SettingsCollector.kt @@ -5,6 +5,7 @@ import androidx.appcompat.app.AppCompatDelegate import com.hippo.ehviewer.client.EhEngine import com.hippo.ehviewer.client.EhTagDatabase import com.hippo.ehviewer.dailycheck.updateDailyCheckWork +import com.hippo.ehviewer.spider.trimHAtHBlacklist import com.hippo.ehviewer.ui.keepNoMediaFileStatus import com.hippo.ehviewer.util.isAtLeastS import eu.kanade.tachiyomi.util.lang.launchIO @@ -67,3 +68,9 @@ fun updateWhenGallerySiteChanges() { fun updateWhenTagTranslationChanges() { EhTagDatabase.launchUpdate() } + +fun updateWhenHAtHBlacklistChanges() { + collectScope.launchIO { + trimHAtHBlacklist() + } +} diff --git a/app/src/main/kotlin/com/hippo/ehviewer/download/DownloadManager.kt b/app/src/main/kotlin/com/hippo/ehviewer/download/DownloadManager.kt index cdc96b581..8f8a3bfa6 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/download/DownloadManager.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/download/DownloadManager.kt @@ -33,6 +33,7 @@ import com.hippo.ehviewer.dao.DownloadArtist import com.hippo.ehviewer.dao.DownloadInfo import com.hippo.ehviewer.dao.DownloadLabel import com.hippo.ehviewer.spider.COMIC_INFO_FILE +import com.hippo.ehviewer.spider.SpeedTracker import com.hippo.ehviewer.spider.SpiderQueen import com.hippo.ehviewer.spider.SpiderQueen.Companion.SPIDER_INFO_FILENAME import com.hippo.ehviewer.spider.SpiderQueen.OnSpiderListener @@ -40,7 +41,6 @@ import com.hippo.ehviewer.spider.downloadDirname import com.hippo.ehviewer.spider.readComicInfo import com.hippo.ehviewer.spider.readCompatFromPath import com.hippo.ehviewer.spider.toSimpleTags -import com.hippo.ehviewer.ui.tools.SpeedTracker import com.hippo.ehviewer.util.AppConfig import com.hippo.ehviewer.util.insertWith import com.hippo.ehviewer.util.mapNotNull diff --git a/app/src/main/kotlin/com/hippo/ehviewer/spider/HAtHBlacklist.kt b/app/src/main/kotlin/com/hippo/ehviewer/spider/HAtHBlacklist.kt new file mode 100644 index 000000000..d9d1c389e --- /dev/null +++ b/app/src/main/kotlin/com/hippo/ehviewer/spider/HAtHBlacklist.kt @@ -0,0 +1,37 @@ +package com.hippo.ehviewer.spider + +import com.hippo.files.read +import com.hippo.files.write +import io.ktor.utils.io.core.writeFully +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant +import kotlinx.io.readByteArray +import kotlinx.serialization.Serializable +import kotlinx.serialization.cbor.Cbor +import kotlinx.serialization.decodeFromByteArray +import kotlinx.serialization.encodeToByteArray +import okio.Path.Companion.toOkioPath +import splitties.init.appCtx + +@Serializable +class HAtHBlacklist(val blacklist: MutableMap = mutableMapOf()) + +private val cache = appCtx.cacheDir.toOkioPath() / "H@H_blacklist" + +val blacklist by lazy { + runCatching { + cache.read { Cbor.decodeFromByteArray(readByteArray()) } + }.getOrElse { + cache.write { HAtHBlacklist().also { writeFully(Cbor.encodeToByteArray(it)) } } + } +} + +fun appendHAtHBlacklist(host: String) = synchronized(blacklist) { + blacklist.blacklist[host] = Clock.System.now() + cache.write { writeFully(Cbor.encodeToByteArray(blacklist)) } +} + +fun trimHAtHBlacklist(time: Instant = Clock.System.now()) = synchronized(blacklist) { + blacklist.blacklist.values.removeIf { i -> i < time } + cache.write { writeFully(Cbor.encodeToByteArray(blacklist)) } +} diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/tools/SpeedTracker.kt b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpeedTracker.kt similarity index 90% rename from app/src/main/kotlin/com/hippo/ehviewer/ui/tools/SpeedTracker.kt rename to app/src/main/kotlin/com/hippo/ehviewer/spider/SpeedTracker.kt index 792edb0f1..23b2cd156 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/tools/SpeedTracker.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpeedTracker.kt @@ -1,4 +1,4 @@ -package com.hippo.ehviewer.ui.tools +package com.hippo.ehviewer.spider import androidx.collection.MutableObjectIntMap import androidx.collection.mutableObjectIntMapOf @@ -17,25 +17,26 @@ import io.ktor.client.statement.HttpStatement import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds -import kotlin.time.TimeSource import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant class SpeedTracker(val window: Duration = 1.seconds) { private val mutex = Mutex() - private val received = mutableObjectIntMapOf() - private var start: TimeSource.Monotonic.ValueTimeMark? = null + private val received = mutableObjectIntMapOf() + private var start: Instant? = null suspend fun reset() = mutex.withLock { received.clear() start = null } - suspend fun track(bytes: Int) = TimeSource.Monotonic.markNow().let { now -> + suspend fun track(bytes: Int) = Clock.System.now().let { now -> mutex.withLock { start = start ?: now received[now] = bytes @@ -45,7 +46,7 @@ class SpeedTracker(val window: Duration = 1.seconds) { fun speedFlow(sample: Duration = 1.seconds) = fixedRate(sample).map { mutex.withLock { start?.let { start -> - val now = TimeSource.Monotonic.markNow() + val now = Clock.System.now() val passed = now - start val window = passed.coerceIn(10.milliseconds, window) val cutoff = now - window diff --git a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt index e0d26e981..66da7fbea 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt @@ -37,7 +37,6 @@ import com.hippo.ehviewer.download.downloadLocation import com.hippo.ehviewer.download.tempDownloadDir import com.hippo.ehviewer.image.PathSource import com.hippo.ehviewer.jni.archiveFdBatch -import com.hippo.ehviewer.ui.tools.timeoutBySpeed import com.hippo.ehviewer.util.FileUtils import com.hippo.ehviewer.util.copyTo import com.hippo.ehviewer.util.sendTo diff --git a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt index 737404da4..f52f69c65 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt @@ -38,8 +38,10 @@ import com.hippo.ehviewer.client.parser.GalleryDetailParser.parsePreviewPages import com.hippo.ehviewer.client.parser.GalleryMultiPageViewerPTokenParser import com.hippo.ehviewer.client.parser.GalleryPageUrlParser import com.hippo.ehviewer.util.displayString +import com.hippo.ehviewer.util.isHAtHTimeoutException import com.hippo.files.find import eu.kanade.tachiyomi.util.system.logcat +import io.ktor.http.Url import java.util.concurrent.atomic.AtomicInteger import kotlin.time.Duration.Companion.milliseconds import kotlin.time.TimeSource @@ -576,6 +578,7 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS var forceHtml = false val original = Settings.downloadOriginImage || orgImg runSuspendCatching { + var lasthost: String? = null repeat(3) { retries -> var imageUrl: String? = null var localShowKey: String? @@ -635,6 +638,20 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS } checkNotNull(targetImageUrl) + val host = Url(targetImageUrl).host + synchronized(blacklist) { + if (lasthost == host) { + blacklist.blacklist.remove(host) + logcat(WORKER_DEBUG_TAG) { "Drop $host from H@H blacklist as its resource is unique for now!" } + } + + if (Settings.hAtHBlacklist && blacklist.blacklist.containsKey(host)) { + logcat(WORKER_DEBUG_TAG) { "$host hits H@H blacklist!" } + lasthost = host + return@repeat + } + } + runCatching { logcat(WORKER_DEBUG_TAG) { "Start download image $index" } spiderDen.makeHttpCallAndSaveImage( @@ -648,7 +665,10 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS return }.onFailure { spiderDen.removeIntermediateFiles(index) - logcat(WORKER_DEBUG_TAG) { "Download image $index attempt" } + logcat(WORKER_DEBUG_TAG) { "Download image $index failed" } + if (Settings.hAtHBlacklist && it.isHAtHTimeoutException()) { + appendHAtHBlacklist(host) + } when (it) { is CancellationException, is FileNotFoundException -> throw it } diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/EnStrings.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/EnStrings.kt index 4baf8c0e9..8619543b2 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/EnStrings.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/EnStrings.kt @@ -310,6 +310,7 @@ object EnStrings : Strings { override val settingsDownloadDownloadDelaySummary = { a: Int -> "Delay $a ms per download" } override val settingsDownloadConnectionTimeout = "Connection timeout (in seconds)" override val settingsDownloadTimeoutSpeed = "Minimum response speed (in KB/s)" + override val settingsDownloadHAtHBlacklist = "Enable H@H blacklist" override val settingsDownloadPreloadImage = "Preload image" override val settingsDownloadPreloadImageSummary = { a: Int -> "Preload next $a image" } override val settingsDownloadDownloadOriginImage = "Download original image" diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/Strings.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/Strings.kt index a8da47f51..1bf83fe56 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/Strings.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/Strings.kt @@ -315,6 +315,7 @@ interface Strings { val settingsDownloadDownloadDelaySummary: (Int) -> String val settingsDownloadConnectionTimeout: String val settingsDownloadTimeoutSpeed: String + val settingsDownloadHAtHBlacklist: String val settingsDownloadPreloadImage: String val settingsDownloadPreloadImageSummary: (Int) -> String val settingsDownloadDownloadOriginImage: String diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt index 60e54a532..ad22e000d 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt @@ -408,6 +408,7 @@ object ZhCnStrings : Strings by EnStrings { override val settingsDownloadDownloadDelaySummary = { a: Int -> "每次下载延时 $a 毫秒" } override val settingsDownloadConnectionTimeout = "连接超时（秒）" override val settingsDownloadTimeoutSpeed = "最低响应速度（KB/s）" + override val settingsDownloadHAtHBlacklist = "启用H@H黑名单" override val noBrowserInstalled = "请安装一个浏览器。" override val toplistAlltime = "从始至终" override val toplistPastyear = "过去一年" diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/settings/DownloadScreen.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/settings/DownloadScreen.kt index a9170145d..4ed3682ac 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/settings/DownloadScreen.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/settings/DownloadScreen.kt @@ -193,6 +193,10 @@ fun AnimatedVisibilityScope.DownloadScreen(navigator: DestinationsNavigator) = c title = settingsDownloadTimeoutSpeed, value = Settings::timeoutSpeed, ) + SwitchPreference( + title = settingsDownloadHAtHBlacklist, + value = Settings::hAtHBlacklist, + ) val preloadImage = Settings::preloadImage.observed SimpleMenuPreferenceInt( title = settingsDownloadPreloadImage, diff --git a/app/src/main/kotlin/com/hippo/ehviewer/updater/AppUpdater.kt b/app/src/main/kotlin/com/hippo/ehviewer/updater/AppUpdater.kt index ce663e59c..8c11803d8 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/updater/AppUpdater.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/updater/AppUpdater.kt @@ -4,7 +4,7 @@ import com.hippo.ehviewer.BuildConfig import com.hippo.ehviewer.EhApplication.Companion.ktorClient import com.hippo.ehviewer.Settings import com.hippo.ehviewer.client.executeAndParseAs -import com.hippo.ehviewer.ui.tools.timeoutBySpeed +import com.hippo.ehviewer.spider.timeoutBySpeed import com.hippo.ehviewer.util.copyTo import com.hippo.files.write import io.ktor.client.request.HttpRequestBuilder diff --git a/app/src/main/kotlin/com/hippo/ehviewer/util/ExceptionUtils.kt b/app/src/main/kotlin/com/hippo/ehviewer/util/ExceptionUtils.kt index d98130a50..615a39292 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/util/ExceptionUtils.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/util/ExceptionUtils.kt @@ -20,9 +20,15 @@ import eu.kanade.tachiyomi.util.system.logcat import io.ktor.client.network.sockets.ConnectTimeoutException import io.ktor.client.network.sockets.SocketTimeoutException import io.ktor.client.plugins.HttpRequestTimeoutException +import javax.net.ssl.SSLHandshakeException import kotlinx.io.IOException import splitties.init.appCtx +fun Throwable.isHAtHTimeoutException() = when (this) { + is HttpRequestTimeoutException, is ConnectTimeoutException, is SocketTimeoutException, is LowSpeedException, is SSLHandshakeException -> true + else -> false +} + class LowSpeedException( url: String, speed: Long,