From b7948f79652161a2b15eaffd94bbf05cb52cc596 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <99822064+revonateB0T@users.noreply.github.com> Date: Tue, 31 Dec 2024 15:25:00 +0800 Subject: [PATCH] Introduce speed based timeout strategy (#2055) * CronetEngine throw connTimeout properly * Apply LowSpeedDetector * Add switch * Relax * Refine * Extract common part * Apply to AppUpdater * Fix * Adjust * Log conn timeout as well --------- Co-authored-by: Dude so hot --- .../kotlin/com/hippo/ehviewer/Settings.kt | 2 +- .../com/hippo/ehviewer/ktor/CronetEngine.kt | 7 +-- .../com/hippo/ehviewer/spider/SpiderDen.kt | 29 +++------- .../com/hippo/ehviewer/spider/SpiderQueen.kt | 4 +- .../com/hippo/ehviewer/ui/i18n/EnStrings.kt | 2 +- .../com/hippo/ehviewer/ui/i18n/JaStrings.kt | 1 - .../com/hippo/ehviewer/ui/i18n/Strings.kt | 2 +- .../com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt | 2 +- .../com/hippo/ehviewer/ui/i18n/ZhTwStrings.kt | 1 - .../ehviewer/ui/settings/DownloadScreen.kt | 14 ++--- .../hippo/ehviewer/ui/tools/SpeedTracker.kt | 53 +++++++++++++++++++ .../com/hippo/ehviewer/updater/AppUpdater.kt | 34 ++++++------ .../com/hippo/ehviewer/util/ExceptionUtils.kt | 16 +++--- 13 files changed, 99 insertions(+), 68 deletions(-) diff --git a/app/src/main/kotlin/com/hippo/ehviewer/Settings.kt b/app/src/main/kotlin/com/hippo/ehviewer/Settings.kt index 112bcf51c..3b5ed071d 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/Settings.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/Settings.kt @@ -141,8 +141,8 @@ object Settings : DataStorePreferences(null) { var downloadDelay by intPref("download_delay_3", 1000) var multiThreadDownload by intPref("download_thread_2", 3) var preloadImage by intPref("preload_image_2", 5) - var downloadTimeout by intPref("download_timeout", 60) var connTimeout by intPref("conn_timeout", 10) + var timeoutSpeed by intPref("timeout_speed", 128) var theme by intPref("theme_2", -1).observed { updateWhenThemeChanges() } var readCacheSize by intPref("read_cache_size_2", 640) var launchPage by intPref("launch_page_2", 0) diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ktor/CronetEngine.kt b/app/src/main/kotlin/com/hippo/ehviewer/ktor/CronetEngine.kt index 5cb11e349..be094e6cc 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ktor/CronetEngine.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ktor/CronetEngine.kt @@ -14,6 +14,7 @@ import com.hippo.ehviewer.Settings import com.hippo.ehviewer.util.restartApplication import io.ktor.client.engine.HttpClientEngineBase import io.ktor.client.engine.callContext +import io.ktor.client.plugins.ConnectTimeoutException import io.ktor.client.plugins.HttpTimeoutCapability import io.ktor.client.request.HttpRequestData import io.ktor.client.request.HttpResponseData @@ -41,7 +42,7 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.consumeEach import kotlinx.coroutines.job import kotlinx.coroutines.suspendCancellableCoroutine -import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.withTimeoutOrNull import splitties.preferences.edit class CronetEngine(override val config: CronetConfig) : HttpClientEngineBase("Cronet") { @@ -59,9 +60,9 @@ class CronetEngine(override val config: CronetConfig) : HttpClientEngineBase("Cr @InternalAPI override suspend fun execute(data: HttpRequestData) = data.getCapabilityOrNull(HttpTimeoutCapability)!!.let { cfg -> val connTimeout = cfg.connectTimeoutMillis!! - withTimeout(connTimeout.milliseconds) { + withTimeoutOrNull(connTimeout.milliseconds) { executeHttpRequest(callContext(), data) - } + } ?: throw ConnectTimeoutException(data) } private suspend fun executeHttpRequest( diff --git a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt index cea4dd2bd..7cc8cb0c2 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt @@ -28,7 +28,6 @@ import com.hippo.ehviewer.client.EhUtils.getSuitableTitle import com.hippo.ehviewer.client.data.GalleryDetail import com.hippo.ehviewer.client.data.GalleryInfo import com.hippo.ehviewer.client.ehRequest -import com.hippo.ehviewer.client.executeSafely import com.hippo.ehviewer.client.getImageKey import com.hippo.ehviewer.coil.read import com.hippo.ehviewer.coil.suspendEdit @@ -38,6 +37,7 @@ import com.hippo.ehviewer.download.downloadLocation import com.hippo.ehviewer.download.tempDownloadDir import com.hippo.ehviewer.image.PathSource import com.hippo.ehviewer.jni.archiveFdBatch +import com.hippo.ehviewer.ui.tools.timeoutBySpeed import com.hippo.ehviewer.util.FileUtils import com.hippo.ehviewer.util.copyTo import com.hippo.ehviewer.util.sendTo @@ -51,13 +51,9 @@ import com.hippo.files.mkdirs import com.hippo.files.moveTo import com.hippo.files.openFileDescriptor import eu.kanade.tachiyomi.util.system.logcat -import io.ktor.client.plugins.onDownload -import io.ktor.client.plugins.timeout import io.ktor.client.statement.HttpResponse import io.ktor.client.statement.bodyAsChannel import io.ktor.client.statement.request -import io.ktor.http.isSuccess -import io.ktor.utils.io.copyTo import java.util.concurrent.locks.ReentrantReadWriteLock import kotlin.concurrent.read import kotlin.concurrent.write @@ -72,7 +68,6 @@ class SpiderDen(val info: GalleryInfo) { private var tempDownloadDir: Path? = null private val saveAsCbz = Settings.saveAsCbz private val archiveName = "$gid.cbz" - private val downloadTimeout = Settings.downloadTimeout * 1000L private val lock = ReentrantReadWriteLock() @@ -170,23 +165,11 @@ class SpiderDen(val info: GalleryInfo) { url: String, referer: String?, notifyProgress: (Long, Long, Int) -> Unit, - ) = ehRequest(url, referer) { - var prev = 0L - onDownload { done, total -> - notifyProgress(total!!, done, (done - prev).toInt()) - prev = done - } - timeout { - connectTimeoutMillis = Settings.connTimeout * 1000L - requestTimeoutMillis = downloadTimeout - } - }.executeSafely { - if (it.status.isSuccess()) { - saveFromHttpResponse(index, it) - } else { - false - } - } + ) = timeoutBySpeed( + { ehRequest(url, referer, builder = it) }, + { a, b, c -> notifyProgress(a, b, c) }, + { resp -> check(saveFromHttpResponse(index, resp)) }, + ) private suspend inline fun saveResponseMeta( index: Int, diff --git a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt index 8458a748a..065476135 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt @@ -638,14 +638,12 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS repeat(2) { times -> runCatching { logcat(WORKER_DEBUG_TAG) { "Start download image $index attempt #$times" } - val success = spiderDen.makeHttpCallAndSaveImage( + spiderDen.makeHttpCallAndSaveImage( index, targetImageUrl, referer, this@SpiderQueen::notifyPageDownload.partially1(index), ) - - check(success) logcat(WORKER_DEBUG_TAG) { "Download image $index succeed" } updatePageState(index, STATE_FINISHED) return diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/EnStrings.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/EnStrings.kt index df507bffa..4baf8c0e9 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/EnStrings.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/EnStrings.kt @@ -308,8 +308,8 @@ object EnStrings : Strings { override val settingsDownloadConcurrencySummary = { a: Int -> "Up to $a images" } override val settingsDownloadDownloadDelay = "Download delay" override val settingsDownloadDownloadDelaySummary = { a: Int -> "Delay $a ms per download" } - override val settingsDownloadDownloadTimeout = "Download timeout (in seconds)" override val settingsDownloadConnectionTimeout = "Connection timeout (in seconds)" + override val settingsDownloadTimeoutSpeed = "Minimum response speed (in KB/s)" override val settingsDownloadPreloadImage = "Preload image" override val settingsDownloadPreloadImageSummary = { a: Int -> "Preload next $a image" } override val settingsDownloadDownloadOriginImage = "Download original image" diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/JaStrings.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/JaStrings.kt index a19c55907..9da628259 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/JaStrings.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/JaStrings.kt @@ -286,7 +286,6 @@ object JaStrings : Strings by EnStrings { override val settingsDownloadConcurrencySummary = { a: Int -> "同時に最大 $a 枚の画像をダウンロードします" } override val settingsDownloadDownloadDelay = "ダウンロードの遅延" override val settingsDownloadDownloadDelaySummary = { a: Int -> "ダウンロードで $a ミリ秒の遅延をさせます" } - override val settingsDownloadDownloadTimeout = "ダウンロードのタイムアウト (秒単位)" override val settingsDownloadPreloadImage = "画像をプリロード" override val settingsDownloadPreloadImageSummary = { a: Int -> "$a 枚の画像をプリロードします" } override val settingsDownloadDownloadOriginImage = "オリジナルの画像をダウンロードする" diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/Strings.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/Strings.kt index 1eacf41dd..a8da47f51 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/Strings.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/Strings.kt @@ -313,8 +313,8 @@ interface Strings { val settingsDownloadConcurrencySummary: (Int) -> String val settingsDownloadDownloadDelay: String val settingsDownloadDownloadDelaySummary: (Int) -> String - val settingsDownloadDownloadTimeout: String val settingsDownloadConnectionTimeout: String + val settingsDownloadTimeoutSpeed: String val settingsDownloadPreloadImage: String val settingsDownloadPreloadImageSummary: (Int) -> String val settingsDownloadDownloadOriginImage: String diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt index 4fe5cae7f..60e54a532 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhCnStrings.kt @@ -406,8 +406,8 @@ object ZhCnStrings : Strings by EnStrings { override val downloadStartAllReversed = "全部开始（倒序）" override val settingsDownloadDownloadDelay = "下载延时" override val settingsDownloadDownloadDelaySummary = { a: Int -> "每次下载延时 $a 毫秒" } - override val settingsDownloadDownloadTimeout = "下载超时（秒）" override val settingsDownloadConnectionTimeout = "连接超时（秒）" + override val settingsDownloadTimeoutSpeed = "最低响应速度（KB/s）" override val noBrowserInstalled = "请安装一个浏览器。" override val toplistAlltime = "从始至终" override val toplistPastyear = "过去一年" diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhTwStrings.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhTwStrings.kt index 7a864e84d..be6c313a4 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhTwStrings.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/i18n/ZhTwStrings.kt @@ -397,7 +397,6 @@ object ZhTwStrings : Strings by EnStrings { override val downloadStartAllReversed = "全部開始（倒序）" override val settingsDownloadDownloadDelay = "下載延時" override val settingsDownloadDownloadDelaySummary = { a: Int -> "每次下載延時 $a 毫秒" } - override val settingsDownloadDownloadTimeout = "下載超時（秒）" override val noBrowserInstalled = "請安裝一個瀏覽器。" override val toplistAlltime = "從始至終" override val toplistPastyear = "過去一年" diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/settings/DownloadScreen.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/settings/DownloadScreen.kt index 8ddb378da..a9170145d 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/settings/DownloadScreen.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/settings/DownloadScreen.kt @@ -180,18 +180,18 @@ fun AnimatedVisibilityScope.DownloadScreen(navigator: DestinationsNavigator) = c value = downloadDelay, ) IntSliderPreference( - maxValue = 20, + maxValue = 10, minValue = 2, - step = 17, + step = 7, title = settingsDownloadConnectionTimeout, value = Settings::connTimeout, ) IntSliderPreference( - maxValue = 120, - minValue = 10, - step = 10, - title = settingsDownloadDownloadTimeout, - value = Settings::downloadTimeout, + maxValue = 1024, + minValue = 0, + step = 7, + title = settingsDownloadTimeoutSpeed, + value = Settings::timeoutSpeed, ) val preloadImage = Settings::preloadImage.observed SimpleMenuPreferenceInt( diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ui/tools/SpeedTracker.kt b/app/src/main/kotlin/com/hippo/ehviewer/ui/tools/SpeedTracker.kt index 115a563ed..1dd05abcb 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ui/tools/SpeedTracker.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ui/tools/SpeedTracker.kt @@ -3,11 +3,25 @@ package com.hippo.ehviewer.ui.tools import androidx.collection.MutableObjectIntMap import androidx.collection.mutableObjectIntMapOf import arrow.fx.coroutines.fixedRate +import com.hippo.ehviewer.Settings +import com.hippo.ehviewer.client.executeSafely +import com.hippo.ehviewer.util.LowSpeedException +import com.hippo.ehviewer.util.ensureSuccess +import io.ktor.client.plugins.onDownload +import io.ktor.client.plugins.timeout +import io.ktor.client.request.HttpRequestBuilder +import io.ktor.client.statement.HttpResponse +import io.ktor.client.statement.HttpStatement +import io.ktor.client.statement.request import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds import kotlin.time.TimeSource +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.map +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -42,6 +56,45 @@ class SpeedTracker(val window: Duration = 1.seconds) { } } +suspend inline fun timeoutBySpeed( + crossinline req: suspend (HttpRequestBuilder.() -> Unit) -> HttpStatement, + crossinline l: suspend (Long, Long, Int) -> Unit, + crossinline f: suspend (HttpResponse) -> R, +) = with(SpeedTracker(2.seconds)) { + var prev = 0L + req { + onDownload { done, total -> + val bytesRead = (done - prev).toInt() + track(bytesRead) + l(total!!, done, bytesRead) + prev = done + } + timeout { connectTimeoutMillis = Settings.connTimeout * 1000L } + }.executeSafely { resp -> + resp.status.ensureSuccess() + val timeoutSpeed = Settings.timeoutSpeed * 1024.0 + coroutineScope { + val work = async { f(resp) } + val job = launch { + delay(2.seconds) // Tolerant 4 secs for receiving + speedFlow(1.seconds).collect { speed -> + if (timeoutSpeed != 0.0 && speed < timeoutSpeed) { + throw LowSpeedException( + resp.request.url.toString(), + speed.toLong(), + ) + } + } + } + try { + work.await() + } finally { + job.cancel() + } + } + } +} + inline fun MutableObjectIntMap.fold(initial: R, operation: (acc: R, T, Int) -> R): R { var accumulator = initial forEach { key, value -> accumulator = operation(accumulator, key, value) } diff --git a/app/src/main/kotlin/com/hippo/ehviewer/updater/AppUpdater.kt b/app/src/main/kotlin/com/hippo/ehviewer/updater/AppUpdater.kt index 55ef14473..a0623989f 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/updater/AppUpdater.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/updater/AppUpdater.kt @@ -4,11 +4,9 @@ import com.hippo.ehviewer.BuildConfig import com.hippo.ehviewer.EhApplication.Companion.ktorClient import com.hippo.ehviewer.Settings import com.hippo.ehviewer.client.executeAndParseAs -import com.hippo.ehviewer.client.executeSafely +import com.hippo.ehviewer.ui.tools.timeoutBySpeed import com.hippo.ehviewer.util.copyTo -import com.hippo.ehviewer.util.ensureSuccess import com.hippo.files.write -import io.ktor.client.plugins.timeout import io.ktor.client.request.HttpRequestBuilder import io.ktor.client.request.header import io.ktor.client.request.prepareGet @@ -70,24 +68,22 @@ object AppUpdater { return null } - suspend fun downloadUpdate(url: String, path: Path) = ghStatement(url) { - timeout { - connectTimeoutMillis = Settings.connTimeout * 1000L - requestTimeoutMillis = 60_000 - } - }.executeSafely { response -> - response.status.ensureSuccess() - if (url.endsWith("zip")) { - response.bodyAsChannel().toInputStream().use { stream -> - ZipInputStream(stream).use { zip -> - zip.nextEntry - path.write { transferFrom(zip.asSource()) } + suspend fun downloadUpdate(url: String, path: Path) = timeoutBySpeed( + { ghStatement(url, builder = it) }, + { _, _, _ -> }, + { response -> + if (url.endsWith("zip")) { + response.bodyAsChannel().toInputStream().use { stream -> + ZipInputStream(stream).use { zip -> + zip.nextEntry + path.write { transferFrom(zip.asSource()) } + } } + } else { + response.bodyAsChannel().copyTo(path) } - } else { - response.bodyAsChannel().copyTo(path) - } - } + }, + ) } private suspend inline fun ghStatement( diff --git a/app/src/main/kotlin/com/hippo/ehviewer/util/ExceptionUtils.kt b/app/src/main/kotlin/com/hippo/ehviewer/util/ExceptionUtils.kt index 956f13efa..d98130a50 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/util/ExceptionUtils.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/util/ExceptionUtils.kt @@ -20,15 +20,17 @@ import eu.kanade.tachiyomi.util.system.logcat import io.ktor.client.network.sockets.ConnectTimeoutException import io.ktor.client.network.sockets.SocketTimeoutException import io.ktor.client.plugins.HttpRequestTimeoutException +import kotlinx.io.IOException import splitties.init.appCtx -fun Throwable.displayString(): String = when (this) { - is HttpRequestTimeoutException, - is ConnectTimeoutException, is SocketTimeoutException, - -> appCtx.getString(R.string.error_timeout) +class LowSpeedException( + url: String, + speed: Long, +) : IOException("Response speed too slow [url=$url, speed=${FileUtils.humanReadableByteCount(speed)}]") - else -> { - logcat(this) - message ?: appCtx.getString(R.string.error_unknown) +fun Throwable.displayString(): String = logcat(this).let { + when (this) { + is HttpRequestTimeoutException, is ConnectTimeoutException, is SocketTimeoutException, is LowSpeedException -> appCtx.getString(R.string.error_timeout) + else -> message ?: appCtx.getString(R.string.error_unknown) } }