From 79c6b63c566ef9cdcbd0d65d25dde0a2e56a06ef Mon Sep 17 00:00:00 2001 From: FooIbar <118464521+FooIbar@users.noreply.github.com> Date: Thu, 19 Oct 2023 14:49:06 +0800 Subject: [PATCH] Refactor image download and improve retry strategy --- app/build.gradle.kts | 1 - .../exception/QuotaExceededException.kt | 5 + .../com/hippo/ehviewer/spider/SpiderQueen.kt | 225 ++++++++---------- 3 files changed, 98 insertions(+), 133 deletions(-) create mode 100644 app/src/main/kotlin/com/hippo/ehviewer/client/exception/QuotaExceededException.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index b475f5f76..962bf7443 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -134,7 +134,6 @@ android { "-progressive", "-Xjvm-default=all", "-Xlambdas=indy", - "-XXLanguage:+BreakContinueInInlineLambdas", "-opt-in=coil.annotation.ExperimentalCoilApi", "-opt-in=androidx.compose.foundation.layout.ExperimentalLayoutApi", diff --git a/app/src/main/kotlin/com/hippo/ehviewer/client/exception/QuotaExceededException.kt b/app/src/main/kotlin/com/hippo/ehviewer/client/exception/QuotaExceededException.kt new file mode 100644 index 000000000..eecd09bc2 --- /dev/null +++ b/app/src/main/kotlin/com/hippo/ehviewer/client/exception/QuotaExceededException.kt @@ -0,0 +1,5 @@ +package com.hippo.ehviewer.client.exception + +import com.hippo.ehviewer.R + +class QuotaExceededException : EhException(R.string.error_509) diff --git a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt index 6fef4b8d1..25367a05e 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt @@ -20,6 +20,7 @@ import android.util.Log import androidx.annotation.IntDef import androidx.collection.LongSparseArray import androidx.collection.set +import arrow.core.partially1 import com.hippo.ehviewer.R import com.hippo.ehviewer.Settings import com.hippo.ehviewer.client.EhEngine @@ -29,8 +30,8 @@ import com.hippo.ehviewer.client.EhUrl.getGalleryMultiPageViewerUrl import com.hippo.ehviewer.client.EhUrl.referer import com.hippo.ehviewer.client.data.GalleryInfo import com.hippo.ehviewer.client.ehRequest -import com.hippo.ehviewer.client.exception.EhException import com.hippo.ehviewer.client.exception.ParseException +import com.hippo.ehviewer.client.exception.QuotaExceededException import com.hippo.ehviewer.client.execute import com.hippo.ehviewer.client.parser.GalleryDetailParser.parsePages import com.hippo.ehviewer.client.parser.GalleryDetailParser.parsePreviewList @@ -612,157 +613,117 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS } var skipHathKey: String? = null - val skipHathKeys = mutableListOf() var originImageUrl: String? = null var error: String? = null var forceHtml = false - var leakSkipHathKey = false - loop@ for (i in 1..3) { - var imageUrl: String? = null - var localShowKey: String? - - showKeyLock.withLock { - localShowKey = showKey - if (localShowKey == null || forceHtml) { - if (leakSkipHathKey) break - var pageUrl = EhUrl.getPageUrl(mSpiderInfo.gid, index, pToken) - // Add skipHathKey - if (skipHathKey != null) { - pageUrl += if ("?" in pageUrl) { - "&nl=$skipHathKey" - } else { - "?nl=$skipHathKey" - } - } - runSuspendCatching { - EhEngine.getGalleryPage(pageUrl, mSpiderInfo.gid, mSpiderInfo.token) - .also { - if (check509(it.imageUrl)) { - // Get 509 - notifyGet509(index) - error = ERROR_509 - break@loop - } - } - }.onSuccess { result -> - imageUrl = result.imageUrl - skipHathKey = result.skipHathKey?.takeIf { it.isNotBlank() } - originImageUrl = result.originImageUrl - localShowKey = result.showKey + runCatching { + repeat(2) { + var imageUrl: String? = null + var localShowKey: String? + showKeyLock.withLock { + localShowKey = showKey + if (localShowKey == null || forceHtml) { + var pageUrl = EhUrl.getPageUrl(mSpiderInfo.gid, index, pToken) + // Skipping H@H costs 50 points, only use it as last resort if (skipHathKey != null) { - if (skipHathKey in skipHathKeys) { - // Duplicate skip hath key - leakSkipHathKey = true + pageUrl += if ("?" in pageUrl) { + "&nl=$skipHathKey" } else { - skipHathKeys.add(skipHathKey!!) + "?nl=$skipHathKey" } - } else { - leakSkipHathKey = true } + EhEngine.getGalleryPage(pageUrl, mSpiderInfo.gid, mSpiderInfo.token) + .let { result -> + check509(result.imageUrl) + imageUrl = result.imageUrl + skipHathKey = result.skipHathKey + originImageUrl = result.originImageUrl + localShowKey = result.showKey + showKey = result.showKey + } + } + } - showKey = result.showKey + if (imageUrl == null) { + runCatching { + EhEngine.getGalleryPageApi( + mSpiderInfo.gid, + index, + pToken, + localShowKey, + previousPToken, + ).let { + check509(it.imageUrl) + imageUrl = it.imageUrl + skipHathKey = it.skipHathKey + originImageUrl = it.originImageUrl + } }.onFailure { - error = ExceptionUtils.getReadableString(it) - break - } - } - } - - if (imageUrl == null) { - if (localShowKey == null) { - error = "ShowKey error" - break - } - runSuspendCatching { - EhEngine.getGalleryPageApi( - mSpiderInfo.gid, - index, - pToken, - localShowKey, - previousPToken, - ).also { - if (check509(it.imageUrl)) { - // Get 509 - notifyGet509(index) - error = ERROR_509 - break@loop + if (it is ParseException && "Key mismatch" == it.message) { + // Show key is wrong, enter a new loop to get the new show key + if (showKey == localShowKey) showKey = null + return@repeat + } else { + throw it } } - }.onFailure { - if (it is ParseException && "Key mismatch" == it.message) { - // Show key is wrong, enter a new loop to get the new show key - if (showKey == localShowKey) showKey = null - continue - } else { - error = ExceptionUtils.getReadableString(it) - break - } - }.onSuccess { - imageUrl = it.imageUrl - skipHathKey = it.skipHathKey - originImageUrl = it.originImageUrl - } - } - - val targetImageUrl: String? - val referer: String? - - if ((Settings.downloadOriginImage || orgImg) && !originImageUrl.isNullOrBlank()) { - val pageUrl = EhUrl.getPageUrl(mSpiderInfo.gid, index, pToken) - targetImageUrl = runSuspendCatching { - EhEngine.getOriginalImageUrl(originImageUrl!!, pageUrl) - }.getOrElse { - error = ExceptionUtils.getReadableString(it) - if (it is EhException) break else continue - } - referer = EhUrl.referer - } else { - targetImageUrl = imageUrl - referer = null - } - if (targetImageUrl == null) { - error = "TargetImageUrl error" - break - } - Log.d(WORKER_DEBUG_TAG, targetImageUrl) - - runCatching { - Log.d(WORKER_DEBUG_TAG, "Start download image $index") - val success: Boolean = mSpiderDen.makeHttpCallAndSaveImage( - index, - targetImageUrl, - referer, - ) { contentLength: Long, receivedSize: Long, bytesRead: Int -> - notifyPageDownload(index, contentLength, receivedSize, bytesRead) } - if (!success) { - Log.e(WORKER_DEBUG_TAG, "Can't download all of image data") - error = "Incomplete" + val targetImageUrl: String? + val referer: String? + + if ((Settings.downloadOriginImage || orgImg) && !originImageUrl.isNullOrBlank()) { + val pageUrl = EhUrl.getPageUrl(mSpiderInfo.gid, index, pToken) + targetImageUrl = EhEngine.getOriginalImageUrl(originImageUrl!!, pageUrl) + referer = EhUrl.referer + } else { + // Original image url won't change, so only set forceHtml in this case forceHtml = true - continue@loop + targetImageUrl = imageUrl + referer = null } + checkNotNull(targetImageUrl) + Log.d(WORKER_DEBUG_TAG, targetImageUrl) - Log.d(WORKER_DEBUG_TAG, "Download image succeed $index") - updatePageState(index, STATE_FINISHED) - return - }.onFailure { - if (it is CancellationException) { - Log.d(WORKER_DEBUG_TAG, "Download image cancelled $index") + repeat(3) { times -> + runCatching { + Log.d(WORKER_DEBUG_TAG, "Start download image $index attempt #$times") + val success = mSpiderDen.makeHttpCallAndSaveImage( + index, + targetImageUrl, + referer, + this@SpiderQueen::notifyPageDownload.partially1(index), + ) + + check(success) + Log.d(WORKER_DEBUG_TAG, "Download image $index succeed") + updatePageState(index, STATE_FINISHED) + return + }.onFailure { + mSpiderDen.remove(index) + if (it is CancellationException) { + throw it + } else { + Log.d(WORKER_DEBUG_TAG, "Download image $index attempt #$times failed") + error = ExceptionUtils.getReadableString(it) + } + } + } + } + }.onFailure { + when (it) { + is CancellationException -> { + Log.d(WORKER_DEBUG_TAG, "Download image $index cancelled") error = "Cancelled" - mSpiderDen.remove(index) updatePageState(index, STATE_FAILED, error) throw it - } else { - it.printStackTrace() - error = NETWORK_ERROR - forceHtml = true } + + is QuotaExceededException -> notifyGet509(index) } - Log.d(WORKER_DEBUG_TAG, "End download image $index") + error = ExceptionUtils.getReadableString(it) } - mSpiderDen.remove(index) updatePageState(index, STATE_FAILED, error) } @@ -808,8 +769,6 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS } private val PTOKEN_FAILED_MESSAGE = appCtx.getString(R.string.error_get_ptoken_error) -private val ERROR_509 = appCtx.getString(R.string.error_509) -private val NETWORK_ERROR = appCtx.getString(R.string.error_socket) private val DECODE_ERROR = appCtx.getString(R.string.error_decoding_failed) private val URL_509_ARRAY = arrayOf( "https://ehgt.org/g/509.gif", @@ -819,4 +778,6 @@ private val URL_509_ARRAY = arrayOf( ) private const val WORKER_DEBUG_TAG = "SpiderQueenWorker" -private fun check509(url: String) = url in URL_509_ARRAY +private fun check509(url: String) { + if (url in URL_509_ARRAY) throw QuotaExceededException() +}