From 933a95023ce8e92888bbede1539a0aa91babe185 Mon Sep 17 00:00:00 2001 From: tarsin Date: Sun, 5 Feb 2023 12:31:30 +0800 Subject: [PATCH] Tweak readahead logic --- .../hippo/ehviewer/gallery/EhPageLoader.java | 4 +- .../hippo/ehviewer/spider/SpiderQueen.java | 28 +- .../ehviewer/spider/SpiderQueenWorker.kt | 389 ++++++++++-------- 3 files changed, 217 insertions(+), 204 deletions(-) diff --git a/app/src/main/java/com/hippo/ehviewer/gallery/EhPageLoader.java b/app/src/main/java/com/hippo/ehviewer/gallery/EhPageLoader.java index 7e227d5ac..07c56717a 100644 --- a/app/src/main/java/com/hippo/ehviewer/gallery/EhPageLoader.java +++ b/app/src/main/java/com/hippo/ehviewer/gallery/EhPageLoader.java @@ -124,7 +124,7 @@ public class EhPageLoader extends PageLoader2 implements SpiderQueen.OnSpiderLis @Override protected void onRequest(int index) { if (mSpiderQueen != null) { - Object object = mSpiderQueen.request(index, true); + Object object = mSpiderQueen.request(index); if (object instanceof Float) { notifyPagePercent(index, (Float) object); } else if (object instanceof String) { @@ -185,7 +185,7 @@ public class EhPageLoader extends PageLoader2 implements SpiderQueen.OnSpiderLis @Override public void onPageSuccess(int index, int finished, int downloaded, int total) { if (mSpiderQueen != null) { - mSpiderQueen.request(index, false); + mSpiderQueen.request(index); } } diff --git a/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueen.java b/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueen.java index 228911f29..ac7e77b05 100644 --- a/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueen.java +++ b/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueen.java @@ -28,7 +28,6 @@ import androidx.collection.LongSparseArray; import com.hippo.ehviewer.EhApplication; import com.hippo.ehviewer.GetText; import com.hippo.ehviewer.R; -import com.hippo.ehviewer.Settings; import com.hippo.ehviewer.client.EhRequestBuilder; import com.hippo.ehviewer.client.EhUrl; import com.hippo.ehviewer.client.data.GalleryInfo; @@ -40,7 +39,6 @@ import com.hippo.ehviewer.client.parser.GalleryPageUrlParser; import com.hippo.image.Image; import com.hippo.unifile.UniFile; import com.hippo.util.ExceptionUtils; -import com.hippo.yorozuya.MathUtils; import com.hippo.yorozuya.OSUtils; import com.hippo.yorozuya.thread.PriorityThread; @@ -100,11 +98,9 @@ public final class SpiderQueen implements Runnable { private final Object mPageStateLock = new Object(); private final AtomicInteger mDownloadedPages = new AtomicInteger(0); private final AtomicInteger mFinishedPages = new AtomicInteger(0); - private final Object showKeyLock = new Object(); // Store page error private final ConcurrentHashMap mPageErrorMap = new ConcurrentHashMap<>(); private final List mSpiderListeners = new ArrayList<>(); - private final int mPreloadNumber; public volatile int[] mPageStateArray; private int mReadReference = 0; private int mDownloadReference = 0; @@ -121,8 +117,6 @@ public final class SpiderQueen implements Runnable { mGalleryInfo = galleryInfo; mSpiderDen = new SpiderDen(mGalleryInfo); - mPreloadNumber = MathUtils.clamp(Settings.getPreloadImage(), 0, 100); - for (int i = 0; i < DECODE_THREAD_NUM; i++) { mDecodeIndexArray[i] = -1; } @@ -388,11 +382,11 @@ public final class SpiderQueen implements Runnable { } public Object forceRequest(int index) { - return request(index, true, true, false); + return request(index, true, true); } - public Object request(int index, boolean addNeighbor) { - return request(index, true, false, addNeighbor); + public Object request(int index) { + return request(index, true, false); } private int getPageState(int index) { @@ -410,7 +404,7 @@ public final class SpiderQueen implements Runnable { return; } - mWorkerScope.removeRequested(index); + mWorkerScope.cancel(index); synchronized (mDecodeRequestQueue) { mDecodeRequestQueue.remove(index); } @@ -421,9 +415,7 @@ public final class SpiderQueen implements Runnable { return; } - for (int i : pages) { - mWorkerScope.launchIndexed(i); - } + mWorkerScope.updateRAList(pages); } /** @@ -431,7 +423,7 @@ public final class SpiderQueen implements Runnable { * Float for download percent
* null for wait */ - private Object request(int index, boolean ignoreError, boolean force, boolean addNeighbor) { + private Object request(int index, boolean ignoreError, boolean force) { if (mQueenThread == null) { return null; } @@ -447,10 +439,7 @@ public final class SpiderQueen implements Runnable { state = STATE_NONE; } - mWorkerScope.launchIndexed(index, force); - for (int i = index + 1, n = index + 1 + mPreloadNumber; i < n && i < mPageStateArray.length; i++) { - mWorkerScope.launchIndexed(i); - } + mWorkerScope.launch(index, force); Object result; @@ -797,6 +786,7 @@ public final class SpiderQueen implements Runnable { @Retention(RetentionPolicy.SOURCE) public @interface State { } + public interface OnSpiderListener { void onGetPages(int pages); @@ -873,7 +863,7 @@ public final class SpiderQueen implements Runnable { // Can't find the file, it might be removed from cache, // Reset it state and request it updatePageState(index, STATE_NONE, null); - request(index, false, false, false); + request(index, false, false); continue; } diff --git a/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueenWorker.kt b/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueenWorker.kt index 93d8a21c1..72f6675b8 100644 --- a/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueenWorker.kt +++ b/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueenWorker.kt @@ -42,200 +42,223 @@ class SpiderQueenWorker(private val queen: SpiderQueen) : CoroutineScope { override val coroutineContext: CoroutineContext get() = Dispatchers.IO.limitedParallelism(maxParallelismSize) + SupervisorJob() - fun removeRequested(index: Int) { - mJobMap[index]?.cancel() + fun cancel(index: Int) { + synchronized(mJobMap) { + mJobMap.remove(index)?.cancel() + } + } + + fun updateRAList(list: List) { + synchronized(mJobMap) { + sequence { + mJobMap.forEach { (i, job) -> + if (i !in list) { + job.cancel() + } + if (!job.isActive) yield(i) + } + }.toSet().forEach { mJobMap.remove(it) } + list.forEach { + if (mJobMap[it]?.isActive != true) + launch(it) + } + } } @JvmOverloads - @Synchronized - fun launchIndexed(index: Int, force: Boolean = false) { + fun launch(index: Int, force: Boolean = false) { check(index in 0 until size) val state = queen.mPageStateArray[index] - val currentJob = mJobMap[index] if (!force && state == STATE_FINISHED) return - if (force) currentJob?.cancel() - if (currentJob?.isActive != true) { - mJobMap[index] = launch { - fun getPToken(index: Int): String? { - if (index !in 0 until size) return null - return spiderInfo.pTokenMap[index].takeIf { it != TOKEN_FAILED } - ?: queen.getPTokenFromInternet(index) - ?: queen.getPTokenFromInternet(index) - ?: queen.getPTokenFromMultiPageViewer(index) - } - queen.updatePageState(index, SpiderQueen.STATE_DOWNLOADING) - if (!force && index in spiderDen) { - queen.updatePageState(index, STATE_FINISHED) - return@launch - } - if (force) { - synchronized(pTokenLock) { - val pToken = spiderInfo.pTokenMap[index] - if (pToken == TOKEN_FAILED) spiderInfo.pTokenMap.remove(index) - } - } - var pToken: String? - var previousPToken: String? - currentCoroutineContext().ensureActive() - synchronized(pTokenLock) { - pToken = getPToken(index) - if (pToken == null) { - spiderInfo.pTokenMap[index] = TOKEN_FAILED - queen.updatePageState(index, STATE_FAILED, PTOKEN_FAILED_MESSAGE) - return@launch - } - previousPToken = getPToken(index - 1) - } - - var skipHathKey: String? = null - val skipHathKeys = mutableListOf() - var originImageUrl: String? = null - val pageUrl: String? = null - var error: String? = null - var forceHtml = false - var leakSkipHathKey = false - repeat(3) { - var imageUrl: String? = null - var localShowKey: String? - - currentCoroutineContext().ensureActive() - synchronized(showKeyLock) { - localShowKey = showKey - if (localShowKey == null || forceHtml) { - if (leakSkipHathKey) return@repeat - val pageUrl1 = - getPageUrl(spiderInfo.gid, index, pToken!!, pageUrl, skipHathKey) - runCatching { - fetchPageResultFromHtml(index, pageUrl1) - }.onSuccess { - imageUrl = it.imageUrl - skipHathKey = it.skipHathKey - originImageUrl = it.originImageUrl - localShowKey = it.showKey - - if (!skipHathKey.isNullOrBlank()) { - if (skipHathKeys.contains(skipHathKey)) { - // Duplicate skip hath key - leakSkipHathKey = true - } else { - skipHathKeys.add(skipHathKey!!) - } - } else { - leakSkipHathKey = true - } - - showKey = it.showKey - }.onFailure { - if (it is Image509Exception) { - error = ERROR_509 - return@repeat - } - if (it is ParseException && "Key mismatch" == it.message) { - // Show key is wrong, enter a new loop to get the new show key - if (showKey == localShowKey) showKey = null - return@repeat - } else { - error = ExceptionUtils.getReadableString(it) - return@repeat - } - } - } - } - - currentCoroutineContext().ensureActive() - if (imageUrl == null) { - if (localShowKey == null) { - error = "ShowKey error" - return@repeat - } - runCatching { - fetchPageResultFromApi( - spiderInfo.gid, - index, - pToken, - localShowKey, - previousPToken - ) - }.onFailure { - if (it is Image509Exception) { - error = ERROR_509 - return@repeat - } - if (it is ParseException && "Key mismatch" == it.message) { - // Show key is wrong, enter a new loop to get the new show key - if (showKey == localShowKey) showKey = null - return@repeat - } else { - error = ExceptionUtils.getReadableString(it) - return@repeat - } - }.onSuccess { - imageUrl = it.imageUrl - skipHathKey = it.skipHathKey - originImageUrl = it.originImageUrl - } - } - - val targetImageUrl: String? - val referer: String? - - if (Settings.getDownloadOriginImage() && !originImageUrl.isNullOrBlank()) { - targetImageUrl = originImageUrl - referer = getPageUrl(spiderInfo.gid, index, pToken!!) - } else { - targetImageUrl = imageUrl - referer = null - } - if (targetImageUrl == null) { - error = "TargetImageUrl error" - return@repeat - } - Log.d(DEBUG_TAG, targetImageUrl) - - currentCoroutineContext().ensureActive() - runCatching { - Log.d(DEBUG_TAG, "Start download image $index") - val success: Boolean = spiderDen.makeHttpCallAndSaveImage( - index, - targetImageUrl, - referer - ) { contentLength: Long, receivedSize: Long, bytesRead: Int -> - queen.mPagePercentMap[index] = receivedSize.toFloat() / contentLength - queen.notifyPageDownload(index, contentLength, receivedSize, bytesRead) - } - - if (!success) { - Log.e(DEBUG_TAG, "Can't download all of image data") - error = "Incomplete" - forceHtml = true - return@repeat - } - - if (spiderDen.checkPlainText(index)) { - error = "" - forceHtml = true - return@repeat - } - - Log.d(DEBUG_TAG, "Download image succeed $index") - - queen.updatePageState(index, STATE_FINISHED) - delay(mDownloadDelay.toLong()) - return@launch - }.onFailure { - if (it is CancellationException) throw it - it.printStackTrace() - error = NETWORK_ERROR - } - Log.d(DEBUG_TAG, "End download image $index") - } - spiderDen.remove(index) - queen.updatePageState(index, STATE_FAILED, error) + synchronized(mJobMap) { + val currentJob = mJobMap[index] + if (force) currentJob?.cancel() + if (currentJob?.isActive != true) { + mJobMap[index] = launch { doInJob(index, force) } } } } + private suspend fun doInJob(index: Int, force: Boolean) { + fun getPToken(index: Int): String? { + if (index !in 0 until size) return null + return spiderInfo.pTokenMap[index].takeIf { it != TOKEN_FAILED } + ?: queen.getPTokenFromInternet(index) + ?: queen.getPTokenFromInternet(index) + ?: queen.getPTokenFromMultiPageViewer(index) + } + queen.updatePageState(index, SpiderQueen.STATE_DOWNLOADING) + if (!force && index in spiderDen) { + queen.updatePageState(index, STATE_FINISHED) + return + } + if (force) { + synchronized(pTokenLock) { + val pToken = spiderInfo.pTokenMap[index] + if (pToken == TOKEN_FAILED) spiderInfo.pTokenMap.remove(index) + } + } + var pToken: String? + var previousPToken: String? + + currentCoroutineContext().ensureActive() + synchronized(pTokenLock) { + pToken = getPToken(index) + if (pToken == null) { + spiderInfo.pTokenMap[index] = TOKEN_FAILED + queen.updatePageState(index, STATE_FAILED, PTOKEN_FAILED_MESSAGE) + return + } + previousPToken = getPToken(index - 1) + } + + var skipHathKey: String? = null + val skipHathKeys = mutableListOf() + var originImageUrl: String? = null + val pageUrl: String? = null + var error: String? = null + var forceHtml = false + var leakSkipHathKey = false + repeat(3) { + var imageUrl: String? = null + var localShowKey: String? + + currentCoroutineContext().ensureActive() + synchronized(showKeyLock) { + localShowKey = showKey + if (localShowKey == null || forceHtml) { + if (leakSkipHathKey) return@repeat + val pageUrl1 = + getPageUrl(spiderInfo.gid, index, pToken!!, pageUrl, skipHathKey) + runCatching { + fetchPageResultFromHtml(index, pageUrl1) + }.onSuccess { + imageUrl = it.imageUrl + skipHathKey = it.skipHathKey + originImageUrl = it.originImageUrl + localShowKey = it.showKey + + if (!skipHathKey.isNullOrBlank()) { + if (skipHathKeys.contains(skipHathKey)) { + // Duplicate skip hath key + leakSkipHathKey = true + } else { + skipHathKeys.add(skipHathKey!!) + } + } else { + leakSkipHathKey = true + } + + showKey = it.showKey + }.onFailure { + if (it is Image509Exception) { + error = ERROR_509 + return@repeat + } + if (it is ParseException && "Key mismatch" == it.message) { + // Show key is wrong, enter a new loop to get the new show key + if (showKey == localShowKey) showKey = null + return@repeat + } else { + error = ExceptionUtils.getReadableString(it) + return@repeat + } + } + } + } + + currentCoroutineContext().ensureActive() + if (imageUrl == null) { + if (localShowKey == null) { + error = "ShowKey error" + return@repeat + } + runCatching { + fetchPageResultFromApi( + spiderInfo.gid, + index, + pToken, + localShowKey, + previousPToken + ) + }.onFailure { + if (it is Image509Exception) { + error = ERROR_509 + return@repeat + } + if (it is ParseException && "Key mismatch" == it.message) { + // Show key is wrong, enter a new loop to get the new show key + if (showKey == localShowKey) showKey = null + return@repeat + } else { + error = ExceptionUtils.getReadableString(it) + return@repeat + } + }.onSuccess { + imageUrl = it.imageUrl + skipHathKey = it.skipHathKey + originImageUrl = it.originImageUrl + } + } + + val targetImageUrl: String? + val referer: String? + + if (Settings.getDownloadOriginImage() && !originImageUrl.isNullOrBlank()) { + targetImageUrl = originImageUrl + referer = getPageUrl(spiderInfo.gid, index, pToken!!) + } else { + targetImageUrl = imageUrl + referer = null + } + if (targetImageUrl == null) { + error = "TargetImageUrl error" + return@repeat + } + Log.d(DEBUG_TAG, targetImageUrl) + + currentCoroutineContext().ensureActive() + runCatching { + Log.d(DEBUG_TAG, "Start download image $index") + val success: Boolean = spiderDen.makeHttpCallAndSaveImage( + index, + targetImageUrl, + referer + ) { contentLength: Long, receivedSize: Long, bytesRead: Int -> + queen.mPagePercentMap[index] = receivedSize.toFloat() / contentLength + queen.notifyPageDownload(index, contentLength, receivedSize, bytesRead) + } + + if (!success) { + Log.e(DEBUG_TAG, "Can't download all of image data") + error = "Incomplete" + forceHtml = true + return@repeat + } + + if (spiderDen.checkPlainText(index)) { + error = "" + forceHtml = true + return@repeat + } + + Log.d(DEBUG_TAG, "Download image succeed $index") + + queen.updatePageState(index, STATE_FINISHED) + delay(mDownloadDelay.toLong()) + return + }.onFailure { + if (it is CancellationException) throw it + it.printStackTrace() + error = NETWORK_ERROR + } + Log.d(DEBUG_TAG, "End download image $index") + } + spiderDen.remove(index) + queen.updatePageState(index, STATE_FAILED, error) + } + private fun getPageUrl( gid: Long, index: Int, pToken: String, oldPageUrl: String?, skipHathKey: String?