From 52f314f6b03bcbfabf62d8903fbce66f4b51195b Mon Sep 17 00:00:00 2001 From: Vladimir Dostoyevskiy <99822064+VDostoyevskiy@users.noreply.github.com> Date: Sat, 2 Dec 2023 09:33:46 +0800 Subject: [PATCH] Move to Ktor, Part 1 (#415) * Ktor 3.0.0-beta-1 * Drop chunker since we no more se okhttp * Move getGalleryDetail to ktor * Migration * Migration * Migration * Migration * Migration * Migration * Migration * Migration --------- Co-authored-by: GG Bond --- app/build.gradle.kts | 3 - app/proguard-rules.pro | 3 + .../com/hippo/ehviewer/EhApplication.kt | 12 +- .../com/hippo/ehviewer/client/EhEngine.kt | 230 ++++++++++-------- .../hippo/ehviewer/client/EhRequestBuilder.kt | 105 +++----- .../com/hippo/ehviewer/ktbuilder/Okhttp.kt | 3 - .../com/hippo/ehviewer/spider/SpiderDen.kt | 52 ++-- .../com/hippo/ehviewer/spider/SpiderQueen.kt | 22 +- gradle/libs.versions.toml | 11 +- 9 files changed, 225 insertions(+), 216 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index ce5298009..90038ef41 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -269,9 +269,6 @@ dependencies { implementation(libs.kotlinx.datetime) implementation(libs.jsoup) - debugImplementation(libs.chucker) - releaseImplementation(libs.chucker.nop) - coreLibraryDesugaring(libs.desugar) "gmsImplementation"(libs.bundles.cronet) diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index a363c1c21..354f15f55 100644 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -16,5 +16,8 @@ setWindowFocused(boolean); } +# Ktor logger +-dontwarn org.slf4j.impl.StaticLoggerBinder + -allowaccessmodification -repackageclasses diff --git a/app/src/main/kotlin/com/hippo/ehviewer/EhApplication.kt b/app/src/main/kotlin/com/hippo/ehviewer/EhApplication.kt index 537f75107..bfe30d6cc 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/EhApplication.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/EhApplication.kt @@ -35,7 +35,6 @@ import com.hippo.ehviewer.dailycheck.checkDawn import com.hippo.ehviewer.dao.SearchDatabase import com.hippo.ehviewer.download.DownloadManager import com.hippo.ehviewer.ktbuilder.cache -import com.hippo.ehviewer.ktbuilder.chunker import com.hippo.ehviewer.ktbuilder.diskCache import com.hippo.ehviewer.ktbuilder.httpClient import com.hippo.ehviewer.ktbuilder.imageLoader @@ -57,6 +56,8 @@ import eu.kanade.tachiyomi.network.interceptor.UncaughtExceptionInterceptor import eu.kanade.tachiyomi.util.lang.launchIO import eu.kanade.tachiyomi.util.lang.withUIContext import eu.kanade.tachiyomi.util.system.logcat +import io.ktor.client.HttpClient +import io.ktor.client.engine.okhttp.OkHttp import kotlinx.coroutines.launch import moe.tarsin.kt.unreachable import okhttp3.AsyncDns @@ -178,13 +179,20 @@ class EhApplication : Application(), ImageLoaderFactory { } companion object { + val ktorClient by lazy { + HttpClient(OkHttp) { + engine { + preconfigured = baseOkHttpClient + } + } + } + val baseOkHttpClient by lazy { httpClient { cookieJar(EhCookieStore) if (isAtLeastQ) { dns(AsyncDns.toDns(AndroidAsyncDns.IPv4, AndroidAsyncDns.IPv6)) } - chunker { alwaysReadResponseBody(false) } addInterceptor(UncaughtExceptionInterceptor()) addInterceptor(CloudflareInterceptor(appCtx)) } diff --git a/app/src/main/kotlin/com/hippo/ehviewer/client/EhEngine.kt b/app/src/main/kotlin/com/hippo/ehviewer/client/EhEngine.kt index b85aae375..59d57b8e4 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/client/EhEngine.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/client/EhEngine.kt @@ -16,6 +16,9 @@ package com.hippo.ehviewer.client import android.util.Log +import arrow.core.Either +import arrow.core.left +import arrow.core.right import arrow.fx.coroutines.parMap import arrow.fx.coroutines.parZip import com.hippo.ehviewer.R @@ -54,6 +57,11 @@ import com.hippo.ehviewer.dailycheck.today import com.hippo.ehviewer.util.AppConfig import com.hippo.ehviewer.util.StatusCodeException import com.hippo.ehviewer.util.isCronetSupported +import eu.kanade.tachiyomi.util.system.logcat +import io.ktor.client.statement.HttpStatement +import io.ktor.client.statement.bodyAsChannel +import io.ktor.client.statement.bodyAsText +import io.ktor.client.statement.request import io.ktor.utils.io.pool.DirectByteBufferPool import io.ktor.utils.io.pool.useInstance import java.io.File @@ -70,23 +78,26 @@ import moe.tarsin.coroutines.runSuspendCatching import okhttp3.MediaType import okhttp3.MediaType.Companion.toMediaType import okhttp3.MultipartBody -import okhttp3.Request import okhttp3.RequestBody import okhttp3.RequestBody.Companion.asRequestBody import org.jsoup.Jsoup import splitties.init.appCtx -private const val TAG = "EhEngine" +const val TAG = "EhEngine" private const val MAX_REQUEST_SIZE = 25 private const val U_CONFIG_TEXT = "Selected Profile" private val MEDIA_TYPE_JPEG: MediaType = "image/jpeg".toMediaType() -private fun rethrowExactly(code: Int, body: String, e: Throwable): Nothing { +fun rethrowExactly(code: Int, body: Either, e: Throwable): Nothing { // Don't translate coroutine cancellation if (e is CancellationException) throw e // Check sad panda(without panda) - if (body.isEmpty()) { + val empty = body.fold( + { it.isEmpty() }, + { !it.hasRemaining() }, + ) + if (empty) { if (EhUtils.isExHentai) { throw EhException("Sad Panda\n(without panda)") } else { @@ -95,10 +106,12 @@ private fun rethrowExactly(code: Int, body: String, e: Throwable): Nothing { } // Check Gallery Not Available - if (body.contains("Gallery Not Available - ")) { - val error = GalleryNotAvailableParser.parse(body) - if (!error.isNullOrBlank()) { - throw EhException(error) + body.onLeft { + if ("Gallery Not Available - " in it) { + val error = GalleryNotAvailableParser.parse(it) + if (!error.isNullOrBlank()) { + throw EhException(error) + } } } @@ -108,12 +121,17 @@ private fun rethrowExactly(code: Int, body: String, e: Throwable): Nothing { } if (e is ParseException || e is SerializationException) { - if ("<" !in body) { - throw EhException(body) - } else { - if (Settings.saveParseErrorBody) AppConfig.saveParseErrorBody(e, body) - throw EhException(appCtx.getString(R.string.error_parse_error), e) + body.onLeft { + if ("<" !in it) { + throw EhException(it) + } else { + if (Settings.saveParseErrorBody) AppConfig.saveParseErrorBody(e, it) + throw EhException(appCtx.getString(R.string.error_parse_error), e) + } } + body.onRight { + } + // Save content for ByteBuffer as well } // We can't translate it, rethrow it anyway @@ -172,16 +190,26 @@ suspend fun fetchCompat( } } -@Suppress("REDUNDANT_INLINE_SUSPEND_FUNCTION_TYPE") -private suspend inline fun Request.executeAndParsingWith(block: suspend String.() -> T): T { - Log.d(TAG, url.toString()) - return execute { - val body = body.string() - try { - block(body) - } catch (e: Exception) { - rethrowExactly(code, body, e) - } +suspend inline fun HttpStatement.parseString(crossinline block: suspend String.() -> T) = execute { response -> + logcat(tag = TAG) { response.request.url.toString() } + val body = response.bodyAsText() + runSuspendCatching { + block(body) + }.onFailure { + rethrowExactly(response.status.value, body.left(), it) + }.getOrThrow() +} + +suspend inline fun HttpStatement.parseByteBuffer(crossinline block: suspend ByteBuffer.() -> T) = execute { response -> + logcat(tag = TAG) { response.request.url.toString() } + httpContentPool.useInstance { buffer -> + with(response.bodyAsChannel()) { while (!isClosedForRead) readAvailable(buffer) } + buffer.flip() + runSuspendCatching { + block(buffer) + }.onFailure { + rethrowExactly(response.status.value, buffer.right(), it) + }.getOrThrow() } } @@ -199,37 +227,37 @@ object EhEngine { suspend fun getTorrentList(url: String, gid: Long, token: String?): TorrentResult { val referer = EhUrl.getGalleryDetailUrl(gid, token) - return fetchCompat(url, referer, parser = TorrentParser::parse) + return statement(url, referer).parseByteBuffer(TorrentParser::parse) } - suspend fun getArchiveList(url: String, gid: Long, token: String?) = ehRequest(url, EhUrl.getGalleryDetailUrl(gid, token)) - .executeAndParsingWith(ArchiveParser::parse) - .apply { funds = funds ?: ehRequest(EhUrl.URL_FUNDS).executeAndParsingWith(HomeParser::parseFunds) } + suspend fun getArchiveList(url: String, gid: Long, token: String?) = statement(url, EhUrl.getGalleryDetailUrl(gid, token)) + .parseString(ArchiveParser::parse) + .apply { funds = funds ?: statement(EhUrl.URL_FUNDS).parseString(HomeParser::parseFunds) } suspend fun getImageLimits() = parZip( - { fetchCompat(EhUrl.URL_HOME, parser = HomeParser::parse) }, - { ehRequest(EhUrl.URL_FUNDS).executeAndParsingWith(HomeParser::parseFunds) }, + { statement(EhUrl.URL_HOME).parseByteBuffer(HomeParser::parse) }, + { statement(EhUrl.URL_FUNDS).parseString(HomeParser::parseFunds) }, { limits, funds -> HomeParser.Result(limits, funds) }, ) - suspend fun getNews(parse: Boolean) = ehRequest(EhUrl.URL_NEWS, EhUrl.REFERER_E) - .executeAndParsingWith { if (parse) EventPaneParser.parse(this) else null } + suspend fun getNews(parse: Boolean) = statement(EhUrl.URL_NEWS, EhUrl.REFERER_E) + .parseString { if (parse) EventPaneParser.parse(this) else null } suspend fun getProfile(): ProfileParser.Result { - val url = ehRequest(EhUrl.URL_FORUMS).executeAndParsingWith(ForumsParser::parse) - return ehRequest(url, EhUrl.URL_FORUMS).executeAndParsingWith(ProfileParser::parse) + val url = statement(EhUrl.URL_FORUMS).parseString(ForumsParser::parse) + return statement(url, EhUrl.URL_FORUMS).parseString(ProfileParser::parse) } suspend fun getUConfig(url: String = EhUrl.uConfigUrl) { runSuspendCatching { - ehRequest(url).executeAndParsingWith { check(contains(U_CONFIG_TEXT)) { "Unable to load config from $url!" } } - }.onFailure { + statement(url).parseString { check(U_CONFIG_TEXT in this) { "Unable to load config from $url!" } } + }.onFailure { throwable -> // It may get redirected when accessing ex for the first time if (url == EhUrl.URL_UCONFIG_EX) { - it.printStackTrace() - ehRequest(url).executeAndParsingWith { check(contains(U_CONFIG_TEXT)) { "Unable to load config from $url!" } } + throwable.printStackTrace() + statement(url).parseString { check(U_CONFIG_TEXT in this) { "Unable to load config from $url!" } } } else { - throw it + throw throwable } } } @@ -240,54 +268,55 @@ object EhEngine { token: String?, ): GalleryPageParser.Result { val referer = EhUrl.getGalleryDetailUrl(gid, token) - return ehRequest(url, referer).executeAndParsingWith(GalleryPageParser::parse) + return statement(url, referer).parseString(GalleryPageParser::parse) } - suspend fun getGalleryList(url: String) = fetchCompat(url, EhUrl.referer, parser = GalleryListParser::parse) + suspend fun getGalleryList(url: String) = statement(url, EhUrl.referer).parseByteBuffer(GalleryListParser::parse) .apply { fillGalleryList(galleryInfoList, url, true) } .takeUnless { it.galleryInfoList.isEmpty() } ?: GalleryListParser.emptyResult - suspend fun getGalleryDetail(url: String) = ehRequest(url, EhUrl.referer).executeAndParsingWith { - EventPaneParser.parse(this)?.let { + suspend fun getGalleryDetail(url: String) = statement(url, EhUrl.referer).parseString { + val eventPane = EventPaneParser.parse(this) + if (eventPane != null) { Settings.lastDawnDay = today - showEventNotification(it) + showEventNotification(eventPane) } GalleryDetailParser.parse(this) } - suspend fun getPreviewList(url: String) = ehRequest(url, EhUrl.referer).executeAndParsingWith { + suspend fun getPreviewList(url: String) = statement(url, EhUrl.referer).parseString { GalleryDetailParser.parsePreviewList(this) to GalleryDetailParser.parsePreviewPages(this) } - suspend fun getFavorites(url: String) = fetchCompat(url, EhUrl.referer, parser = FavoritesParser::parse) + suspend fun getFavorites(url: String) = statement(url, EhUrl.referer).parseByteBuffer(FavoritesParser::parse) .apply { fillGalleryList(galleryInfoList, url) } suspend fun signIn(username: String, password: String): String { val referer = "https://forums.e-hentai.org/index.php?act=Login&CODE=00" val url = EhUrl.API_SIGN_IN val origin = "https://forums.e-hentai.org" - return ehRequest(url, referer, origin) { + return statement(url, referer, origin) { formBody { - add("referer", referer) - add("b", "") - add("bt", "") - add("UserName", username) - add("PassWord", password) - add("CookieDate", "1") + append("referer", referer) + append("b", "") + append("bt", "") + append("UserName", username) + append("PassWord", password) + append("CookieDate", "1") } - }.executeAndParsingWith(SignInParser::parse) + }.parseString(SignInParser::parse) } - suspend fun commentGallery(url: String, comment: String, id: Long = -1) = ehRequest(url, url, EhUrl.origin) { + suspend fun commentGallery(url: String, comment: String, id: Long = -1) = statement(url, url, EhUrl.origin) { formBody { if (id == -1L) { - add("commenttext_new", comment) + append("commenttext_new", comment) } else { - add("commenttext_edit", comment) - add("edit_comment", id.toString()) + append("commenttext_edit", comment) + append("edit_comment", id.toString()) } } - }.executeAndParsingWith { + }.parseString { val document = Jsoup.parse(this) val elements = document.select("#chd + p") if (elements.size > 0) { @@ -303,14 +332,15 @@ object EhEngine { else -> throw EhException("Invalid dstCat: $dstCat") } val url = EhUrl.getAddFavorites(gid, token) - val body = formBody { - add("favcat", catStr) - add("favnote", note) - // apply=Add+to+Favorites is not necessary, just use apply=Apply+Changes all the time - add("apply", "Apply Changes") - add("update", "1") - } - fetchCompat(url, url, EhUrl.origin, body) { } + statement(url, url, EhUrl.origin) { + formBody { + append("favcat", catStr) + append("favnote", note) + // apply=Add+to+Favorites is not necessary, just use apply=Apply+Changes all the time + append("apply", "Apply Changes") + append("update", "1") + } + }.execute { } } suspend fun downloadArchive( @@ -328,26 +358,26 @@ object EhEngine { } val url = EhUrl.getDownloadArchive(gid, token, or) val referer = EhUrl.getGalleryDetailUrl(gid, token) - val request = ehRequest(url, referer, EhUrl.origin) { + val request = statement(url, referer, EhUrl.origin) { formBody { if (isHAtH) { - add("hathdl_xres", res) + append("hathdl_xres", res) } else { - add("dltype", res) + append("dltype", res) if (res == "org") { - add("dlcheck", "Download Original Archive") + append("dlcheck", "Download Original Archive") } else { - add("dlcheck", "Download Resample Archive") + append("dlcheck", "Download Resample Archive") } } } } - var result = request.executeAndParsingWith(ArchiveParser::parseArchiveUrl) + var result = request.parseString(ArchiveParser::parseArchiveUrl) if (!isHAtH) { if (result == null) { // Wait for the server to prepare archives delay(1000) - result = request.executeAndParsingWith(ArchiveParser::parseArchiveUrl) + result = request.parseString(ArchiveParser::parseArchiveUrl) if (result == null) { throw EhException("Archive unavailable") } @@ -357,11 +387,11 @@ object EhEngine { return null } - suspend fun resetImageLimits() = ehRequest(EhUrl.URL_HOME) { + suspend fun resetImageLimits() = statement(EhUrl.URL_HOME) { formBody { - add("reset_imagelimit", "Reset Limit") + append("reset_imagelimit", "Reset Limit") } - }.executeAndParsingWith(HomeParser::parseResetLimits) + }.parseString(HomeParser::parseResetLimits) suspend fun modifyFavorites(gidArray: LongArray, srcCat: Int, dstCat: Int): FavoritesParser.Result { val url = ehUrl { @@ -375,12 +405,12 @@ object EhEngine { in 0..9 -> "fav$dstCat" else -> throw EhException("Invalid dstCat: $dstCat") } - val body = formBody { - add("ddact", catStr) - gidArray.forEach { add("modifygids[]", it.toString()) } - } - return fetchCompat(url, url, EhUrl.origin, body, FavoritesParser::parse) - .apply { fillGalleryList(galleryInfoList, url) } + return statement(url, url, EhUrl.origin) { + formBody { + append("ddact", catStr) + gidArray.forEach { append("modifygids[]", it.toString()) } + } + }.parseByteBuffer(FavoritesParser::parse).apply { fillGalleryList(galleryInfoList, url) } } suspend fun getGalleryPageApi( @@ -391,7 +421,7 @@ object EhEngine { previousPToken: String?, ): GalleryPageParser.Result { val referer = if (index > 0 && previousPToken != null) EhUrl.getPageUrl(gid, index - 1, previousPToken) else null - return ehRequest(EhUrl.apiUrl, referer, EhUrl.origin) { + return statement(EhUrl.apiUrl, referer, EhUrl.origin) { jsonBody { put("method", "showpage") put("gid", gid) @@ -399,7 +429,7 @@ object EhEngine { put("imgkey", pToken) put("showkey", showKey) } - }.executeAndParsingWith { GalleryPageParser.parse(filterNot { it == '\\' }) } + }.parseString { GalleryPageParser.parse(filterNot { it == '\\' }) } } suspend fun rateGallery( @@ -408,7 +438,7 @@ object EhEngine { gid: Long, token: String?, rating: Float, - ): RateGalleryResult = ehRequest(EhUrl.apiUrl, EhUrl.getGalleryDetailUrl(gid, token), EhUrl.origin) { + ): RateGalleryResult = statement(EhUrl.apiUrl, EhUrl.getGalleryDetailUrl(gid, token), EhUrl.origin) { jsonBody { put("method", "rategallery") put("apiuid", apiUid) @@ -417,11 +447,11 @@ object EhEngine { put("token", token) put("rating", ceil((rating * 2).toDouble()).toInt()) } - }.executeAndParsingWith(String::parseAs) + }.parseString(String::parseAs) suspend fun fillGalleryListByApi(galleryInfoList: List, referer: String) = galleryInfoList.chunked(MAX_REQUEST_SIZE).parMap(concurrency = Settings.multiThreadDownload) { - ehRequest(EhUrl.apiUrl, referer, EhUrl.origin) { + statement(EhUrl.apiUrl, referer, EhUrl.origin) { jsonBody { put("method", "gdata") array("gidlist") { @@ -434,7 +464,7 @@ object EhEngine { } put("namespace", 1) } - }.executeAndParsingWith { GalleryApiParser.parse(this, it) } + }.parseString { GalleryApiParser.parse(this, it) } } suspend fun voteComment( @@ -444,7 +474,7 @@ object EhEngine { token: String?, commentId: Long, commentVote: Int, - ): VoteCommentResult = ehRequest(EhUrl.apiUrl, EhUrl.referer, EhUrl.origin) { + ): VoteCommentResult = statement(EhUrl.apiUrl, EhUrl.referer, EhUrl.origin) { jsonBody { put("method", "votecomment") put("apiuid", apiUid) @@ -454,7 +484,7 @@ object EhEngine { put("comment_id", commentId) put("comment_vote", commentVote) } - }.executeAndParsingWith(String::parseAs) + }.parseString(String::parseAs) suspend fun voteTag( apiUid: Long, @@ -463,7 +493,7 @@ object EhEngine { token: String?, tags: String?, vote: Int, - ) = ehRequest(EhUrl.apiUrl, EhUrl.referer, EhUrl.origin) { + ) = statement(EhUrl.apiUrl, EhUrl.referer, EhUrl.origin) { jsonBody { put("method", "taggallery") put("apiuid", apiUid) @@ -473,9 +503,9 @@ object EhEngine { put("tags", tags) put("vote", vote) } - }.executeAndParsingWith(VoteTagParser::parse) + }.parseString(VoteTagParser::parse) - suspend fun getGalleryToken(gid: Long, gtoken: String?, page: Int) = ehRequest(EhUrl.apiUrl, EhUrl.referer, EhUrl.origin) { + suspend fun getGalleryToken(gid: Long, gtoken: String?, page: Int) = statement(EhUrl.apiUrl, EhUrl.referer, EhUrl.origin) { jsonBody { put("method", "gtoken") array("pagelist") { @@ -486,7 +516,7 @@ object EhEngine { } } } - }.executeAndParsingWith(GalleryTokenApiParser::parse) + }.parseString(GalleryTokenApiParser::parse) /** * @param image Must be jpeg @@ -506,7 +536,6 @@ object EhEngine { ).apply { fillGalleryList(galleryInfoList, EhUrl.imageSearchUrl) } private suspend fun fillGalleryList(list: MutableList, url: String, filter: Boolean = false) { - // Filter title and uploader if (filter) list.removeAllSuspend { EhFilter.filterTitle(it) || EhFilter.filterUploader(it) } var hasTags = false @@ -526,11 +555,16 @@ object EhEngine { val needApi = filter && EhFilter.needTags() && !hasTags || Settings.showGalleryPages && !hasPages || hasRated if (needApi) fillGalleryListByApi(list, url) - // Filter tag, thumbnail mode need filter uploader again - if (filter) list.removeAllSuspend { EhFilter.filterUploader(it) || EhFilter.filterTag(it) || EhFilter.filterTagNamespace(it) } + if (filter) { + list.removeAllSuspend { + EhFilter.filterUploader(it) || EhFilter.filterTag(it) || EhFilter.filterTagNamespace(it) + } + } } suspend fun addFavorites(galleryList: List>, dstCat: Int) { - galleryList.parMap(concurrency = Settings.multiThreadDownload) { (gid, token) -> modifyFavorites(gid, token, dstCat) } + galleryList.parMap(concurrency = Settings.multiThreadDownload) { (gid, token) -> + modifyFavorites(gid, token, dstCat) + } } } diff --git a/app/src/main/kotlin/com/hippo/ehviewer/client/EhRequestBuilder.kt b/app/src/main/kotlin/com/hippo/ehviewer/client/EhRequestBuilder.kt index b797efd23..025acdbaa 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/client/EhRequestBuilder.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/client/EhRequestBuilder.kt @@ -15,31 +15,31 @@ */ package com.hippo.ehviewer.client -import android.util.Log -import com.hippo.ehviewer.EhApplication.Companion.baseOkHttpClient +import com.hippo.ehviewer.EhApplication.Companion.ktorClient import com.hippo.ehviewer.EhApplication.Companion.noRedirectOkHttpClient import com.hippo.ehviewer.EhApplication.Companion.okHttpClient import com.hippo.ehviewer.Settings +import io.ktor.client.request.HttpRequestBuilder +import io.ktor.client.request.forms.FormDataContent +import io.ktor.client.request.header +import io.ktor.client.request.prepareRequest +import io.ktor.client.request.setBody +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpMethod +import io.ktor.http.ParametersBuilder +import io.ktor.http.content.TextContent +import io.ktor.http.userAgent import kotlin.contracts.InvocationKind import kotlin.contracts.contract -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.launch -import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArrayBuilder import kotlinx.serialization.json.JsonObjectBuilder import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.buildJsonObject import kotlinx.serialization.json.okio.decodeFromBufferedSource -import okhttp3.Call -import okhttp3.FormBody -import okhttp3.MediaType -import okhttp3.MediaType.Companion.toMediaType import okhttp3.MultipartBody import okhttp3.Request -import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response import okhttp3.executeAsync import okio.BufferedSource @@ -52,69 +52,17 @@ inline fun ehRequest(url: String, referer: String? = null, origin: String? = nul origin?.let { addHeader("Origin", it) } }.apply(builder).build() -inline fun formBody(builder: FormBody.Builder.() -> Unit) = FormBody.Builder().apply(builder).build() -inline fun Request.Builder.formBody(builder: FormBody.Builder.() -> Unit) = post(FormBody.Builder().apply(builder).build()) - inline fun multipartBody(builder: MultipartBody.Builder.() -> Unit) = MultipartBody.Builder().apply(builder).build() -val MEDIA_TYPE_JSON: MediaType = "application/json; charset=utf-8".toMediaType() - inline fun JsonObjectBuilder.array(name: String, builder: JsonArrayBuilder.() -> Unit) = put(name, buildJsonArray(builder)) -inline fun Request.Builder.jsonBody(builder: JsonObjectBuilder.() -> Unit) = post(buildJsonObject(builder).toString().toRequestBody(MEDIA_TYPE_JSON)) - -const val TAG_CALL = "CancellableCallReadingScope" - -// This scoped response reading scope suspend function aimed at cancelling the call immediately once coroutine is cancelled. -// i.e., bind a call with a suspended coroutine, and launch another coroutine to execute reading -// Once origin coroutine is cancelled, then call cancelled, socket is closed, the reading coroutine will encounter [IOException] -// Since Okio is not suspendable/cancellable, also okhttp, [executeAsync] only make [Request -> Response] cancellable, reading a Response is still not cancellable -// Considering interrupting is not safe and performant, See https://github.com/Kotlin/kotlinx.coroutines/issues/3551#issuecomment-1353245978, we have such [usingCancellable] -// TODO: Remove logging after this function stable -suspend inline fun Call.usingCancellable(crossinline block: suspend Response.() -> R): R = executeAsync().use { - val call = this - Log.d(TAG_CALL, "Got response of call$call") - coroutineScope { - suspendCancellableCoroutine { cont -> - launch { - val r = runCatching { - Log.d(TAG_CALL, "Reading response of call$call") - block(it) - } - if (!isCanceled() && cont.isActive) { - r.exceptionOrNull()?.let { - Log.e(TAG_CALL, "Reading response of call$call failed!") - cont.resumeWithException(it) - } - r.getOrNull()?.let { - Log.d(TAG_CALL, "Reading response of call$call succeed!") - cont.resume(it) - } - } else if (isCanceled() && cont.isActive) { - Log.e(TAG_CALL, "call$call cancelled but coroutine is Active!") - } else if (!isCanceled() && !cont.isActive) { - Log.e(TAG_CALL, "call$call not cancelled but coroutine is Dead!") - } else { - Log.d(TAG_CALL, "Reading response of call$call cancelled") - } - } - cont.invokeOnCancellation { - Log.d(TAG_CALL, "Cancelling call$call") - cancel() - } - } - }.apply { - Log.d(TAG_CALL, "Reading response of call$call finished!") - } -} - suspend inline fun Request.execute(block: Response.() -> R): R { contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) } return okHttpClient.newCall(this).executeAsync().use(block) } -suspend inline fun Request.executeNonCache(crossinline block: suspend Response.() -> R) = baseOkHttpClient.newCall(this).usingCancellable(block) + suspend inline fun Request.executeNoRedirect(block: Response.() -> R) = noRedirectOkHttpClient.newCall(this).executeAsync().use(block) suspend inline fun Request.executeAndParseAs() = execute { parseAs() } @@ -128,3 +76,30 @@ val json = Json { ignoreUnknownKeys = true } const val CHROME_USER_AGENT = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36" const val CHROME_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" const val CHROME_ACCEPT_LANGUAGE = "en-US,en;q=0.5" + +suspend inline fun statement( + url: String, + referer: String? = null, + origin: String? = null, + builder: HttpRequestBuilder.() -> Unit = {}, +) = ktorClient.prepareRequest(url) { + method = HttpMethod.Get + header("Referer", referer) + header("Origin", origin) + userAgent(Settings.userAgent) + header(HttpHeaders.Accept, CHROME_ACCEPT) + header(HttpHeaders.AcceptLanguage, CHROME_ACCEPT_LANGUAGE) + apply(builder) +} + +fun HttpRequestBuilder.formBody(builder: ParametersBuilder.() -> Unit) { + method = HttpMethod.Post + val parameters = ParametersBuilder().apply(builder).build() + setBody(FormDataContent(parameters)) +} + +fun HttpRequestBuilder.jsonBody(builder: JsonObjectBuilder.() -> Unit) { + method = HttpMethod.Post + val json = buildJsonObject(builder).toString() + setBody(TextContent(text = json, contentType = ContentType.Application.Json)) +} diff --git a/app/src/main/kotlin/com/hippo/ehviewer/ktbuilder/Okhttp.kt b/app/src/main/kotlin/com/hippo/ehviewer/ktbuilder/Okhttp.kt index 68e0e3bc1..ac8d46393 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/ktbuilder/Okhttp.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/ktbuilder/Okhttp.kt @@ -1,13 +1,10 @@ package com.hippo.ehviewer.ktbuilder -import com.chuckerteam.chucker.api.ChuckerInterceptor import okhttp3.Cache import okhttp3.OkHttpClient import okio.FileSystem import okio.Path -import splitties.init.appCtx inline fun httpClient(builder: OkHttpClient.Builder.() -> Unit): OkHttpClient = OkHttpClient.Builder().apply(builder).build() inline fun httpClient(client: OkHttpClient, builder: OkHttpClient.Builder.() -> Unit): OkHttpClient = client.newBuilder().apply(builder).build() fun OkHttpClient.Builder.cache(directory: Path, maxSize: Long, fileSystem: FileSystem = FileSystem.SYSTEM) = cache(Cache(directory, maxSize, fileSystem)) -inline fun OkHttpClient.Builder.chunker(builder: ChuckerInterceptor.Builder.() -> Unit) = addInterceptor(ChuckerInterceptor.Builder(appCtx).apply(builder).build()) diff --git a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt index fc2ce48b9..7fe9a7592 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderDen.kt @@ -21,9 +21,8 @@ import com.hippo.ehviewer.EhApplication.Companion.imageCache as sCache import com.hippo.ehviewer.EhDB import com.hippo.ehviewer.client.EhUtils.getSuitableTitle import com.hippo.ehviewer.client.data.GalleryInfo -import com.hippo.ehviewer.client.ehRequest -import com.hippo.ehviewer.client.executeNonCache import com.hippo.ehviewer.client.getImageKey +import com.hippo.ehviewer.client.statement import com.hippo.ehviewer.coil.read import com.hippo.ehviewer.coil.suspendEdit import com.hippo.ehviewer.cronet.copyToChannel @@ -39,13 +38,15 @@ import com.hippo.ehviewer.util.isCronetSupported import com.hippo.ehviewer.util.sendTo import com.hippo.unifile.UniFile import com.hippo.unifile.openOutputStream +import io.ktor.client.plugins.onDownload +import io.ktor.client.statement.HttpResponse +import io.ktor.client.statement.bodyAsChannel +import io.ktor.http.contentType +import io.ktor.http.isSuccess +import io.ktor.utils.io.jvm.nio.copyTo import java.util.Locale import kotlin.io.path.readText import okhttp3.MediaType.Companion.toMediaType -import okhttp3.Response -import okio.Buffer -import okio.ForwardingSink -import okio.sink class SpiderDen(mGalleryInfo: GalleryInfo) { private val mGid = mGalleryInfo.gid @@ -141,7 +142,7 @@ class SpiderDen(mGalleryInfo: GalleryInfo) { val headers = info.getHeadersMap() val type = headers["Content-Type"]?.first()?.toMediaType()?.subtype ?: "jpg" val length = headers["Content-Length"]!!.first().toLong() - saveResponseMeta(index, type, length) { file -> + saveResponseMeta(index, type) { file -> file.openOutputStream().use { os -> val chan = os.channel chan.truncate(0) @@ -152,17 +153,23 @@ class SpiderDen(mGalleryInfo: GalleryInfo) { } } } else { - ehRequest(url, referer).executeNonCache { - if (code >= 400) { - false + statement(url, referer) { + var prev = 0L + onDownload { done, total -> + notifyProgress(total!!, done, (done - prev).toInt()) + prev = done + } + }.execute { + if (it.status.isSuccess()) { + saveFromHttpResponse(index, it) } else { - saveFromHttpResponse(index, this, notifyProgress) + false } } } } - private suspend fun saveResponseMeta(index: Int, ext: String, length: Long, fops: suspend (UniFile) -> Unit): Boolean { + private suspend fun saveResponseMeta(index: Int, ext: String, fops: suspend (UniFile) -> Unit): Boolean { findDownloadFileForIndex(index, ext)?.run { fops(this) return true @@ -179,19 +186,14 @@ class SpiderDen(mGalleryInfo: GalleryInfo) { return false } - private suspend fun saveFromHttpResponse(index: Int, response: Response, notifyProgress: (Long, Long, Int) -> Unit): Boolean { - val contentType = response.body.contentType() - val extension = contentType?.subtype ?: "jpg" - val length = response.body.contentLength() - return saveResponseMeta(index, extension, length) { outFile -> - object : ForwardingSink(outFile.openOutputStream().also { it.channel.truncate(0) }.sink()) { - var totalBytesRead = 0L - override fun write(source: Buffer, byteCount: Long) { - super.write(source, byteCount) - totalBytesRead += byteCount - notifyProgress(length, totalBytesRead, byteCount.toInt()) - } - }.use { sink -> response.body.source().use { source -> source.readAll(sink) } } + private suspend fun saveFromHttpResponse(index: Int, response: HttpResponse): Boolean { + val contentType = response.contentType() + val extension = contentType?.contentSubtype ?: "jpg" + return saveResponseMeta(index, extension) { outFile -> + outFile.openOutputStream().use { + val chan = it.channel.apply { truncate(0) } + response.bodyAsChannel().copyTo(chan) + } } } diff --git a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt index bd0fe2858..714f129fc 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/spider/SpiderQueen.kt @@ -29,14 +29,14 @@ import com.hippo.ehviewer.client.EhUrl.getGalleryDetailUrl import com.hippo.ehviewer.client.EhUrl.getGalleryMultiPageViewerUrl import com.hippo.ehviewer.client.EhUrl.referer import com.hippo.ehviewer.client.data.GalleryInfo -import com.hippo.ehviewer.client.ehRequest import com.hippo.ehviewer.client.exception.QuotaExceededException -import com.hippo.ehviewer.client.execute +import com.hippo.ehviewer.client.parseString import com.hippo.ehviewer.client.parser.GalleryDetailParser.parsePages import com.hippo.ehviewer.client.parser.GalleryDetailParser.parsePreviewList import com.hippo.ehviewer.client.parser.GalleryDetailParser.parsePreviewPages import com.hippo.ehviewer.client.parser.GalleryMultiPageViewerPTokenParser import com.hippo.ehviewer.client.parser.GalleryPageUrlParser +import com.hippo.ehviewer.client.statement import com.hippo.ehviewer.image.Image import com.hippo.ehviewer.util.ExceptionUtils import com.hippo.unifile.UniFile @@ -363,15 +363,14 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS private suspend fun readSpiderInfoFromInternet(): SpiderInfo? { return runSuspendCatching { - ehRequest( + statement( getGalleryDetailUrl(galleryInfo.gid, galleryInfo.token, 0, false), referer, - ).execute { - val bodyStr = body.string() - val pages = parsePages(bodyStr) + ).parseString { + val pages = parsePages(this) val spiderInfo = SpiderInfo(galleryInfo.gid, pages) spiderInfo.token = galleryInfo.token - readPreviews(bodyStr, 0, spiderInfo) + readPreviews(this, 0, spiderInfo) spiderInfo } }.onFailure { @@ -386,9 +385,8 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS galleryInfo.token!!, ) return try { - ehRequest(url, referer).execute { - val bodyStr = body.string() - GalleryMultiPageViewerPTokenParser.parse(bodyStr).forEachIndexed { index, s -> + statement(url, referer).parseString { + GalleryMultiPageViewerPTokenParser.parse(this).forEachIndexed { index, s -> spiderInfo.pTokenMap[index] = s } spiderInfo.pTokenMap[index] @@ -419,8 +417,8 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS false, ) return try { - ehRequest(url, referer).execute { - readPreviews(body.string(), previewIndex, spiderInfo) + statement(url, referer).parseString { + readPreviews(this, previewIndex, spiderInfo) spiderInfo.pTokenMap[index] } } catch (e: Throwable) { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b4fb909f2..c84756159 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,7 +17,6 @@ androidx-recyclerview = "1.3.2" androidx-room = "2.6.1" androidx-work = "2.9.0" arrow = "1.2.1" -chucker = "4.0.0" coil = "2.5.0" compose-bom = "2023.12.00-alpha03" compose-compiler = "1.5.5-dev-k2.0.0-Beta1-06b8ae672a4" @@ -30,7 +29,7 @@ kotlinx-datetime = "0.5.0" kotlinx-serialization = "1.6.2" ksp = "2.0.0-Beta1-1.0.15" ktlint = "1.0.1" -ktor = "2.3.6" +ktor = "3.0.0-beta-1" okhttp3 = "5.0.0-alpha.11" okio = "3.6.0" splitties = "3.0.0" @@ -69,9 +68,6 @@ androidx-work-runtime = { module = "androidx.work:work-runtime-ktx", version.ref arrow-fx-coroutines = { module = "io.arrow-kt:arrow-fx-coroutines" } arrow-stack = { module = "io.arrow-kt:arrow-stack", version.ref = "arrow" } -chucker = { module = "com.github.chuckerteam.chucker:library", version.ref = "chucker" } -chucker-nop = { module = "com.github.chuckerteam.chucker:library-no-op", version.ref = "chucker" } - coil-bom = { module = "io.coil-kt:coil-bom", version.ref = "coil" } coil-compose = { module = "io.coil-kt:coil-compose" } coil-gif = { module = "io.coil-kt:coil-gif" } @@ -107,8 +103,7 @@ kotlinx-serialization-json-okio = { module = "org.jetbrains.kotlinx:kotlinx-seri # For renovate ktlint = { module = "com.pinterest:ktlint", version.ref = "ktlint" } -ktor-io-jvm = { module = "io.ktor:ktor-io-jvm", version.ref = "ktor" } -ktor-utils-jvm = { module = "io.ktor:ktor-utils-jvm", version.ref = "ktor" } +ktor-client-okhttp = { module = "io.ktor:ktor-client-okhttp", version.ref = "ktor" } material = { module = "com.google.android.material:material", version.ref = "google-android-material" } @@ -136,7 +131,7 @@ coil = ["coil-compose", "coil-gif"] compose = ["compose-foundation", "compose-material-icons-extended", "compose-material3", "compose-material3-windowsizeclass", "compose-ui-util"] cronet = ["cronet-api", "cronet-gms"] kotlinx-serialization = ["kotlinx-serialization-cbor", "kotlinx-serialization-json", "kotlinx-serialization-json-okio"] -ktor = ["ktor-io-jvm", "ktor-utils-jvm"] +ktor = ["ktor-client-okhttp"] okhttp = ["okhttp-android", "okhttp-coroutines"] splitties = ["splitties-appctx", "splitties-arch-room", "splitties-preferences", "splitties-systemservices"]