From c5c11703831735e52f3701c1d753a5ae82a3ec3a Mon Sep 17 00:00:00 2001 From: tarsin Date: Wed, 24 May 2023 20:12:24 +0800 Subject: [PATCH] Uniform html parse behaviour --- .../com/hippo/ehviewer/client/EhEngine.kt | 95 ++++++++++--------- .../ehviewer/client/parser/ArchiveParser.kt | 5 +- .../ehviewer/client/parser/EventPaneParser.kt | 6 +- .../ehviewer/client/parser/FavoritesParser.kt | 7 +- .../client/parser/GalleryDetailParser.kt | 5 +- .../client/parser/GalleryListParser.kt | 8 +- .../ehviewer/client/parser/HomeParser.kt | 12 +-- .../ehviewer/client/parser/ProfileParser.kt | 7 +- .../ehviewer/client/parser/TorrentParser.kt | 5 +- 9 files changed, 72 insertions(+), 78 deletions(-) diff --git a/app/src/main/java/com/hippo/ehviewer/client/EhEngine.kt b/app/src/main/java/com/hippo/ehviewer/client/EhEngine.kt index d31c2037c..f71839bcb 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/EhEngine.kt +++ b/app/src/main/java/com/hippo/ehviewer/client/EhEngine.kt @@ -58,6 +58,7 @@ import okhttp3.RequestBody.Companion.asRequestBody import okio.ByteString.Companion.decodeHex import org.json.JSONArray import org.jsoup.Jsoup +import org.jsoup.nodes.Document import org.jsoup.nodes.Element import org.jsoup.nodes.TextNode import java.io.File @@ -121,12 +122,14 @@ private fun rethrowExactly(code: Int, headers: Headers, body: String, e: Throwab throw e } -private const val EMAIL_PROTECTED = "email protected" +private const val EMAIL_PROTECTED = "[email protected]" private const val RAW_EMAIL_PROTECTED = "[email protected]" -private fun resolveEmailProtected(origin: String): String { - if (RAW_EMAIL_PROTECTED !in origin) return origin - val resolvedEmailList = mutableListOf() +private fun resolveEmailProtected(origin: String): Document { + // Temporary workaround, see https://github.com/jhy/jsoup/issues/1850 + val doc = Jsoup.parse(origin.replace("del>", "s>")) + + if (RAW_EMAIL_PROTECTED !in origin) return doc fun Element.decodeProtectedEmail() { if (EMAIL_PROTECTED in text()) { if (EMAIL_PROTECTED in ownText()) { @@ -137,7 +140,8 @@ private fun resolveEmailProtected(origin: String): String { val encoded = attr("data-cfemail").chunked(2).toMutableList() val k = encoded.removeFirst().decodeHex()[0] val email = encoded.map { it.decodeHex()[0] xor k }.toByteArray().decodeToString() - resolvedEmailList.add(email) + val text = node.text() + node.text(text.replace(EMAIL_PROTECTED, email)) } } }, @@ -148,16 +152,18 @@ private fun resolveEmailProtected(origin: String): String { } } } - Jsoup.parse(origin).decodeProtectedEmail() - var ret = origin - resolvedEmailList.forEach { ret = ret.replaceFirst(RAW_EMAIL_PROTECTED, it) } - return ret + return doc.apply { decodeProtectedEmail() } } -private suspend inline fun Request.executeAndParsingWith(block: String.() -> T): T { +private suspend inline fun Request.executeParsingDocument(block: (String, Document) -> T) = executeParsing { + val doc = resolveEmailProtected(this) + block(this, doc) +} + +private suspend inline fun Request.executeParsing(block: String.() -> T): T { Log.d(TAG, url.toString()) return execute { - val body = resolveEmailProtected(body.string()) + val body = body.string() try { block(body) } catch (e: Exception) { @@ -169,35 +175,35 @@ private suspend inline fun Request.executeAndParsingWith(block: String.() -> object EhEngine { suspend fun getTorrentList(url: String, gid: Long, token: String?): List { val referer = EhUrl.getGalleryDetailUrl(gid, token) - return ehRequest(url, referer).executeAndParsingWith(TorrentParser::parse) + return ehRequest(url, referer).executeParsingDocument { _, d -> TorrentParser.parse(d) } } suspend fun getArchiveList(url: String, gid: Long, token: String?) = ehRequest(url, EhUrl.getGalleryDetailUrl(gid, token)) - .executeAndParsingWith(ArchiveParser::parse) - .apply { funds = funds ?: ehRequest(EhUrl.URL_FUNDS).executeAndParsingWith(HomeParser::parseFunds) } + .executeParsingDocument(ArchiveParser::parse) + .apply { funds = funds ?: ehRequest(EhUrl.URL_FUNDS).executeParsingDocument { s, _ -> HomeParser.parseFunds(s) } } suspend fun getImageLimits() = parZip( - { ehRequest(EhUrl.URL_HOME).executeAndParsingWith(HomeParser::parse) }, - { ehRequest(EhUrl.URL_FUNDS).executeAndParsingWith(HomeParser::parseFunds) }, + { ehRequest(EhUrl.URL_HOME).executeParsingDocument { _, d -> HomeParser.parse(d) } }, + { ehRequest(EhUrl.URL_FUNDS).executeParsingDocument { s, _ -> HomeParser.parseFunds(s) } }, { limits, funds -> HomeParser.Result(limits, funds) }, ) suspend fun getNews(parse: Boolean) = ehRequest(EhUrl.URL_NEWS, EhUrl.REFERER_E) - .executeAndParsingWith { if (parse) EventPaneParser.parse(this) else null } + .executeParsingDocument { _, d -> if (parse) EventPaneParser.parse(d) else null } suspend fun getProfile(): ProfileParser.Result { - val url = ehRequest(EhUrl.URL_FORUMS).executeAndParsingWith(ForumsParser::parse) - return ehRequest(url, EhUrl.URL_FORUMS).executeAndParsingWith(ProfileParser::parse) + val url = ehRequest(EhUrl.URL_FORUMS).executeParsing(ForumsParser::parse) + return ehRequest(url, EhUrl.URL_FORUMS).executeParsingDocument { _, d -> ProfileParser.parse(d) } } suspend fun getUConfig(url: String = EhUrl.uConfigUrl) { runSuspendCatching { - ehRequest(url).executeAndParsingWith { check(contains(U_CONFIG_TEXT)) { "U_CONFIG_TEXT not found!" } } + ehRequest(url).executeParsing { check(U_CONFIG_TEXT in this) { "U_CONFIG_TEXT not found!" } } }.onFailure { // It may get redirected when accessing ex for the first time if (EhUtils.isExHentai) { it.printStackTrace() - ehRequest(url).executeAndParsingWith { check(contains(U_CONFIG_TEXT)) { "U_CONFIG_TEXT not found!" } } + ehRequest(url).executeParsing { check(U_CONFIG_TEXT in this) { "U_CONFIG_TEXT not found!" } } } else { throw it } @@ -210,27 +216,27 @@ object EhEngine { token: String?, ): GalleryPageParser.Result { val referer = EhUrl.getGalleryDetailUrl(gid, token) - return ehRequest(url, referer).executeAndParsingWith(GalleryPageParser::parse) + return ehRequest(url, referer).executeParsing(GalleryPageParser::parse) } suspend fun getGalleryList(url: String) = ehRequest(url, EhUrl.referer) - .executeAndParsingWith(GalleryListParser::parse) + .executeParsingDocument(GalleryListParser::parse) .apply { fillGalleryList(galleryInfoList, url, true) } - suspend fun getGalleryDetail(url: String) = ehRequest(url, EhUrl.referer).executeAndParsingWith { - EventPaneParser.parse(this)?.let { + suspend fun getGalleryDetail(url: String) = ehRequest(url, EhUrl.referer).executeParsingDocument { s, d -> + EventPaneParser.parse(d)?.let { Settings.lastDawnDay = today showEventNotification(it) } - GalleryDetailParser.parse(this) + GalleryDetailParser.parse(s, d) } - suspend fun getPreviewList(url: String) = ehRequest(url, EhUrl.referer).executeAndParsingWith { + suspend fun getPreviewList(url: String) = ehRequest(url, EhUrl.referer).executeParsing { GalleryDetailParser.parsePreviewList(this) to GalleryDetailParser.parsePreviewPages(this) } suspend fun getFavorites(url: String) = ehRequest(url, EhUrl.referer) - .executeAndParsingWith(FavoritesParser::parse) + .executeParsingDocument(FavoritesParser::parse) .apply { fillGalleryList(galleryInfoList, url, false) } suspend fun signIn(username: String, password: String): String { @@ -246,7 +252,7 @@ object EhEngine { add("PassWord", password) add("CookieDate", "1") } - }.executeAndParsingWith(SignInParser::parse) + }.executeParsing(SignInParser::parse) } suspend fun commentGallery(url: String, comment: String, id: String?) = ehRequest(url, url, EhUrl.origin) { @@ -258,13 +264,12 @@ object EhEngine { add("edit_comment", id) } } - }.executeAndParsingWith { - val document = Jsoup.parse(this) - val elements = document.select("#chd + p") + }.executeParsingDocument { _, d -> + val elements = d.select("#chd + p") if (elements.size > 0) { throw EhException(elements[0].text()) } - GalleryDetailParser.parseComments(document) + GalleryDetailParser.parseComments(d) } /** @@ -291,7 +296,7 @@ object EhEngine { add("submit", "Apply Changes") add("update", "1") } - }.executeAndParsingWith { } + }.executeParsing { } } suspend fun downloadArchive( @@ -323,12 +328,12 @@ object EhEngine { } } } - var result = request.executeAndParsingWith(ArchiveParser::parseArchiveUrl) + var result = request.executeParsing(ArchiveParser::parseArchiveUrl) if (!isHAtH) { if (result == null) { // Wait for the server to prepare archives delay(1000) - result = request.executeAndParsingWith(ArchiveParser::parseArchiveUrl) + result = request.executeParsing(ArchiveParser::parseArchiveUrl) if (result == null) { throw EhException("Archive unavailable") } @@ -344,7 +349,7 @@ object EhEngine { add("act", "limits") add("reset", "Reset Limit") } - }.executeAndParsingWith(HomeParser::parseResetLimits) + }.executeParsingDocument(HomeParser::parseResetLimits) } suspend fun modifyFavorites(url: String, gidArray: LongArray, dstCat: Int): FavoritesParser.Result { @@ -359,7 +364,7 @@ object EhEngine { gidArray.forEach { add("modifygids[]", it.toString()) } add("apply", "Apply") } - }.executeAndParsingWith(FavoritesParser::parse).apply { fillGalleryList(galleryInfoList, url, false) } + }.executeParsingDocument(FavoritesParser::parse).apply { fillGalleryList(galleryInfoList, url, false) } } suspend fun getGalleryPageApi( @@ -378,7 +383,7 @@ object EhEngine { put("imgkey", pToken) put("showkey", showKey) } - }.executeAndParsingWith(GalleryPageApiParser::parse) + }.executeParsing(GalleryPageApiParser::parse) } suspend fun rateGallery(apiUid: Long, apiKey: String?, gid: Long, token: String?, rating: Float) = ehRequest(EhUrl.apiUrl, EhUrl.getGalleryDetailUrl(gid, token), EhUrl.origin) { @@ -390,7 +395,7 @@ object EhEngine { put("token", token) put("rating", ceil((rating * 2).toDouble()).toInt()) } - }.executeAndParsingWith(RateGalleryParser::parse) + }.executeParsing(RateGalleryParser::parse) suspend fun fillGalleryListByApi(galleryInfoList: List, referer: String) = galleryInfoList.chunked(MAX_REQUEST_SIZE).parMap { ehRequest(EhUrl.apiUrl, referer, EhUrl.origin) { @@ -399,7 +404,7 @@ object EhEngine { array("gidlist") { it.forEach { put(jsonArrayOf(it.gid, it.token)) } } put("namespace", 1) } - }.executeAndParsingWith { GalleryApiParser.parse(this, it) } + }.executeParsing { GalleryApiParser.parse(this, it) } } suspend fun voteComment( @@ -419,7 +424,7 @@ object EhEngine { put("comment_id", commentId) put("comment_vote", commentVote) } - }.executeAndParsingWith { VoteCommentParser.parse(this, commentVote) } + }.executeParsing { VoteCommentParser.parse(this, commentVote) } suspend fun voteTag( apiUid: Long, @@ -438,14 +443,14 @@ object EhEngine { put("tags", tags) put("vote", vote) } - }.executeAndParsingWith(VoteTagParser::parse) + }.executeParsing(VoteTagParser::parse) suspend fun getGalleryToken(gid: Long, gtoken: String?, page: Int) = ehRequest(EhUrl.apiUrl, EhUrl.referer, EhUrl.origin) { jsonBody { put("method", "gtoken") put("pagelist", JSONArray().put(JSONArray().put(gid).put(gtoken).put(page + 1))) } - }.executeAndParsingWith(GalleryTokenApiParser::parse) + }.executeParsing(GalleryTokenApiParser::parse) /** * @param image Must be jpeg @@ -458,7 +463,7 @@ object EhEngine { if (osc) addFormDataPart("fs_covers", "on") addFormDataPart("f_sfile", "File Search") } - }.executeAndParsingWith(GalleryListParser::parse).apply { fillGalleryList(galleryInfoList, EhUrl.imageSearchUrl, true) } + }.executeParsingDocument(GalleryListParser::parse).apply { fillGalleryList(galleryInfoList, EhUrl.imageSearchUrl, true) } private suspend fun fillGalleryList(list: MutableList, url: String, filter: Boolean) { // Filter title and uploader diff --git a/app/src/main/java/com/hippo/ehviewer/client/parser/ArchiveParser.kt b/app/src/main/java/com/hippo/ehviewer/client/parser/ArchiveParser.kt index f78b228b1..f19f73dc9 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/parser/ArchiveParser.kt +++ b/app/src/main/java/com/hippo/ehviewer/client/parser/ArchiveParser.kt @@ -17,6 +17,7 @@ package com.hippo.ehviewer.client.parser import com.hippo.ehviewer.client.exception.NoHAtHClientException import org.jsoup.Jsoup +import org.jsoup.nodes.Document object ArchiveParser { private val PATTERN_ARCHIVE_URL = @@ -30,11 +31,11 @@ object ArchiveParser { private val PATTERN_NEED_HATH_CLIENT = Regex("You must have a H@H client assigned to your account to use this feature\\.") - fun parse(body: String): Result { + fun parse(body: String, doc: Document): Result { val m = PATTERN_HATH_FORM.find(body)!! val paramOr = m.groupValues[1] val archiveList = ArrayList() - Jsoup.parse(body).select("#db>div>div").forEach { element -> + doc.select("#db>div>div").forEach { element -> if (element.childrenSize() > 0 && !element.attr("style").contains("color:#CCCCCC")) { runCatching { val res = element.selectFirst("form>input")!!.attr("value") diff --git a/app/src/main/java/com/hippo/ehviewer/client/parser/EventPaneParser.kt b/app/src/main/java/com/hippo/ehviewer/client/parser/EventPaneParser.kt index bd90cd89e..601b1ca86 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/parser/EventPaneParser.kt +++ b/app/src/main/java/com/hippo/ehviewer/client/parser/EventPaneParser.kt @@ -17,10 +17,10 @@ */ package com.hippo.ehviewer.client.parser -import org.jsoup.Jsoup +import org.jsoup.nodes.Document object EventPaneParser { - fun parse(body: String): String? { - return Jsoup.parse(body).getElementById("eventpane")?.html() + fun parse(doc: Document): String? { + return doc.getElementById("eventpane")?.html() } } diff --git a/app/src/main/java/com/hippo/ehviewer/client/parser/FavoritesParser.kt b/app/src/main/java/com/hippo/ehviewer/client/parser/FavoritesParser.kt index 74529b4d5..b0210b1d0 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/parser/FavoritesParser.kt +++ b/app/src/main/java/com/hippo/ehviewer/client/parser/FavoritesParser.kt @@ -21,16 +21,15 @@ import com.hippo.ehviewer.client.exception.EhException import com.hippo.ehviewer.client.exception.ParseException import com.hippo.ehviewer.util.ExceptionUtils import com.hippo.ehviewer.util.JsoupUtils -import org.jsoup.Jsoup +import org.jsoup.nodes.Document object FavoritesParser { - fun parse(body: String): Result { + fun parse(body: String, d: Document): Result { if (body.contains("This page requires you to log on.

")) { throw EhException(getString(R.string.need_sign_in)) } val catArray = arrayOfNulls(10) val countArray = IntArray(10) - val d = Jsoup.parse(body) runCatching { val ido = JsoupUtils.getElementByClass(d, "ido") val fps = ido!!.getElementsByClass("fp") @@ -46,7 +45,7 @@ object FavoritesParser { it.printStackTrace() throw ParseException("Parse favorites error", body) } - val result = GalleryListParser.parse(d, body) + val result = GalleryListParser.parse(body, d) return Result(catArray, countArray, result) } diff --git a/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.kt b/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.kt index d582f12a1..cf2552771 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.kt +++ b/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.kt @@ -40,7 +40,6 @@ import com.hippo.ehviewer.yorozuya.NumberUtils import com.hippo.ehviewer.yorozuya.StringUtils import com.hippo.ehviewer.yorozuya.trimAnd import com.hippo.ehviewer.yorozuya.unescapeXml -import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.nodes.Element import org.jsoup.nodes.Node @@ -83,7 +82,7 @@ object GalleryDetailParser { private const val PINING_STRING = "

This gallery is pining for the fjords.

" @Throws(EhException::class) - fun parse(body: String): GalleryDetail { + fun parse(body: String, document: Document): GalleryDetail { if (body.contains(OFFENSIVE_STRING)) { throw OffensiveException() } @@ -93,8 +92,6 @@ object GalleryDetailParser { // Error info PATTERN_ERROR.find(body)?.run { throw EhException(groupValues[1]) } - // Temporary workaround, see https://github.com/jhy/jsoup/issues/1850 - val document = Jsoup.parse(body.replace("del>", "s>")) val galleryDetail = GalleryDetail( tags = parseTagGroups(document), comments = parseComments(document), diff --git a/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryListParser.kt b/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryListParser.kt index 92d4bb0fd..6c3722cf8 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryListParser.kt +++ b/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryListParser.kt @@ -25,7 +25,6 @@ import com.hippo.ehviewer.client.getThumbKey import com.hippo.ehviewer.util.ExceptionUtils import com.hippo.ehviewer.util.JsoupUtils import com.hippo.ehviewer.yorozuya.NumberUtils -import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.nodes.Element import java.util.regex.Pattern @@ -261,12 +260,7 @@ object GalleryListParser { return gi } - fun parse(body: String): Result { - val d = Jsoup.parse(body) - return parse(d, body) - } - - fun parse(d: Document, body: String): Result { + fun parse(body: String, d: Document): Result { val result = Result() try { val prev = d.getElementById("uprev") diff --git a/app/src/main/java/com/hippo/ehviewer/client/parser/HomeParser.kt b/app/src/main/java/com/hippo/ehviewer/client/parser/HomeParser.kt index afbda63ad..4979874d4 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/parser/HomeParser.kt +++ b/app/src/main/java/com/hippo/ehviewer/client/parser/HomeParser.kt @@ -1,15 +1,15 @@ package com.hippo.ehviewer.client.parser import com.hippo.ehviewer.client.exception.ParseException -import org.jsoup.Jsoup +import org.jsoup.nodes.Document object HomeParser { private val PATTERN_FUNDS = Regex("Available: ([\\d,]+) Credits.*Available: ([\\d,]+) kGP", RegexOption.DOT_MATCHES_ALL) private const val RESET_SUCCEED = "Image limit was successfully reset." - fun parse(body: String): Limits { - Jsoup.parse(body).selectFirst("div.homebox")?.let { + fun parse(doc: Document): Limits { + doc.selectFirst("div.homebox")?.let { val es = it.select("p > strong") if (es.size == 3) { val current = ParserUtils.parseInt(es[0].text(), 0) @@ -18,14 +18,14 @@ object HomeParser { return Limits(current, maximum, resetCost) } } - throw ParseException("Parse image limits error", body) + throw ParseException("Parse image limits error", doc.html()) } - fun parseResetLimits(body: String): Limits? { + fun parseResetLimits(body: String, doc: Document): Limits? { return if (body.contains(RESET_SUCCEED)) { null } else { - parse(body) + parse(doc) } } diff --git a/app/src/main/java/com/hippo/ehviewer/client/parser/ProfileParser.kt b/app/src/main/java/com/hippo/ehviewer/client/parser/ProfileParser.kt index afbfc57e6..d135b80f5 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/parser/ProfileParser.kt +++ b/app/src/main/java/com/hippo/ehviewer/client/parser/ProfileParser.kt @@ -19,13 +19,12 @@ import android.util.Log import com.hippo.ehviewer.client.EhUrl import com.hippo.ehviewer.client.exception.ParseException import com.hippo.ehviewer.util.ExceptionUtils -import org.jsoup.Jsoup +import org.jsoup.nodes.Document object ProfileParser { private val TAG = ProfileParser::class.java.simpleName - fun parse(body: String): Result { + fun parse(d: Document): Result { return runCatching { - val d = Jsoup.parse(body) val profilename = d.getElementById("profilename") val displayName = profilename!!.child(0).text() val avatar = runCatching { @@ -46,7 +45,7 @@ object ProfileParser { Result(displayName, avatar) }.getOrElse { ExceptionUtils.throwIfFatal(it) - throw ParseException("Parse forums error", body) + throw ParseException("Parse forums error", d.html()) } } diff --git a/app/src/main/java/com/hippo/ehviewer/client/parser/TorrentParser.kt b/app/src/main/java/com/hippo/ehviewer/client/parser/TorrentParser.kt index 347063060..625a24de7 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/parser/TorrentParser.kt +++ b/app/src/main/java/com/hippo/ehviewer/client/parser/TorrentParser.kt @@ -15,16 +15,15 @@ */ package com.hippo.ehviewer.client.parser -import org.jsoup.Jsoup +import org.jsoup.nodes.Document import java.util.regex.Pattern object TorrentParser { private val PATTERN_TORRENT = Pattern.compile(" ([0-9-]+) [0-9:]+[\\s\\S]+ ([0-9.]+ [KMGT]B)[\\s\\S]+ ([0-9]+)[\\s\\S]+ ([0-9]+)[\\s\\S]+ ([0-9]+)[\\s\\S]+([^<]+)[\\s\\S]+onclick=\"document.location='([^\"]+)'[^<]+>([^<]+)") - fun parse(body: String): List { + fun parse(d: Document): List { val torrentList = ArrayList() - val d = Jsoup.parse(body) val es = d.select("form>div>table") for (e in es) { val m = PATTERN_TORRENT.matcher(e.html())