yenru0/EhViewer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Uniform html parse behaviour

Browse Source
This commit is contained in:
tarsin committed 2023-05-24 20:12:24 +08:00
1 parent 6ed3fb29d9
commit c5c1170383
9 files changed
+72 -78
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/src/main/java/com/hippo/ehviewer/client/EhEngine.kt
+50 -45
View File
@@ -58,6 +58,7 @@ import okhttp3.RequestBody.Companion.asRequestBody
import okio.ByteString.Companion.decodeHex
import org.json.JSONArray
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.nodes.TextNode
import java.io.File
@@ -121,12 +122,14 @@ private fun rethrowExactly(code: Int, headers: Headers, body: String, e: Throwab
throw e
}
private const val EMAIL_PROTECTED = "email protected"
private const val EMAIL_PROTECTED = "[email protected]"
private const val RAW_EMAIL_PROTECTED = "[email protected]"
private fun resolveEmailProtected(origin: String): String {
if (RAW_EMAIL_PROTECTED !in origin) return origin
val resolvedEmailList = mutableListOf<String>()
private fun resolveEmailProtected(origin: String): Document {
// Temporary workaround, see https://github.com/jhy/jsoup/issues/1850
val doc = Jsoup.parse(origin.replace("del>", "s>"))
if (RAW_EMAIL_PROTECTED !in origin) return doc
fun Element.decodeProtectedEmail() {
if (EMAIL_PROTECTED in text()) {
if (EMAIL_PROTECTED in ownText()) {
@@ -137,7 +140,8 @@ private fun resolveEmailProtected(origin: String): String {
val encoded = attr("data-cfemail").chunked(2).toMutableList()
val k = encoded.removeFirst().decodeHex()[0]
val email = encoded.map { it.decodeHex()[0] xor k }.toByteArray().decodeToString()
resolvedEmailList.add(email)
val text = node.text()
node.text(text.replace(EMAIL_PROTECTED, email))
}
}
},
@@ -148,16 +152,18 @@ private fun resolveEmailProtected(origin: String): String {
}
}
}
Jsoup.parse(origin).decodeProtectedEmail()
var ret = origin
resolvedEmailList.forEach { ret = ret.replaceFirst(RAW_EMAIL_PROTECTED, it) }
return ret
return doc.apply { decodeProtectedEmail() }
}
private suspend inline fun <T> Request.executeAndParsingWith(block: String.() -> T): T {
private suspend inline fun <T> Request.executeParsingDocument(block: (String, Document) -> T) = executeParsing {
val doc = resolveEmailProtected(this)
block(this, doc)
}
private suspend inline fun <T> Request.executeParsing(block: String.() -> T): T {
Log.d(TAG, url.toString())
return execute {
val body = resolveEmailProtected(body.string())
val body = body.string()
try {
block(body)
} catch (e: Exception) {
@@ -169,35 +175,35 @@ private suspend inline fun <T> Request.executeAndParsingWith(block: String.() ->
object EhEngine {
suspend fun getTorrentList(url: String, gid: Long, token: String?): List<TorrentParser.Result> {
val referer = EhUrl.getGalleryDetailUrl(gid, token)
return ehRequest(url, referer).executeAndParsingWith(TorrentParser::parse)
return ehRequest(url, referer).executeParsingDocument { _, d -> TorrentParser.parse(d) }
}
suspend fun getArchiveList(url: String, gid: Long, token: String?) = ehRequest(url, EhUrl.getGalleryDetailUrl(gid, token))
.executeAndParsingWith(ArchiveParser::parse)
.apply { funds = funds ?: ehRequest(EhUrl.URL_FUNDS).executeAndParsingWith(HomeParser::parseFunds) }
.executeParsingDocument(ArchiveParser::parse)
.apply { funds = funds ?: ehRequest(EhUrl.URL_FUNDS).executeParsingDocument { s, _ -> HomeParser.parseFunds(s) } }
suspend fun getImageLimits() = parZip(
{ ehRequest(EhUrl.URL_HOME).executeAndParsingWith(HomeParser::parse) },
{ ehRequest(EhUrl.URL_FUNDS).executeAndParsingWith(HomeParser::parseFunds) },
{ ehRequest(EhUrl.URL_HOME).executeParsingDocument { _, d -> HomeParser.parse(d) } },
{ ehRequest(EhUrl.URL_FUNDS).executeParsingDocument { s, _ -> HomeParser.parseFunds(s) } },
{ limits, funds -> HomeParser.Result(limits, funds) },
)
suspend fun getNews(parse: Boolean) = ehRequest(EhUrl.URL_NEWS, EhUrl.REFERER_E)
.executeAndParsingWith { if (parse) EventPaneParser.parse(this) else null }
.executeParsingDocument { _, d -> if (parse) EventPaneParser.parse(d) else null }
suspend fun getProfile(): ProfileParser.Result {
val url = ehRequest(EhUrl.URL_FORUMS).executeAndParsingWith(ForumsParser::parse)
return ehRequest(url, EhUrl.URL_FORUMS).executeAndParsingWith(ProfileParser::parse)
val url = ehRequest(EhUrl.URL_FORUMS).executeParsing(ForumsParser::parse)
return ehRequest(url, EhUrl.URL_FORUMS).executeParsingDocument { _, d -> ProfileParser.parse(d) }
}
suspend fun getUConfig(url: String = EhUrl.uConfigUrl) {
runSuspendCatching {
ehRequest(url).executeAndParsingWith { check(contains(U_CONFIG_TEXT)) { "U_CONFIG_TEXT not found!" } }
ehRequest(url).executeParsing { check(U_CONFIG_TEXT in this) { "U_CONFIG_TEXT not found!" } }
}.onFailure {
// It may get redirected when accessing ex for the first time
if (EhUtils.isExHentai) {
it.printStackTrace()
ehRequest(url).executeAndParsingWith { check(contains(U_CONFIG_TEXT)) { "U_CONFIG_TEXT not found!" } }
ehRequest(url).executeParsing { check(U_CONFIG_TEXT in this) { "U_CONFIG_TEXT not found!" } }
} else {
throw it
}
@@ -210,27 +216,27 @@ object EhEngine {
token: String?,
): GalleryPageParser.Result {
val referer = EhUrl.getGalleryDetailUrl(gid, token)
return ehRequest(url, referer).executeAndParsingWith(GalleryPageParser::parse)
return ehRequest(url, referer).executeParsing(GalleryPageParser::parse)
}
suspend fun getGalleryList(url: String) = ehRequest(url, EhUrl.referer)
.executeAndParsingWith(GalleryListParser::parse)
.executeParsingDocument(GalleryListParser::parse)
.apply { fillGalleryList(galleryInfoList, url, true) }
suspend fun getGalleryDetail(url: String) = ehRequest(url, EhUrl.referer).executeAndParsingWith {
EventPaneParser.parse(this)?.let {
suspend fun getGalleryDetail(url: String) = ehRequest(url, EhUrl.referer).executeParsingDocument { s, d ->
EventPaneParser.parse(d)?.let {
Settings.lastDawnDay = today
showEventNotification(it)
}
GalleryDetailParser.parse(this)
GalleryDetailParser.parse(s, d)
}
suspend fun getPreviewList(url: String) = ehRequest(url, EhUrl.referer).executeAndParsingWith {
suspend fun getPreviewList(url: String) = ehRequest(url, EhUrl.referer).executeParsing {
GalleryDetailParser.parsePreviewList(this) to GalleryDetailParser.parsePreviewPages(this)
}
suspend fun getFavorites(url: String) = ehRequest(url, EhUrl.referer)
.executeAndParsingWith(FavoritesParser::parse)
.executeParsingDocument(FavoritesParser::parse)
.apply { fillGalleryList(galleryInfoList, url, false) }
suspend fun signIn(username: String, password: String): String {
@@ -246,7 +252,7 @@ object EhEngine {
add("PassWord", password)
add("CookieDate", "1")
}
}.executeAndParsingWith(SignInParser::parse)
}.executeParsing(SignInParser::parse)
}
suspend fun commentGallery(url: String, comment: String, id: String?) = ehRequest(url, url, EhUrl.origin) {
@@ -258,13 +264,12 @@ object EhEngine {
add("edit_comment", id)
}
}
}.executeAndParsingWith {
val document = Jsoup.parse(this)
val elements = document.select("#chd + p")
}.executeParsingDocument { _, d ->
val elements = d.select("#chd + p")
if (elements.size > 0) {
throw EhException(elements[0].text())
}
GalleryDetailParser.parseComments(document)
GalleryDetailParser.parseComments(d)
}
/**
@@ -291,7 +296,7 @@ object EhEngine {
add("submit", "Apply Changes")
add("update", "1")
}
}.executeAndParsingWith { }
}.executeParsing { }
}
suspend fun downloadArchive(
@@ -323,12 +328,12 @@ object EhEngine {
}
}
}
var result = request.executeAndParsingWith(ArchiveParser::parseArchiveUrl)
var result = request.executeParsing(ArchiveParser::parseArchiveUrl)
if (!isHAtH) {
if (result == null) {
// Wait for the server to prepare archives
delay(1000)
result = request.executeAndParsingWith(ArchiveParser::parseArchiveUrl)
result = request.executeParsing(ArchiveParser::parseArchiveUrl)
if (result == null) {
throw EhException("Archive unavailable")
}
@@ -344,7 +349,7 @@ object EhEngine {
add("act", "limits")
add("reset", "Reset Limit")
}
}.executeAndParsingWith(HomeParser::parseResetLimits)
}.executeParsingDocument(HomeParser::parseResetLimits)
}
suspend fun modifyFavorites(url: String, gidArray: LongArray, dstCat: Int): FavoritesParser.Result {
@@ -359,7 +364,7 @@ object EhEngine {
gidArray.forEach { add("modifygids[]", it.toString()) }
add("apply", "Apply")
}
}.executeAndParsingWith(FavoritesParser::parse).apply { fillGalleryList(galleryInfoList, url, false) }
}.executeParsingDocument(FavoritesParser::parse).apply { fillGalleryList(galleryInfoList, url, false) }
}
suspend fun getGalleryPageApi(
@@ -378,7 +383,7 @@ object EhEngine {
put("imgkey", pToken)
put("showkey", showKey)
}
}.executeAndParsingWith(GalleryPageApiParser::parse)
}.executeParsing(GalleryPageApiParser::parse)
}
suspend fun rateGallery(apiUid: Long, apiKey: String?, gid: Long, token: String?, rating: Float) = ehRequest(EhUrl.apiUrl, EhUrl.getGalleryDetailUrl(gid, token), EhUrl.origin) {
@@ -390,7 +395,7 @@ object EhEngine {
put("token", token)
put("rating", ceil((rating * 2).toDouble()).toInt())
}
}.executeAndParsingWith(RateGalleryParser::parse)
}.executeParsing(RateGalleryParser::parse)
suspend fun fillGalleryListByApi(galleryInfoList: List<GalleryInfo>, referer: String) = galleryInfoList.chunked(MAX_REQUEST_SIZE).parMap {
ehRequest(EhUrl.apiUrl, referer, EhUrl.origin) {
@@ -399,7 +404,7 @@ object EhEngine {
array("gidlist") { it.forEach { put(jsonArrayOf(it.gid, it.token)) } }
put("namespace", 1)
}
}.executeAndParsingWith { GalleryApiParser.parse(this, it) }
}.executeParsing { GalleryApiParser.parse(this, it) }
}
suspend fun voteComment(
@@ -419,7 +424,7 @@ object EhEngine {
put("comment_id", commentId)
put("comment_vote", commentVote)
}
}.executeAndParsingWith { VoteCommentParser.parse(this, commentVote) }
}.executeParsing { VoteCommentParser.parse(this, commentVote) }
suspend fun voteTag(
apiUid: Long,
@@ -438,14 +443,14 @@ object EhEngine {
put("tags", tags)
put("vote", vote)
}
}.executeAndParsingWith(VoteTagParser::parse)
}.executeParsing(VoteTagParser::parse)
suspend fun getGalleryToken(gid: Long, gtoken: String?, page: Int) = ehRequest(EhUrl.apiUrl, EhUrl.referer, EhUrl.origin) {
jsonBody {
put("method", "gtoken")
put("pagelist", JSONArray().put(JSONArray().put(gid).put(gtoken).put(page + 1)))
}
}.executeAndParsingWith(GalleryTokenApiParser::parse)
}.executeParsing(GalleryTokenApiParser::parse)
/**
* @param image Must be jpeg
@@ -458,7 +463,7 @@ object EhEngine {
if (osc) addFormDataPart("fs_covers", "on")
addFormDataPart("f_sfile", "File Search")
}
}.executeAndParsingWith(GalleryListParser::parse).apply { fillGalleryList(galleryInfoList, EhUrl.imageSearchUrl, true) }
}.executeParsingDocument(GalleryListParser::parse).apply { fillGalleryList(galleryInfoList, EhUrl.imageSearchUrl, true) }
private suspend fun fillGalleryList(list: MutableList<GalleryInfo>, url: String, filter: Boolean) {
// Filter title and uploader
app/src/main/java/com/hippo/ehviewer/client/parser/ArchiveParser.kt
+3 -2
View File
@@ -17,6 +17,7 @@ package com.hippo.ehviewer.client.parser
import com.hippo.ehviewer.client.exception.NoHAtHClientException
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
object ArchiveParser {
private val PATTERN_ARCHIVE_URL =
@@ -30,11 +31,11 @@ object ArchiveParser {
private val PATTERN_NEED_HATH_CLIENT =
Regex("You must have a H@H client assigned to your account to use this feature\\.")
fun parse(body: String): Result {
fun parse(body: String, doc: Document): Result {
val m = PATTERN_HATH_FORM.find(body)!!
val paramOr = m.groupValues[1]
val archiveList = ArrayList<Archive>()
Jsoup.parse(body).select("#db>div>div").forEach { element ->
doc.select("#db>div>div").forEach { element ->
if (element.childrenSize() > 0 && !element.attr("style").contains("color:#CCCCCC")) {
runCatching {
val res = element.selectFirst("form>input")!!.attr("value")
app/src/main/java/com/hippo/ehviewer/client/parser/EventPaneParser.kt
+3 -3
View File
@@ -17,10 +17,10 @@
*/
package com.hippo.ehviewer.client.parser
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
object EventPaneParser {
fun parse(body: String): String? {
return Jsoup.parse(body).getElementById("eventpane")?.html()
fun parse(doc: Document): String? {
return doc.getElementById("eventpane")?.html()
}
}
app/src/main/java/com/hippo/ehviewer/client/parser/FavoritesParser.kt
+3 -4
View File
@@ -21,16 +21,15 @@ import com.hippo.ehviewer.client.exception.EhException
import com.hippo.ehviewer.client.exception.ParseException
import com.hippo.ehviewer.util.ExceptionUtils
import com.hippo.ehviewer.util.JsoupUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
object FavoritesParser {
fun parse(body: String): Result {
fun parse(body: String, d: Document): Result {
if (body.contains("This page requires you to log on.</p>")) {
throw EhException(getString(R.string.need_sign_in))
}
val catArray = arrayOfNulls<String>(10)
val countArray = IntArray(10)
val d = Jsoup.parse(body)
runCatching {
val ido = JsoupUtils.getElementByClass(d, "ido")
val fps = ido!!.getElementsByClass("fp")
@@ -46,7 +45,7 @@ object FavoritesParser {
it.printStackTrace()
throw ParseException("Parse favorites error", body)
}
val result = GalleryListParser.parse(d, body)
val result = GalleryListParser.parse(body, d)
return Result(catArray, countArray, result)
}
app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.kt
+1 -4
View File
@@ -40,7 +40,6 @@ import com.hippo.ehviewer.yorozuya.NumberUtils
import com.hippo.ehviewer.yorozuya.StringUtils
import com.hippo.ehviewer.yorozuya.trimAnd
import com.hippo.ehviewer.yorozuya.unescapeXml
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.nodes.Node
@@ -83,7 +82,7 @@ object GalleryDetailParser {
private const val PINING_STRING = "<p>This gallery is pining for the fjords.</p>"
@Throws(EhException::class)
fun parse(body: String): GalleryDetail {
fun parse(body: String, document: Document): GalleryDetail {
if (body.contains(OFFENSIVE_STRING)) {
throw OffensiveException()
}
@@ -93,8 +92,6 @@ object GalleryDetailParser {
// Error info
PATTERN_ERROR.find(body)?.run { throw EhException(groupValues[1]) }
// Temporary workaround, see https://github.com/jhy/jsoup/issues/1850
val document = Jsoup.parse(body.replace("del>", "s>"))
val galleryDetail = GalleryDetail(
tags = parseTagGroups(document),
comments = parseComments(document),
app/src/main/java/com/hippo/ehviewer/client/parser/GalleryListParser.kt
+1 -7
View File
@@ -25,7 +25,6 @@ import com.hippo.ehviewer.client.getThumbKey
import com.hippo.ehviewer.util.ExceptionUtils
import com.hippo.ehviewer.util.JsoupUtils
import com.hippo.ehviewer.yorozuya.NumberUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import java.util.regex.Pattern
@@ -261,12 +260,7 @@ object GalleryListParser {
return gi
}
fun parse(body: String): Result {
val d = Jsoup.parse(body)
return parse(d, body)
}
fun parse(d: Document, body: String): Result {
fun parse(body: String, d: Document): Result {
val result = Result()
try {
val prev = d.getElementById("uprev")
app/src/main/java/com/hippo/ehviewer/client/parser/HomeParser.kt
+6 -6
View File
@@ -1,15 +1,15 @@
package com.hippo.ehviewer.client.parser
import com.hippo.ehviewer.client.exception.ParseException
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
object HomeParser {
private val PATTERN_FUNDS =
Regex("Available: ([\\d,]+) Credits.*Available: ([\\d,]+) kGP", RegexOption.DOT_MATCHES_ALL)
private const val RESET_SUCCEED = "Image limit was successfully reset."
fun parse(body: String): Limits {
Jsoup.parse(body).selectFirst("div.homebox")?.let {
fun parse(doc: Document): Limits {
doc.selectFirst("div.homebox")?.let {
val es = it.select("p > strong")
if (es.size == 3) {
val current = ParserUtils.parseInt(es[0].text(), 0)
@@ -18,14 +18,14 @@ object HomeParser {
return Limits(current, maximum, resetCost)
}
}
throw ParseException("Parse image limits error", body)
throw ParseException("Parse image limits error", doc.html())
}
fun parseResetLimits(body: String): Limits? {
fun parseResetLimits(body: String, doc: Document): Limits? {
return if (body.contains(RESET_SUCCEED)) {
null
} else {
parse(body)
parse(doc)
}
}
app/src/main/java/com/hippo/ehviewer/client/parser/ProfileParser.kt
+3 -4
View File
@@ -19,13 +19,12 @@ import android.util.Log
import com.hippo.ehviewer.client.EhUrl
import com.hippo.ehviewer.client.exception.ParseException
import com.hippo.ehviewer.util.ExceptionUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
object ProfileParser {
private val TAG = ProfileParser::class.java.simpleName
fun parse(body: String): Result {
fun parse(d: Document): Result {
return runCatching {
val d = Jsoup.parse(body)
val profilename = d.getElementById("profilename")
val displayName = profilename!!.child(0).text()
val avatar = runCatching {
@@ -46,7 +45,7 @@ object ProfileParser {
Result(displayName, avatar)
}.getOrElse {
ExceptionUtils.throwIfFatal(it)
throw ParseException("Parse forums error", body)
throw ParseException("Parse forums error", d.html())
}
}
app/src/main/java/com/hippo/ehviewer/client/parser/TorrentParser.kt
+2 -3
View File
@@ -15,16 +15,15 @@
*/
package com.hippo.ehviewer.client.parser
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import java.util.regex.Pattern
object TorrentParser {
private val PATTERN_TORRENT =
Pattern.compile("</span> ([0-9-]+) [0-9:]+</td>[\\s\\S]+</span> ([0-9.]+ [KMGT]B)</td>[\\s\\S]+</span> ([0-9]+)</td>[\\s\\S]+</span> ([0-9]+)</td>[\\s\\S]+</span> ([0-9]+)</td>[\\s\\S]+</span>([^<]+)</td>[\\s\\S]+onclick=\"document.location='([^\"]+)'[^<]+>([^<]+)</a>")
fun parse(body: String): List<Result> {
fun parse(d: Document): List<Result> {
val torrentList = ArrayList<Result>()
val d = Jsoup.parse(body)
val es = d.select("form>div>table")
for (e in es) {
val m = PATTERN_TORRENT.matcher(e.html())