Migrate GalleryDetailParser to Rust Part 1/N

This commit is contained in:
FooIbar committed 2025-06-10 11:39:42 +08:00
1 parent 734bffd279
commit 12374fb573
23 files changed
+727 -148

No files matched your search

@@ -41,7 +41,6 @@ import com.hippo.ehviewer.client.parser.FavoritesParser
import com.hippo.ehviewer.client.parser.GalleryApiParser
import com.hippo.ehviewer.client.parser.GalleryDetailParser
import com.hippo.ehviewer.client.parser.GalleryListParser
import com.hippo.ehviewer.client.parser.GalleryNotAvailableParser
import com.hippo.ehviewer.client.parser.GalleryPageParser
import com.hippo.ehviewer.client.parser.GalleryTokenApiParser
import com.hippo.ehviewer.client.parser.HomeParser
@@ -121,11 +120,19 @@ fun rethrowExactly(response: HttpResponse, body: Either<String, ByteBuffer>, e:
// Check Gallery Not Available
body.onLeft {
if ("Gallery Not Available - " in it) {
val error = GalleryNotAvailableParser.parse(it)
if (!error.isNullOrBlank()) {
throw EhException(error)
}
if ("Gallery Not Available - " in it) throw e
}
// Check parse error thrown by rust
val cause = e.cause
if (e is ParseException && cause is RuntimeException) {
when (val message = cause.message) {
"0" -> throw NoHitsFoundException()
"1" -> throw EhException(R.string.gallery_list_empty_hit_subscription)
"2" -> throw NotLoggedInException()
"3" -> throw NoHathClientException()
"4" -> throw InsufficientFundsException()
is String if message.startsWith('5') -> throw EhException(message.drop(1))
}
}
@@ -134,14 +141,6 @@ fun rethrowExactly(response: HttpResponse, body: Either<String, ByteBuffer>, e:
if (e is ParseException || e is SerializationException) {
body.onLeft { if ("<" !in it) throw EhException(it) }
when (val message = e.cause?.message) {
"0" -> throw NoHitsFoundException()
"1" -> throw EhException(R.string.gallery_list_empty_hit_subscription)
"2" -> throw NotLoggedInException()
"3" -> throw NoHathClientException()
"4" -> throw InsufficientFundsException()
is String if message.startsWith('5') -> throw EhException(message.drop(1))
}
if (Settings.saveParseErrorBody) body.saveParseError(e)
throw EhException(appCtx.getString(R.string.error_parse_error), e)
}
@@ -205,7 +204,7 @@ object EhEngine {
private suspend fun getFunds() = ehRequest(EhUrl.URL_FUNDS).fetchUsingAsText(HomeParser::parseFunds)
suspend fun getNews(parse: Boolean) = ehRequest(EhUrl.URL_NEWS, EhUrl.REFERER_E)
.fetchUsingAsText { if (parse) EventPaneParser.parse(this) else null }
.fetchUsingAsByteBuffer { if (parse) EventPaneParser.parse(this) else null }
suspend fun getProfile(): ProfileParser.Result {
val url = ehRequest(EhUrl.URL_FORUMS).fetchUsingAsByteBuffer(ProfileParser::parseProfileUrl)
@@ -235,13 +234,16 @@ object EhEngine {
.apply { galleryInfoList.fillInfo(url, true) }
.takeUnless { it.galleryInfoList.isEmpty() } ?: GalleryListParser.emptyResult
suspend fun getGalleryDetail(url: String) = ehRequest(url, EhUrl.referer).fetchUsingAsText {
val eventPane = EventPaneParser.parse(this)
if (eventPane != null) {
suspend fun getGalleryDetail(url: String) = ehRequest(url, EhUrl.referer).fetchUsingAsByteBuffer(GalleryDetailParser::parseRust).run {
event?.let {
Settings.lastDawnDays = today
showEventNotification(eventPane)
showEventNotification(it)
}
detail.apply {
// Fill info for database
fillInfo()
filterComments()
}
GalleryDetailParser.parse(this)
}
suspend fun getPreviewList(url: String) = ehRequest(url, EhUrl.referer).fetchUsingAsText {
@@ -15,6 +15,9 @@
*/
package com.hippo.ehviewer.client.data
import kotlinx.serialization.Serializable
@Serializable
data class GalleryComment(
val id: Long,
val score: Int,
@@ -15,6 +15,9 @@
*/
package com.hippo.ehviewer.client.data
import kotlinx.serialization.Serializable
@Serializable
data class GalleryCommentList(
val comments: List<GalleryComment>,
val hasMore: Boolean,
@@ -16,7 +16,10 @@
package com.hippo.ehviewer.client.data
import androidx.compose.ui.util.fastFlatMap
import com.hippo.ehviewer.client.data.GalleryInfo.Companion.S_LANGS
import com.hippo.ehviewer.EhDB
import com.hippo.ehviewer.Settings
import com.hippo.ehviewer.client.EhFilter
import kotlinx.serialization.Serializable
private val LANGUAGES = arrayOf(
"English",
@@ -35,6 +38,7 @@ private val LANGUAGES = arrayOf(
"Dutch",
)
@Serializable
data class GalleryDetail(
val galleryInfo: BaseGalleryInfo = BaseGalleryInfo(),
var apiUid: Long = -1L,
@@ -53,11 +57,28 @@ data class GalleryDetail(
var comments: GalleryCommentList,
val previewList: List<GalleryPreview>,
) : GalleryInfo by galleryInfo {
fun fillInfo() {
suspend fun fillInfo() {
val index = LANGUAGES.indexOf(language)
if (index != -1) simpleLanguage = S_LANGS[index]
if (index != -1) simpleLanguage = GalleryInfo.S_LANGS[index]
simpleTags = tagGroups.fastFlatMap(GalleryTagGroup::tags).map { (text, power, _) ->
if (power == PowerStatus.WEAK) "_$text" else text
if (power == PowerStatus.Weak) "_$text" else text
}
if (favoriteSlot == GalleryInfo.NOT_FAVORITED && EhDB.containLocalFavorites(gid)) {
favoriteSlot = GalleryInfo.LOCAL_FAVORITED
}
}
suspend fun filterComments() {
comments = with(comments) {
val scoreThreshold = Settings.commentThreshold
copy(
comments = comments.filter {
it.uploader ||
it.score > scoreThreshold &&
!EhFilter.filterCommenter(it.user.orEmpty()) &&
!EhFilter.filterComment(it.comment)
},
)
}
}
}
@@ -17,7 +17,10 @@ package com.hippo.ehviewer.client.data
import com.hippo.ehviewer.client.getThumbKey
import com.hippo.ehviewer.client.getV2PreviewKey
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
@Serializable
sealed interface GalleryPreview {
val url: String
val imageKey: String
@@ -25,6 +28,8 @@ sealed interface GalleryPreview {
val pToken: String
}
@Serializable
@SerialName("V1")
data class V1GalleryPreview(
override val url: String,
override val position: Int,
@@ -33,6 +38,8 @@ data class V1GalleryPreview(
override val imageKey get() = getThumbKey(url)
}
@Serializable
@SerialName("V2")
data class V2GalleryPreview(
override val url: String,
override val position: Int,
@@ -1,17 +1,20 @@
package com.hippo.ehviewer.client.data
import kotlinx.serialization.Serializable
enum class VoteStatus(val display: String) {
NONE(""),
UP("+1"),
DOWN("-1"),
None(""),
Up("+1"),
Down("-1"),
}
enum class PowerStatus {
SOLID,
ACTIVE,
WEAK,
Solid,
Active,
Weak,
}
@Serializable
data class GalleryTag(
val text: String,
val power: PowerStatus,
@@ -15,7 +15,10 @@
*/
package com.hippo.ehviewer.client.data
import kotlinx.serialization.Serializable
@Serializable
data class GalleryTagGroup(
val nameSpace: TagNamespace,
val namespace: TagNamespace,
val tags: List<GalleryTag>,
)
@@ -1,5 +1,14 @@
package com.hippo.ehviewer.client.data
import kotlinx.serialization.KSerializer
import kotlinx.serialization.Serializable
import kotlinx.serialization.SerializationException
import kotlinx.serialization.descriptors.PrimitiveKind
import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
import kotlinx.serialization.encoding.Decoder
import kotlinx.serialization.encoding.Encoder
@Serializable(TagNamespaceSerializer::class)
sealed class TagNamespace(val value: String, val prefix: String?) {
object Artist : TagNamespace("artist", "a")
object Cosplayer : TagNamespace("cosplayer", "cos")
@@ -32,3 +41,12 @@ sealed class TagNamespace(val value: String, val prefix: String?) {
}
}
}
object TagNamespaceSerializer : KSerializer<TagNamespace> {
override val descriptor = PrimitiveSerialDescriptor("TagNamespace", PrimitiveKind.STRING)
override fun deserialize(decoder: Decoder): TagNamespace {
val string = decoder.decodeString()
return TagNamespace.from(string) ?: throw SerializationException("Unknown tag namespace $string")
}
override fun serialize(encoder: Encoder, value: TagNamespace) = encoder.encodeString(value.value)
}
@@ -17,8 +17,10 @@
*/
package com.hippo.ehviewer.client.parser
import org.jsoup.Jsoup
import java.nio.ByteBuffer
object EventPaneParser {
fun parse(body: String): String? = Jsoup.parse(body).getElementById("eventpane")?.html()
private external fun parseEventPane(body: ByteBuffer, size: Int = body.limit()): Int
fun parse(body: ByteBuffer): String? = unmarshalParsingAs(body, ::parseEventPane)
}
@@ -17,16 +17,11 @@ package com.hippo.ehviewer.client.parser
import arrow.core.Either
import arrow.core.getOrElse
import com.hippo.ehviewer.EhDB
import com.hippo.ehviewer.Settings
import com.hippo.ehviewer.client.EhFilter
import com.hippo.ehviewer.client.EhUtils
import com.hippo.ehviewer.client.data.BaseGalleryInfo
import com.hippo.ehviewer.client.data.GalleryComment
import com.hippo.ehviewer.client.data.GalleryCommentList
import com.hippo.ehviewer.client.data.GalleryDetail
import com.hippo.ehviewer.client.data.GalleryInfo.Companion.LOCAL_FAVORITED
import com.hippo.ehviewer.client.data.GalleryInfo.Companion.NOT_FAVORITED
import com.hippo.ehviewer.client.data.GalleryTag
import com.hippo.ehviewer.client.data.GalleryTagGroup
import com.hippo.ehviewer.client.data.PowerStatus
@@ -35,9 +30,7 @@ import com.hippo.ehviewer.client.data.V1GalleryPreview
import com.hippo.ehviewer.client.data.V2GalleryPreview
import com.hippo.ehviewer.client.data.VoteStatus
import com.hippo.ehviewer.client.exception.EhException
import com.hippo.ehviewer.client.exception.OffensiveException
import com.hippo.ehviewer.client.exception.ParseException
import com.hippo.ehviewer.client.exception.PiningException
import com.hippo.ehviewer.client.getThumbKey
import com.hippo.ehviewer.util.toEpochMillis
import com.hippo.ehviewer.util.toFloatOrDefault
@@ -45,16 +38,19 @@ import com.hippo.ehviewer.util.toIntOrDefault
import com.hippo.ehviewer.util.trimAnd
import com.hippo.ehviewer.util.unescapeXml
import eu.kanade.tachiyomi.util.system.logcat
import java.nio.ByteBuffer
import kotlinx.datetime.LocalDateTime
import kotlinx.datetime.format.MonthNames
import kotlinx.datetime.format.char
import kotlinx.serialization.Serializable
import kotlinx.serialization.cbor.CborArray
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.NodeTraversor
object GalleryDetailParser {
private val PATTERN_ERROR = Regex("<div class=\"d\">\n<p>([^<]+)</p>")
private val PATTERN_ERROR = Regex("<div class=\"d\">[^<]*<p>([^<]+)</p>")
private val PATTERN_DETAIL =
Regex("var gid = (\\d+);\\s+var token = \"([a-f0-9]+)\";\\s+var apiuid = ([\\-\\d]+);\\s+var apikey = \"([a-f0-9]+)\";")
private val PATTERN_TORRENT =
@@ -83,20 +79,9 @@ object GalleryDetailParser {
char(':')
minute()
}
private const val OFFENSIVE_STRING =
"<p>(And if you choose to ignore this warning, you lose all rights to complain about it in the future.)</p>"
private const val PINING_STRING = "<p>This gallery is pining for the fjords.</p>"
suspend fun parse(body: String): GalleryDetail {
if (body.contains(OFFENSIVE_STRING)) {
throw OffensiveException()
}
if (body.contains(PINING_STRING)) {
throw PiningException()
}
// Error info
PATTERN_ERROR.find(body)?.run { throw EhException(groupValues[1]) }
fun parse(body: String): GalleryDetail {
PATTERN_ERROR.find(body)?.run { throw EhException(groupValues[1].replace("<br>", "\n")) }
val document = Jsoup.parse(body)
val galleryDetail = GalleryDetail(
tagGroups = parseTagGroups(document),
@@ -104,13 +89,10 @@ object GalleryDetailParser {
previewList = parsePreviewList(body),
)
parseDetail(galleryDetail, document, body)
// Fill info for database
galleryDetail.fillInfo()
return galleryDetail
}
private suspend fun parseDetail(gd: GalleryDetail, d: Document, body: String) {
private fun parseDetail(gd: GalleryDetail, d: Document, body: String) {
PATTERN_DETAIL.find(body)?.apply {
gd.gid = groupValues[1].toLongOrNull() ?: -1L
gd.token = groupValues[2]
@@ -206,9 +188,6 @@ object GalleryDetailParser {
}
}
}
if (gd.favoriteSlot == NOT_FAVORITED && EhDB.containLocalFavorites(gd.gid)) {
gd.favoriteSlot = LOCAL_FAVORITED
}
}.getOrElse {
throw ParseException("Can't parse gallery detail")
}
@@ -279,14 +258,14 @@ object GalleryDetailParser {
// Sometimes parody tag is followed with '|' and english translate, just remove them
text = e.text().substringBefore('|').trim(),
power = when {
e.hasClass("gtw") -> PowerStatus.WEAK
e.hasClass("gtl") -> PowerStatus.ACTIVE
else -> PowerStatus.SOLID
e.hasClass("gtw") -> PowerStatus.Weak
e.hasClass("gtl") -> PowerStatus.Active
else -> PowerStatus.Solid
},
vote = when {
e.child(0).hasClass("tup") -> VoteStatus.UP
e.child(0).hasClass("tdn") -> VoteStatus.DOWN
else -> VoteStatus.NONE
e.child(0).hasClass("tup") -> VoteStatus.Up
e.child(0).hasClass("tdn") -> VoteStatus.Down
else -> VoteStatus.None
},
)
}
@@ -308,7 +287,7 @@ object GalleryDetailParser {
emptyList()
}
private suspend fun parseComment(element: Element): GalleryComment? = Either.catch {
private fun parseComment(element: Element): GalleryComment? = Either.catch {
// Id
val a = element.previousElementSibling()
val name = a!!.attr("name")
@@ -365,13 +344,6 @@ object GalleryDetailParser {
}
}
val commentHtml = c6.html()
// filter comment
if (!uploader) {
val sEhFilter = EhFilter
if (score <= Settings.commentThreshold || sEhFilter.filterCommenter(user.orEmpty()) || sEhFilter.filterComment(commentHtml)) {
return null
}
}
// last edited
val c8 = element.getElementsByClass("c8").first()
val lastEdited = c8?.children()?.first()?.run { WEB_COMMENT_DATE_FORMAT.parse(text()).toEpochMillis() } ?: 0
@@ -387,7 +359,7 @@ object GalleryDetailParser {
/**
* Parse comments with html parser
*/
suspend fun parseComments(document: Document): GalleryCommentList = Either.catch {
fun parseComments(document: Document): GalleryCommentList = Either.catch {
// Disable pretty print to get comments in raw html
document.outputSettings().prettyPrint(false)
val cdiv = document.getElementById("cdiv")!!
@@ -430,4 +402,16 @@ object GalleryDetailParser {
}.toList().apply {
if (isEmpty()) throw ParseException("Parse preview list error")
}
fun parseRust(body: ByteBuffer) = Either.catch {
unmarshalParsingAs<Result>(body, ::parseGalleryDetail)
}.getOrElse {
throw ParseException("Failed to parse gallery detail", it)
}
@Serializable
@CborArray
class Result(val detail: GalleryDetail, val event: String?)
private external fun parseGalleryDetail(body: ByteBuffer, size: Int = body.limit()): Int
}
@@ -1,33 +0,0 @@
/*
* Copyright 2022 Tarsin Norbin
*
* This file is part of EhViewer
*
* EhViewer is free software: you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* EhViewer is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with EhViewer.
* If not, see <https://www.gnu.org/licenses/>.
*/
package com.hippo.ehviewer.client.parser
import arrow.core.Either
import arrow.core.getOrElse
import eu.kanade.tachiyomi.util.system.logcat
import org.jsoup.Jsoup
object GalleryNotAvailableParser {
fun parse(body: String): String? = Either.catch {
val e = Jsoup.parse(body).getElementsByClass("d").first()
e!!.child(0).html().replace("<br>", "\n")
}.getOrElse {
logcat(it)
null
}
}
@@ -54,8 +54,8 @@ fun GalleryInfo.getComicInfo(): ComicInfo {
with(TagNamespace) {
when (this@getComicInfo) {
is GalleryDetail -> tagGroups.forEach { group ->
val list = group.tags.filterNot { (text, power, _) -> text == TAG_ORIGINAL || power == PowerStatus.WEAK }.map(GalleryTag::text)
when (val ns = group.nameSpace) {
val list = group.tags.filterNot { (text, power, _) -> text == TAG_ORIGINAL || power == PowerStatus.Weak }.map(GalleryTag::text)
when (val ns = group.namespace) {
Artist, Cosplayer -> artists.addAll(list)
Group -> groups.addAll(list)
Character -> characters.addAll(list)
@@ -58,7 +58,7 @@ fun GalleryTags(
Box {
BaseRoundText(
text = translation,
weak = power == PowerStatus.WEAK,
weak = power == PowerStatus.Weak,
modifier = Modifier.combinedClickable(
onClick = { onTagClick(tag) },
onLongClick = {
@@ -67,7 +67,7 @@ fun GalleryTags(
},
),
)
if (vote != VoteStatus.NONE && showVote) {
if (vote != VoteStatus.None && showVote) {
Text(
text = vote.display,
modifier = Modifier.align(Alignment.TopEnd).padding(horizontal = 2.dp),
@@ -243,9 +243,8 @@ fun AnimatedVisibilityScope.GalleryCommentsScreen(gid: Long, navigator: Destinat
snackbar(filterAdded)
}
suspend fun showCommentVoteStatus(comment: GalleryComment) {
val statusStr = comment.voteState ?: return
val data = statusStr.split(',').map {
suspend fun showCommentVoteStatus(status: String) {
val data = status.split(',').map {
val str = it.trim()
val index = str.lastIndexOf(' ')
if (index < 0) {
@@ -400,7 +399,7 @@ fun AnimatedVisibilityScope.GalleryCommentsScreen(gid: Long, navigator: Destinat
}
if (!comment.voteState.isNullOrEmpty()) {
onSelect(checkVoteStatus) {
showCommentVoteStatus(comment)
showCommentVoteStatus(comment.voteState)
}
}
}()
@@ -743,12 +743,12 @@ fun BelowHeader(galleryDetail: GalleryDetail, voteTag: VoteTag) {
}
if (galleryDetail.apiUid >= 0) {
when (vote) {
VoteStatus.NONE -> {
VoteStatus.None -> {
onSelect(upTag) { galleryDetail.voteTag(tag, 1) }
onSelect(downTag) { galleryDetail.voteTag(tag, -1) }
}
VoteStatus.UP -> onSelect(withDraw) { galleryDetail.voteTag(tag, -1) }
VoteStatus.DOWN -> onSelect(withDraw) { galleryDetail.voteTag(tag, 1) }
VoteStatus.Up -> onSelect(withDraw) { galleryDetail.voteTag(tag, -1) }
VoteStatus.Down -> onSelect(withDraw) { galleryDetail.voteTag(tag, 1) }
}
}
}()
@@ -765,7 +765,6 @@ fun BelowHeader(galleryDetail: GalleryDetail, voteTag: VoteTag) {
@StringRes
private fun getRatingText(rating: Float): Int = when ((rating * 2).roundToInt()) {
0 -> R.string.rating0
1 -> R.string.rating1
2 -> R.string.rating2
3 -> R.string.rating3
+1
View File
@@ -269,6 +269,7 @@ version = "0.0.0"
dependencies = [
"android_logger",
"anyhow",
"chrono",
"image",
"jni",
"jni_fn",
+1
View File
@@ -15,6 +15,7 @@ android-26 = ["android", "ndk/api-level-26"]
regex-lite = "0.1"
jni = { version = "0.21", optional = true }
jni_fn = { version = "0.1", optional = true }
chrono = { version = "0.4", default-features = false, features = ["std"] }
quick-xml = { default-features = false, version = "0.37" }
tl = { version = "0.7", features = ["simd"] }
log = "0.4"
+38 -3
View File
@@ -2,6 +2,7 @@
use crate::parser::archive::{parse_archive_url, parse_archives, parse_archives_with_funds};
use crate::parser::config::parse_fav_cat;
use crate::parser::detail::{parse_event_pane, parse_gallery_detail};
use crate::parser::fav::parse_fav;
use crate::parser::home::parse_limit;
use crate::parser::list::parse_info_list;
@@ -49,6 +50,25 @@ pub fn parseGalleryInfoList(mut env: JNIEnv, _: JClass, buffer: JByteBuffer, lim
})
}
#[no_mangle]
#[allow(non_snake_case)]
#[jni_fn("com.hippo.ehviewer.client.parser.GalleryDetailParser")]
pub fn parseGalleryDetail(mut env: JNIEnv, _: JClass, buffer: JByteBuffer, limit: jint) -> jint {
let options = ParserOptions::default().track_ids();
parse_marshal_inplace_with_options(&mut env, buffer, limit, options, |dom, html| {
parse_gallery_detail(dom, html).map(|detail| (detail, parse_event_pane(dom, dom.parser())))
})
}
#[no_mangle]
#[allow(non_snake_case)]
#[jni_fn("com.hippo.ehviewer.client.parser.EventPaneParser")]
pub fn parseEventPane(mut env: JNIEnv, _class: JClass, input: JByteBuffer, limit: jint) -> jint {
parse_marshal_inplace(&mut env, input, limit, |dom, _| {
Ok(parse_event_pane(dom, dom.parser()))
})
}
#[no_mangle]
#[allow(non_snake_case)]
#[jni_fn("com.hippo.ehviewer.client.parser.TorrentParserKt")]
@@ -165,9 +185,24 @@ where
}
}
#[inline]
pub fn parse_marshal_inplace<F, R>(env: &mut JNIEnv, str: JByteBuffer, limit: jint, f: F) -> i32
where
F: Fn(&VDom, &str) -> Result<R>,
F: Fn(&mut VDom, &str) -> Result<R>,
R: Serialize,
{
parse_marshal_inplace_with_options(env, str, limit, ParserOptions::default(), f)
}
pub fn parse_marshal_inplace_with_options<F, R>(
env: &mut JNIEnv,
str: JByteBuffer,
limit: jint,
options: ParserOptions,
f: F,
) -> i32
where
F: Fn(&mut VDom, &str) -> Result<R>,
R: Serialize,
{
jni_throwing(env, |env| {
@@ -176,9 +211,9 @@ where
// SAFETY: ktor client ensure html content is valid utf-8.
let html = unsafe { from_utf8_unchecked(&buffer[..limit as usize]) };
let dom = tl::parse(html, ParserOptions::default())?;
let mut dom = tl::parse(html, options)?;
ensure!(dom.version().is_some(), EhError::Error(html.to_string()));
f(&dom, html)?
f(&mut dom, html)?
};
let mut cursor = Cursor::new(buffer);
serde_cbor::to_writer(&mut cursor, &value)?;
+10
View File
@@ -36,6 +36,8 @@ fn get_first_element_by_class_name<'a>(
handle.get(parser)
}
// tl does not support selecting child elements
// Workaround for https://github.com/y21/tl/issues/22
fn select_first<'a>(tag: &'a HTMLTag, parser: &'a Parser, id: &str) -> Option<&'a HTMLTag<'a>> {
for n in tag.children().top().iter() {
let tag = n.get(parser).and_then(Node::as_tag);
@@ -46,6 +48,14 @@ fn select_first<'a>(tag: &'a HTMLTag, parser: &'a Parser, id: &str) -> Option<&'
None
}
fn get_first_child<'a>(tag: &'a HTMLTag, parser: &'a Parser) -> Option<&'a HTMLTag<'a>> {
tag.children()
.top()
.iter()
.filter_map(|n| n.get(parser)?.as_tag())
.next()
}
fn get_element_by_id<'b, S>(node: &'b Node, parser: &'b Parser, id: S) -> Option<&'b Node<'b>>
where
S: Into<Bytes<'b>>,
+1
View File
@@ -1,5 +1,6 @@
pub mod archive;
pub mod config;
pub mod detail;
pub mod fav;
pub mod home;
pub mod list;
+507
View File
@@ -0,0 +1,507 @@
use crate::parser::list::{get_thumb_key, parse_token_and_gid, to_category_i32, BaseGalleryInfo};
use crate::{get_first_child, get_first_element_by_class_name, get_node_handle_attr, get_tag_attr};
use crate::{get_vdom_first_element_by_class_name, regex, EhError};
use anyhow::{bail, Context, Result};
use chrono::naive::NaiveDateTime;
use quick_xml::escape::unescape;
use serde::Serialize;
use tl::{Node, NodeHandle, Parser, VDom};
#[derive(Serialize, Debug, Default)]
#[allow(non_snake_case)]
pub struct GalleryDetail {
pub galleryInfo: BaseGalleryInfo,
pub apiUid: i64,
pub apiKey: Option<String>,
pub torrentCount: i32,
pub torrentUrl: Option<String>,
pub archiveUrl: Option<String>,
pub parent: Option<String>,
pub newerVersions: Vec<BaseGalleryInfo>,
pub visible: Option<String>,
pub language: Option<String>,
pub size: Option<String>,
pub favoriteCount: i32,
pub ratingCount: i32,
pub tagGroups: Vec<GalleryTagGroup>,
pub comments: GalleryCommentList,
pub previewList: Vec<GalleryPreview>,
}
#[derive(Serialize, Debug)]
pub enum PowerStatus {
Solid,
Weak,
Active,
}
#[derive(Serialize, Debug)]
pub enum VoteStatus {
None,
Up,
Down,
}
#[derive(Serialize, Debug)]
pub struct GalleryTag {
pub text: String,
pub power: PowerStatus,
pub vote: VoteStatus,
}
#[derive(Serialize, Debug)]
pub struct GalleryTagGroup {
pub namespace: String,
pub tags: Vec<GalleryTag>,
}
#[derive(Serialize, Debug)]
#[allow(non_snake_case)]
pub struct GalleryComment {
pub id: i64,
pub score: i32,
pub editable: bool,
pub voteUpAble: bool,
pub voteUpEd: bool,
pub voteDownAble: bool,
pub voteDownEd: bool,
pub uploader: bool,
pub voteState: Option<String>,
pub time: i64,
pub user: Option<String>,
pub comment: String,
pub lastEdited: i64,
}
#[derive(Serialize, Debug, Default)]
#[allow(non_snake_case)]
pub struct GalleryCommentList {
pub comments: Vec<GalleryComment>,
pub hasMore: bool,
}
#[derive(Serialize, Debug)]
#[allow(non_snake_case)]
pub struct PreviewData {
pub url: String,
pub position: i32,
pub pToken: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub offsetX: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub clipWidth: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub clipHeight: Option<i32>,
}
// Should use internally tagged enum, but kotlinx.serialization only support array polymorphism for CBOR
// https://github.com/Kotlin/kotlinx.serialization/issues/2058
#[derive(Serialize, Debug)]
pub struct GalleryPreview(pub String, pub PreviewData);
fn parse_detail_info(gd: &mut GalleryDetail, node: &NodeHandle, parser: &Parser) -> Option<()> {
let children = node.get(parser)?.children()?;
let mut iter = children
.top()
.iter()
.filter_map(|n| n.get(parser)?.as_tag());
let key_tag = iter.next()?;
let value_tag = iter.next()?;
let key = key_tag.inner_text(parser);
let value = value_tag.children().top()[0]
.get(parser)?
.inner_text(parser);
match key.as_ref() {
"Posted:" => gd.galleryInfo.posted = value.to_string(),
"Parent:" => {
gd.parent = get_first_child(value_tag, parser)
.and_then(|a| get_tag_attr(a, "href").map(str::to_string))
}
"Visible:" => gd.visible = Some(value.to_string()),
"Language:" => gd.language = value.strip_suffix(" &nbsp;").map(str::to_string),
"File Size:" => gd.size = Some(value.to_string()),
"Length:" => gd.galleryInfo.pages = value.strip_suffix(" pages")?.parse().ok()?,
"Favorited:" => {
gd.favoriteCount = match value.as_ref() {
"Never" => 0,
"Once" => 1,
_ => value.strip_suffix(" times")?.parse().ok()?,
}
}
_ => {}
}
Some(())
}
fn parse_detail(gd: &mut GalleryDetail, dom: &VDom, parser: &Parser, body: &str) -> Option<()> {
let detail_regex = regex!(
r#"var gid = (\d+);\s*var token = "([a-f0-9]+)";\s*var apiuid = ([-\d]+);\s*var apikey = "([a-f0-9]+)";\s*var average_rating = ([.\d]+);"#
);
let caps = detail_regex.captures(body)?;
gd.galleryInfo.gid = caps[1].parse().ok()?;
gd.galleryInfo.token = caps[2].to_string();
gd.apiUid = caps[3].parse().ok()?;
gd.apiKey = Some(caps[4].to_string());
gd.galleryInfo.rating = caps[5].parse().ok()?;
let torrent_regex = regex!(
r#"<a[^<>]*onclick="return popUp\('([^']+)'[^)]+\)">Torrent Download \((\d+)\)</a>"#
);
let caps = torrent_regex.captures(body)?;
gd.torrentUrl = unescape(&caps[1]).ok().map(|s| s.to_string());
gd.torrentCount = caps[2].parse().ok()?;
let archive_regex =
regex!(r#"<a[^<>]*onclick="return popUp\('([^']+)'[^)]+\)">Archive Download</a>"#);
let caps = archive_regex.captures(body)?;
gd.archiveUrl = unescape(&caps[1]).ok().map(|s| s.to_string());
let gd1 = dom.get_element_by_id("gd1")?.get(parser)?.as_tag()?;
let style = get_tag_attr(get_first_child(gd1, parser)?, "style")?;
let cover_regex = regex!(r#"[^(]+\(([^)]+)"#);
let caps = cover_regex.captures(style)?;
gd.galleryInfo.thumbKey = get_thumb_key(&caps[1]);
let title = dom.get_element_by_id("gn")?.get(parser)?.inner_text(parser);
gd.galleryInfo.title = unescape(&title).ok()?.to_string();
let title_jpn = dom.get_element_by_id("gj")?.get(parser)?.inner_text(parser);
gd.galleryInfo.titleJpn = unescape(&title_jpn).ok().map(|s| s.to_string());
let category = dom
.get_element_by_id("gdc")?
.get(parser)?
.inner_text(parser);
gd.galleryInfo.category = to_category_i32(&category.to_lowercase());
gd.galleryInfo.uploader = dom.get_element_by_id("gdn").and_then(|n| {
let gdn = n.get(parser)?.as_tag()?;
gd.galleryInfo.disowned =
get_tag_attr(gdn, "style").is_some_and(|s| s.contains("opacity:0.5"));
let raw = get_first_child(gdn, parser)
.unwrap_or(gdn)
.inner_text(parser);
unescape(&raw).ok().map(|s| s.to_string())
});
dom.get_element_by_id("gdd")?
.get(parser)?
.as_tag()?
.query_selector(parser, "tr")?
.for_each(|node| {
parse_detail_info(gd, &node, parser);
});
gd.ratingCount = dom
.get_element_by_id("rating_count")?
.get(parser)?
.inner_text(parser)
.parse()
.ok()?;
let fav = dom.get_element_by_id("fav")?.get(parser)?.as_tag()?;
gd.galleryInfo.favoriteSlot = get_first_child(fav, parser)
.and_then(|div| {
gd.galleryInfo.favoriteName = get_tag_attr(div, "title")
.and_then(|s| unescape(s).ok())
.map(|s| s.to_string());
let fav_slot_regex = regex!(r#"\D+\d+\D+(\d+)"#);
fav_slot_regex
.captures(get_tag_attr(div, "style")?)
.map(|caps| (caps[1].parse().unwrap_or(2) - 2) / 19)
})
.unwrap_or(-2);
gd.newerVersions = dom
.get_element_by_id("gnd")
.and_then(|n| n.get(parser))
.map(|gnd_node| {
let newer_versions_regex = regex!(r#"<a href="([^"]+)">([^<]+)</a>, added ([^<]+)<"#);
newer_versions_regex
.captures_iter(&gnd_node.inner_html(parser))
.filter_map(|caps| {
let (gid, token) = parse_token_and_gid(&caps[1])?;
Some(BaseGalleryInfo {
gid,
token,
title: unescape(&caps[2]).ok()?.to_string(),
posted: caps[3].to_string(),
..Default::default()
})
})
.collect()
})
.unwrap_or_default();
Some(())
}
pub fn parse_gallery_detail(dom: &mut VDom, body: &str) -> Result<GalleryDetail> {
let parser = dom.parser();
let error = get_vdom_first_element_by_class_name(dom, "d");
if let Some(tag) = error.and_then(|n| get_first_child(n.as_tag()?, parser)) {
let node = tag.children().top()[0].get(parser).unwrap();
bail!(EhError::Error(node.inner_text(parser).to_string()));
}
let mut gallery_detail = GalleryDetail {
tagGroups: parse_tag_groups(dom, parser).context("Failed to parse tag groups")?,
previewList: parse_preview_list(dom, parser).context("Failed to parse preview list")?,
comments: parse_comments(dom).context("Failed to parse comments")?,
..Default::default()
};
parse_detail(&mut gallery_detail, dom, dom.parser(), body)
.context("Failed to parse gallery detail")?;
Ok(gallery_detail)
}
pub fn parse_event_pane(dom: &VDom, parser: &Parser) -> Option<String> {
dom.get_element_by_id("eventpane")?
.get(parser)
.map(|n| n.inner_html(parser).to_string())
}
fn parse_tag_groups(dom: &VDom, parser: &Parser) -> Option<Vec<GalleryTagGroup>> {
dom.get_element_by_id("taglist")?
.get(parser)?
.as_tag()?
.query_selector(parser, "tr")?
.filter_map(|n| n.get(parser)?.children())
.map(|children| {
let mut iter = children
.top()
.iter()
.filter_map(|n| n.get(parser)?.as_tag());
let namespace = iter
.next()?
.inner_text(parser)
.strip_suffix(':')?
.to_string();
let tags = iter
.next()?
.children()
.top()
.iter()
.filter_map(|n| n.get(parser)?.as_tag())
.map(|tag| {
let text = unescape(&tag.inner_text(parser))
.ok()?
.split('|')
.next()?
.trim()
.to_string();
let power = match tag.attributes().class()?.try_as_utf8_str()? {
"gtw" => PowerStatus::Weak,
"gtl" => PowerStatus::Active,
_ => PowerStatus::Solid,
};
let vote = match get_first_child(tag, parser)?
.attributes()
.class()?
.try_as_utf8_str()?
{
"tup" => VoteStatus::Up,
"tdn" => VoteStatus::Down,
_ => VoteStatus::None,
};
Some(GalleryTag { text, power, vote })
})
.collect::<Option<Vec<_>>>()?;
Some(GalleryTagGroup { namespace, tags })
})
.collect::<Option<Vec<_>>>()
}
fn parse_comment_time(str: &str, prefix: &str) -> Option<i64> {
str.strip_prefix(prefix)
.and_then(|s| NaiveDateTime::parse_and_remainder(s, "%d %B %Y, %R").ok())
.map(|(dt, _)| dt.and_utc().timestamp_millis())
}
fn parse_comments(dom: &mut VDom) -> Result<GalleryCommentList> {
let cdiv = dom
.get_element_by_id("cdiv")
.context("Failed to find cdiv")?;
let parser = dom.parser_mut();
let comment_nodes = cdiv
.get(parser)
.and_then(|n| n.as_tag()?.query_selector(parser, ".c1"))
.unwrap()
.collect::<Vec<_>>();
let comments = comment_nodes
.iter()
.map(|node_handle| {
let node = node_handle.get(parser).unwrap();
let c3 = get_first_element_by_class_name(node, parser, "c3")
.and_then(Node::as_tag)
.context("Failed to find c3")?;
let posted_node = c3.children().top()[0].get(parser).unwrap();
let posted = parse_comment_time(&posted_node.inner_text(parser), "Posted on ")
.context("Failed to parse posted date")?;
let user = get_first_child(c3, parser)
.map(|n| unescape(&n.inner_text(parser)).map(|s| s.to_string()))
.transpose()?;
let mut editable = false;
let mut vote_up_able = false;
let mut vote_up_ed = false;
let mut vote_down_able = false;
let mut vote_down_ed = false;
if let Some(c4) =
get_first_element_by_class_name(node, parser, "c4").and_then(Node::as_tag)
{
c4.children()
.top()
.iter()
.filter_map(|n| n.get(parser)?.as_tag())
.for_each(|tag| match tag.inner_text(parser).as_ref() {
"Vote+" => {
vote_up_able = true;
vote_up_ed = tag.attributes().contains("style");
}
"Vote-" => {
vote_down_able = true;
vote_down_ed = tag.attributes().contains("style");
}
"Edit" => editable = true,
_ => {}
})
};
let score = get_first_element_by_class_name(node, parser, "c5")
.and_then(|c5| {
get_first_child(c5.as_tag()?, parser)?
.inner_text(parser)
.parse()
.ok()
})
.unwrap_or(0);
let vote_state = get_first_element_by_class_name(node, parser, "c7")
.map(|c7| unescape(&c7.inner_text(parser)).map(|s| s.to_string()))
.transpose()?;
let last_edited = get_first_element_by_class_name(node, parser, "c8")
.and_then(|c8| parse_comment_time(&c8.inner_text(parser), "Last edited on "))
.unwrap_or(0);
let c6_handle = node
.find_node(parser, &mut |n| {
n.as_tag()
.is_some_and(|tag| tag.attributes().is_class_member("c6"))
})
.context("Failed to find c6")?;
let c6 = c6_handle.get(parser).and_then(Node::as_tag).unwrap();
let nodes = c6
// Should be "span[style='text-decoration:underline;']" but it's bugged in tl
// https://github.com/y21/tl/issues/46
.query_selector(parser, "span[style^='text-decoration:underline']")
.unwrap()
.collect::<Vec<_>>();
for node in nodes {
let tag = node.get_mut(parser).and_then(Node::as_tag_mut).unwrap();
*tag.name_mut() = "u".into();
tag.attributes_mut().remove("style");
}
let c6 = c6_handle.get(parser).and_then(Node::as_tag).unwrap();
let id = c6
.attributes()
.id()
.and_then(|id| id.try_as_utf8_str()?.strip_prefix("comment_")?.parse().ok())
.context("Failed to parse comment id")?;
let comment = c6.inner_html(parser).to_string();
Ok(GalleryComment {
id,
score,
editable,
voteUpAble: vote_up_able,
voteUpEd: vote_up_ed,
voteDownAble: vote_down_able,
voteDownEd: vote_down_ed,
uploader: id == 0,
voteState: vote_state,
time: posted,
user,
comment,
lastEdited: last_edited,
})
})
.collect::<Result<Vec<_>>>()?;
let parser = dom.parser();
let has_more = dom
.get_element_by_id("chd")
.and_then(|n| {
let tag = n
.get(parser)?
.children()?
.top()
.iter()
.filter_map(|n| n.get(parser)?.as_tag())
.next()?;
tag.attributes().id()
})
.is_some_and(|id| id.ne("postnewcomment"));
Ok(GalleryCommentList {
comments,
hasMore: has_more,
})
}
fn parse_preview_list(dom: &VDom, parser: &Parser) -> Option<Vec<GalleryPreview>> {
dom.get_element_by_id("gdt")?
.get(parser)?
.children()?
.top()
.iter()
.filter_map(|n| n.get(parser)?.as_tag())
.map(|a| {
let href_regex = regex!(r#"https[^s#"]+(?:s/([0-9a-f]{10})/\d+-|#page)(\d+)"#);
let caps = href_regex.captures(get_tag_attr(a, "href")?)?;
let p_token = caps.get(1).map_or("", |m| m.as_str()).to_string();
let position = caps[2].parse::<i32>().unwrap() - 1;
let div = a.query_selector(parser, "div[title]")?.next()?;
let style_regex = regex!(r#"\D+(\d+)\D+(\d+)[^(]+\(([^)]+)\)(?: -(\d+))?"#);
let caps = style_regex.captures(get_node_handle_attr(&div, parser, "style")?)?;
let url = caps[3].to_string();
let preview = if let Some(offset) = caps.get(4) {
GalleryPreview(
"V2".to_string(),
PreviewData {
url,
position,
pToken: p_token,
offsetX: offset.as_str().parse().ok(),
clipWidth: caps[1].parse().ok(),
clipHeight: caps[2].parse().ok(),
},
)
} else {
GalleryPreview(
"V1".to_string(),
PreviewData {
url,
position,
pToken: p_token,
offsetX: None,
clipWidth: None,
clipHeight: None,
},
)
};
Some(preview)
})
.collect::<Option<Vec<_>>>()
}
+24 -24
View File
@@ -10,28 +10,28 @@ use std::borrow::Cow;
use std::ops::Index;
use tl::{Node, Parser, VDom};
#[derive(Serialize, Debug)]
#[derive(Serialize, Debug, Default)]
#[allow(non_snake_case)]
struct BaseGalleryInfo {
gid: i64,
token: String,
title: String,
titleJpn: Option<String>,
thumbKey: String,
category: i32,
posted: String,
uploader: Option<String>,
disowned: bool,
rating: f32,
rated: bool,
simpleTags: Vec<String>,
pages: i32,
thumbWidth: i32,
thumbHeight: i32,
simpleLanguage: Option<String>,
favoriteSlot: i32,
favoriteName: Option<String>,
favoriteNote: Option<String>,
pub struct BaseGalleryInfo {
pub gid: i64,
pub token: String,
pub title: String,
pub titleJpn: Option<String>,
pub thumbKey: String,
pub category: i32,
pub posted: String,
pub uploader: Option<String>,
pub disowned: bool,
pub rating: f32,
pub rated: bool,
pub simpleTags: Vec<String>,
pub pages: i32,
pub thumbWidth: i32,
pub thumbHeight: i32,
pub simpleLanguage: Option<String>,
pub favoriteSlot: i32,
pub favoriteName: Option<String>,
pub favoriteNote: Option<String>,
}
#[derive(Serialize, Debug)]
@@ -42,7 +42,7 @@ pub struct GalleryListResult {
galleryInfoList: Vec<BaseGalleryInfo>,
}
fn to_category_i32(category: &str) -> i32 {
pub fn to_category_i32(category: &str) -> i32 {
match category {
"misc" => 0x1,
"doujinshi" => 0x2,
@@ -75,14 +75,14 @@ fn parse_rating(str: &str) -> f32 {
}
}
fn get_thumb_key(url: &str) -> String {
pub fn get_thumb_key(url: &str) -> String {
url.trim_start_matches(EHGT_PREFIX)
.trim_start_matches(EX_PREFIX)
.trim_start_matches("t/")
.to_string()
}
fn parse_token_and_gid(str: &str) -> Option<(i64, String)> {
pub fn parse_token_and_gid(str: &str) -> Option<(i64, String)> {
let grp =
regex!("https?://(?:exhentai.org|e-hentai.org(?:/lofi)?)/(?:g|mpv)/(\\d+)/([0-9a-f]{10})")
.captures(str)?;
@@ -1,6 +1,7 @@
#[cfg(test)]
mod tests {
use crate::parser::archive::{parse_archive_url, parse_archives_with_funds};
use crate::parser::detail::parse_gallery_detail;
use crate::parser::list::parse_info_list;
use crate::parser::profile::{parse_profile, parse_profile_url};
use reqwest::get;
@@ -52,4 +53,16 @@ mod tests {
Some("https://forums.e-hentai.org/ehgt/jdk_180.png".to_string())
);
}
#[tokio::test]
async fn test_parse_gallery_detail() {
let resp = get("https://e-hentai.org/g/530350/8b3c7e4a21/")
.await
.expect("Failed to get!");
let body = resp.text().await.expect("Failed to receive!");
let mut dom =
tl::parse(&body, ParserOptions::default().track_ids()).expect("Failed to parse HTML");
let result = parse_gallery_detail(&mut dom, &body).expect("Failed to parse gallery detail");
dbg!(result);
}
}