Migrate GalleryDetailParser to Rust Part 3/N

This commit is contained in:
FooIbar committed 2025-06-13 22:25:47 +08:00
1 parent 1feca411b6
commit 4c359e50f8
14 files changed
+183 -109

No files matched your search

@@ -245,9 +245,7 @@ object EhEngine {
}
}
suspend fun getPreviewList(url: String) = ehRequest(url, EhUrl.referer).fetchUsingAsText {
GalleryDetailParser.parsePreviewList(this)
}
suspend fun getPreviewList(url: String) = ehRequest(url, EhUrl.referer).fetchUsingAsByteBuffer(GalleryDetailParser::parsePreviewsRust)
suspend fun getFavorites(url: String) = ehRequest(url, EhUrl.referer).fetchUsingAsByteBuffer(FavoritesParser::parse)
.apply { galleryInfoList.fillInfo(url) }
@@ -426,7 +424,7 @@ object EhEngine {
put("tags", tags)
put("vote", vote)
}
}.fetchUsingAsText(VoteTagParser::parse)
}.fetchUsingAsByteBuffer(VoteTagParser::parse)
suspend fun getGalleryToken(gid: Long, gtoken: String, page: Int) = ehRequest(EhUrl.apiUrl, EhUrl.referer, EhUrl.origin) {
jsonBody {
@@ -19,6 +19,7 @@ import com.hippo.ehviewer.client.getThumbKey
import com.hippo.ehviewer.client.getV2PreviewKey
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.cbor.CborArray
@Serializable
sealed interface GalleryPreview {
@@ -50,3 +51,10 @@ data class V2GalleryPreview(
) : GalleryPreview {
override val imageKey get() = getV2PreviewKey(url)
}
@Serializable
@CborArray
class GalleryPreviewList(
val previews: List<GalleryPreview>,
val total: Int,
)
@@ -22,6 +22,7 @@ import com.hippo.ehviewer.client.data.BaseGalleryInfo
import com.hippo.ehviewer.client.data.GalleryComment
import com.hippo.ehviewer.client.data.GalleryCommentList
import com.hippo.ehviewer.client.data.GalleryDetail
import com.hippo.ehviewer.client.data.GalleryPreviewList
import com.hippo.ehviewer.client.data.GalleryTag
import com.hippo.ehviewer.client.data.GalleryTagGroup
import com.hippo.ehviewer.client.data.PowerStatus
@@ -404,21 +405,28 @@ object GalleryDetailParser {
}
fun parseRust(body: ByteBuffer) = Either.catch {
unmarshalParsingAs<Result>(body, ::parseGalleryDetail)
unmarshalParsingAs<Result>(body, ::nativeParse)
}.getOrElse {
throw ParseException("Failed to parse gallery detail", it)
}
fun parseCommentsRust(body: ByteBuffer) = Either.catch {
unmarshalParsingAs<GalleryCommentList>(body, ::parseGalleryComments)
unmarshalParsingAs<GalleryCommentList>(body, ::nativeParseComments)
}.getOrElse {
throw ParseException("Failed to parse gallery comments", it)
throw ParseException("Failed to parse comments", it)
}
fun parsePreviewsRust(body: ByteBuffer) = Either.catch {
unmarshalParsingAs<GalleryPreviewList>(body, ::nativeParsePreviews)
}.getOrElse {
throw ParseException("Failed to parse previews", it)
}
@Serializable
@CborArray
class Result(val detail: GalleryDetail, val event: String?)
private external fun parseGalleryDetail(body: ByteBuffer, size: Int = body.limit()): Int
private external fun parseGalleryComments(body: ByteBuffer, size: Int = body.limit()): Int
private external fun nativeParse(body: ByteBuffer, size: Int = body.limit()): Int
private external fun nativeParseComments(body: ByteBuffer, size: Int = body.limit()): Int
private external fun nativeParsePreviews(body: ByteBuffer, size: Int = body.limit()): Int
}
@@ -17,15 +17,18 @@
*/
package com.hippo.ehviewer.client.parser
import arrow.core.Either.Companion.catch
import arrow.core.getOrElse
import com.hippo.ehviewer.client.data.GalleryTagGroup
import com.hippo.ehviewer.client.parseAs
import org.jsoup.Jsoup
import com.hippo.ehviewer.client.exception.ParseException
import java.nio.ByteBuffer
object VoteTagParser {
// {"error":"The tag \"neko\" is not allowed. Use character:neko or artist:neko"}
fun parse(body: String): List<GalleryTagGroup> = body.parseAs<VoteTagResult>().let { (error, tagpane) ->
error?.let { error(error) }
checkNotNull(tagpane)
GalleryDetailParser.parseTagGroups(Jsoup.parse("<div id=\"taglist\">$tagpane</div>"))
fun parse(body: ByteBuffer) = catch {
unmarshalParsingAs<List<GalleryTagGroup>>(body, ::nativeParse)
}.getOrElse {
throw ParseException("Failed to parse vote tag result", it)
}
private external fun nativeParse(body: ByteBuffer, size: Int = body.limit()): Int
}
@@ -29,11 +29,10 @@ import com.hippo.ehviewer.client.EhUrl.getGalleryMultiPageViewerUrl
import com.hippo.ehviewer.client.EhUrl.referer
import com.hippo.ehviewer.client.EhUtils
import com.hippo.ehviewer.client.data.GalleryInfo
import com.hippo.ehviewer.client.data.GalleryPreview
import com.hippo.ehviewer.client.ehRequest
import com.hippo.ehviewer.client.exception.QuotaExceededException
import com.hippo.ehviewer.client.fetchUsingAsText
import com.hippo.ehviewer.client.parser.GalleryDetailParser.parsePages
import com.hippo.ehviewer.client.parser.GalleryDetailParser.parsePreviewList
import com.hippo.ehviewer.client.parser.GalleryMultiPageViewerPTokenParser
import com.hippo.ehviewer.util.displayString
import com.hippo.files.find
@@ -318,27 +317,24 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS
}
?: readFromCache(galleryInfo.gid)?.takeIf { it.gid == galleryInfo.gid && it.token == galleryInfo.token }
private fun readPreviews(body: String, index: Int, spiderInfo: SpiderInfo) {
val previewList = parsePreviewList(body)
private fun readPreviews(previews: List<GalleryPreview>, index: Int, spiderInfo: SpiderInfo) {
if (index == 0) {
spiderInfo.previewPerPage = previewList.size
spiderInfo.previewPerPage = previews.size
} else {
spiderInfo.previewPerPage = previewList[0].position / index
spiderInfo.previewPerPage = previews[0].position / index
}
previewList.forEach {
previews.forEach {
if (it.pToken.isNotEmpty()) {
spiderInfo.pTokenMap[it.position] = it.pToken
}
}
}
private suspend fun readSpiderInfoFromInternet() = ehRequest(
private suspend fun readSpiderInfoFromInternet() = EhEngine.getPreviewList(
getGalleryDetailUrl(galleryInfo.gid, galleryInfo.token, 0, false),
referer,
).fetchUsingAsText {
val pages = parsePages(this)
val spiderInfo = SpiderInfo(galleryInfo.gid, galleryInfo.token, pages)
readPreviews(this, 0, spiderInfo)
).run {
val spiderInfo = SpiderInfo(galleryInfo.gid, galleryInfo.token, total)
readPreviews(previews, 0, spiderInfo)
spiderInfo
}
@@ -376,8 +372,8 @@ class SpiderQueen private constructor(val galleryInfo: GalleryInfo) : CoroutineS
false,
)
return runSuspendCatching {
ehRequest(url, referer).fetchUsingAsText {
readPreviews(this, previewIndex, spiderInfo)
EhEngine.getPreviewList(url).run {
readPreviews(previews, previewIndex, spiderInfo)
spiderInfo.pTokenMap[index]
}
}.getOrElse {
@@ -804,7 +804,7 @@ private fun GalleryDetail.collectPreviewItems() = rememberInVM(previewList) {
(up..end).filterNot { it in previewPagesMap }.map { it / pageSize }.toSet()
.parMap(concurrency = Settings.multiThreadDownload) { page ->
val url = EhUrl.getGalleryDetailUrl(gid, token, page, false)
EhEngine.getPreviewList(url)
EhEngine.getPreviewList(url).previews
}.flattenForEach {
previewPagesMap[it.position] = it
if (Settings.preloadThumbAggressively) {
+1
View File
@@ -281,6 +281,7 @@ dependencies = [
"rxing",
"serde",
"serde_cbor",
"serde_json",
"tl",
"tokio",
]
+1
View File
@@ -21,6 +21,7 @@ tl = { git = "https://github.com/y21/tl.git", rev = "13ed128e35ce04c4f16c41cb1e3
log = "0.4"
android_logger = { default-features = false, version = "0.15" }
serde_cbor = "0.11"
serde_json = "1.0"
serde = { version = "1.0", features = ["derive"] }
anyhow = "1.0"
image = { default-features = false, version = "0.25" }
+34 -7
View File
@@ -1,9 +1,11 @@
#![cfg(feature = "jvm")]
use crate::EhError;
use crate::parser::api::parse_vote_tag;
use crate::parser::archive::{parse_archive_url, parse_archives, parse_archives_with_funds};
use crate::parser::config::parse_fav_cat;
use crate::parser::detail::{parse_comments, parse_event_pane, parse_gallery_detail};
use crate::parser::detail::{parse_pages, parse_preview_list};
use crate::parser::fav::parse_fav;
use crate::parser::home::parse_limit;
use crate::parser::list::parse_info_list;
@@ -45,7 +47,7 @@ pub fn parseGalleryInfoList(mut env: JNIEnv, _: JClass, buffer: JByteBuffer, lim
}
#[jni_fn("com.hippo.ehviewer.client.parser.GalleryDetailParser")]
pub fn parseGalleryDetail(mut env: JNIEnv, _: JClass, buffer: JByteBuffer, limit: jint) -> jint {
pub fn nativeParse(mut env: JNIEnv, _: JClass, buffer: JByteBuffer, limit: jint) -> jint {
let options = ParserOptions::default().track_ids();
parse_marshal_inplace_with_options(&mut env, buffer, limit, options, |dom, html| {
parse_gallery_detail(dom, html).map(|detail| (detail, parse_event_pane(dom, dom.parser())))
@@ -53,13 +55,29 @@ pub fn parseGalleryDetail(mut env: JNIEnv, _: JClass, buffer: JByteBuffer, limit
}
#[jni_fn("com.hippo.ehviewer.client.parser.GalleryDetailParser")]
pub fn parseGalleryComments(mut env: JNIEnv, _: JClass, buffer: JByteBuffer, limit: jint) -> jint {
pub fn nativeParseComments(mut env: JNIEnv, _: JClass, buffer: JByteBuffer, limit: jint) -> jint {
let options = ParserOptions::default().track_ids();
parse_marshal_inplace_with_options(&mut env, buffer, limit, options, |dom, _| {
parse_comments(dom)
})
}
#[jni_fn("com.hippo.ehviewer.client.parser.GalleryDetailParser")]
pub fn nativeParsePreviews(mut env: JNIEnv, _: JClass, buffer: JByteBuffer, limit: jint) -> jint {
let options = ParserOptions::default().track_ids();
parse_marshal_inplace_with_options(&mut env, buffer, limit, options, |dom, _| {
Ok((
parse_preview_list(dom, dom.parser())?,
parse_pages(dom, dom.parser())?,
))
})
}
#[jni_fn("com.hippo.ehviewer.client.parser.VoteTagParser")]
pub fn nativeParse(mut env: JNIEnv, _: JClass, buffer: JByteBuffer, limit: jint) -> jint {
parse_raw_marshal_inplace(&mut env, buffer, limit, parse_vote_tag)
}
#[jni_fn("com.hippo.ehviewer.client.parser.EventPaneParser")]
pub fn parseEventPane(mut env: JNIEnv, _class: JClass, input: JByteBuffer, limit: jint) -> jint {
parse_marshal_inplace(&mut env, input, limit, |dom, _| {
@@ -190,16 +208,25 @@ pub fn parse_marshal_inplace_with_options<F, R>(
where
F: Fn(&mut VDom, &str) -> Result<R>,
R: Serialize,
{
parse_raw_marshal_inplace(env, str, limit, |html| {
let mut dom = tl::parse(html, options)?;
ensure!(dom.version().is_some(), EhError::Error(html.to_string()));
f(&mut dom, html)
})
}
pub fn parse_raw_marshal_inplace<F, R>(env: &mut JNIEnv, str: JByteBuffer, limit: jint, f: F) -> i32
where
F: Fn(&str) -> Result<R>,
R: Serialize,
{
jni_throwing(env, |env| {
let buffer = deref_mut_direct_bytebuffer(env, str)?;
let value = {
// SAFETY: ktor client ensure html content is valid utf-8.
let html = unsafe { from_utf8_unchecked(&buffer[..limit as usize]) };
let mut dom = tl::parse(html, options)?;
ensure!(dom.version().is_some(), EhError::Error(html.to_string()));
f(&mut dom, html)?
let body = unsafe { from_utf8_unchecked(&buffer[..limit as usize]) };
f(body)?
};
let mut cursor = Cursor::new(buffer);
serde_cbor::to_writer(&mut cursor, &value)?;
+1
View File
@@ -1,3 +1,4 @@
pub mod api;
pub mod archive;
pub mod config;
pub mod detail;
+18
View File
@@ -0,0 +1,18 @@
use crate::EhError;
use crate::parser::detail::{GalleryTagGroup, parse_tag_groups};
use anyhow::{Context, Result, bail};
use serde_json::Value;
use tl::ParserOptions;
pub fn parse_vote_tag(str: &str) -> Result<Vec<GalleryTagGroup>> {
let result = serde_json::from_str::<Value>(str)?;
if let Some(error) = result.get("error").and_then(Value::as_str) {
bail!(EhError::Error(error.to_string()))
};
let tagpane = result
.get("tagpane")
.and_then(Value::as_str)
.context("Failed to get tagpane")?;
let dom = tl::parse(tagpane, ParserOptions::default())?;
parse_tag_groups(&dom, dom.parser(), true)
}
+74 -70
View File
@@ -55,7 +55,7 @@ pub struct GalleryTagGroup {
pub tags: Vec<GalleryTag>,
}
#[derive(Serialize, Debug)]
#[derive(Serialize, Debug, Default)]
#[allow(non_snake_case)]
pub struct GalleryComment {
pub id: i64,
@@ -80,7 +80,7 @@ pub struct GalleryCommentList {
pub hasMore: bool,
}
#[derive(Serialize, Debug)]
#[derive(Serialize, Debug, Default)]
#[allow(non_snake_case)]
pub struct PreviewData {
pub url: String,
@@ -248,9 +248,9 @@ pub fn parse_gallery_detail(dom: &mut VDom, body: &str) -> Result<GalleryDetail>
}
let mut gallery_detail = GalleryDetail {
tagGroups: parse_tag_groups(dom, parser).context("Failed to parse tag groups")?,
previewList: parse_preview_list(dom, parser).context("Failed to parse preview list")?,
comments: parse_comments(dom).context("Failed to parse comments")?,
tagGroups: parse_tag_groups(dom, parser, false)?,
previewList: parse_preview_list(dom, parser)?,
comments: parse_comments(dom)?,
..Default::default()
};
@@ -266,11 +266,23 @@ pub fn parse_event_pane(dom: &VDom, parser: &Parser) -> Option<String> {
.map(|n| n.inner_html(parser).to_string())
}
fn parse_tag_groups(dom: &VDom, parser: &Parser) -> Option<Vec<GalleryTagGroup>> {
dom.get_element_by_id("taglist")?
.get(parser)?
.as_tag()?
.query_selector(parser, "tr")?
pub fn parse_tag_groups(
dom: &VDom,
parser: &Parser,
is_root: bool,
) -> Result<Vec<GalleryTagGroup>> {
let nodes: Vec<_> = if is_root {
dom.query_selector("tr").unwrap().collect()
} else {
dom.get_element_by_id("taglist")
.and_then(|n| n.get(parser)?.as_tag())
.context("Failed to find taglist")?
.query_selector(parser, "tr")
.unwrap()
.collect()
};
nodes
.iter()
.filter_map(|n| n.get(parser)?.children())
.map(|children| {
let mut iter = children
@@ -317,6 +329,7 @@ fn parse_tag_groups(dom: &VDom, parser: &Parser) -> Option<Vec<GalleryTagGroup>>
Some(GalleryTagGroup { namespace, tags })
})
.collect::<Option<Vec<_>>>()
.context("Failed to parse tag groups")
}
fn parse_comment_time(str: &str, prefix: &str) -> Option<i64> {
@@ -363,22 +376,18 @@ pub fn parse_comments(dom: &mut VDom) -> Result<GalleryCommentList> {
.map(|node_handle| {
let parser = dom.parser_mut();
let node = node_handle.get(parser).unwrap();
let mut c = GalleryComment::default();
let c3 = get_first_element_by_class_name(node, parser, "c3")
.and_then(Node::as_tag)
.context("Failed to find c3")?;
let posted_node = c3.children().top()[0].get(parser).unwrap();
let posted = parse_comment_time(&posted_node.inner_text(parser), "Posted on ")
c.time = parse_comment_time(&posted_node.inner_text(parser), "Posted on ")
.context("Failed to parse posted date")?;
let user = get_first_child(c3, parser)
c.user = get_first_child(c3, parser)
.map(|n| unescape(&n.inner_text(parser)).map(|s| s.to_string()))
.transpose()?;
let mut editable = false;
let mut vote_up_able = false;
let mut vote_up_ed = false;
let mut vote_down_able = false;
let mut vote_down_ed = false;
if let Some(c4) =
get_first_element_by_class_name(node, parser, "c4").and_then(Node::as_tag)
{
@@ -388,19 +397,19 @@ pub fn parse_comments(dom: &mut VDom) -> Result<GalleryCommentList> {
.filter_map(|n| n.get(parser)?.as_tag())
.for_each(|tag| match tag.inner_text(parser).as_ref() {
"Vote+" => {
vote_up_able = true;
vote_up_ed = tag.attributes().contains("style");
c.voteUpAble = true;
c.voteUpEd = tag.attributes().contains("style");
}
"Vote-" => {
vote_down_able = true;
vote_down_ed = tag.attributes().contains("style");
c.voteDownAble = true;
c.voteDownEd = tag.attributes().contains("style");
}
"Edit" => editable = true,
"Edit" => c.editable = true,
_ => {}
})
};
let score = get_first_element_by_class_name(node, parser, "c5")
c.score = get_first_element_by_class_name(node, parser, "c5")
.and_then(|c5| {
get_first_child(c5.as_tag()?, parser)?
.inner_text(parser)
@@ -409,11 +418,11 @@ pub fn parse_comments(dom: &mut VDom) -> Result<GalleryCommentList> {
})
.unwrap_or(0);
let vote_state = get_first_element_by_class_name(node, parser, "c7")
c.voteState = get_first_element_by_class_name(node, parser, "c7")
.map(|c7| unescape(&c7.inner_text(parser)).map(|s| s.to_string()))
.transpose()?;
let last_edited = get_first_element_by_class_name(node, parser, "c8")
c.lastEdited = get_first_element_by_class_name(node, parser, "c8")
.and_then(|c8| parse_comment_time(&c8.inner_text(parser), "Last edited on "))
.unwrap_or(0);
@@ -434,28 +443,15 @@ pub fn parse_comments(dom: &mut VDom) -> Result<GalleryCommentList> {
tag.attributes_mut().remove("style");
}
let c6 = c6_handle.get(parser).and_then(Node::as_tag).unwrap();
let id = c6
c.id = c6
.attributes()
.id()
.and_then(|id| id.try_as_utf8_str()?.strip_prefix("comment_")?.parse().ok())
.context("Failed to parse comment id")?;
let comment = c6.inner_html(parser).to_string();
c.uploader = c.id == 0;
c.comment = c6.inner_html(parser).to_string();
Ok(GalleryComment {
id,
score,
editable,
voteUpAble: vote_up_able,
voteUpEd: vote_up_ed,
voteDownAble: vote_down_able,
voteDownEd: vote_down_ed,
uploader: id == 0,
voteState: vote_state,
time: posted,
user,
comment,
lastEdited: last_edited,
})
Ok(c)
})
.collect::<Result<Vec<_>>>()?;
@@ -465,10 +461,28 @@ pub fn parse_comments(dom: &mut VDom) -> Result<GalleryCommentList> {
})
}
fn parse_preview_list(dom: &VDom, parser: &Parser) -> Option<Vec<GalleryPreview>> {
dom.get_element_by_id("gdt")?
.get(parser)?
.children()?
pub fn parse_pages(dom: &VDom, parser: &Parser) -> Result<i32> {
dom.get_element_by_id("gdd")
.and_then(|gdd| {
gdd.get(parser)?
.as_tag()?
.query_selector(parser, "td.gdt2")?
.find_map(|n| {
n.get(parser)?.children()?.top()[0]
.get(parser)?
.inner_text(parser)
.strip_suffix(" pages")?
.parse()
.ok()
})
})
.context("Failed to parse pages")
}
pub fn parse_preview_list(dom: &VDom, parser: &Parser) -> Result<Vec<GalleryPreview>> {
dom.get_element_by_id("gdt")
.and_then(|n| n.get(parser)?.children())
.context("Failed to find gdt")?
.top()
.iter()
.filter_map(|n| n.get(parser)?.as_tag())
@@ -482,33 +496,23 @@ fn parse_preview_list(dom: &VDom, parser: &Parser) -> Option<Vec<GalleryPreview>
let style_regex = regex!(r#"\D+(\d+)\D+(\d+)[^(]+\(([^)]+)\)(?: -(\d+))?"#);
let caps = style_regex.captures(get_node_handle_attr(&div, parser, "style")?)?;
let url = caps[3].to_string();
let preview = if let Some(offset) = caps.get(4) {
GalleryPreview(
"V2".to_string(),
PreviewData {
url,
position,
pToken: p_token,
offsetX: offset.as_str().parse().ok(),
clipWidth: caps[1].parse().ok(),
clipHeight: caps[2].parse().ok(),
},
)
let mut data = PreviewData {
url,
position,
pToken: p_token,
..Default::default()
};
let ty = if let Some(offset) = caps.get(4) {
data.offsetX = offset.as_str().parse().ok();
data.clipWidth = caps[1].parse().ok();
data.clipHeight = caps[2].parse().ok();
"V2"
} else {
GalleryPreview(
"V1".to_string(),
PreviewData {
url,
position,
pToken: p_token,
offsetX: None,
clipWidth: None,
clipHeight: None,
},
)
"V1"
};
Some(preview)
Some(GalleryPreview(ty.to_string(), data))
})
.collect::<Option<Vec<_>>>()
.context("Failed to parse preview list")
}
@@ -1,5 +1,6 @@
#[cfg(test)]
mod tests {
use crate::parser::api::parse_vote_tag;
use crate::parser::archive::{parse_archive_url, parse_archives_with_funds};
use crate::parser::detail::parse_gallery_detail;
use crate::parser::list::parse_info_list;
@@ -65,4 +66,11 @@ mod tests {
let result = parse_gallery_detail(&mut dom, &body).expect("Failed to parse gallery detail");
dbg!(result);
}
#[test]
fn test_parse_tag_gallery() {
let json = fs::read_to_string("test_data/vote_tag.json").expect("Failed to read json file");
let result = parse_vote_tag(&json).expect_err("Should fail to parse");
dbg!(result);
}
}
@@ -0,0 +1 @@
{"error":"The tag \"neko\" has been blocked for use. Did you mean \"character:neko\" or \"artist:neko\" ?","tagpane":"No tags have been added for this gallery yet."}