From 3bad1d06f40150d590cfacdf7c1dbd58db92f140 Mon Sep 17 00:00:00 2001 From: Vladimir Dostoyevskiy <99822064+VDostoyevskiy@users.noreply.github.com> Date: Sun, 12 Nov 2023 19:29:31 +0800 Subject: [PATCH] Migrate more to cronet directly (#247) * Convert GalleryListParser to parse a DirectByteBuffer * Convert FavoritesParser to parse a DirectByteBuffer * Limit buffer size --------- Co-authored-by: Lokesh Gidra --- .../com/hippo/ehviewer/client/EhEngine.kt | 26 +++++++++----- .../ehviewer/client/parser/FavoritesParser.kt | 14 ++++---- .../client/parser/GalleryListParser.kt | 34 +++++++++++-------- app/src/main/rust/src/lib.rs | 20 +++-------- app/src/main/rust/src/parser/fav.rs | 19 ++++++++--- app/src/main/rust/src/parser/home.rs | 2 +- app/src/main/rust/src/parser/list.rs | 21 +++++++++--- app/src/main/rust/src/parser/torrent.rs | 2 +- 8 files changed, 81 insertions(+), 57 deletions(-) diff --git a/app/src/main/kotlin/com/hippo/ehviewer/client/EhEngine.kt b/app/src/main/kotlin/com/hippo/ehviewer/client/EhEngine.kt index e51b795be..c7307334f 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/client/EhEngine.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/client/EhEngine.kt @@ -116,9 +116,9 @@ private fun rethrowExactly(code: Int, body: String, e: Throwable): Nothing { throw e } -val httpContentPool = DirectByteBufferPool(16, 0x20000) +val httpContentPool = DirectByteBufferPool(8, 0x80000) -suspend inline fun fetchCompat(url: String, referer: String? = null, crossinline parser: (ByteBuffer) -> T): T { +suspend inline fun fetchCompat(url: String, referer: String? = null, crossinline parser: suspend (ByteBuffer) -> T): T { return if (isCronetSupported) { cronetRequest(url, referer).execute { httpContentPool.useInstance { buffer -> @@ -213,8 +213,7 @@ object EhEngine { return ehRequest(url, referer).executeAndParsingWith(GalleryPageParser::parse) } - suspend fun getGalleryList(url: String) = ehRequest(url, EhUrl.referer) - .executeAndParsingWith(GalleryListParser::parse) + suspend fun getGalleryList(url: String) = fetchCompat(url, EhUrl.referer) { GalleryListParser.parse(it) } .apply { fillGalleryList(galleryInfoList, url, true) } .takeUnless { it.galleryInfoList.isEmpty() } ?: GalleryListParser.emptyResult @@ -230,8 +229,7 @@ object EhEngine { GalleryDetailParser.parsePreviewList(this) to GalleryDetailParser.parsePreviewPages(this) } - suspend fun getFavorites(url: String) = ehRequest(url, EhUrl.referer) - .executeAndParsingWith(FavoritesParser::parse) + suspend fun getFavorites(url: String) = fetchCompat(url, EhUrl.referer, FavoritesParser::parse) .apply { fillGalleryList(galleryInfoList, url) } suspend fun signIn(username: String, password: String): String { @@ -356,7 +354,13 @@ object EhEngine { add("ddact", catStr) gidArray.forEach { add("modifygids[]", it.toString()) } } - }.executeAndParsingWith(FavoritesParser::parse).apply { fillGalleryList(galleryInfoList, url) } + }.executeAndParsingWith { + httpContentPool.useInstance { + it.put(ByteBuffer.wrap(toByteArray())) + it.flip() + FavoritesParser.parse(it) + } + }.apply { fillGalleryList(galleryInfoList, url) } } suspend fun getGalleryPageApi( @@ -475,7 +479,13 @@ object EhEngine { if (osc) addFormDataPart("fs_covers", "on") addFormDataPart("f_sfile", "File Search") } - }.executeAndParsingWith(GalleryListParser::parse).apply { fillGalleryList(galleryInfoList, EhUrl.imageSearchUrl) } + }.executeAndParsingWith { + httpContentPool.useInstance { + it.put(ByteBuffer.wrap(toByteArray())) + it.flip() + GalleryListParser.parse(it) + } + }.apply { fillGalleryList(galleryInfoList, EhUrl.imageSearchUrl) } private suspend fun fillGalleryList(list: MutableList, url: String, filter: Boolean = false) { // Filter title and uploader diff --git a/app/src/main/kotlin/com/hippo/ehviewer/client/parser/FavoritesParser.kt b/app/src/main/kotlin/com/hippo/ehviewer/client/parser/FavoritesParser.kt index 48151fc03..a652fc3f5 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/client/parser/FavoritesParser.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/client/parser/FavoritesParser.kt @@ -2,16 +2,16 @@ package com.hippo.ehviewer.client.parser import com.hippo.ehviewer.client.exception.NotLoggedInException import com.hippo.ehviewer.client.exception.ParseException +import java.nio.ByteBuffer +import moe.tarsin.coroutines.runSuspendCatching object FavoritesParser { - suspend fun parse(body: String): Result { - if (body.contains("This page requires you to log on.

")) { - throw NotLoggedInException() - } + suspend fun parse(body: ByteBuffer): Result { val catArray = arrayOfNulls(10) - val countArray = runCatching { - parseFav(body, catArray).also { check(it.isNotEmpty()) } + val countArray = runSuspendCatching { + parseFav(body, favCat = catArray).also { check(it.isNotEmpty()) } }.getOrElse { + if (it is RuntimeException && it.message == "Not logged in!") throw NotLoggedInException() throw ParseException("Parse favorites error", it) } val result = GalleryListParser.parse(body) @@ -29,4 +29,4 @@ object FavoritesParser { } } -private external fun parseFav(body: String, favCat: Array): IntArray +private external fun parseFav(body: ByteBuffer, limit: Int = body.limit(), favCat: Array): IntArray diff --git a/app/src/main/kotlin/com/hippo/ehviewer/client/parser/GalleryListParser.kt b/app/src/main/kotlin/com/hippo/ehviewer/client/parser/GalleryListParser.kt index a7c70392f..4aa907c4d 100644 --- a/app/src/main/kotlin/com/hippo/ehviewer/client/parser/GalleryListParser.kt +++ b/app/src/main/kotlin/com/hippo/ehviewer/client/parser/GalleryListParser.kt @@ -8,25 +8,29 @@ import com.hippo.ehviewer.client.data.GalleryInfo.Companion.LOCAL_FAVORITED import com.hippo.ehviewer.client.data.GalleryInfo.Companion.NOT_FAVORITED import com.hippo.ehviewer.client.exception.EhException import com.hippo.ehviewer.client.exception.ParseException +import java.nio.ByteBuffer +import moe.tarsin.coroutines.runSuspendCatching object GalleryListParser { - suspend fun parse(body: String) = runCatching { - parseGalleryInfoList(body).apply { - galleryInfoList.onEach { - if (it.favoriteSlot == NOT_FAVORITED && EhDB.containLocalFavorites(it.gid)) { - it.favoriteSlot = LOCAL_FAVORITED - } - it.generateSLang() + suspend fun parse(body: ByteBuffer) = runSuspendCatching { + parseGalleryInfoList(body) + }.onFailure { + if (it is RuntimeException) { + if (it.message == "No watched tags!") { + throw EhException(R.string.gallery_list_empty_hit_subscription) + } + if (it.message == "No hits found!") { + throw EhException(R.string.gallery_list_empty_hit) } } - }.getOrElse { - if (body.contains("

You do not have any watched tags")) { - throw EhException(R.string.gallery_list_empty_hit_subscription) - } - if (body.contains("No hits found

")) { - throw EhException(R.string.gallery_list_empty_hit) - } throw ParseException("Can't parse gallery list", it) + }.getOrThrow().apply { + galleryInfoList.onEach { + if (it.favoriteSlot == NOT_FAVORITED && EhDB.containLocalFavorites(it.gid)) { + it.favoriteSlot = LOCAL_FAVORITED + } + it.generateSLang() + } } val emptyResult = GalleryListResult(null, null, arrayListOf()) @@ -38,4 +42,4 @@ class GalleryListResult @Keep constructor( val galleryInfoList: ArrayList, ) -private external fun parseGalleryInfoList(e: String): GalleryListResult +private external fun parseGalleryInfoList(body: ByteBuffer, size: Int = body.limit()): GalleryListResult diff --git a/app/src/main/rust/src/lib.rs b/app/src/main/rust/src/lib.rs index 8216c79a7..fbebc8ba5 100644 --- a/app/src/main/rust/src/lib.rs +++ b/app/src/main/rust/src/lib.rs @@ -13,13 +13,13 @@ extern crate regex_lite; extern crate tl; use android_logger::Config; -use jnix::jni::objects::{JByteBuffer, JString}; +use jnix::jni::objects::JByteBuffer; use jnix::jni::sys::{jint, JavaVM, JNI_VERSION_1_6}; use jnix::JnixEnv; use log::LevelFilter; use std::ffi::c_void; use std::ptr::slice_from_raw_parts; -use std::str::from_utf8; +use std::str::from_utf8_unchecked; use tl::{Bytes, Node, NodeHandle, Parser, VDom}; #[macro_export] @@ -62,28 +62,18 @@ where handle.get(parser) } -fn parse_jni_string(env: &mut JnixEnv, str: &JString, mut f: F) -> Option -where - F: FnMut(&VDom, &Parser, &JnixEnv) -> Option, -{ - let html = env.get_string(*str).ok()?; - let dom = tl::parse(html.to_str().ok()?, tl::ParserOptions::default()).ok()?; - let parser = dom.parser(); - f(&dom, parser, env) -} - fn parse_bytebuffer(env: &mut JnixEnv, str: JByteBuffer, limit: jint, mut f: F) -> Option where - F: FnMut(&VDom, &Parser, &JnixEnv) -> Option, + F: FnMut(&VDom, &Parser, &JnixEnv, &str) -> Option, { let ptr = env.get_direct_buffer_address(str).ok()?; let html = unsafe { let buff = slice_from_raw_parts(ptr, limit as usize); - from_utf8(&*buff).ok()? + from_utf8_unchecked(&*buff) }; let dom = tl::parse(html, tl::ParserOptions::default()).ok()?; let parser = dom.parser(); - f(&dom, parser, env) + f(&dom, parser, env, html) } fn get_node_handle_attr<'a>( diff --git a/app/src/main/rust/src/parser/fav.rs b/app/src/main/rust/src/parser/fav.rs index a06e5d48e..dc8bf11de 100644 --- a/app/src/main/rust/src/parser/fav.rs +++ b/app/src/main/rust/src/parser/fav.rs @@ -1,20 +1,29 @@ use apply::Also; use catch_panic::catch_panic; use jni_fn::jni_fn; -use jnix::jni::objects::{JClass, JString}; -use jnix::jni::sys::{jintArray, jobjectArray}; +use jnix::jni::objects::{JByteBuffer, JClass}; +use jnix::jni::sys::{jint, jintArray, jobjectArray}; use jnix::jni::JNIEnv; use jnix::JnixEnv; -use parse_jni_string; +use parse_bytebuffer; use quick_xml::escape::unescape; #[no_mangle] #[catch_panic(default = "std::ptr::null_mut()")] #[allow(non_snake_case)] #[jni_fn("com.hippo.ehviewer.client.parser.FavoritesParserKt")] -pub fn parseFav(env: JNIEnv, _class: JClass, input: JString, str: jobjectArray) -> jintArray { +pub fn parseFav( + env: JNIEnv, + _class: JClass, + input: JByteBuffer, + limit: jint, + str: jobjectArray, +) -> jintArray { let mut env = JnixEnv { env }; - let vec = parse_jni_string(&mut env, &input, |dom, parser, env| { + let vec = parse_bytebuffer(&mut env, input, limit, |dom, parser, env, html| { + if html.contains("This page requires you to log on.

") { + panic!("Not logged in!") + } let vec: Vec = dom .get_elements_by_class_name("fp") .enumerate() diff --git a/app/src/main/rust/src/parser/home.rs b/app/src/main/rust/src/parser/home.rs index d09e40f07..9fc3d5309 100644 --- a/app/src/main/rust/src/parser/home.rs +++ b/app/src/main/rust/src/parser/home.rs @@ -22,7 +22,7 @@ pub struct Limits { #[jni_fn("com.hippo.ehviewer.client.parser.HomeParserKt")] pub fn parseLimit(env: JNIEnv, _class: JClass, input: JByteBuffer, limit: jint) -> jobject { let mut env = JnixEnv { env }; - parse_bytebuffer(&mut env, input, limit, |dom, parser, _env| { + parse_bytebuffer(&mut env, input, limit, |dom, parser, _, _| { let iter = get_vdom_first_element_by_class_name(dom, "homebox")? .as_tag()? .query_selector(parser, "strong")?; diff --git a/app/src/main/rust/src/parser/list.rs b/app/src/main/rust/src/parser/list.rs index 482da675f..a546fce25 100644 --- a/app/src/main/rust/src/parser/list.rs +++ b/app/src/main/rust/src/parser/list.rs @@ -1,11 +1,11 @@ use catch_panic::catch_panic; use jni_fn::jni_fn; -use jnix::jni::objects::{JClass, JString}; -use jnix::jni::sys::jobject; +use jnix::jni::objects::{JByteBuffer, JClass}; +use jnix::jni::sys::{jint, jobject}; use jnix::jni::JNIEnv; use jnix::{IntoJava, JnixEnv}; use jnix_macros::IntoJava; -use parse_jni_string; +use parse_bytebuffer; use quick_xml::escape::unescape; use std::borrow::Cow; use std::ops::Index; @@ -197,9 +197,20 @@ fn parse_gallery_info(node: &Node, parser: &Parser) -> Option { #[catch_panic(default = "std::ptr::null_mut()")] #[allow(non_snake_case)] #[jni_fn("com.hippo.ehviewer.client.parser.GalleryListParserKt")] -pub fn parseGalleryInfoList(env: JNIEnv, _class: JClass, input: JString) -> jobject { +pub fn parseGalleryInfoList( + env: JNIEnv, + _class: JClass, + buffer: JByteBuffer, + limit: jint, +) -> jobject { let mut env = JnixEnv { env }; - parse_jni_string(&mut env, &input, |dom, parser, _env| { + parse_bytebuffer(&mut env, buffer, limit, |dom, parser, _, str| { + if str.contains("

You do not have any watched tags") { + panic!("No watched tags!") + } + if str.contains("No hits found

") { + panic!("No hits found!") + } let itg = get_vdom_first_element_by_class_name(dom, "itg")?; let children = itg.children()?; let iter = children.top().iter(); diff --git a/app/src/main/rust/src/parser/torrent.rs b/app/src/main/rust/src/parser/torrent.rs index f422c20aa..ec1a4e699 100644 --- a/app/src/main/rust/src/parser/torrent.rs +++ b/app/src/main/rust/src/parser/torrent.rs @@ -27,7 +27,7 @@ pub struct Torrent { #[jni_fn("com.hippo.ehviewer.client.parser.TorrentParserKt")] pub fn parseTorrent(env: JNIEnv, _class: JClass, buffer: JByteBuffer, limit: jint) -> jobject { let mut env = JnixEnv { env }; - parse_bytebuffer(&mut env, buffer, limit, |dom, parser, _env| { + parse_bytebuffer(&mut env, buffer, limit, |dom, parser, _, _| { Some(dom.query_selector("table")?.filter_map(|e| { let html = e.get(parser)?.inner_html(parser); if html.contains("Expunged") {