Move to Ktor Cookie handle, Part 2 (#419)

* Migration

* Migration

* Migration

* Migration

* Make EhCookieStore a CookiesStorage

* Drop CookieDB migration

* Drop one Okhttp cookie usage

* Drop one Okhttp cookie usage

* Remove typealias

* Move EhTagDatabase to use ktor

* Make cronet just use Dispatchers.IO

* Make all okhttpClient internally used

* Cleanup

* Refine

---------

Co-authored-by: GG Bond <whatever@tagkong.com>
This commit is contained in:
Vladimir DostoyevskiyandGG Bond authored and GitHub committed 2023-12-02 05:42:02 +00:00
1 parent 52f314f6b0
commit 0fdbfbb1ae
22 files changed
+177 -546

No files matched your search

@@ -1,88 +0,0 @@
{
"formatVersion": 1,
"database": {
"version": 1,
"identityHash": "c801ee62729ba20a203b150211234d5c",
"entities": [
{
"tableName": "OK_HTTP_3_COOKIE",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`NAME` TEXT, `VALUE` TEXT, `EXPIRES_AT` INTEGER NOT NULL, `DOMAIN` TEXT, `PATH` TEXT, `SECURE` INTEGER NOT NULL, `HTTP_ONLY` INTEGER NOT NULL, `PERSISTENT` INTEGER NOT NULL, `HOST_ONLY` INTEGER NOT NULL, `_id` INTEGER NOT NULL, PRIMARY KEY(`_id`))",
"fields": [
{
"fieldPath": "name",
"columnName": "NAME",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "value",
"columnName": "VALUE",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "expiresAt",
"columnName": "EXPIRES_AT",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "domain",
"columnName": "DOMAIN",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "path",
"columnName": "PATH",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "secure",
"columnName": "SECURE",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "httpOnly",
"columnName": "HTTP_ONLY",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "persistent",
"columnName": "PERSISTENT",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "hostOnly",
"columnName": "HOST_ONLY",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "id",
"columnName": "_id",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"_id"
]
},
"indices": [],
"foreignKeys": []
}
],
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'c801ee62729ba20a203b150211234d5c')"
]
}
}
@@ -1,88 +0,0 @@
{
"formatVersion": 1,
"database": {
"version": 2,
"identityHash": "1b80cd29939b0f43934721f1289ba94d",
"entities": [
{
"tableName": "OK_HTTP_3_COOKIE",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`NAME` TEXT NOT NULL, `VALUE` TEXT NOT NULL, `EXPIRES_AT` INTEGER NOT NULL, `DOMAIN` TEXT NOT NULL, `PATH` TEXT NOT NULL, `SECURE` INTEGER NOT NULL, `HTTP_ONLY` INTEGER NOT NULL, `PERSISTENT` INTEGER NOT NULL, `HOST_ONLY` INTEGER NOT NULL, `_id` INTEGER, PRIMARY KEY(`_id`))",
"fields": [
{
"fieldPath": "name",
"columnName": "NAME",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "value",
"columnName": "VALUE",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "expiresAt",
"columnName": "EXPIRES_AT",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "domain",
"columnName": "DOMAIN",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "path",
"columnName": "PATH",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "secure",
"columnName": "SECURE",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "httpOnly",
"columnName": "HTTP_ONLY",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "persistent",
"columnName": "PERSISTENT",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "hostOnly",
"columnName": "HOST_ONLY",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "id",
"columnName": "_id",
"affinity": "INTEGER",
"notNull": false
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"_id"
]
},
"indices": [],
"foreignKeys": []
}
],
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '1b80cd29939b0f43934721f1289ba94d')"
]
}
}
@@ -13,7 +13,6 @@ import kotlin.coroutines.resumeWithException
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.RequestBody
import okio.Buffer
import okio.Path.Companion.toOkioPath
@@ -64,7 +63,7 @@ class CronetRequest {
inline fun cronetRequest(url: String, referer: String? = null, origin: String? = null, conf: UrlRequest.Builder.() -> Unit = {}) = CronetRequest().apply {
request = cronetHttpClient.newUrlRequestBuilder(url, callback, cronetHttpClientExecutor).apply {
addHeader("Cookie", EhCookieStore.getCookieHeader(url.toHttpUrl()))
addHeader("Cookie", EhCookieStore.getCookieHeader(url))
addHeader("Accept", CHROME_ACCEPT)
addHeader("Accept-Language", CHROME_ACCEPT_LANGUAGE)
referer?.let { addHeader("Referer", it) }
@@ -34,12 +34,10 @@ import com.hippo.ehviewer.coil.installCronetHttpUriFetcher
import com.hippo.ehviewer.dailycheck.checkDawn
import com.hippo.ehviewer.dao.SearchDatabase
import com.hippo.ehviewer.download.DownloadManager
import com.hippo.ehviewer.ktbuilder.cache
import com.hippo.ehviewer.ktbuilder.diskCache
import com.hippo.ehviewer.ktbuilder.httpClient
import com.hippo.ehviewer.ktbuilder.imageLoader
import com.hippo.ehviewer.legacy.cleanObsoleteCache
import com.hippo.ehviewer.legacy.migrateCookies
import com.hippo.ehviewer.ui.keepNoMediaFileStatus
import com.hippo.ehviewer.ui.lockObserver
import com.hippo.ehviewer.util.AppConfig
@@ -58,6 +56,8 @@ import eu.kanade.tachiyomi.util.lang.withUIContext
import eu.kanade.tachiyomi.util.system.logcat
import io.ktor.client.HttpClient
import io.ktor.client.engine.okhttp.OkHttp
import io.ktor.client.plugins.cookies.HttpCookies
import io.ktor.http.HttpHeaders
import kotlinx.coroutines.launch
import moe.tarsin.kt.unreachable
import okhttp3.AsyncDns
@@ -98,9 +98,6 @@ class EhApplication : Application(), ImageLoaderFactory {
System.loadLibrary("ehviewer_rust")
ReadableTime.initialize(this)
lifecycleScope.launchIO {
launchIO {
migrateCookies()
}
launchIO {
EhTagDatabase
}
@@ -164,7 +161,7 @@ class EhApplication : Application(), ImageLoaderFactory {
callFactory { unreachable() }
installCronetHttpUriFetcher()
} else {
callFactory(baseOkHttpClient)
callFactory(coilClient)
}
if (isAtLeastP) {
add { result, options, _ -> ImageDecoderDecoder(result.source, options, false) }
@@ -184,12 +181,21 @@ class EhApplication : Application(), ImageLoaderFactory {
engine {
preconfigured = baseOkHttpClient
}
install(HttpCookies) {
storage = EhCookieStore
}
}
}
val baseOkHttpClient by lazy {
val noRedirectKtorClient by lazy {
HttpClient(ktorClient.engine) {
followRedirects = false
}
}
// Fallback to CIO when cronet unavailable after coil 3.0 release
private val baseOkHttpClient by lazy {
httpClient {
cookieJar(EhCookieStore)
if (isAtLeastQ) {
dns(AsyncDns.toDns(AndroidAsyncDns.IPv4, AndroidAsyncDns.IPv6))
}
@@ -198,19 +204,16 @@ class EhApplication : Application(), ImageLoaderFactory {
}
}
val noRedirectOkHttpClient by lazy {
// Use KtorClient directly when coil 3.0 released
private val coilClient by lazy {
httpClient(baseOkHttpClient) {
followRedirects(false)
}
}
// Never use this okhttp client to download large blobs!!!
val okHttpClient by lazy {
httpClient(baseOkHttpClient) {
cache(
appCtx.cacheDir.toOkioPath() / "http_cache",
20L * 1024L * 1024L,
)
addInterceptor {
val req = it.request()
val newReq = req.newBuilder().apply {
addHeader(HttpHeaders.Cookie, EhCookieStore.getCookieHeader(req.url.toString()))
}.build()
it.proceed(newReq)
}
}
}
@@ -1,50 +1,23 @@
/*
* Copyright 2016 Hippo Seven
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hippo.ehviewer.client
import android.webkit.CookieManager
import androidx.compose.ui.util.fastAny
import androidx.compose.ui.util.fastFilter
import androidx.compose.ui.util.fastForEach
import androidx.compose.ui.util.fastJoinToString
import androidx.compose.ui.util.fastMapNotNull
import okhttp3.Cookie
import okhttp3.CookieJar
import io.ktor.client.plugins.cookies.CookiesStorage
import io.ktor.http.Cookie
import io.ktor.http.Url
import io.ktor.http.parseClientCookiesHeader
import io.ktor.http.renderSetCookieHeader
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
object EhCookieStore : CookieJar {
object EhCookieStore : CookiesStorage {
private val manager = CookieManager.getInstance()
fun signOut() = manager.removeAllCookies(null)
fun contains(url: HttpUrl, name: String) = get(url).fastAny { it.name == name }
fun get(url: HttpUrl): List<Cookie> {
val cookies = manager.getCookie(url.toString())
return if (!cookies.isNullOrEmpty()) {
cookies.split(";").fastMapNotNull { setCookie ->
Cookie.parse(url, setCookie)?.takeUnless { it.name == KEY_UTMP_NAME }
}
} else {
emptyList()
}
}
fun contains(url: String, name: String) = load(Url(url)).fastAny { it.name == name }
fun hasSignedIn(): Boolean {
val url = EhUrl.HOST_E.toHttpUrl()
val url = EhUrl.HOST_E
return contains(url, KEY_IPB_MEMBER_ID) && contains(url, KEY_IPB_PASS_HASH)
}
@@ -55,16 +28,13 @@ object EhCookieStore : CookieJar {
private const val KEY_CONTENT_WARNING = "nw"
private const val CONTENT_WARNING_NOT_SHOW = "1"
private const val KEY_UTMP_NAME = "__utmp"
private val sTipsCookie = Cookie.Builder().apply {
name(KEY_CONTENT_WARNING)
value(CONTENT_WARNING_NOT_SHOW)
domain(EhUrl.DOMAIN_E)
path("/")
expiresAt(Long.MAX_VALUE)
}.build()
private val sTipsCookie = Cookie(
name = KEY_CONTENT_WARNING,
value = CONTENT_WARNING_NOT_SHOW,
)
fun copyNecessaryCookies() {
val cookies = get(EhUrl.HOST_E.toHttpUrl())
val cookies = load(Url(EhUrl.HOST_E))
cookies.fastForEach {
if (it.name == KEY_STAR || it.name == KEY_IPB_MEMBER_ID || it.name == KEY_IPB_PASS_HASH) {
manager.setCookie(EhUrl.HOST_EX, it.toString())
@@ -76,33 +46,41 @@ object EhCookieStore : CookieJar {
manager.setCookie(url.toString(), "$name=;Max-Age=0")
}
fun addCookie(cookie: Cookie) {
manager.setCookie(if (EhUrl.DOMAIN_E == cookie.domain) EhUrl.HOST_E else EhUrl.HOST_EX, cookie.toString())
fun addCookie(k: String, v: String, domain: String) {
val cookie = Cookie(name = k, value = v, domain = domain, maxAge = Int.MAX_VALUE)
val url = if (EhUrl.DOMAIN_E == cookie.domain) EhUrl.HOST_E else EhUrl.HOST_EX
manager.setCookie(url, renderSetCookieHeader(cookie))
}
fun flush() = manager.flush()
fun getCookieHeader(url: HttpUrl): String {
return loadForRequest(url).fastJoinToString("; ") {
fun getCookieHeader(url: String): String {
return load(Url(url)).fastJoinToString("; ") {
"${it.name}=${it.value}"
}
}
// See https://github.com/Ehviewer-Overhauled/Ehviewer/issues/873
override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) = cookies.fastForEach {
if (it.name != KEY_UTMP_NAME) {
manager.setCookie(url.toString(), it.toString())
override suspend fun addCookie(requestUrl: Url, cookie: Cookie) {
if (cookie.name != KEY_UTMP_NAME) {
manager.setCookie(requestUrl.toString(), renderSetCookieHeader(cookie))
}
}
override fun loadForRequest(url: HttpUrl): List<Cookie> {
override fun close() = Unit
fun load(url: Url): List<Cookie> {
val cookies = manager.getCookie(url.toString()) ?: return emptyList()
val checkTips = EhUrl.DOMAIN_E in url.host
return get(url).run {
if (checkTips) {
fastFilter { it.name != KEY_CONTENT_WARNING }.toMutableList().apply { add(sTipsCookie) }
} else {
this
return parseClientCookiesHeader(cookies)
.map { Cookie(it.key, it.value, maxAge = Int.MAX_VALUE) }
.filterTo(mutableListOf()) { it.name != KEY_UTMP_NAME }.apply {
if (checkTips) {
removeAll { it.name == KEY_CONTENT_WARNING }
add(sTipsCookie)
}
}
}
}
override suspend fun get(requestUrl: Url) = load(requestUrl)
}
@@ -15,7 +15,6 @@
*/
package com.hippo.ehviewer.client
import android.util.Log
import arrow.core.Either
import arrow.core.left
import arrow.core.right
@@ -48,20 +47,18 @@ import com.hippo.ehviewer.client.parser.TorrentParser
import com.hippo.ehviewer.client.parser.TorrentResult
import com.hippo.ehviewer.client.parser.VoteCommentResult
import com.hippo.ehviewer.client.parser.VoteTagParser
import com.hippo.ehviewer.cronet.awaitBodyFully
import com.hippo.ehviewer.cronet.cronetRequest
import com.hippo.ehviewer.cronet.execute
import com.hippo.ehviewer.cronet.withRequestBody
import com.hippo.ehviewer.dailycheck.showEventNotification
import com.hippo.ehviewer.dailycheck.today
import com.hippo.ehviewer.util.AppConfig
import com.hippo.ehviewer.util.StatusCodeException
import com.hippo.ehviewer.util.isCronetSupported
import eu.kanade.tachiyomi.util.system.logcat
import io.ktor.client.request.forms.append
import io.ktor.client.statement.HttpStatement
import io.ktor.client.statement.bodyAsChannel
import io.ktor.client.statement.bodyAsText
import io.ktor.client.statement.request
import io.ktor.http.ContentType
import io.ktor.utils.io.core.writeFully
import io.ktor.utils.io.pool.DirectByteBufferPool
import io.ktor.utils.io.pool.useInstance
import java.io.File
@@ -75,18 +72,12 @@ import kotlinx.serialization.json.addJsonArray
import kotlinx.serialization.json.put
import moe.tarsin.coroutines.removeAllSuspend
import moe.tarsin.coroutines.runSuspendCatching
import okhttp3.MediaType
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.MultipartBody
import okhttp3.RequestBody
import okhttp3.RequestBody.Companion.asRequestBody
import org.jsoup.Jsoup
import splitties.init.appCtx
const val TAG = "EhEngine"
private const val MAX_REQUEST_SIZE = 25
private const val U_CONFIG_TEXT = "Selected Profile"
private val MEDIA_TYPE_JPEG: MediaType = "image/jpeg".toMediaType()
fun rethrowExactly(code: Int, body: Either<String, ByteBuffer>, e: Throwable): Nothing {
// Don't translate coroutine cancellation
@@ -140,56 +131,6 @@ fun rethrowExactly(code: Int, body: Either<String, ByteBuffer>, e: Throwable): N
val httpContentPool = DirectByteBufferPool(8, 0x80000)
suspend fun <T> fetchCompat(
url: String,
referer: String? = null,
origin: String? = null,
reqBody: RequestBody? = null,
parser: suspend (ByteBuffer) -> T,
): T {
Log.d(TAG, url)
fun commonChecks(body: ByteBuffer) {
// Check sad panda(without panda)
if (!body.hasRemaining()) {
if (EhUtils.isExHentai) {
throw EhException("Sad Panda\n(without panda)")
} else {
throw EhException("IP banned")
}
}
}
return if (isCronetSupported) {
cronetRequest(url, referer, origin) {
reqBody?.let { withRequestBody(it) }
}.execute {
@Suppress("NewApi")
val code = it.httpStatusCode
if (code >= 400) {
throw StatusCodeException(code)
}
httpContentPool.useInstance { buffer ->
awaitBodyFully(buffer)
buffer.flip()
commonChecks(buffer)
parser(buffer)
}
}
} else {
ehRequest(url, referer, origin) {
reqBody?.let { post(it) }
}.execute {
httpContentPool.useInstance { buffer ->
while (true) {
if (body.source().read(buffer) == -1) break
}
buffer.flip()
commonChecks(buffer)
parser(buffer)
}
}
}
}
suspend inline fun <T> HttpStatement.parseString(crossinline block: suspend String.() -> T) = execute { response ->
logcat(tag = TAG) { response.request.url.toString() }
val body = response.bodyAsText()
@@ -215,12 +156,9 @@ suspend inline fun <T> HttpStatement.parseByteBuffer(crossinline block: suspend
object EhEngine {
suspend fun getOriginalImageUrl(url: String, referer: String?): String {
Log.d(TAG, url)
return ehRequest(url, referer).executeNoRedirect {
header("Location")?.apply {
if (contains("bounce_login")) {
throw NotLoggedInException()
}
return noRedirectStatement(url, referer).execute {
it.headers["Location"]?.apply {
if (contains("bounce_login")) throw NotLoggedInException()
} ?: throw InsufficientFundsException()
}
}
@@ -521,19 +459,16 @@ object EhEngine {
/**
* @param image Must be jpeg
*/
suspend fun imageSearch(image: File, uss: Boolean, osc: Boolean) = fetchCompat(
EhUrl.imageSearchUrl,
EhUrl.referer,
EhUrl.origin,
suspend fun imageSearch(image: File, uss: Boolean, osc: Boolean) = statement(EhUrl.imageSearchUrl, EhUrl.referer, EhUrl.origin) {
multipartBody {
setType(MultipartBody.FORM)
addFormDataPart("sfile", "a.jpg", image.asRequestBody(MEDIA_TYPE_JPEG))
if (uss) addFormDataPart("fs_similar", "on")
if (osc) addFormDataPart("fs_covers", "on")
addFormDataPart("f_sfile", "File Search")
},
parser = GalleryListParser::parse,
).apply { fillGalleryList(galleryInfoList, EhUrl.imageSearchUrl) }
append("sfile", "a.jpg", ContentType.Image.JPEG, image.length()) {
writeFully(image.readBytes())
}
if (uss) append("fs_similar", "on")
if (osc) append("fs_covers", "on")
append("f_sfile", "File Search")
}
}.parseByteBuffer(GalleryListParser::parse).apply { fillGalleryList(galleryInfoList, EhUrl.imageSearchUrl) }
private suspend fun fillGalleryList(list: MutableList<BaseGalleryInfo>, url: String, filter: Boolean = false) {
if (filter) list.removeAllSuspend { EhFilter.filterTitle(it) || EhFilter.filterUploader(it) }
@@ -16,58 +16,38 @@
package com.hippo.ehviewer.client
import com.hippo.ehviewer.EhApplication.Companion.ktorClient
import com.hippo.ehviewer.EhApplication.Companion.noRedirectOkHttpClient
import com.hippo.ehviewer.EhApplication.Companion.okHttpClient
import com.hippo.ehviewer.EhApplication.Companion.noRedirectKtorClient
import com.hippo.ehviewer.Settings
import io.ktor.client.request.HttpRequestBuilder
import io.ktor.client.request.forms.FormBuilder
import io.ktor.client.request.forms.FormDataContent
import io.ktor.client.request.forms.MultiPartFormDataContent
import io.ktor.client.request.forms.formData
import io.ktor.client.request.header
import io.ktor.client.request.prepareRequest
import io.ktor.client.request.setBody
import io.ktor.client.statement.HttpResponse
import io.ktor.client.statement.HttpStatement
import io.ktor.client.statement.bodyAsText
import io.ktor.http.ContentType
import io.ktor.http.HttpHeaders
import io.ktor.http.HttpMethod
import io.ktor.http.ParametersBuilder
import io.ktor.http.content.TextContent
import io.ktor.http.userAgent
import kotlin.contracts.InvocationKind
import kotlin.contracts.contract
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArrayBuilder
import kotlinx.serialization.json.JsonObjectBuilder
import kotlinx.serialization.json.buildJsonArray
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.okio.decodeFromBufferedSource
import okhttp3.MultipartBody
import okhttp3.Request
import okhttp3.Response
import okhttp3.executeAsync
import okio.BufferedSource
inline fun ehRequest(url: String, referer: String? = null, origin: String? = null, builder: Request.Builder.() -> Unit = {}) = Request.Builder().url(url).apply {
addHeader("User-Agent", Settings.userAgent)
addHeader("Accept", CHROME_ACCEPT)
addHeader("Accept-Language", CHROME_ACCEPT_LANGUAGE)
referer?.let { addHeader("Referer", it) }
origin?.let { addHeader("Origin", it) }
}.apply(builder).build()
inline fun multipartBody(builder: MultipartBody.Builder.() -> Unit) = MultipartBody.Builder().apply(builder).build()
inline fun JsonObjectBuilder.array(name: String, builder: JsonArrayBuilder.() -> Unit) = put(name, buildJsonArray(builder))
suspend inline fun <R> Request.execute(block: Response.() -> R): R {
contract {
callsInPlace(block, InvocationKind.EXACTLY_ONCE)
}
return okHttpClient.newCall(this).executeAsync().use(block)
}
suspend inline fun <reified T> HttpStatement.executeAndParseAs() = execute { it.parseAs<T>() }
suspend inline fun <R> Request.executeNoRedirect(block: Response.() -> R) = noRedirectOkHttpClient.newCall(this).executeAsync().use(block)
suspend inline fun <reified T> Request.executeAndParseAs() = execute { parseAs<T>() }
inline fun <reified T> Response.parseAs(): T = body.source().parseAs()
suspend inline fun <reified T> HttpResponse.parseAs(): T = bodyAsText().parseAs()
inline fun <reified T> BufferedSource.parseAs(): T = json.decodeFromBufferedSource(this)
inline fun <reified T> String.parseAs(): T = json.decodeFromString(this)
@@ -77,18 +57,32 @@ const val CHROME_USER_AGENT = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/53
const val CHROME_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
const val CHROME_ACCEPT_LANGUAGE = "en-US,en;q=0.5"
suspend inline fun statement(
url: String,
referer: String? = null,
origin: String? = null,
builder: HttpRequestBuilder.() -> Unit = {},
) = ktorClient.prepareRequest(url) {
fun HttpRequestBuilder.applyEhConfig(referer: String?, origin: String?) {
method = HttpMethod.Get
header("Referer", referer)
header("Origin", origin)
userAgent(Settings.userAgent)
header(HttpHeaders.Accept, CHROME_ACCEPT)
header(HttpHeaders.AcceptLanguage, CHROME_ACCEPT_LANGUAGE)
}
suspend inline fun statement(
url: String,
referer: String? = null,
origin: String? = null,
builder: HttpRequestBuilder.() -> Unit = {},
) = ktorClient.prepareRequest(url) {
applyEhConfig(referer, origin)
apply(builder)
}
suspend inline fun noRedirectStatement(
url: String,
referer: String? = null,
origin: String? = null,
builder: HttpRequestBuilder.() -> Unit = {},
) = noRedirectKtorClient.prepareRequest(url) {
applyEhConfig(referer, origin)
apply(builder)
}
@@ -103,3 +97,8 @@ fun HttpRequestBuilder.jsonBody(builder: JsonObjectBuilder.() -> Unit) {
val json = buildJsonObject(builder).toString()
setBody(TextContent(text = json, contentType = ContentType.Application.Json))
}
fun HttpRequestBuilder.multipartBody(builder: FormBuilder.() -> Unit) {
method = HttpMethod.Post
setBody(MultiPartFormDataContent(formData(builder)))
}
@@ -19,11 +19,18 @@ import android.content.Context
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
import com.hippo.ehviewer.EhApplication.Companion.baseOkHttpClient
import com.hippo.ehviewer.EhApplication.Companion.ktorClient
import com.hippo.ehviewer.R
import com.hippo.ehviewer.util.AppConfig
import com.hippo.ehviewer.util.FileUtils
import com.hippo.ehviewer.util.copyToFile
import com.hippo.ehviewer.util.StatusCodeException
import io.ktor.client.HttpClient
import io.ktor.client.request.prepareGet
import io.ktor.client.statement.bodyAsChannel
import io.ktor.http.isSuccess
import io.ktor.util.cio.writeChannel
import io.ktor.utils.io.close
import io.ktor.utils.io.copyTo
import java.io.File
import java.nio.charset.StandardCharsets
import kotlinx.coroutines.CoroutineScope
@@ -35,9 +42,6 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import moe.tarsin.coroutines.runSuspendCatching
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.executeAsync
import okio.BufferedSource
import okio.HashingSink.Companion.sha1
import okio.blackholeSink
@@ -175,24 +179,23 @@ object EhTagDatabase : CoroutineScope {
return sha1 != null && sha1 == getFileSha1(data)
}
private suspend fun save(client: OkHttpClient, url: String, file: File): Boolean {
val request: Request = Request.Builder().url(url).build()
val call = client.newCall(request)
private suspend fun save(client: HttpClient, url: String, file: File) {
runCatching {
call.executeAsync().use { response ->
if (!response.isSuccessful) {
return false
client.prepareGet(url).execute {
if (it.status.isSuccess()) {
val channel = file.writeChannel()
try {
it.bodyAsChannel().copyTo(channel)
} finally {
channel.close()
}
} else {
throw StatusCodeException(it.status.value)
}
response.body.use {
it.copyToFile(file)
}
return true
}
}.onFailure {
file.delete()
it.printStackTrace()
}
return false
}.getOrThrow()
}
suspend fun update() {
@@ -246,7 +249,7 @@ object EhTagDatabase : CoroutineScope {
// Save new sha1
val tempSha1File = File(dir, "$sha1Name.tmp")
check(save(baseOkHttpClient, sha1Url, tempSha1File))
save(ktorClient, sha1Url, tempSha1File)
val tempSha1 = getFileContent(tempSha1File)
// Check new sha1 and current sha1
@@ -258,7 +261,7 @@ object EhTagDatabase : CoroutineScope {
// Save new data
val tempDataFile = File(dir, "$dataName.tmp")
check(save(baseOkHttpClient, dataUrl, tempDataFile))
save(ktorClient, dataUrl, tempDataFile)
// Check new sha1 and new data
if (!checkData(tempSha1, tempDataFile)) {
@@ -1,16 +1,17 @@
package com.hippo.ehviewer.cronet
import com.hippo.ehviewer.EhApplication
import io.ktor.utils.io.pool.DirectByteBufferPool
import io.ktor.utils.io.pool.useInstance
import java.nio.ByteBuffer
import java.nio.channels.FileChannel
import kotlin.coroutines.resumeWithException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.asExecutor
import kotlinx.coroutines.suspendCancellableCoroutine
val pool = DirectByteBufferPool(32)
val cronetHttpClientExecutor = EhApplication.baseOkHttpClient.dispatcher.executorService
val cronetHttpClientExecutor = Dispatchers.IO.asExecutor()
@Suppress("NewApi")
suspend inline fun CronetRequest.awaitBodyFully(crossinline callback: (ByteBuffer) -> Unit) {
@@ -1,30 +0,0 @@
package com.hippo.ehviewer.dao
import androidx.room.ColumnInfo
import androidx.room.Entity
import androidx.room.PrimaryKey
@Entity(tableName = "OK_HTTP_3_COOKIE")
data class Cookie(
@ColumnInfo(name = "NAME")
var name: String,
@ColumnInfo(name = "VALUE")
var value: String,
@ColumnInfo(name = "EXPIRES_AT")
var expiresAt: Long,
@ColumnInfo(name = "DOMAIN")
var domain: String,
@ColumnInfo(name = "PATH")
var path: String,
@ColumnInfo(name = "SECURE")
var secure: Boolean,
@ColumnInfo(name = "HTTP_ONLY")
var httpOnly: Boolean,
@ColumnInfo(name = "PERSISTENT")
var persistent: Boolean,
@ColumnInfo(name = "HOST_ONLY")
var hostOnly: Boolean,
@PrimaryKey
@ColumnInfo(name = "_id")
var id: Long?,
)
@@ -1,10 +0,0 @@
package com.hippo.ehviewer.dao
import androidx.room.Dao
import androidx.room.Query
@Dao
interface CookiesDao {
@Query("SELECT * FROM OK_HTTP_3_COOKIE")
suspend fun list(): List<Cookie>
}
@@ -181,21 +181,6 @@ abstract class EhDatabase : RoomDatabase() {
abstract fun quickSearchDao(): QuickSearchDao
}
// 1 -> 2 some nullability changes
@Database(
entities = [Cookie::class],
version = 2,
autoMigrations = [
AutoMigration(
from = 1,
to = 2,
),
],
)
abstract class CookiesDatabase : RoomDatabase() {
abstract fun cookiesDao(): CookiesDao
}
@Database(
entities = [Search::class],
version = 2,
@@ -1,10 +1,6 @@
package com.hippo.ehviewer.ktbuilder
import okhttp3.Cache
import okhttp3.OkHttpClient
import okio.FileSystem
import okio.Path
inline fun httpClient(builder: OkHttpClient.Builder.() -> Unit): OkHttpClient = OkHttpClient.Builder().apply(builder).build()
inline fun httpClient(client: OkHttpClient, builder: OkHttpClient.Builder.() -> Unit): OkHttpClient = client.newBuilder().apply(builder).build()
fun OkHttpClient.Builder.cache(directory: Path, maxSize: Long, fileSystem: FileSystem = FileSystem.SYSTEM) = cache(Cache(directory, maxSize, fileSystem))
@@ -1,38 +0,0 @@
package com.hippo.ehviewer.legacy
import arrow.fx.coroutines.release
import arrow.fx.coroutines.resource
import com.hippo.ehviewer.client.EhCookieStore
import com.hippo.ehviewer.dao.Cookie
import com.hippo.ehviewer.dao.CookiesDatabase
import splitties.arch.room.roomDb
import splitties.init.appCtx
private const val LEGACY_COOKIES_DB = "okhttp3-cookie.db"
private fun Cookie.toOkHttp3Cookie(): okhttp3.Cookie {
return okhttp3.Cookie.Builder().apply {
name(name)
value(value)
expiresAt(expiresAt)
if (hostOnly) hostOnlyDomain(domain) else domain(domain)
path(path)
if (secure) secure()
if (httpOnly) httpOnly()
}.build()
}
suspend fun migrateCookies() {
if (LEGACY_COOKIES_DB !in appCtx.databaseList()) return
resource {
roomDb<CookiesDatabase>(LEGACY_COOKIES_DB)
} release {
it.close()
appCtx.deleteDatabase(LEGACY_COOKIES_DB)
} use { database ->
database.cookiesDao().list().forEach {
EhCookieStore.addCookie(it.toOkHttp3Cookie())
}
EhCookieStore.flush()
}
}
@@ -68,7 +68,6 @@ import eu.kanade.tachiyomi.util.lang.withUIContext
import java.util.Locale
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import okhttp3.Cookie
@Destination
@Composable
@@ -93,15 +92,13 @@ fun CookieSignInScene(navigator: DestinationsNavigator) {
val noCookies = stringResource(R.string.from_clipboard_error)
fun newCookie(name: String, value: String, domain: String) = Cookie.Builder().name(name).value(value).domain(domain).expiresAt(Long.MAX_VALUE).build()
fun storeCookie(id: String, hash: String, igneous: String) {
EhUtils.signOut()
EhCookieStore.addCookie(newCookie(EhCookieStore.KEY_IPB_MEMBER_ID, id, EhUrl.DOMAIN_E))
EhCookieStore.addCookie(newCookie(EhCookieStore.KEY_IPB_MEMBER_ID, id, EhUrl.DOMAIN_EX))
EhCookieStore.addCookie(newCookie(EhCookieStore.KEY_IPB_PASS_HASH, hash, EhUrl.DOMAIN_E))
EhCookieStore.addCookie(newCookie(EhCookieStore.KEY_IPB_PASS_HASH, hash, EhUrl.DOMAIN_EX))
if (igneous.isNotBlank()) EhCookieStore.addCookie(newCookie(EhCookieStore.KEY_IGNEOUS, igneous, EhUrl.DOMAIN_EX))
EhCookieStore.addCookie(EhCookieStore.KEY_IPB_MEMBER_ID, id, EhUrl.DOMAIN_E)
EhCookieStore.addCookie(EhCookieStore.KEY_IPB_MEMBER_ID, id, EhUrl.DOMAIN_EX)
EhCookieStore.addCookie(EhCookieStore.KEY_IPB_PASS_HASH, hash, EhUrl.DOMAIN_E)
EhCookieStore.addCookie(EhCookieStore.KEY_IPB_PASS_HASH, hash, EhUrl.DOMAIN_EX)
if (igneous.isNotBlank()) EhCookieStore.addCookie(EhCookieStore.KEY_IGNEOUS, igneous, EhUrl.DOMAIN_EX)
}
fun login() {
@@ -22,7 +22,7 @@ import com.ramcosta.composedestinations.navigation.DestinationsNavigator
import eu.kanade.tachiyomi.util.lang.launchIO
import eu.kanade.tachiyomi.util.lang.withNonCancellableContext
import eu.kanade.tachiyomi.util.lang.withUIContext
import okhttp3.HttpUrl.Companion.toHttpUrl
import io.ktor.http.Url
@Destination
@Composable
@@ -39,7 +39,7 @@ fun WebViewSignInScreen(navigator: DestinationsNavigator) {
}
var getId = false
var getHash = false
EhCookieStore.loadForRequest(EhUrl.HOST_E.toHttpUrl()).forEach {
EhCookieStore.load(Url(EhUrl.HOST_E)).forEach {
if (EhCookieStore.KEY_IPB_MEMBER_ID == it.name) {
getId = true
} else if (EhCookieStore.KEY_IPB_PASS_HASH == it.name) {
@@ -170,7 +170,6 @@ import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.parcelize.Parcelize
import moe.tarsin.coroutines.runSuspendCatching
import okhttp3.HttpUrl.Companion.toHttpUrl
import splitties.systemservices.downloadManager
import splitties.systemservices.layoutInflater
@@ -673,7 +672,7 @@ fun GalleryDetailScreen(args: GalleryDetailScreenArgs, navigator: DestinationsNa
FileUtils.sanitizeFilename("$name.torrent"),
)
r.setNotificationVisibility(DownloadManager.Request.VISIBILITY_VISIBLE_NOTIFY_COMPLETED)
r.addRequestHeader("Cookie", EhCookieStore.getCookieHeader(itemUrl.toHttpUrl()))
r.addRequestHeader("Cookie", EhCookieStore.getCookieHeader(itemUrl))
try {
downloadManager.enqueue(r)
activity.showTip(R.string.download_torrent_started)
@@ -54,11 +54,11 @@ import com.hippo.ehviewer.util.copyTextToClipboard
import com.ramcosta.composedestinations.annotation.Destination
import com.ramcosta.composedestinations.navigation.DestinationsNavigator
import eu.kanade.tachiyomi.util.lang.withUIContext
import io.ktor.http.Url
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.launch
import moe.tarsin.coroutines.runSuspendCatching
import okhttp3.HttpUrl.Companion.toHttpUrl
@Destination
@Composable
@@ -91,8 +91,8 @@ fun EhScreen(navigator: DestinationsNavigator) {
title = stringResource(id = R.string.account_name),
summary = Settings.displayName ?: touristMode,
) {
val eCookies = EhCookieStore.loadForRequest(EhUrl.HOST_E.toHttpUrl())
val exCookies = EhCookieStore.loadForRequest(EhUrl.HOST_EX.toHttpUrl())
val eCookies = EhCookieStore.load(Url(EhUrl.HOST_E))
val exCookies = EhCookieStore.load(Url(EhUrl.HOST_EX))
var ipbMemberId: String? = null
var ipbPassHash: String? = null
var igneous: String? = null
@@ -1,9 +1,17 @@
package com.hippo.ehviewer.updater
import com.hippo.ehviewer.BuildConfig
import com.hippo.ehviewer.EhApplication.Companion.ktorClient
import com.hippo.ehviewer.Settings
import com.hippo.ehviewer.client.execute
import com.hippo.ehviewer.client.executeAndParseAs
import io.ktor.client.request.header
import io.ktor.client.request.prepareGet
import io.ktor.client.statement.bodyAsChannel
import io.ktor.client.statement.bodyAsText
import io.ktor.util.cio.writeChannel
import io.ktor.utils.io.close
import io.ktor.utils.io.copyTo
import io.ktor.utils.io.jvm.javaio.toInputStream
import java.io.File
import java.time.Instant
import java.time.temporal.ChronoUnit
@@ -11,8 +19,6 @@ import java.util.zip.ZipInputStream
import kotlin.io.encoding.Base64
import kotlin.io.encoding.ExperimentalEncodingApi
import moe.tarsin.coroutines.runSuspendCatching
import okhttp3.Request
import okio.sink
import org.json.JSONObject
import tachiyomi.data.release.GithubArtifacts
import tachiyomi.data.release.GithubCommitComparison
@@ -31,18 +37,16 @@ object AppUpdater {
Settings.lastUpdateDay = now.epochSecond
if (Settings.useCIUpdateChannel) {
val curSha = BuildConfig.COMMIT_SHA
val branch = ghRequest(API_URL).execute {
JSONObject(body.string()).getString("default_branch")
}
val branch = ghStatement(API_URL).execute { JSONObject(it.bodyAsText()).getString("default_branch") }
val workflowRunsUrl = "$API_URL/actions/workflows/ci.yml/runs?branch=$branch&event=push&status=success&per_page=1"
val workflowRun = ghRequest(workflowRunsUrl).executeAndParseAs<GithubWorkflowRuns>().workflowRuns[0]
val workflowRun = ghStatement(workflowRunsUrl).executeAndParseAs<GithubWorkflowRuns>().workflowRuns[0]
val shortSha = workflowRun.headSha.take(7)
if (shortSha != curSha) {
val artifacts = ghRequest(workflowRun.artifactsUrl).executeAndParseAs<GithubArtifacts>()
val artifacts = ghStatement(workflowRun.artifactsUrl).executeAndParseAs<GithubArtifacts>()
val archiveUrl = artifacts.getDownloadLink()
val changelog = runSuspendCatching {
val commitComparisonUrl = "$API_URL/compare/$curSha...$shortSha"
val result = ghRequest(commitComparisonUrl).executeAndParseAs<GithubCommitComparison>()
val result = ghStatement(commitComparisonUrl).executeAndParseAs<GithubCommitComparison>()
// TODO: Prettier format, Markdown?
result.commits.joinToString("\n") { commit ->
"${commit.commit.message.takeWhile { it != '\n' }} (@${commit.author.name})"
@@ -52,7 +56,7 @@ object AppUpdater {
}
} else {
val curVersion = BuildConfig.VERSION_NAME.substringBefore('-')
val release = ghRequest(LATEST_RELEASE_URL).executeAndParseAs<GithubRelease>()
val release = ghStatement(LATEST_RELEASE_URL).executeAndParseAs<GithubRelease>()
val latestVersion = release.version
val description = release.info
val downloadUrl = release.getDownloadLink()
@@ -65,29 +69,34 @@ object AppUpdater {
}
suspend fun downloadUpdate(url: String, file: File) =
ghRequest(url).execute {
ghStatement(url).execute { response ->
if (url.endsWith("zip")) {
ZipInputStream(body.byteStream()).use { zip ->
zip.nextEntry
file.outputStream().use {
zip.copyTo(it)
response.bodyAsChannel().toInputStream().use { stream ->
ZipInputStream(stream).use { zip ->
zip.nextEntry
file.outputStream().use {
zip.copyTo(it)
}
}
}
} else {
file.sink().use {
body.source().readAll(it)
val channel = file.writeChannel()
try {
response.bodyAsChannel().copyTo(channel)
} finally {
channel.close()
}
}
}
}
@OptIn(ExperimentalEncodingApi::class)
private inline fun ghRequest(url: String, builder: Request.Builder.() -> Unit = {}) = Request.Builder().url(url).apply {
private suspend inline fun ghStatement(url: String) = ktorClient.prepareGet(url) {
val token = "github_" + "pat_11AXZS" + "T4A0k3TArCGakP3t_7DzUE5S" + "mr1zw8rmmzVtCeRq62" + "A4qkuDMw6YQm5ZUtHSLZ2MLI3J4VSifLXZ"
val user = "nullArrayList"
val base64 = Base64.encode("$user:$token".toByteArray())
addHeader("Authorization", "Basic $base64")
}.apply(builder).build()
header("Authorization", "Basic $base64")
}
data class Release(
val version: String,
@@ -1,17 +0,0 @@
package com.hippo.ehviewer.util
import java.io.File
import okhttp3.ResponseBody
import okio.buffer
import okio.sink
fun ResponseBody.copyToFile(file: File) {
file.outputStream().use { os ->
// Prior to the adoption of OpenJDK, transferFrom will call ByteBuffer.allocate((int) count)
if (isAtLeastN) {
os.channel.transferFrom(source(), 0, Long.MAX_VALUE)
} else {
os.sink().buffer().use { it.writeAll(source()) }
}
}
}
@@ -8,8 +8,8 @@ import android.webkit.WebViewClient
import androidx.core.content.ContextCompat
import com.hippo.ehviewer.client.EhCookieStore
import com.hippo.ehviewer.client.exception.CloudflareBypassException
import io.ktor.http.Url
import java.util.concurrent.CountDownLatch
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
@@ -52,8 +52,7 @@ class CloudflareInterceptor(context: Context) : WebViewInterceptor(context) {
webview?.webViewClient = object : WebViewClient() {
override fun onPageFinished(view: WebView, url: String) {
fun isCloudFlareBypassed(): Boolean {
return EhCookieStore.get(origRequestUrl.toHttpUrl())
.any { it.name == COOKIE_NAME }
return EhCookieStore.load(Url(origRequestUrl)).any { it.name == COOKIE_NAME }
}
if (isCloudFlareBypassed()) {
@@ -21,7 +21,6 @@ import kotlin.coroutines.resumeWithException
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.RequestBody
import okio.Buffer
import okio.Path.Companion.toOkioPath
@@ -70,7 +69,7 @@ class CronetRequest {
inline fun cronetRequest(url: String, referer: String? = null, origin: String? = null, conf: UrlRequest.Builder.() -> Unit = {}) = CronetRequest().apply {
request = cronetHttpClient.newUrlRequestBuilder(url, cronetHttpClientExecutor, callback).apply {
addHeader("Cookie", EhCookieStore.getCookieHeader(url.toHttpUrl()))
addHeader("Cookie", EhCookieStore.getCookieHeader(url))
addHeader("Accept", CHROME_ACCEPT)
addHeader("Accept-Language", CHROME_ACCEPT_LANGUAGE)
referer?.let { addHeader("Referer", it) }