yenru0/EhViewer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New SpiderInfo serialization format

Browse Source
This commit is contained in:
tarsin committed 2023-01-28 15:58:45 +08:00
1 parent 71f7771a97
commit 30c7f9e020
7 files changed
+107 -230
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/build.gradle
+2
View File
@@ -2,6 +2,7 @@ plugins {
id 'com.android.application'
id 'kotlin-android'
id 'kotlin-parcelize'
id 'org.jetbrains.kotlin.plugin.serialization'
id 'com.google.devtools.ksp'
}
@@ -159,6 +160,7 @@ dependencies {
// implementation 'io.coil-kt:coil:2.2.2'
implementation 'com.github.Ehviewer-Overhauled.coil:coil-compose:d65881d'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.6.4'
implementation 'org.jetbrains.kotlinx:kotlinx-serialization-cbor:1.5.0-RC'
implementation 'org.jsoup:jsoup:1.15.3'
}
app/src/main/java/com/hippo/ehviewer/EhApplication.kt
+1 -1
View File
@@ -331,7 +331,7 @@ class EhApplication : Application(), DefaultLifecycleObserver, ImageLoaderFactor
@JvmStatic
val spiderInfoCache by lazy {
DiskCache.Builder().directory(File(application.cacheDir, "spider_info"))
DiskCache.Builder().directory(File(application.cacheDir, "spider_info_v2"))
.maxSizeBytes(20 * 1024 * 1024).build()
}
app/src/main/java/com/hippo/ehviewer/download/DownloadManager.kt
+4 -4
View File
@@ -18,8 +18,8 @@ package com.hippo.ehviewer.download
import android.annotation.SuppressLint
import android.os.AsyncTask
import android.util.Log
import android.util.SparseLongArray
import androidx.collection.LongSparseArray
import androidx.collection.SparseArrayCompat
import com.hippo.ehviewer.EhDB
import com.hippo.ehviewer.client.data.BaseGalleryInfo
import com.hippo.ehviewer.client.data.GalleryInfo
@@ -584,7 +584,7 @@ class DownloadManager : OnSpiderListener {
val spiderInfo = SpiderInfo.read(file) ?: continue
spiderInfo.startPage = 0
try {
spiderInfo.write(file.openOutputStream())
spiderInfo.write(file)
} catch (e: IOException) {
Log.e(TAG, "Can't write SpiderInfo", e)
}
@@ -1130,8 +1130,8 @@ class DownloadManager : OnSpiderListener {
}
internal inner class SpeedReminder : Runnable {
private val mContentLengthMap = SparseArrayCompat<Long>()
private val mReceivedSizeMap = SparseArrayCompat<Long>()
private val mContentLengthMap = SparseLongArray()
private val mReceivedSizeMap = SparseLongArray()
private var mStop = true
private var mBytesRead: Long = 0
private var oldSpeed: Long = -1
app/src/main/java/com/hippo/ehviewer/preference/RestoreDownloadPreference.kt
+1 -5
View File
@@ -53,10 +53,8 @@ class RestoreDownloadPreference constructor(
return null
}
val siFile = file.findFile(SpiderQueen.SPIDER_INFO_FILENAME) ?: return null
var `is`: InputStream? = null
return try {
`is` = siFile.openInputStream()
val spiderInfo = SpiderInfo.read(`is`) ?: return null
val spiderInfo = SpiderInfo.read(siFile) ?: return null
val gid = spiderInfo.gid
val dirname = file.name
if (mManager.containDownloadInfo(gid)) {
@@ -76,8 +74,6 @@ class RestoreDownloadPreference constructor(
restoreItem
} catch (e: IOException) {
null
} finally {
IOUtils.closeQuietly(`is`)
}
}
app/src/main/java/com/hippo/ehviewer/spider/SpiderInfo.kt
+78 -181
View File
@@ -1,207 +1,104 @@
/*
* Copyright 2016 Hippo Seven
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:OptIn(ExperimentalSerializationApi::class)
package com.hippo.ehviewer.spider;
package com.hippo.ehviewer.spider
import android.text.TextUtils;
import android.util.Log;
import android.util.SparseArray;
import com.hippo.unifile.UniFile
import com.hippo.yorozuya.IOUtils
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.Serializable
import kotlinx.serialization.cbor.Cbor
import kotlinx.serialization.decodeFromByteArray
import kotlinx.serialization.encodeToByteArray
import java.io.File
import java.io.InputStream
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
@Suppress("ArrayInDataClass")
@Serializable
data class SpiderInfo @JvmOverloads constructor(
@JvmField
val gid: Long,
import com.hippo.unifile.UniFile;
import com.hippo.yorozuya.IOUtils;
import com.hippo.yorozuya.NumberUtils;
@JvmField
val pages: Int,
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
@JvmField
var pTokenMap: Array<String?> = arrayOfNulls(pages),
public class SpiderInfo {
@JvmField
var startPage: Int = 0,
static final String TOKEN_FAILED = "failed";
private static final String TAG = SpiderInfo.class.getSimpleName();
private static final String VERSION_STR = "VERSION";
private static final int VERSION = 2;
public int startPage = 0;
public long gid = -1;
public String token = null;
public int pages = -1;
public int previewPages = -1;
public int previewPerPage = -1;
public SparseArray<String> pTokenMap = null;
@JvmField
var token: String? = null,
public static SpiderInfo read(@Nullable UniFile file) {
if (file == null) {
return null;
}
@JvmField
var previewPages: Int = -1,
InputStream is = null;
try {
is = file.openInputStream();
return read(is);
} catch (IOException e) {
return null;
} finally {
IOUtils.closeQuietly(is);
@JvmField
var previewPerPage: Int = -1
) {
fun write(file: UniFile) {
file.openOutputStream().use {
it.write(Cbor.encodeToByteArray(this))
}
}
private static int getStartPage(String str) {
if (null == str) {
return 0;
}
int startPage = 0;
for (int i = 0, n = str.length(); i < n; i++) {
startPage *= 16;
char ch = str.charAt(i);
if (ch >= '0' && ch <= '9') {
startPage += ch - '0';
} else if (ch >= 'a' && ch <= 'f') {
startPage += ch - 'a' + 10;
}
}
return Math.max(startPage, 0);
fun write(file: File) {
file.writeBytes(Cbor.encodeToByteArray(this))
}
private static int getVersion(String str) {
if (null == str) {
return -1;
}
if (str.startsWith(VERSION_STR)) {
return NumberUtils.parseIntSafely(str.substring(VERSION_STR.length()), -1);
} else {
return 1;
}
}
@Nullable
@SuppressWarnings("InfiniteLoopStatement")
public static SpiderInfo read(@Nullable InputStream is) {
if (null == is) {
return null;
companion object {
@JvmStatic
fun read(file: UniFile): SpiderInfo? {
return runCatching {
file.openInputStream().use {
Cbor.decodeFromByteArray<SpiderInfo>(it.readBytes())
}
}.getOrNull() ?: runCatching {
file.openInputStream().use { readV1(it) }
}.getOrNull()
}
SpiderInfo spiderInfo = null;
try {
spiderInfo = new SpiderInfo();
// Get version
String line = IOUtils.readAsciiLine(is);
int version = getVersion(line);
if (version == VERSION) {
// Read next line
line = IOUtils.readAsciiLine(is);
} else if (version == 1) {
// pass
} else {
// Invalid version
return null;
// This path is for diskcache only, no need to compatible with old data
@JvmStatic
fun read(file: File): SpiderInfo? {
return Cbor.decodeFromByteArray(file.readBytes())
}
private fun readV1(inputStream: InputStream): SpiderInfo {
fun read(): String {
return IOUtils.readAsciiLine(inputStream)
}
// Start page
spiderInfo.startPage = getStartPage(line);
// Gid
spiderInfo.gid = Long.parseLong(IOUtils.readAsciiLine(is));
// Token
spiderInfo.token = IOUtils.readAsciiLine(is);
// Deprecated, mode, skip it
IOUtils.readAsciiLine(is);
// Preview pages
spiderInfo.previewPages = Integer.parseInt(IOUtils.readAsciiLine(is));
// Preview pre page
line = IOUtils.readAsciiLine(is);
if (version == 1) {
// Skip it
} else {
spiderInfo.previewPerPage = Integer.parseInt(line);
fun readInt(): Int {
return read().toInt()
}
// Pages
spiderInfo.pages = Integer.parseInt(IOUtils.readAsciiLine(is));
// Check pages
if (spiderInfo.pages <= 0) {
return null;
fun readLong(): Long {
return read().toLong()
}
// PToken
spiderInfo.pTokenMap = new SparseArray<>(spiderInfo.pages);
while (true) { // EOFException will raise
line = IOUtils.readAsciiLine(is);
int pos = line.indexOf(" ");
repeat(2) { read() } // We assert that only info v2
val gid = readLong()
val token = read()
read()
val previewPages = readInt()
val previewPerPage = readInt()
val pages = read().toInt()
val pTokenMap = arrayOfNulls<String>(pages)
val info = SpiderInfo(gid, pages, pTokenMap, 0, token, previewPages, previewPerPage)
runCatching {
val line = read()
val pos = line.indexOf(" ")
if (pos > 0) {
int index = Integer.parseInt(line.substring(0, pos));
String pToken = line.substring(pos + 1);
if (!TextUtils.isEmpty(pToken)) {
spiderInfo.pTokenMap.put(index, pToken);
val index = line.substring(0, pos).toInt()
val pToken = line.substring(pos + 1)
if (pToken.isNotEmpty()) {
pTokenMap[index] = pToken
}
} else {
Log.e(TAG, "Can't parse index and pToken, index = " + pos);
}
}
} catch (IOException | NumberFormatException e) {
// Ignore
return info
}
if (spiderInfo == null || spiderInfo.gid == -1 || spiderInfo.token == null ||
spiderInfo.pages == -1 || spiderInfo.pTokenMap == null) {
return null;
} else {
return spiderInfo;
}
const val TOKEN_FAILED = "failed"
}
public void write(@NonNull OutputStream os) {
OutputStreamWriter writer = null;
try {
writer = new OutputStreamWriter(os);
writer.write(VERSION_STR);
writer.write(Integer.toString(VERSION));
writer.write("\n");
writer.write(String.format("%08x", Math.max(startPage, 0))); // Avoid negative
writer.write("\n");
writer.write(Long.toString(gid));
writer.write("\n");
writer.write(token);
writer.write("\n");
writer.write("1");
writer.write("\n");
writer.write(Integer.toString(previewPages));
writer.write("\n");
writer.write(Integer.toString(previewPerPage));
writer.write("\n");
writer.write(Integer.toString(pages));
writer.write("\n");
for (int i = 0; i < pTokenMap.size(); i++) {
Integer key = pTokenMap.keyAt(i);
String value = pTokenMap.valueAt(i);
if (TOKEN_FAILED.equals(value) || TextUtils.isEmpty(value)) {
continue;
}
writer.write(Integer.toString(key));
writer.write(" ");
writer.write(value);
writer.write("\n");
}
writer.flush();
} catch (IOException e) {
// Ignore
} finally {
IOUtils.closeQuietly(writer);
IOUtils.closeQuietly(os);
}
}
}
}
app/src/main/java/com/hippo/ehviewer/spider/SpiderQueen.java
+20 -39
View File
@@ -21,7 +21,6 @@ import android.os.AsyncTask;
import android.os.Process;
import android.text.TextUtils;
import android.util.Log;
import android.util.SparseArray;
import android.webkit.MimeTypeMap;
import androidx.annotation.IntDef;
@@ -60,7 +59,6 @@ import com.hippo.yorozuya.thread.PriorityThread;
import com.hippo.yorozuya.thread.PriorityThreadFactory;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -705,13 +703,13 @@ public final class SpiderQueen implements Runnable {
// Read from cache
var entry = mSpiderInfoCache.get(Long.toString(mGalleryInfo.getGid()));
if (entry == null) return null;
try (entry; InputStream is = new FileInputStream(entry.getData().toFile())) {
spiderInfo = SpiderInfo.read(is);
try (entry) {
spiderInfo = SpiderInfo.read(entry.getData().toFile());
if (spiderInfo != null && spiderInfo.gid == mGalleryInfo.getGid() &&
spiderInfo.token.equals(mGalleryInfo.getToken())) {
return spiderInfo;
}
} catch (IOException e) {
} catch (Throwable e) {
e.printStackTrace();
}
@@ -719,7 +717,6 @@ public final class SpiderQueen implements Runnable {
}
private void readPreviews(String body, int index, SpiderInfo spiderInfo) throws ParseException {
spiderInfo.pages = GalleryDetailParser.parsePages(body);
spiderInfo.previewPages = GalleryDetailParser.parsePreviewPages(body);
PreviewSet previewSet = GalleryDetailParser.parsePreviewSet(body);
@@ -735,24 +732,21 @@ public final class SpiderQueen implements Runnable {
GalleryPageUrlParser.Result result = GalleryPageUrlParser.parse(previewSet.getPageUrlAt(i));
if (result != null) {
synchronized (mPTokenLock) {
spiderInfo.pTokenMap.put(result.page, result.pToken);
spiderInfo.pTokenMap[result.page] = result.pToken;
}
}
}
}
private SpiderInfo readSpiderInfoFromInternet() {
SpiderInfo spiderInfo = new SpiderInfo();
spiderInfo.gid = mGalleryInfo.getGid();
spiderInfo.token = mGalleryInfo.getToken();
Request request = new EhRequestBuilder(EhUrl.getGalleryDetailUrl(
mGalleryInfo.getGid(), mGalleryInfo.getToken(), 0, false), EhUrl.getReferer()).build();
try (Response response = mHttpClient.newCall(request).execute()) {
String body = response.body().string();
spiderInfo.pages = GalleryDetailParser.parsePages(body);
spiderInfo.pTokenMap = new SparseArray<>(spiderInfo.pages);
var pages = GalleryDetailParser.parsePages(body);
SpiderInfo spiderInfo = new SpiderInfo(mGalleryInfo.getGid(), pages);
spiderInfo.token = mGalleryInfo.getToken();
readPreviews(body, 0, spiderInfo);
return spiderInfo;
} catch (Throwable e) {
@@ -781,13 +775,13 @@ public final class SpiderQueen implements Runnable {
for (int i = 0; i < list.size(); i++) {
synchronized (mPTokenLock) {
spiderInfo.pTokenMap.put(i, list.get(i));
spiderInfo.pTokenMap[i] = list.get(i);
}
}
String pToken;
synchronized (mPTokenLock) {
pToken = spiderInfo.pTokenMap.get(index);
pToken = spiderInfo.pTokenMap[index];
}
return pToken;
} catch (Throwable e) {
@@ -830,7 +824,7 @@ public final class SpiderQueen implements Runnable {
String pToken;
synchronized (mPTokenLock) {
pToken = spiderInfo.pTokenMap.get(index);
pToken = spiderInfo.pTokenMap[index];
}
return pToken;
} catch (Throwable e) {
@@ -851,24 +845,14 @@ public final class SpiderQueen implements Runnable {
UniFile downloadDir = mSpiderDen.getDownloadDir();
if (downloadDir != null) {
UniFile file = downloadDir.createFile(SPIDER_INFO_FILENAME);
try {
spiderInfo.write(file.openOutputStream());
} catch (IOException e) {
ExceptionUtils.throwIfFatal(e);
// Ignore
}
spiderInfo.write(file);
}
// Write to cache
var editor = mSpiderInfoCache.edit(Long.toString(mGalleryInfo.getGid()));
if (editor == null) return;
try (OutputStream os = new FileOutputStream(editor.getData().toFile())) {
spiderInfo.write(os);
editor.commit();
} catch (IOException e) {
editor.abort();
e.printStackTrace();
}
spiderInfo.write(editor.getData().toFile());
editor.commit();
}
private void runInternal() {
@@ -944,7 +928,7 @@ public final class SpiderQueen implements Runnable {
// Check it in spider info
String pToken;
synchronized (mPTokenLock) {
pToken = spiderInfo.pTokenMap.get(index);
pToken = spiderInfo.pTokenMap[index];
}
if (pToken != null) {
// Get pToken from spider info, notify worker
@@ -969,7 +953,7 @@ public final class SpiderQueen implements Runnable {
if (null == pToken) {
// If failed, set the pToken "failed"
synchronized (mPTokenLock) {
spiderInfo.pTokenMap.put(index, SpiderInfo.TOKEN_FAILED);
spiderInfo.pTokenMap[index] = SpiderInfo.TOKEN_FAILED;
}
}
@@ -1469,12 +1453,9 @@ public final class SpiderQueen implements Runnable {
// Clear TOKEN_FAILED for force request
if (force) {
synchronized (mPTokenLock) {
int i = spiderInfo.pTokenMap.indexOfKey(index);
if (i >= 0) {
String pToken = spiderInfo.pTokenMap.valueAt(i);
if (SpiderInfo.TOKEN_FAILED.equals(pToken)) {
spiderInfo.pTokenMap.remove(i);
}
String pToken = spiderInfo.pTokenMap[index];
if (SpiderInfo.TOKEN_FAILED.equals(pToken)) {
spiderInfo.pTokenMap[index] = null;
}
}
}
@@ -1483,7 +1464,7 @@ public final class SpiderQueen implements Runnable {
// Get token
while (!mStoped) {
synchronized (mPTokenLock) {
pToken = spiderInfo.pTokenMap.get(index);
pToken = spiderInfo.pTokenMap[index];
}
if (pToken == null) {
mRequestPTokenQueue.add(index);
@@ -1522,7 +1503,7 @@ public final class SpiderQueen implements Runnable {
// Get token
while (previousIndex >= 0 && !mStoped) {
synchronized (mPTokenLock) {
previousPToken = spiderInfo.pTokenMap.get(previousIndex);
previousPToken = spiderInfo.pTokenMap[previousIndex];
}
if (previousPToken == null) {
mRequestPTokenQueue.add(previousIndex);
build.gradle
+1
View File
@@ -1,6 +1,7 @@
plugins {
id 'com.android.application' version '8.0.0-beta01' apply false
id 'org.jetbrains.kotlin.android' version '1.8.0' apply false
id 'org.jetbrains.kotlin.plugin.serialization' version '1.8.0' apply false
id 'com.google.devtools.ksp' version '1.8.0-1.0.9' apply false
}