yenru0/EhViewer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change tag database format to json for better maintainability

Browse Source
This commit is contained in:
FooIbar authored and tarsin committed 2022-12-15 11:38:32 +08:00
1 parent 55c7f6b896
commit 11d3799b03
5 files changed
+73 -170
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/src/main/java/com/hippo/beerbelly/SimpleDiskCache.java
+2 -20
View File
@@ -23,6 +23,7 @@ import androidx.annotation.Nullable;
import com.hippo.streampipe.InputStreamPipe;
import com.hippo.streampipe.OutputStreamPipe;
import com.hippo.util.HashCodeUtils;
import java.io.File;
import java.io.IOException;
@@ -85,32 +86,13 @@ public class SimpleDiskCache {
try {
final MessageDigest digest = MessageDigest.getInstance("MD5");
digest.update(key.getBytes());
cacheKey = bytesToHexString(digest.digest());
cacheKey = HashCodeUtils.bytesToHexString(digest.digest());
} catch (final NoSuchAlgorithmException e) {
cacheKey = String.valueOf(key.hashCode());
}
return cacheKey;
}
/**
* http://stackoverflow.com/questions/332079
*
* @param bytes The bytes to convert.
* @return A {@link String} converted from the bytes of a hashable key used
* to store a filename on the disk, to hex digits.
*/
private static String bytesToHexString(final byte[] bytes) {
final StringBuilder builder = new StringBuilder();
for (final byte b : bytes) {
final String hex = Integer.toHexString(0xFF & b);
if (hex.length() == 1) {
builder.append('0');
}
builder.append(hex);
}
return builder.toString();
}
public synchronized boolean isValid() {
return mDiskLruCache != null;
}
app/src/main/java/com/hippo/ehviewer/client/EhTagDatabase.java
+45 -146
View File
@@ -17,7 +17,6 @@
package com.hippo.ehviewer.client;
import android.content.Context;
import android.util.Base64;
import androidx.annotation.Nullable;
import androidx.core.util.Pair;
@@ -27,10 +26,14 @@ import com.hippo.ehviewer.EhApplication;
import com.hippo.ehviewer.R;
import com.hippo.ehviewer.Settings;
import com.hippo.util.ExceptionUtils;
import com.hippo.util.HashCodeUtils;
import com.hippo.util.IoThreadPoolExecutor;
import com.hippo.yorozuya.FileUtils;
import com.hippo.yorozuya.IOUtils;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -60,6 +63,7 @@ public class EhTagDatabase {
// TODO more lock for different language
private static final Lock lock = new ReentrantLock();
private static volatile EhTagDatabase instance;
private static boolean translate;
static {
NAMESPACE_TO_PREFIX.put("artist", "a:");
@@ -76,28 +80,31 @@ public class EhTagDatabase {
}
private final String name;
private final byte[] tags;
private final JSONObject tags;
private final ArrayList<Pair<String, String>> tagList;
public EhTagDatabase(String name, BufferedSource source) throws IOException {
this.name = name;
String[] tmp;
StringBuilder buffer = new StringBuilder();
source.readInt();
for (String i : source.readUtf8().split("\n")) {
tmp = i.split("\r", 2);
buffer.append(tmp[0]);
buffer.append("\r");
try {
buffer.append(new String(Base64.decode(tmp[1], Base64.DEFAULT), StandardCharsets.UTF_8));
} catch (Exception e) {
buffer.append(tmp[1]);
}
buffer.append("\n");
JSONObject tmpTags = null;
try {
tmpTags = new JSONObject(source.readString(StandardCharsets.UTF_8));
} catch (JSONException e) {
e.printStackTrace();
}
byte[] b = buffer.toString().getBytes(StandardCharsets.UTF_8);
int totalBytes = b.length;
tags = new byte[totalBytes];
System.arraycopy(b, 0, tags, 0, totalBytes);
tags = tmpTags;
if (tags != null) {
tagList = new ArrayList<>();
tags.keys().forEachRemaining(k -> {
try {
tagList.add(new Pair<>(k, tags.getString(k)));
} catch (JSONException e) {
e.printStackTrace();
}
});
} else {
tagList = null;
}
translate = Settings.getShowTagTranslations();
}
@Nullable
@@ -105,6 +112,10 @@ public class EhTagDatabase {
return instance;
}
public static void setTranslate(boolean value) {
translate = value;
}
@Nullable
public static String namespaceToPrefix(String namespace) {
return NAMESPACE_TO_PREFIX.get(namespace);
@@ -124,18 +135,16 @@ public class EhTagDatabase {
}
@Nullable
private static byte[] getFileContent(File file, int length) {
private static String getFileContent(File file) {
try (BufferedSource source = Okio.buffer(Okio.source(file))) {
byte[] content = new byte[length];
source.readFully(content);
return content;
return source.readString(StandardCharsets.UTF_8);
} catch (IOException e) {
return null;
}
}
@Nullable
private static byte[] getFileSha1(File file) {
private static String getFileSha1(File file) {
try (InputStream is = new FileInputStream(file)) {
MessageDigest digest = MessageDigest.getInstance("SHA-1");
int n;
@@ -143,45 +152,24 @@ public class EhTagDatabase {
while ((n = is.read(buffer)) != -1) {
digest.update(buffer, 0, n);
}
return digest.digest();
return HashCodeUtils.bytesToHexString(digest.digest());
} catch (IOException | NoSuchAlgorithmException e) {
return null;
}
}
private static boolean equals(byte[] b1, byte[] b2) {
if (b1 == null && b2 == null) {
return true;
}
if (b1 == null || b2 == null) {
return false;
}
if (b1.length != b2.length) {
return false;
}
for (int i = 0; i < b1.length; i++) {
if (b1[i] != b2[i]) {
return false;
}
}
return true;
}
private static boolean checkData(File sha1File, File dataFile) {
byte[] s1 = getFileContent(sha1File, 20);
var s1 = getFileContent(sha1File);
if (s1 == null) {
return false;
}
byte[] s2 = getFileSha1(dataFile);
var s2 = getFileSha1(dataFile);
if (s2 == null) {
return false;
}
return equals(s1, s2);
return s1.equals(s2);
}
private static boolean save(OkHttpClient client, String url, File file) {
@@ -302,108 +290,20 @@ public class EhTagDatabase {
}
public String getTranslation(String tag) {
return search(tags, tag.getBytes(StandardCharsets.UTF_8));
}
@Nullable
private String search(byte[] tags, byte[] tag) {
int low = 0;
int high = tags.length;
while (low < high) {
int start = (low + high) / 2;
// Look for the starting '\n'
while (start > -1 && tags[start] != '\n') {
start--;
}
start++;
// Look for the middle '\r'.
int middle = 1;
while (tags[start + middle] != '\r') {
middle++;
}
// Look for the ending '\n'
int end = middle + 1;
while (tags[start + end] != '\n') {
end++;
}
int compare;
int tagIndex = 0;
int curIndex = start;
for (; ; ) {
int tagByte = tag[tagIndex] & 0xff;
int curByte = tags[curIndex] & 0xff;
compare = tagByte - curByte;
if (compare != 0) {
break;
}
tagIndex++;
curIndex++;
if (tagIndex == tag.length && curIndex == start + middle) {
break;
}
if (tagIndex == tag.length) {
compare = -1;
break;
}
if (curIndex == start + middle) {
compare = 1;
break;
}
}
if (compare < 0) {
high = start - 1;
} else if (compare > 0) {
low = start + end + 1;
} else {
return new String(tags, start + middle + 1, end - middle - 1, StandardCharsets.UTF_8);
}
try {
return tags.getString(tag);
} catch (JSONException e) {
e.printStackTrace();
}
return null;
}
public ArrayList<Pair<String, String>> suggest(String keyword) {
return searchTag(tags, keyword);
}
private ArrayList<Pair<String, String>> searchTag(byte[] tags, String keyword) {
ArrayList<Pair<String, String>> searchHints = new ArrayList<>();
int begin = 0;
while (begin < tags.length - 1) {
int start = begin;
// Look for the starting '\n'
while (tags[start] != '\n') {
start++;
}
// Look for the middle '\r'.
int middle = 1;
while (tags[start + middle] != '\r') {
middle++;
}
// Look for the ending '\n'
int end = middle + 1;
while (tags[start + end] != '\n') {
end++;
}
begin = start + end;
String hint = null;
if (Settings.getShowTagTranslations()) {
byte[] hintBytes = new byte[end - middle - 1];
System.arraycopy(tags, start + middle + 1, hintBytes, 0, end - middle - 1);
hint = new String(hintBytes, StandardCharsets.UTF_8);
}
byte[] tagBytes = new byte[middle];
System.arraycopy(tags, start + 1, tagBytes, 0, middle);
String tag = new String(tagBytes, StandardCharsets.UTF_8);
for (int i = 0; i < tagList.size(); i++) {
var tmp = tagList.get(i);
String tag = tmp.first;
String hint = translate ? tmp.second : null;
int index = tag.indexOf(':');
boolean keywordMatches;
if (index == -1 || index >= tag.length() - 1 || keyword.length() > 2) {
@@ -421,7 +321,6 @@ public class EhTagDatabase {
if (searchHints.size() > 20) {
break;
}
}
return searchHints;
}
app/src/main/java/com/hippo/ehviewer/ui/fragment/EhFragment.kt
+3
View File
@@ -76,6 +76,9 @@ class EhFragment : BasePreferenceFragment() {
} else if (Settings.KEY_SHOW_TAG_TRANSLATIONS == key) {
if (java.lang.Boolean.TRUE == newValue) {
EhTagDatabase.update(requireActivity())
EhTagDatabase.setTranslate(true)
} else {
EhTagDatabase.setTranslate(false)
}
} else if (Settings.KEY_BLACK_DARK_THEME == key) {
if (requireActivity().resources.configuration.uiMode and Configuration.UI_MODE_NIGHT_YES > 0) {
app/src/main/java/com/hippo/util/HashCodeUtils.java
+19
View File
@@ -39,4 +39,23 @@ public class HashCodeUtils {
}
return hash;
}
/**
* http://stackoverflow.com/questions/332079
*
* @param bytes The bytes to convert.
* @return A {@link String} converted from the bytes of a hashable key used
* to store a filename on the disk, to hex digits.
*/
public static String bytesToHexString(final byte[] bytes) {
final StringBuilder builder = new StringBuilder();
for (final byte b : bytes) {
final String hex = Integer.toHexString(0xFF & b);
if (hex.length() == 1) {
builder.append('0');
}
builder.append(hex);
}
return builder.toString();
}
}
app/src/main/res/values/arrays.xml
+4 -4
View File
@@ -209,10 +209,10 @@
</string-array>
<string-array name="tag_translation_metadata" tools:ignore="InconsistentArrays">
<item>tag-translations-zh-rCN.sha1</item>
<item>https://raw.githubusercontent.com/Mapaler/EhViewer/tag-translations/tag-translations/tag-translations-zh-rCN.sha1</item>
<item>tag-translations-zh-rCN</item>
<item>https://raw.githubusercontent.com/Mapaler/EhViewer/tag-translations/tag-translations/tag-translations-zh-rCN</item>
<item>tag-translations-zh-rCN.json.sha1</item>
<item>https://raw.githubusercontent.com/FooIbar/EhTagTranslation/tag-translations/tag-translations/tag-translations-zh-rCN.json.sha1</item>
<item>tag-translations-zh-rCN.json</item>
<item>https://raw.githubusercontent.com/FooIbar/EhTagTranslation/tag-translations/tag-translations/tag-translations-zh-rCN.json</item>
</string-array>
<string-array name="proxy_types">