From 11d3799b032827c8d8adc8aca42d6fb315ae6176 Mon Sep 17 00:00:00 2001 From: FooIbar <118464521+FooIbar@users.noreply.github.com> Date: Thu, 15 Dec 2022 08:14:19 +0800 Subject: [PATCH] Change tag database format to json for better maintainability --- .../com/hippo/beerbelly/SimpleDiskCache.java | 22 +- .../hippo/ehviewer/client/EhTagDatabase.java | 191 +++++------------- .../hippo/ehviewer/ui/fragment/EhFragment.kt | 3 + .../java/com/hippo/util/HashCodeUtils.java | 19 ++ app/src/main/res/values/arrays.xml | 8 +- 5 files changed, 73 insertions(+), 170 deletions(-) diff --git a/app/src/main/java/com/hippo/beerbelly/SimpleDiskCache.java b/app/src/main/java/com/hippo/beerbelly/SimpleDiskCache.java index b3c162130..6813316d3 100644 --- a/app/src/main/java/com/hippo/beerbelly/SimpleDiskCache.java +++ b/app/src/main/java/com/hippo/beerbelly/SimpleDiskCache.java @@ -23,6 +23,7 @@ import androidx.annotation.Nullable; import com.hippo.streampipe.InputStreamPipe; import com.hippo.streampipe.OutputStreamPipe; +import com.hippo.util.HashCodeUtils; import java.io.File; import java.io.IOException; @@ -85,32 +86,13 @@ public class SimpleDiskCache { try { final MessageDigest digest = MessageDigest.getInstance("MD5"); digest.update(key.getBytes()); - cacheKey = bytesToHexString(digest.digest()); + cacheKey = HashCodeUtils.bytesToHexString(digest.digest()); } catch (final NoSuchAlgorithmException e) { cacheKey = String.valueOf(key.hashCode()); } return cacheKey; } - /** - * http://stackoverflow.com/questions/332079 - * - * @param bytes The bytes to convert. - * @return A {@link String} converted from the bytes of a hashable key used - * to store a filename on the disk, to hex digits. - */ - private static String bytesToHexString(final byte[] bytes) { - final StringBuilder builder = new StringBuilder(); - for (final byte b : bytes) { - final String hex = Integer.toHexString(0xFF & b); - if (hex.length() == 1) { - builder.append('0'); - } - builder.append(hex); - } - return builder.toString(); - } - public synchronized boolean isValid() { return mDiskLruCache != null; } diff --git a/app/src/main/java/com/hippo/ehviewer/client/EhTagDatabase.java b/app/src/main/java/com/hippo/ehviewer/client/EhTagDatabase.java index 87065311e..530a368a3 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/EhTagDatabase.java +++ b/app/src/main/java/com/hippo/ehviewer/client/EhTagDatabase.java @@ -17,7 +17,6 @@ package com.hippo.ehviewer.client; import android.content.Context; -import android.util.Base64; import androidx.annotation.Nullable; import androidx.core.util.Pair; @@ -27,10 +26,14 @@ import com.hippo.ehviewer.EhApplication; import com.hippo.ehviewer.R; import com.hippo.ehviewer.Settings; import com.hippo.util.ExceptionUtils; +import com.hippo.util.HashCodeUtils; import com.hippo.util.IoThreadPoolExecutor; import com.hippo.yorozuya.FileUtils; import com.hippo.yorozuya.IOUtils; +import org.json.JSONException; +import org.json.JSONObject; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -60,6 +63,7 @@ public class EhTagDatabase { // TODO more lock for different language private static final Lock lock = new ReentrantLock(); private static volatile EhTagDatabase instance; + private static boolean translate; static { NAMESPACE_TO_PREFIX.put("artist", "a:"); @@ -76,28 +80,31 @@ public class EhTagDatabase { } private final String name; - private final byte[] tags; + private final JSONObject tags; + private final ArrayList> tagList; public EhTagDatabase(String name, BufferedSource source) throws IOException { this.name = name; - String[] tmp; - StringBuilder buffer = new StringBuilder(); - source.readInt(); - for (String i : source.readUtf8().split("\n")) { - tmp = i.split("\r", 2); - buffer.append(tmp[0]); - buffer.append("\r"); - try { - buffer.append(new String(Base64.decode(tmp[1], Base64.DEFAULT), StandardCharsets.UTF_8)); - } catch (Exception e) { - buffer.append(tmp[1]); - } - buffer.append("\n"); + JSONObject tmpTags = null; + try { + tmpTags = new JSONObject(source.readString(StandardCharsets.UTF_8)); + } catch (JSONException e) { + e.printStackTrace(); } - byte[] b = buffer.toString().getBytes(StandardCharsets.UTF_8); - int totalBytes = b.length; - tags = new byte[totalBytes]; - System.arraycopy(b, 0, tags, 0, totalBytes); + tags = tmpTags; + if (tags != null) { + tagList = new ArrayList<>(); + tags.keys().forEachRemaining(k -> { + try { + tagList.add(new Pair<>(k, tags.getString(k))); + } catch (JSONException e) { + e.printStackTrace(); + } + }); + } else { + tagList = null; + } + translate = Settings.getShowTagTranslations(); } @Nullable @@ -105,6 +112,10 @@ public class EhTagDatabase { return instance; } + public static void setTranslate(boolean value) { + translate = value; + } + @Nullable public static String namespaceToPrefix(String namespace) { return NAMESPACE_TO_PREFIX.get(namespace); @@ -124,18 +135,16 @@ public class EhTagDatabase { } @Nullable - private static byte[] getFileContent(File file, int length) { + private static String getFileContent(File file) { try (BufferedSource source = Okio.buffer(Okio.source(file))) { - byte[] content = new byte[length]; - source.readFully(content); - return content; + return source.readString(StandardCharsets.UTF_8); } catch (IOException e) { return null; } } @Nullable - private static byte[] getFileSha1(File file) { + private static String getFileSha1(File file) { try (InputStream is = new FileInputStream(file)) { MessageDigest digest = MessageDigest.getInstance("SHA-1"); int n; @@ -143,45 +152,24 @@ public class EhTagDatabase { while ((n = is.read(buffer)) != -1) { digest.update(buffer, 0, n); } - return digest.digest(); + return HashCodeUtils.bytesToHexString(digest.digest()); } catch (IOException | NoSuchAlgorithmException e) { return null; } } - private static boolean equals(byte[] b1, byte[] b2) { - if (b1 == null && b2 == null) { - return true; - } - if (b1 == null || b2 == null) { - return false; - } - - if (b1.length != b2.length) { - return false; - } - - for (int i = 0; i < b1.length; i++) { - if (b1[i] != b2[i]) { - return false; - } - } - - return true; - } - private static boolean checkData(File sha1File, File dataFile) { - byte[] s1 = getFileContent(sha1File, 20); + var s1 = getFileContent(sha1File); if (s1 == null) { return false; } - byte[] s2 = getFileSha1(dataFile); + var s2 = getFileSha1(dataFile); if (s2 == null) { return false; } - return equals(s1, s2); + return s1.equals(s2); } private static boolean save(OkHttpClient client, String url, File file) { @@ -302,108 +290,20 @@ public class EhTagDatabase { } public String getTranslation(String tag) { - return search(tags, tag.getBytes(StandardCharsets.UTF_8)); - } - - @Nullable - private String search(byte[] tags, byte[] tag) { - int low = 0; - int high = tags.length; - while (low < high) { - int start = (low + high) / 2; - // Look for the starting '\n' - while (start > -1 && tags[start] != '\n') { - start--; - } - start++; - - // Look for the middle '\r'. - int middle = 1; - while (tags[start + middle] != '\r') { - middle++; - } - - // Look for the ending '\n' - int end = middle + 1; - while (tags[start + end] != '\n') { - end++; - } - - int compare; - int tagIndex = 0; - int curIndex = start; - - for (; ; ) { - int tagByte = tag[tagIndex] & 0xff; - int curByte = tags[curIndex] & 0xff; - compare = tagByte - curByte; - if (compare != 0) { - break; - } - - tagIndex++; - curIndex++; - if (tagIndex == tag.length && curIndex == start + middle) { - break; - } - if (tagIndex == tag.length) { - compare = -1; - break; - } - if (curIndex == start + middle) { - compare = 1; - break; - } - } - - if (compare < 0) { - high = start - 1; - } else if (compare > 0) { - low = start + end + 1; - } else { - return new String(tags, start + middle + 1, end - middle - 1, StandardCharsets.UTF_8); - } + try { + return tags.getString(tag); + } catch (JSONException e) { + e.printStackTrace(); } return null; } public ArrayList> suggest(String keyword) { - return searchTag(tags, keyword); - } - - private ArrayList> searchTag(byte[] tags, String keyword) { ArrayList> searchHints = new ArrayList<>(); - int begin = 0; - while (begin < tags.length - 1) { - int start = begin; - // Look for the starting '\n' - while (tags[start] != '\n') { - start++; - } - - // Look for the middle '\r'. - int middle = 1; - while (tags[start + middle] != '\r') { - middle++; - } - - // Look for the ending '\n' - int end = middle + 1; - while (tags[start + end] != '\n') { - end++; - } - - begin = start + end; - - String hint = null; - if (Settings.getShowTagTranslations()) { - byte[] hintBytes = new byte[end - middle - 1]; - System.arraycopy(tags, start + middle + 1, hintBytes, 0, end - middle - 1); - hint = new String(hintBytes, StandardCharsets.UTF_8); - } - byte[] tagBytes = new byte[middle]; - System.arraycopy(tags, start + 1, tagBytes, 0, middle); - String tag = new String(tagBytes, StandardCharsets.UTF_8); + for (int i = 0; i < tagList.size(); i++) { + var tmp = tagList.get(i); + String tag = tmp.first; + String hint = translate ? tmp.second : null; int index = tag.indexOf(':'); boolean keywordMatches; if (index == -1 || index >= tag.length() - 1 || keyword.length() > 2) { @@ -421,7 +321,6 @@ public class EhTagDatabase { if (searchHints.size() > 20) { break; } - } return searchHints; } diff --git a/app/src/main/java/com/hippo/ehviewer/ui/fragment/EhFragment.kt b/app/src/main/java/com/hippo/ehviewer/ui/fragment/EhFragment.kt index 5cb4f1584..b74b29f4c 100644 --- a/app/src/main/java/com/hippo/ehviewer/ui/fragment/EhFragment.kt +++ b/app/src/main/java/com/hippo/ehviewer/ui/fragment/EhFragment.kt @@ -76,6 +76,9 @@ class EhFragment : BasePreferenceFragment() { } else if (Settings.KEY_SHOW_TAG_TRANSLATIONS == key) { if (java.lang.Boolean.TRUE == newValue) { EhTagDatabase.update(requireActivity()) + EhTagDatabase.setTranslate(true) + } else { + EhTagDatabase.setTranslate(false) } } else if (Settings.KEY_BLACK_DARK_THEME == key) { if (requireActivity().resources.configuration.uiMode and Configuration.UI_MODE_NIGHT_YES > 0) { diff --git a/app/src/main/java/com/hippo/util/HashCodeUtils.java b/app/src/main/java/com/hippo/util/HashCodeUtils.java index 13e9b7f0f..dee884947 100644 --- a/app/src/main/java/com/hippo/util/HashCodeUtils.java +++ b/app/src/main/java/com/hippo/util/HashCodeUtils.java @@ -39,4 +39,23 @@ public class HashCodeUtils { } return hash; } + + /** + * http://stackoverflow.com/questions/332079 + * + * @param bytes The bytes to convert. + * @return A {@link String} converted from the bytes of a hashable key used + * to store a filename on the disk, to hex digits. + */ + public static String bytesToHexString(final byte[] bytes) { + final StringBuilder builder = new StringBuilder(); + for (final byte b : bytes) { + final String hex = Integer.toHexString(0xFF & b); + if (hex.length() == 1) { + builder.append('0'); + } + builder.append(hex); + } + return builder.toString(); + } } diff --git a/app/src/main/res/values/arrays.xml b/app/src/main/res/values/arrays.xml index f58820028..efa99cd60 100644 --- a/app/src/main/res/values/arrays.xml +++ b/app/src/main/res/values/arrays.xml @@ -209,10 +209,10 @@ - tag-translations-zh-rCN.sha1 - https://raw.githubusercontent.com/Mapaler/EhViewer/tag-translations/tag-translations/tag-translations-zh-rCN.sha1 - tag-translations-zh-rCN - https://raw.githubusercontent.com/Mapaler/EhViewer/tag-translations/tag-translations/tag-translations-zh-rCN + tag-translations-zh-rCN.json.sha1 + https://raw.githubusercontent.com/FooIbar/EhTagTranslation/tag-translations/tag-translations/tag-translations-zh-rCN.json.sha1 + tag-translations-zh-rCN.json + https://raw.githubusercontent.com/FooIbar/EhTagTranslation/tag-translations/tag-translations/tag-translations-zh-rCN.json