Support tag translations

2019-01-20 00:49:54 +08:00
1 parent 3904406d74
commit c580938806
@@ -156,4 +156,19 @@ public class AppConfig {
            IOUtils.closeQuietly(os);
        }
    }
+
+    @Nullable
+    public static File getFilesDir(String name) {
+        File dir = sContext.getFilesDir();
+        if (dir == null) {
+            return null;
+        }
+
+        dir = new File(dir, name);
+        if (dir.isDirectory() || dir.mkdirs()) {
+            return dir;
+        } else {
+            return null;
+        }
+    }
 }
@@ -78,6 +78,8 @@ public class EhApplication extends RecordingApplication {
    private static final boolean DEBUG_PRINT_IMAGE_COUNT = false;
    private static final long DEBUG_PRINT_INTERVAL = 3000L;

+    private static EhApplication instance;
+
    private final IntIdGenerator mIdGenerator = new IntIdGenerator();
    private final HashMap<Integer, Object> mGlobalStuffMap = new HashMap<>();
    private EhCookieStore mEhCookieStore;
@@ -92,9 +94,15 @@ public class EhApplication extends RecordingApplication {

    private final List<Activity> mActivityList = new ArrayList<>();

+    public static EhApplication getInstance() {
+        return instance;
+    }
+
    @SuppressLint("StaticFieldLeak")
    @Override
    public void onCreate() {
+        instance = this;
+
        super.onCreate();

        GetText.initialize(this);
@@ -340,6 +340,13 @@ public class Settings {
        return getBoolean(KEY_SHOW_GALLERY_PAGES, DEFAULT_SHOW_GALLERY_PAGES);
    }

+    public static final String KEY_SHOW_TAG_TRANSLATIONS = "show_tag_translations";
+    private static final boolean DEFAULT_SHOW_TAG_TRANSLATIONS = false;
+
+    public static boolean getShowTagTranslations() {
+        return getBoolean(KEY_SHOW_TAG_TRANSLATIONS, DEFAULT_SHOW_TAG_TRANSLATIONS);
+    }
+
    public static final String KEY_DEFAULT_CATEGORIES = "default_categories";
    public static final int DEFAULT_DEFAULT_CATEGORIES = EhUtils.ALL_CATEGORY;

@@ -0,0 +1,355 @@
+/*
+ * Copyright 2019 Hippo Seven
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hippo.ehviewer.client;
+
+import android.support.annotation.Nullable;
+import android.util.Base64;
+import com.hippo.ehviewer.AppConfig;
+import com.hippo.ehviewer.EhApplication;
+import com.hippo.util.IoThreadPoolExecutor;
+import com.hippo.util.TextUrl;
+import com.hippo.yorozuya.FileUtils;
+import com.hippo.yorozuya.IOUtils;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import okhttp3.Call;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
+import okhttp3.ResponseBody;
+import okio.BufferedSource;
+import okio.Okio;
+
+public class EhTagDatabase {
+
+  private final String name;
+  private final byte[] tags;
+
+  public EhTagDatabase(String name, BufferedSource source) throws IOException {
+    this.name = name;
+    int totalBytes = source.readInt();
+    tags = new byte[totalBytes];
+    source.readFully(tags);
+  }
+
+  public String getTranslation(String tag) {
+    return search(tags, tag.getBytes(TextUrl.UTF_8));
+  }
+
+  @Nullable
+  private String search(byte[] tags, byte[] tag) {
+    int low = 0;
+    int high = tags.length;
+    while (low < high) {
+      int start = (low + high) / 2;
+      // Look for the starting '\n'
+      while (start > -1 && tags[start] != '\n') {
+        start--;
+      }
+      start++;
+
+      // Look for the middle '\r'.
+      int middle = 1;
+      while (tags[start + middle] != '\r') {
+        middle++;
+      }
+
+      // Look for the ending '\n'
+      int end = middle + 1;
+      while (tags[start + end] != '\n') {
+        end++;
+      }
+
+      int compare;
+      int tagIndex = 0;
+      int curIndex = start;
+
+      for (;;) {
+        int tagByte = tag[tagIndex] & 0xff;
+        int curByte = tags[curIndex] & 0xff;
+        compare = tagByte - curByte;
+        if (compare != 0) {
+          break;
+        }
+
+        tagIndex++;
+        curIndex++;
+        if (tagIndex == tag.length && curIndex == start + middle) {
+          break;
+        }
+        if (tagIndex == tag.length) {
+          compare = -1;
+          break;
+        }
+        if (curIndex == start + middle) {
+          compare = 1;
+          break;
+        }
+      }
+
+      if (compare < 0) {
+        high = start - 1;
+      } else if (compare > 0) {
+        low = start + end + 1;
+      } else {
+        byte[] bytes = Base64.decode(tags, start + middle + 1, end - middle - 1, Base64.DEFAULT);
+        return new String(bytes, TextUrl.UTF_8);
+      }
+    }
+    return null;
+  }
+
+
+  private static final String[] METADATA_ZH_CN = new String[] {
+      "tag-translations-zh-rCN.sha1",
+      "https://github.com/seven332/EhViewer/raw/tag-translations/tag-translations/tag-translations-zh-rCN.sha1",
+      "tag-translations-zh-rCN",
+      "https://github.com/seven332/EhViewer/raw/tag-translations/tag-translations/tag-translations-zh-rCN",
+  };
+
+  private static final Map<String, String> NAMESPACE_TO_PREFIX = new HashMap<>();
+
+  static {
+    NAMESPACE_TO_PREFIX.put("artist", "a:");
+    NAMESPACE_TO_PREFIX.put("character", "c:");
+    NAMESPACE_TO_PREFIX.put("female", "f:");
+    NAMESPACE_TO_PREFIX.put("group", "g:");
+    NAMESPACE_TO_PREFIX.put("language", "l:");
+    NAMESPACE_TO_PREFIX.put("male", "m:");
+    NAMESPACE_TO_PREFIX.put("misc", "");
+    NAMESPACE_TO_PREFIX.put("parody", "p:");
+    NAMESPACE_TO_PREFIX.put("reclass", "r:");
+  }
+
+  private static volatile EhTagDatabase instance;
+  // TODO more lock for different language
+  private static Lock lock = new ReentrantLock();
+
+  @Nullable
+  public static EhTagDatabase getInstance() {
+    return instance;
+  }
+
+  @Nullable
+  public static String namespaceToPrefix(String namespace) {
+    return NAMESPACE_TO_PREFIX.get(namespace);
+  }
+
+  private static String[] getMetadata() {
+    Locale locale = Locale.getDefault();
+    if ("zh".equals(locale.getLanguage()) && "CN".equals(locale.getCountry())) {
+      return METADATA_ZH_CN;
+    }
+    return null;
+  }
+
+  public static boolean isPossible() {
+    return getMetadata() != null;
+  }
+
+  @Nullable
+  private static byte[] getFileContent(File file, int length) {
+    try (BufferedSource source = Okio.buffer(Okio.source(file))) {
+      byte[] content = new byte[length];
+      source.readFully(content);
+      return content;
+    } catch (IOException e) {
+      return null;
+    }
+  }
+
+  @Nullable
+  private static byte[] getFileSha1(File file) {
+    try (InputStream is = new FileInputStream(file)) {
+      MessageDigest digest = MessageDigest.getInstance("SHA-1");
+      int n;
+      byte[] buffer = new byte[4 * 1024];
+      while ((n = is.read(buffer)) != -1) {
+        digest.update(buffer, 0, n);
+      }
+      return digest.digest();
+    } catch (IOException e) {
+      return null;
+    } catch (NoSuchAlgorithmException e) {
+      return null;
+    }
+  }
+
+  private static boolean equals(byte[] b1, byte[] b2) {
+    if (b1 == null && b2 == null) {
+      return true;
+    }
+    if (b1 == null || b2 == null) {
+      return false;
+    }
+
+    if (b1.length != b2.length) {
+      return false;
+    }
+
+    for (int i = 0; i < b1.length; i++) {
+      if (b1[i] != b2[i]) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  private static boolean checkData(File sha1File, File dataFile) {
+    byte[] s1 = getFileContent(sha1File, 20);
+    if (s1 == null) {
+      return false;
+    }
+
+    byte[] s2 = getFileSha1(dataFile);
+    if (s2 == null) {
+      return false;
+    }
+
+    return equals(s1, s2);
+  }
+
+  private static boolean save(OkHttpClient client, String url, File file) {
+    Request request = new Request.Builder().url(url).build();
+    Call call = client.newCall(request);
+    try (Response response = call.execute()) {
+      if (!response.isSuccessful()) {
+        return false;
+      }
+      ResponseBody body = response.body();
+      if (body == null) {
+        return false;
+      }
+
+      try (InputStream is = body.byteStream(); OutputStream os = new FileOutputStream(file)) {
+        IOUtils.copy(is, os);
+      }
+
+      return true;
+    } catch (IOException e) {
+      return false;
+    }
+  }
+
+  public static void update() {
+    String[] urls = getMetadata();
+    if (urls == null || urls.length != 4) {
+      // Clear tags if it's not possible
+      instance = null;
+      return;
+    }
+
+    String sha1Name = urls[0];
+    String sha1Url = urls[1];
+    String dataName = urls[2];
+    String dataUrl = urls[3];
+
+    // Clear tags if name if different
+    EhTagDatabase tmp = instance;
+    if (tmp != null && !tmp.name.equals(dataName)) {
+      instance = null;
+    }
+
+    IoThreadPoolExecutor.getInstance().execute(() -> {
+      if (!lock.tryLock()) {
+        return;
+      }
+
+      try {
+        File dir = AppConfig.getFilesDir("tag-translations");
+        if (dir == null) {
+          return;
+        }
+
+        // Check current sha1 and current data
+        File sha1File = new File(dir, sha1Name);
+        File dataFile = new File(dir, dataName);
+        if (!checkData(sha1File, dataFile)) {
+          FileUtils.delete(sha1File);
+          FileUtils.delete(dataFile);
+        }
+
+        // Read current EhTagDatabase
+        if (instance == null && dataFile.exists()) {
+          try (BufferedSource source = Okio.buffer(Okio.source(dataFile))) {
+            instance = new EhTagDatabase(dataName, source);
+          } catch (IOException e) {
+            FileUtils.delete(sha1File);
+            FileUtils.delete(dataFile);
+          }
+        }
+
+        OkHttpClient client = EhApplication.getOkHttpClient(EhApplication.getInstance());
+
+        // Save new sha1
+        File tempSha1File = new File(dir, sha1Name + ".tmp");
+        if (!save(client, sha1Url, tempSha1File)) {
+          FileUtils.delete(tempSha1File);
+          return;
+        }
+
+        // Check new sha1 and current data
+        if (checkData(tempSha1File, dataFile)) {
+          // The data is the same
+          FileUtils.delete(tempSha1File);
+          return;
+        }
+
+        // Save new data
+        File tempDataFile = new File(dir, dataName + ".tmp");
+        if (!save(client, dataUrl, tempDataFile)) {
+          FileUtils.delete(tempDataFile);
+          return;
+        }
+
+        // Check new sha1 and new data
+        if (!checkData(tempSha1File, tempDataFile)) {
+          FileUtils.delete(tempSha1File);
+          FileUtils.delete(tempDataFile);
+          return;
+        }
+
+        // Replace current sha1 and current data with new sha1 and new data
+        FileUtils.delete(sha1File);
+        FileUtils.delete(dataFile);
+        tempSha1File.renameTo(sha1File);
+        tempDataFile.renameTo(dataFile);
+
+        // Read new EhTagDatabase
+        try (BufferedSource source = Okio.buffer(Okio.source(dataFile))) {
+          instance = new EhTagDatabase(dataName, source);
+        } catch (IOException e) {
+          // Ignore
+        }
+      } finally {
+        lock.unlock();
+      }
+    });
+  }
+}
@@ -48,6 +48,7 @@ import com.hippo.ehviewer.AppConfig;
 import com.hippo.ehviewer.EhApplication;
 import com.hippo.ehviewer.R;
 import com.hippo.ehviewer.Settings;
+import com.hippo.ehviewer.client.EhTagDatabase;
 import com.hippo.ehviewer.client.EhUrlOpener;
 import com.hippo.ehviewer.client.EhUtils;
 import com.hippo.ehviewer.client.data.ListUrlBuilder;
@@ -362,6 +363,8 @@ public final class MainActivity extends StageActivity
        } else {
            onRestore(savedInstanceState);
        }
+
+        EhTagDatabase.update();
    }

    private String getThemeText() {
@@ -24,6 +24,7 @@ import android.preference.PreferenceFragment;
 import com.hippo.ehviewer.EhApplication;
 import com.hippo.ehviewer.R;
 import com.hippo.ehviewer.Settings;
+import com.hippo.ehviewer.client.EhTagDatabase;

 public class EhFragment extends PreferenceFragment
        implements Preference.OnPreferenceChangeListener {
@@ -39,6 +40,7 @@ public class EhFragment extends PreferenceFragment
        Preference listMode = findPreference(Settings.KEY_LIST_MODE);
        Preference detailSize = findPreference(Settings.KEY_DETAIL_SIZE);
        Preference thumbSize = findPreference(Settings.KEY_THUMB_SIZE);
+        Preference showTagTranslations = findPreference(Settings.KEY_SHOW_TAG_TRANSLATIONS);

        theme.setOnPreferenceChangeListener(this);
        applyNavBarThemeColor.setOnPreferenceChangeListener(this);
@@ -46,10 +48,15 @@ public class EhFragment extends PreferenceFragment
        listMode.setOnPreferenceChangeListener(this);
        detailSize.setOnPreferenceChangeListener(this);
        thumbSize.setOnPreferenceChangeListener(this);
+        showTagTranslations.setOnPreferenceChangeListener(this);

        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
            getPreferenceScreen().removePreference(applyNavBarThemeColor);
        }
+
+        if (!EhTagDatabase.isPossible()) {
+            getPreferenceScreen().removePreference(showTagTranslations);
+        }
    }

    @Override
@@ -71,6 +78,10 @@ public class EhFragment extends PreferenceFragment
            getActivity().setResult(Activity.RESULT_OK);
        } else if (Settings.KEY_THUMB_SIZE.equals(key)) {
            getActivity().setResult(Activity.RESULT_OK);
+        } else if (Settings.KEY_SHOW_TAG_TRANSLATIONS.equals(key)) {
+            if (Boolean.TRUE.equals(newValue)) {
+                EhTagDatabase.update();
+            }
        }
        return true;
    }
@@ -69,6 +69,7 @@ import com.hippo.ehviewer.client.EhCacheKeyFactory;
 import com.hippo.ehviewer.client.EhClient;
 import com.hippo.ehviewer.client.EhFilter;
 import com.hippo.ehviewer.client.EhRequest;
+import com.hippo.ehviewer.client.EhTagDatabase;
 import com.hippo.ehviewer.client.EhUrl;
 import com.hippo.ehviewer.client.EhUtils;
 import com.hippo.ehviewer.client.data.GalleryComment;
@@ -868,6 +869,7 @@ public class GalleryDetailScene extends BaseScene implements View.OnClickListene
            mNoTags.setVisibility(View.GONE);
        }

+        EhTagDatabase ehTags = Settings.getShowTagTranslations() ? EhTagDatabase.getInstance() : null;
        int colorTag = AttrResources.getAttrColor(context, R.attr.tagBackgroundColor);
        int colorName = AttrResources.getAttrColor(context, R.attr.tagGroupBackgroundColor);
        for (GalleryTagGroup tg : tagGroups) {
@@ -875,18 +877,34 @@ public class GalleryDetailScene extends BaseScene implements View.OnClickListene
            ll.setOrientation(LinearLayout.HORIZONTAL);
            mTags.addView(ll);

+            String readableTagName = null;
+            if (ehTags != null) {
+                readableTagName = ehTags.getTranslation("n:" + tg.groupName);
+            }
+
            TextView tgName = (TextView) inflater.inflate(R.layout.item_gallery_tag, ll, false);
            ll.addView(tgName);
-            tgName.setText(tg.groupName);
+            tgName.setText(readableTagName != null ? readableTagName : tg.groupName);
            tgName.setBackgroundDrawable(new RoundSideRectDrawable(colorName));

+            String prefix = EhTagDatabase.namespaceToPrefix(tg.groupName);
+            if (prefix == null) {
+                prefix = "";
+            }
+
            AutoWrapLayout awl = new AutoWrapLayout(context);
            ll.addView(awl, ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT);
            for (int j = 0, z = tg.size(); j < z; j++) {
                TextView tag = (TextView) inflater.inflate(R.layout.item_gallery_tag, awl, false);
                awl.addView(tag);
                String tagStr = tg.getTagAt(j);
-                tag.setText(tagStr);
+
+                String readableTag = null;
+                if (ehTags != null) {
+                    readableTag = ehTags.getTranslation(prefix + tagStr);
+                }
+
+                tag.setText(readableTag != null ? readableTag : tagStr);
                tag.setBackgroundDrawable(new RoundSideRectDrawable(colorTag));
                tag.setTag(R.id.tag, tg.groupName + ":" + tagStr);
                tag.setOnClickListener(this);
@@ -20,11 +20,14 @@ import android.text.Spannable;
 import android.text.SpannableString;
 import android.text.style.URLSpan;

+import java.nio.charset.Charset;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 public final class TextUrl {

+    public static final Charset UTF_8 = Charset.forName("UTF-8");
+
    private static final Pattern URL_PATTERN = Pattern.compile("(http|https)://[a-z0-9A-Z%-]+(\\.[a-z0-9A-Z%-]+)+(:\\d{1,5})?(/[a-zA-Z0-9-_~:#@!&',;=%/\\*\\.\\?\\+\\$\\[\\]\\(\\)]+)?/?");

    public static CharSequence handleTextUrl(CharSequence content) {