From c58093880647a8fc64bf0a4e3b01ad753d83bcc2 Mon Sep 17 00:00:00 2001 From: seven332 Date: Sun, 20 Jan 2019 00:49:54 +0800 Subject: [PATCH] Support tag translations --- .../java/com/hippo/ehviewer/AppConfig.java | 15 + .../com/hippo/ehviewer/EhApplication.java | 8 + .../java/com/hippo/ehviewer/Settings.java | 7 + .../hippo/ehviewer/client/EhTagDatabase.java | 355 ++++++++++++++++++ .../com/hippo/ehviewer/ui/MainActivity.java | 3 + .../ehviewer/ui/fragment/EhFragment.java | 11 + .../ehviewer/ui/scene/GalleryDetailScene.java | 22 +- app/src/main/java/com/hippo/util/TextUrl.java | 3 + app/src/main/res/values-zh-rCN/strings.xml | 2 + app/src/main/res/values-zh-rHK/strings.xml | 2 + app/src/main/res/values-zh-rTW/strings.xml | 2 + app/src/main/res/values/strings.xml | 2 + app/src/main/res/xml/eh_settings.xml | 6 + .../ehviewer/client/EhTagDatabaseTest.java | 55 +++ .../hippo/ehviewer/client/EhTagDatabaseTest | Bin 0 -> 80 bytes 15 files changed, 491 insertions(+), 2 deletions(-) create mode 100644 app/src/main/java/com/hippo/ehviewer/client/EhTagDatabase.java create mode 100644 app/src/test/java/com/hippo/ehviewer/client/EhTagDatabaseTest.java create mode 100644 app/src/test/resources/com/hippo/ehviewer/client/EhTagDatabaseTest diff --git a/app/src/main/java/com/hippo/ehviewer/AppConfig.java b/app/src/main/java/com/hippo/ehviewer/AppConfig.java index 15898725f..9a5d848e6 100644 --- a/app/src/main/java/com/hippo/ehviewer/AppConfig.java +++ b/app/src/main/java/com/hippo/ehviewer/AppConfig.java @@ -156,4 +156,19 @@ public class AppConfig { IOUtils.closeQuietly(os); } } + + @Nullable + public static File getFilesDir(String name) { + File dir = sContext.getFilesDir(); + if (dir == null) { + return null; + } + + dir = new File(dir, name); + if (dir.isDirectory() || dir.mkdirs()) { + return dir; + } else { + return null; + } + } } diff --git a/app/src/main/java/com/hippo/ehviewer/EhApplication.java b/app/src/main/java/com/hippo/ehviewer/EhApplication.java index 64e67ec0b..554082212 100644 --- a/app/src/main/java/com/hippo/ehviewer/EhApplication.java +++ b/app/src/main/java/com/hippo/ehviewer/EhApplication.java @@ -78,6 +78,8 @@ public class EhApplication extends RecordingApplication { private static final boolean DEBUG_PRINT_IMAGE_COUNT = false; private static final long DEBUG_PRINT_INTERVAL = 3000L; + private static EhApplication instance; + private final IntIdGenerator mIdGenerator = new IntIdGenerator(); private final HashMap mGlobalStuffMap = new HashMap<>(); private EhCookieStore mEhCookieStore; @@ -92,9 +94,15 @@ public class EhApplication extends RecordingApplication { private final List mActivityList = new ArrayList<>(); + public static EhApplication getInstance() { + return instance; + } + @SuppressLint("StaticFieldLeak") @Override public void onCreate() { + instance = this; + super.onCreate(); GetText.initialize(this); diff --git a/app/src/main/java/com/hippo/ehviewer/Settings.java b/app/src/main/java/com/hippo/ehviewer/Settings.java index 612e65e25..e582a3d2f 100755 --- a/app/src/main/java/com/hippo/ehviewer/Settings.java +++ b/app/src/main/java/com/hippo/ehviewer/Settings.java @@ -340,6 +340,13 @@ public class Settings { return getBoolean(KEY_SHOW_GALLERY_PAGES, DEFAULT_SHOW_GALLERY_PAGES); } + public static final String KEY_SHOW_TAG_TRANSLATIONS = "show_tag_translations"; + private static final boolean DEFAULT_SHOW_TAG_TRANSLATIONS = false; + + public static boolean getShowTagTranslations() { + return getBoolean(KEY_SHOW_TAG_TRANSLATIONS, DEFAULT_SHOW_TAG_TRANSLATIONS); + } + public static final String KEY_DEFAULT_CATEGORIES = "default_categories"; public static final int DEFAULT_DEFAULT_CATEGORIES = EhUtils.ALL_CATEGORY; diff --git a/app/src/main/java/com/hippo/ehviewer/client/EhTagDatabase.java b/app/src/main/java/com/hippo/ehviewer/client/EhTagDatabase.java new file mode 100644 index 000000000..2d9eb1f40 --- /dev/null +++ b/app/src/main/java/com/hippo/ehviewer/client/EhTagDatabase.java @@ -0,0 +1,355 @@ +/* + * Copyright 2019 Hippo Seven + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hippo.ehviewer.client; + +import android.support.annotation.Nullable; +import android.util.Base64; +import com.hippo.ehviewer.AppConfig; +import com.hippo.ehviewer.EhApplication; +import com.hippo.util.IoThreadPoolExecutor; +import com.hippo.util.TextUrl; +import com.hippo.yorozuya.FileUtils; +import com.hippo.yorozuya.IOUtils; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import okhttp3.Call; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.BufferedSource; +import okio.Okio; + +public class EhTagDatabase { + + private final String name; + private final byte[] tags; + + public EhTagDatabase(String name, BufferedSource source) throws IOException { + this.name = name; + int totalBytes = source.readInt(); + tags = new byte[totalBytes]; + source.readFully(tags); + } + + public String getTranslation(String tag) { + return search(tags, tag.getBytes(TextUrl.UTF_8)); + } + + @Nullable + private String search(byte[] tags, byte[] tag) { + int low = 0; + int high = tags.length; + while (low < high) { + int start = (low + high) / 2; + // Look for the starting '\n' + while (start > -1 && tags[start] != '\n') { + start--; + } + start++; + + // Look for the middle '\r'. + int middle = 1; + while (tags[start + middle] != '\r') { + middle++; + } + + // Look for the ending '\n' + int end = middle + 1; + while (tags[start + end] != '\n') { + end++; + } + + int compare; + int tagIndex = 0; + int curIndex = start; + + for (;;) { + int tagByte = tag[tagIndex] & 0xff; + int curByte = tags[curIndex] & 0xff; + compare = tagByte - curByte; + if (compare != 0) { + break; + } + + tagIndex++; + curIndex++; + if (tagIndex == tag.length && curIndex == start + middle) { + break; + } + if (tagIndex == tag.length) { + compare = -1; + break; + } + if (curIndex == start + middle) { + compare = 1; + break; + } + } + + if (compare < 0) { + high = start - 1; + } else if (compare > 0) { + low = start + end + 1; + } else { + byte[] bytes = Base64.decode(tags, start + middle + 1, end - middle - 1, Base64.DEFAULT); + return new String(bytes, TextUrl.UTF_8); + } + } + return null; + } + + + private static final String[] METADATA_ZH_CN = new String[] { + "tag-translations-zh-rCN.sha1", + "https://github.com/seven332/EhViewer/raw/tag-translations/tag-translations/tag-translations-zh-rCN.sha1", + "tag-translations-zh-rCN", + "https://github.com/seven332/EhViewer/raw/tag-translations/tag-translations/tag-translations-zh-rCN", + }; + + private static final Map NAMESPACE_TO_PREFIX = new HashMap<>(); + + static { + NAMESPACE_TO_PREFIX.put("artist", "a:"); + NAMESPACE_TO_PREFIX.put("character", "c:"); + NAMESPACE_TO_PREFIX.put("female", "f:"); + NAMESPACE_TO_PREFIX.put("group", "g:"); + NAMESPACE_TO_PREFIX.put("language", "l:"); + NAMESPACE_TO_PREFIX.put("male", "m:"); + NAMESPACE_TO_PREFIX.put("misc", ""); + NAMESPACE_TO_PREFIX.put("parody", "p:"); + NAMESPACE_TO_PREFIX.put("reclass", "r:"); + } + + private static volatile EhTagDatabase instance; + // TODO more lock for different language + private static Lock lock = new ReentrantLock(); + + @Nullable + public static EhTagDatabase getInstance() { + return instance; + } + + @Nullable + public static String namespaceToPrefix(String namespace) { + return NAMESPACE_TO_PREFIX.get(namespace); + } + + private static String[] getMetadata() { + Locale locale = Locale.getDefault(); + if ("zh".equals(locale.getLanguage()) && "CN".equals(locale.getCountry())) { + return METADATA_ZH_CN; + } + return null; + } + + public static boolean isPossible() { + return getMetadata() != null; + } + + @Nullable + private static byte[] getFileContent(File file, int length) { + try (BufferedSource source = Okio.buffer(Okio.source(file))) { + byte[] content = new byte[length]; + source.readFully(content); + return content; + } catch (IOException e) { + return null; + } + } + + @Nullable + private static byte[] getFileSha1(File file) { + try (InputStream is = new FileInputStream(file)) { + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + int n; + byte[] buffer = new byte[4 * 1024]; + while ((n = is.read(buffer)) != -1) { + digest.update(buffer, 0, n); + } + return digest.digest(); + } catch (IOException e) { + return null; + } catch (NoSuchAlgorithmException e) { + return null; + } + } + + private static boolean equals(byte[] b1, byte[] b2) { + if (b1 == null && b2 == null) { + return true; + } + if (b1 == null || b2 == null) { + return false; + } + + if (b1.length != b2.length) { + return false; + } + + for (int i = 0; i < b1.length; i++) { + if (b1[i] != b2[i]) { + return false; + } + } + + return true; + } + + private static boolean checkData(File sha1File, File dataFile) { + byte[] s1 = getFileContent(sha1File, 20); + if (s1 == null) { + return false; + } + + byte[] s2 = getFileSha1(dataFile); + if (s2 == null) { + return false; + } + + return equals(s1, s2); + } + + private static boolean save(OkHttpClient client, String url, File file) { + Request request = new Request.Builder().url(url).build(); + Call call = client.newCall(request); + try (Response response = call.execute()) { + if (!response.isSuccessful()) { + return false; + } + ResponseBody body = response.body(); + if (body == null) { + return false; + } + + try (InputStream is = body.byteStream(); OutputStream os = new FileOutputStream(file)) { + IOUtils.copy(is, os); + } + + return true; + } catch (IOException e) { + return false; + } + } + + public static void update() { + String[] urls = getMetadata(); + if (urls == null || urls.length != 4) { + // Clear tags if it's not possible + instance = null; + return; + } + + String sha1Name = urls[0]; + String sha1Url = urls[1]; + String dataName = urls[2]; + String dataUrl = urls[3]; + + // Clear tags if name if different + EhTagDatabase tmp = instance; + if (tmp != null && !tmp.name.equals(dataName)) { + instance = null; + } + + IoThreadPoolExecutor.getInstance().execute(() -> { + if (!lock.tryLock()) { + return; + } + + try { + File dir = AppConfig.getFilesDir("tag-translations"); + if (dir == null) { + return; + } + + // Check current sha1 and current data + File sha1File = new File(dir, sha1Name); + File dataFile = new File(dir, dataName); + if (!checkData(sha1File, dataFile)) { + FileUtils.delete(sha1File); + FileUtils.delete(dataFile); + } + + // Read current EhTagDatabase + if (instance == null && dataFile.exists()) { + try (BufferedSource source = Okio.buffer(Okio.source(dataFile))) { + instance = new EhTagDatabase(dataName, source); + } catch (IOException e) { + FileUtils.delete(sha1File); + FileUtils.delete(dataFile); + } + } + + OkHttpClient client = EhApplication.getOkHttpClient(EhApplication.getInstance()); + + // Save new sha1 + File tempSha1File = new File(dir, sha1Name + ".tmp"); + if (!save(client, sha1Url, tempSha1File)) { + FileUtils.delete(tempSha1File); + return; + } + + // Check new sha1 and current data + if (checkData(tempSha1File, dataFile)) { + // The data is the same + FileUtils.delete(tempSha1File); + return; + } + + // Save new data + File tempDataFile = new File(dir, dataName + ".tmp"); + if (!save(client, dataUrl, tempDataFile)) { + FileUtils.delete(tempDataFile); + return; + } + + // Check new sha1 and new data + if (!checkData(tempSha1File, tempDataFile)) { + FileUtils.delete(tempSha1File); + FileUtils.delete(tempDataFile); + return; + } + + // Replace current sha1 and current data with new sha1 and new data + FileUtils.delete(sha1File); + FileUtils.delete(dataFile); + tempSha1File.renameTo(sha1File); + tempDataFile.renameTo(dataFile); + + // Read new EhTagDatabase + try (BufferedSource source = Okio.buffer(Okio.source(dataFile))) { + instance = new EhTagDatabase(dataName, source); + } catch (IOException e) { + // Ignore + } + } finally { + lock.unlock(); + } + }); + } +} diff --git a/app/src/main/java/com/hippo/ehviewer/ui/MainActivity.java b/app/src/main/java/com/hippo/ehviewer/ui/MainActivity.java index e745e5b91..cc7e7f6d7 100644 --- a/app/src/main/java/com/hippo/ehviewer/ui/MainActivity.java +++ b/app/src/main/java/com/hippo/ehviewer/ui/MainActivity.java @@ -48,6 +48,7 @@ import com.hippo.ehviewer.AppConfig; import com.hippo.ehviewer.EhApplication; import com.hippo.ehviewer.R; import com.hippo.ehviewer.Settings; +import com.hippo.ehviewer.client.EhTagDatabase; import com.hippo.ehviewer.client.EhUrlOpener; import com.hippo.ehviewer.client.EhUtils; import com.hippo.ehviewer.client.data.ListUrlBuilder; @@ -362,6 +363,8 @@ public final class MainActivity extends StageActivity } else { onRestore(savedInstanceState); } + + EhTagDatabase.update(); } private String getThemeText() { diff --git a/app/src/main/java/com/hippo/ehviewer/ui/fragment/EhFragment.java b/app/src/main/java/com/hippo/ehviewer/ui/fragment/EhFragment.java index 3c1a251f6..14ab727c4 100644 --- a/app/src/main/java/com/hippo/ehviewer/ui/fragment/EhFragment.java +++ b/app/src/main/java/com/hippo/ehviewer/ui/fragment/EhFragment.java @@ -24,6 +24,7 @@ import android.preference.PreferenceFragment; import com.hippo.ehviewer.EhApplication; import com.hippo.ehviewer.R; import com.hippo.ehviewer.Settings; +import com.hippo.ehviewer.client.EhTagDatabase; public class EhFragment extends PreferenceFragment implements Preference.OnPreferenceChangeListener { @@ -39,6 +40,7 @@ public class EhFragment extends PreferenceFragment Preference listMode = findPreference(Settings.KEY_LIST_MODE); Preference detailSize = findPreference(Settings.KEY_DETAIL_SIZE); Preference thumbSize = findPreference(Settings.KEY_THUMB_SIZE); + Preference showTagTranslations = findPreference(Settings.KEY_SHOW_TAG_TRANSLATIONS); theme.setOnPreferenceChangeListener(this); applyNavBarThemeColor.setOnPreferenceChangeListener(this); @@ -46,10 +48,15 @@ public class EhFragment extends PreferenceFragment listMode.setOnPreferenceChangeListener(this); detailSize.setOnPreferenceChangeListener(this); thumbSize.setOnPreferenceChangeListener(this); + showTagTranslations.setOnPreferenceChangeListener(this); if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { getPreferenceScreen().removePreference(applyNavBarThemeColor); } + + if (!EhTagDatabase.isPossible()) { + getPreferenceScreen().removePreference(showTagTranslations); + } } @Override @@ -71,6 +78,10 @@ public class EhFragment extends PreferenceFragment getActivity().setResult(Activity.RESULT_OK); } else if (Settings.KEY_THUMB_SIZE.equals(key)) { getActivity().setResult(Activity.RESULT_OK); + } else if (Settings.KEY_SHOW_TAG_TRANSLATIONS.equals(key)) { + if (Boolean.TRUE.equals(newValue)) { + EhTagDatabase.update(); + } } return true; } diff --git a/app/src/main/java/com/hippo/ehviewer/ui/scene/GalleryDetailScene.java b/app/src/main/java/com/hippo/ehviewer/ui/scene/GalleryDetailScene.java index b6a5db191..611cbdd8b 100755 --- a/app/src/main/java/com/hippo/ehviewer/ui/scene/GalleryDetailScene.java +++ b/app/src/main/java/com/hippo/ehviewer/ui/scene/GalleryDetailScene.java @@ -69,6 +69,7 @@ import com.hippo.ehviewer.client.EhCacheKeyFactory; import com.hippo.ehviewer.client.EhClient; import com.hippo.ehviewer.client.EhFilter; import com.hippo.ehviewer.client.EhRequest; +import com.hippo.ehviewer.client.EhTagDatabase; import com.hippo.ehviewer.client.EhUrl; import com.hippo.ehviewer.client.EhUtils; import com.hippo.ehviewer.client.data.GalleryComment; @@ -868,6 +869,7 @@ public class GalleryDetailScene extends BaseScene implements View.OnClickListene mNoTags.setVisibility(View.GONE); } + EhTagDatabase ehTags = Settings.getShowTagTranslations() ? EhTagDatabase.getInstance() : null; int colorTag = AttrResources.getAttrColor(context, R.attr.tagBackgroundColor); int colorName = AttrResources.getAttrColor(context, R.attr.tagGroupBackgroundColor); for (GalleryTagGroup tg : tagGroups) { @@ -875,18 +877,34 @@ public class GalleryDetailScene extends BaseScene implements View.OnClickListene ll.setOrientation(LinearLayout.HORIZONTAL); mTags.addView(ll); + String readableTagName = null; + if (ehTags != null) { + readableTagName = ehTags.getTranslation("n:" + tg.groupName); + } + TextView tgName = (TextView) inflater.inflate(R.layout.item_gallery_tag, ll, false); ll.addView(tgName); - tgName.setText(tg.groupName); + tgName.setText(readableTagName != null ? readableTagName : tg.groupName); tgName.setBackgroundDrawable(new RoundSideRectDrawable(colorName)); + String prefix = EhTagDatabase.namespaceToPrefix(tg.groupName); + if (prefix == null) { + prefix = ""; + } + AutoWrapLayout awl = new AutoWrapLayout(context); ll.addView(awl, ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); for (int j = 0, z = tg.size(); j < z; j++) { TextView tag = (TextView) inflater.inflate(R.layout.item_gallery_tag, awl, false); awl.addView(tag); String tagStr = tg.getTagAt(j); - tag.setText(tagStr); + + String readableTag = null; + if (ehTags != null) { + readableTag = ehTags.getTranslation(prefix + tagStr); + } + + tag.setText(readableTag != null ? readableTag : tagStr); tag.setBackgroundDrawable(new RoundSideRectDrawable(colorTag)); tag.setTag(R.id.tag, tg.groupName + ":" + tagStr); tag.setOnClickListener(this); diff --git a/app/src/main/java/com/hippo/util/TextUrl.java b/app/src/main/java/com/hippo/util/TextUrl.java index f9461dae9..a7cd35d3d 100644 --- a/app/src/main/java/com/hippo/util/TextUrl.java +++ b/app/src/main/java/com/hippo/util/TextUrl.java @@ -20,11 +20,14 @@ import android.text.Spannable; import android.text.SpannableString; import android.text.style.URLSpan; +import java.nio.charset.Charset; import java.util.regex.Matcher; import java.util.regex.Pattern; public final class TextUrl { + public static final Charset UTF_8 = Charset.forName("UTF-8"); + private static final Pattern URL_PATTERN = Pattern.compile("(http|https)://[a-z0-9A-Z%-]+(\\.[a-z0-9A-Z%-]+)+(:\\d{1,5})?(/[a-zA-Z0-9-_~:#@!&',;=%/\\*\\.\\?\\+\\$\\[\\]\\(\\)]+)?/?"); public static CharSequence handleTextUrl(CharSequence content) { diff --git a/app/src/main/res/values-zh-rCN/strings.xml b/app/src/main/res/values-zh-rCN/strings.xml index 7b5964be1..4a7599d2c 100755 --- a/app/src/main/res/values-zh-rCN/strings.xml +++ b/app/src/main/res/values-zh-rCN/strings.xml @@ -433,6 +433,8 @@ 显示默认标题。一般是英文标题 显示画廊页数 在画廊列表中显示页数。这会使获取画廊列表花费更多时间 + 显示标签翻译 + 显示翻译后的标签而非原始文字（需花费时间来下载数据文件） 过滤 过滤标题，上传者，标签和标签组 移动网络警告 diff --git a/app/src/main/res/values-zh-rHK/strings.xml b/app/src/main/res/values-zh-rHK/strings.xml index 8f3e6cd91..96997333f 100755 --- a/app/src/main/res/values-zh-rHK/strings.xml +++ b/app/src/main/res/values-zh-rHK/strings.xml @@ -432,6 +432,8 @@ 顯示默認標題。一般是英文標題 顯示畫廊頁數 在畫廊列表中顯示頁數。這會使獲取畫廊列表花費更多時間 + 顯示標籤翻譯 + 顯示翻譯後的標籤而非原始文字（需花費時間來下載數據文件） 過濾 過濾標題，上傳者，標籤和標籤組 移動網絡警告 diff --git a/app/src/main/res/values-zh-rTW/strings.xml b/app/src/main/res/values-zh-rTW/strings.xml index 2e721d1cc..c58581c36 100755 --- a/app/src/main/res/values-zh-rTW/strings.xml +++ b/app/src/main/res/values-zh-rTW/strings.xml @@ -432,6 +432,8 @@ 顯示預設的標題。通常是英文的 顯示圖庫頁數 在圖庫列表中顯示頁數。這會使取得圖庫列表時間增加 + 顯示標籤翻譯 + 顯示翻譯後的標籤而非原始文字（需花費時間來下載資料檔案） 篩選 篩選標題，上傳者，標籤和標籤命名空間 行動數據警告 diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index c0cd25d6b..7336da064 100755 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -476,6 +476,8 @@ Show default titles, which are usually in English. Show gallery pages Display the number of pages in the gallery list. It takes more time to load the gallery list + Show tag translations + Show tag translations instead of the original text (It takes time to download the data file) Filter Filter by title, uploader, tag and tag namespace Cellular network warning diff --git a/app/src/main/res/xml/eh_settings.xml b/app/src/main/res/xml/eh_settings.xml index b5f569227..b8749c2be 100644 --- a/app/src/main/res/xml/eh_settings.xml +++ b/app/src/main/res/xml/eh_settings.xml @@ -106,6 +106,12 @@ android:summary="@string/settings_eh_show_gallery_pages_summary" android:defaultValue="false"/> + +