From cf0659db85ae5d00dfba7d8711da865b14b20934 Mon Sep 17 00:00:00 2001 From: seven332 Date: Thu, 24 Mar 2016 18:48:03 +0800 Subject: [PATCH] Improve gallery detail parsing --- .../client/parser/GalleryDetailParser.java | 298 +++++++++++------- yorozuya | 2 +- 2 files changed, 188 insertions(+), 112 deletions(-) diff --git a/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.java b/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.java index 0b8bb5fa1..4cc907675 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.java +++ b/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.java @@ -32,6 +32,7 @@ import com.hippo.ehviewer.client.exception.ParseException; import com.hippo.ehviewer.client.exception.PiningException; import com.hippo.util.JsoupUtils; import com.hippo.yorozuya.NumberUtils; +import com.hippo.yorozuya.StringUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -51,55 +52,22 @@ import java.util.regex.Pattern; public class GalleryDetailParser { - private static final DateFormat WEB_COMMENT_DATE_FORMAT = new SimpleDateFormat("dd MMMMM yyyy, HH:mm z", Locale.US); - - private static final Pattern ERROR_PATTERN = Pattern.compile("
\n

([^<]+)

"); - private static final Pattern DETAIL_PATTERN = Pattern.compile( - "var gid = (\\d+)" // 1 gid - + ".+?" - + "var token = \"([a-z0-9A]+)\"" // 2 token - + ".+?" - + "
]+>
" // 3 thumb - + "
" - + "
" - + "

([^<>]+)

" // 4 title - + "

([^<>]*)

" // 5 title_jpn might be empty string - + "
" - + ".+?" - + "]*onclick=\"return popUp\\('([^']+)'[^)]+\\)\">Torrent Download \\( (\\d+) \\)" // 6 torrentUrl, 7 torrentCount - + ".+?" - + "
]+><[^<>]*alt=\"([\\w\\-]+)\"[^<>]*>
" // 8 category - + "
]+>([^<>]+)" // 9 uploader - + ".+?" - + "]*>Posted:]*>([\\w\\-\\s:]+)" // 10 posted - + "]*>Parent:]*>(?:]*>)?([^<>]+)(?:)?" // 11 parent - + "]*>Visible:]*>([^<>]+)" // 12 visible - + "]*>Language:]*>([^<>]+)(?:]*>[^<>]*)?" // 13 language - + "]*>File Size:]*>([^<>]+)(?:]*>([^<>]+))?" // 14 File size, 15 resize - + "]*>Length:]*>([\\d,]+) pages" // 16 pageCount - + "]*>Favorited:<[^<>]*>([^<>]+)" // 17 Favorite times "([\d,]+) times" or "Once" or "Never" - + ".+?" - + "([\\d,]+)" // 18 ratedTimes - + "" - + "]*>([^<>]+)", // 19 rating "Average: x.xx" or "Not Yet Rated" - Pattern.DOTALL); - - private static final Pattern IS_FAVORED_PATTERN = Pattern.compile("]*>(.+?)"); // isFavored "Favorite Gallery" for favorite - - private static final Pattern TAG_GROUP_PATTERN = Pattern.compile("]+>([\\w\\s]+):(?:]+>]+>[\\w\\s]+
)+"); - private static final Pattern TAG_PATTERN = Pattern.compile("]+>]+>([\\w\\s]+)"); - - - private static final Pattern COMMENT_PATTERN = Pattern.compile("
Posted on ([^<>]+) by:   ]+>([^<>]+).+?
]*>(.+?)
]*>Length:]*>([\\d,]+) pages"); - public static final Pattern PREVIEW_PAGES_PATTERN = Pattern.compile("]+>]+>([\\d,]+)]+>(?:]+>)?>(?:)?"); - private static final Pattern NORMAL_PREVIEW_PATTERN = Pattern.compile("
]*>]*width:(\\d+)[^<>]*height:(\\d+)[^<>]*\\((.+?)\\)[^<>]*-(\\d+)px[^<>]*>]*href=\"(.+?)\"[^<>]*>\"([\\d,]+)\"");\"([\\d,]+)\".+?src=\"(.+?)\"");\n

([^<]+)

"); + private static final Pattern PATTERN_DETAIL = Pattern.compile("var gid = (\\d+).+?var token = \"([a-f0-9]+)\"", Pattern.DOTALL); + private static final Pattern PATTERN_TORRENT = Pattern.compile("]*onclick=\"return popUp\\('([^']+)'[^)]+\\)\">Torrent Download \\( (\\d+) \\)"); + private static final Pattern PATTERN_TAG_GROUP = Pattern.compile("]+>([\\w\\s]+):(?:]+>]+>[\\w\\s]+
)+"); + private static final Pattern PATTERN_TAG = Pattern.compile("]+>]+>([\\w\\s]+)
"); + private static final Pattern PATTERN_COMMENT = Pattern.compile("
Posted on ([^<>]+) by:   ]+>([^<>]+).+?
]*>(.+?)
]*>Length:]*>([\\d,]+) pages"); + private static final Pattern PATTERN_PREVIEW_PAGES = Pattern.compile("]+>]+>([\\d,]+)]+>(?:]+>)?>(?:)?"); + private static final Pattern PATTERN_NORMAL_PREVIEW = Pattern.compile("
]*>]*width:(\\d+)[^<>]*height:(\\d+)[^<>]*\\((.+?)\\)[^<>]*-(\\d+)px[^<>]*>]*href=\"(.+?)\"[^<>]*>\"([\\d,]+)\"");\"([\\d,]+)\".+?src=\"(.+?)\"");= ratingStr.length()) { - gd.rating = 0f; + throw new ParseException("Can't parse gallery detail", body); + } + + matcher = PATTERN_TORRENT.matcher(body); + if (matcher.find()) { + gd.torrentUrl = StringUtils.unescapeXml(StringUtils.trim(matcher.group(1))); + gd.torrentCount = NumberUtils.parseIntSafely(matcher.group(2), 0); + } else { + gd.torrentCount = 0; + gd.torrentUrl = ""; + } + + try { + Element gm = JsoupUtils.getElementByClass(d, "gm"); + + // Thumb url + Element gd1 = gm.getElementById("gd1"); + try { + gd.thumb = StringUtils.trim(gd1.child(0).attr("src")); + } catch (Exception e) { + gd.thumb = ""; + } + + // Title + Element gn = gm.getElementById("gn"); + if (null != gn) { + gd.title = StringUtils.trim(gn.text()); } else { - gd.rating = NumberUtils.parseFloatSafely(ratingStr.substring(index + 1), 0f); + gd.title = ""; + } + + // Jpn title + Element gj = gm.getElementById("gj"); + if (null != gj) { + gd.titleJpn = StringUtils.trim(gj.text()); + } else { + gd.titleJpn = ""; + } + + // Category + Element gdc = gm.getElementById("gdc"); + try { + gd.category = EhUtils.getCategory(gdc.child(0).child(0).attr("alt")); + } catch (Exception e) { + gd.category = EhUtils.UNKNOWN; + } + + // Uploader + Element gdn = gm.getElementById("gdn"); + if (null != gdn) { + gd.uploader = StringUtils.trim(gdn.text()); + } else { + gd.uploader = ""; + } + + Element gdd = gm.getElementById("gdd"); + gd.posted = ""; + gd.parent = ""; + gd.visible = ""; + gd.visible = ""; + gd.size = ""; + gd.pages = 0; + gd.favoredTimes = 0; + try { + Elements es = gdd.child(0).child(0).children(); + for (int i = 0, n = es.size(); i < n; i++) { + parseDetailInfo(gd, es.get(i), body); + } + } catch (Exception e) { + // Ignore + } + + // Rating count + Element rating_count = gm.getElementById("rating_count"); + if (null != rating_count) { + gd.ratedTimes = NumberUtils.parseIntSafely( + StringUtils.trim(rating_count.text()), 0); + } else { + gd.ratedTimes = 0; + } + + // Rating + Element rating_label = gm.getElementById("rating_label"); + if (null != rating_label) { + String ratingStr = StringUtils.trim(rating_label.text()); + if ("Not Yet Rated".equals(ratingStr)) { + gd.rating = Float.NaN; + } else { + int index = ratingStr.indexOf(' '); + if (index == -1 || index >= ratingStr.length()) { + gd.rating = 0f; + } else { + gd.rating = NumberUtils.parseFloatSafely(ratingStr.substring(index + 1), 0f); + } + } + } else { + gd.rating = Float.NaN; + } + + // isFavored + Element gdf = gm.getElementById("gdf"); + gd.isFavored = null != gdf && !StringUtils.trim(gdf.text()).equals("Add to Favorites"); + } catch (Exception e) { + throw new ParseException("Can't parse gallery detail", body); + } + } + + private static void parseDetailInfo(GalleryDetail gd, Element e, String body) { + Elements es = e.children(); + if (es.size() < 2) { + return; + } + + String key = StringUtils.trim(es.get(0).text()); + String value = StringUtils.trim(es.get(1).ownText()); + if (key.startsWith("Posted")) { + gd.posted = value; + } else if (key.startsWith("Parent")) { + gd.parent = value; + } else if (key.startsWith("Visible")) { + gd.visible = value; + } else if (key.startsWith("Language")) { + gd.language = value; + } else if (key.startsWith("File Size")) { + gd.size = value; + } else if (key.startsWith("Length")) { + int index = value.indexOf(' '); + if (index >= 0) { + gd.pages = NumberUtils.parseIntSafely(value.substring(0, index), 1); + } else { + gd.pages = 1; + } + } else if (key.startsWith("Favorited")) { + switch (value) { + case "Never": + gd.favoredTimes = 0; + break; + case "Once": + gd.favoredTimes = 1; + break; + default: + int index = value.indexOf(' '); + if (index == -1) { + gd.favoredTimes = 0; + } else { + gd.favoredTimes = NumberUtils.parseIntSafely(value.substring(0, index), 0); + } + break; } } - - m = IS_FAVORED_PATTERN.matcher(body); - gd.isFavored = m.find() && "Favorite Gallery".equals(ParserUtils.trim(m.group(1))); } @Nullable @@ -251,7 +319,7 @@ public class GalleryDetailParser { private static GalleryTagGroup[] parseTagGroups(String body) throws EhException { List list = new LinkedList<>(); - Matcher m = TAG_GROUP_PATTERN.matcher(body); + Matcher m = PATTERN_TAG_GROUP.matcher(body); while (m.find()) { GalleryTagGroup tagGroup = new GalleryTagGroup(); tagGroup.groupName = ParserUtils.trim(m.group(1)); @@ -263,7 +331,7 @@ public class GalleryDetailParser { } private static void parseGroup(GalleryTagGroup tagGroup, String body) { - Matcher m = TAG_PATTERN.matcher(body); + Matcher m = PATTERN_TAG.matcher(body); while (m.find()) { tagGroup.addTag(ParserUtils.trim(m.group(1))); } @@ -317,7 +385,7 @@ public class GalleryDetailParser { public static GalleryComment[] parseComments(String body) { List list = new LinkedList<>(); - Matcher m = COMMENT_PATTERN.matcher(body); + Matcher m = PATTERN_COMMENT.matcher(body); while (m.find()) { String webDateString = ParserUtils.trim(m.group(1)); Date date; @@ -353,7 +421,7 @@ public class GalleryDetailParser { * Parse preview pages with regular expressions */ public static int parsePreviewPages(String body) throws ParseException { - Matcher m = PREVIEW_PAGES_PATTERN.matcher(body); + Matcher m = PATTERN_PREVIEW_PAGES.matcher(body); int previewPages = -1; if (m.find()) { previewPages = ParserUtils.parseInt(m.group(1)); @@ -370,7 +438,7 @@ public class GalleryDetailParser { * Parse pages with regular expressions */ public static int parsePages(String body) throws ParseException { - Matcher m = PAGES_PATTERN.matcher(body); + Matcher m = PATTERN_PAGES.matcher(body); if (m.find()) { return ParserUtils.parseInt(m.group(1)); } else { @@ -378,6 +446,14 @@ public class GalleryDetailParser { } } + public static PreviewSet parsePreviewSet(Document d, String body) throws ParseException { + try { + return parseLargePreviewSet(d, body); + } catch (ParseException e) { + return parseNormalPreviewSet(body); + } + } + public static PreviewSet parsePreviewSet(String body) throws ParseException { try { return parseLargePreviewSet(body); @@ -389,10 +465,10 @@ public class GalleryDetailParser { /** * Parse large previews with regular expressions */ - private static LargePreviewSet parseLargePreviewSet(Document document, String body) throws ParseException { + private static LargePreviewSet parseLargePreviewSet(Document d, String body) throws ParseException { try { LargePreviewSet largePreviewSet = new LargePreviewSet(); - Element gdt = document.getElementById("gdt"); + Element gdt = d.getElementById("gdt"); Elements gdtls = gdt.getElementsByClass("gdtl"); int n = gdtls.size(); if (n <= 0) { @@ -417,7 +493,7 @@ public class GalleryDetailParser { * Parse large previews with regular expressions */ private static LargePreviewSet parseLargePreviewSet(String body) throws ParseException { - Matcher m = LARGE_PREVIEW_PATTERN.matcher(body); + Matcher m = PATTERN_LARGE_PREVIEW.matcher(body); LargePreviewSet largePreviewSet = new LargePreviewSet(); while (m.find()) { @@ -436,7 +512,7 @@ public class GalleryDetailParser { * Parse normal previews with regular expressions */ private static NormalPreviewSet parseNormalPreviewSet(String body) throws ParseException { - Matcher m = NORMAL_PREVIEW_PATTERN.matcher(body); + Matcher m = PATTERN_NORMAL_PREVIEW.matcher(body); NormalPreviewSet normalPreviewSet = new NormalPreviewSet(); while (m.find()) { normalPreviewSet.addItem(ParserUtils.parseInt(m.group(6)) - 1, @@ -446,7 +522,7 @@ public class GalleryDetailParser { } if (normalPreviewSet.size() == 0) { - throw new ParseException("Can't parse large preview", body); + throw new ParseException("Can't parse normal preview", body); } return normalPreviewSet; diff --git a/yorozuya b/yorozuya index 467ca2a2d..33e84adb9 160000 --- a/yorozuya +++ b/yorozuya @@ -1 +1 @@ -Subproject commit 467ca2a2dd0965ee5072e4f3ae1845369dd122ac +Subproject commit 33e84adb9cac19ca40eaa26045781a41a3a34579