From 59e704c992ac426efcd4ed9dc3a93094b40e5ae0 Mon Sep 17 00:00:00 2001 From: seven332 Date: Thu, 17 Mar 2016 20:50:13 +0800 Subject: [PATCH] Add more jsoup parser --- .../com/hippo/ehviewer/client/EhEngine.java | 10 +- .../ehviewer/client/data/GalleryTagGroup.java | 3 +- .../client/parser/GalleryDetailParser.java | 175 +++++++++++++++++- .../hippo/ehviewer/spider/SpiderQueen.java | 2 +- 4 files changed, 177 insertions(+), 13 deletions(-) diff --git a/app/src/main/java/com/hippo/ehviewer/client/EhEngine.java b/app/src/main/java/com/hippo/ehviewer/client/EhEngine.java index cff9620a1..bef4e8666 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/EhEngine.java +++ b/app/src/main/java/com/hippo/ehviewer/client/EhEngine.java @@ -49,6 +49,8 @@ import com.hippo.yorozuya.IOUtils; import org.json.JSONArray; import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; import java.io.File; import java.io.FileOutputStream; @@ -296,8 +298,9 @@ public class EhEngine { code = response.code(); headers = response.headers(); body = response.body().string(); - return Pair.create(GalleryDetailParser.parseLargePreview(body), - GalleryDetailParser.parsePreviewPages(body)); + Document document = Jsoup.parse(body); + return Pair.create(GalleryDetailParser.parseLargePreviewSet(document, body), + GalleryDetailParser.parsePreviewPages(document, body)); } catch (Exception e) { throwException(call, code, headers, body, e); throw e; @@ -361,7 +364,8 @@ public class EhEngine { code = response.code(); headers = response.headers(); body = response.body().string(); - return GalleryDetailParser.parseComment(body); + Document document = Jsoup.parse(body); + return GalleryDetailParser.parseComments(document); } catch (Exception e) { throwException(call, code, headers, body, e); throw e; diff --git a/app/src/main/java/com/hippo/ehviewer/client/data/GalleryTagGroup.java b/app/src/main/java/com/hippo/ehviewer/client/data/GalleryTagGroup.java index 02f7d247a..c46d41e8d 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/data/GalleryTagGroup.java +++ b/app/src/main/java/com/hippo/ehviewer/client/data/GalleryTagGroup.java @@ -21,10 +21,11 @@ import android.os.Parcelable; import java.util.ArrayList; +// TODO Add url field? public class GalleryTagGroup implements Parcelable { public String groupName; - private ArrayList mTagList; + private final ArrayList mTagList; public void addTag(String tag) { mTagList.add(tag); diff --git a/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.java b/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.java index e769f38ce..305db2f94 100644 --- a/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.java +++ b/app/src/main/java/com/hippo/ehviewer/client/parser/GalleryDetailParser.java @@ -16,6 +16,9 @@ package com.hippo.ehviewer.client.parser; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; + import com.hippo.ehviewer.client.EhUtils; import com.hippo.ehviewer.client.data.GalleryComment; import com.hippo.ehviewer.client.data.GalleryDetail; @@ -26,10 +29,17 @@ import com.hippo.ehviewer.client.exception.EhException; import com.hippo.ehviewer.client.exception.OffensiveException; import com.hippo.ehviewer.client.exception.ParseException; import com.hippo.ehviewer.client.exception.PiningException; +import com.hippo.util.JsoupUtils; import com.hippo.yorozuya.NumberUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; import java.util.LinkedList; import java.util.List; @@ -86,6 +96,9 @@ public class GalleryDetailParser { private static final Pattern NORMAL_PREVIEW_PATTERN = Pattern.compile("
]*>]*width:(\\d+)[^<>]*height:(\\d+)[^<>]*\\((.+?)\\)[^<>]*-(\\d+)px[^<>]*>]*href=\"(.+?)\"[^<>]*>\"([\\d,]+)\"");\"([\\d,]+)\".+?src=\"(.+?)\""); 0 ? group : null; + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + /** + * Parse tag groups with html parser + */ + @NonNull + public static GalleryTagGroup[] parseTagGroups(Document document) { + try { + Element taglist = document.getElementById("taglist"); + Elements tagGroups = taglist.child(0).child(0).children(); + + List list = new ArrayList<>(tagGroups.size()); + for (int i = 0, n = tagGroups.size(); i < n; i++) { + GalleryTagGroup group = parseTagGroup(tagGroups.get(i)); + if (null != group) { + list.add(group); + } + } + return list.toArray(new GalleryTagGroup[list.size()]); + } catch (Exception e) { + e.printStackTrace(); + return EMPTY_GALLERY_TAG_GROUP_ARRAY; + } + } + + /** + * Parse tag groups with regular expressions + */ + @NonNull + private static GalleryTagGroup[] parseTagGroups(String body) throws EhException { List list = new LinkedList<>(); Matcher m = TAG_GROUP_PATTERN.matcher(body); @@ -199,7 +262,52 @@ public class GalleryDetailParser { } } - public static GalleryComment[] parseComment(String body) { + @Nullable + @SuppressWarnings("ConstantConditions") + public static GalleryComment parseComment(Element element) { + try { + GalleryComment comment = new GalleryComment(); + Element c3 = JsoupUtils.getElementByClass(element, "c3"); + String temp = c3.ownText(); + temp = temp.substring("Posted on ".length(), temp.length() - " by:  ".length()); + comment.time = WEB_COMMENT_DATE_FORMAT.parse(temp).getTime(); + comment.user = c3.child(0).text(); + comment.comment = JsoupUtils.getElementByClass(element, "c6").html(); + return comment; + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + /** + * Parse comments with html parser + */ + @NonNull + public static GalleryComment[] parseComments(Document document) { + try { + Element cdiv = document.getElementById("cdiv"); + Elements c1s = cdiv.getElementsByClass("c1"); + + List list = new ArrayList<>(c1s.size()); + for (int i = 0, n = c1s.size(); i < n; i++) { + GalleryComment comment = parseComment(c1s.get(i)); + if (null != comment) { + list.add(comment); + } + } + return list.toArray(new GalleryComment[list.size()]); + } catch (Exception e) { + e.printStackTrace(); + return EMPTY_GALLERY_COMMENT_ARRAY; + } + } + + /** + * Parse comments with regular expressions + */ + @NonNull + public static GalleryComment[] parseComments(String body) { List list = new LinkedList<>(); Matcher m = COMMENT_PATTERN.matcher(body); @@ -221,6 +329,22 @@ public class GalleryDetailParser { return list.toArray(new GalleryComment[list.size()]); } + /** + * Parse preview pages with html parser + */ + public static int parsePreviewPages(Document document, String body) throws ParseException { + try { + Elements elements = document.getElementsByClass("ptt").first().child(0).child(0).children(); + return Integer.parseInt(elements.get(elements.size() - 2).text()); + } catch (Exception e) { + e.printStackTrace(); + throw new ParseException("Can't parse preview pages", body); + } + } + + /** + * Parse preview pages with regular expressions + */ public static int parsePreviewPages(String body) throws ParseException { Matcher m = PREVIEW_PAGES_PATTERN.matcher(body); int previewPages = -1; @@ -235,6 +359,9 @@ public class GalleryDetailParser { return previewPages; } + /** + * Parse pages with regular expressions + */ public static int parsePages(String body) throws ParseException { Matcher m = PAGES_PATTERN.matcher(body); if (m.find()) { @@ -244,7 +371,37 @@ public class GalleryDetailParser { } } - public static LargePreviewSet parseLargePreview(String body) throws ParseException { + /** + * Parse large previews with regular expressions + */ + public static LargePreviewSet parseLargePreviewSet(Document document, String body) throws ParseException { + try { + LargePreviewSet largePreviewSet = new LargePreviewSet(); + Element gdt = document.getElementById("gdt"); + Elements gdtls = gdt.getElementsByClass("gdtl"); + int n = gdtls.size(); + if (n <= 0) { + throw new ParseException("Can't parse large preview", body); + } + for (int i = 0; i < n; i++) { + Element element = gdtls.get(i).child(0); + String pageUrl = element.attr("href"); + element = element.child(0); + String imageUrl = element.attr("src"); + int index = Integer.parseInt(element.attr("alt")) - 1; + largePreviewSet.addItem(index, imageUrl, pageUrl); + } + return largePreviewSet; + } catch (Exception e) { + e.printStackTrace(); + throw new ParseException("Can't parse large preview", body); + } + } + + /** + * Parse large previews with regular expressions + */ + public static LargePreviewSet parseLargePreviewSet(String body) throws ParseException { Matcher m = LARGE_PREVIEW_PATTERN.matcher(body); LargePreviewSet largePreviewSet = new LargePreviewSet(); @@ -260,6 +417,9 @@ public class GalleryDetailParser { return largePreviewSet; } + /** + * Parse normal previews with regular expressions + */ public static NormalPreviewSet parseNormalPreviewSet(String body) throws ParseException { Matcher m = NORMAL_PREVIEW_PATTERN.matcher(body); NormalPreviewSet normalPreviewSet = new NormalPreviewSet(); @@ -276,5 +436,4 @@ public class GalleryDetailParser { return normalPreviewSet; } - } diff --git a/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueen.java b/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueen.java index 763c227ff..b648c424c 100644 --- a/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueen.java +++ b/app/src/main/java/com/hippo/ehviewer/spider/SpiderQueen.java @@ -689,7 +689,7 @@ public class SpiderQueen implements Runnable { } } } catch (ParseException e) { - LargePreviewSet previewSet = GalleryDetailParser.parseLargePreview(body); + LargePreviewSet previewSet = GalleryDetailParser.parseLargePreviewSet(body); spiderInfo.previewPerPage = previewSet.size(); for (int i = 0, n = previewSet.size(); i < n; i++) { GalleryPageUrlParser.Result result = GalleryPageUrlParser.parse(previewSet.getPageUrlAt(i));