add extract by url

webmagic-core/src/main/java/us/codecraft/webmagic/annotation/ExtractByUrl.java

@@ -0,0 +1,18 @@
+package us.codecraft.webmagic.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+/**
+ * @author yihua.huang@dianping.com <br>
+ * @date: 13-8-1 <br>
+ * Time: 下午8:40 <br>
+ */
+@Retention(java.lang.annotation.RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD})
+public @interface ExtractByUrl {
+
+    String value() default "";
+
+}

webmagic-core/src/main/java/us/codecraft/webmagic/annotation/FieldExtractor.java

@@ -15,9 +15,20 @@ class FieldExtractor {
 
     private final Selector selector;
 
-    FieldExtractor(Field field, Selector selector) {
+    private final Source source;
+
+    static enum Source {Html, Url}
+
+    public FieldExtractor(Field field, Selector selector) {
+        this.field = field;
+        this.selector = selector;
+        this.source = Source.Html;
+    }
+
+    public FieldExtractor(Field field, Selector selector, Source source) {
         this.field = field;
         this.selector = selector;
+        this.source = source;
     }
 
     Field getField() {
@@ -27,4 +38,8 @@ class FieldExtractor {
     Selector getSelector() {
         return selector;
     }
+
+    Source getSource() {
+        return source;
+    }
 }

webmagic-core/src/main/java/us/codecraft/webmagic/annotation/ObjectPageProcessor.java

@@ -47,6 +47,7 @@ public class ObjectPageProcessor implements PageProcessor {
     public void process(Page page) {
         for (PageModelExtractor pageModelExtractor : pageModelExtractorList) {
             Object process = pageModelExtractor.process(page);
+            postProcessPageModel(pageModelExtractor.getClazz(), process);
             page.putField(pageModelExtractor.getClazz().getCanonicalName(), process);
         }
         for (String link : page.getHtml().links().all()) {
@@ -58,6 +59,9 @@ public class ObjectPageProcessor implements PageProcessor {
         }
     }
 
+    protected void postProcessPageModel(Class clazz, Object object){
+    }
+
     @Override
     public Site getSite() {
         return site;

webmagic-core/src/main/java/us/codecraft/webmagic/annotation/PageModelExtractor.java

@@ -38,6 +38,7 @@ class PageModelExtractor {
         for (Field field : clazz.getDeclaredFields()) {
             field.setAccessible(true);
             ExtractBy extractBy = field.getAnnotation(ExtractBy.class);
+            if (extractBy != null) {
                 String value = extractBy.value();
                 Selector selector;
                 switch (extractBy.type()) {
@@ -55,6 +56,15 @@ class PageModelExtractor {
                 }
                 fieldExtractors.add(new FieldExtractor(field, selector));
             }
+            ExtractByUrl extractByUrl = field.getAnnotation(ExtractByUrl.class);
+            if (extractByUrl != null) {
+                String regexPattern = extractByUrl.value();
+                if (regexPattern.trim().equals("")) {
+                    regexPattern = ".*";
+                }
+                fieldExtractors.add(new FieldExtractor(field, new RegexSelector(regexPattern), FieldExtractor.Source.Url));
+            }
+        }
     }
 
     private void initTargetUrlPatterns() {
@@ -84,7 +94,15 @@ class PageModelExtractor {
         try {
             o = clazz.newInstance();
             for (FieldExtractor fieldExtractor : fieldExtractors) {
+                switch (fieldExtractor.getSource()) {
+                    case Html:
                         fieldExtractor.getField().set(o, fieldExtractor.getSelector().select(page.getHtml().toString()));
+                        break;
+                    case Url:
+                        fieldExtractor.getField().set(o, fieldExtractor.getSelector().select(page.getUrl().toString()));
+                        break;
+                }
+
             }
         } catch (InstantiationException e) {
             e.printStackTrace();

webmagic-core/src/test/java/us/codecraft/webmagic/annotation/OschinaBlog.java

@@ -6,7 +6,7 @@ package us.codecraft.webmagic.annotation;
  * Time: 下午10:18 <br>
  */
 @TargetUrl("http://my.oschina.net/flashsword/blog/*")
-public class Blog {
+public class OschinaBlog {
 
     @ExtractBy("//title")
     private String title;
@@ -16,7 +16,7 @@ public class Blog {
 
     @Override
     public String toString() {
-        return "Blog{" +
+        return "OschinaBlog{" +
                 "title='" + title + '\'' +
                 ", content='" + content + '\'' +
                 '}';

webmagic-core/src/test/java/us/codecraft/webmagic/annotation/TestFetcher.java

@@ -15,7 +15,7 @@ public class TestFetcher {
     @Ignore("takes long")
     @Test
     public void test() {
-        Spider.create(ObjectPageProcessor.create(Site.me().addStartUrl("http://my.oschina.net/flashsword/blog/145796"), Blog.class)).run();
+        Spider.create(ObjectPageProcessor.create(Site.me().addStartUrl("http://my.oschina.net/flashsword/blog/145796"), OschinaBlog.class)).run();
 
     }
 

webmagic-core/src/test/java/us/codecraft/webmagic/selector/XpathSelectorTest.java

@@ -1168,7 +1168,7 @@ public class XpathSelectorTest {
                 + "    var location = window.location;\n"
                 + "    source_url = location.protocol + \"//\" + location.host + location.pathname + location.search;\n"
                 + "    pre.writeAttribute('codeable_id', post_id);\n"
-                + "    pre.writeAttribute('codeable_type', \"Blog\");\n"
+                + "    pre.writeAttribute('codeable_type', \"OschinaBlog\");\n"
                 + "    pre.writeAttribute('source_url', source_url);\n"
                 + "    pre.writeAttribute('pre_index', index);\n"
                 + "    pre.writeAttribute('title', 'jsoup 解析页面商品信息');\n"