add iteye sample

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -18,23 +18,24 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * <pre>
- *webmagic爬虫的入口类。
+ * webmagic爬虫的入口类。
  *
- *示例：
- *定义一个最简单的爬虫：
+ * 示例：
+ * 定义一个最简单的爬虫：
  *      Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*")).run();
  *
- *使用FilePipeline保存结果到文件:
+ * 使用FilePipeline保存结果到文件:
  *      Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*"))
  *          .pipeline(new FilePipeline("/data/temp/webmagic/")).run();
  *
- *使用FileCacheQueueScheduler缓存URL，关闭爬虫后下次自动从停止的页面继续抓取:
+ * 使用FileCacheQueueScheduler缓存URL，关闭爬虫后下次自动从停止的页面继续抓取:
  *      Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*"))
  *          .scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).run();
  * </pre>
+ *
  * @author code4crafter@gmail.com <br>
- * Date: 13-4-21
- * Time: 上午6:53
+ *         Date: 13-4-21
+ *         Time: 上午6:53
  */
 public class Spider implements Runnable, Task {
 
@@ -66,6 +67,7 @@ public class Spider implements Runnable, Task {
 
     /**
      * 使用已定义的抽取规则新建一个Spider。
+     *
      * @param pageProcessor 已定义的抽取规则
      */
     public Spider(PageProcessor pageProcessor) {
@@ -76,6 +78,7 @@ public class Spider implements Runnable, Task {
 
     /**
      * 使用已定义的抽取规则新建一个Spider。
+     *
      * @param pageProcessor 已定义的抽取规则
      * @return 新建的Spider
      */
@@ -85,6 +88,7 @@ public class Spider implements Runnable, Task {
 
     /**
      * 重新设置startUrls，会覆盖Site本身的startUrls。
+     *
      * @param startUrls
      * @return this
      */
@@ -96,6 +100,7 @@ public class Spider implements Runnable, Task {
 
     /**
      * 为爬虫设置一个唯一ID，用于标志任务，默认情况下使用domain作为uuid，对于单domain多任务的情况，请为重复任务设置不同的ID。
+     *
      * @param uuid 唯一ID
      * @return this
      */
@@ -106,6 +111,7 @@ public class Spider implements Runnable, Task {
 
     /**
      * 设置调度器。调度器用于保存待抓取URL，并可以进行去重、同步、持久化等工作。默认情况下使用内存中的阻塞队列进行调度。
+     *
      * @param scheduler 调度器
      * @return this
      */
@@ -117,6 +123,7 @@ public class Spider implements Runnable, Task {
 
     /**
      * 设置处理管道。处理管道用于最终抽取结果的后处理，例如：保存到文件、保存到数据库等。默认情况下会输出到控制台。
+     *
      * @param pipeline 处理管道
      * @return this
      */
@@ -148,7 +155,7 @@ public class Spider implements Runnable, Task {
             pipelines.add(new ConsolePipeline());
         }
         //singel thread
-        if (executorService==null){
+        if (executorService == null) {
             while (request != null) {
                 processRequest(request);
                 request = scheduler.poll(this);
@@ -217,13 +224,13 @@ public class Spider implements Runnable, Task {
         }
     }
 
-    private void checkIfNotRunning(){
-        if (!stat.compareAndSet(STAT_INIT,STAT_INIT)){
+    private void checkIfNotRunning() {
+        if (!stat.compareAndSet(STAT_INIT, STAT_INIT)) {
             throw new IllegalStateException("Spider is already running!");
         }
     }
 
-    public void runAsync(){
+    public void runAsync() {
         Thread thread = new Thread(this);
         thread.setDaemon(false);
         thread.start();
@@ -231,15 +238,19 @@ public class Spider implements Runnable, Task {
 
     /**
      * 建立多个线程下载
+     *
      * @param threadNum 线程数
      * @return this
      */
     public Spider thread(int threadNum) {
         checkIfNotRunning();
-        if (threadNum <= 1) {
+        if (threadNum <= 0) {
             throw new IllegalArgumentException("threadNum should be more than one!");
         }
-        synchronized (this){
+        if (threadNum == 1) {
+            return this;
+        }
+        synchronized (this) {
             this.executorService = ThreadUtils.newFixedThreadPool(threadNum);
         }
         return this;

webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java

@@ -46,7 +46,7 @@ public class FilePipeline implements Pipeline {
             file.mkdirs();
         }
         try {
-            PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl())));
+            PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl())+".html"));
             printWriter.println("url:\t" + resultItems.getRequest().getUrl());
             for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
                 printWriter.println(entry.getKey()+":\t"+entry.getValue());

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/IteyeBlogProcessor.java

@@ -0,0 +1,38 @@
+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.pipeline.FilePipeline;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+/**
+ * @author yihua.huang@dianping.com <br>
+ * @date: 13-7-26 <br>
+ * Time: 上午7:31 <br>
+ */
+public class IteyeBlogProcessor implements PageProcessor {
+
+    private Site site;
+
+    @Override
+    public void process(Page page) {
+        page.addTargetRequests(page.getHtml().links().regex(".*yanghaoli\\.iteye\\.com/blog/\\d+").all());
+        page.putField("title",page.getHtml().xpath("//title").toString());
+        page.putField("content",page.getHtml().smartContent().toString());
+    }
+
+    @Override
+    public Site getSite() {
+        if (site == null) {
+            site = Site.me().setDomain("yanghaoli.iteye.com").addStartUrl("http://yanghaoli.iteye.com/").
+                    setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31")
+            .setSleepTime(100).setRetryTimes(3);
+        }
+        return site;
+    }
+
+    public static void main(String[] args) {
+        Spider.create(new IteyeBlogProcessor()).thread(5).pipeline(new FilePipeline()).run();
+    }
+}

webmagic-samples/src/main/resources/log4j.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+    <appender name="stdout" class="org.apache.log4j.ConsoleAppender">
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
+        </layout>
+    </appender>
+
+    <logger name="org.springframework" additivity="false">
+        <level value="warn" />
+        <appender-ref ref="stdout" />
+    </logger>
+
+    <logger name="net.sf.ehcache" additivity="false">
+        <level value="warn" />
+        <appender-ref ref="stdout" />
+    </logger>
+
+    <root>
+        <level value="info" />
+        <appender-ref ref="stdout" />
+    </root>
+
+</log4j:configuration>