From c2142f872b0a21be06c7802a82912d0b0e474f03 Mon Sep 17 00:00:00 2001 From: "yihua.huang" Date: Fri, 26 Jul 2013 08:24:08 +0800 Subject: [PATCH] add iteye sample --- .../java/us/codecraft/webmagic/Spider.java | 37 +++++++++++------- .../webmagic/pipeline/FilePipeline.java | 2 +- .../webmagic/samples/IteyeBlogProcessor.java | 38 +++++++++++++++++++ webmagic-samples/src/main/resources/log4j.xml | 26 +++++++++++++ 4 files changed, 89 insertions(+), 14 deletions(-) create mode 100644 webmagic-samples/src/main/java/us/codecraft/webmagic/samples/IteyeBlogProcessor.java create mode 100644 webmagic-samples/src/main/resources/log4j.xml diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java index a51ed960..f3065422 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java @@ -18,23 +18,24 @@ import java.util.concurrent.atomic.AtomicInteger; /** * 
- *webmagic爬虫的入口类。
+ * webmagic爬虫的入口类。
  *
- *示例：
- *定义一个最简单的爬虫：
+ * 示例：
+ * 定义一个最简单的爬虫：
  *      Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*")).run();
  *
- *使用FilePipeline保存结果到文件:
+ * 使用FilePipeline保存结果到文件:
  *      Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*"))
  *          .pipeline(new FilePipeline("/data/temp/webmagic/")).run();
  *
- *使用FileCacheQueueScheduler缓存URL，关闭爬虫后下次自动从停止的页面继续抓取:
+ * 使用FileCacheQueueScheduler缓存URL，关闭爬虫后下次自动从停止的页面继续抓取:
  *      Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*"))
  *          .scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).run();
  *
+ * * @author code4crafter@gmail.com
- * Date: 13-4-21 - * Time: 上午6:53 + * Date: 13-4-21 + * Time: 上午6:53 */ public class Spider implements Runnable, Task { @@ -66,6 +67,7 @@ public class Spider implements Runnable, Task { /** * 使用已定义的抽取规则新建一个Spider。 + * * @param pageProcessor 已定义的抽取规则 */ public Spider(PageProcessor pageProcessor) { @@ -76,6 +78,7 @@ public class Spider implements Runnable, Task { /** * 使用已定义的抽取规则新建一个Spider。 + * * @param pageProcessor 已定义的抽取规则 * @return 新建的Spider */ @@ -85,6 +88,7 @@ public class Spider implements Runnable, Task { /** * 重新设置startUrls，会覆盖Site本身的startUrls。 + * * @param startUrls * @return this */ @@ -96,6 +100,7 @@ public class Spider implements Runnable, Task { /** * 为爬虫设置一个唯一ID，用于标志任务，默认情况下使用domain作为uuid，对于单domain多任务的情况，请为重复任务设置不同的ID。 + * * @param uuid 唯一ID * @return this */ @@ -106,6 +111,7 @@ public class Spider implements Runnable, Task { /** * 设置调度器。调度器用于保存待抓取URL，并可以进行去重、同步、持久化等工作。默认情况下使用内存中的阻塞队列进行调度。 + * * @param scheduler 调度器 * @return this */ @@ -117,6 +123,7 @@ public class Spider implements Runnable, Task { /** * 设置处理管道。处理管道用于最终抽取结果的后处理，例如：保存到文件、保存到数据库等。默认情况下会输出到控制台。 + * * @param pipeline 处理管道 * @return this */ @@ -148,7 +155,7 @@ public class Spider implements Runnable, Task { pipelines.add(new ConsolePipeline()); } //singel thread - if (executorService==null){ + if (executorService == null) { while (request != null) { processRequest(request); request = scheduler.poll(this); @@ -217,13 +224,13 @@ public class Spider implements Runnable, Task { } } - private void checkIfNotRunning(){ - if (!stat.compareAndSet(STAT_INIT,STAT_INIT)){ + private void checkIfNotRunning() { + if (!stat.compareAndSet(STAT_INIT, STAT_INIT)) { throw new IllegalStateException("Spider is already running!"); } } - public void runAsync(){ + public void runAsync() { Thread thread = new Thread(this); thread.setDaemon(false); thread.start(); @@ -231,15 +238,19 @@ public class Spider implements Runnable, Task { /** * 建立多个线程下载 + * * @param threadNum 线程数 * @return this */ public Spider thread(int threadNum) { checkIfNotRunning(); - if (threadNum <= 1) { + if (threadNum <= 0) { throw new IllegalArgumentException("threadNum should be more than one!"); } - synchronized (this){ + if (threadNum == 1) { + return this; + } + synchronized (this) { this.executorService = ThreadUtils.newFixedThreadPool(threadNum); } return this; diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java b/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java index 0948bfe0..10d97a83 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java @@ -46,7 +46,7 @@ public class FilePipeline implements Pipeline { file.mkdirs(); } try { - PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl()))); + PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl())+".html")); printWriter.println("url:\t" + resultItems.getRequest().getUrl()); for (Map.Entry entry : resultItems.getAll().entrySet()) { printWriter.println(entry.getKey()+":\t"+entry.getValue()); diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/IteyeBlogProcessor.java b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/IteyeBlogProcessor.java new file mode 100644 index 00000000..188f3a1f --- /dev/null +++ b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/IteyeBlogProcessor.java @@ -0,0 +1,38 @@ +package us.codecraft.webmagic.samples; + +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.pipeline.FilePipeline; +import us.codecraft.webmagic.processor.PageProcessor; + +/** + * @author yihua.huang@dianping.com
+ * @date: 13-7-26
+ * Time: 上午7:31
+ */ +public class IteyeBlogProcessor implements PageProcessor { + + private Site site; + + @Override + public void process(Page page) { + page.addTargetRequests(page.getHtml().links().regex(".*yanghaoli\\.iteye\\.com/blog/\\d+").all()); + page.putField("title",page.getHtml().xpath("//title").toString()); + page.putField("content",page.getHtml().smartContent().toString()); + } + + @Override + public Site getSite() { + if (site == null) { + site = Site.me().setDomain("yanghaoli.iteye.com").addStartUrl("http://yanghaoli.iteye.com/"). + setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31") + .setSleepTime(100).setRetryTimes(3); + } + return site; + } + + public static void main(String[] args) { + Spider.create(new IteyeBlogProcessor()).thread(5).pipeline(new FilePipeline()).run(); + } +} diff --git a/webmagic-samples/src/main/resources/log4j.xml b/webmagic-samples/src/main/resources/log4j.xml new file mode 100644 index 00000000..a6630f81 --- /dev/null +++ b/webmagic-samples/src/main/resources/log4j.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + +