diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
index a51ed960..f3065422 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
@@ -18,23 +18,24 @@ import java.util.concurrent.atomic.AtomicInteger;
/**
*
- *webmagic爬虫的入口类。
+ * webmagic爬虫的入口类。
*
- *示例:
- *定义一个最简单的爬虫:
+ * 示例:
+ * 定义一个最简单的爬虫:
* Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*")).run();
*
- *使用FilePipeline保存结果到文件:
+ * 使用FilePipeline保存结果到文件:
* Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*"))
* .pipeline(new FilePipeline("/data/temp/webmagic/")).run();
*
- *使用FileCacheQueueScheduler缓存URL,关闭爬虫后下次自动从停止的页面继续抓取:
+ * 使用FileCacheQueueScheduler缓存URL,关闭爬虫后下次自动从停止的页面继续抓取:
* Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*"))
* .scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).run();
*
+ *
* @author code4crafter@gmail.com
- * Date: 13-4-21
- * Time: 上午6:53
+ * Date: 13-4-21
+ * Time: 上午6:53
*/
public class Spider implements Runnable, Task {
@@ -66,6 +67,7 @@ public class Spider implements Runnable, Task {
/**
* 使用已定义的抽取规则新建一个Spider。
+ *
* @param pageProcessor 已定义的抽取规则
*/
public Spider(PageProcessor pageProcessor) {
@@ -76,6 +78,7 @@ public class Spider implements Runnable, Task {
/**
* 使用已定义的抽取规则新建一个Spider。
+ *
* @param pageProcessor 已定义的抽取规则
* @return 新建的Spider
*/
@@ -85,6 +88,7 @@ public class Spider implements Runnable, Task {
/**
* 重新设置startUrls,会覆盖Site本身的startUrls。
+ *
* @param startUrls
* @return this
*/
@@ -96,6 +100,7 @@ public class Spider implements Runnable, Task {
/**
* 为爬虫设置一个唯一ID,用于标志任务,默认情况下使用domain作为uuid,对于单domain多任务的情况,请为重复任务设置不同的ID。
+ *
* @param uuid 唯一ID
* @return this
*/
@@ -106,6 +111,7 @@ public class Spider implements Runnable, Task {
/**
* 设置调度器。调度器用于保存待抓取URL,并可以进行去重、同步、持久化等工作。默认情况下使用内存中的阻塞队列进行调度。
+ *
* @param scheduler 调度器
* @return this
*/
@@ -117,6 +123,7 @@ public class Spider implements Runnable, Task {
/**
* 设置处理管道。处理管道用于最终抽取结果的后处理,例如:保存到文件、保存到数据库等。默认情况下会输出到控制台。
+ *
* @param pipeline 处理管道
* @return this
*/
@@ -148,7 +155,7 @@ public class Spider implements Runnable, Task {
pipelines.add(new ConsolePipeline());
}
//singel thread
- if (executorService==null){
+ if (executorService == null) {
while (request != null) {
processRequest(request);
request = scheduler.poll(this);
@@ -217,13 +224,13 @@ public class Spider implements Runnable, Task {
}
}
- private void checkIfNotRunning(){
- if (!stat.compareAndSet(STAT_INIT,STAT_INIT)){
+ private void checkIfNotRunning() {
+ if (!stat.compareAndSet(STAT_INIT, STAT_INIT)) {
throw new IllegalStateException("Spider is already running!");
}
}
- public void runAsync(){
+ public void runAsync() {
Thread thread = new Thread(this);
thread.setDaemon(false);
thread.start();
@@ -231,15 +238,19 @@ public class Spider implements Runnable, Task {
/**
* 建立多个线程下载
+ *
* @param threadNum 线程数
* @return this
*/
public Spider thread(int threadNum) {
checkIfNotRunning();
- if (threadNum <= 1) {
+ if (threadNum <= 0) {
throw new IllegalArgumentException("threadNum should be more than one!");
}
- synchronized (this){
+ if (threadNum == 1) {
+ return this;
+ }
+ synchronized (this) {
this.executorService = ThreadUtils.newFixedThreadPool(threadNum);
}
return this;
diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java b/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java
index 0948bfe0..10d97a83 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java
@@ -46,7 +46,7 @@ public class FilePipeline implements Pipeline {
file.mkdirs();
}
try {
- PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl())));
+ PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl())+".html"));
printWriter.println("url:\t" + resultItems.getRequest().getUrl());
for (Map.Entry entry : resultItems.getAll().entrySet()) {
printWriter.println(entry.getKey()+":\t"+entry.getValue());
diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/IteyeBlogProcessor.java b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/IteyeBlogProcessor.java
new file mode 100644
index 00000000..188f3a1f
--- /dev/null
+++ b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/IteyeBlogProcessor.java
@@ -0,0 +1,38 @@
+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.pipeline.FilePipeline;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+/**
+ * @author yihua.huang@dianping.com
+ * @date: 13-7-26
+ * Time: 上午7:31
+ */
+public class IteyeBlogProcessor implements PageProcessor {
+
+ private Site site;
+
+ @Override
+ public void process(Page page) {
+ page.addTargetRequests(page.getHtml().links().regex(".*yanghaoli\\.iteye\\.com/blog/\\d+").all());
+ page.putField("title",page.getHtml().xpath("//title").toString());
+ page.putField("content",page.getHtml().smartContent().toString());
+ }
+
+ @Override
+ public Site getSite() {
+ if (site == null) {
+ site = Site.me().setDomain("yanghaoli.iteye.com").addStartUrl("http://yanghaoli.iteye.com/").
+ setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31")
+ .setSleepTime(100).setRetryTimes(3);
+ }
+ return site;
+ }
+
+ public static void main(String[] args) {
+ Spider.create(new IteyeBlogProcessor()).thread(5).pipeline(new FilePipeline()).run();
+ }
+}
diff --git a/webmagic-samples/src/main/resources/log4j.xml b/webmagic-samples/src/main/resources/log4j.xml
new file mode 100644
index 00000000..a6630f81
--- /dev/null
+++ b/webmagic-samples/src/main/resources/log4j.xml
@@ -0,0 +1,26 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+