|
|
|
@ -3,7 +3,7 @@ package us.codecraft.webmagic.example;
|
|
|
|
|
import us.codecraft.webmagic.Spider;
|
|
|
|
|
import us.codecraft.webmagic.monitor.SpiderMonitor;
|
|
|
|
|
import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;
|
|
|
|
|
import us.codecraft.webmagic.processor.example.OschinaBlogPageProcessor;
|
|
|
|
|
import us.codecraft.webmagic.processor.example.ZhihuPageProcessor;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @author code4crafer@gmail.com
|
|
|
|
@ -13,14 +13,14 @@ public class MonitorExample {
|
|
|
|
|
|
|
|
|
|
public static void main(String[] args) throws Exception {
|
|
|
|
|
|
|
|
|
|
Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor())
|
|
|
|
|
Spider zhihuPageProcessor = Spider.create(new ZhihuPageProcessor())
|
|
|
|
|
.addUrl("http://my.oschina.net/flashsword/blog");
|
|
|
|
|
Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
|
|
|
|
|
.addUrl("https://github.com/code4craft");
|
|
|
|
|
|
|
|
|
|
SpiderMonitor.instance().register(oschinaSpider);
|
|
|
|
|
SpiderMonitor.instance().register(zhihuPageProcessor);
|
|
|
|
|
SpiderMonitor.instance().register(githubSpider);
|
|
|
|
|
oschinaSpider.start();
|
|
|
|
|
zhihuPageProcessor.start();
|
|
|
|
|
githubSpider.start();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|