|
|
@ -6,6 +6,7 @@ import us.codecraft.webmagic.model.OOSpider;
|
|
|
|
import us.codecraft.webmagic.model.annotation.ExtractBy;
|
|
|
|
import us.codecraft.webmagic.model.annotation.ExtractBy;
|
|
|
|
import us.codecraft.webmagic.model.annotation.HelpUrl;
|
|
|
|
import us.codecraft.webmagic.model.annotation.HelpUrl;
|
|
|
|
import us.codecraft.webmagic.model.annotation.TargetUrl;
|
|
|
|
import us.codecraft.webmagic.model.annotation.TargetUrl;
|
|
|
|
|
|
|
|
import us.codecraft.webmagic.scheduler.RedisScheduler;
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* @author code4crafter@gmail.com
|
|
|
|
* @author code4crafter@gmail.com
|
|
|
@ -23,7 +24,9 @@ public class JokejiModel {
|
|
|
|
public static void main(String[] args) {
|
|
|
|
public static void main(String[] args) {
|
|
|
|
OOSpider.create(Site.me().setDomain("www.jokeji.cn").setCharset("gbk").setSleepTime(100).setTimeOut(3000)
|
|
|
|
OOSpider.create(Site.me().setDomain("www.jokeji.cn").setCharset("gbk").setSleepTime(100).setTimeOut(3000)
|
|
|
|
.setUserAgent("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)")
|
|
|
|
.setUserAgent("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)")
|
|
|
|
, new ConsolePageModelPipeline(), JokejiModel.class).addUrl("http://www.jokeji.cn/").thread(2).run();
|
|
|
|
, new ConsolePageModelPipeline(), JokejiModel.class).addUrl("http://www.jokeji.cn/").thread(2)
|
|
|
|
|
|
|
|
.scheduler(new RedisScheduler("127.0.0.1"))
|
|
|
|
|
|
|
|
.run();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|