|
|
|
@ -28,10 +28,10 @@ public class SpiderTest {
|
|
|
|
|
// PageProcessor pageProcessor = new MeicanProcessor();
|
|
|
|
|
// Spider.me().pipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
|
|
|
|
|
// processor(pageProcessor).run();
|
|
|
|
|
SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space", "http://www.diaoyuweng.com/thread-*-1-1.html");
|
|
|
|
|
SimplePageProcessor pageProcessor2 = new SimplePageProcessor( "http://www.diaoyuweng.com/thread-*-1-1.html");
|
|
|
|
|
System.out.println(pageProcessor2.getSite().getCharset());
|
|
|
|
|
pageProcessor2.getSite().setSleepTime(500);
|
|
|
|
|
Spider.create(pageProcessor2).addPipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
|
|
|
|
Spider.create(pageProcessor2).addUrl("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space").addPipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
|
|
|
|
run();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|