add iteye sample

pull/17/head
yihua.huang 12 years ago
parent 2a19d80311
commit c2142f872b

@ -18,23 +18,24 @@ import java.util.concurrent.atomic.AtomicInteger;
/**
* <pre>
*webmagic
* webmagic
*
*
*
*
*
* Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*")).run();
*
*使FilePipeline:
* 使FilePipeline:
* Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*"))
* .pipeline(new FilePipeline("/data/temp/webmagic/")).run();
*
*使FileCacheQueueSchedulerURL:
* 使FileCacheQueueSchedulerURL:
* Spider.create(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*blog/*"))
* .scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).run();
* </pre>
*
* @author code4crafter@gmail.com <br>
* Date: 13-4-21
* Time: 6:53
* Date: 13-4-21
* Time: 6:53
*/
public class Spider implements Runnable, Task {
@ -66,6 +67,7 @@ public class Spider implements Runnable, Task {
/**
* 使Spider
*
* @param pageProcessor
*/
public Spider(PageProcessor pageProcessor) {
@ -76,6 +78,7 @@ public class Spider implements Runnable, Task {
/**
* 使Spider
*
* @param pageProcessor
* @return Spider
*/
@ -85,6 +88,7 @@ public class Spider implements Runnable, Task {
/**
* startUrlsSitestartUrls
*
* @param startUrls
* @return this
*/
@ -96,6 +100,7 @@ public class Spider implements Runnable, Task {
/**
* ID使domainuuiddomainID
*
* @param uuid ID
* @return this
*/
@ -106,6 +111,7 @@ public class Spider implements Runnable, Task {
/**
* URL使
*
* @param scheduler
* @return this
*/
@ -117,6 +123,7 @@ public class Spider implements Runnable, Task {
/**
*
*
* @param pipeline
* @return this
*/
@ -148,7 +155,7 @@ public class Spider implements Runnable, Task {
pipelines.add(new ConsolePipeline());
}
//singel thread
if (executorService==null){
if (executorService == null) {
while (request != null) {
processRequest(request);
request = scheduler.poll(this);
@ -217,13 +224,13 @@ public class Spider implements Runnable, Task {
}
}
private void checkIfNotRunning(){
if (!stat.compareAndSet(STAT_INIT,STAT_INIT)){
private void checkIfNotRunning() {
if (!stat.compareAndSet(STAT_INIT, STAT_INIT)) {
throw new IllegalStateException("Spider is already running!");
}
}
public void runAsync(){
public void runAsync() {
Thread thread = new Thread(this);
thread.setDaemon(false);
thread.start();
@ -231,15 +238,19 @@ public class Spider implements Runnable, Task {
/**
* 线
*
* @param threadNum 线
* @return this
*/
public Spider thread(int threadNum) {
checkIfNotRunning();
if (threadNum <= 1) {
if (threadNum <= 0) {
throw new IllegalArgumentException("threadNum should be more than one!");
}
synchronized (this){
if (threadNum == 1) {
return this;
}
synchronized (this) {
this.executorService = ThreadUtils.newFixedThreadPool(threadNum);
}
return this;

@ -46,7 +46,7 @@ public class FilePipeline implements Pipeline {
file.mkdirs();
}
try {
PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl())));
PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl())+".html"));
printWriter.println("url:\t" + resultItems.getRequest().getUrl());
for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
printWriter.println(entry.getKey()+":\t"+entry.getValue());

@ -0,0 +1,38 @@
package us.codecraft.webmagic.samples;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
/**
* @author yihua.huang@dianping.com <br>
* @date: 13-7-26 <br>
* Time: 7:31 <br>
*/
public class IteyeBlogProcessor implements PageProcessor {
private Site site;
@Override
public void process(Page page) {
page.addTargetRequests(page.getHtml().links().regex(".*yanghaoli\\.iteye\\.com/blog/\\d+").all());
page.putField("title",page.getHtml().xpath("//title").toString());
page.putField("content",page.getHtml().smartContent().toString());
}
@Override
public Site getSite() {
if (site == null) {
site = Site.me().setDomain("yanghaoli.iteye.com").addStartUrl("http://yanghaoli.iteye.com/").
setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31")
.setSleepTime(100).setRetryTimes(3);
}
return site;
}
public static void main(String[] args) {
Spider.create(new IteyeBlogProcessor()).thread(5).pipeline(new FilePipeline()).run();
}
}

@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<logger name="org.springframework" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<logger name="net.sf.ehcache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<root>
<level value="info" />
<appender-ref ref="stdout" />
</root>
</log4j:configuration>
Loading…
Cancel
Save