github-forks
/
webmagic
mirror of https://github.com/code4craft/webmagic.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add example

Browse Source
pull/88/head
yihua.huang 12 years ago
parent 95ab4edec3
commit b131878123
8 changed files with 161 additions and 20 deletions
Show Stats Download Patch File Download Diff File

36
webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/GithubRepoPageProcesser.java
Unescape Escape View File

@ -0,0 +1,36 @@
package us.codecraft.webmagic.processor.example;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
/**
* @author code4crafter@gmail.com <br>
* @since 0.3.2
*/
public class GithubRepoPageProcesser implements PageProcessor {
private Site site = Site.me().addStartUrl("https://github.com/code4craft").setRetryTimes(3).setSleepTime(100);
@Override
public void process(Page page) {
page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all());
page.putField("author", page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString());
page.putField("name", page.getHtml().xpath("//h1[@class='entry-title public']/strong/a/text()").toString());
if (page.getResultItems().get("name")==null){
//skip this page
page.setSkip(true);
}
page.putField("readme", page.getHtml().xpath("//div[@id='readme']/tidyText()"));
}
@Override
public Site getSite() {
return site;
}
public static void main(String[] args) {
Spider.create(new GithubRepoPageProcesser()).thread(5).run();
}
}

39
webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/OschinaBlogPageProcesser.java
Unescape Escape View File

@ -0,0 +1,39 @@
package us.codecraft.webmagic.processor.example;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List;
/**
* @author code4crafter@gmail.com <br>
*/
public class OschinaBlogPageProcesser implements PageProcessor {
private Site site = Site.me().setDomain("my.oschina.net").addStartUrl("http://my.oschina.net/flashsword/blog");
@Override
public void process(Page page) {
List<String> links = page.getHtml().links().regex("http://my\\.oschina\\.net/flashsword/blog/\\d+").all();
page.addTargetRequests(links);
page.putField("title", page.getHtml().xpath("//div[@class='BlogEntity']/div[@class='BlogTitle']/h1/text()").toString());
if (page.getResultItems().get("title") == null) {
//skip this page
page.setSkip(true);
}
page.putField("content", page.getHtml().xpath("//div[@class='BlogContent']/tidyText()").toString());
page.putField("tags", page.getHtml().xpath("//div[@class='BlogTags']/a/text()").all());
}
@Override
public Site getSite() {
return site;
}
public static void main(String[] args) {
Spider.create(new OschinaBlogPageProcesser()).thread(2).run();
}
}

13
webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepo.java
Unescape Escape View File

@ -1,6 +1,9 @@
package us.codecraft.webmagic.example;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.model.ConsolePageModelPipeline;
import us.codecraft.webmagic.model.HasKey;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.model.annotation.ExtractByUrl;
import us.codecraft.webmagic.model.annotation.HelpUrl;
@ -10,6 +13,7 @@ import java.util.List;
/**
* @author code4crafter@gmail.com <br>
* @since 0.3.2
*/
@TargetUrl("https://github.com/\\w+/\\w+")
@HelpUrl({"https://github.com/\\w+\\?tab=repositories", "https://github.com/\\w+", "https://github.com/explore/*"})
@ -27,15 +31,20 @@ public class GithubRepo implements HasKey {
@ExtractBy(value = "//div[@class='repository-lang-stats']//li//span[@class='lang']/text()", multi = true)
private List<String> language;
@ExtractBy("//ul[@class='pagehead-actions']/li[2]//a[@class='social-count js-social-count']/text()")
@ExtractBy("//ul[@class='pagehead-actions']/li[1]//a[@class='social-count js-social-count']/text()")
private int star;
@ExtractBy("//ul[@class='pagehead-actions']/li[3]//a[@class='social-count']/text()")
@ExtractBy("//ul[@class='pagehead-actions']/li[2]//a[@class='social-count']/text()")
private int fork;
@ExtractByUrl
private String url;
public static void main(String[] args) {
OOSpider.create(Site.me().addStartUrl("https://github.com/code4craft").setSleepTime(100)
, new ConsolePageModelPipeline(), GithubRepo.class).thread(10).run();
}
@Override
public String key() {
return author + ":" + name;

25
webmagic-extension/src/main/java/us/codecraft/webmagic/example/OschinaBlog.java
Unescape Escape View File

@ -1,38 +1,38 @@
package us.codecraft.webmagic.example;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.model.AfterExtractor;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.model.annotation.Formatter;
import us.codecraft.webmagic.model.annotation.TargetUrl;
import us.codecraft.webmagic.pipeline.JsonFilePageModelPipeline;
import java.util.Date;
import java.util.List;
/**
* @author code4crafter@gmail.com <br>
* @since 0.3.2
*/
@TargetUrl("http://my.oschina.net/flashsword/blog/\\d+")
public class OschinaBlog implements AfterExtractor{
public class OschinaBlog {
@ExtractBy("//title/text()")
private String title;
@ExtractBy(value = "div.BlogContent",type = ExtractBy.Type.Css)
@ExtractBy(value = "div.BlogContent", type = ExtractBy.Type.Css)
private String content;
@ExtractBy(value = "//div[@class='BlogTags']/a/text()", multi = true)
private List<String> tags;
@Formatter("YYYY-MM-dd HH:mm")
@Formatter("yyyy-MM-dd HH:mm")
@ExtractBy("//div[@class='BlogStat']/regex('\\d+-\\d+-\\d+\\s+\\d+:\\d+')")
private String date;
private Date date;
public static void main(String[] args) {
OOSpider.create(Site.me().addStartUrl("http://my.oschina.net/flashsword/blog")
,new JsonFilePageModelPipeline("/data/webmagic/"), OschinaBlog.class).run();
, new JsonFilePageModelPipeline("/data/webmagic/"), OschinaBlog.class).run();
}
public String getTitle() {
@ -47,13 +47,8 @@ public class OschinaBlog implements AfterExtractor{
return tags;
}
// public Date getDate() {
// return date;
// }
@Override
public void afterProcess(Page page) {
System.out.println(date);
System.out.println(title);
public Date getDate() {
return date;
}
}

15
webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java
Unescape Escape View File

@ -105,7 +105,8 @@ class PageModelExtractor {
Formatter formatter = field.getAnnotation(Formatter.class);
if (formatter != null) {
if (!formatter.formatter().equals(ObjectFormatter.class)) {
return initFormatter(formatter.formatter());
ObjectFormatter objectFormatter = initFormatter(formatter.formatter());
objectFormatter.initParam(formatter.value());
}
}
return initFormatter(ObjectFormatters.get(fieldClazz));
@ -311,6 +312,9 @@ class PageModelExtractor {
}
if (fieldExtractor.getObjectFormatter() != null) {
Object converted = convert(value, fieldExtractor.getObjectFormatter());
if (converted == null && fieldExtractor.isNotNull()) {
return null;
}
setField(o, fieldExtractor, converted);
} else {
setField(o, fieldExtractor, value);
@ -332,7 +336,11 @@ class PageModelExtractor {
private Object convert(String value, ObjectFormatter objectFormatter) {
try {
return objectFormatter.format(value);
Object format = objectFormatter.format(value);
if (logger.isDebugEnabled()) {
logger.debug("String " + value + " is converted to " + format);
}
return format;
} catch (Exception e) {
logger.error("convert " + value + " to " + objectFormatter.clazz() + " error!", e);
}
@ -351,6 +359,9 @@ class PageModelExtractor {
}
private void setField(Object o, FieldExtractor fieldExtractor, Object value) throws IllegalAccessException, InvocationTargetException {
if (value==null){
return;
}
if (fieldExtractor.getSetterMethod() != null) {
fieldExtractor.getSetterMethod().invoke(o, value);
}

2
webmagic-extension/src/main/java/us/codecraft/webmagic/model/formatter/DateFormatter.java
Unescape Escape View File

@ -10,7 +10,7 @@ import java.util.Date;
*/
public class DateFormatter implements ObjectFormatter<Date> {
private String[] datePatterns = new String[]{"YYYY-MM-dd HH:mm"};
private String[] datePatterns = new String[]{"yyyy-MM-dd HH:mm"};
@Override
public Date format(String raw) throws Exception {

20
webmagic-extension/src/test/java/us/codecraft/webmagic/formatter/DateFormatterTest.java
Unescape Escape View File

@ -0,0 +1,20 @@
package us.codecraft.webmagic.formatter;
import org.junit.Test;
import us.codecraft.webmagic.model.formatter.DateFormatter;
import java.util.Date;
/**
* @author code4crafter@gmail.com
*/
public class DateFormatterTest {
@Test
public void testDateFormatter() throws Exception {
DateFormatter dateFormatter = new DateFormatter();
dateFormatter.initParam(new String[]{"yyyy-MM-dd HH:mm"});
Date format = dateFormatter.format("2013-09-10 22:11");
System.out.println(format);
}
}

31
webmagic-extension/src/test/resouces/log4j.xml
Unescape Escape View File

@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<logger name="org.springframework" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<logger name="org.apache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<logger name="net.sf.ehcache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<root>
<level value="debug" />
<appender-ref ref="stdout" />
</root>
</log4j:configuration>
Write Preview
Loading…
Cancel
Save