complete test cases
yihua.huang
12 years ago
parent
c17a31a21d
commit
d7c7a78177
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,14 @@
|
||||
package us.codecraft.webmagic;
|
||||
|
||||
import junit.framework.Assert;
|
||||
import us.codecraft.webmagic.model.PageModelPipeline;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com
|
||||
*/
|
||||
public class MockPageModelPipeline implements PageModelPipeline{
|
||||
@Override
|
||||
public void process(Object o, Task task) {
|
||||
Assert.assertNotNull(o);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
package us.codecraft.webmagic;
|
||||
|
||||
import us.codecraft.webmagic.pipeline.Pipeline;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com
|
||||
*/
|
||||
public class MockPipeline implements Pipeline{
|
||||
@Override
|
||||
public void process(ResultItems resultItems, Task task) {
|
||||
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,87 @@
|
||||
package us.codecraft.webmagic.model;
|
||||
|
||||
import junit.framework.Assert;
|
||||
import org.junit.Test;
|
||||
import us.codecraft.webmagic.MockDownloader;
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.Task;
|
||||
import us.codecraft.webmagic.model.annotation.ExtractBy;
|
||||
import us.codecraft.webmagic.model.annotation.ExtractByUrl;
|
||||
import us.codecraft.webmagic.model.annotation.HelpUrl;
|
||||
import us.codecraft.webmagic.model.annotation.TargetUrl;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com <br>
|
||||
*/
|
||||
@TargetUrl("https://github.com/\\w+/\\w+")
|
||||
@HelpUrl({"https://github.com/\\w+\\?tab=repositories", "https://github.com/\\w+", "https://github.com/explore/*"})
|
||||
public class GithubRepo implements HasKey {
|
||||
|
||||
@ExtractBy(value = "//h1[@class='entry-title public']/strong/a/text()", notNull = true)
|
||||
private String name;
|
||||
|
||||
@ExtractByUrl("https://github\\.com/(\\w+)/.*")
|
||||
private String author;
|
||||
|
||||
@ExtractBy("//div[@id='readme']")
|
||||
private String readme;
|
||||
|
||||
@ExtractBy(value = "//div[@class='repository-lang-stats']//li//span[@class='lang']", multi = true)
|
||||
private List<String> language;
|
||||
|
||||
@ExtractBy("//ul[@class='pagehead-actions']/li[2]//a[@class='social-count js-social-count']/text()")
|
||||
private String star;
|
||||
|
||||
@ExtractBy("//ul[@class='pagehead-actions']/li[3]//a[@class='social-count']/text()")
|
||||
private String fork;
|
||||
|
||||
@ExtractByUrl
|
||||
private String url;
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
OOSpider.create(Site.me().addStartUrl("https://github.com/code4craft/webmagic").setSleepTime(0)
|
||||
, new PageModelPipeline<GithubRepo>() {
|
||||
@Override
|
||||
public void process(GithubRepo o, Task task) {
|
||||
Assert.assertEquals("78",o.getStar().trim());
|
||||
Assert.assertEquals("65",o.getFork().trim());
|
||||
}
|
||||
}, GithubRepo.class).setDownloader(new MockDownloader()).test("https://github.com/code4craft/webmagic");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String key() {
|
||||
return author + ":" + name;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public String getReadme() {
|
||||
return readme;
|
||||
}
|
||||
|
||||
public String getAuthor() {
|
||||
return author;
|
||||
}
|
||||
|
||||
public List<String> getLanguage() {
|
||||
return language;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public String getStar() {
|
||||
return star;
|
||||
}
|
||||
|
||||
public String getFork() {
|
||||
return fork;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package us.codecraft.webmagic.processor;
|
||||
|
||||
import junit.framework.Assert;
|
||||
import org.junit.Test;
|
||||
import us.codecraft.webmagic.*;
|
||||
import us.codecraft.webmagic.model.OOSpider;
|
||||
import us.codecraft.webmagic.pipeline.Pipeline;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com
|
||||
*/
|
||||
public class GithubRepoProcessor implements PageProcessor {
|
||||
@Override
|
||||
public void process(Page page) {
|
||||
page.putField("star",page.getHtml().xpath("//ul[@class='pagehead-actions']/li[2]//a[@class='social-count js-social-count']/text()").toString());
|
||||
page.putField("fork",page.getHtml().xpath("//ul[@class='pagehead-actions']/li[3]//a[@class='social-count']/text()").toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Site getSite() {
|
||||
return Site.me().addStartUrl("https://github.com/code4craft/webmagic");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
OOSpider.create(new GithubRepoProcessor()).addPipeline(new Pipeline() {
|
||||
@Override
|
||||
public void process(ResultItems resultItems, Task task) {
|
||||
Assert.assertEquals("78",((String)resultItems.get("star")).trim());
|
||||
Assert.assertEquals("65",((String)resultItems.get("fork")).trim());
|
||||
}
|
||||
}).setDownloader(new MockDownloader()).test("https://github.com/code4craft/webmagic");
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
package us.codecraft.webmagic.samples;
|
||||
|
||||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.Spider;
|
||||
import us.codecraft.webmagic.processor.PageProcessor;
|
||||
import us.codecraft.webmagic.selector.PlainText;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午8:08
|
||||
*/
|
||||
public class DiaoyuwengProcessor implements PageProcessor {
|
||||
|
||||
private Site site;
|
||||
|
||||
@Override
|
||||
public void process(Page page) {
|
||||
List<String> requests = page.getHtml().links().regex("(http://www\\.diaoyuweng\\.com/home\\.php\\?mod=space&uid=88304&do=thread&view=me&type=thread&order=dateline&from=space&page=\\d+)").all();
|
||||
page.addTargetRequests(requests);
|
||||
requests = page.getHtml().links().regex("(http://www\\.diaoyuweng\\.com/thread-\\d+-1-1.html)").all();
|
||||
page.addTargetRequests(requests);
|
||||
if (page.getUrl().toString().contains("thread")){
|
||||
page.putField("title", page.getHtml().xpath("//a[@id='thread_subject']"));
|
||||
page.putField("content", page.getHtml().xpath("//div[@class='pcb']//tbody/tidyText()"));
|
||||
page.putField("date",page.getHtml().regex("发表于 (\\d{4}-\\d+-\\d+ \\d+:\\d+:\\d+)"));
|
||||
page.putField("id",new PlainText("1000"+page.getUrl().regex("http://www\\.diaoyuweng\\.com/thread-(\\d+)-1-1.html").toString()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Site getSite() {
|
||||
if (site==null){
|
||||
site= Site.me().setDomain("www.diaoyuweng.com").addStartUrl("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space").
|
||||
setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31").setCharset("GBK").setSleepTime(500);
|
||||
}
|
||||
return site;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
Spider.create(new DiaoyuwengProcessor()).run();
|
||||
}
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
package us.codecraft.webmagic.samples;
|
||||
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.Spider;
|
||||
import us.codecraft.webmagic.processor.PageProcessor;
|
||||
import us.codecraft.webmagic.scheduler.RedisScheduler;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:48
|
||||
*/
|
||||
public class F58PageProcesser implements PageProcessor {
|
||||
|
||||
@Override
|
||||
public void process(Page page) {
|
||||
List<String> strings = page.getHtml().links().regex(".*/yewu/.*").all();
|
||||
page.addTargetRequests(strings);
|
||||
page.putField("title",page.getHtml().regex("<title>(.*)</title>"));
|
||||
page.putField("body",page.getHtml().xpath("//dd"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Site getSite() {
|
||||
return Site.me().setDomain("sh.58.com").addStartUrl("http://sh1.51a8.com/").setCycleRetryTimes(2); //To change body of implemented methods use File | Settings | File Templates.
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
Spider.create(new F58PageProcesser()).setScheduler(new RedisScheduler("localhost")).run();
|
||||
}
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
package us.codecraft.webmagic.samples;
|
||||
|
||||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.Spider;
|
||||
import us.codecraft.webmagic.processor.PageProcessor;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-5-20
|
||||
* Time: 下午5:31
|
||||
*/
|
||||
public class KaichibaProcessor implements PageProcessor {
|
||||
@Override
|
||||
public void process(Page page) {
|
||||
//http://progressdaily.diandian.com/post/2013-01-24/40046867275
|
||||
int i = Integer.valueOf(page.getUrl().regex("shop/(\\d+)").toString()) + 1;
|
||||
page.addTargetRequest("http://kaichiba.com/shop/" + i);
|
||||
page.putField("title",page.getHtml().xpath("//Title"));
|
||||
page.putField("items", page.getHtml().xpath("//li[@class=\"foodTitle\"]").replace("^\\s+", "").replace("\\s+$", "").replace("<span>.*?</span>", ""));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Site getSite() {
|
||||
return Site.me().setDomain("kaichiba.com").addStartUrl("http://kaichiba.com/shop/41725781").setCharset("utf-8").
|
||||
setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
Spider.create(new KaichibaProcessor()).run();
|
||||
}
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
package us.codecraft.webmagic.samples;
|
||||
|
||||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.Spider;
|
||||
import us.codecraft.webmagic.processor.PageProcessor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-5-20
|
||||
* Time: 下午5:31
|
||||
*/
|
||||
public class MeicanProcessor implements PageProcessor {
|
||||
@Override
|
||||
public void process(Page page) {
|
||||
//http://progressdaily.diandian.com/post/2013-01-24/40046867275
|
||||
List<String> requests = page.getHtml().xpath("//a[@class=\"area_link flat_btn\"]/@href").all();
|
||||
if (requests.size() > 2) {
|
||||
requests = requests.subList(0, 2);
|
||||
}
|
||||
page.addTargetRequests(requests);
|
||||
page.addTargetRequests(page.getHtml().links().regex("(.*/restaurant/[^#]+)").all());
|
||||
page.putField("items", page.getHtml().xpath("//ul[@class=\"dishes menu_dishes\"]/li/span[@class=\"name\"]/text()"));
|
||||
page.putField("prices", page.getHtml().xpath("//ul[@class=\"dishes menu_dishes\"]/li/span[@class=\"price_outer\"]/span[@class=\"price\"]/text()"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Site getSite() {
|
||||
return Site.me().setDomain("meican.com").addStartUrl("http://www.meican.com/shanghai/districts").setCharset("utf-8").
|
||||
setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
Spider.create(new MeicanProcessor()).run();
|
||||
}
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
#!/bin/sh
|
||||
touch wordpress.xml
|
||||
cat wp-head.xml >> wordpress.xml
|
||||
for f in `ls`;
|
||||
do
|
||||
cat ${f} >> ../wordpress.xml
|
||||
done;
|
||||
cat wp-bottom.xml >> wordpress.xml
|
||||
@ -1,22 +0,0 @@
|
||||
<item>
|
||||
<title>${title}</title>
|
||||
<link>http://127.0.0.1/wordpress/?p=${id}</link>
|
||||
<pubDate>${date}</pubDate>
|
||||
<dc:creator>admin</dc:creator>
|
||||
<guid isPermaLink="false">http://127.0.0.1/wordpress/?p=${id}</guid>
|
||||
<description></description>
|
||||
<content:encoded><![CDATA[${content}]]></content:encoded>
|
||||
<excerpt:encoded><![CDATA[]]></excerpt:encoded>
|
||||
<wp:post_id>${id}</wp:post_id>
|
||||
<wp:post_date>${date}</wp:post_date>
|
||||
<wp:post_date_gmt>${date}</wp:post_date_gmt>
|
||||
<wp:comment_status>open</wp:comment_status>
|
||||
<wp:ping_status>open</wp:ping_status>
|
||||
<wp:post_name>${title}</wp:post_name>
|
||||
<wp:status>publish</wp:status>
|
||||
<wp:post_parent>0</wp:post_parent>
|
||||
<wp:menu_order>0</wp:menu_order>
|
||||
<wp:post_type>post</wp:post_type>
|
||||
<wp:post_password></wp:post_password>
|
||||
<wp:is_sticky>0</wp:is_sticky>
|
||||
</item>
|
||||
@ -1,2 +0,0 @@
|
||||
</channel>
|
||||
</rss>
|
||||
@ -1,35 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<!-- This is a WordPress eXtended RSS file generated by WordPress as an export of your site. -->
|
||||
<!-- It contains information about your site's posts, pages, comments, categories, and other content. -->
|
||||
<!-- You may use this file to transfer that content from one site to another. -->
|
||||
<!-- This file is not intended to serve as a complete backup of your site. -->
|
||||
|
||||
<!-- To import this information into a WordPress site follow these steps: -->
|
||||
<!-- 1. Log in to that site as an administrator. -->
|
||||
<!-- 2. Go to Tools: Import in the WordPress admin panel. -->
|
||||
<!-- 3. Install the "WordPress" importer from the list. -->
|
||||
<!-- 4. Activate & Run Importer. -->
|
||||
<!-- 5. Upload this file using the form provided on that page. -->
|
||||
<!-- 6. You will first be asked to map the authors in this export file to users -->
|
||||
<!-- on the site. For each author, you may choose to map to an -->
|
||||
<!-- existing user on the site or to create a new user. -->
|
||||
<!-- 7. WordPress will then import each of the posts, pages, comments, categories, etc. -->
|
||||
<!-- contained in this file into your site. -->
|
||||
|
||||
<!-- generator="WordPress/3.3.1" created="2012-06-10 09:15" -->
|
||||
<rss version="2.0"
|
||||
xmlns:excerpt="http://wordpress.org/export/1.1/excerpt/"
|
||||
xmlns:content="http://purl.org/rss/1.0/modules/content/"
|
||||
xmlns:wfw="http://wellformedweb.org/CommentAPI/"
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:wp="http://wordpress.org/export/1.1/"
|
||||
>
|
||||
<channel>
|
||||
<wp:wxr_version>1.1</wp:wxr_version>
|
||||
<wp:base_site_url>http://127.0.0.1/wordpress</wp:base_site_url>
|
||||
<wp:base_blog_url>http://127.0.0.1/wordpress</wp:base_blog_url>
|
||||
|
||||
<wp:author><wp:author_id>1</wp:author_id><wp:author_login>admin</wp:author_login><wp:author_email>flashsword20@163.com</wp:author_email><wp:author_display_name><![CDATA[admin]]></wp:author_display_name><wp:author_first_name><![CDATA[]]></wp:author_first_name><wp:author_last_name><![CDATA[]]></wp:author_last_name></wp:author>
|
||||
|
||||
|
||||
<generator>http://wordpress.org/?v=3.3.1</generator>
|
||||
@ -1,28 +0,0 @@
|
||||
package us.codecraft.webmagic.processor;
|
||||
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import us.codecraft.webmagic.Spider;
|
||||
import us.codecraft.webmagic.pipeline.FilePipeline;
|
||||
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
|
||||
import us.codecraft.webmagic.samples.DiaoyuwengProcessor;
|
||||
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-6-9
|
||||
* Time: 上午8:02
|
||||
*/
|
||||
public class DiaoyuwengProcessorTest {
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void test() throws IOException {
|
||||
DiaoyuwengProcessor diaoyuwengProcessor = new DiaoyuwengProcessor();
|
||||
JsonFilePipeline pipeline = new JsonFilePipeline("/data/webmagic/");
|
||||
Spider.create(diaoyuwengProcessor).pipeline(new FilePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||
run();
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue