add tests before refactor #586
yihua.huang
8 years ago
parent
b363ee6a9d
commit
b1ef61b278
@ -0,0 +1,18 @@
|
||||
package us.codecraft.webmagic.model;
|
||||
|
||||
import us.codecraft.webmagic.model.annotation.ExtractBy;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com
|
||||
* Date: 2017/6/3
|
||||
* Time: 下午9:07
|
||||
*/
|
||||
public class GithubRepoApi {
|
||||
|
||||
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.name",source = ExtractBy.Source.RawText)
|
||||
private String name;
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
@ -1,13 +0,0 @@
|
||||
package us.codecraft.webmagic.model;
|
||||
|
||||
import us.codecraft.webmagic.model.annotation.HelpUrl;
|
||||
import us.codecraft.webmagic.model.annotation.TargetUrl;
|
||||
|
||||
/**
|
||||
* @author code4crafer@gmail.com
|
||||
*/
|
||||
@TargetUrl(value = "http://webmagic.io/post/\\d+",sourceRegion = "//li[@class='post']")
|
||||
@HelpUrl(value = "http://webmagic.io/list/\\d+",sourceRegion = "//li[@class='list']")
|
||||
public class MockModel {
|
||||
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
package us.codecraft.webmagic.model;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.Request;
|
||||
import us.codecraft.webmagic.selector.PlainText;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com
|
||||
* Date: 2017/6/3
|
||||
* Time: 下午9:08
|
||||
*/
|
||||
public class PageMocker {
|
||||
|
||||
public Page getMockJsonPage() throws IOException {
|
||||
Page page = new Page();
|
||||
page.setRawText(IOUtils.toString(PageMocker.class.getClassLoader().getResourceAsStream("json/mock-githubrepo.json")));
|
||||
page.setRequest(new Request("https://api.github.com/repos/code4craft/webmagic"));
|
||||
page.setUrl(new PlainText("https://api.github.com/repos/code4craft/webmagic"));
|
||||
return page;
|
||||
}
|
||||
|
||||
public Page getMockPage() throws IOException {
|
||||
Page page = new Page();
|
||||
page.setRawText(IOUtils.toString(PageMocker.class.getClassLoader().getResourceAsStream("html/mock-webmagic.html")));
|
||||
page.setRequest(new Request("http://webmagic.io/list/0"));
|
||||
page.setUrl(new PlainText("http://webmagic.io/list/0"));
|
||||
return page;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,103 @@
|
||||
package us.codecraft.webmagic.model;
|
||||
|
||||
import org.apache.commons.lang3.time.DateFormatUtils;
|
||||
import org.junit.Test;
|
||||
import us.codecraft.webmagic.model.annotation.ExtractBy;
|
||||
import us.codecraft.webmagic.model.annotation.Formatter;
|
||||
import us.codecraft.webmagic.model.formatter.DateFormatter;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com
|
||||
* Date: 2017/6/3
|
||||
* Time: 下午9:06
|
||||
*/
|
||||
public class PageModelExtractorTest {
|
||||
|
||||
private PageMocker pageMocker = new PageMocker();
|
||||
|
||||
public static class ModelDateStr {
|
||||
|
||||
@ExtractBy(value = "//div[@class='date']/text()", notNull = true)
|
||||
private String dateStr;
|
||||
|
||||
}
|
||||
|
||||
public static class ModelDate {
|
||||
|
||||
@Formatter(value = "yyyyMMdd", formatter = DateFormatter.class)
|
||||
@ExtractBy(value = "//div[@class='date']/text()", notNull = true)
|
||||
private Date date;
|
||||
|
||||
}
|
||||
|
||||
public static class ModelInt {
|
||||
|
||||
@ExtractBy(value = "//div[@class='number']/text()", notNull = true)
|
||||
private int number;
|
||||
|
||||
}
|
||||
|
||||
public static class ModelStringList {
|
||||
|
||||
@ExtractBy("//a/@href")
|
||||
private List<String> links;
|
||||
|
||||
}
|
||||
|
||||
public static class ModelIntList {
|
||||
|
||||
@Formatter(subClazz = Integer.class)
|
||||
@ExtractBy("//li[@class='numbers']/text()")
|
||||
private List<Integer> numbers;
|
||||
|
||||
}
|
||||
|
||||
public static class ModelDateList {
|
||||
|
||||
@Formatter(subClazz = Date.class, value = "yyyyMMdd")
|
||||
@ExtractBy("//li[@class='dates']/text()")
|
||||
private List<Date> dates;
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testXpath() throws Exception {
|
||||
ModelDateStr modelDate = (ModelDateStr) PageModelExtractor.create(ModelDateStr.class).process(pageMocker.getMockPage());
|
||||
assertThat(modelDate.dateStr).isEqualTo("20170603");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExtractDate() throws Exception {
|
||||
ModelDate modelDate = (ModelDate) PageModelExtractor.create(ModelDate.class).process(pageMocker.getMockPage());
|
||||
assertThat(DateFormatUtils.format(modelDate.date,"yyyyMMdd")).isEqualTo("20170603");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExtractInt() throws Exception {
|
||||
ModelInt modelDate = (ModelInt) PageModelExtractor.create(ModelInt.class).process(pageMocker.getMockPage());
|
||||
assertThat(modelDate.number).isEqualTo(12);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExtractList() throws Exception {
|
||||
ModelStringList modelDate = (ModelStringList) PageModelExtractor.create(ModelStringList.class).process(pageMocker.getMockPage());
|
||||
assertThat(modelDate.links).hasSize(8);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExtractIntList() throws Exception {
|
||||
ModelIntList modelDate = (ModelIntList) PageModelExtractor.create(ModelIntList.class).process(pageMocker.getMockPage());
|
||||
assertThat(modelDate.numbers).hasSize(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExtractDateList() throws Exception {
|
||||
ModelDateList modelDate = (ModelDateList) PageModelExtractor.create(ModelDateList.class).process(pageMocker.getMockPage());
|
||||
assertThat(modelDate.dates).hasSize(4);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue