fix jsonpath in AngularJSProcessor

pull/102/head
yihua.huang 11 years ago
parent 84b897f83b
commit b14f0ee479

@ -3,6 +3,7 @@ package us.codecraft.webmagic.samples;
import org.apache.commons.collections.CollectionUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.JsonPathSelector;
@ -23,15 +24,15 @@ public class AngularJSProcessor implements PageProcessor {
@Override
public void process(Page page) {
if (page.getUrl().regex(LIST_URL).match()) {
List<String> ids = new JsonPathSelector("$.data._id").selectList(page.getRawText());
List<String> ids = new JsonPathSelector("$.data[*]._id").selectList(page.getRawText());
if (CollectionUtils.isNotEmpty(ids)) {
for (String id : ids) {
page.addTargetRequest("http://angularjs\\.cn/api/article/" + id);
page.addTargetRequest("http://angularjs.cn/api/article/" + id);
}
}
} else {
page.putField("title", new JsonPathSelector("$.title").select(page.getRawText()));
page.putField("content", new JsonPathSelector("$.content").select(page.getRawText()));
page.putField("title", new JsonPathSelector("$.data.title").select(page.getRawText()));
page.putField("content", new JsonPathSelector("$.data.content").select(page.getRawText()));
}
}
@ -40,4 +41,8 @@ public class AngularJSProcessor implements PageProcessor {
public Site getSite() {
return site;
}
public static void main(String[] args) {
Spider.create(new AngularJSProcessor()).addUrl("http://angularjs.cn/api/article/latest?p=1&s=20").run();
}
}

Loading…
Cancel
Save