From 49de9374cd17b7443a9f7f229be97e3c1b5b28eb Mon Sep 17 00:00:00 2001 From: "yihua.huang" Date: Sat, 27 May 2017 17:30:19 +0800 Subject: [PATCH] new SimpleHttpClient #576 --- .../java/us/codecraft/webmagic/Request.java | 9 ++-- .../codecraft/webmagic/SimpleHttpClient.java | 51 +++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 webmagic-extension/src/main/java/us/codecraft/webmagic/SimpleHttpClient.java diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java index d2ea247b..f29ccb32 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java @@ -93,12 +93,14 @@ public class Request implements Serializable { return extras; } - public void setExtras(Map extras) { + public Request setExtras(Map extras) { this.extras = extras; + return this; } - public void setUrl(String url) { + public Request setUrl(String url) { this.url = url; + return this; } /** @@ -111,8 +113,9 @@ public class Request implements Serializable { return method; } - public void setMethod(String method) { + public Request setMethod(String method) { this.method = method; + return this; } @Override diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/SimpleHttpClient.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/SimpleHttpClient.java new file mode 100644 index 00000000..67f4348e --- /dev/null +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/SimpleHttpClient.java @@ -0,0 +1,51 @@ +package us.codecraft.webmagic; + +import us.codecraft.webmagic.downloader.HttpClientDownloader; +import us.codecraft.webmagic.model.PageMapper; +import us.codecraft.webmagic.proxy.ProxyProvider; + +/** + * @author code4crafter@gmail.com + * Date: 2017/5/27 + * @since 0.7.0 + */ +public class SimpleHttpClient { + + private final HttpClientDownloader httpClientDownloader; + + private final Site site; + + public SimpleHttpClient() { + this(Site.me()); + } + + public SimpleHttpClient(Site site) { + this.site = site; + this.httpClientDownloader = new HttpClientDownloader(); + } + + public void setProxyProvider(ProxyProvider proxyProvider){ + this.httpClientDownloader.setProxyProvider(proxyProvider); + } + + public T get(String url, Class clazz) { + return get(new Request(url), clazz); + } + + public T get(Request request, Class clazz) { + Page page = httpClientDownloader.download(request, site.toTask()); + if (!page.isDownloadSuccess()) { + return null; + } + return new PageMapper(clazz).get(page); + } + + public Page get(String url) { + return httpClientDownloader.download(new Request(url), site.toTask()); + } + + public Page get(Request request) { + return httpClientDownloader.download(request, site.toTask()); + } + +}