From 40bf8ca58fc7d52efafb2707cb266a23c213997f Mon Sep 17 00:00:00 2001 From: "yihua.huang" Date: Wed, 28 May 2014 07:53:23 +0800 Subject: [PATCH] change return proxy from spider to httpclientdownloader #128 --- .../src/main/java/us/codecraft/webmagic/Spider.java | 5 ----- .../codecraft/webmagic/downloader/HttpClientDownloader.java | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java index b6ec472c..81cf1794 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java @@ -2,7 +2,6 @@ package us.codecraft.webmagic; import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; -import org.apache.http.HttpHost; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import us.codecraft.webmagic.downloader.Downloader; @@ -325,10 +324,6 @@ public class Spider implements Runnable, Task { onError(requestFinal); logger.error("process request " + requestFinal + " error", e); } finally { - if (site.getHttpProxyPool().isEnable()) { - site.returnHttpProxyToPool((HttpHost) requestFinal.getExtra(Request.PROXY), (Integer) requestFinal - .getExtra(Request.STATUS_CODE)); - } pageCount.incrementAndGet(); signalNewUrl(); } diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index bdafea79..f2f6d39e 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -117,6 +117,10 @@ public class HttpClientDownloader extends AbstractDownloader { } catch (IOException e) { logger.warn("close response fail", e); } + if (site.getHttpProxyPool().isEnable()) { + site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request + .getExtra(Request.STATUS_CODE)); + } } }