diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 1d308bcb..f204e394 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -32,21 +32,24 @@ import us.codecraft.webmagic.utils.HttpClientUtils; */ public class HttpClientDownloader extends AbstractDownloader { + private Logger logger = LoggerFactory.getLogger(getClass()); + private final Map httpClients = new HashMap(); - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final HttpClientGenerator httpClientGenerator = new HttpClientGenerator(); + + private HttpClientGenerator httpClientGenerator = new HttpClientGenerator(); private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter(); private ProxyProvider proxyProvider; - private final boolean responseHeader = true; + private boolean responseHeader = true; + private volatile boolean refreshProxyOnError = false; - private Predicate refreshProxyOnError = t -> false; + private Predicate throwablePredicate = t->false; - public void setRefreshProxyOnError(Predicate proxyOnError) { - this.refreshProxyOnError = refreshProxyOnError; + public void setThrowablePredicate(Predicate predicate){ + this.throwablePredicate = predicate; } public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) { @@ -116,7 +119,7 @@ public class HttpClientDownloader extends AbstractDownloader { String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue(); Page page = new Page(); page.setBytes(bytes); - if (!request.isBinaryContent()) { + if (!request.isBinaryContent()){ if (charset == null) { charset = getHtmlCharset(contentType, bytes); }