|
|
|
@ -32,21 +32,24 @@ import us.codecraft.webmagic.utils.HttpClientUtils;
|
|
|
|
|
*/
|
|
|
|
|
public class HttpClientDownloader extends AbstractDownloader {
|
|
|
|
|
|
|
|
|
|
private Logger logger = LoggerFactory.getLogger(getClass());
|
|
|
|
|
|
|
|
|
|
private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
|
|
|
|
|
private final Logger logger = LoggerFactory.getLogger(getClass());
|
|
|
|
|
private final HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
|
|
|
|
|
|
|
|
|
|
private HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
|
|
|
|
|
|
|
|
|
|
private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter();
|
|
|
|
|
|
|
|
|
|
private ProxyProvider proxyProvider;
|
|
|
|
|
|
|
|
|
|
private final boolean responseHeader = true;
|
|
|
|
|
private boolean responseHeader = true;
|
|
|
|
|
|
|
|
|
|
private volatile boolean refreshProxyOnError = false;
|
|
|
|
|
|
|
|
|
|
private Predicate<Throwable> refreshProxyOnError = t -> false;
|
|
|
|
|
private Predicate<Throwable> throwablePredicate = t->false;
|
|
|
|
|
|
|
|
|
|
public void setRefreshProxyOnError(Predicate<Throwable> proxyOnError) {
|
|
|
|
|
this.refreshProxyOnError = refreshProxyOnError;
|
|
|
|
|
public void setThrowablePredicate(Predicate<Throwable> predicate){
|
|
|
|
|
this.throwablePredicate = predicate;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) {
|
|
|
|
|