|
|
@ -32,24 +32,21 @@ import java.util.function.Predicate;
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
public class HttpClientDownloader extends AbstractDownloader {
|
|
|
|
public class HttpClientDownloader extends AbstractDownloader {
|
|
|
|
|
|
|
|
|
|
|
|
private Logger logger = LoggerFactory.getLogger(getClass());
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
|
|
|
|
private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
|
|
|
|
|
|
|
|
private final Logger logger = LoggerFactory.getLogger(getClass());
|
|
|
|
private HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
|
|
|
|
private final HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
|
|
|
|
|
|
|
|
|
|
|
|
private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter();
|
|
|
|
private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter();
|
|
|
|
|
|
|
|
|
|
|
|
private ProxyProvider proxyProvider;
|
|
|
|
private ProxyProvider proxyProvider;
|
|
|
|
|
|
|
|
|
|
|
|
private boolean responseHeader = true;
|
|
|
|
private final boolean responseHeader = true;
|
|
|
|
|
|
|
|
|
|
|
|
private volatile boolean refreshProxyOnError = false;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private Predicate<Throwable> throwablePredicate = t->false;
|
|
|
|
private Predicate<Throwable> refreshProxyOnError = t -> false;
|
|
|
|
|
|
|
|
|
|
|
|
public void setThrowablePredicate(Predicate<Throwable> predicate){
|
|
|
|
public void setRefreshProxyOnError(Predicate<Throwable> proxyOnError) {
|
|
|
|
this.throwablePredicate = predicate;
|
|
|
|
this.refreshProxyOnError = refreshProxyOnError;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) {
|
|
|
|
public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) {
|
|
|
@ -97,7 +94,7 @@ public class HttpClientDownloader extends AbstractDownloader {
|
|
|
|
} catch (IOException e) {
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.warn("download page {} error", request.getUrl(), e);
|
|
|
|
logger.warn("download page {} error", request.getUrl(), e);
|
|
|
|
onError(request, e, proxyProvider);
|
|
|
|
onError(request, e, proxyProvider);
|
|
|
|
if(proxyProvider != null && refreshProxyOnError && throwablePredicate.test(e)){
|
|
|
|
if (proxyProvider != null && refreshProxyOnError.test(e)) {
|
|
|
|
proxyProvider.refreshProxy(task);
|
|
|
|
proxyProvider.refreshProxy(task);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return page;
|
|
|
|
return page;
|
|
|
|