|
|
|
@ -32,21 +32,24 @@ import us.codecraft.webmagic.utils.HttpClientUtils;
|
|
|
|
|
*/
|
|
|
|
|
public class HttpClientDownloader extends AbstractDownloader {
|
|
|
|
|
|
|
|
|
|
private Logger logger = LoggerFactory.getLogger(getClass());
|
|
|
|
|
|
|
|
|
|
private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
|
|
|
|
|
private final Logger logger = LoggerFactory.getLogger(getClass());
|
|
|
|
|
private final HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
|
|
|
|
|
|
|
|
|
|
private HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
|
|
|
|
|
|
|
|
|
|
private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter();
|
|
|
|
|
|
|
|
|
|
private ProxyProvider proxyProvider;
|
|
|
|
|
|
|
|
|
|
private final boolean responseHeader = true;
|
|
|
|
|
private boolean responseHeader = true;
|
|
|
|
|
|
|
|
|
|
private volatile boolean refreshProxyOnError = false;
|
|
|
|
|
|
|
|
|
|
private Predicate<Throwable> refreshProxyOnError = t -> false;
|
|
|
|
|
private Predicate<Throwable> throwablePredicate = t->false;
|
|
|
|
|
|
|
|
|
|
public void setRefreshProxyOnError(Predicate<Throwable> proxyOnError) {
|
|
|
|
|
this.refreshProxyOnError = refreshProxyOnError;
|
|
|
|
|
public void setThrowablePredicate(Predicate<Throwable> predicate){
|
|
|
|
|
this.throwablePredicate = predicate;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) {
|
|
|
|
@ -116,7 +119,7 @@ public class HttpClientDownloader extends AbstractDownloader {
|
|
|
|
|
String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue();
|
|
|
|
|
Page page = new Page();
|
|
|
|
|
page.setBytes(bytes);
|
|
|
|
|
if (!request.isBinaryContent()) {
|
|
|
|
|
if (!request.isBinaryContent()){
|
|
|
|
|
if (charset == null) {
|
|
|
|
|
charset = getHtmlCharset(contentType, bytes);
|
|
|
|
|
}
|
|
|
|
|