|
|
|
@ -1,8 +1,9 @@
|
|
|
|
|
package us.codecraft.webmagic.downloader;
|
|
|
|
|
|
|
|
|
|
import org.apache.http.*;
|
|
|
|
|
import org.apache.http.HttpException;
|
|
|
|
|
import org.apache.http.HttpRequest;
|
|
|
|
|
import org.apache.http.HttpRequestInterceptor;
|
|
|
|
|
import org.apache.http.client.CookieStore;
|
|
|
|
|
import org.apache.http.client.protocol.ResponseContentEncoding;
|
|
|
|
|
import org.apache.http.config.Registry;
|
|
|
|
|
import org.apache.http.config.RegistryBuilder;
|
|
|
|
|
import org.apache.http.config.SocketConfig;
|
|
|
|
@ -66,24 +67,6 @@ public class HttpClientGenerator {
|
|
|
|
|
}
|
|
|
|
|
SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
|
|
|
|
|
httpClientBuilder.setDefaultSocketConfig(socketConfig);
|
|
|
|
|
// Http client has some problem handling compressing entity for redirect
|
|
|
|
|
// So I disable it and do it manually
|
|
|
|
|
// https://issues.apache.org/jira/browse/HTTPCLIENT-1432
|
|
|
|
|
httpClientBuilder.disableContentCompression();
|
|
|
|
|
httpClientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {
|
|
|
|
|
|
|
|
|
|
private ResponseContentEncoding contentEncoding = new ResponseContentEncoding();
|
|
|
|
|
|
|
|
|
|
public void process(
|
|
|
|
|
final HttpResponse response,
|
|
|
|
|
final HttpContext context) throws HttpException, IOException {
|
|
|
|
|
if (response.getStatusLine().getStatusCode() == 301 || response.getStatusLine().getStatusCode() == 302) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
contentEncoding.process(response, context);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
});
|
|
|
|
|
if (site != null) {
|
|
|
|
|
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
|
|
|
|
|
}
|
|
|
|
|