|
|
@ -1,5 +1,18 @@
|
|
|
|
package us.codecraft.webmagic.downloader;
|
|
|
|
package us.codecraft.webmagic.downloader;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
|
|
|
import java.security.KeyManagementException;
|
|
|
|
|
|
|
|
import java.security.NoSuchAlgorithmException;
|
|
|
|
|
|
|
|
import java.security.cert.CertificateException;
|
|
|
|
|
|
|
|
import java.security.cert.X509Certificate;
|
|
|
|
|
|
|
|
import java.util.Map;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import javax.net.ssl.SSLContext;
|
|
|
|
|
|
|
|
import javax.net.ssl.TrustManager;
|
|
|
|
|
|
|
|
import javax.net.ssl.X509TrustManager;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import org.apache.commons.lang3.JavaVersion;
|
|
|
|
|
|
|
|
import org.apache.commons.lang3.SystemUtils;
|
|
|
|
import org.apache.http.HttpException;
|
|
|
|
import org.apache.http.HttpException;
|
|
|
|
import org.apache.http.HttpRequest;
|
|
|
|
import org.apache.http.HttpRequest;
|
|
|
|
import org.apache.http.HttpRequestInterceptor;
|
|
|
|
import org.apache.http.HttpRequestInterceptor;
|
|
|
@ -11,23 +24,18 @@ import org.apache.http.conn.socket.ConnectionSocketFactory;
|
|
|
|
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
|
|
|
|
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
|
|
|
|
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
|
|
|
|
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
|
|
|
|
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
|
|
|
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
|
|
|
import org.apache.http.impl.client.*;
|
|
|
|
import org.apache.http.impl.client.BasicCookieStore;
|
|
|
|
|
|
|
|
import org.apache.http.impl.client.CloseableHttpClient;
|
|
|
|
|
|
|
|
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
|
|
|
|
|
|
|
|
import org.apache.http.impl.client.HttpClientBuilder;
|
|
|
|
|
|
|
|
import org.apache.http.impl.client.HttpClients;
|
|
|
|
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
|
|
|
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
|
|
|
import org.apache.http.impl.cookie.BasicClientCookie;
|
|
|
|
import org.apache.http.impl.cookie.BasicClientCookie;
|
|
|
|
import org.apache.http.protocol.HttpContext;
|
|
|
|
import org.apache.http.protocol.HttpContext;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import us.codecraft.webmagic.Site;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import javax.net.ssl.SSLContext;
|
|
|
|
import us.codecraft.webmagic.Site;
|
|
|
|
import javax.net.ssl.TrustManager;
|
|
|
|
|
|
|
|
import javax.net.ssl.X509TrustManager;
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
|
|
|
import java.security.KeyManagementException;
|
|
|
|
|
|
|
|
import java.security.NoSuchAlgorithmException;
|
|
|
|
|
|
|
|
import java.security.cert.CertificateException;
|
|
|
|
|
|
|
|
import java.security.cert.X509Certificate;
|
|
|
|
|
|
|
|
import java.util.Map;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* @author code4crafter@gmail.com <br>
|
|
|
|
* @author code4crafter@gmail.com <br>
|
|
|
@ -50,7 +58,15 @@ public class HttpClientGenerator {
|
|
|
|
|
|
|
|
|
|
|
|
private SSLConnectionSocketFactory buildSSLConnectionSocketFactory() {
|
|
|
|
private SSLConnectionSocketFactory buildSSLConnectionSocketFactory() {
|
|
|
|
try {
|
|
|
|
try {
|
|
|
|
return new SSLConnectionSocketFactory(createIgnoreVerifySSL(), new String[]{"SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"},
|
|
|
|
SSLContext sslContext = createIgnoreVerifySSL();
|
|
|
|
|
|
|
|
String[] supportedProtocols;
|
|
|
|
|
|
|
|
if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_11)) {
|
|
|
|
|
|
|
|
supportedProtocols = new String[] { "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" };
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
supportedProtocols = new String[] { "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2" };
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info("supportedProtocols: {}", String.join(", ", supportedProtocols));
|
|
|
|
|
|
|
|
return new SSLConnectionSocketFactory(sslContext, supportedProtocols,
|
|
|
|
null,
|
|
|
|
null,
|
|
|
|
new DefaultHostnameVerifier()); // 优先绕过安全证书
|
|
|
|
new DefaultHostnameVerifier()); // 优先绕过安全证书
|
|
|
|
} catch (KeyManagementException e) {
|
|
|
|
} catch (KeyManagementException e) {
|
|
|
|