diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Site.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Site.java index bc786fc0..698265ce 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Site.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Site.java @@ -4,6 +4,7 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.Table; import org.apache.http.HttpHost; +import org.apache.http.auth.UsernamePasswordCredentials; import us.codecraft.webmagic.proxy.ProxyPool; import us.codecraft.webmagic.utils.UrlUtils; @@ -51,6 +52,8 @@ public class Site { private HttpHost httpProxy; + private UsernamePasswordCredentials usernamePasswordCredentials; //代理用户名密码设置 + private ProxyPool httpProxyPool; private boolean useGzip = true; @@ -479,6 +482,15 @@ public class Site { return this; } + public UsernamePasswordCredentials getUsernamePasswordCredentials() { + return usernamePasswordCredentials; + } + + public Site setUsernamePasswordCredentials(UsernamePasswordCredentials usernamePasswordCredentials) { + this.usernamePasswordCredentials = usernamePasswordCredentials; + return this; + } + public ProxyPool getHttpProxyPool() { return httpProxyPool; } diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java index 136d9c57..4bf5269b 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java @@ -3,7 +3,10 @@ package us.codecraft.webmagic.downloader; import org.apache.http.HttpException; import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CookieStore; +import org.apache.http.client.CredentialsProvider; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.config.SocketConfig; @@ -65,6 +68,15 @@ public class HttpClientGenerator { } }); } + + if(site!=null&&site.getHttpProxy()!=null&&site.getUsernamePasswordCredentials()!=null){ + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials( + new AuthScope(site.getHttpProxy()),//可以访问的范围 + site.getUsernamePasswordCredentials());//用户名和密码 + httpClientBuilder.setDefaultCredentialsProvider(credsProvider); + } + SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build(); httpClientBuilder.setDefaultSocketConfig(socketConfig); if (site != null) {