diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 8cc52dbf..1d308bcb 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -2,8 +2,8 @@ package us.codecraft.webmagic.downloader; import java.io.IOException; import java.nio.charset.Charset; +import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; import org.apache.commons.io.IOUtils; @@ -24,7 +24,6 @@ import us.codecraft.webmagic.selector.PlainText; import us.codecraft.webmagic.utils.CharsetUtils; import us.codecraft.webmagic.utils.HttpClientUtils; - /** * The http downloader based on HttpClient. * @@ -33,7 +32,7 @@ import us.codecraft.webmagic.utils.HttpClientUtils; */ public class HttpClientDownloader extends AbstractDownloader { - private final Map httpClients = new ConcurrentHashMap<>(); + private final Map httpClients = new HashMap(); private final Logger logger = LoggerFactory.getLogger(getClass()); private final HttpClientGenerator httpClientGenerator = new HttpClientGenerator(); @@ -46,13 +45,6 @@ public class HttpClientDownloader extends AbstractDownloader { private Predicate refreshProxyOnError = t -> false; - - private Predicate refreshClientOnError = t -> false; - - - public void setRefreshClientOnError(Predicate clientOnError){ - this.refreshClientOnError = clientOnError; - } public void setRefreshProxyOnError(Predicate proxyOnError) { this.refreshProxyOnError = refreshProxyOnError; } @@ -70,8 +62,17 @@ public class HttpClientDownloader extends AbstractDownloader { return httpClientGenerator.getClient(null); } String domain = site.getDomain(); - return httpClients.computeIfAbsent(domain,k->httpClientGenerator.getClient(site)); - + CloseableHttpClient httpClient = httpClients.get(domain); + if (httpClient == null) { + synchronized (this) { + httpClient = httpClients.get(domain); + if (httpClient == null) { + httpClient = httpClientGenerator.getClient(site); + httpClients.put(domain, httpClient); + } + } + } + return httpClient; } @Override