Merge pull request #3 from code4craft/master

合并最新代码

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -86,10 +86,10 @@ public class HttpClientDownloader extends AbstractDownloader {
         try {
             HttpHost proxyHost = null;
             Proxy proxy = null; //TODO
-            if (site.getHttpProxyPool() != null && site.getHttpProxyPool().isEnable()) {
+            if (site != null && site.getHttpProxyPool() != null && site.getHttpProxyPool().isEnable()) {
                 proxy = site.getHttpProxyFromPool();
                 proxyHost = proxy.getHttpHost();
-            } else if(site.getHttpProxy()!= null){
+            } else if (site != null && site.getHttpProxy() != null){
                 proxyHost = site.getHttpProxy();
             }
             
@@ -107,24 +107,20 @@ public class HttpClientDownloader extends AbstractDownloader {
             }
         } catch (IOException e) {
             logger.warn("download page {} error", request.getUrl(), e);
-            if (site.getCycleRetryTimes() > 0) {
+            if (site != null && site.getCycleRetryTimes() > 0) {
                 return addToCycleRetry(request, site);
             }
             onError(request);
             return null;
         } finally {
-        	request.putExtra(Request.STATUS_CODE, statusCode);
-            if (site.getHttpProxyPool()!=null && site.getHttpProxyPool().isEnable()) {
-                site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request
-                        .getExtra(Request.STATUS_CODE));
-            }
-            try {
             if (httpResponse != null) {
                 //ensure the connection is released back to pool
-                    EntityUtils.consume(httpResponse.getEntity());
+                EntityUtils.consumeQuietly(httpResponse.getEntity());
             }
-            } catch (IOException e) {
+        	request.putExtra(Request.STATUS_CODE, statusCode);
-                logger.warn("close response fail", e);
+            if (site != null && site.getHttpProxyPool() != null && site.getHttpProxyPool().isEnable()) {
+                site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request
+                        .getExtra(Request.STATUS_CODE));
             }
         }
     }
@@ -145,11 +141,15 @@ public class HttpClientDownloader extends AbstractDownloader {
                 requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
             }
         }
-        RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
+
-                .setConnectionRequestTimeout(site.getTimeOut())
+        RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
+        if (site != null) {
+            requestConfigBuilder.setConnectionRequestTimeout(site.getTimeOut())
                     .setSocketTimeout(site.getTimeOut())
                     .setConnectTimeout(site.getTimeOut())
                     .setCookieSpec(CookieSpecs.BEST_MATCH);
+        }
+
         if (proxy != null) {
 			requestConfigBuilder.setProxy(proxy);
 			request.putExtra(Request.PROXY, proxy);

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java

@@ -138,13 +138,18 @@ public class HttpClientGenerator {
         //解决post/redirect/post 302跳转问题
         httpClientBuilder.setRedirectStrategy(new CustomRedirectStrategy());
 
-        SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(site.getTimeOut()).setSoKeepAlive(true).setTcpNoDelay(true).build();
+        SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
+        socketConfigBuilder.setSoKeepAlive(true).setTcpNoDelay(true);
+        if (site != null) {
+            socketConfigBuilder.setSoTimeout(site.getTimeOut());
+        }
+        SocketConfig socketConfig = socketConfigBuilder.build();
         httpClientBuilder.setDefaultSocketConfig(socketConfig);
         connectionManager.setDefaultSocketConfig(socketConfig);
         if (site != null) {
             httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
-        }
             generateCookie(httpClientBuilder, site);
+        }
         return httpClientBuilder.build();
     }
 

webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java

@@ -145,4 +145,20 @@ public class HttpClientDownloaderTest {
             }
         });
     }
+
+    @Test
+    public void test_download_when_task_is_null() throws Exception {
+        HttpServer server = httpserver(12306);
+        server.response("foo");
+        Runner.running(server, new Runnable() {
+            @Override
+            public void run() throws Exception {
+                final HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
+                Request request = new Request();
+                request.setUrl("http://127.0.0.1:12306/");
+                Page page = httpClientDownloader.download(request, null);
+                assertThat(page.getRawText()).isEqualTo("foo");
+            }
+        });
+    }
 }