diff --git a/pom.xml b/pom.xml index 2fb56e7b..eecbdb9e 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,7 @@ us.codecraft - 0.10.0 + 0.10.1 4.0.0 pom @@ -30,7 +30,7 @@ 1.3.0 1.2.0 11.4 - 4.14.1 + 3.141.59 2.0.4 4.0.0.RELEASE 0.3.5 diff --git a/webmagic-core/pom.xml b/webmagic-core/pom.xml index 021a83f3..dcf2e4c4 100644 --- a/webmagic-core/pom.xml +++ b/webmagic-core/pom.xml @@ -3,7 +3,7 @@ us.codecraft webmagic-parent - 0.10.0 + 0.10.1 4.0.0 diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 80e7b72c..31eebc72 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.Optional; import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; @@ -85,13 +86,13 @@ public class HttpClientDownloader extends AbstractDownloader { page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task); onSuccess(page, task); - logger.info("downloading page success {}", request.getUrl()); + logger.info("Download page success: {}", request.getUrl()); return page; } catch (IOException e) { onError(page, task, e); - logger.info("download page {} error", request.getUrl(), e); + logger.info("Download page error: {}", request.getUrl(), e); return page; } finally { @@ -111,7 +112,8 @@ public class HttpClientDownloader extends AbstractDownloader { } protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task) throws IOException { - byte[] bytes = IOUtils.toByteArray(httpResponse.getEntity().getContent()); + HttpEntity entity = httpResponse.getEntity(); + byte[] bytes = entity != null ? IOUtils.toByteArray(entity.getContent()) : new byte[0];; String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue(); Page page = new Page(); page.setBytes(bytes); diff --git a/webmagic-coverage/pom.xml b/webmagic-coverage/pom.xml index 4109c49f..3e441ecd 100644 --- a/webmagic-coverage/pom.xml +++ b/webmagic-coverage/pom.xml @@ -8,7 +8,7 @@ us.codecraft webmagic-parent - 0.10.0 + 0.10.1 webmagic-coverage diff --git a/webmagic-extension/pom.xml b/webmagic-extension/pom.xml index b47ae361..d99bf826 100644 --- a/webmagic-extension/pom.xml +++ b/webmagic-extension/pom.xml @@ -3,7 +3,7 @@ us.codecraft webmagic-parent - 0.10.0 + 0.10.1 4.0.0 diff --git a/webmagic-samples/pom.xml b/webmagic-samples/pom.xml index 906606fb..bae376ef 100644 --- a/webmagic-samples/pom.xml +++ b/webmagic-samples/pom.xml @@ -3,7 +3,7 @@ webmagic-parent us.codecraft - 0.10.0 + 0.10.1 4.0.0 diff --git a/webmagic-saxon/pom.xml b/webmagic-saxon/pom.xml index 4a2b358d..9709a04f 100644 --- a/webmagic-saxon/pom.xml +++ b/webmagic-saxon/pom.xml @@ -3,7 +3,7 @@ webmagic-parent us.codecraft - 0.10.0 + 0.10.1 4.0.0 diff --git a/webmagic-scripts/pom.xml b/webmagic-scripts/pom.xml index 92914655..2939b824 100644 --- a/webmagic-scripts/pom.xml +++ b/webmagic-scripts/pom.xml @@ -3,7 +3,7 @@ webmagic-parent us.codecraft - 0.10.0 + 0.10.1 4.0.0 diff --git a/webmagic-selenium/pom.xml b/webmagic-selenium/pom.xml index 5c2e50b2..cd121304 100644 --- a/webmagic-selenium/pom.xml +++ b/webmagic-selenium/pom.xml @@ -3,7 +3,7 @@ webmagic-parent us.codecraft - 0.10.0 + 0.10.1 4.0.0