Merge branch 'hotfix/0.10.1'

pull/1121/merge WebMagic-0.10.1
Sutra Zhou 11 months ago
commit 2994716ba5

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>us.codecraft</groupId>
<version>0.10.0</version>
<version>0.10.1</version>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
<properties>
@ -30,7 +30,7 @@
<moco.version>1.3.0</moco.version>
<phantomjsdriver.version>1.2.0</phantomjsdriver.version>
<saxon-he.version>11.4</saxon-he.version>
<selenium-java.version>4.14.1</selenium-java.version>
<selenium-java.version>3.141.59</selenium-java.version>
<slf4j.version>2.0.4</slf4j.version>
<spring-version>4.0.0.RELEASE</spring-version>
<xsoup.version>0.3.5</xsoup.version>

@ -3,7 +3,7 @@
<parent>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.10.0</version>
<version>0.10.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>

@ -7,6 +7,7 @@ import java.util.Map;
import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.impl.client.CloseableHttpClient;
@ -85,13 +86,13 @@ public class HttpClientDownloader extends AbstractDownloader {
page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
onSuccess(page, task);
logger.info("downloading page success {}", request.getUrl());
logger.info("Download page success: {}", request.getUrl());
return page;
} catch (IOException e) {
onError(page, task, e);
logger.info("download page {} error", request.getUrl(), e);
logger.info("Download page error: {}", request.getUrl(), e);
return page;
} finally {
@ -111,7 +112,8 @@ public class HttpClientDownloader extends AbstractDownloader {
}
protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task) throws IOException {
byte[] bytes = IOUtils.toByteArray(httpResponse.getEntity().getContent());
HttpEntity entity = httpResponse.getEntity();
byte[] bytes = entity != null ? IOUtils.toByteArray(entity.getContent()) : new byte[0];;
String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue();
Page page = new Page();
page.setBytes(bytes);

@ -8,7 +8,7 @@
<parent>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.10.0</version>
<version>0.10.1</version>
</parent>
<artifactId>webmagic-coverage</artifactId>

@ -3,7 +3,7 @@
<parent>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.10.0</version>
<version>0.10.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>

@ -3,7 +3,7 @@
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.10.0</version>
<version>0.10.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>

@ -3,7 +3,7 @@
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.10.0</version>
<version>0.10.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>

@ -3,7 +3,7 @@
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.10.0</version>
<version>0.10.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>

@ -3,7 +3,7 @@
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.10.0</version>
<version>0.10.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Loading…
Cancel
Save