github-forks
/
webmagic
mirror of https://github.com/code4craft/webmagic.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into master

Browse Source
pull/944/head
apaqi 5 years ago committed by GitHub
parent 78a1621831 96ebe608cc
commit c4c48d3522
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 516 additions and 257 deletions
Show Stats Download Patch File Download Diff File

2
.travis.yml
Unescape Escape View File

@ -1,3 +1,3 @@
language: java language: java
jdk: jdk:
- oraclejdk7 - openjdk9

4
README-zh.md
Unescape Escape View File

@ -93,7 +93,7 @@ webmagic还包含两个可用的扩展包，因为这两个包都依赖了比较
PageProcessor是webmagic-core的一部分，定制一个PageProcessor即可实现自己的爬虫逻辑。以下是抓取osc博客的一段代码： PageProcessor是webmagic-core的一部分，定制一个PageProcessor即可实现自己的爬虫逻辑。以下是抓取osc博客的一段代码：
```java ```java
public class OschinaBlogPageProcesser implements PageProcessor { public class OschinaBlogPageProcessor implements PageProcessor {
private Site site = Site.me().setDomain("my.oschina.net"); private Site site = Site.me().setDomain("my.oschina.net");
@ -113,7 +113,7 @@ public class OschinaBlogPageProcesser implements PageProcessor {
} }
public static void main(String[] args) { public static void main(String[] args) {
Spider.create(new OschinaBlogPageProcesser()).addUrl("http://my.oschina.net/flashsword/blog") Spider.create(new OschinaBlogPageProcessor()).addUrl("http://my.oschina.net/flashsword/blog")
.addPipeline(new ConsolePipeline()).run(); .addPipeline(new ConsolePipeline()).run();
} }
} }

132
pom.xml
Unescape Escape View File

@ -1,10 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.sonatype.oss</groupId>
<artifactId>oss-parent</artifactId>
<version>7</version>
</parent>
<groupId>us.codecraft</groupId> <groupId>us.codecraft</groupId>
<version>0.7.3</version> <version>0.7.3</version>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
@ -12,8 +7,8 @@
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
<spring-version>4.0.0.RELEASE</spring-version> <spring-version>4.0.0.RELEASE</spring-version>
</properties> </properties>
<artifactId>webmagic-parent</artifactId> <artifactId>webmagic-parent</artifactId>
<name>webmagic-parent</name> <name>webmagic-parent</name>
@ -39,7 +34,7 @@
<developerConnection>scm:git:git@github.com:code4craft/webmagic.git</developerConnection> <developerConnection>scm:git:git@github.com:code4craft/webmagic.git</developerConnection>
<url>git@github.com:code4craft/webmagic.git</url> <url>git@github.com:code4craft/webmagic.git</url>
<tag>webmagic-parent-0.6.1</tag> <tag>webmagic-parent-0.6.1</tag>
</scm> </scm>
<licenses> <licenses>
<license> <license>
<name>Apache License, Version 2.0</name> <name>Apache License, Version 2.0</name>
@ -61,7 +56,7 @@
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
<version>4.11</version> <version>4.13</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
@ -73,12 +68,17 @@
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId> <artifactId>httpclient</artifactId>
<version>4.5.2</version> <version>4.5.12</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.13</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.guava</groupId> <groupId>com.google.guava</groupId>
<artifactId>guava</artifactId> <artifactId>guava</artifactId>
<version>15.0</version> <version>29.0-jre</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.jayway.jsonpath</groupId> <groupId>com.jayway.jsonpath</groupId>
@ -88,12 +88,12 @@
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId> <artifactId>slf4j-api</artifactId>
<version>1.7.6</version> <version>1.7.30</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId> <artifactId>slf4j-log4j12</artifactId>
<version>1.7.6</version> <version>1.7.30</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>us.codecraft</groupId>
@ -103,12 +103,12 @@
<dependency> <dependency>
<groupId>com.alibaba</groupId> <groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId> <artifactId>fastjson</artifactId>
<version>1.2.28</version> <version>1.2.68</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.github.dreamhead</groupId> <groupId>com.github.dreamhead</groupId>
<artifactId>moco-core</artifactId> <artifactId>moco-core</artifactId>
<version>0.11.0</version> <version>1.1.0</version>
<scope>test</scope> <scope>test</scope>
<exclusions> <exclusions>
<exclusion> <exclusion>
@ -125,13 +125,13 @@
<dependency> <dependency>
<groupId>org.assertj</groupId> <groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId> <artifactId>assertj-core</artifactId>
<version>1.5.0</version> <version>3.16.1</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
<version>3.1</version> <version>3.10</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>commons-collections</groupId> <groupId>commons-collections</groupId>
@ -139,9 +139,19 @@
<version>3.2.2</version> <version>3.2.2</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>commons-io</groupId>
<artifactId>commons-io</artifactId> <artifactId>commons-io</artifactId>
<version>1.3.2</version> <version>2.7</version>
</dependency>
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
<version>2.4.19</version>
</dependency>
<dependency>
<groupId>org.jruby</groupId>
<artifactId>jruby</artifactId>
<version>9.2.11.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.jsoup</groupId> <groupId>org.jsoup</groupId>
@ -149,20 +159,69 @@
<version>1.10.3</version> <version>1.10.3</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.mockito</groupId> <groupId>org.python</groupId>
<artifactId>mockito-all</artifactId> <artifactId>jython</artifactId>
<version>1.9.5</version> <version>2.7.2</version>
<scope>test</scope> </dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>3.141.59</version>
</dependency>
<dependency>
<groupId>net.sf.saxon</groupId>
<artifactId>Saxon-HE</artifactId>
<version>10.1</version>
</dependency>
<dependency>
<groupId>net.sourceforge.htmlcleaner</groupId>
<artifactId>htmlcleaner</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>com.github.detro</groupId>
<artifactId>phantomjsdriver</artifactId>
<version>1.2.0</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.9.3</version>
</dependency> </dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
<build> <build>
<plugins> <plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>3.0.0-M3</version>
<executions>
<execution>
<id>enforce-maven</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireMavenVersion>
<version>3.0.5</version>
</requireMavenVersion>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.18</version> <version>3.0.0-M4</version>
<configuration> <configuration>
<forkCount>0</forkCount> <forkCount>0</forkCount>
</configuration> </configuration>
@ -170,11 +229,10 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId> <artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version> <version>3.8.1</version>
<configuration> <configuration>
<source>1.6</source> <source>${java.version}</source>
<target>1.6</target> <target>${java.version}</target>
<encoding>UTF-8</encoding>
</configuration> </configuration>
</plugin> </plugin>
<!--<plugin>--> <!--<plugin>-->
@ -200,14 +258,12 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId> <artifactId>maven-resources-plugin</artifactId>
<version>2.6</version> <version>3.1.0</version>
<configuration>
<encoding>UTF-8</encoding>
</configuration>
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId> <artifactId>maven-jar-plugin</artifactId>
<version>3.2.0</version>
<configuration> <configuration>
<excludes> <excludes>
<exclude>log4j.xml</exclude> <exclude>log4j.xml</exclude>
@ -217,7 +273,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId> <artifactId>maven-source-plugin</artifactId>
<version>2.2.1</version> <version>3.2.1</version>
<executions> <executions>
<execution> <execution>
<id>attach-sources</id> <id>attach-sources</id>
@ -230,11 +286,15 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId> <artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.4</version> <version>3.2.0</version>
<configuration> <configuration>
<encoding>UTF-8</encoding> <encoding>UTF-8</encoding>
<doctitle>WebMagic 0.7.3</doctitle> <doctitle>WebMagic 0.7.3</doctitle>
<locale>en_US</locale> <locale>en_US</locale>
<!-- avoid the issue: https://bugs.openjdk.java.net/browse/JDK-8212233 -->
<detectJavaApiLink>false</detectJavaApiLink>
</configuration> </configuration>
<executions> <executions>
<execution> <execution>
@ -255,7 +315,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId> <artifactId>maven-release-plugin</artifactId>
<version>2.4.1</version> <version>3.0.0-M1</version>
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
@ -310,7 +370,7 @@
<plugin> <plugin>
<groupId>org.sonatype.plugins</groupId> <groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId> <artifactId>nexus-staging-maven-plugin</artifactId>
<version>1.6</version> <version>1.6.8</version>
<extensions>true</extensions> <extensions>true</extensions>
<configuration> <configuration>
<serverId>sonatype-nexus-staging</serverId> <serverId>sonatype-nexus-staging</serverId>

5
webmagic-core/pom.xml
Unescape Escape View File

@ -48,6 +48,7 @@
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId> <artifactId>slf4j-log4j12</artifactId>
<optional>true</optional>
</dependency> </dependency>
<dependency> <dependency>
@ -66,7 +67,7 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>commons-io</groupId>
<artifactId>commons-io</artifactId> <artifactId>commons-io</artifactId>
</dependency> </dependency>
@ -82,4 +83,4 @@
</dependencies> </dependencies>
</project> </project>

7
webmagic-core/src/main/java/us/codecraft/webmagic/Request.java
Unescape Escape View File

@ -78,14 +78,15 @@ public class Request implements Serializable {
return this; return this;
} }
public Object getExtra(String key) { @SuppressWarnings("unchecked")
public <T> T getExtra(String key) {
if (extras == null) { if (extras == null) {
return null; return null;
} }
return extras.get(key); return (T) extras.get(key);
} }
public Request putExtra(String key, Object value) { public <T> Request putExtra(String key, T value) {
if (extras == null) { if (extras == null) {
extras = new HashMap<String, Object>(); extras = new HashMap<String, Object>();
} }

2
webmagic-core/src/main/java/us/codecraft/webmagic/ResultItems.java
Unescape Escape View File

@ -1,6 +1,5 @@
package us.codecraft.webmagic; package us.codecraft.webmagic;
import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
@ -21,6 +20,7 @@ public class ResultItems {
private boolean skip; private boolean skip;
@SuppressWarnings("unchecked")
public <T> T get(String key) { public <T> T get(String key) {
Object o = fields.get(key); Object o = fields.get(key);
if (o == null) { if (o == null) {

4
webmagic-core/src/main/java/us/codecraft/webmagic/Site.java
Unescape Escape View File

@ -203,7 +203,7 @@ public class Site {
/** /**
* Set the interval between the processing of two pages.<br> * Set the interval between the processing of two pages.<br>
* Time unit is micro seconds.<br> * Time unit is milliseconds.<br>
* *
* @param sleepTime sleepTime * @param sleepTime sleepTime
* @return this * @return this
@ -215,7 +215,7 @@ public class Site {
/** /**
* Get the interval between the processing of two pages.<br> * Get the interval between the processing of two pages.<br>
* Time unit is micro seconds.<br> * Time unit is milliseconds.<br>
* *
* @return the interval between the processing of two pages, * @return the interval between the processing of two pages,
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
Unescape Escape View File

@ -2,7 +2,6 @@ package us.codecraft.webmagic.downloader;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.annotation.ThreadSafe;
import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
@ -30,7 +29,6 @@ import java.util.Map;
* @author code4crafter@gmail.com <br> * @author code4crafter@gmail.com <br>
* @since 0.1.0 * @since 0.1.0
*/ */
@ThreadSafe
public class HttpClientDownloader extends AbstractDownloader { public class HttpClientDownloader extends AbstractDownloader {
private Logger logger = LoggerFactory.getLogger(getClass()); private Logger logger = LoggerFactory.getLogger(getClass());

103
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java
Unescape Escape View File

@ -1,5 +1,18 @@
package us.codecraft.webmagic.downloader; package us.codecraft.webmagic.downloader;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Map;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.apache.commons.lang3.JavaVersion;
import org.apache.commons.lang3.SystemUtils;
import org.apache.http.HttpException; import org.apache.http.HttpException;
import org.apache.http.HttpRequest; import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpRequestInterceptor;
@ -9,34 +22,30 @@ import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig; import org.apache.http.config.SocketConfig;
import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.*; import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.impl.cookie.BasicClientCookie; import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpContext;
import org.apache.http.ssl.SSLContexts; import org.apache.http.ssl.SSLContexts;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Site;
import javax.net.ssl.SSLContext; import us.codecraft.webmagic.Site;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Map;
/** /**
* @author code4crafter@gmail.com <br> * @author code4crafter@gmail.com <br>
* @since 0.4.0 * @since 0.4.0
*/ */
public class HttpClientGenerator { public class HttpClientGenerator {
private transient Logger logger = LoggerFactory.getLogger(getClass()); private transient Logger logger = LoggerFactory.getLogger(getClass());
private PoolingHttpClientConnectionManager connectionManager; private PoolingHttpClientConnectionManager connectionManager;
public HttpClientGenerator() { public HttpClientGenerator() {
@ -48,41 +57,51 @@ public class HttpClientGenerator {
connectionManager.setDefaultMaxPerRoute(100); connectionManager.setDefaultMaxPerRoute(100);
} }
private SSLConnectionSocketFactory buildSSLConnectionSocketFactory() { private SSLConnectionSocketFactory buildSSLConnectionSocketFactory() {
try { try {
return new SSLConnectionSocketFactory(createIgnoreVerifySSL()); // 优先绕过安全证书 SSLContext sslContext = createIgnoreVerifySSL();
} catch (KeyManagementException e) { String[] supportedProtocols;
if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_11)) {
supportedProtocols = new String[] { "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" };
} else {
supportedProtocols = new String[] { "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2" };
}
logger.debug("supportedProtocols: {}", String.join(", ", supportedProtocols));
return new SSLConnectionSocketFactory(sslContext, supportedProtocols,
null,
new DefaultHostnameVerifier()); // 优先绕过安全证书
} catch (KeyManagementException e) {
logger.error("ssl connection fail", e); logger.error("ssl connection fail", e);
} catch (NoSuchAlgorithmException e) { } catch (NoSuchAlgorithmException e) {
logger.error("ssl connection fail", e); logger.error("ssl connection fail", e);
} }
return SSLConnectionSocketFactory.getSocketFactory(); return SSLConnectionSocketFactory.getSocketFactory();
} }
private SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException { private SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
// 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法 // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
X509TrustManager trustManager = new X509TrustManager() { X509TrustManager trustManager = new X509TrustManager() {
@Override @Override
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
} }
@Override @Override
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
} }
@Override @Override
public X509Certificate[] getAcceptedIssuers() { public X509Certificate[] getAcceptedIssuers() {
return null; return null;
} }
}; };
SSLContext sc = SSLContext.getInstance("SSLv3"); SSLContext sc = SSLContext.getInstance("SSLv3");
sc.init(null, new TrustManager[] { trustManager }, null); sc.init(null, new TrustManager[] { trustManager }, null);
return sc; return sc;
} }
public HttpClientGenerator setPoolSize(int poolSize) { public HttpClientGenerator setPoolSize(int poolSize) {
connectionManager.setMaxTotal(poolSize); connectionManager.setMaxTotal(poolSize);
return this; return this;
@ -94,7 +113,7 @@ public class HttpClientGenerator {
private CloseableHttpClient generateClient(Site site) { private CloseableHttpClient generateClient(Site site) {
HttpClientBuilder httpClientBuilder = HttpClients.custom(); HttpClientBuilder httpClientBuilder = HttpClients.custom();
httpClientBuilder.setConnectionManager(connectionManager); httpClientBuilder.setConnectionManager(connectionManager);
if (site.getUserAgent() != null) { if (site.getUserAgent() != null) {
httpClientBuilder.setUserAgent(site.getUserAgent()); httpClientBuilder.setUserAgent(site.getUserAgent());

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpUriRequestConverter.java
Unescape Escape View File

@ -74,7 +74,7 @@ public class HttpUriRequestConverter {
} }
if (proxy != null) { if (proxy != null) {
requestConfigBuilder.setProxy(new HttpHost(proxy.getHost(), proxy.getPort())); requestConfigBuilder.setProxy(new HttpHost(proxy.getHost(), proxy.getPort(), proxy.getScheme()));
} }
requestBuilder.setConfig(requestConfigBuilder.build()); requestBuilder.setConfig(requestConfigBuilder.build());
HttpUriRequest httpUriRequest = requestBuilder.build(); HttpUriRequest httpUriRequest = requestBuilder.build();

3
webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java
Unescape Escape View File

@ -1,10 +1,8 @@
package us.codecraft.webmagic.pipeline; package us.codecraft.webmagic.pipeline;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.http.annotation.ThreadSafe;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.ResultItems; import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task; import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.utils.FilePersistentBase; import us.codecraft.webmagic.utils.FilePersistentBase;
@ -24,6 +22,7 @@ import java.util.Map;
@ThreadSafe @ThreadSafe
public class FilePipeline extends FilePersistentBase implements Pipeline { public class FilePipeline extends FilePersistentBase implements Pipeline {
private Logger logger = LoggerFactory.getLogger(getClass()); private Logger logger = LoggerFactory.getLogger(getClass());
/** /**

188
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/Proxy.java
Unescape Escape View File

@ -1,73 +1,135 @@
package us.codecraft.webmagic.proxy; package us.codecraft.webmagic.proxy;
/** import java.io.UnsupportedEncodingException;
* import java.net.URI;
*/ import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import org.apache.commons.lang3.StringUtils;
public class Proxy { public class Proxy {
private String host; private String scheme;
private int port;
private String username; private String host;
private String password;
private int port;
private String username;
private String password;
public Proxy(String host, int port) { public static Proxy create(final URI uri) {
this.host = host; Proxy proxy = new Proxy(uri.getHost(), uri.getPort(), uri.getScheme());
this.port = port; String userInfo = uri.getUserInfo();
} if (userInfo != null) {
String[] up = userInfo.split(":");
if (up.length == 1) {
proxy.username = up[0].isEmpty() ? null : up[0];
} else {
proxy.username = up[0].isEmpty() ? null : up[0];
proxy.password = up[1].isEmpty() ? null : up[1];
}
}
return proxy;
}
public Proxy(String host, int port, String username, String password) { public Proxy(String host, int port) {
this.host = host; this(host, port, null);
this.port = port; }
this.username = username;
this.password = password; public Proxy(String host, int port, String scheme) {
} this.host = host;
this.port = port;
this.scheme = scheme;
}
public Proxy(String host, int port, String username, String password) {
this.host = host;
this.port = port;
this.username = username;
this.password = password;
}
public String getScheme() {
return scheme;
}
public void setScheme(String scheme) {
this.scheme = scheme;
}
public String getHost() { public String getHost() {
return host; return host;
} }
public int getPort() { public int getPort() {
return port; return port;
} }
public String getUsername() { public String getUsername() {
return username; return username;
} }
public String getPassword() { public String getPassword() {
return password; return password;
} }
@Override public URI toURI() {
public boolean equals(Object o) { final StringBuilder userInfoBuffer = new StringBuilder();
if (this == o) return true; if (username != null) {
if (o == null || getClass() != o.getClass()) return false; userInfoBuffer.append(urlencode(username));
}
Proxy proxy = (Proxy) o; if (password != null) {
userInfoBuffer.append(":").append(urlencode(password));
if (port != proxy.port) return false; }
if (host != null ? !host.equals(proxy.host) : proxy.host != null) return false; final String userInfo = StringUtils.defaultIfEmpty(userInfoBuffer.toString(), null);
if (username != null ? !username.equals(proxy.username) : proxy.username != null) return false; URI uri;
return password != null ? password.equals(proxy.password) : proxy.password == null; try {
} uri = new URI(scheme, userInfo, host, port, null, null, null);
} catch (URISyntaxException e) {
@Override throw new IllegalArgumentException(e.getMessage(), e);
public int hashCode() { }
int result = host != null ? host.hashCode() : 0; return uri;
result = 31 * result + port; }
result = 31 * result + (username != null ? username.hashCode() : 0);
result = 31 * result + (password != null ? password.hashCode() : 0); private String urlencode(String s) {
return result; String enc = StandardCharsets.UTF_8.name();
} try {
return URLEncoder.encode(s, enc);
@Override } catch (UnsupportedEncodingException e) {
public String toString() { throw new IllegalArgumentException(e);
return "Proxy{" + }
"host='" + host + '\'' + }
", port=" + port +
", username='" + username + '\'' + @Override
", password='" + password + '\'' + public boolean equals(Object o) {
'}'; if (this == o) return true;
} if (o == null || getClass() != o.getClass()) return false;
Proxy proxy = (Proxy) o;
if (port != proxy.port) return false;
if (host != null ? !host.equals(proxy.host) : proxy.host != null) return false;
if (scheme != null ? !scheme.equals(proxy.scheme) : proxy.scheme != null) return false;
if (username != null ? !username.equals(proxy.username) : proxy.username != null) return false;
return password != null ? password.equals(proxy.password) : proxy.password == null;
}
@Override
public int hashCode() {
int result = host != null ? host.hashCode() : 0;
result = 31 * result + port;
result = 31 * result + (scheme != null ? scheme.hashCode() : 0);
result = 31 * result + (username != null ? username.hashCode() : 0);
result = 31 * result + (password != null ? password.hashCode() : 0);
return result;
}
@Override
public String toString() {
return this.toURI().toString();
}
} }

2
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/ProxyProvider.java
Unescape Escape View File

@ -25,5 +25,5 @@ public interface ProxyProvider {
* @return proxy * @return proxy
*/ */
Proxy getProxy(Task task); Proxy getProxy(Task task);
} }

1
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/SimpleProxyProvider.java
Unescape Escape View File

@ -59,4 +59,5 @@ public class SimpleProxyProvider implements ProxyProvider {
} }
return p % size; return p % size;
} }
} }

2
webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/PriorityScheduler.java
Unescape Escape View File

@ -1,6 +1,5 @@
package us.codecraft.webmagic.scheduler; package us.codecraft.webmagic.scheduler;
import org.apache.http.annotation.ThreadSafe;
import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task; import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.utils.NumberUtils; import us.codecraft.webmagic.utils.NumberUtils;
@ -16,7 +15,6 @@ import java.util.concurrent.PriorityBlockingQueue;
* @author code4crafter@gmail.com <br> * @author code4crafter@gmail.com <br>
* @since 0.2.1 * @since 0.2.1
*/ */
@ThreadSafe
public class PriorityScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler { public class PriorityScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {
public static final int INITIAL_CAPACITY = 5; public static final int INITIAL_CAPACITY = 5;

2
webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/QueueScheduler.java
Unescape Escape View File

@ -1,6 +1,5 @@
package us.codecraft.webmagic.scheduler; package us.codecraft.webmagic.scheduler;
import org.apache.http.annotation.ThreadSafe;
import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task; import us.codecraft.webmagic.Task;
@ -15,7 +14,6 @@ import java.util.concurrent.LinkedBlockingQueue;
* @author code4crafter@gmail.com <br> * @author code4crafter@gmail.com <br>
* @since 0.1.0 * @since 0.1.0
*/ */
@ThreadSafe
public class QueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler { public class QueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {
private BlockingQueue<Request> queue = new LinkedBlockingQueue<Request>(); private BlockingQueue<Request> queue = new LinkedBlockingQueue<Request>();

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/RegexSelector.java
Unescape Escape View File

@ -41,7 +41,7 @@ public class RegexSelector implements Selector {
/** /**
* Create a RegexSelector. When there is no capture group, the value is set to 0 else set to 1. * Create a RegexSelector. When there is no capture group, the value is set to 0 else set to 1.
* @param regexStr * @param regexStr the regular expression.
*/ */
public RegexSelector(String regexStr) { public RegexSelector(String regexStr) {
this.compileRegex(regexStr); this.compileRegex(regexStr);

1
webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java
Unescape Escape View File

@ -322,4 +322,5 @@ public class HttpClientDownloaderTest {
}); });
} }
} }

26
webmagic-core/src/test/java/us/codecraft/webmagic/downloader/SSLCompatibilityTest.java
Unescape Escape View File

@ -0,0 +1,26 @@
package us.codecraft.webmagic.downloader;
import org.junit.Test;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author code4crafter@gmail.com
* Date: 2017/11/29
* Time: 下午1:32
*/
public class SSLCompatibilityTest {
@Test
public void test_tls12() throws Exception {
HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
Task task = Site.me().setCycleRetryTimes(5).toTask();
Request request = new Request("https://juejin.im/");
Page page = httpClientDownloader.download(request, task);
assertThat(page.isDownloadSuccess()).isTrue();
}
}

114
webmagic-core/src/test/java/us/codecraft/webmagic/proxy/ProxyTest.java
Unescape Escape View File

@ -1,45 +1,97 @@
package us.codecraft.webmagic.proxy; package us.codecraft.webmagic.proxy;
import org.apache.http.HttpHost; import static org.junit.Assert.assertEquals;
import org.junit.BeforeClass; import static org.junit.Assert.assertNull;
import java.net.URI;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.apache.http.HttpHost;
import org.junit.BeforeClass;
import org.junit.Test;
/** /**
* @author yxssfxwzy@sina.com May 30, 2014 * @author yxssfxwzy@sina.com May 30, 2014
* *
*/ */
public class ProxyTest { public class ProxyTest {
private static List<String[]> httpProxyList = new ArrayList<String[]>(); private static List<String[]> httpProxyList = new ArrayList<String[]>();
@BeforeClass @BeforeClass
public static void before() { public static void before() {
// String[] source = { "0.0.0.1:0", "0.0.0.2:0", "0.0.0.3:0", // String[] source = { "0.0.0.1:0", "0.0.0.2:0", "0.0.0.3:0",
// "0.0.0.4:0" }; // "0.0.0.4:0" };
String[] source = { "::0.0.0.1:0", "::0.0.0.2:0", "::0.0.0.3:0", "::0.0.0.4:0" }; String[] source = { "::0.0.0.1:0", "::0.0.0.2:0", "::0.0.0.3:0", "::0.0.0.4:0" };
for (String line : source) { for (String line : source) {
httpProxyList.add(new String[] {line.split(":")[0], line.split(":")[1], line.split(":")[2], line.split(":")[3] }); httpProxyList.add(new String[] {line.split(":")[0], line.split(":")[1], line.split(":")[2], line.split(":")[3] });
} }
} }
class Fetch extends Thread { class Fetch extends Thread {
HttpHost hp; HttpHost hp;
public Fetch(HttpHost hp) { public Fetch(HttpHost hp) {
this.hp = hp; this.hp = hp;
} }
@Override @Override
public void run() { public void run() {
try { try {
System.out.println("fetch web page use proxy: " + hp.getHostName() + ":" + hp.getPort()); System.out.println("fetch web page use proxy: " + hp.getHostName() + ":" + hp.getPort());
sleep(500); sleep(500);
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
} }
@Test
public void testCreate() {
Proxy proxy = Proxy.create(URI.create("//127.0.0.1:8080"));
assertNull(proxy.getScheme());
assertNull(proxy.getUsername());
assertNull(proxy.getPassword());
assertEquals("127.0.0.1", proxy.getHost());
assertEquals(8080, proxy.getPort());
proxy = Proxy.create(URI.create("http://127.0.0.1:8080"));
assertEquals("http", proxy.getScheme());
assertNull(proxy.getUsername());
assertNull(proxy.getPassword());
assertEquals("127.0.0.1", proxy.getHost());
assertEquals(8080, proxy.getPort());
proxy = Proxy.create(URI.create("//username:password@127.0.0.1:8080"));
assertNull(proxy.getScheme());
assertEquals("username", proxy.getUsername());
assertEquals("password", proxy.getPassword());
assertEquals("127.0.0.1", proxy.getHost());
assertEquals(8080, proxy.getPort());
proxy = Proxy.create(URI.create("//username@127.0.0.1:8080"));
assertNull(proxy.getScheme());
assertEquals("username", proxy.getUsername());
assertNull(proxy.getPassword());
assertEquals("127.0.0.1", proxy.getHost());
assertEquals(8080, proxy.getPort());
proxy = Proxy.create(URI.create("//:password@127.0.0.1:8080"));
assertNull(proxy.getScheme());
assertNull(proxy.getUsername());
assertEquals("password", proxy.getPassword());
assertEquals("127.0.0.1", proxy.getHost());
assertEquals(8080, proxy.getPort());
}
@Test
public void testToString() {
assertEquals("//127.0.0.1:8080", new Proxy("127.0.0.1", 8080).toString());
assertEquals("http://127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "http").toString());
assertEquals("//username:password@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "username", "password").toString());
assertEquals("//username@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "username", null).toString());
assertEquals("//:password@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, null, "password").toString());
}
} }

16
webmagic-core/src/test/java/us/codecraft/webmagic/utils/NumberUtilsTest.java
Unescape Escape View File

@ -0,0 +1,16 @@
package us.codecraft.webmagic.utils;
import org.junit.Assert;
import org.junit.Test;
public class NumberUtilsTest {
@Test
public void testCompareLong() {
Assert.assertEquals(0, NumberUtils.compareLong(0L, 0L));
Assert.assertEquals(1, NumberUtils.compareLong(9L, 0L));
Assert.assertEquals(-1, NumberUtils.compareLong(0L, 9L));
Assert.assertEquals(-1, NumberUtils.compareLong(-9L, 0L));
Assert.assertEquals(1, NumberUtils.compareLong(0L, -9L));
}
}

6
webmagic-extension/pom.xml
Unescape Escape View File

@ -13,16 +13,14 @@
<dependency> <dependency>
<groupId>redis.clients</groupId> <groupId>redis.clients</groupId>
<artifactId>jedis</artifactId> <artifactId>jedis</artifactId>
<version>2.9.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.guava</groupId> <groupId>com.google.guava</groupId>
<artifactId>guava</artifactId> <artifactId>guava</artifactId>
<version>15.0</version>
<optional>true</optional> <optional>true</optional>
</dependency> </dependency>
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>${project.groupId}</groupId>
<artifactId>webmagic-core</artifactId> <artifactId>webmagic-core</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
@ -32,4 +30,4 @@
</dependency> </dependency>
</dependencies> </dependencies>
</project> </project>

2
webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java
Unescape Escape View File

@ -1,6 +1,5 @@
package us.codecraft.webmagic.downloader; package us.codecraft.webmagic.downloader;
import org.apache.http.annotation.ThreadSafe;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Page;
@ -16,7 +15,6 @@ import java.io.*;
* @author dolphineor@gmail.com * @author dolphineor@gmail.com
* @version 0.5.3 * @version 0.5.3
*/ */
@ThreadSafe
public class PhantomJSDownloader extends AbstractDownloader { public class PhantomJSDownloader extends AbstractDownloader {
private static Logger logger = LoggerFactory.getLogger(PhantomJSDownloader.class); private static Logger logger = LoggerFactory.getLogger(PhantomJSDownloader.class);

6
webmagic-extension/src/main/java/us/codecraft/webmagic/example/PatternProcessorExample.java
Unescape Escape View File

@ -1,6 +1,8 @@
package us.codecraft.webmagic.example; package us.codecraft.webmagic.example;
import org.apache.log4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.*; import us.codecraft.webmagic.*;
import us.codecraft.webmagic.handler.CompositePageProcessor; import us.codecraft.webmagic.handler.CompositePageProcessor;
import us.codecraft.webmagic.handler.CompositePipeline; import us.codecraft.webmagic.handler.CompositePipeline;
@ -15,7 +17,7 @@ import us.codecraft.webmagic.handler.RequestMatcher;
*/ */
public class PatternProcessorExample { public class PatternProcessorExample {
private static Logger log = Logger.getLogger(PatternProcessorExample.class); private static Logger log = LoggerFactory.getLogger(PatternProcessorExample.class);
public static void main(String... args) { public static void main(String... args) {

35
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
Unescape Escape View File

@ -1,12 +1,13 @@
package us.codecraft.webmagic.scheduler; package us.codecraft.webmagic.scheduler;
import org.apache.commons.io.IOUtils; import java.io.BufferedReader;
import org.apache.commons.lang3.math.NumberUtils; import java.io.Closeable;
import us.codecraft.webmagic.Request; import java.io.File;
import us.codecraft.webmagic.Task; import java.io.FileNotFoundException;
import us.codecraft.webmagic.scheduler.component.DuplicateRemover; import java.io.FileReader;
import java.io.FileWriter;
import java.io.*; import java.io.IOException;
import java.io.PrintWriter;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.Set; import java.util.Set;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
@ -17,6 +18,13 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.math.NumberUtils;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.scheduler.component.DuplicateRemover;
/** /**
* Store urls and cursor in files so that a Spider can resume the status when shutdown.<br> * Store urls and cursor in files so that a Spider can resume the status when shutdown.<br>
@ -141,7 +149,7 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
urls.add(line.trim()); urls.add(line.trim());
lineReaded++; lineReaded++;
if (lineReaded > cursor.get()) { if (lineReaded > cursor.get()) {
queue.add(new Request(line)); queue.add(deserializeRequest(line));
} }
} }
} finally { } finally {
@ -183,7 +191,7 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
init(task); init(task);
} }
queue.add(request); queue.add(request);
fileUrlWriter.println(request.getUrl()); fileUrlWriter.println(serializeRequest(request));
} }
@Override @Override
@ -204,4 +212,13 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
public int getTotalRequestsCount(Task task) { public int getTotalRequestsCount(Task task) {
return getDuplicateRemover().getTotalRequestsCount(task); return getDuplicateRemover().getTotalRequestsCount(task);
} }
protected String serializeRequest(Request request) {
return request.getUrl();
}
protected Request deserializeRequest(String line) {
return new Request(line);
}
} }

37
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java
Unescape Escape View File

@ -2,6 +2,7 @@ package us.codecraft.webmagic.scheduler;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig; import redis.clients.jedis.JedisPoolConfig;
@ -60,14 +61,41 @@ public class RedisScheduler extends DuplicateRemovedScheduler implements Monitor
Jedis jedis = pool.getResource(); Jedis jedis = pool.getResource();
try { try {
jedis.rpush(getQueueKey(task), request.getUrl()); jedis.rpush(getQueueKey(task), request.getUrl());
if (request.getExtras() != null) { if (checkForAdditionalInfo(request)) {
String field = DigestUtils.shaHex(request.getUrl()); String field = DigestUtils.shaHex(request.getUrl());
String value = JSON.toJSONString(request); String value = JSON.toJSONString(request);
jedis.hset((ITEM_PREFIX + task.getUUID()), field, value); jedis.hset((ITEM_PREFIX + task.getUUID()), field, value);
} }
} finally { } finally {
pool.returnResource(jedis); jedis.close();
}
}
private boolean checkForAdditionalInfo(Request request) {
if (request == null) {
return false;
}
if (!request.getHeaders().isEmpty() || !request.getCookies().isEmpty()) {
return true;
}
if (StringUtils.isNotBlank(request.getCharset()) || StringUtils.isNotBlank(request.getMethod())) {
return true;
}
if (request.isBinaryContent() || request.getRequestBody() != null) {
return true;
} }
if (request.getExtras() != null && !request.getExtras().isEmpty()) {
return true;
}
if (request.getPriority() != 0L) {
return true;
}
return false;
} }
@Override @Override
@ -85,7 +113,7 @@ public class RedisScheduler extends DuplicateRemovedScheduler implements Monitor
Request o = JSON.parseObject(new String(bytes), Request.class); Request o = JSON.parseObject(new String(bytes), Request.class);
return o; return o;
} }
Request request = new Request(url); Request request = new Request(url);
return request; return request;
} finally { } finally {
pool.returnResource(jedis); pool.returnResource(jedis);
@ -100,8 +128,7 @@ public class RedisScheduler extends DuplicateRemovedScheduler implements Monitor
return QUEUE_PREFIX + task.getUUID(); return QUEUE_PREFIX + task.getUUID();
} }
protected String getItemKey(Task task) protected String getItemKey(Task task) {
{
return ITEM_PREFIX + task.getUUID(); return ITEM_PREFIX + task.getUUID();
} }

4
webmagic-samples/pom.xml
Unescape Escape View File

@ -11,12 +11,12 @@
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>${project.groupId}</groupId>
<artifactId>webmagic-core</artifactId> <artifactId>webmagic-core</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>${project.groupId}</groupId>
<artifactId>webmagic-extension</artifactId> <artifactId>webmagic-extension</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>

6
webmagic-saxon/pom.xml
Unescape Escape View File

@ -11,19 +11,17 @@
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>${project.groupId}</groupId>
<artifactId>webmagic-core</artifactId> <artifactId>webmagic-core</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>net.sourceforge.htmlcleaner</groupId> <groupId>net.sourceforge.htmlcleaner</groupId>
<artifactId>htmlcleaner</artifactId> <artifactId>htmlcleaner</artifactId>
<version>2.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>net.sf.saxon</groupId> <groupId>net.sf.saxon</groupId>
<artifactId>Saxon-HE</artifactId> <artifactId>Saxon-HE</artifactId>
<version>9.5.1-1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
@ -34,7 +32,9 @@
<build> <build>
<plugins> <plugins>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId> <artifactId>maven-deploy-plugin</artifactId>
<version>3.0.0-M1</version>
<configuration> <configuration>
<skip>true</skip> <skip>true</skip>
</configuration> </configuration>

5
webmagic-saxon/src/main/java/us/codecraft/webmagic/selector/Xpath2Selector.java
Unescape Escape View File

@ -2,11 +2,12 @@ package us.codecraft.webmagic.selector;
import net.sf.saxon.lib.NamespaceConstant; import net.sf.saxon.lib.NamespaceConstant;
import net.sf.saxon.xpath.XPathEvaluator; import net.sf.saxon.xpath.XPathEvaluator;
import org.apache.log4j.Logger;
import org.htmlcleaner.CleanerProperties; import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer; import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner; import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode; import org.htmlcleaner.TagNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document; import org.w3c.dom.Document;
import org.w3c.dom.Node; import org.w3c.dom.Node;
import org.w3c.dom.NodeList; import org.w3c.dom.NodeList;
@ -40,7 +41,7 @@ public class Xpath2Selector implements Selector {
private XPathExpression xPathExpression; private XPathExpression xPathExpression;
private Logger logger = Logger.getLogger(getClass()); private Logger logger = LoggerFactory.getLogger(getClass());
public Xpath2Selector(String xpathStr) { public Xpath2Selector(String xpathStr) {
this.xpathStr = xpathStr; this.xpathStr = xpathStr;

32
webmagic-scripts/pom.xml
Unescape Escape View File

@ -7,7 +7,6 @@
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-scripts</artifactId> <artifactId>webmagic-scripts</artifactId>
<properties> <properties>
<kotlin.version>1.1.2-2</kotlin.version> <kotlin.version>1.1.2-2</kotlin.version>
@ -17,27 +16,23 @@
<dependency> <dependency>
<groupId>org.jruby</groupId> <groupId>org.jruby</groupId>
<artifactId>jruby</artifactId> <artifactId>jruby</artifactId>
<version>1.7.6</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.jetbrains.kotlin</groupId> <groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId> <artifactId>kotlin-stdlib</artifactId>
<version>${kotlin.version}</version> <version>${kotlin.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.codehaus.groovy</groupId> <groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId> <artifactId>groovy-all</artifactId>
<version>2.1.6</version>
</dependency> </dependency>
<dependency><groupId>org.python</groupId> <dependency>
<groupId>org.python</groupId>
<artifactId>jython</artifactId> <artifactId>jython</artifactId>
<version>2.5.3</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>commons-cli</groupId> <groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId> <artifactId>commons-cli</artifactId>
<version>1.2</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
@ -45,12 +40,16 @@
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>${project.groupId}</groupId>
<artifactId>webmagic-core</artifactId> <artifactId>webmagic-core</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>webmagic-extension</artifactId> <artifactId>webmagic-extension</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
@ -59,21 +58,6 @@
<build> <build>
<sourceDirectory>${project.basedir}/src/main/java</sourceDirectory> <sourceDirectory>${project.basedir}/src/main/java</sourceDirectory>
<plugins> <plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.6</source>
<target>1.6</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<configuration>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId> <artifactId>maven-jar-plugin</artifactId>

9
webmagic-selenium/pom.xml
Unescape Escape View File

@ -13,21 +13,16 @@
<dependency> <dependency>
<groupId>org.seleniumhq.selenium</groupId> <groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId> <artifactId>selenium-java</artifactId>
<version>2.41.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>${project.groupId}</groupId>
<artifactId>webmagic-core</artifactId> <artifactId>webmagic-core</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.github.detro</groupId> <groupId>com.github.detro</groupId>
<artifactId>phantomjsdriver</artifactId> <artifactId>phantomjsdriver</artifactId>
<version>1.2.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
@ -37,7 +32,9 @@
<build> <build>
<plugins> <plugins>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId> <artifactId>maven-deploy-plugin</artifactId>
<version>3.0.0-M1</version>
<configuration> <configuration>
<skip>true</skip> <skip>true</skip>
</configuration> </configuration>

6
webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java
Unescape Escape View File

@ -1,10 +1,12 @@
package us.codecraft.webmagic.downloader.selenium; package us.codecraft.webmagic.downloader.selenium;
import org.apache.log4j.Logger;
import org.openqa.selenium.By; import org.openqa.selenium.By;
import org.openqa.selenium.Cookie; import org.openqa.selenium.Cookie;
import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement; import org.openqa.selenium.WebElement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
@ -29,7 +31,7 @@ public class SeleniumDownloader implements Downloader, Closeable {
private volatile WebDriverPool webDriverPool; private volatile WebDriverPool webDriverPool;
private Logger logger = Logger.getLogger(getClass()); private Logger logger = LoggerFactory.getLogger(getClass());
private int sleepTime = 0; private int sleepTime = 0;

5
webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java
Unescape Escape View File

@ -1,6 +1,5 @@
package us.codecraft.webmagic.downloader.selenium; package us.codecraft.webmagic.downloader.selenium;
import org.apache.log4j.Logger;
import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver; import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.firefox.FirefoxDriver; import org.openqa.selenium.firefox.FirefoxDriver;
@ -8,6 +7,8 @@ import org.openqa.selenium.phantomjs.PhantomJSDriver;
import org.openqa.selenium.phantomjs.PhantomJSDriverService; import org.openqa.selenium.phantomjs.PhantomJSDriverService;
import org.openqa.selenium.remote.DesiredCapabilities; import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.remote.RemoteWebDriver; import org.openqa.selenium.remote.RemoteWebDriver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileReader; import java.io.FileReader;
import java.io.IOException; import java.io.IOException;
@ -27,7 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* Time: 下午1:41 <br> * Time: 下午1:41 <br>
*/ */
class WebDriverPool { class WebDriverPool {
private Logger logger = Logger.getLogger(getClass()); private Logger logger = LoggerFactory.getLogger(getClass());
private final static int DEFAULT_CAPACITY = 5; private final static int DEFAULT_CAPACITY = 5;

Write Preview
Loading…
Cancel
Save