Merge branch 'master' into master
apaqi
5 years ago
committed by
GitHub
commit
c4c48d3522
@ -1,3 +1,3 @@
|
||||
language: java
|
||||
jdk:
|
||||
- oraclejdk7
|
||||
- openjdk9
|
||||
|
||||
@ -1,73 +1,135 @@
|
||||
package us.codecraft.webmagic.proxy;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
public class Proxy {
|
||||
|
||||
private String host;
|
||||
private int port;
|
||||
private String username;
|
||||
private String password;
|
||||
private String scheme;
|
||||
|
||||
private String host;
|
||||
|
||||
private int port;
|
||||
|
||||
private String username;
|
||||
|
||||
private String password;
|
||||
|
||||
public Proxy(String host, int port) {
|
||||
this.host = host;
|
||||
this.port = port;
|
||||
}
|
||||
public static Proxy create(final URI uri) {
|
||||
Proxy proxy = new Proxy(uri.getHost(), uri.getPort(), uri.getScheme());
|
||||
String userInfo = uri.getUserInfo();
|
||||
if (userInfo != null) {
|
||||
String[] up = userInfo.split(":");
|
||||
if (up.length == 1) {
|
||||
proxy.username = up[0].isEmpty() ? null : up[0];
|
||||
} else {
|
||||
proxy.username = up[0].isEmpty() ? null : up[0];
|
||||
proxy.password = up[1].isEmpty() ? null : up[1];
|
||||
}
|
||||
}
|
||||
return proxy;
|
||||
}
|
||||
|
||||
public Proxy(String host, int port, String username, String password) {
|
||||
this.host = host;
|
||||
this.port = port;
|
||||
this.username = username;
|
||||
this.password = password;
|
||||
}
|
||||
public Proxy(String host, int port) {
|
||||
this(host, port, null);
|
||||
}
|
||||
|
||||
public Proxy(String host, int port, String scheme) {
|
||||
this.host = host;
|
||||
this.port = port;
|
||||
this.scheme = scheme;
|
||||
}
|
||||
|
||||
public Proxy(String host, int port, String username, String password) {
|
||||
this.host = host;
|
||||
this.port = port;
|
||||
this.username = username;
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public String getScheme() {
|
||||
return scheme;
|
||||
}
|
||||
|
||||
public void setScheme(String scheme) {
|
||||
this.scheme = scheme;
|
||||
}
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public int getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
|
||||
Proxy proxy = (Proxy) o;
|
||||
|
||||
if (port != proxy.port) return false;
|
||||
if (host != null ? !host.equals(proxy.host) : proxy.host != null) return false;
|
||||
if (username != null ? !username.equals(proxy.username) : proxy.username != null) return false;
|
||||
return password != null ? password.equals(proxy.password) : proxy.password == null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = host != null ? host.hashCode() : 0;
|
||||
result = 31 * result + port;
|
||||
result = 31 * result + (username != null ? username.hashCode() : 0);
|
||||
result = 31 * result + (password != null ? password.hashCode() : 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Proxy{" +
|
||||
"host='" + host + '\'' +
|
||||
", port=" + port +
|
||||
", username='" + username + '\'' +
|
||||
", password='" + password + '\'' +
|
||||
'}';
|
||||
}
|
||||
return host;
|
||||
}
|
||||
|
||||
public int getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public URI toURI() {
|
||||
final StringBuilder userInfoBuffer = new StringBuilder();
|
||||
if (username != null) {
|
||||
userInfoBuffer.append(urlencode(username));
|
||||
}
|
||||
if (password != null) {
|
||||
userInfoBuffer.append(":").append(urlencode(password));
|
||||
}
|
||||
final String userInfo = StringUtils.defaultIfEmpty(userInfoBuffer.toString(), null);
|
||||
URI uri;
|
||||
try {
|
||||
uri = new URI(scheme, userInfo, host, port, null, null, null);
|
||||
} catch (URISyntaxException e) {
|
||||
throw new IllegalArgumentException(e.getMessage(), e);
|
||||
}
|
||||
return uri;
|
||||
}
|
||||
|
||||
private String urlencode(String s) {
|
||||
String enc = StandardCharsets.UTF_8.name();
|
||||
try {
|
||||
return URLEncoder.encode(s, enc);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
|
||||
Proxy proxy = (Proxy) o;
|
||||
|
||||
if (port != proxy.port) return false;
|
||||
if (host != null ? !host.equals(proxy.host) : proxy.host != null) return false;
|
||||
if (scheme != null ? !scheme.equals(proxy.scheme) : proxy.scheme != null) return false;
|
||||
if (username != null ? !username.equals(proxy.username) : proxy.username != null) return false;
|
||||
return password != null ? password.equals(proxy.password) : proxy.password == null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = host != null ? host.hashCode() : 0;
|
||||
result = 31 * result + port;
|
||||
result = 31 * result + (scheme != null ? scheme.hashCode() : 0);
|
||||
result = 31 * result + (username != null ? username.hashCode() : 0);
|
||||
result = 31 * result + (password != null ? password.hashCode() : 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return this.toURI().toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
package us.codecraft.webmagic.downloader;
|
||||
|
||||
import org.junit.Test;
|
||||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.Request;
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.Task;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com
|
||||
* Date: 2017/11/29
|
||||
* Time: 下午1:32
|
||||
*/
|
||||
public class SSLCompatibilityTest {
|
||||
|
||||
@Test
|
||||
public void test_tls12() throws Exception {
|
||||
HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
|
||||
Task task = Site.me().setCycleRetryTimes(5).toTask();
|
||||
Request request = new Request("https://juejin.im/");
|
||||
Page page = httpClientDownloader.download(request, task);
|
||||
assertThat(page.isDownloadSuccess()).isTrue();
|
||||
}
|
||||
}
|
||||
@ -1,45 +1,97 @@
|
||||
package us.codecraft.webmagic.proxy;
|
||||
|
||||
import org.apache.http.HttpHost;
|
||||
import org.junit.BeforeClass;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.http.HttpHost;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* @author yxssfxwzy@sina.com May 30, 2014
|
||||
*
|
||||
*/
|
||||
public class ProxyTest {
|
||||
|
||||
private static List<String[]> httpProxyList = new ArrayList<String[]>();
|
||||
|
||||
@BeforeClass
|
||||
public static void before() {
|
||||
// String[] source = { "0.0.0.1:0", "0.0.0.2:0", "0.0.0.3:0",
|
||||
// "0.0.0.4:0" };
|
||||
String[] source = { "::0.0.0.1:0", "::0.0.0.2:0", "::0.0.0.3:0", "::0.0.0.4:0" };
|
||||
for (String line : source) {
|
||||
httpProxyList.add(new String[] {line.split(":")[0], line.split(":")[1], line.split(":")[2], line.split(":")[3] });
|
||||
}
|
||||
}
|
||||
|
||||
class Fetch extends Thread {
|
||||
HttpHost hp;
|
||||
|
||||
public Fetch(HttpHost hp) {
|
||||
this.hp = hp;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
System.out.println("fetch web page use proxy: " + hp.getHostName() + ":" + hp.getPort());
|
||||
sleep(500);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
private static List<String[]> httpProxyList = new ArrayList<String[]>();
|
||||
|
||||
@BeforeClass
|
||||
public static void before() {
|
||||
// String[] source = { "0.0.0.1:0", "0.0.0.2:0", "0.0.0.3:0",
|
||||
// "0.0.0.4:0" };
|
||||
String[] source = { "::0.0.0.1:0", "::0.0.0.2:0", "::0.0.0.3:0", "::0.0.0.4:0" };
|
||||
for (String line : source) {
|
||||
httpProxyList.add(new String[] {line.split(":")[0], line.split(":")[1], line.split(":")[2], line.split(":")[3] });
|
||||
}
|
||||
}
|
||||
|
||||
class Fetch extends Thread {
|
||||
HttpHost hp;
|
||||
|
||||
public Fetch(HttpHost hp) {
|
||||
this.hp = hp;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
System.out.println("fetch web page use proxy: " + hp.getHostName() + ":" + hp.getPort());
|
||||
sleep(500);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreate() {
|
||||
Proxy proxy = Proxy.create(URI.create("//127.0.0.1:8080"));
|
||||
assertNull(proxy.getScheme());
|
||||
assertNull(proxy.getUsername());
|
||||
assertNull(proxy.getPassword());
|
||||
assertEquals("127.0.0.1", proxy.getHost());
|
||||
assertEquals(8080, proxy.getPort());
|
||||
|
||||
proxy = Proxy.create(URI.create("http://127.0.0.1:8080"));
|
||||
assertEquals("http", proxy.getScheme());
|
||||
assertNull(proxy.getUsername());
|
||||
assertNull(proxy.getPassword());
|
||||
assertEquals("127.0.0.1", proxy.getHost());
|
||||
assertEquals(8080, proxy.getPort());
|
||||
|
||||
proxy = Proxy.create(URI.create("//username:password@127.0.0.1:8080"));
|
||||
assertNull(proxy.getScheme());
|
||||
assertEquals("username", proxy.getUsername());
|
||||
assertEquals("password", proxy.getPassword());
|
||||
assertEquals("127.0.0.1", proxy.getHost());
|
||||
assertEquals(8080, proxy.getPort());
|
||||
|
||||
proxy = Proxy.create(URI.create("//username@127.0.0.1:8080"));
|
||||
assertNull(proxy.getScheme());
|
||||
assertEquals("username", proxy.getUsername());
|
||||
assertNull(proxy.getPassword());
|
||||
assertEquals("127.0.0.1", proxy.getHost());
|
||||
assertEquals(8080, proxy.getPort());
|
||||
|
||||
proxy = Proxy.create(URI.create("//:password@127.0.0.1:8080"));
|
||||
assertNull(proxy.getScheme());
|
||||
assertNull(proxy.getUsername());
|
||||
assertEquals("password", proxy.getPassword());
|
||||
assertEquals("127.0.0.1", proxy.getHost());
|
||||
assertEquals(8080, proxy.getPort());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testToString() {
|
||||
assertEquals("//127.0.0.1:8080", new Proxy("127.0.0.1", 8080).toString());
|
||||
assertEquals("http://127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "http").toString());
|
||||
assertEquals("//username:password@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "username", "password").toString());
|
||||
assertEquals("//username@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "username", null).toString());
|
||||
assertEquals("//:password@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, null, "password").toString());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,16 @@
|
||||
package us.codecraft.webmagic.utils;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class NumberUtilsTest {
|
||||
|
||||
@Test
|
||||
public void testCompareLong() {
|
||||
Assert.assertEquals(0, NumberUtils.compareLong(0L, 0L));
|
||||
Assert.assertEquals(1, NumberUtils.compareLong(9L, 0L));
|
||||
Assert.assertEquals(-1, NumberUtils.compareLong(0L, 9L));
|
||||
Assert.assertEquals(-1, NumberUtils.compareLong(-9L, 0L));
|
||||
Assert.assertEquals(1, NumberUtils.compareLong(0L, -9L));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue