diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/proxy/Proxy.java b/webmagic-core/src/main/java/us/codecraft/webmagic/proxy/Proxy.java index a38ccaa7..c5f10073 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/proxy/Proxy.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/proxy/Proxy.java @@ -38,4 +38,36 @@ public class Proxy { public String getPassword() { return password; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Proxy proxy = (Proxy) o; + + if (port != proxy.port) return false; + if (host != null ? !host.equals(proxy.host) : proxy.host != null) return false; + if (username != null ? !username.equals(proxy.username) : proxy.username != null) return false; + return password != null ? password.equals(proxy.password) : proxy.password == null; + } + + @Override + public int hashCode() { + int result = host != null ? host.hashCode() : 0; + result = 31 * result + port; + result = 31 * result + (username != null ? username.hashCode() : 0); + result = 31 * result + (password != null ? password.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "Proxy{" + + "host='" + host + '\'' + + ", port=" + port + + ", username='" + username + '\'' + + ", password='" + password + '\'' + + '}'; + } } diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/proxy/SimpleProxyProvider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/proxy/SimpleProxyProvider.java new file mode 100644 index 00000000..7fbc9485 --- /dev/null +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/proxy/SimpleProxyProvider.java @@ -0,0 +1,58 @@ +package us.codecraft.webmagic.proxy; + +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Task; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A simple ProxyProvider. Provide proxy as round-robin without heartbeat and error check. It can be used when all proxies are stable. + * @author code4crafter@gmail.com + * Date: 17/4/16 + * Time: 10:18 + * @since 0.7.0 + */ +public class SimpleProxyProvider implements ProxyProvider { + + private final List proxies; + + private final AtomicInteger pointer; + + private SimpleProxyProvider(List proxies, AtomicInteger pointer) { + this.proxies = proxies; + this.pointer = pointer; + } + + public static SimpleProxyProvider from(Proxy... proxies) { + List proxiesTemp = new ArrayList(proxies.length); + for (Proxy proxy : proxies) { + proxiesTemp.add(proxy); + } + return new SimpleProxyProvider(Collections.unmodifiableList(proxiesTemp), new AtomicInteger(-1)); + } + + @Override + public void returnProxy(Proxy proxy, Page page, Task task) { + //Donothing + } + + @Override + public Proxy getProxy(Task task) { + return proxies.get(incrForLoop()); + } + + private int incrForLoop() { + int p = pointer.incrementAndGet(); + int size = proxies.size(); + if (p < size) { + return p; + } + while (!pointer.compareAndSet(p, p % size)) { + p = pointer.get(); + } + return p % size; + } +} diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/proxy/SimpleProxyProviderTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/proxy/SimpleProxyProviderTest.java new file mode 100644 index 00000000..6495b16b --- /dev/null +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/proxy/SimpleProxyProviderTest.java @@ -0,0 +1,30 @@ +package us.codecraft.webmagic.proxy; + +import org.junit.Test; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Task; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author code4crafter@gmail.com + * Date: 17/4/16 + * Time: 上午10:29 + */ +public class SimpleProxyProviderTest { + + public static final Task TASK = Site.me().toTask(); + + @Test + public void test_get_proxy() throws Exception { + Proxy originProxy1 = new Proxy("127.0.0.1", 1087); + Proxy originProxy2 = new Proxy("127.0.0.1", 1088); + SimpleProxyProvider proxyProvider = SimpleProxyProvider.from(originProxy1, originProxy2); + Proxy proxy = proxyProvider.getProxy(TASK); + assertThat(proxy).isEqualTo(originProxy1); + proxy = proxyProvider.getProxy(TASK); + assertThat(proxy).isEqualTo(originProxy2); + proxy = proxyProvider.getProxy(TASK); + assertThat(proxy).isEqualTo(originProxy1); + } +}