fix getProxy to make poll and offer atomic

webmagic-core/src/main/java/us/codecraft/webmagic/proxy/EffectiveProxyProvider.java

@@ -27,6 +27,8 @@ public abstract class EffectiveProxyProvider implements ProxyProvider {
 
     private final ReentrantLock addProxyLock = new ReentrantLock();
 
+    private final ReentrantLock pollAndOfferLock = new ReentrantLock();
+
     private int expandPoolSize = DEFAULT_EXPAND_POOL_SIZE;
 
     private ProxyPageValidator proxyPageValidator;
@@ -59,32 +61,43 @@ public abstract class EffectiveProxyProvider implements ProxyProvider {
     @Override
     public Proxy getProxy(Task task) {
 
-        //async addProxy and avoid invoke extra times
+        //make atomic poll and offer
-        if (validProxyQueue.size() <= expandPoolSize) {
+        pollAndOfferLock.lock();
-            addProxyPool.submit(new Runnable() {
+        try {
-                @Override public void run() {
+            Proxy proxy = validProxyQueue.poll();
-                    if (addProxyLock.tryLock()) {
+            if (proxy != null) {
-                        try {
+                //put tail realize loop
-                            List<Proxy> newProxies = addProxies();
+                validProxyQueue.offer(proxy);
-                            if (CollectionUtils.isNotEmpty(newProxies)) {
+            }
-                                validProxyQueue.addAll(newProxies);
+
-                            }
+            //get more proxies when queue capacity less than expect
-                        }finally {
+            if (validProxyQueue.size() <= expandPoolSize) {
-                            addProxyLock.unlock();
+                expand();
-                        }
+            }
-                    }
+            return proxy;
-                }
+
-            });
+        }finally {
+            pollAndOfferLock.unlock();
         }
+    }
 
-        Proxy proxy = validProxyQueue.poll();
+    //async addProxy and avoid invoke extra times
-        if (proxy == null) {
+    public void expand(){
-            return null;
-        }
-        //put tail realize loop
-        validProxyQueue.offer(proxy);
 
-        return proxy;
+        if (addProxyLock.tryLock()) {
+            try {
+                addProxyPool.submit(new Runnable() {
+                    @Override public void run() {
+                        List<Proxy> newProxies = addProxies();
+                        if (CollectionUtils.isNotEmpty(newProxies)) {
+                            validProxyQueue.addAll(newProxies);
+                        }
+                    }
+                });
+            } finally {
+                addProxyLock.unlock();
+            }
+        }
     }
 
     @Override public void returnProxy(Proxy proxy, Page page, Task task) {