change check-and-set to atomic sadd for redis DuplicateRemover #368

webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java

@@ -48,11 +48,7 @@ public class RedisScheduler extends DuplicateRemovedScheduler implements Monitor
     public boolean isDuplicate(Request request, Task task) {
         Jedis jedis = pool.getResource();
         try {
-            boolean isDuplicate = jedis.sismember(getSetKey(task), request.getUrl());
+            return jedis.sadd(getSetKey(task), request.getUrl()) > 0;
-            if (!isDuplicate) {
-                jedis.sadd(getSetKey(task), request.getUrl());
-            }
-            return isDuplicate;
         } finally {
             pool.returnResource(jedis);
         }