change check-and-set to atomic sadd for redis DuplicateRemover #368

8 years ago · d87c73b472
parent d6cd92b1a8
commit d87c73b472
1 changed files with 1 additions and 5 deletions
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java
@ -48,11 +48,7 @@ public class RedisScheduler extends DuplicateRemovedScheduler implements Monitor
    public boolean isDuplicate(Request request, Task task) {
        Jedis jedis = pool.getResource();
        try {
-            boolean isDuplicate = jedis.sismember(getSetKey(task), request.getUrl());
-            if (!isDuplicate) {
-                jedis.sadd(getSetKey(task), request.getUrl());
-            }
-            return isDuplicate;
+            return jedis.sadd(getSetKey(task), request.getUrl()) > 0;
        } finally {
            pool.returnResource(jedis);
        }