Merge branch 'master' of github.com:code4craft/webmagic

11 years ago · f9b157951d
parent 22c394e629 3efa774191
commit f9b157951d
1 changed files with 2 additions and 1 deletions
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/LocalDuplicatedRemovedScheduler.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/LocalDuplicatedRemovedScheduler.java
@ -23,8 +23,9 @@ public abstract class LocalDuplicatedRemovedScheduler implements Scheduler {

    @Override
    public void push(Request request, Task task) {
-        logger.debug("push to queue " + request.getUrl());
+        logger.trace("get a candidate url " + request.getUrl());
        if (request.getExtra(Request.CYCLE_TRIED_TIMES) != null || urls.add(request.getUrl())) {
+        	logger.debug("push to queue " + request.getUrl());
            pushWhenNoDuplicate(request, task);
        }
    }