update urls.contains to DuplicateRemover in FileCacheQueueScheduler #157

pull/158/head
yihua.huang 11 years ago
parent 689e89a9b2
commit 42a30074c9

@ -4,6 +4,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.math.NumberUtils;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.scheduler.component.DuplicateRemover;
import java.io.*;
import java.util.LinkedHashSet;
@ -68,6 +69,26 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
logger.info("init cache scheduler success");
}
private void initDuplicateRemover() {
setDuplicateRemover(
new DuplicateRemover() {
@Override
public boolean isDuplicate(Request request, Task task) {
return !urls.add(request.getUrl());
}
@Override
public void resetDuplicateCheck(Task task) {
urls.clear();
}
@Override
public int getTotalRequestsCount(Task task) {
return urls.size();
}
});
}
private void initFlushThread() {
Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {
@Override
@ -92,6 +113,7 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
urls = new LinkedHashSet<String>();
readCursorFile();
readUrlFile();
initDuplicateRemover();
} catch (FileNotFoundException e) {
//init
logger.info("init cache file " + getFileName(fileUrlAllName));
@ -145,8 +167,6 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
if (!inited.get()) {
init(task);
}
if(urls.contains(request.getUrl())) //已存在此URL 表示已抓取过 跳过
return;
queue.add(request);
fileUrlWriter.println(request.getUrl());
}

Loading…
Cancel
Save