From 08fa3b01c14225a52c78f806055c4a6abf37ce85 Mon Sep 17 00:00:00 2001
From: Bo LIANG <ccliangbo@gmail.com>
Date: Thu, 17 Apr 2014 17:53:12 +0800
Subject: [PATCH] when download error, throw an exception instead of calling
 onError and returning peacefully. #105

---
 .../src/main/java/us/codecraft/webmagic/Spider.java        | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
index c53afcbe..a03dee17 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
@@ -1,9 +1,11 @@
 package us.codecraft.webmagic;
 
 import com.google.common.collect.Lists;
+
 import org.apache.commons.collections.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import us.codecraft.webmagic.downloader.Downloader;
 import us.codecraft.webmagic.downloader.HttpClientDownloader;
 import us.codecraft.webmagic.monitor.SpiderListener;
@@ -318,7 +320,7 @@ public class Spider implements Runnable, Task {
                             onSuccess(requestFinal);
                         } catch (Exception e) {
                             onError(requestFinal);
-                            logger.error("download " + requestFinal + " error", e);
+                            logger.error("process request " + requestFinal + " error", e);
                         } finally {
                             threadAlive.decrementAndGet();
                             pageCount.incrementAndGet();
@@ -399,9 +401,8 @@ public class Spider implements Runnable, Task {
     protected void processRequest(Request request) {
         Page page = downloader.download(request, this);
         if (page == null) {
-            onError(request);
             sleep(site.getSleepTime());
-            return;
+            throw new IllegalStateException("download error");
         }
         // for cycle retry
         if (page.isNeedCycleRetry()) {