Fix Spider main thread any problem with redis connection (when working with redis-based schedulers) leads to unhandled exception and release the main (control) thread

webmagic-core/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.7.3</version>
+        <version>0.7.5</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -305,7 +305,13 @@ public class Spider implements Runnable, Task {
         initComponent();
         logger.info("Spider {} started!",getUUID());
         while (!Thread.currentThread().isInterrupted() && stat.get() == STAT_RUNNING) {
-            final Request request = scheduler.poll(this);
+            final Request request;
+            try {
+                request = scheduler.poll(this);
+            } catch (Exception e) {
+                logger.warn(e.getMessage());
+                continue;
+            }
             if (request == null) {
                 if (threadPool.getThreadAlive() == 0 && exitWhenComplete) {
                     break;