From 12a6390cbdea540acb88fabdb6b39fa20bc0d010 Mon Sep 17 00:00:00 2001 From: "yihua.huang" Date: Wed, 18 Dec 2013 01:02:59 +0800 Subject: [PATCH] update spring4 configuration --- pom.xml | 5 ++ webmagic-extension/pom.xml | 1 - webmagic-worker/pom.xml | 23 ++----- .../codecraft/webmagic/worker/Bootstrap.java | 19 ------ .../webmagic/worker/SpiderManager.java | 64 ------------------- .../webmagic/worker/web/SpiderController.java | 9 ++- .../applicationContext-webmagic-worker.xml | 18 ------ .../resources/spring/applicationContext.xml | 30 +++++---- 8 files changed, 35 insertions(+), 134 deletions(-) delete mode 100644 webmagic-worker/src/main/java/us/codecraft/webmagic/worker/Bootstrap.java delete mode 100644 webmagic-worker/src/main/java/us/codecraft/webmagic/worker/SpiderManager.java delete mode 100755 webmagic-worker/src/main/resources/spring/applicationContext-webmagic-worker.xml diff --git a/pom.xml b/pom.xml index 0ee156a3..3fcbd449 100644 --- a/pom.xml +++ b/pom.xml @@ -83,6 +83,11 @@ Saxon-HE 9.5.1-1 + + com.alibaba + fastjson + 1.1.37 + log4j log4j diff --git a/webmagic-extension/pom.xml b/webmagic-extension/pom.xml index c6af14f6..c11f0f1f 100644 --- a/webmagic-extension/pom.xml +++ b/webmagic-extension/pom.xml @@ -13,7 +13,6 @@ com.alibaba fastjson - 1.1.35 redis.clients diff --git a/webmagic-worker/pom.xml b/webmagic-worker/pom.xml index 58386f1d..e9a2146b 100644 --- a/webmagic-worker/pom.xml +++ b/webmagic-worker/pom.xml @@ -21,11 +21,6 @@ 0.4.3-SNAPSHOT - - org.codehaus.jackson - jackson-core-lgpl - 1.9.8 - org.mybatis mybatis @@ -38,13 +33,6 @@ 1.1.1 - - - org.codehaus.jackson - jackson-mapper-lgpl - 1.9.7 - - org.freemarker freemarker @@ -73,11 +61,6 @@ aspectjweaver 1.7.2 - - us.codecraft - express.java - 0.1.0 - org.springframework spring-core @@ -104,6 +87,12 @@ spring-context-support ${spring-version} + + com.alibaba + fastjson + 1.1.37 + + diff --git a/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/Bootstrap.java b/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/Bootstrap.java deleted file mode 100644 index 155588e6..00000000 --- a/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/Bootstrap.java +++ /dev/null @@ -1,19 +0,0 @@ -package us.codecraft.webmagic.worker; - -import org.springframework.context.support.ClassPathXmlApplicationContext; -import us.codecraft.express.WebServer; - -/** - * @author code4crafter@gmail.com - */ -public class Bootstrap { - - public static void main(String[] args) throws Exception { - ClassPathXmlApplicationContext classPathXmlApplicationContext = new ClassPathXmlApplicationContext( - new String[]{"classpath*:/spring/applicationContext-*.xml"} - ); - WebServer webServer = classPathXmlApplicationContext.getBean(WebServer.class); - webServer.port(11111).start(); - } - -} diff --git a/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/SpiderManager.java b/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/SpiderManager.java deleted file mode 100644 index 067c0252..00000000 --- a/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/SpiderManager.java +++ /dev/null @@ -1,64 +0,0 @@ -package us.codecraft.webmagic.worker; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import us.codecraft.express.WebServer; -import us.codecraft.express.controller.AjaxController; -import us.codecraft.express.controller.ParamMap; -import us.codecraft.express.controller.ResultMap; -import us.codecraft.webmagic.Spider; -import us.codecraft.webmagic.scripts.Language; -import us.codecraft.webmagic.scripts.ScriptProcessor; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * @author code4crafter@gmail.com - */ -@Component -public class SpiderManager implements InitializingBean { - - @Autowired - private WebServer webServer; - - private Map spiderMap = new ConcurrentHashMap(); - - public Spider newSpider(ParamMap params) { - Spider spider = Spider - .create(new ScriptProcessor(Language.JavaScript, params.get("script"), params.getInt("thread"))) - .thread(params.getInt("thread")).addUrl(params.get("url")); - spider.start(); - return spider; - } - - @Override - public void afterPropertiesSet() throws Exception { - AjaxController newController = new AjaxController() { - @Override - public Object ajax(ParamMap params) { - try { - Spider spider = newSpider(params); - spiderMap.put(params.get("uuid"), spider); - return ResultMap.create().put("code", 200).put("msg", "success"); - } catch (Exception e) { - // If you provide worker to user, DO NOT return - // e.getMessage()! - return ResultMap.create().put("code", 500).put("msg", e.getMessage()); - } - } - }; - webServer.post("/new/${uuid}", newController); - webServer.get("/new/${uuid}", newController); - webServer.get("/status/${uuid}", new AjaxController() { - @Override - public Object ajax(ParamMap params) { - Spider spider = spiderMap.get(params.get("uuid")); - ResultMap put = ResultMap.create().put("pageCount", spider.getPageCount()) - .put("status", spider.getStatus().name()).put("thread", spider.getThreadAlive()); - return put; - } - }); - } -} diff --git a/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/web/SpiderController.java b/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/web/SpiderController.java index 905064ec..01fb789e 100644 --- a/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/web/SpiderController.java +++ b/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/web/SpiderController.java @@ -5,6 +5,9 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.servlet.ModelAndView; +import java.util.HashMap; +import java.util.Map; + /** * @author code4crafter@gmail.com */ @@ -14,7 +17,9 @@ public class SpiderController { @RequestMapping("create") @ResponseBody - public ModelAndView create(){ - return null; + public Map create() { + HashMap map = new HashMap(); + map.put("code", 200); + return map; } } diff --git a/webmagic-worker/src/main/resources/spring/applicationContext-webmagic-worker.xml b/webmagic-worker/src/main/resources/spring/applicationContext-webmagic-worker.xml deleted file mode 100755 index adc334f9..00000000 --- a/webmagic-worker/src/main/resources/spring/applicationContext-webmagic-worker.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - diff --git a/webmagic-worker/src/main/resources/spring/applicationContext.xml b/webmagic-worker/src/main/resources/spring/applicationContext.xml index 85f9974e..c988f9c1 100644 --- a/webmagic-worker/src/main/resources/spring/applicationContext.xml +++ b/webmagic-worker/src/main/resources/spring/applicationContext.xml @@ -4,11 +4,11 @@ xmlns:context="http://www.springframework.org/schema/context" xmlns:mvc="http://www.springframework.org/schema/mvc" xsi:schemaLocation="http://www.springframework.org/schema/mvc - http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd + http://www.springframework.org/schema/mvc/spring-mvc-4.0.xsd http://www.springframework.org/schema/beans - http://www.springframework.org/schema/beans/spring-beans-3.0.xsd + http://www.springframework.org/schema/beans/spring-beans-4.0.xsd http://www.springframework.org/schema/context - http://www.springframework.org/schema/context/spring-context-3.0.xsd"> + http://www.springframework.org/schema/context/spring-context-4.0.xsd"> @@ -19,25 +19,29 @@ - - + + - + + + + text/html;charset=UTF-8 + + + + - - - - - + + + - + \ No newline at end of file