github-forks
/
webmagic
mirror of https://github.com/code4craft/webmagic.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

update spring4 configuration

Browse Source
pull/51/head
yihua.huang 11 years ago
parent 7fd27d9b22
commit 12a6390cbd
8 changed files with 35 additions and 134 deletions
Show Stats Download Patch File Download Diff File

5
pom.xml
Unescape Escape View File

@ -83,6 +83,11 @@
<artifactId>Saxon-HE</artifactId> <artifactId>Saxon-HE</artifactId>
<version>9.5.1-1</version> <version>9.5.1-1</version>
</dependency> </dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.1.37</version>
</dependency>
<dependency> <dependency>
<groupId>log4j</groupId> <groupId>log4j</groupId>
<artifactId>log4j</artifactId> <artifactId>log4j</artifactId>

1
webmagic-extension/pom.xml
Unescape Escape View File

@ -13,7 +13,6 @@
<dependency> <dependency>
<groupId>com.alibaba</groupId> <groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId> <artifactId>fastjson</artifactId>
<version>1.1.35</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>redis.clients</groupId> <groupId>redis.clients</groupId>

23
webmagic-worker/pom.xml
Unescape Escape View File

@ -21,11 +21,6 @@
<version>0.4.3-SNAPSHOT</version> <version>0.4.3-SNAPSHOT</version>
</dependency> </dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-lgpl</artifactId>
<version>1.9.8</version>
</dependency>
<dependency> <dependency>
<groupId>org.mybatis</groupId> <groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId> <artifactId>mybatis</artifactId>
@ -38,13 +33,6 @@
<version>1.1.1</version> <version>1.1.1</version>
</dependency> </dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-lgpl</artifactId>
<version>1.9.7</version>
</dependency>
<dependency> <dependency>
<groupId>org.freemarker</groupId> <groupId>org.freemarker</groupId>
<artifactId>freemarker</artifactId> <artifactId>freemarker</artifactId>
@ -73,11 +61,6 @@
<artifactId>aspectjweaver</artifactId> <artifactId>aspectjweaver</artifactId>
<version>1.7.2</version> <version>1.7.2</version>
</dependency> </dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>express.java</artifactId>
<version>0.1.0</version>
</dependency>
<dependency> <dependency>
<groupId>org.springframework</groupId> <groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId> <artifactId>spring-core</artifactId>
@ -104,6 +87,12 @@
<artifactId>spring-context-support</artifactId> <artifactId>spring-context-support</artifactId>
<version>${spring-version}</version> <version>${spring-version}</version>
</dependency> </dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.1.37</version>
</dependency>
</dependencies> </dependencies>

19
webmagic-worker/src/main/java/us/codecraft/webmagic/worker/Bootstrap.java
Unescape Escape View File

@ -1,19 +0,0 @@
package us.codecraft.webmagic.worker;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import us.codecraft.express.WebServer;
/**
* @author code4crafter@gmail.com
*/
public class Bootstrap {
public static void main(String[] args) throws Exception {
ClassPathXmlApplicationContext classPathXmlApplicationContext = new ClassPathXmlApplicationContext(
new String[]{"classpath*:/spring/applicationContext-*.xml"}
);
WebServer webServer = classPathXmlApplicationContext.getBean(WebServer.class);
webServer.port(11111).start();
}
}

64
webmagic-worker/src/main/java/us/codecraft/webmagic/worker/SpiderManager.java
Unescape Escape View File

@ -1,64 +0,0 @@
package us.codecraft.webmagic.worker;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.express.WebServer;
import us.codecraft.express.controller.AjaxController;
import us.codecraft.express.controller.ParamMap;
import us.codecraft.express.controller.ResultMap;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scripts.Language;
import us.codecraft.webmagic.scripts.ScriptProcessor;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author code4crafter@gmail.com
*/
@Component
public class SpiderManager implements InitializingBean {
@Autowired
private WebServer webServer;
private Map<String, Spider> spiderMap = new ConcurrentHashMap<String, Spider>();
public Spider newSpider(ParamMap params) {
Spider spider = Spider
.create(new ScriptProcessor(Language.JavaScript, params.get("script"), params.getInt("thread")))
.thread(params.getInt("thread")).addUrl(params.get("url"));
spider.start();
return spider;
}
@Override
public void afterPropertiesSet() throws Exception {
AjaxController newController = new AjaxController() {
@Override
public Object ajax(ParamMap params) {
try {
Spider spider = newSpider(params);
spiderMap.put(params.get("uuid"), spider);
return ResultMap.create().put("code", 200).put("msg", "success");
} catch (Exception e) {
// If you provide worker to user, DO NOT return
// e.getMessage()!
return ResultMap.create().put("code", 500).put("msg", e.getMessage());
}
}
};
webServer.post("/new/${uuid}", newController);
webServer.get("/new/${uuid}", newController);
webServer.get("/status/${uuid}", new AjaxController() {
@Override
public Object ajax(ParamMap params) {
Spider spider = spiderMap.get(params.get("uuid"));
ResultMap put = ResultMap.create().put("pageCount", spider.getPageCount())
.put("status", spider.getStatus().name()).put("thread", spider.getThreadAlive());
return put;
}
});
}
}

9
webmagic-worker/src/main/java/us/codecraft/webmagic/worker/web/SpiderController.java
Unescape Escape View File

@ -5,6 +5,9 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.ModelAndView;
import java.util.HashMap;
import java.util.Map;
/** /**
* @author code4crafter@gmail.com * @author code4crafter@gmail.com
*/ */
@ -14,7 +17,9 @@ public class SpiderController {
@RequestMapping("create") @RequestMapping("create")
@ResponseBody @ResponseBody
public ModelAndView create(){ public Map<String, Object> create() {
return null; HashMap<String, Object> map = new HashMap<String, Object>();
map.put("code", 200);
return map;
} }
} }

18
webmagic-worker/src/main/resources/spring/applicationContext-webmagic-worker.xml
Unescape Escape View File

@ -1,18 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util"
xmlns:mvc="http://www.springframework.org/schema/mvc"
xsi:schemaLocation="http://www.springframework.org/schema/mvc
http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
<beans>
<bean name="server" class="us.codecraft.express.connector.jetty.JettyWebServer">
</bean>
</beans>
</beans>

30
webmagic-worker/src/main/resources/spring/applicationContext.xml
Unescape Escape View File

@ -4,11 +4,11 @@
xmlns:context="http://www.springframework.org/schema/context" xmlns:context="http://www.springframework.org/schema/context"
xmlns:mvc="http://www.springframework.org/schema/mvc" xmlns:mvc="http://www.springframework.org/schema/mvc"
xsi:schemaLocation="http://www.springframework.org/schema/mvc xsi:schemaLocation="http://www.springframework.org/schema/mvc
http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd http://www.springframework.org/schema/mvc/spring-mvc-4.0.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-3.0.xsd"> http://www.springframework.org/schema/context/spring-context-4.0.xsd">
<context:annotation-config/> <context:annotation-config/>
<bean id="messageSource" class="org.springframework.context.support.ResourceBundleMessageSource"> <bean id="messageSource" class="org.springframework.context.support.ResourceBundleMessageSource">
@ -19,25 +19,29 @@
</property> </property>
</bean> </bean>
<context:component-scan base-package="us.codecraft.webmagic.worker"/> <context:component-scan base-package="us.codecraft.webmagic.worker"/>
<bean
class="org.springframework.web.servlet.mvc.annotation.AnnotationMethodHandlerAdapter"> <bean class="org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter">
<property name="messageConverters"> <property name="messageConverters">
<list> <list>
<ref bean="jsonHttpMessageConverter"/> <bean id="fastJsonHttpMessageConverter" class="com.alibaba.fastjson.support.spring.FastJsonHttpMessageConverter">
<property name="supportedMediaTypes">
<list>
<value>text/html;charset=UTF-8</value>
</list>
</property>
</bean>
</list> </list>
</property> </property>
</bean> </bean>
<bean id="jsonHttpMessageConverter" class="org.springframework.http.converter.json.MappingJacksonHttpMessageConverter"/>
<mvc:resources mapping="/js/**" location="/js/" /> <mvc:resources mapping="/static/**" location="/static/" />
<mvc:resources mapping="/css/**" location="/css/" />
<mvc:resources mapping="/img/**" location="/img/" /> <mvc:annotation-driven>
<mvc:resources mapping="/font/**" location="/font/" />
<mvc:annotation-driven /> </mvc:annotation-driven>
</beans> </beans>
Write Preview
Loading…
Cancel
Save