github-forks
/
webmagic
mirror of https://github.com/code4craft/webmagic.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add spring 4 mvc for worker

Browse Source
pull/51/head
yihua.huang 11 years ago
parent 31fb0048a1
commit 7fd27d9b22
12 changed files with 333 additions and 47 deletions
Show Stats Download Patch File Download Diff File

2
pom.xml
Unescape Escape View File

@ -12,7 +12,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-version>3.1.1.RELEASE</spring-version>
<spring-version>4.0.0.RELEASE</spring-version>
</properties>
<artifactId>webmagic-parent</artifactId>

67
webmagic-worker/pom.xml
Unescape Escape View File

@ -12,6 +12,7 @@
<groupId>us.codecraft</groupId>
<artifactId>webmagic-worker</artifactId>
<version>0.4.3-SNAPSHOT</version>
<packaging>war</packaging>
<dependencies>
<dependency>
@ -19,6 +20,59 @@
<artifactId>webmagic-scripts</artifactId>
<version>0.4.3-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-lgpl</artifactId>
<version>1.9.8</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
<version>3.1.1</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-lgpl</artifactId>
<version>1.9.7</version>
</dependency>
<dependency>
<groupId>org.freemarker</groupId>
<artifactId>freemarker</artifactId>
<version>2.3.19</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>${spring-version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-aop</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjrt</artifactId>
<version>1.7.2</version>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
<version>1.7.2</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>express.java</artifactId>
@ -31,14 +85,25 @@
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-asm</artifactId>
<artifactId>spring-webmvc</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>
<version>${spring-version}</version>
</dependency>
</dependencies>

68
webmagic-worker/src/main/java/us/codecraft/webmagic/worker/SpiderManager.java
Unescape Escape View File

@ -20,45 +20,45 @@ import java.util.concurrent.ConcurrentHashMap;
@Component
public class SpiderManager implements InitializingBean {
@Autowired
private WebServer webServer;
@Autowired
private WebServer webServer;
private Map<String, Spider> spiderMap = new ConcurrentHashMap<String, Spider>();
private Map<String, Spider> spiderMap = new ConcurrentHashMap<String, Spider>();
public Spider newSpider(ParamMap params) {
Spider spider = Spider
.create(new ScriptProcessor(Language.JavaScript, params.get("script"), params.getInt("thread")))
.thread(params.getInt("thread")).addUrl(params.get("url"));
spider.start();
return spider;
}
public Spider newSpider(ParamMap params) {
Spider spider = Spider
.create(new ScriptProcessor(Language.JavaScript, params.get("script"), params.getInt("thread")))
.thread(params.getInt("thread")).addUrl(params.get("url"));
spider.start();
return spider;
}
@Override
public void afterPropertiesSet() throws Exception {
AjaxController newController = new AjaxController() {
@Override
public Object ajax(ParamMap params) {
try {
Spider spider = newSpider(params);
spiderMap.put(params.get("uuid"), spider);
return ResultMap.create().put("code", 200).put("msg", "success");
} catch (Exception e) {
// If you provide worker to user, DO NOT return
// e.getMessage()!
return ResultMap.create().put("code", 500).put("msg", e.getMessage());
}
}
};
webServer.post("/new/${uuid}", newController);
webServer.get("/new/${uuid}", newController);
webServer.get("/status/${uuid}", new AjaxController() {
@Override
public Object ajax(ParamMap params) {
Spider spider = spiderMap.get(params.get("uuid"));
@Override
public void afterPropertiesSet() throws Exception {
AjaxController newController = new AjaxController() {
@Override
public Object ajax(ParamMap params) {
try {
Spider spider = newSpider(params);
spiderMap.put(params.get("uuid"), spider);
return ResultMap.create().put("code", 200).put("msg", "success");
} catch (Exception e) {
// If you provide worker to user, DO NOT return
// e.getMessage()!
return ResultMap.create().put("code", 500).put("msg", e.getMessage());
}
}
};
webServer.post("/new/${uuid}", newController);
webServer.get("/new/${uuid}", newController);
webServer.get("/status/${uuid}", new AjaxController() {
@Override
public Object ajax(ParamMap params) {
Spider spider = spiderMap.get(params.get("uuid"));
ResultMap put = ResultMap.create().put("pageCount", spider.getPageCount())
.put("status", spider.getStatus().name()).put("thread", spider.getThreadAlive());
return put;
}
});
}
});
}
}

20
webmagic-worker/src/main/java/us/codecraft/webmagic/worker/web/SpiderController.java
Unescape Escape View File

@ -0,0 +1,20 @@
package us.codecraft.webmagic.worker.web;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.ModelAndView;
/**
* @author code4crafter@gmail.com
*/
@Controller("spider")
@RequestMapping("spider")
public class SpiderController {
@RequestMapping("create")
@ResponseBody
public ModelAndView create(){
return null;
}
}

21
webmagic-worker/src/main/resources/log/log4j.xml
Unescape Escape View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<logger name="org.apache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<root>
<level value="info" />
<appender-ref ref="stdout" />
</root>
</log4j:configuration>

34
webmagic-worker/src/main/resources/spring/applicationContext-freemarker.xml
Unescape Escape View File

@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd">
<bean id="freemarkerConfigurer"
class="org.springframework.web.servlet.view.freemarker.FreeMarkerConfigurer">
<property name="templateLoaderPath" value="/WEB-INF/pages/" />
<property name="defaultEncoding" value="utf-8" />
<property name="freemarkerSettings">
<props>
<prop key="template_update_delay">0</prop>
<prop key="locale">zh_CN</prop>
<prop key="datetime_format">yyyy-MM-dd HH:mm:ss</prop>
<prop key="date_format">yyyy-MM-dd</prop>
<prop key="number_format">#.##</prop>
</props>
</property>
</bean>
<!-- FreeMarker视图解析 如返回userinfo。。在这里配置后缀名ftl和视图解析器。。 -->
<bean id="viewResolver"
class="org.springframework.web.servlet.view.freemarker.FreeMarkerViewResolver">
<property name="viewClass"
value="org.springframework.web.servlet.view.freemarker.FreeMarkerView" />
<property name="suffix" value=".ftl" />
<property name="contentType" value="text/html;charset=utf-8" />
<property name="exposeRequestAttributes" value="true" />
<property name="exposeSessionAttributes" value="true" />
<property name="exposeSpringMacroHelpers" value="true" />
</bean>
</beans>

21
webmagic-worker/src/main/resources/spring/applicationContext-myBatis.xml
Unescape Escape View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd">
<!--<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">-->
<!--<property name="dataSource" ref="dataSource" />-->
<!--</bean>-->
<!--<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">-->
<!--<property name="basePackage" value="us.codecraft.blackhole.suite.dao" />-->
<!--</bean>-->
<!--<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource"-->
<!--destroy-method="close">-->
<!--<property name="driverClassName" value="org.sqlite.JDBC" />-->
<!--<property name="url" value="jdbc:sqlite:/usr/local/hostd/zonesfile.db" />-->
<!--</bean>-->
</beans>

21
webmagic-worker/src/main/resources/spring/applicationContext-webmagic-worker.xml
Unescape Escape View File

@ -1,19 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util"
xmlns:mvc="http://www.springframework.org/schema/mvc"
xsi:schemaLocation="http://www.springframework.org/schema/mvc
http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
<context:annotation-config />
<context:component-scan base-package="us.codecraft.webmagic.worker" />
<beans>
<bean name="server" class="us.codecraft.express.connector.jetty.JettyWebServer">
</bean>
</beans>
<beans>
<bean name="server" class="us.codecraft.express.connector.jetty.JettyWebServer">
</bean>
</beans>
</beans>

43
webmagic-worker/src/main/resources/spring/applicationContext.xml
Unescape Escape View File

@ -0,0 +1,43 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:mvc="http://www.springframework.org/schema/mvc"
xsi:schemaLocation="http://www.springframework.org/schema/mvc
http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-3.0.xsd">
<context:annotation-config/>
<bean id="messageSource" class="org.springframework.context.support.ResourceBundleMessageSource">
<property name="basenames">
<list>
<value>web_messages</value>
</list>
</property>
</bean>
<context:component-scan base-package="us.codecraft.webmagic.worker"/>
<bean
class="org.springframework.web.servlet.mvc.annotation.AnnotationMethodHandlerAdapter">
<property name="messageConverters">
<list>
<ref bean="jsonHttpMessageConverter"/>
</list>
</property>
</bean>
<bean id="jsonHttpMessageConverter" class="org.springframework.http.converter.json.MappingJacksonHttpMessageConverter"/>
<mvc:resources mapping="/js/**" location="/js/" />
<mvc:resources mapping="/css/**" location="/css/" />
<mvc:resources mapping="/img/**" location="/img/" />
<mvc:resources mapping="/font/**" location="/font/" />
<mvc:annotation-driven />
</beans>

12
webmagic-worker/src/main/webapp/WEB-INF/jsp/404.jsp
Unescape Escape View File

@ -0,0 +1,12 @@
<%@ page language="java" contentType="text/html; charset=utf8"
pageEncoding="utf8"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf8">
<title>404</title>
</head>
<body>
<h1>Page not found!</h1>
</body>
</html>

18
webmagic-worker/src/main/webapp/WEB-INF/jsp/500.jsp
Unescape Escape View File

@ -0,0 +1,18 @@
<%@ page language="java" contentType="text/html; charset=utf8"
pageEncoding="utf8" isErrorPage="true" import="java.io.*"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf8">
<title>500</title>
</head>
<body>
页面出错啦！
<%
StringWriter stringWriter = new StringWriter();
exception.printStackTrace(new PrintWriter(stringWriter));
out.println(stringWriter.toString());
%>
</body>
</html>

53
webmagic-worker/src/main/webapp/WEB-INF/web.xml
Unescape Escape View File

@ -0,0 +1,53 @@
<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd">
<display-name>Archetype Created Web Application</display-name>
<context-param>
<param-name>contextConfigLocation</param-name>
<param-value>
classpath*:spring/applicationContext*.xml,
</param-value>
</context-param>
<context-param>
<param-name>contextClass</param-name>
<param-value>org.springframework.web.context.support.XmlWebApplicationContext</param-value>
</context-param>
<!--由Sprng载入的Log4j配置文件位置 -->
<context-param>
<param-name>log4jConfigLocation</param-name>
<param-value>classpath:log/log4j.xml</param-value>
</context-param>
<context-param>
<param-name>log4jRefreshInterval</param-name>
<param-value>60000</param-value>
</context-param>
<servlet>
<servlet-name>spring</servlet-name>
<servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
<init-param>
<param-name>contextConfigLocation</param-name>
<param-value>classpath:/spring/applicationContext*.xml</param-value>
</init-param>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
<servlet-name>spring</servlet-name>
<url-pattern>/</url-pattern>
</servlet-mapping>
<error-page>
<error-code>404</error-code>
<location>/WEB-INF/jsp/404.jsp</location>
</error-page>
<error-page>
<error-code>500</error-code>
<location>/WEB-INF/jsp/500.jsp</location>
</error-page>
</web-app>
Write Preview
Loading…
Cancel
Save