github-forks
/
webmagic
mirror of https://github.com/code4craft/webmagic.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add worker

Browse Source
pull/51/head
yihua.huang 11 years ago
parent b51fb2696b
commit 31fb0048a1
7 changed files with 196 additions and 0 deletions
Show Stats Download Patch File Download Diff File

4
pom.xml
Unescape Escape View File

@ -12,6 +12,8 @@
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-version>3.1.1.RELEASE</spring-version>
</properties> </properties>
<artifactId>webmagic-parent</artifactId> <artifactId>webmagic-parent</artifactId>
<name>webmagic-parent</name> <name>webmagic-parent</name>
@ -49,6 +51,8 @@
<module>webmagic-core</module> <module>webmagic-core</module>
<module>webmagic-extension/</module> <module>webmagic-extension/</module>
<module>webmagic-scripts/</module> <module>webmagic-scripts/</module>
<module>webmagic-panel</module>
<module>webmagic-worker</module>
</modules> </modules>
<dependencyManagement> <dependencyManagement>

20
webmagic-panel/README.md
Unescape Escape View File

@ -0,0 +1,20 @@
Worker:
任务执行者，提供Http接口，监控运行状态，终止和开始job
队列：
仍然使用redis
Panel:
提供Web管理后台，管理
1. 新建任务
1. 通过脚本
2. 配置
3. 分配机器
2. 已有任务
3. 任务查看

25
webmagic-panel/pom.xml
Unescape Escape View File

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.4.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-panel</artifactId>
<version>0.4.3-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-scripts</artifactId>
<version>0.4.3-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

45
webmagic-worker/pom.xml
Unescape Escape View File

@ -0,0 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.4.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-worker</artifactId>
<version>0.4.3-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-scripts</artifactId>
<version>0.4.3-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>express.java</artifactId>
<version>0.1.0</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-asm</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring-version}</version>
</dependency>
</dependencies>
</project>

19
webmagic-worker/src/main/java/us/codecraft/webmagic/worker/Bootstrap.java
Unescape Escape View File

@ -0,0 +1,19 @@
package us.codecraft.webmagic.worker;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import us.codecraft.express.WebServer;
/**
* @author code4crafter@gmail.com
*/
public class Bootstrap {
public static void main(String[] args) throws Exception {
ClassPathXmlApplicationContext classPathXmlApplicationContext = new ClassPathXmlApplicationContext(
new String[]{"classpath*:/spring/applicationContext-*.xml"}
);
WebServer webServer = classPathXmlApplicationContext.getBean(WebServer.class);
webServer.port(11111).start();
}
}

64
webmagic-worker/src/main/java/us/codecraft/webmagic/worker/SpiderManager.java
Unescape Escape View File

@ -0,0 +1,64 @@
package us.codecraft.webmagic.worker;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.express.WebServer;
import us.codecraft.express.controller.AjaxController;
import us.codecraft.express.controller.ParamMap;
import us.codecraft.express.controller.ResultMap;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scripts.Language;
import us.codecraft.webmagic.scripts.ScriptProcessor;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author code4crafter@gmail.com
*/
@Component
public class SpiderManager implements InitializingBean {
@Autowired
private WebServer webServer;
private Map<String, Spider> spiderMap = new ConcurrentHashMap<String, Spider>();
public Spider newSpider(ParamMap params) {
Spider spider = Spider
.create(new ScriptProcessor(Language.JavaScript, params.get("script"), params.getInt("thread")))
.thread(params.getInt("thread")).addUrl(params.get("url"));
spider.start();
return spider;
}
@Override
public void afterPropertiesSet() throws Exception {
AjaxController newController = new AjaxController() {
@Override
public Object ajax(ParamMap params) {
try {
Spider spider = newSpider(params);
spiderMap.put(params.get("uuid"), spider);
return ResultMap.create().put("code", 200).put("msg", "success");
} catch (Exception e) {
// If you provide worker to user, DO NOT return
// e.getMessage()!
return ResultMap.create().put("code", 500).put("msg", e.getMessage());
}
}
};
webServer.post("/new/${uuid}", newController);
webServer.get("/new/${uuid}", newController);
webServer.get("/status/${uuid}", new AjaxController() {
@Override
public Object ajax(ParamMap params) {
Spider spider = spiderMap.get(params.get("uuid"));
ResultMap put = ResultMap.create().put("pageCount", spider.getPageCount())
.put("status", spider.getStatus().name()).put("thread", spider.getThreadAlive());
return put;
}
});
}
}

19
webmagic-worker/src/main/resources/spring/applicationContext-webmagic-worker.xml
Unescape Escape View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
<context:annotation-config />
<context:component-scan base-package="us.codecraft.webmagic.worker" />
<beans>
<bean name="server" class="us.codecraft.express.connector.jetty.JettyWebServer">
</bean>
</beans>
</beans>
Write Preview
Loading…
Cancel
Save