From f1573b40a2601a1ebf05b952e4398ff4e2e9f4a7 Mon Sep 17 00:00:00 2001
From: "yihua.huang" <code4crafter@gmail.com>
Date: Wed, 7 Aug 2013 14:44:52 +0800
Subject: [PATCH] set selenium dep to seperate package

---
 pom.xml                                       | 11 ++++---
 webmagic-extension/pom.xml                    |  5 ---
 webmagic-samples/assembly.xml                 | 21 -------------
 webmagic-selenium/README.md                   |  3 ++
 webmagic-selenium/pom.xml                     | 31 +++++++++++++++++++
 .../selenium/SeleniumDownloader.java          |  0
 .../downloader/selenium/WebDriverPool.java    |  0
 .../webmagic/downloader/SeleniumTest.java     |  0
 .../selenium/SeleniumDownloaderTest.java      |  0
 .../selenium/WebDriverPoolTest.java           |  0
 .../webmagic/samples}/HuabanProcessor.java    |  4 +--
 11 files changed, 41 insertions(+), 34 deletions(-)
 delete mode 100644 webmagic-samples/assembly.xml
 create mode 100644 webmagic-selenium/README.md
 create mode 100644 webmagic-selenium/pom.xml
 rename {webmagic-extension => webmagic-selenium}/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java (100%)
 rename {webmagic-extension => webmagic-selenium}/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java (100%)
 rename {webmagic-extension => webmagic-selenium}/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java (100%)
 rename {webmagic-extension => webmagic-selenium}/src/test/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloaderTest.java (100%)
 rename {webmagic-extension => webmagic-selenium}/src/test/java/us/codecraft/webmagic/downloader/selenium/WebDriverPoolTest.java (100%)
 rename {webmagic-samples/src/main/java/us/codecraft/webmagic/samples/selenium => webmagic-selenium/src/test/java/us/codecraft/webmagic/samples}/HuabanProcessor.java (89%)

diff --git a/pom.xml b/pom.xml
index 5033977b..1497bb0c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -8,10 +8,12 @@
     <packaging>pom</packaging>
     <artifactId>webmagic</artifactId>
 
-    <modules>  
-    <module>webmagic-core</module>
-    <module>webmagic-extension/</module>
-    <module>webmagic-samples/</module>
+    <modules>
+        <module>webmagic-core</module>
+        <module>webmagic-extension/</module>
+        <module>webmagic-samples/</module>
+        <module>webmagic-selenium/</module>
+        <module>webmagic-lucene/</module>
     </modules>
 
     <dependencyManagement>
@@ -143,5 +145,4 @@
     </build>
 
 
-
 </project>
diff --git a/webmagic-extension/pom.xml b/webmagic-extension/pom.xml
index 2a3590d3..63034f23 100644
--- a/webmagic-extension/pom.xml
+++ b/webmagic-extension/pom.xml
@@ -22,11 +22,6 @@
             <artifactId>jedis</artifactId>
             <version>2.0.0</version>
         </dependency>
-        <dependency>
-            <groupId>org.seleniumhq.selenium</groupId>
-            <artifactId>selenium-java</artifactId>
-            <version>2.33.0</version>
-        </dependency>
         <dependency>
             <groupId>us.codecraft</groupId>
             <artifactId>webmagic-core</artifactId>
diff --git a/webmagic-samples/assembly.xml b/webmagic-samples/assembly.xml
deleted file mode 100644
index 83370a25..00000000
--- a/webmagic-samples/assembly.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<assembly
-	xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
-	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-	<id>jar-with-dependencies</id>
-	<formats>
-		<format>jar</format>
-	</formats>
-	<includeBaseDirectory>false</includeBaseDirectory>
-	<fileSets>
-		<fileSet>
-			<directory>${project.basedir}/target/classes</directory>
-			<outputDirectory>/</outputDirectory>
-		</fileSet>
-	</fileSets>
-	<dependencySets>
-		<dependencySet>
-			<unpack>false</unpack>
-		</dependencySet>
-	</dependencySets>
-</assembly>
\ No newline at end of file
diff --git a/webmagic-selenium/README.md b/webmagic-selenium/README.md
new file mode 100644
index 00000000..71d3c48b
--- /dev/null
+++ b/webmagic-selenium/README.md
@@ -0,0 +1,3 @@
+webmagic-extension
+-------
+webmagic的扩展模块。包括注解格式定义爬虫、JSON、分布式等支持。
\ No newline at end of file
diff --git a/webmagic-selenium/pom.xml b/webmagic-selenium/pom.xml
new file mode 100644
index 00000000..814b7b31
--- /dev/null
+++ b/webmagic-selenium/pom.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <groupId>us.codecraft</groupId>
+        <artifactId>webmagic</artifactId>
+        <version>0.2.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>webmagic-selenium</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.seleniumhq.selenium</groupId>
+            <artifactId>selenium-java</artifactId>
+            <version>2.33.0</version>
+        </dependency>
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>webmagic-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java b/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java
similarity index 100%
rename from webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java
rename to webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java b/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java
similarity index 100%
rename from webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java
rename to webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java
diff --git a/webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java b/webmagic-selenium/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java
similarity index 100%
rename from webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java
rename to webmagic-selenium/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java
diff --git a/webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloaderTest.java b/webmagic-selenium/src/test/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloaderTest.java
similarity index 100%
rename from webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloaderTest.java
rename to webmagic-selenium/src/test/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloaderTest.java
diff --git a/webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/selenium/WebDriverPoolTest.java b/webmagic-selenium/src/test/java/us/codecraft/webmagic/downloader/selenium/WebDriverPoolTest.java
similarity index 100%
rename from webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/selenium/WebDriverPoolTest.java
rename to webmagic-selenium/src/test/java/us/codecraft/webmagic/downloader/selenium/WebDriverPoolTest.java
diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/selenium/HuabanProcessor.java b/webmagic-selenium/src/test/java/us/codecraft/webmagic/samples/HuabanProcessor.java
similarity index 89%
rename from webmagic-samples/src/main/java/us/codecraft/webmagic/samples/selenium/HuabanProcessor.java
rename to webmagic-selenium/src/test/java/us/codecraft/webmagic/samples/HuabanProcessor.java
index dfff0c6c..6b3d8a07 100644
--- a/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/selenium/HuabanProcessor.java
+++ b/webmagic-selenium/src/test/java/us/codecraft/webmagic/samples/HuabanProcessor.java
@@ -1,4 +1,4 @@
-package us.codecraft.webmagic.samples.selenium;
+package us.codecraft.webmagic.samples;
 
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Site;
@@ -6,7 +6,6 @@ import us.codecraft.webmagic.Spider;
 import us.codecraft.webmagic.downloader.selenium.SeleniumDownloader;
 import us.codecraft.webmagic.pipeline.FilePipeline;
 import us.codecraft.webmagic.processor.PageProcessor;
-import us.codecraft.webmagic.scheduler.RedisScheduler;
 
 /**
  * 花瓣网抽取器。<br>
@@ -39,7 +38,6 @@ public class HuabanProcessor implements PageProcessor {
 
     public static void main(String[] args) {
         Spider.create(new HuabanProcessor()).thread(5)
-                .scheduler(new RedisScheduler("localhost"))
                 .pipeline(new FilePipeline("/data/webmagic/test/"))
                 .downloader(new SeleniumDownloader("/Users/yihua/Downloads/chromedriver"))
                 .runAsync();