diff --git a/pom.xml b/pom.xml index fa369f4a..b269adee 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ webmagic-core - webmagic-plugin/ + webmagic-extension/ webmagic-samples/ diff --git a/webmagic-plugin/webmagic-misc/pom.xml b/webmagic-extension/pom.xml similarity index 70% rename from webmagic-plugin/webmagic-misc/pom.xml rename to webmagic-extension/pom.xml index 130e4000..cbff0b85 100644 --- a/webmagic-plugin/webmagic-misc/pom.xml +++ b/webmagic-extension/pom.xml @@ -4,12 +4,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> us.codecraft - webmagic-plugin + webmagic 0.1.0 4.0.0 - webmagic-misc + webmagic-extension @@ -27,6 +27,15 @@ selenium-java 2.33.0 + + us.codecraft + webmagic-core + ${project.version} + + + junit + junit + \ No newline at end of file diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/PagedModel.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/PagedModel.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/PagedModel.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/PagedModel.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/FileDownloader.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/FileDownloader.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/FileDownloader.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/FileDownloader.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/AfterExtractor.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/AfterExtractor.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/AfterExtractor.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/AfterExtractor.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ConsolePageModelPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ConsolePageModelPipeline.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ConsolePageModelPipeline.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/ConsolePageModelPipeline.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ExtractBy.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ExtractBy.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ExtractBy.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/ExtractBy.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ExtractBy2.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ExtractBy2.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ExtractBy2.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/ExtractBy2.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ExtractBy3.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ExtractBy3.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ExtractBy3.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/ExtractBy3.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ExtractByRaw.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ExtractByRaw.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ExtractByRaw.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/ExtractByRaw.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ExtractByUrl.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ExtractByUrl.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ExtractByUrl.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/ExtractByUrl.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/Extractor.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/Extractor.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/Extractor.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/Extractor.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/FieldExtractor.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/FieldExtractor.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/FieldExtractor.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/FieldExtractor.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/HelpUrl.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/HelpUrl.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/HelpUrl.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/HelpUrl.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ModelPageProcessor.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPageProcessor.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ModelPageProcessor.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPageProcessor.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/OOSpider.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/OOSpider.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/OOSpider.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/OOSpider.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/PageModelPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelPipeline.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/PageModelPipeline.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelPipeline.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/TargetUrl.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/TargetUrl.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/TargetUrl.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/TargetUrl.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/package.html b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/package.html similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/package.html rename to webmagic-extension/src/main/java/us/codecraft/webmagic/model/package.html diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePipeline.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePipeline.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePipeline.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/pipeline/PagedPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/PagedPipeline.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/pipeline/PagedPipeline.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/PagedPipeline.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/utils/DoubleKeyMap.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/utils/DoubleKeyMap.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/utils/DoubleKeyMap.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/utils/DoubleKeyMap.java diff --git a/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/utils/MultiKeyMapBase.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/utils/MultiKeyMapBase.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/utils/MultiKeyMapBase.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/utils/MultiKeyMapBase.java diff --git a/webmagic-plugin/webmagic-misc/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java b/webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java rename to webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java diff --git a/webmagic-plugin/webmagic-misc/src/test/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloaderTest.java b/webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloaderTest.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/test/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloaderTest.java rename to webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloaderTest.java diff --git a/webmagic-plugin/webmagic-misc/src/test/java/us/codecraft/webmagic/downloader/selenium/WebDriverPoolTest.java b/webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/selenium/WebDriverPoolTest.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/test/java/us/codecraft/webmagic/downloader/selenium/WebDriverPoolTest.java rename to webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/selenium/WebDriverPoolTest.java diff --git a/webmagic-plugin/webmagic-misc/src/test/java/us/codecraft/webmagic/scheduler/RedisSchedulerTest.java b/webmagic-extension/src/test/java/us/codecraft/webmagic/scheduler/RedisSchedulerTest.java similarity index 100% rename from webmagic-plugin/webmagic-misc/src/test/java/us/codecraft/webmagic/scheduler/RedisSchedulerTest.java rename to webmagic-extension/src/test/java/us/codecraft/webmagic/scheduler/RedisSchedulerTest.java diff --git a/webmagic-plugin/webmagic-lucene/pom.xml b/webmagic-lucene/pom.xml similarity index 70% rename from webmagic-plugin/webmagic-lucene/pom.xml rename to webmagic-lucene/pom.xml index b072472d..512d1896 100644 --- a/webmagic-plugin/webmagic-lucene/pom.xml +++ b/webmagic-lucene/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - webmagic-plugin + webmagic us.codecraft 0.1.0 @@ -22,6 +22,15 @@ lucene-queryparser 4.4.0 + + us.codecraft + webmagic-core + ${project.version} + + + junit + junit + diff --git a/webmagic-plugin/webmagic-lucene/src/main/java/us/codecraft/webmagic/pipeline/LucenePipeline.java b/webmagic-lucene/src/main/java/us/codecraft/webmagic/pipeline/LucenePipeline.java similarity index 100% rename from webmagic-plugin/webmagic-lucene/src/main/java/us/codecraft/webmagic/pipeline/LucenePipeline.java rename to webmagic-lucene/src/main/java/us/codecraft/webmagic/pipeline/LucenePipeline.java diff --git a/webmagic-plugin/README.md b/webmagic-plugin/README.md deleted file mode 100644 index 536d5960..00000000 --- a/webmagic-plugin/README.md +++ /dev/null @@ -1,6 +0,0 @@ -webmagic-plugin -------- -webmagic的插件模块。 -目前仅实现了freemarker模板渲染，和redis实现分布式爬虫。 - -另外有一个使用Selenium来动态渲染页面的模块在开发中。 \ No newline at end of file diff --git a/webmagic-plugin/pom.xml b/webmagic-plugin/pom.xml deleted file mode 100644 index 54c69ec5..00000000 --- a/webmagic-plugin/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - us.codecraft - webmagic - 0.1.0 - - pom - 4.0.0 - - webmagic-misc - webmagic-selenium - webmagic-lucene - - - webmagic-plugin - - - - us.codecraft - webmagic-core - ${project.version} - - - junit - junit - - - - \ No newline at end of file diff --git a/webmagic-samples/pom.xml b/webmagic-samples/pom.xml index ff37e210..de3b4aaf 100644 --- a/webmagic-samples/pom.xml +++ b/webmagic-samples/pom.xml @@ -19,7 +19,7 @@ us.codecraft - webmagic-misc + webmagic-extension ${project.version}