diff --git a/webmagic-plugin/webmagic-misc/pom.xml b/webmagic-plugin/webmagic-misc/pom.xml index 5e241731..130e4000 100644 --- a/webmagic-plugin/webmagic-misc/pom.xml +++ b/webmagic-plugin/webmagic-misc/pom.xml @@ -22,6 +22,11 @@ jedis 2.0.0 + + org.seleniumhq.selenium + selenium-java + 2.33.0 + \ No newline at end of file diff --git a/webmagic-plugin/webmagic-selenium/src/main/java/us/codecraft/webmagic/selenium/downloader/SeleniumDownloader.java b/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java similarity index 98% rename from webmagic-plugin/webmagic-selenium/src/main/java/us/codecraft/webmagic/selenium/downloader/SeleniumDownloader.java rename to webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java index 002dcc9e..e95f27c4 100644 --- a/webmagic-plugin/webmagic-selenium/src/main/java/us/codecraft/webmagic/selenium/downloader/SeleniumDownloader.java +++ b/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java @@ -1,4 +1,4 @@ -package us.codecraft.webmagic.selenium.downloader; +package us.codecraft.webmagic.downloader.selenium; import org.apache.log4j.Logger; import org.openqa.selenium.By; diff --git a/webmagic-plugin/webmagic-selenium/src/main/java/us/codecraft/webmagic/selenium/downloader/WebDriverPool.java b/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java similarity index 97% rename from webmagic-plugin/webmagic-selenium/src/main/java/us/codecraft/webmagic/selenium/downloader/WebDriverPool.java rename to webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java index fdd978df..c763a99a 100644 --- a/webmagic-plugin/webmagic-selenium/src/main/java/us/codecraft/webmagic/selenium/downloader/WebDriverPool.java +++ b/webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java @@ -1,4 +1,4 @@ -package us.codecraft.webmagic.selenium.downloader; +package us.codecraft.webmagic.downloader.selenium; import org.openqa.selenium.WebDriver; import org.openqa.selenium.chrome.ChromeDriver; diff --git a/webmagic-plugin/webmagic-selenium/README.md b/webmagic-plugin/webmagic-selenium/README.md deleted file mode 100644 index 5e5ce821..00000000 --- a/webmagic-plugin/webmagic-selenium/README.md +++ /dev/null @@ -1,3 +0,0 @@ -webmagic-selenium -------- -尝试使用selenium来进行页面动态渲染，开发中。 \ No newline at end of file diff --git a/webmagic-plugin/webmagic-selenium/pom.xml b/webmagic-plugin/webmagic-selenium/pom.xml deleted file mode 100644 index 0da4504f..00000000 --- a/webmagic-plugin/webmagic-selenium/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - us.codecraft - webmagic-plugin - 0.1.0 - - 4.0.0 - webmagic-selenium - - - - org.seleniumhq.selenium - selenium-java - 2.33.0 - - - - - \ No newline at end of file diff --git a/webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/SeleniumTest.java b/webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/SeleniumTest.java deleted file mode 100644 index a6de8471..00000000 --- a/webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/SeleniumTest.java +++ /dev/null @@ -1,41 +0,0 @@ -package us.codecraft.webmagic.selenium; - -import org.junit.Ignore; -import org.junit.Test; -import org.openqa.selenium.By; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.chrome.ChromeDriver; -import org.openqa.selenium.remote.DesiredCapabilities; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -/** - * @author code4crafter@gmail.com
- * @date: 13-7-26
- * Time: 下午12:27
- */ -public class SeleniumTest { - - @Ignore("need chrome driver") - @Test - public void testSelenium() { - System.getProperties().setProperty("webdriver.chrome.driver", "/Users/yihua/Downloads/chromedriver"); - Map contentSettings = new HashMap(); - contentSettings.put("images", 2); - - Map preferences = new HashMap(); - preferences.put("profile.default_content_settings", contentSettings); - - DesiredCapabilities caps = DesiredCapabilities.chrome(); - caps.setCapability("chrome.prefs", preferences); - caps.setCapability("chrome.switches", Arrays.asList("--user-data-dir=/Users/yihua/temp/chrome")); - WebDriver webDriver = new ChromeDriver(caps); - webDriver.get("http://huaban.com/"); - WebElement webElement = webDriver.findElement(By.xpath("/html")); - System.out.println(webElement.getAttribute("outerHTML")); - webDriver.close(); - } -} diff --git a/webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/downloader/SeleniumDownloaderTest.java b/webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/downloader/SeleniumDownloaderTest.java deleted file mode 100644 index 23711fa9..00000000 --- a/webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/downloader/SeleniumDownloaderTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package us.codecraft.webmagic.selenium.downloader; - -import org.junit.Ignore; -import org.junit.Test; -import us.codecraft.webmagic.Page; -import us.codecraft.webmagic.Request; -import us.codecraft.webmagic.Site; -import us.codecraft.webmagic.Task; - -/** - * @author code4crafter@gmail.com
- * @date: 13-7-26
- * Time: 下午2:46
- */ -public class SeleniumDownloaderTest { - - private String chromeDriverPath = "/Users/yihua/Downloads/chromedriver"; - - @Ignore("need chrome driver") - @Test - public void test() { - SeleniumDownloader seleniumDownloader = new SeleniumDownloader(chromeDriverPath); - long time1 = System.currentTimeMillis(); - for (int i = 0; i < 100; i++) { - Page page = seleniumDownloader.download(new Request("http://huaban.com/"), new Task() { - @Override - public String getUUID() { - return "huaban.com"; - } - - @Override - public Site getSite() { - return Site.me(); - } - }); - System.out.println(page.getHtml().$("#waterfall").links().regex(".*pins.*").all()); - } - System.out.println(System.currentTimeMillis() - time1); - } - - @Ignore - @Test - public void testBaiduWenku() { - SeleniumDownloader seleniumDownloader = new SeleniumDownloader(chromeDriverPath); - seleniumDownloader.setSleepTime(10000); - long time1 = System.currentTimeMillis(); - Page page = seleniumDownloader.download(new Request("http://wenku.baidu.com/view/462933ff04a1b0717fd5ddc2.html"), new Task() { - @Override - public String getUUID() { - return "huaban.com"; - } - - @Override - public Site getSite() { - return Site.me(); - } - }); - System.out.println(page.getHtml().$("div.inner").replace("<[^<>]+>","").replace("&nsbp;","").all()); - } - -} diff --git a/webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/downloader/WebDriverPoolTest.java b/webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/downloader/WebDriverPoolTest.java deleted file mode 100644 index cbf38606..00000000 --- a/webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/downloader/WebDriverPoolTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package us.codecraft.webmagic.selenium.downloader; - -import org.junit.Ignore; -import org.junit.Test; -import org.openqa.selenium.WebDriver; - -/** - * @author code4crafter@gmail.com
- * @date: 13-7-26
- * Time: 下午2:12
- */ -public class WebDriverPoolTest { - - private String chromeDriverPath = "/Users/yihua/Downloads/chromedriver"; - - @Ignore("need chrome driver") - @Test - public void test() { - System.getProperties().setProperty("webdriver.chrome.driver", chromeDriverPath); - WebDriverPool webDriverPool = new WebDriverPool(5); - for (int i = 0; i < 5; i++) { - try { - WebDriver webDriver = webDriverPool.get(); - System.out.println(i); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - webDriverPool.closeAll(); - } -} diff --git a/webmagic-samples/pom.xml b/webmagic-samples/pom.xml index a38e872d..ff37e210 100644 --- a/webmagic-samples/pom.xml +++ b/webmagic-samples/pom.xml @@ -22,11 +22,6 @@ webmagic-misc ${project.version} - - us.codecraft - webmagic-selenium - ${project.version} - org.freemarker freemarker