orgnize dep
yihua.huang
12 years ago
parent
36494bcfa5
commit
ac917039a3
@ -1,4 +1,4 @@
|
||||
package us.codecraft.webmagic.selenium.downloader;
|
||||
package us.codecraft.webmagic.downloader.selenium;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.openqa.selenium.By;
|
||||
@ -1,4 +1,4 @@
|
||||
package us.codecraft.webmagic.selenium.downloader;
|
||||
package us.codecraft.webmagic.downloader.selenium;
|
||||
|
||||
import org.openqa.selenium.WebDriver;
|
||||
import org.openqa.selenium.chrome.ChromeDriver;
|
||||
@ -1,23 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<parent>
|
||||
<groupId>us.codecraft</groupId>
|
||||
<artifactId>webmagic-plugin</artifactId>
|
||||
<version>0.1.0</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>webmagic-selenium</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.seleniumhq.selenium</groupId>
|
||||
<artifactId>selenium-java</artifactId>
|
||||
<version>2.33.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
</project>
|
||||
@ -1,41 +0,0 @@
|
||||
package us.codecraft.webmagic.selenium;
|
||||
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.openqa.selenium.By;
|
||||
import org.openqa.selenium.WebDriver;
|
||||
import org.openqa.selenium.WebElement;
|
||||
import org.openqa.selenium.chrome.ChromeDriver;
|
||||
import org.openqa.selenium.remote.DesiredCapabilities;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* @date: 13-7-26 <br>
|
||||
* Time: 下午12:27 <br>
|
||||
*/
|
||||
public class SeleniumTest {
|
||||
|
||||
@Ignore("need chrome driver")
|
||||
@Test
|
||||
public void testSelenium() {
|
||||
System.getProperties().setProperty("webdriver.chrome.driver", "/Users/yihua/Downloads/chromedriver");
|
||||
Map<String, Object> contentSettings = new HashMap<String, Object>();
|
||||
contentSettings.put("images", 2);
|
||||
|
||||
Map<String, Object> preferences = new HashMap<String, Object>();
|
||||
preferences.put("profile.default_content_settings", contentSettings);
|
||||
|
||||
DesiredCapabilities caps = DesiredCapabilities.chrome();
|
||||
caps.setCapability("chrome.prefs", preferences);
|
||||
caps.setCapability("chrome.switches", Arrays.asList("--user-data-dir=/Users/yihua/temp/chrome"));
|
||||
WebDriver webDriver = new ChromeDriver(caps);
|
||||
webDriver.get("http://huaban.com/");
|
||||
WebElement webElement = webDriver.findElement(By.xpath("/html"));
|
||||
System.out.println(webElement.getAttribute("outerHTML"));
|
||||
webDriver.close();
|
||||
}
|
||||
}
|
||||
@ -1,61 +0,0 @@
|
||||
package us.codecraft.webmagic.selenium.downloader;
|
||||
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.Request;
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.Task;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* @date: 13-7-26 <br>
|
||||
* Time: 下午2:46 <br>
|
||||
*/
|
||||
public class SeleniumDownloaderTest {
|
||||
|
||||
private String chromeDriverPath = "/Users/yihua/Downloads/chromedriver";
|
||||
|
||||
@Ignore("need chrome driver")
|
||||
@Test
|
||||
public void test() {
|
||||
SeleniumDownloader seleniumDownloader = new SeleniumDownloader(chromeDriverPath);
|
||||
long time1 = System.currentTimeMillis();
|
||||
for (int i = 0; i < 100; i++) {
|
||||
Page page = seleniumDownloader.download(new Request("http://huaban.com/"), new Task() {
|
||||
@Override
|
||||
public String getUUID() {
|
||||
return "huaban.com";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Site getSite() {
|
||||
return Site.me();
|
||||
}
|
||||
});
|
||||
System.out.println(page.getHtml().$("#waterfall").links().regex(".*pins.*").all());
|
||||
}
|
||||
System.out.println(System.currentTimeMillis() - time1);
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testBaiduWenku() {
|
||||
SeleniumDownloader seleniumDownloader = new SeleniumDownloader(chromeDriverPath);
|
||||
seleniumDownloader.setSleepTime(10000);
|
||||
long time1 = System.currentTimeMillis();
|
||||
Page page = seleniumDownloader.download(new Request("http://wenku.baidu.com/view/462933ff04a1b0717fd5ddc2.html"), new Task() {
|
||||
@Override
|
||||
public String getUUID() {
|
||||
return "huaban.com";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Site getSite() {
|
||||
return Site.me();
|
||||
}
|
||||
});
|
||||
System.out.println(page.getHtml().$("div.inner").replace("<[^<>]+>","").replace("&nsbp;","").all());
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
package us.codecraft.webmagic.selenium.downloader;
|
||||
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.openqa.selenium.WebDriver;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* @date: 13-7-26 <br>
|
||||
* Time: 下午2:12 <br>
|
||||
*/
|
||||
public class WebDriverPoolTest {
|
||||
|
||||
private String chromeDriverPath = "/Users/yihua/Downloads/chromedriver";
|
||||
|
||||
@Ignore("need chrome driver")
|
||||
@Test
|
||||
public void test() {
|
||||
System.getProperties().setProperty("webdriver.chrome.driver", chromeDriverPath);
|
||||
WebDriverPool webDriverPool = new WebDriverPool(5);
|
||||
for (int i = 0; i < 5; i++) {
|
||||
try {
|
||||
WebDriver webDriver = webDriverPool.get();
|
||||
System.out.println(i);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
webDriverPool.closeAll();
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue