github-forks
/
webmagic
mirror of https://github.com/code4craft/webmagic.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

code smells fixes

Browse Source
pull/1015/head
Vivian Delannoy 4 years ago
parent fcffe42bd1
commit 6b6a419ef9
11 changed files with 68 additions and 71 deletions
Show Stats Download Patch File Download Diff File

3
webmagic-core/src/main/java/us/codecraft/webmagic/Page.java
Unescape Escape View File

@ -49,7 +49,7 @@ public class Page {
private byte[] bytes;
private List<Request> targetRequests = new ArrayList<Request>();
private List<Request> targetRequests = new ArrayList<>();
private String charset;
@ -108,6 +108,7 @@ public class Page {
* @deprecated since 0.4.0
* The html is parse just when first time of calling {@link #getHtml()}, so use {@link #setRawText(String)} instead.
*/
@Deprecated
public void setHtml(Html html) {
this.html = html;
}

6
webmagic-core/src/main/java/us/codecraft/webmagic/Request.java
Unescape Escape View File

@ -40,9 +40,9 @@ public class Request implements Serializable {
/**
* cookies for current url, if not set use Site's cookies
*/
private Map<String, String> cookies = new HashMap<String, String>();
private Map<String, String> cookies = new HashMap<>();
private Map<String, String> headers = new HashMap<String, String>();
private Map<String, String> headers = new HashMap<>();
/**
* Priority of the request.<br>
@ -94,7 +94,7 @@ public class Request implements Serializable {
public <T> Request putExtra(String key, T value) {
if (extras == null) {
extras = new HashMap<String, Object>();
extras = new HashMap<>();
}
extras.put(key, value);
return this;

2
webmagic-core/src/main/java/us/codecraft/webmagic/ResultItems.java
Unescape Escape View File

@ -14,7 +14,7 @@ import java.util.Map;
*/
public class ResultItems {
private Map<String, Object> fields = new LinkedHashMap<String, Object>();
private Map<String, Object> fields = new LinkedHashMap<>();
private Request request;

10
webmagic-core/src/main/java/us/codecraft/webmagic/Site.java
Unescape Escape View File

@ -22,9 +22,9 @@ public class Site {
private String userAgent;
private Map<String, String> defaultCookies = new LinkedHashMap<String, String>();
private Map<String, String> defaultCookies = new LinkedHashMap<>();
private Map<String, Map<String, String>> cookies = new HashMap<String, Map<String, String>>();
private Map<String, Map<String, String>> cookies = new HashMap<>();
private String charset;
@ -38,11 +38,11 @@ public class Site {
private int timeOut = 5000;
private static final Set<Integer> DEFAULT_STATUS_CODE_SET = new HashSet<Integer>();
private static final Set<Integer> DEFAULT_STATUS_CODE_SET = new HashSet<>();
private Set<Integer> acceptStatCode = DEFAULT_STATUS_CODE_SET;
private Map<String, String> headers = new HashMap<String, String>();
private Map<String, String> headers = new HashMap<>();
private boolean useGzip = true;
@ -83,7 +83,7 @@ public class Site {
*/
public Site addCookie(String domain, String name, String value) {
if (!cookies.containsKey(domain)){
cookies.put(domain,new HashMap<String, String>());
cookies.put(domain,new HashMap<>());
}
cookies.get(domain).put(name, value);
return this;

17
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
Unescape Escape View File

@ -62,7 +62,7 @@ public class Spider implements Runnable, Task {
protected Downloader downloader;
protected List<Pipeline> pipelines = new ArrayList<Pipeline>();
protected List<Pipeline> pipelines = new ArrayList<>();
protected PageProcessor pageProcessor;
@ -86,11 +86,11 @@ public class Spider implements Runnable, Task {
protected boolean exitWhenComplete = true;
protected final static int STAT_INIT = 0;
protected static final int STAT_INIT = 0;
protected final static int STAT_RUNNING = 1;
protected static final int STAT_RUNNING = 1;
protected final static int STAT_STOPPED = 2;
protected static final int STAT_STOPPED = 2;
protected boolean spawnUrl = true;
@ -248,7 +248,7 @@ public class Spider implements Runnable, Task {
* @return this
*/
public Spider clearPipeline() {
pipelines = new ArrayList<Pipeline>();
pipelines = new ArrayList<>();
return this;
}
@ -439,7 +439,6 @@ public class Spider implements Runnable, Task {
logger.info("page status code error, page {} , code: {}", request.getUrl(), page.getStatusCode());
}
sleep(site.getSleepTime());
return;
}
private void onDownloaderFail(Request request) {
@ -545,7 +544,7 @@ public class Spider implements Runnable, Task {
public <T> T get(String url) {
List<String> urls = WMCollections.newArrayList(url);
List<T> resultItemses = getAll(urls);
if (resultItemses != null && resultItemses.size() > 0) {
if (resultItemses != null && !(resultItemses.isEmpty())) {
return resultItemses.get(0);
} else {
return null;
@ -678,7 +677,7 @@ public class Spider implements Runnable, Task {
public enum Status {
Init(0), Running(1), Stopped(2);
INIT(0), RUNNING(1), STOPPED(2);
private Status(int value) {
this.value = value;
@ -697,7 +696,7 @@ public class Spider implements Runnable, Task {
}
}
//default value
return Init;
return INIT;
}
}

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/AbstractDownloader.java
Unescape Escape View File

@ -32,7 +32,7 @@ public abstract class AbstractDownloader implements Downloader {
*/
public Html download(String url, String charset) {
Page page = download(new Request(url), Site.me().setCharset(charset).toTask());
return (Html) page.getHtml();
return page.getHtml();
}
protected void onSuccess(Request request) {

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
Unescape Escape View File

@ -33,7 +33,7 @@ public class HttpClientDownloader extends AbstractDownloader {
private Logger logger = LoggerFactory.getLogger(getClass());
private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
private final Map<String, CloseableHttpClient> httpClients = new HashMap<>();
private HttpClientGenerator httpClientGenerator = new HttpClientGenerator();

8
webmagic-core/src/main/java/us/codecraft/webmagic/model/HttpRequestBody.java
Unescape Escape View File

@ -16,6 +16,8 @@ import java.util.Map;
*/
public class HttpRequestBody implements Serializable {
private static final String ILL_ENC = "illegal encoding ";
private static final long serialVersionUID = 5659170945717023595L;
public static abstract class ContentType {
@ -68,7 +70,7 @@ public class HttpRequestBody implements Serializable {
try {
return new HttpRequestBody(json.getBytes(encoding), ContentType.JSON, encoding);
} catch (UnsupportedEncodingException e) {
throw new IllegalArgumentException("illegal encoding " + encoding, e);
throw new IllegalArgumentException(ILL_ENC + encoding, e);
}
}
@ -76,7 +78,7 @@ public class HttpRequestBody implements Serializable {
try {
return new HttpRequestBody(xml.getBytes(encoding), ContentType.XML, encoding);
} catch (UnsupportedEncodingException e) {
throw new IllegalArgumentException("illegal encoding " + encoding, e);
throw new IllegalArgumentException(ILL_ENC + encoding, e);
}
}
@ -92,7 +94,7 @@ public class HttpRequestBody implements Serializable {
try {
return new HttpRequestBody(URLEncodedUtils.format(nameValuePairs, encoding).getBytes(encoding), ContentType.FORM, encoding);
} catch (UnsupportedEncodingException e) {
throw new IllegalArgumentException("illegal encoding " + encoding, e);
throw new IllegalArgumentException(ILL_ENC + encoding, e);
}
}

7
webmagic-core/src/main/java/us/codecraft/webmagic/utils/CharsetUtils.java
Unescape Escape View File

@ -19,6 +19,7 @@ import java.nio.charset.Charset;
*/
public abstract class CharsetUtils {
private static final String CHR = "charset";
private static Logger logger = LoggerFactory.getLogger(CharsetUtils.class);
public static String detectCharset(String contentType, byte[] contentBytes) throws IOException {
@ -40,9 +41,9 @@ public abstract class CharsetUtils {
for (Element link : links) {
// 2.1、html4.01 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
String metaContent = link.attr("content");
String metaCharset = link.attr("charset");
if (metaContent.indexOf("charset") != -1) {
metaContent = metaContent.substring(metaContent.indexOf("charset"), metaContent.length());
String metaCharset = link.attr(CHR);
if (metaContent.indexOf(CHR) != -1) {
metaContent = metaContent.substring(metaContent.indexOf(CHR), metaContent.length());
charset = metaContent.split("=")[1];
break;
}

1
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/scheduler/ZipCodePageProcessor.java
Unescape Escape View File

@ -19,6 +19,7 @@ import static us.codecraft.webmagic.selector.Selectors.xpath;
*/
public class ZipCodePageProcessor implements PageProcessor {
private Site site = Site.me().setCharset("gb2312")
.setSleepTime(100);

81
webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java
Unescape Escape View File

@ -63,54 +63,47 @@ class WebDriverPool {
* @throws IOException
*/
public void configure() throws IOException {
try{
// Read config file
sConfig = new Properties();
String configFile = DEFAULT_CONFIG_FILE;
if (System.getProperty("selenuim_config")!=null){
configFile = System.getProperty("selenuim_config");
// Read config file
sConfig = new Properties();
String configFile = DEFAULT_CONFIG_FILE;
if (System.getProperty("selenuim_config")!=null){
configFile = System.getProperty("selenuim_config");
}
sConfig.load(new FileReader(configFile));
// Prepare capabilities
sCaps = new DesiredCapabilities();
sCaps.setJavascriptEnabled(true);
sCaps.setCapability("takesScreenshot", false);
String driver = sConfig.getProperty("driver", DRIVER_PHANTOMJS);
// Fetch PhantomJS-specific configuration parameters
if (driver.equals(DRIVER_PHANTOMJS)) {
// "phantomjs_exec_path"
if (sConfig.getProperty("phantomjs_exec_path") != null) {
sCaps.setCapability(
PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY,
sConfig.getProperty("phantomjs_exec_path"));
} else {
throw new IOException(
String.format(
"Property '%s' not set!",
PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY));
}
FileReader configFileReader = new FileReader(configFile)
sConfig.load(configFileReader);
// Prepare capabilities
sCaps = new DesiredCapabilities();
sCaps.setJavascriptEnabled(true);
sCaps.setCapability("takesScreenshot", false);
String driver = sConfig.getProperty("driver", DRIVER_PHANTOMJS);
// Fetch PhantomJS-specific configuration parameters
if (driver.equals(DRIVER_PHANTOMJS)) {
// "phantomjs_exec_path"
if (sConfig.getProperty("phantomjs_exec_path") != null) {
sCaps.setCapability(
PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY,
sConfig.getProperty("phantomjs_exec_path"));
} else {
throw new IOException(
String.format(
"Property '%s' not set!",
PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY));
}
// "phantomjs_driver_path"
if (sConfig.getProperty("phantomjs_driver_path") != null) {
System.out.println("Test will use an external GhostDriver");
sCaps.setCapability(
PhantomJSDriverService.PHANTOMJS_GHOSTDRIVER_PATH_PROPERTY,
sConfig.getProperty("phantomjs_driver_path"));
} else {
System.out
.println("Test will use PhantomJS internal GhostDriver");
}
// "phantomjs_driver_path"
if (sConfig.getProperty("phantomjs_driver_path") != null) {
System.out.println("Test will use an external GhostDriver");
sCaps.setCapability(
PhantomJSDriverService.PHANTOMJS_GHOSTDRIVER_PATH_PROPERTY,
sConfig.getProperty("phantomjs_driver_path"));
} else {
System.out
.println("Test will use PhantomJS internal GhostDriver");
}
}catch(Exception e){
throw new IOException("Can not load config file properly");
}finally{
configFileReader.close();
}
// Disable "web-security", enable all possible "ssl-protocols" and
// "ignore-ssl-errors" for PhantomJSDriver
// sCaps.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS, new

Write Preview
Loading…
Cancel
Save