重构一部分httpclient
yihua.huang
8 years ago
parent
221c155060
commit
a7f9e7cad5
@ -0,0 +1,98 @@
|
||||
package us.codecraft.webmagic.downloader;
|
||||
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.config.CookieSpecs;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.HttpUriRequest;
|
||||
import org.apache.http.client.methods.RequestBuilder;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import us.codecraft.webmagic.Request;
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.proxy.Proxy;
|
||||
import us.codecraft.webmagic.utils.HttpConstant;
|
||||
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com
|
||||
* Date: 17/3/18
|
||||
* Time: 上午11:28
|
||||
*/
|
||||
public class HttpUriRequestConverter {
|
||||
|
||||
public HttpUriRequest convert(Request request, Site site, Proxy proxy) {
|
||||
return null;
|
||||
}
|
||||
|
||||
private HttpUriRequest getHttpUriRequest(Request request, Site site, Map<String, String> headers, HttpHost proxy) {
|
||||
RequestBuilder requestBuilder = selectRequestMethod(request).setUri(request.getUrl());
|
||||
if (headers != null) {
|
||||
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
|
||||
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
|
||||
if (site != null) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(site.getTimeOut())
|
||||
.setSocketTimeout(site.getTimeOut())
|
||||
.setConnectTimeout(site.getTimeOut())
|
||||
.setCookieSpec(CookieSpecs.BEST_MATCH);
|
||||
}
|
||||
|
||||
if (proxy != null) {
|
||||
requestConfigBuilder.setProxy(proxy);
|
||||
}
|
||||
requestBuilder.setConfig(requestConfigBuilder.build());
|
||||
return requestBuilder.build();
|
||||
}
|
||||
|
||||
private RequestBuilder selectRequestMethod(Request request) {
|
||||
String method = request.getMethod();
|
||||
if (method == null || method.equalsIgnoreCase(HttpConstant.Method.GET)) {
|
||||
//default get
|
||||
return addQueryParams(RequestBuilder.get(),request.getParams());
|
||||
} else if (method.equalsIgnoreCase(HttpConstant.Method.POST)) {
|
||||
return addFormParams(RequestBuilder.post(), (NameValuePair[]) request.getExtra("nameValuePair"), request.getParams());
|
||||
} else if (method.equalsIgnoreCase(HttpConstant.Method.HEAD)) {
|
||||
return addQueryParams(RequestBuilder.head(),request.getParams());
|
||||
} else if (method.equalsIgnoreCase(HttpConstant.Method.PUT)) {
|
||||
return addFormParams(RequestBuilder.put(), (NameValuePair[]) request.getExtra("nameValuePair"), request.getParams());
|
||||
} else if (method.equalsIgnoreCase(HttpConstant.Method.DELETE)) {
|
||||
return addQueryParams(RequestBuilder.delete(),request.getParams());
|
||||
} else if (method.equalsIgnoreCase(HttpConstant.Method.TRACE)) {
|
||||
return addQueryParams(RequestBuilder.trace(),request.getParams());
|
||||
}
|
||||
throw new IllegalArgumentException("Illegal HTTP Method " + method);
|
||||
}
|
||||
|
||||
private RequestBuilder addFormParams(RequestBuilder requestBuilder, NameValuePair[] nameValuePair, Map<String, String> params) {
|
||||
List<NameValuePair> allNameValuePair=new ArrayList<NameValuePair>();
|
||||
if (nameValuePair != null && nameValuePair.length > 0) {
|
||||
allNameValuePair= Arrays.asList(nameValuePair);
|
||||
}
|
||||
if (params != null) {
|
||||
for (String key : params.keySet()) {
|
||||
allNameValuePair.add(new BasicNameValuePair(key, params.get(key)));
|
||||
}
|
||||
}
|
||||
requestBuilder.setEntity(new UrlEncodedFormEntity(allNameValuePair, Charset.forName("utf8")));
|
||||
return requestBuilder;
|
||||
}
|
||||
|
||||
private RequestBuilder addQueryParams(RequestBuilder requestBuilder, Map<String, String> params) {
|
||||
if (params != null) {
|
||||
for (Map.Entry<String, String> entry : params.entrySet()) {
|
||||
requestBuilder.addParameter(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
return requestBuilder;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,199 +1,47 @@
|
||||
package us.codecraft.webmagic.proxy;
|
||||
|
||||
import org.apache.http.HttpHost;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Delayed;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* >>>> Proxy lifecycle
|
||||
|
||||
+----------+ +-----+
|
||||
| last use | | new |
|
||||
+-----+----+ +---+-+
|
||||
| +------+ |
|
||||
+->| init |<--+
|
||||
+--+---+
|
||||
|
|
||||
v
|
||||
+--------+
|
||||
+--->| borrow |
|
||||
| +---+----+
|
||||
| |+------------------+
|
||||
| v
|
||||
| +--------+
|
||||
| | in use | Respone Time
|
||||
| +---+----+
|
||||
| |+------------------+
|
||||
| v
|
||||
| +--------+
|
||||
| | return |
|
||||
| +---+----+
|
||||
| |+-------------------+
|
||||
| v
|
||||
| +-------+ reuse interval
|
||||
| | delay | (delay time)
|
||||
| +---+---+
|
||||
| |+-------------------+
|
||||
| v
|
||||
| +------+
|
||||
| | idle | idle time
|
||||
| +---+--+
|
||||
| |+-------------------+
|
||||
+--------+
|
||||
*/
|
||||
|
||||
/**
|
||||
* Object has these status of lifecycle above.<br>
|
||||
*
|
||||
* @author yxssfxwzy@sina.com <br>
|
||||
* @since 0.5.1
|
||||
* @see SimpleProxyPool
|
||||
*/
|
||||
|
||||
public class Proxy implements Delayed, Serializable {
|
||||
|
||||
private static final long serialVersionUID = 228939737383625551L;
|
||||
public static final int ERROR_403 = 403;
|
||||
public static final int ERROR_404 = 404;
|
||||
public static final int ERROR_BANNED = 10000;// banned by website
|
||||
public static final int ERROR_Proxy = 10001;// the proxy itself failed
|
||||
public static final int SUCCESS = 200;
|
||||
public class Proxy {
|
||||
|
||||
private final HttpHost httpHost;
|
||||
private ProxyHost proxyHost;
|
||||
private String user;
|
||||
private String password;
|
||||
|
||||
|
||||
private int reuseTimeInterval = 1500;// ms
|
||||
private Long canReuseTime = 0L;
|
||||
private Long lastBorrowTime = System.currentTimeMillis();
|
||||
private Long responseTime = 0L;
|
||||
|
||||
private int failedNum = 0;
|
||||
private int successNum = 0;
|
||||
private int borrowNum = 0;
|
||||
|
||||
private List<Integer> failedErrorType = new ArrayList<Integer>();
|
||||
|
||||
public Proxy(HttpHost httpHost, String user, String password) {
|
||||
this.httpHost = httpHost;
|
||||
this.user = user;
|
||||
this.password = password;
|
||||
this.canReuseTime = System.nanoTime() + TimeUnit.NANOSECONDS.convert(reuseTimeInterval, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
public Proxy(HttpHost httpHost, int reuseInterval, String user, String password) {
|
||||
this.httpHost = httpHost;
|
||||
public Proxy(ProxyHost proxyHost, String user, String password) {
|
||||
this.proxyHost = proxyHost;
|
||||
this.user = user;
|
||||
this.password = password;
|
||||
this.canReuseTime = System.nanoTime() + TimeUnit.NANOSECONDS.convert(reuseInterval, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
public int getSuccessNum() {
|
||||
return successNum;
|
||||
}
|
||||
|
||||
public void successNumIncrement(int increment) {
|
||||
this.successNum += increment;
|
||||
}
|
||||
|
||||
public Long getLastUseTime() {
|
||||
return lastBorrowTime;
|
||||
}
|
||||
|
||||
public void setLastBorrowTime(Long lastBorrowTime) {
|
||||
this.lastBorrowTime = lastBorrowTime;
|
||||
}
|
||||
|
||||
public void recordResponse() {
|
||||
this.responseTime = (System.currentTimeMillis() - lastBorrowTime + responseTime) / 2;
|
||||
this.lastBorrowTime = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
public List<Integer> getFailedErrorType() {
|
||||
return failedErrorType;
|
||||
}
|
||||
|
||||
public void setFailedErrorType(List<Integer> failedErrorType) {
|
||||
this.failedErrorType = failedErrorType;
|
||||
}
|
||||
|
||||
public void fail(int failedErrorType) {
|
||||
this.failedNum++;
|
||||
this.failedErrorType.add(failedErrorType);
|
||||
}
|
||||
|
||||
public void setFailedNum(int failedNum) {
|
||||
this.failedNum = failedNum;
|
||||
}
|
||||
|
||||
public int getFailedNum() {
|
||||
return failedNum;
|
||||
}
|
||||
|
||||
public String getFailedType() {
|
||||
String re = "";
|
||||
for (Integer i : this.failedErrorType) {
|
||||
re += i + " . ";
|
||||
}
|
||||
return re;
|
||||
}
|
||||
|
||||
public HttpHost getHttpHost() {
|
||||
return httpHost;
|
||||
}
|
||||
|
||||
public int getReuseTimeInterval() {
|
||||
return reuseTimeInterval;
|
||||
public Proxy(ProxyHost proxyHost) {
|
||||
this.proxyHost = proxyHost;
|
||||
}
|
||||
|
||||
public void setReuseTimeInterval(int reuseTimeInterval) {
|
||||
this.reuseTimeInterval = reuseTimeInterval;
|
||||
this.canReuseTime = System.nanoTime() + TimeUnit.NANOSECONDS.convert(reuseTimeInterval, TimeUnit.MILLISECONDS);
|
||||
|
||||
public ProxyHost getProxyHost() {
|
||||
return proxyHost;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getDelay(TimeUnit unit) {
|
||||
return unit.convert(canReuseTime - System.nanoTime(), TimeUnit.NANOSECONDS);
|
||||
public void setProxyHost(ProxyHost proxyHost) {
|
||||
this.proxyHost = proxyHost;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(Delayed o) {
|
||||
Proxy that = (Proxy) o;
|
||||
return canReuseTime > that.canReuseTime ? 1 : (canReuseTime < that.canReuseTime ? -1 : 0);
|
||||
|
||||
public String getUser() {
|
||||
return user;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
String re = String.format("host: %15s >> %5dms >> success: %-3.2f%% >> borrow: %d", httpHost.getAddress().getHostAddress(), responseTime,
|
||||
successNum * 100.0 / borrowNum, borrowNum);
|
||||
return re;
|
||||
|
||||
public void setUser(String user) {
|
||||
this.user = user;
|
||||
}
|
||||
|
||||
public String getUser()
|
||||
{
|
||||
return user;
|
||||
|
||||
}
|
||||
public String getPassword()
|
||||
{
|
||||
public String getPassword() {
|
||||
return password;
|
||||
|
||||
}
|
||||
|
||||
public void borrowNumIncrement(int increment) {
|
||||
this.borrowNum += increment;
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public int getBorrowNum() {
|
||||
return borrowNum;
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,34 @@
|
||||
package us.codecraft.webmagic.proxy;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com
|
||||
* Date: 17/3/18
|
||||
* Time: 下午12:04
|
||||
*/
|
||||
public class ProxyHost {
|
||||
|
||||
private String host;
|
||||
|
||||
private int port;
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public ProxyHost(String host, int port) {
|
||||
this.host = host;
|
||||
this.port = port;
|
||||
}
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
public int getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
public void setPort(int port) {
|
||||
this.port = port;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,163 @@
|
||||
package us.codecraft.webmagic.proxy;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Delayed;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* >>>> Proxy lifecycle
|
||||
|
||||
+----------+ +-----+
|
||||
| last use | | new |
|
||||
+-----+----+ +---+-+
|
||||
| +------+ |
|
||||
+->| init |<--+
|
||||
+--+---+
|
||||
|
|
||||
v
|
||||
+--------+
|
||||
+--->| borrow |
|
||||
| +---+----+
|
||||
| |+------------------+
|
||||
| v
|
||||
| +--------+
|
||||
| | in use | Respone Time
|
||||
| +---+----+
|
||||
| |+------------------+
|
||||
| v
|
||||
| +--------+
|
||||
| | return |
|
||||
| +---+----+
|
||||
| |+-------------------+
|
||||
| v
|
||||
| +-------+ reuse interval
|
||||
| | delay | (delay time)
|
||||
| +---+---+
|
||||
| |+-------------------+
|
||||
| v
|
||||
| +------+
|
||||
| | idle | idle time
|
||||
| +---+--+
|
||||
| |+-------------------+
|
||||
+--------+
|
||||
*/
|
||||
|
||||
/**
|
||||
* Object has these status of lifecycle above.<br>
|
||||
*
|
||||
* @author yxssfxwzy@sina.com <br>
|
||||
* @since 0.5.1
|
||||
* @see TimerReuseProxyPool
|
||||
*/
|
||||
|
||||
public class TimerReuseProxy extends Proxy implements Delayed, Serializable {
|
||||
|
||||
private static final long serialVersionUID = 228939737383625551L;
|
||||
public static final int ERROR_403 = 403;
|
||||
public static final int ERROR_404 = 404;
|
||||
public static final int ERROR_BANNED = 10000;// banned by website
|
||||
public static final int ERROR_Proxy = 10001;// the proxy itself failed
|
||||
public static final int SUCCESS = 200;
|
||||
|
||||
private int reuseTimeInterval = 1500;// ms
|
||||
private Long canReuseTime = 0L;
|
||||
private Long lastBorrowTime = System.currentTimeMillis();
|
||||
private Long responseTime = 0L;
|
||||
|
||||
private int failedNum = 0;
|
||||
private int successNum = 0;
|
||||
private int borrowNum = 0;
|
||||
|
||||
private List<Integer> failedErrorType = new ArrayList<Integer>();
|
||||
|
||||
public TimerReuseProxy(ProxyHost proxyHost, String user, String password) {
|
||||
super(proxyHost, user, password);
|
||||
}
|
||||
|
||||
public TimerReuseProxy(ProxyHost proxyHost, String user, String password, int reuseTimeInterval) {
|
||||
super(proxyHost, user, password);
|
||||
this.reuseTimeInterval = reuseTimeInterval;
|
||||
}
|
||||
|
||||
public int getSuccessNum() {
|
||||
return successNum;
|
||||
}
|
||||
|
||||
public void successNumIncrement(int increment) {
|
||||
this.successNum += increment;
|
||||
}
|
||||
|
||||
public Long getLastUseTime() {
|
||||
return lastBorrowTime;
|
||||
}
|
||||
|
||||
public void setLastBorrowTime(Long lastBorrowTime) {
|
||||
this.lastBorrowTime = lastBorrowTime;
|
||||
}
|
||||
|
||||
public void recordResponse() {
|
||||
this.responseTime = (System.currentTimeMillis() - lastBorrowTime + responseTime) / 2;
|
||||
this.lastBorrowTime = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
public List<Integer> getFailedErrorType() {
|
||||
return failedErrorType;
|
||||
}
|
||||
|
||||
public void setFailedErrorType(List<Integer> failedErrorType) {
|
||||
this.failedErrorType = failedErrorType;
|
||||
}
|
||||
|
||||
public void fail(int failedErrorType) {
|
||||
this.failedNum++;
|
||||
this.failedErrorType.add(failedErrorType);
|
||||
}
|
||||
|
||||
public void setFailedNum(int failedNum) {
|
||||
this.failedNum = failedNum;
|
||||
}
|
||||
|
||||
public int getFailedNum() {
|
||||
return failedNum;
|
||||
}
|
||||
|
||||
public String getFailedType() {
|
||||
String re = "";
|
||||
for (Integer i : this.failedErrorType) {
|
||||
re += i + " . ";
|
||||
}
|
||||
return re;
|
||||
}
|
||||
|
||||
public int getReuseTimeInterval() {
|
||||
return reuseTimeInterval;
|
||||
}
|
||||
|
||||
public void setReuseTimeInterval(int reuseTimeInterval) {
|
||||
this.reuseTimeInterval = reuseTimeInterval;
|
||||
this.canReuseTime = System.nanoTime() + TimeUnit.NANOSECONDS.convert(reuseTimeInterval, TimeUnit.MILLISECONDS);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getDelay(TimeUnit unit) {
|
||||
return unit.convert(canReuseTime - System.nanoTime(), TimeUnit.NANOSECONDS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(Delayed o) {
|
||||
TimerReuseProxy that = (TimerReuseProxy) o;
|
||||
return canReuseTime > that.canReuseTime ? 1 : (canReuseTime < that.canReuseTime ? -1 : 0);
|
||||
|
||||
}
|
||||
|
||||
public void borrowNumIncrement(int increment) {
|
||||
this.borrowNum += increment;
|
||||
}
|
||||
|
||||
public int getBorrowNum() {
|
||||
return borrowNum;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue