todo bugfix

pull/88/head^2
yihua.huang 12 years ago
parent 583a0eba8c
commit 160a149b05

@ -482,7 +482,7 @@ public class Spider implements Runnable, Task {
/**
* Exit when complete. <br/>
* True: exit when all url of the site is downloaded. <br/>
* False: not exit until call stop manually.<br/>
* False: not exit until call stop() manually.<br/>
*
* @param exitWhenComplete
* @return

@ -90,11 +90,15 @@ public class HttpClientDownloader implements Downloader {
HttpClient httpClient = getHttpClientPool().getClient(site);
try {
HttpGet httpGet = new HttpGet(request.getUrl());
if (headers!=null){
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
httpGet.addHeader(headerEntry.getKey(),headerEntry.getValue());
}
}
if (!httpGet.containsHeader("Accept-Encoding")) {
httpGet.addHeader("Accept-Encoding", "gzip");
}
HttpResponse httpResponse = null;
int tried = 0;
boolean retry;
@ -168,6 +172,7 @@ public class HttpClientDownloader implements Downloader {
HeaderElement[] codecs = ceheader.getElements();
for (HeaderElement codec : codecs) {
if (codec.getName().equalsIgnoreCase("gzip")) {
//todo bugfix
httpResponse.setEntity(
new GzipDecompressingEntity(httpResponse.getEntity()));
}

Loading…
Cancel
Save