修复一个编码错误

src/main/java/us/codecraft/spider/downloader/HttpClientDownloader.java

@@ -32,7 +32,7 @@ public class HttpClientDownloader implements Downloader {
             int statusCode = httpResponse.getStatusLine().getStatusCode();
             if (site.getAcceptStatCode().contains(statusCode)) {
                 String content = IOUtils.toString(httpResponse.getEntity().getContent(),
-                        site.getEncoding() == null ? site.getEncoding() : httpResponse.getEntity().getContentType().getValue());
+                        site.getEncoding() == null ? httpResponse.getEntity().getContentType().getValue() : site.getEncoding());
                 Page page = new Page();
                 page.setHtml(new Html(UrlUtils.fixAllRelativeHrefs(content, request.getUrl())));
                 page.setUrl(new PlainText(request.getUrl()));

src/main/java/us/codecraft/spider/utils/UrlUtils.java

@@ -91,4 +91,15 @@ public class UrlUtils {
         return stringBuilder.toString();
     }
 
+    private static final Pattern patternForCharset = Pattern.compile("charset=([^\\s;]*)");
+
+    public static String getCharset(String contentType) {
+        Matcher matcher = patternForCharset.matcher(contentType);
+        if (matcher.find()) {
+            return matcher.group(1);
+        } else {
+            return null;
+        }
+    }
+
 }

src/test/java/us/codecraft/spider/SpiderTest.java

@@ -25,14 +25,14 @@ public class SpiderTest {
 
     @Test
     public void testGlobalSpider(){
-        SimplePageProcessor pageProcessor = new SimplePageProcessor("http://2012guang.diandian.com/", "http://2012guang.diandian.com/post/*");
+        SimplePageProcessor pageProcessor = new SimplePageProcessor("http://blog.163.com/", "http://blog.163.com/*/blog/static/*");
+        pageProcessor.getSite().setEncoding("gbk");
         Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor.getSite(),"/data/temp/spider/cache/")).
-                processor(pageProcessor).thread().start();
-        SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://lol.duowan.com/", "http://lol.duowan.com/*.html");
-        Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor2.getSite(),"/data/temp/spider/cache/")).
-                processor(pageProcessor2).run();
+                processor(pageProcessor).run();
+//        SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://lol.duowan.com/", "http://lol.duowan.com/*.html");
+//        Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor2.getSite(),"/data/temp/spider/cache/")).
+//                processor(pageProcessor2).run();
 
-        Spider.me().processor(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*/blog/*")).run();
 
     }
 

src/test/java/us/codecraft/spider/samples/QzoneBlogProcessor.java

@@ -0,0 +1,32 @@
+package us.codecraft.spider.samples;
+
+import us.codecraft.spider.Page;
+import us.codecraft.spider.Site;
+import us.codecraft.spider.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午8:08
+ */
+public class QzoneBlogProcessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        //http://progressdaily.diandian.com/post/2013-01-24/40046867275
+
+        //http://b1.cnc.qzone.qq.com/cgi-bin/blognew/get_abs?hostUin=233017404&uin=233017404&blogType=0&statYear=2013&source=0&statYear=2013&g_tk=291639571&g_tk=291639571&reqInfo=7&pos=0&num=15&source=0&rand=0.46480297949165106
+        // &cateName=&cateHex=&statYear=2013&reqInfo=7&pos=0&num=15&sortType=0&source=0&rand=0.46480297949165106&g_tk=291639571&verbose=1&ref=qzone
+        List<String> requests = page.getHtml().rs("<a[^<>]*href=[\"']{1}(http://17dujingdian\\.com/post/[^#]*?)[\"']{1}").toStrings();
+        page.addTargetRequests(requests);
+        page.putField("title",page.getHtml().x("//div[@id='content']//h2/a"));
+        page.putField("content",page.getHtml().sc());
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("www.diandian.com").setStartUrl("http://17dujingdian.com/").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}