update lucene pipeline

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -228,8 +228,10 @@ public class Spider implements Runnable, Task {
         }
         pageProcessor.process(page);
         addRequest(page);
-        for (Pipeline pipeline : pipelines) {
+        if (!page.getResultItems().isSkip()){
-            pipeline.process(page.getResultItems(), this);
+            for (Pipeline pipeline : pipelines) {
+                pipeline.process(page.getResultItems(), this);
+            }
         }
         sleep(site.getSleepTime());
     }

webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ConsolePipeline.java

@@ -15,9 +15,6 @@ public class ConsolePipeline implements  Pipeline{
 
     @Override
     public void process(ResultItems resultItems,Task task) {
-        if (resultItems.isSkip()){
-            return;
-        }
         System.out.println("get page: "+resultItems.getRequest().getUrl());
         for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
             if (entry.getValue() instanceof Iterable) {

webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java

@@ -47,9 +47,6 @@ public class FilePipeline implements Pipeline {
         if (!file.exists()) {
             file.mkdirs();
         }
-        if (resultItems.isSkip()) {
-            return;
-        }
         try {
             PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl()) + ".html"));
             printWriter.println("url:\t" + resultItems.getRequest().getUrl());

webmagic-plugin/webmagic-lucene/src/main/java/us/codecraft/webmagic/pipeline/LucenePipeline.java

@@ -3,20 +3,26 @@ package us.codecraft.webmagic.pipeline;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.queryparser.classic.QueryParser;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.Version;
 import us.codecraft.webmagic.ResultItems;
 import us.codecraft.webmagic.Task;
 
-import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 /**
  * @author yihua.huang@dianping.com <br>
@@ -24,41 +30,64 @@ import java.io.File;
  * Time: 下午2:11 <br>
  */
 public class LucenePipeline implements Pipeline {
-    @Override
-    public void process(ResultItems resultItems, Task task) {
-        try {
 
-        } catch (Exception e) {
+    private Directory directory;
 
-        }
+    private IndexWriter indexWriter;
-    }
+
+    private Analyzer analyzer;
 
-    public static void main(String[] args) throws Exception {
+    private void init() throws IOException {
-        Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_44);
+        analyzer = new StandardAnalyzer(Version.LUCENE_44);
-//        Directory directory = new RAMDirectory();
+        directory = new RAMDirectory();
-        // To store an index on disk, use this instead:
-        Directory directory = FSDirectory.open(new File("/data/webmagic/www.guoxue123.cn/"));
         IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_44, analyzer);
-        IndexWriter iwriter = new IndexWriter(directory, config);
+        indexWriter = new IndexWriter(directory, config);
-        Document doc = new Document();
+        indexWriter.close();
-//        String text = "This is the text to be indexed.";
+    }
-//        doc.add(new Field("fieldname", text, TextField.TYPE_STORED));
+
-//        iwriter.addDocument(doc);
+    public LucenePipeline() {
-        iwriter.close();
+        try {
+            init();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
 
-        // Now search the index:
+    public List<Document> search(String fieldName, String value) throws IOException, ParseException {
+        List<Document> documents = new ArrayList<Document>();
         DirectoryReader ireader = DirectoryReader.open(directory);
         IndexSearcher isearcher = new IndexSearcher(ireader);
         // Parse a simple query that searches for "text":
-        QueryParser parser = new QueryParser(Version.LUCENE_44, "fieldname", analyzer);
+        QueryParser parser = new QueryParser(Version.LUCENE_44, fieldName, analyzer);
-        Query query = parser.parse("经典");
+        Query query = parser.parse(value);
         ScoreDoc[] hits = isearcher.search(query, null, 1000).scoreDocs;
         // Iterate through the results:
         for (int i = 0; i < hits.length; i++) {
             Document hitDoc = isearcher.doc(hits[i].doc);
-            System.out.println(hitDoc);
+            documents.add(hitDoc);
         }
         ireader.close();
         directory.close();
+        return documents;
+    }
+
+    @Override
+    public void process(ResultItems resultItems, Task task) {
+        if (resultItems.isSkip()){
+            return;
+        }
+        Document doc = new Document();
+        Map<String,Object> all = resultItems.getAll();
+        if (all==null){
+            return;
+        }
+        for (Map.Entry<String, Object> objectEntry : all.entrySet()) {
+            doc.add(new Field(objectEntry.getKey(), objectEntry.getValue().toString(), TextField.TYPE_STORED));
+        }
+        try {
+            indexWriter.addDocument(doc);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
     }
 }

webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java

@@ -29,9 +29,6 @@ class ModelPipeline implements Pipeline {
 
     @Override
     public void process(ResultItems resultItems, Task task) {
-        if (resultItems.isSkip()) {
-            return;
-        }
         for (Map.Entry<Class, PageModelPipeline> classPageModelPipelineEntry : pageModelPipelines.entrySet()) {
             Object o = resultItems.get(classPageModelPipelineEntry.getKey().getCanonicalName());
             if (o != null) {

webmagic-samples/src/main/java/us/codecraft/webmagic/pipeline/FreemarkerPipeline.java

@@ -40,9 +40,6 @@ public class FreemarkerPipeline implements Pipeline {
 
     @Override
     public void process(ResultItems resultItems, Task task) {
-        if (resultItems.isSkip()) {
-            return;
-        }
         String path = this.path + "" + task.getUUID() + "/";
         File file = new File(path);
         if (!file.exists()) {