Merge pull request #174 from fukuball/master

讓 jieba 可以切換 idf 語料庫及 stop words 語料庫

extra_dict/stop_words.txt

@@ -0,0 +1,51 @@
+the
+of
+is
+and
+to
+in
+that
+we
+for
+an
+are
+by
+be
+as
+on
+with
+can
+if
+from
+which
+you
+it
+this
+then
+at
+have
+all
+not
+one
+has
+or
+that
+的
+了
+和
+是
+就
+都
+而
+及
+與
+著
+或
+一個
+沒有
+我們
+你們
+妳們
+他們
+她們
+是否

jieba/analyse/__init__.py

@@ -1,3 +1,4 @@
+#encoding=utf-8
 import jieba
 import os
 try:
@@ -6,26 +7,53 @@ except ImportError:
     pass
 
 _curpath = os.path.normpath( os.path.join( os.getcwd(), os.path.dirname(__file__) )  )
-f_name = os.path.join(_curpath,"idf.txt")
+abs_path = os.path.join(_curpath, "idf.txt")
-content = open(f_name,'rb').read().decode('utf-8')
 
+IDF_DICTIONARY = abs_path
+STOP_WORDS = set([
+    "the","of","is","and","to","in","that","we","for","an","are","by","be","as","on","with","can","if","from","which","you","it","this","then","at","have","all","not","one","has","or","that"
+])
+
+def set_idf_path(idf_path):
+    global IDF_DICTIONARY
+    abs_path = os.path.normpath( os.path.join( os.getcwd(), idf_path )  )
+    if not os.path.exists(abs_path):
+        raise Exception("jieba: path does not exist:" + abs_path)
+    IDF_DICTIONARY = abs_path
+    return
+
+def get_idf(abs_path):
+    content = open(abs_path,'rb').read().decode('utf-8')
     idf_freq = {}
     lines = content.split('\n')
     for line in lines:
         word,freq = line.split(' ')
         idf_freq[word] = float(freq)
-
     median_idf = sorted(idf_freq.values())[len(idf_freq)/2]
-stop_words= set([
+    return idf_freq, median_idf
-"the","of","is","and","to","in","that","we","for","an","are","by","be","as","on","with","can","if","from","which","you","it","this","then","at","have","all","not","one","has","or","that"
+
-])
+def set_stop_words(stop_words_path):
+    global STOP_WORDS
+    abs_path = os.path.normpath( os.path.join( os.getcwd(), stop_words_path )  )
+    if not os.path.exists(abs_path):
+        raise Exception("jieba: path does not exist:" + abs_path)
+    content = open(abs_path,'rb').read().decode('utf-8')
+    lines = content.split('\n')
+    for line in lines:
+        STOP_WORDS.add(line)
+    return
 
 def extract_tags(sentence,topK=20):
+    global IDF_DICTIONARY
+    global STOP_WORDS
+
+    idf_freq, median_idf = get_idf(IDF_DICTIONARY)
+
     words = jieba.cut(sentence)
     freq = {}
     for w in words:
         if len(w.strip())<2: continue
-        if w.lower() in stop_words: continue
+        if w.lower() in STOP_WORDS: continue
         freq[w]=freq.get(w,0.0)+1.0
     total = sum(freq.values())
     freq = [(k,v/total) for k,v in freq.iteritems()]

test/extract_tags_idfpath.py

@@ -0,0 +1,32 @@
+import sys
+sys.path.append('../')
+
+import jieba
+import jieba.analyse
+from optparse import OptionParser
+
+USAGE = "usage:    python extract_tags_idfpath.py [file name] -k [top k]"
+
+parser = OptionParser(USAGE)
+parser.add_option("-k", dest="topK")
+opt, args = parser.parse_args()
+
+
+if len(args) < 1:
+    print USAGE
+    sys.exit(1)
+
+file_name = args[0]
+
+if opt.topK is None:
+    topK = 10
+else:
+    topK = int(opt.topK)
+
+content = open(file_name, 'rb').read()
+
+jieba.analyse.set_idf_path("../extra_dict/idf.txt.big");
+
+tags = jieba.analyse.extract_tags(content, topK=topK)
+
+print ",".join(tags)

test/extract_tags_stop_words.py

@@ -0,0 +1,33 @@
+import sys
+sys.path.append('../')
+
+import jieba
+import jieba.analyse
+from optparse import OptionParser
+
+USAGE = "usage:    python extract_tags_stop_words.py [file name] -k [top k]"
+
+parser = OptionParser(USAGE)
+parser.add_option("-k", dest="topK")
+opt, args = parser.parse_args()
+
+
+if len(args) < 1:
+    print USAGE
+    sys.exit(1)
+
+file_name = args[0]
+
+if opt.topK is None:
+    topK = 10
+else:
+    topK = int(opt.topK)
+
+content = open(file_name, 'rb').read()
+
+jieba.analyse.set_stop_words("../extra_dict/stop_words.txt")
+jieba.analyse.set_idf_path("../extra_dict/idf.txt.big");
+
+tags = jieba.analyse.extract_tags(content, topK=topK)
+
+print ",".join(tags)