python 中文分词程序实例

import re
import sys

regex=re.compile(r"(?x) (?: [w-]+  | [x80-xff]{3} )")

def init_wordslist(fn="./words.txt"):
    f=open(fn)
    lines=sorted(f.readlines())
    f.close()
    return lines

def words_2_trie(wordslist):
    d={}
    for word in wordslist:
        ref=d
        chars=regex.findall(word)
        for char in chars:
            ref[char]=ref.has_key(char) and ref[char] or {}
            ref=ref[char]
        ref['']=1
    return d

def search_in_trie(chars, trie):
    ref=trie
    index=0
    for char in chars:
        if ref.has_key(char):
            print char,
            ref=ref[char]
            index+=1
        else:
            if index==0:
                index=1
                print char,
            print '*',
            try:
                chars=chars[index:]
                search_in_trie(chars, trie)
            except:
                pass
            break
def main():
    #init
    words=init_wordslist()
    trie=words_2_trie(words)
    #read content
    fn=sys.argv[1]
    string=open(fn).read()
    chars=regex.findall(string)
   
    #do the job
    search_in_trie(chars, trie)

if __name__=='__main__':
    main()

.def FMM(sentence,diction,result = [],maxwordLength = 4,edcode="utf-8"): 
.    i = 0  
    sentence = PreProcess(sentence,edcode)  
  length = len(sentence)  
   while i < length:  
        # find the ascii word   
        tempi=i  
      tok=sentence[i:i+1]  
     while re.search("[0-9A-Za-z/-/+#@_/.]{1}",tok)<>None:  
          i= i+1  
          tok=sentence[i:i+1]  
      if i-tempi>0:  
            result.append(sentence[tempi:i].lower().encode(edcode))  
       # find chinese word   
       left = len(sentence[i:])  
       if left == 1:  
           """go to 4 step over the FMM"""  
          """should we add the last one? Yes, if not blank"""  
           if sentence[i:] <> " ":  
              result.append(sentence[i:].encode(edcode))  
          return result  
        m = min(left,maxwordLength)  
         
       for j in xrange(m,0,-1):  
         leftword = sentence[i:j+i].encode(edcode)  
     #   print leftword.decode(edcode)   
           if LookUp(leftword,diction):  
              # find the left word in dictionary  
              # it's the right one   
              i = j+i  
                result.append(leftword)  
              break  
          elif j == 1:  
               """only one word, add into result, if not blank"""  
               if leftword.decode(edcode) <> " ":  
                    result.append(leftword)  
              i = i+1  
         else:  
              continue  
   return result 
def LookUp(word,dictionary):  
    if dictionary.has_key(word):  
       return True  
    return False 
def ConvertGBKtoUTF(sentence):  
   return sentence.decode('gbk').encode('utf-8') 
import re
def PreProcess(sentence,edcode="utf-8"):
    sentence = sentence.decode(edcode)
    sentence=re.sub(u"[。，,！……!《》<>/"':：？/?、/|“”‘’；]"," ",sentence)
    return sentence

def FMM(sentence,diction,result = [],maxwordLength = 4,edcode="utf-8"):
    i = 0
    sentence = PreProcess(sentence,edcode)
    length = len(sentence)
    while i < length:
        # find the ascii word
        tempi=i
        tok=sentence[i:i+1]
        while re.search("[0-9A-Za-z/-/+#@_/.]{1}",tok)<>None:
            i= i+1
            tok=sentence[i:i+1]
        if i-tempi>0:
            result.append(sentence[tempi:i].lower().encode(edcode))
        # find chinese word
        left = len(sentence[i:])
        if left == 1:
            """go to 4 step over the FMM"""
            """should we add the last one? Yes, if not blank"""
            if sentence[i:] <> " ":
                result.append(sentence[i:].encode(edcode))
            return result
        m = min(left,maxwordLength)
       
        for j in xrange(m,0,-1):
            leftword = sentence[i:j+i].encode(edcode)
         #   print leftword.decode(edcode)
            if LookUp(leftword,diction):
                # find the left word in dictionary
                # it's the right one
                i = j+i
                result.append(leftword)
                break
            elif j == 1:
                """only one word, add into result, if not blank"""
                if leftword.decode(edcode) <> " ":
                    result.append(leftword)
                i = i+1
            else:
                continue
    return result
def LookUp(word,dictionary):
    if dictionary.has_key(word):
        return True
    return False
def ConvertGBKtoUTF(sentence):
    return sentence.decode('gbk').encode('utf-8')

文本测试代码：

[c-sharp] view plaincopyprint?
.test = open("test.txt","r") 
.for line in test: 
.    s = FMM(CovertGBKtoUTF(line),dictions) 
.    for i in s: 
.        print i.decode("utf-8")