python 安装nltk，使用（英文分词处理，词干化等）（Green VPN）

安装pip命令之后：

sudo pip install -U pyyaml nltk

import nltk
nltk.download()

等待ing

目前访问不了，故使用Green VPN

http://www.evergreenvpn.com/ubuntu-pptp-vpn-setting/

nltk使用

http://www.cnblogs.com/yuxc/archive/2011/08/29/2157415.html

http://blog.csdn.net/huyoo/article/details/12188573

http://www.52nlp.cn/tag/nltk

1.空格进行英文分词.split（python自带）

>>> slower
'we all like the book'
>>> ssplit = slower.split()
>>> ssplit
['we', 'all', 'like', 'the', 'book']
>>> 

或

>>> import nltk
>>> s = u"我们都Like the book"
>>> m = [word for word in nltk.tokenize.word_tokenize(s)]
>>> for word in m:
...     print word
...
我们都Like
the
book

或

>>> tokens = nltk.word_tokenize(s)
>>> tokens
[u'\u6211\u4eec\u90fdLike', u'the', u'book']
>>> for word in tokens
  File "<stdin>", line 1
    for word in tokens
                     ^
SyntaxError: invalid syntax
>>> for word in tokens:
...     print word
...
我们都Like
the
book

2.词性标注

>>> tagged = nltk.pos_tag(tokens)
>>> for word in tagged:
...     print word
...
(u'\u6211\u4eec\u90fdLike', 'IN')
(u'the', 'DT')
(u'book', 'NN')
>>> 

3.句法分析

>>> entities= nltk.chunk.ne_chunk(tagged)
>>> entities
Tree('S', [(u'\u6211\u4eec\u90fdLike', 'IN'), (u'the', 'DT'), (u'book', 'NN')])
>>> 

---------------------------------------------------------------------------------------------------------------------------------------------------------

4.转换为小写（Python自带）

>>> s
'We all like the book'
>>> slower = s.lower()
>>> slower
'we all like the book'
>>> 

5.空格进行英文分词.split（python自带）

>>> slower
'we all like the book'
>>> ssplit = slower.split()
>>> ssplit
['we', 'all', 'like', 'the', 'book']
>>> 

6.标号与单词分离

>>> s
'we all like the book,it\xe2\x80\x98s so interesting.'
>>> s = 'we all like the book, it is so interesting.'
>>> wordtoken = nltk.tokenize.word_tokenize(s)
>>> wordtoken
['we', 'all', 'like', 'the', 'book', ',', 'it', 'is', 'so', 'interesting', '.']
>>> wordtoken = nltk.word_tokenize(s)
>>> wordtoken
['we', 'all', 'like', 'the', 'book', ',', 'it', 'is', 'so', 'interesting', '.']
>>> wordsplit = s.split()
>>> wordsplit
['we', 'all', 'like', 'the', 'book,', 'it', 'is', 'so', 'interesting.']
>>> 

7.去停用词（nltk自带127个英文停用词）

>>> wordEngStop = nltk.corpus.stopwords.words('english')
>>> wordEngStop
[u'i', u'me', u'my', u'myself', u'we', u'our', u'ours', u'ourselves', u'you', u'your', u'yours', u'yourself', u'yourselves', u'he', u'him', u'his', u'himself', u'she', u'her', u'hers', u'herself', u'it', u'its', u'itself', u'they', u'them', u'their', u'theirs', u'themselves', u'what', u'which', u'who', u'whom', u'this', u'that', u'these', u'those', u'am', u'is', u'are', u'was', u'were', u'be', u'been', u'being', u'have', u'has', u'had', u'having', u'do', u'does', u'did', u'doing', u'a', u'an', u'the', u'and', u'but', u'if', u'or', u'because', u'as', u'until', u'while', u'of', u'at', u'by', u'for', u'with', u'about', u'against', u'between', u'into', u'through', u'during', u'before', u'after', u'above', u'below', u'to', u'from', u'up', u'down', u'in', u'out', u'on', u'off', u'over', u'under', u'again', u'further', u'then', u'once', u'here', u'there', u'when', u'where', u'why', u'how', u'all', u'any', u'both', u'each', u'few', u'more', u'most', u'other', u'some', u'such', u'no', u'nor', u'not', u'only', u'own', u'same', u'so', u'than', u'too', u'very', u's', u't', u'can', u'will', u'just', u'don', u'should', u'now']
>>> len(wordEngStop)
127
>>> 

>>> len(wordEngStop)
127
>>> s
'we all like the book, it is so interesting.'
>>> wordtoken
['we', 'all', 'like', 'the', 'book', ',', 'it', 'is', 'so', 'interesting', '.']
>>> for word in wordtoken:
...     if not word in wordEngStop:
...             print word
...
like
book
,
interesting
.
>>> 

8.去标点符号

>>> english_punctuations = [',', '.', ':', ';', '?', '(', ')', '[', ']', '!', '@', '#', '%', '$', '*']
>>> wordtoken
['we', 'all', 'like', 'the', 'book', ',', 'it', 'is', 'so', 'interesting', '.']
>>> for word in wordtoken:
...     if not word in english_punctuations:
...             print word
...
we
all
like
the
book
it
is
so
interesting
>>>

9.词干化

“我们对这些英文单词词干化（Stemming)，NLTK提供了好几个相关工具接口可供选择，具体参考这个页面: http://nltk.org/api/nltk.stem.html , 可选的工具包括Lancaster Stemmer, Porter Stemmer等知名的英文Stemmer。这里我们使用LancasterStemmer:”  来自：我爱自然语言处理  http://www.52nlp.cn/%E5%A6%82%E4%BD%95%E8%AE%A1%E7%AE%97%E4%B8%A4%E4%B8%AA%E6%96%87%E6%A1%A3%E7%9A%84%E7%9B%B8%E4%BC%BC%E5%BA%A6%E4%B8%89

http://lutaf.com/212.htm  词干化的主流方法

http://blog.sina.com.cn/s/blog_6d65717d0100z4hu.html

>>> from nltk.stem.lancaster import LancasterStemmer
>>> st = LancasterStemmer()
>>> wordtoken
['we', 'all', 'like', 'the', 'book', ',', 'it', 'is', 'so', 'interesting', '.']
>>> st.stem(wordtoken)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/dist-packages/nltk/stem/lancaster.py", line 195, in stem
AttributeError: 'list' object has no attribute 'lower'
>>> for word in wordtoken:
...     print st.stem(word)
...
we
al
lik
the
book
,
it
is
so
interest
.
>>>

两者各有优缺点

>>> from nltk.stem import PorterStemmer
>>> wordtoken
['we', 'all', 'like', 'the', 'book', ',', 'it', 'is', 'so', 'interesting', '.']
>>> PorterStemmer().stem(wordtoken)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/dist-packages/nltk/stem/porter.py", line 632, in stem
AttributeError: 'list' object has no attribute 'lower'
>>> PorterStemmer().stem('all')
u'all'
>>> for word in wordtoken:
...     print PorterStemmer().stem(word)
...
we
all
like
the
book
,
it
is
so
interest
.
>>> PorterStemmer().stem("better")
u'better'
>>> PorterStemmer().stem("supplies")
u'suppli'
>>> st.stem('supplies')
u'supply'
>>> 

# -*- coding:utf8 -*-
import nltk
import os

wordEngStop = nltk.corpus.stopwords.words('english')
english_punctuations = [',', '.', ':', ';', '?', '(', ')', '[', ']', '!', '@', '#', '%', '$', '*','=','abstract=', '{', '}']
porterStem=nltk.stem.PorterStemmer()
lancasterStem=nltk.stem.lancaster.LancasterStemmer()

fin = open('/home/xdj/myOutput.txt', 'r')
fout  = open('/home/xdj/myOutputLancasterStemmer.txt','w')
for eachLine in fin:
        eachLine = eachLine.lower().decode('utf-8', 'ignore') #小写
        tokens = nltk.word_tokenize(eachLine)                 #分词（与标点分开）
        wordLine = ''
        for word in tokens:
            if not word in english_punctuations:          #去标点
                if not word in wordEngStop:          #去停用词
                    #word = porterStem.stem(word)
                    word = lancasterStem.stem(word)
                    wordLine+=word+' '
        fout.write(wordLine.encode('utf-8')+'\n')
fin.close()
fout.close()