gensim工具[学习笔记]

平台信息:
PC：ubuntu18.04、i5、anaconda2、cuda9.0、cudnn7.0.5、tensorflow1.10、GTX1060

一、将copy_train.csv文件的内容进行分词，生成process_copy_train.csv文件

 import jieba
 import re
 import os
 import sys
 import gensim
 import sys  

 f1 =  open(u'copy_train.csv')
 for line in f1.readlines():
     with open(u'process_copy_train2.csv','a') as f2:
         seg_list = jieba.cut(line,cut_all=False)
         seg_list = " ".join(seg_list)
         seg_list.encode("utf8")
         seg_list.decode("utf8")
         f2.write(seg_list)
         f2.write("\n")
         #f2.write(line)

 f2.close()
 f1.close()          

二、训练词汇表，并进行测试

 import jieba
 import re
 import os
 import sys
 import gensim
 import sys 

 from gensim.models import word2vec

 reload(sys)
 sys.setdefaultencoding('utf8')  

 sentences=word2vec.Text8Corpus(u'process_copy_train.csv')
 model=word2vec.Word2Vec(sentences, size=50)

 model[u'美元'.decode("utf-8")]
 y2=model.similarity(u"美元", u"美国")
 print(y2)
 y2=model.similarity(u"美元", u"英镑")
 print(y2)
 y2=model.similarity(u"美元", u"美元")
 print(y2)

 for i in model.most_similar(u"银行"):
     print i[0],i[1]

 str4 = model.most_similar(u"银行".decode("utf-8"))

 print str4
 model.save('/tmp/word2vec_model')

 new_model=gensim.models.Word2Vec.load('/tmp/word2vec_model')

测试结果：

 0.21382438
 0.65352416
 1.0
 商业银行 0.724080383778
 券商 0.67235070467
 同业 0.65898835659
 银行业 0.640146613121
 金融机构 0.628186702728
 中资银行 0.624082624912
 流动性 0.589600920677
 中小银行 0.587715625763
 行 0.576077103615
 信贷 0.575850129128