朴素贝叶斯文本分类-在《红楼梦》作者鉴别的应用上（python实现）

朴素贝叶斯算法简单、高效。接下来我们来介绍其如何应用在《红楼梦》作者的鉴别上。

第一步，当然是先得有文本数据，我在网上随便下载了一个txt（当时急着交初稿。。。）。分类肯定是要一个回合一个回合的分，所以我们拿到文本数据后，先进行回合划分。然后就是去标点符号、分词，做词频统计。

 # -*- coding: utf- -*-
 import re
 import jieba
 import string
 import collections as coll
 jieba.load_userdict('E:\\forpython\\红楼梦词汇大全.txt') # 导入搜狗的红楼梦词库

 class textprocesser:
     def __init__(self):
         pass

     # 将小说分成120个章节并单独保存到txt文件中
     def divide_into_chapter(self):
         red=open('E:\\forpython\\红楼梦.txt',encoding='utf-8')
         each_line = red.readline()
         chapter_count =
         chapter_text = ''
         complied_rule = re.compile('第[一二三四五六七八九十百]+回  ')

         while each_line:
             if re.findall(complied_rule,each_line):
                 file_name = 'chap'+str(chapter_count)
                 file_out = open('E:\\forpython\\chapters\\'+file_name+'.txt','a',encoding = 'utf-8')
                 file_out.write(chapter_text)
                 chapter_count +=
                 file_out.close()
                 chapter_text = each_line
             else:
                 chapter_text += each_line

             each_line = red.readline()

         red.close()

     # 对单个章节的分词
     def segmentation(self,text,text_count):
         file_name = 'chap'+str(text_count)+'-words.txt'
         file_out = open('E:\\forpython\\chapter2words\\'+file_name,'a',encoding='utf-8')
         delset = string.punctuation

         line=text.readline()

         while line:
             seg_list = jieba.cut(line,cut_all = False)
             words = " ".join(seg_list)
             words = words.translate(delset) # 去除英文标点
             words = "".join(words.split('\n')) # 去除回车符
             words = self.delCNf(words) # 去除中文标点
             words = re.sub('[ \u3000]+',' ',words) # 去除多余的空格
             file_out.write(words)
             line = text.readline()

         file_out.close()
         text.close()

     # 对所有章节分词
     def do_segmentation(self):
         for loop in range(,):
             file_name = 'chap'+str(loop)+'.txt'
             file_in = open('E:\\forpython\\chapters\\'+file_name,'r',encoding = 'utf-8')

             self.segmentation(file_in,loop)

             file_in.close()

     # 去除中文字符函数
     def delCNf(self,line):
         regex = re.compile('[^\u4e00-\u9fa5a-zA-Z0-9\s]')
         return regex.sub('', line)

     # 去除标点后进行词频统计
     def count_words(self,text,textID):
         line = str(text)
         words = line.split()
         words_dict = coll.Counter(words) # 生成词频字典

         file_name = 'chap'+str(textID)+'-wordcount.txt'
         file_out = open('E:\\forpython\\chapter-wordcount\\'+file_name,'a',encoding = 'utf-8')

         # 排序后写入文本
         sorted_result = sorted(words_dict.items(),key = lambda d:d[],reverse = True)
         for one in sorted_result:
             line = "".join(one[] + '\t' + str(one[]) + '\n')
             file_out.write(line)

         file_out.close()

     def do_wordcount(self):
         for loop in range(,):
             file_name = 'chap'+str(loop)+'-words.txt'
             file_in = open('E:\\forpython\\chapter2words\\'+file_name,'r',encoding = 'utf-8')
             line = file_in.readline()

             text = ''
             while line:
                 text += line
                 line = file_in.readline()
             self.count_words(text,loop)
             file_in.close()

 if __name__ == '__main__':
     processer = textprocesser()
     processer.divide_into_chapter()
     processer.do_segmentation()
     processer.do_wordcount()

文本分类我个人感觉最重要的是选取特征向量，我查阅了相关文献，决定选取五十多个文言虚词和二十多个在120个回合中均出现过的词汇（文言虚词的使用不受情节影响，只与作者写作习惯有关）。下面是生成

特征向量的代码

 # -*- coding: utf- -*-
 import jieba
 import re
 import string
 import collections as coll
 jieba.load_userdict('E:\\forpython\\红楼梦词汇大全.txt') # 导入搜狗的红楼梦词库

 class featureVector:
     def __init__(self):
         pass

      # 去除中文字符函数
     def delCNf(self,line):
         regex = re.compile('[^\u4e00-\u9fa5a-zA-Z0-9\s]')
         return regex.sub('', line)

     # 对整篇文章分词
     def cut_words(self):
         red = open('E:\\forpython\\红楼梦.txt','r',encoding = 'utf-8')
         file_out = open('E:\\forpython\\红楼梦-词.txt','a',encoding = 'utf-8')
         delset = string.punctuation

         line = red.readline()

         while line:
             seg_list = jieba.cut(line,cut_all = False)
             words = ' '.join(seg_list)
             words = words.translate(delset) # 去除英文标点
             words = "".join(words.split('\n')) # 去除回车符
             words = self.delCNf(words) # 去除中文标点
             words = re.sub('[ \u3000]+',' ',words) # 去除多余的空格
             file_out.write(words)
             line = red.readline()

         file_out.close()
         red.close()

     # 统计词频
     def count_words(self):
         data = open('E:\\forpython\\红楼梦-词.txt','r',encoding = 'utf-8')
         line = data.read()
         data.close()
         words = line.split()
         words_dict = coll.Counter(words) # 生成词频字典

         file_out = open('E:\\forpython\\红楼梦-词频.txt','a',encoding = 'utf-8')

         # 排序后写入文本
         sorted_result = sorted(words_dict.items(),key = lambda d:d[],reverse = True)
         for one in sorted_result:
             line = "".join(one[] + '\t' + str(one[]) + '\n')
             file_out.write(line)

         file_out.close()

     def get_featureVector(self):
         # 将分词后的120个章节文本放入一个列表中
         everychapter = []
         for loop in range(,):
             data = open('E:\\forpython\\chapter2words\\chap'+str(loop)+'-words.txt','r',encoding = 'utf-8')
             each_chapter = data.read()
             everychapter.append(each_chapter)
             data.close()

         temp = open('E:\\forpython\\红楼梦-词.txt','r',encoding = 'utf-8')
         word_beg = temp.read()
         word_beg = word_beg.split(' ')
         temp.close()

         # 找出每一个回合都出现的词
         cleanwords = []
         for loop in range(,):
             data = open('E:\\forpython\\chapter2words\\chap'+str(loop)+'-words.txt','r',encoding = 'utf-8')
             words_list = list(set(data.read().split()))
             data.close()
             cleanwords.extend(words_list)

         cleanwords_dict = coll.Counter(cleanwords)

         cleanwords_dict = {k:v for k, v in cleanwords_dict.items() if v >= }

         cleanwords_f = list(cleanwords_dict.keys())

         xuci = open('E:\\forpython\\文言虚词.txt','r',encoding = 'utf-8')
         xuci_list = xuci.read().split()
         xuci.close()
         featureVector = list(set(xuci_list + cleanwords_f))
         featureVector.remove('\ufeff')

         # 写入文本
         file_out = open('E:\\forpython\\红楼梦-特征向量.txt','a',encoding = 'utf-8')
         for one in featureVector:
             line = "".join(one+ '\n')
             file_out.write(line)

         file_out.close()
         return(featureVector)

 if __name__ == '__main__':
     vectorbuilter = featureVector()
     vectorbuilter.cut_words()
     vectorbuilter.count_words()
     vectorbuilter.get_featureVector()

朴素贝叶斯文本分类就是用特征向量的词频作为每个回合的代表（偷个懒，直接截图答辩的ppt）

用特征向量把所有一百二十个回合向量化后，你会得到120×70的一个数组。接下来就简单了。直接挑选训练集，在这我是在前80回中挑选了20至29回标记为第一类（用数字1表示），并将其作为第一类的训练集；在后80回合中挑选了110至119回标记为第二类（用数字2表示），并将其作为第二类的训练集。

 # -*- coding: utf- -*-

 import numpy as np
 from sklearn.naive_bayes import MultinomialNB
 import get_trainset as ts
 x_train = ts.get_train_set().get_all_vector()

 class result:
     def __inti__(self):
         pass

     def have_Xtrainset(self):
         Xtrainset = x_train
         Xtrainset = np.vstack((Xtrainset[:],Xtrainset[:]))
         return(Xtrainset)   

     def as_num(self,x):
         y='{:.10f}'.format(x)
         return(y)

     def built_model(self):
         x_trainset = self.have_Xtrainset()
         y_classset = np.repeat(np.array([,]),[,])

         NBclf = MultinomialNB()
         NBclf.fit(x_trainset,y_classset) # 建立模型

         all_vector = x_train

         result = NBclf.predict(all_vector)
         print('前'+str(len(result[:]))+'回分类结果为：')
         print(result[:])
         print('后'+str(len(result[:]))+'回分类结果为：')
         print(result[:])

         diff_chapter = [,,,,,,,]
         for i in diff_chapter:
             tempr = NBclf.predict_proba(all_vector[i])
             print('第'+str(i+)+'回的分类概率为： ')
             print(str(self.as_num(tempr[][]))+' '+str(self.as_num(tempr[][])))

 if __name__ == '__main__':
     res = result()
     res.built_model()

上面是直接调用了skit-learn的MultinomialNB函数，详细情况我在前一篇中讲过。

得到分类结果：

从最终的分类结果来看，在第82回合左右是有一个比较明显的分界点，这样看来前80回合与后40回合在写作风格上还是有显著的差异的，这个结果和红楼梦学术界的年的推断比较一致。

至于为何在后40回中有8个回合被分到1类中，这8个回合分别是81回、82回、84回、85回、88回、89回、91回还有101回，都是在第80回合附近，这个差异有可能是由于上下文的衔接所导致的，因为本文所使用的《红楼梦》文本是从网上下载得到的，，版本不明，所以也有可能是由于红楼梦的版本所导致的。

代码肯定还有很多可以优化的地方，在这里献丑了。。。。