liblinear使用总结

liblinear是libsvm的线性核的改进版本，专门适用于百万数据量的分类。正好适用于我这次数据挖掘的实验。

liblinear用法和libsvm很相似，我是用的是.exe文件，利用python的subprocess向控制台发送命令即可完成本次试验。

其中核心两句即

train train.txt

predict test.txt train.txt.model output.txt

由于是线性核，没有设置参数c、g

对于50W篇文章模型训练仅需340秒，50W篇文章的预测仅需6秒

 from subprocess import *
 import time

 time = time.time

 start_time = time()
 print("训练")
 cmd = "train train.txt"
 Popen(cmd, shell = True, stdout = PIPE).communicate()
 print("训练结束",str(time() - start_time))

 start_time = time()
 print("预测")
 cmd = "predict test.txt train.txt.model output.txt"
 Popen(cmd, shell = True).communicate()
 print("预测结束",str(time() - start_time))

 #进行统计
 #读测试集真实label
 start_time = time()
 print("统计")
 test_filename = "test.txt"
 f = open(test_filename,"r",encoding = "utf-8")
 real_class = []
 for line in f:
     real_class.append(line[0])

 #总样本
 total_sample = len(real_class)

 #读预测结果label
 predict_filename = "output.txt"
 f_predict = open(predict_filename,"r",encoding = "utf-8")
 s = f_predict.read()
 predict_class = s.split()

 #对预测正确的文章进行计数
 T = 0
 for real, predict in zip(real_class,predict_class):
     if int(real) == int(predict):
         T += 1
 accuracy  = T / total_sample * 100
 print("正确率 为", str(accuracy) + "%")

 # class_label = ["0","1","2","3","4","5","6","7","8","9"]
 num_to_cate = {0:"it",1:"体育",2:"军事",3:"金融",4:"健康",5:"汽车",6:"房产",7:"文化",8:"教育",9:"娱乐"}

 class_label = ["it","体育","军事","金融","健康","汽车","房产","文化","教育","娱乐"]

 predict_precision = dict.fromkeys(class_label,1.0)
 predict_true = dict.fromkeys(class_label,1.0)

 predict_recall = dict.fromkeys(class_label,1.0)
 predict_F = dict.fromkeys(class_label,0.0)
 # print(str(predict_precision))
 # print(str(predict_precision))
 # print(str(predict_recall))
 # print(str(predict_true))
 mat = dict.fromkeys(class_label,{})
 for k,v in mat.items():
     mat[k] = dict.fromkeys(class_label,0)

 # print(str(mat))

 for real, predict in zip(real_class,predict_class):
     real = int(real)
     predict = int(predict)
     # print(num_to_cate[real])
     # print(num_to_cate[predict])
     mat[num_to_cate[real]][num_to_cate[predict]] += 1
     predict_precision[num_to_cate[predict]] += 1
     predict_recall[num_to_cate[real]] += 1

     if int(real) == int(predict):
         predict_true[num_to_cate[predict]] += 1

 # print(str(predict_precision))
 # print(str(predict_recall))
 # print(str(predict_true))

 #输出混淆矩阵
 for k, v in mat.items():
     print(k + ":" + str(v))

 #计算精确率和召回率
 for x in range(len(class_label)):
     # x =  str(x)
     predict_precision[num_to_cate[x]] = predict_true[num_to_cate[x]] / predict_precision[num_to_cate[x]]
     predict_recall[num_to_cate[x]] = predict_true[num_to_cate[x]] / predict_recall[num_to_cate[x]]

 # print(str(predict_precision))
 # print(str(predict_recall))
 # print(str(predict_true))

 #计算F测度
 for x in range(len(class_label)):
     # x = str(x)
     predict_F[num_to_cate[x]] = 2 * predict_recall[num_to_cate[x]] * predict_precision[num_to_cate[x]] / (predict_precision[num_to_cate[x]] + predict_recall[num_to_cate[x]])

 print("统计结束",str(time() - start_time))
 print("精确率为",str(predict_precision))
 print("召回率为",str(predict_recall))
 print("F测度为",str(predict_F))

 print("保存结果")
 final_result_filename = "./finalresult.txt"
 f = open(final_result_filename,"w",encoding = "utf-8")
 for k, v in mat.items():
     f.write(k + ":" + str(v) + "\n")

 f.write("\n")
 f.write("正确率为" + str(accuracy) + "%" + "\n\n")
 f.write("精确率为" + str(predict_precision) + "\n\n")
 f.write("召回率为" + str(predict_recall) + "\n\n")
 f.write("F测度为" + str(predict_F) + "\n\n")
 print("保存结果结束")

 # cate_to_num = {"it":0,"体育":1,"军事":2,"华人":3,"国内":4,"国际":5,"房产":6,"文娱":7,"社会":8,"财经":9}
 # num_to_cate = {0:"it",1:"体育",2:"军事",3:"华人",4:"国内",5:"国际",6:"房产",7:"文娱",8:"社会",9:"财经"}