NLP相似度之tf-idf计算

当然，在学习过程中也是参考了很多其他的资料，代码都是一行一行敲出来的。

一、将多个文件合并成一个文件，避免频繁的打开和关闭

 import sys

 for line in sys.stdin:
     ss = line.strip().split('\t')
     file_name = ss[0].strip()
     file_context = ss[1].strip()
     word_list = file_context.split(' ')

     word_set = set()
     for word in word_list:
         word_set.add(word)

     for word in word_set:
         print '\t'.join([word, ''])

执行命令：就可以得到合并后的文件啦！！！

python convert.py input_tfidf_dir/ > merge_files.data 

tf-idf计算流程图：

二 、计算IDF的值：

map阶段：读取每一行

 import sys

 for line in sys.stdin:
     ss = line.strip().split('\t')
     file_name = ss[0].strip()
     file_context = ss[1].strip()
     word_list = file_context.split(' ')

     word_set = set()
     for word in word_list:
         word_set.add(word)

     for word in word_set:
         print '\t'.join([word, ''])

reduce阶段：

 import sys
 import math

 current_word = None
 doc_cnt = 508
 count_pool = []
 sum = 0

 for line in sys.stdin:
     ss = line.strip().split('\t')
     if len(ss) != 2:
         continue

     word, val = ss
     if current_word == None:
         current_word = word
     if current_word != word:
         for count in count_pool:
             sum += count

         idf_score = math.log(float(doc_cnt) / (float(sum) + 1))
         print '\t'.join([current_word, str(idf_score)])

         current_word = word
         count_pool = []
         sum = 0

     count_pool.append((int(val)))

 for count in count_pool:
     sum += count

 idf_score = math.log(float(doc_cnt) / (float(sum) + 1))
 print '\t'.join([current_word, str(idf_score)])

三、计算TF的值：

 # 计算tf
 # 读取合并后的数据
 # 执行命令 cat merge_files.data | python map_tf.py mapper_func idf.data

 import sys

 word_dict = {}
 idf_dict = {}

 # 读取计算的idf数据文件
 def read_idf_file_func(idf_file_fd):
     with open() as fd:
         for line in fd:
             ss = line.strip().split('\t')
             if len(ss) != 2:
                 continue
             token = ss[0].strip()
             idf_score = ss[1].strip()
             idf_dict[token] = float(idf_score)
     return idf_dict

 # cat merge_files.data | python map_tf.py mapper_func
 def mapper_func(idf_file_fd):
     idf_dict = read_idf_file_func(idf_file_fd)
     # 标准输入
     for line in sys.stdin:
         ss = line.strip().split('\t')
         file_name = ss[0].strip()
         file_context = ss[1].strip()
         word_list = file_context.split(' ')

         for word in word_list:
             if word not in word_dict:
                 word_dict[word] = 1
             else:
                 word_dict[word] += 1

         for k,v in word_dict.item():
             if k not in idf_dict:
                 continue
             print(file_name,k,v,idf_file_fd[k])
             print(k,v)

 if __name__ == "__main__":
     module = sys.modules[__name__]
     func = getattr(module, sys.argv[1])
     args = None
     if len(sys.argv) > 1:
         args = sys.argv[2:]
     func(*args)