dm_analysis

# -*- coding: utf-8-*- #
import sys
import os
import io
import json
reload(sys)
sys.setdefaultencoding('utf-8')

def unicode_convert(input):
    if isinstance(input, dict):
        return {unicode_convert(key): unicode_convert(value) for key, value in input.iteritems()}
    elif isinstance(input, list):
        return [unicode_convert(element) for element in input]
    elif isinstance(input, unicode):
        return input.encode('utf-8')
    else:
        return input

filter=["Dm.log.wf"] #设置过滤后的文件类型 当然可以设置多个类型

def all_path(dirname):

    result = []#所有的文件

    for maindir, subdir, file_name_list in os.walk(dirname):

        # print("1:",maindir) #当前主目录
        # print("2:",subdir) #当前主目录下的所有目录
        # print("3:",file_name_list)  #当前主目录下的所有文件

        for filename in file_name_list:
            apath = os.path.join(maindir, filename)#合并成一个完整路径
            ext = os.path.splitext(apath)[0]  # 获取文件后缀 [0]获取的是除了文件名以外的内容
	    ext = ext.split('/')
	    ext = ext[5]

            if ext in filter:
                result.append(apath)

    return result

if __name__ == "__main__":
	print sys.getdefaultencoding()
	result = all_path("/home/work/yangq/dm_analysis")
	for file_log in result:
		print "*****************************************"
		print file_log
		f = open(file_log, "r")
		f_new = open("/home/work/yangq/dm_analysis/result", "a+")
		f_new_error = open("/home/work/yangq/dm_analysis/error", "a+")
		haproxy_list = f.readlines()
		dm_statistics="dm_statistics"
                f_new.write(file_log + "\n")
                f_new.write("###################################################################" + "\n")
                f_new_error.write(file_log + "\n")
                f_new_error.write("###################################################################" + "\n")
		for line in haproxy_list:
			if dm_statistics in line:
				#print sys._getframe().f_lineno
				#print line
				strlist = line.split('dm_statistics: ')
				sss = strlist[1]
				#test2 = unicode_convert(json.loads(sss))
				try:
					test2 = json.loads(sss.decode('utf-8'))
				except UnicodeDecodeError as w:
					f_new_error.write(line)
                                	f_new_error.write("有特殊自符" + "\n")
                               		continue
				except ValueError:
					f_new_error.write(line)
                                	f_new_error.write("ValueError" + "\n")
                                	continue

				f_new.write("query:"  + str(test2['req']['query']) + "    ")
				if 'recommend_word' in test2['latency']:
					f_new.write("recommend_word:"  + str(test2['latency']['recommend_word']) + "    ")
				if 'um' in test2['latency']:
					f_new.write("um:"  + str(test2['latency']['um']) + "    ")
				if 'se' in test2['latency']:
					f_new.write("se:"  + str(test2['latency']['se']) + "    ")
				if 'ia' in test2['latency']:
					f_new.write("ia:"  + str(test2['latency']['ia']) + "    ")
				if 'ir' in test2['latency']:
					f_new.write("ir:"  + str(test2['latency']['ir']) + "    ")
				if 'dm' in test2['latency']:
					f_new.write("dm:"  + str(test2['latency']['dm']) + "    ")
				if 'chat' in test2['latency']:
					f_new.write("chat:"  + str(test2['latency']['chat']) + "    ")
				if 'ACTION_CALCULATE.calc' in test2['detail_latency']:
					f_new.write("ACTION_CALCULATE.calc: " + str(test2['detail_latency']['nli']['ACTION_CALCULATE.calc']) + "\n")
				f_new.write("\n")
				#try:
				#	f_new.write("query:"  + str(test2['req']['query']) + "    " +  \
				#        	"recommend_word:"  + str(test2['latency']['recommend_word']) +  \
                		#        	"    " "um:"  + str(test2['latency']['um']) + "    " +  \
                		#        	"se:"  + str(test2['latency']['se']) + "    " +  \
                		#        	"ia:"  + str(test2['latency']['ia']) + "    " +  \
                		#        	"ir:"  + str(test2['latency']['ir']) + "    " +  \
               			#        	"dm:"  + str(test2['latency']['dm']) + "    " +  \
                		#        	"chat:"  + str(test2['latency']['chat']) + "    " +  \
                		#        	"ACTION_CALCULATE.calc: " + str(test2['detail_latency']['nli']['ACTION_CALCULATE.calc']) + "\n")
				#except KeyError as e:
				#	f_new_error.write(line)
				#	f_new_error.write("losing key:" + str(e) + "\n")
				#	continue
		f.close()
		f_new.close()
		f_new_error.close()
	pass