python & mysql 操作（compare_sum_fee）

【1】源码

工作中涉及的python工具源码实现

费用比较工具源码：

 #!/usr/bin/python3
 #coding = utf-8
 import time, sqlite3, datetime, threading, pymysql, time, logging, os, contextlib, sys, xlrd, xlwt
 import psutil, py_compile
 from xlutils.copy import copy
 import queue
 db_mq = queue.Queue()

 if True :
     @contextlib.contextmanager
     def billing():
         try:
             if db_mq.empty():
                 conn = pymysql.connect(host = ', db = 'test', charset = 'utf8')
                 conn.autocommit(True)
             else:
                 conn = db_mq.get_nowait()
                 conn.ping(reconnect = True)
             yield conn.cursor()
         finally:
             conn.cursor().close()
             db_mq.put(conn)
 else :
     @contextlib.contextmanager
     def billing():
         try:
             if db_mq.empty():
                 conn = pymysql.connect(host = '127.0.0.1', port = 3306, user = 'root', passwd = '123456', db = 'billing', charset = 'utf8')
                 conn.autocommit(True)
             else:
                 conn = db_mq.get_nowait()
                 conn.ping(reconnect = True)
             yield conn.cursor()
         finally:
             conn.cursor().close()
             db_mq.put(conn)

  # 限制运行数量为1 防止数据冲突
 def isRunning() :
     count = 0
     status = False
     for i in psutil.pids():
         if psutil.Process(i).name() == 'compare_sum_fee_20190412.py' or 'compare_sum_fee_20190412.pyc' in psutil.Process(i).cmdline() :
             count += 1
             if count > 1 :
                 status = True
                 break
     if os.path.exists('/home/work/compare_sum_fee_20190412.py') :
         py_compile.compile('/home/work/compare_sum_fee_20190412.py', '/home/work/compare_sum_fee_20190412.pyc')
     os.system('chmod 777 /home/work/compare_sum_fee_20190412.pyc')
     return status

 # -------------------------------------------------------------
 # (1)
 cycleid = '
 # (2)
 isCreateLog = False
 # (3)是否打印调试日志
 needPrint = False
 # (4)
 needCreateTable = True
 # (5)
 tableSuffix = ''
 # ----------------------------------------------------------------

 # 创建费用比较表
 def create_table_compareFee() :
     dropTable = ''' DROP TABLE IF EXISTS compare_fee_%s; '''
     createTable = ''' CREATE TABLE `compare_fee_%s` (
         `id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '记录流水号',
         `product_id` VARCHAR(32) DEFAULT NULL COMMENT '产品编号',
         `np_id` VARCHAR(36) DEFAULT NULL COMMENT '运营商编号',
         `np_sum_duration` INT(11) DEFAULT NULL COMMENT '运营商话单结算总时长',
         `np_total_times` BIGINT(11) DEFAULT NULL COMMENT '运营商话单总数',
         `np_sum_qty` INT(11) DEFAULT NULL COMMENT '运营商话单总结算量（按公司费率）',
         `np_sum_fee` INT(11) DEFAULT NULL COMMENT '运营商话单总结算费（按公司费率）',
         `un_sum_duration` INT(11) DEFAULT NULL COMMENT '平台话单计费总时长',
         `un_total_times` BIGINT(11) DEFAULT NULL COMMENT '平台话单总数',
         `un_sum_fee` INT(11) DEFAULT NULL COMMENT '平台话单计费总费用（分）',
         `sum_np_fee` INT(11) DEFAULT NULL COMMENT '运营商话单总结算费（按运营商费率）',
         `sum_original_fee` INT(11) DEFAULT NULL COMMENT '原始结算费总和（运营商提供）',
         `reserve_00` VARCHAR(16) DEFAULT NULL COMMENT '扩展字段1',
         `reserve_01` VARCHAR(16) DEFAULT NULL COMMENT '扩展字段2',
         `reserve_02` VARCHAR(32) DEFAULT NULL COMMENT '扩展字段3',
         PRIMARY KEY (`id`),
         KEY `idx_np_id` (`np_id`),
         KEY `idx_product_id` (`product_id`)
         ) ENGINE=INNODB DEFAULT CHARSET=utf8 COMMENT='费用对比详单表'
     '''
     dropSql = dropTable % (cycleid + tableSuffix)
     createSql = createTable % (cycleid + tableSuffix)
     if needCreateTable :
         with billing() as billingDB :
             print('[1]drop create Sql :: %s' % dropSql)
             billingDB.execute(dropSql)
             print('[2]create table Sql :: %s' % createSql)
             billingDB.execute(createSql)
             sql = 'UPDATE acct_offlinefile SET state = 4 WHERE cycle_id = \'' + cycleid + '\';'
             billingDB.execute(sql)
             print('[3]update acct_offlinefile set state = 4 all np_id')

 def get_callingid_code(calling_id) :
     ' :
         calling_id = calling_id[2:(len(calling_id))]

     ' and len(calling_id) == 11 :
         sql = "select area_code from cfg_area_prefix where prefix = '" + calling_id[0:7] + "' limit 1;"
         with billing() as billingDB :
             billingDB.execute(sql)
             results = billingDB.fetchall()
             if results :
                 for record in results :
                     return record['area_code']
             else :
                 return ''

     ' and len(calling_id) >= 4 :
         ' :
             return calling_id[0:3]
         else :
             return calling_id[0:4]
     else :
         is_len = len(calling_id) > 0 and len(calling_id) < 7 and len(calling_id) != 4
         value = int(calling_id)
         is_reserve = len(calling_id) == 4 and value > 0 and (value < 5000 or value > 5999)
         ' or is_len or is_reserve :
             '
         else :
             return ''

 def analysis_cdr_calltype(np_id) :
     sql = 'SELECT calling_id FROM np_cdr_' + cycleid + ' WHERE np_id = \'' + np_id + '\' AND ISNULL(ani_code) GROUP BY calling_id;'
     print('----[1]start : get_callingid_code ......')
     #print('select_calling_id_sql : ' + sql)
     results = ()
     with billing() as billingDB :
         billingDB.execute(sql)
         results = billingDB.fetchall()
     sum_count = 0
     count = 0
     if results :
         sql_const = "UPDATE np_cdr_%s SET ani_code = %s WHERE np_id = %s AND calling_id = %s;"
         param = []
         for record in results :
             cur_code = get_callingid_code(record[0])
             if len(cur_code) != 0 :
                 value = []
                 value.append(int(cycleid))
                 value.append(cur_code)
                 value.append(np_id)
                 value.append(record[0])
                 param.append(value)
                 count += 1
             if 5000 == count :
                 #print('areacode_param5000 : ', param)
                 with billing() as billingDB :
                     billingDB.executemany(sql_const, param)
                 sum_count += count
                 print('------update ani_code count : %d ' % sum_count)
                 count = 0
                 param.clear()
         if count > 0 :
             #print('areacode_param less 5000 : ', param)
             with billing() as billingDB :
                 billingDB.executemany(sql_const, param)
             sum_count += count
             print('------update ani_code sum_count : %d ' % sum_count)
             count = 0
             param.clear()
             #print('id : %d | calling_id : %s | ani_code : %s | affectrow : %d' % (record[0], str(record[1]), cur_code, billingDB.rowcount))
     print('----[1]end : get_callingid_code ......')

     # 分析呼叫类型
     count = 0
     sum_count = 0
     print('----[2]start : analysis_cdr_calltype ......')
     sql = 'SELECT product_id FROM np_cdr_' + cycleid + ' WHERE np_id = \'' + np_id + '\' AND ISNULL(reserve_00) GROUP BY product_id;'
     #print('select_product_id_sql : ' + sql)
     results = ()
     with billing() as billingDB :
         billingDB.execute(sql)
         results = billingDB.fetchall()
     if results :
         sql_result = 'UPDATE np_cdr_%s SET reserve_00 = IF(ani_code = %s, \'市话\', \'长话\') WHERE np_id = %s AND product_id = %s;'
         param = []
         for record in results :
             sql = 'SELECT areacode FROM cfg_productid_areacode WHERE product_id = \'' + record[0] + '\';'
             codeResult = ()
             with billing() as billingDB :
                 billingDB.execute(sql)
                 codeResult = billingDB.fetchall()
             value = []
             value.append(int(cycleid))
             if not codeResult :
                 value.append('')
             else :
                 for codeRecord in codeResult :
                     value.append(codeRecord[0])
             value.append(np_id)
             value.append(record[0])
             param.append(value)
             count += 1
             if 5000 == count :
                 #print('calltype_param5000 : ', param)
                 with billing() as billingDB :
                     billingDB.executemany(sql_result, param)
                 sum_count += count
                 print('------update reserve_00 product count : %d ' % sum_count)
                 count = 0
                 param.clear()
         if count > 0 :
             #print('calltype_param less 5000 : ', param)
             with billing() as billingDB :
                 billingDB.executemany(sql_result, param)
             sum_count += count
             print('------update reserve_00 product sum_count : %d ' % sum_count)
             count = 0
             param.clear()
     print('----[2]end : analysis_cdr_calltype ......')

 # 搜集批价参数
 def collect_union_fee(value, curProductID, unionType, npCdrType) :
     cur_fee_unit = 0
     cur_fee_rate = 0
     sql = 'SELECT fee_unit, fee_rate FROM cfg_fee_rate WHERE product_id = \'' + curProductID + '\' AND call_type = \'' + unionType + '\';'
     unionRate = ()
     with billing() as billingDB :
         billingDB.execute(sql)
         unionRate = billingDB.fetchall()
     if unionRate :
         for rateRecord in unionRate :
             cur_fee_unit = rateRecord[0]
             cur_fee_rate = rateRecord[1]
     element = []
     element.append(int(cycleid))
     element.append(int(cur_fee_unit))
     element.append(int(cur_fee_rate))
     element.append(value)
     element.append(curProductID)
     element.append(npCdrType)
     return element

 # 计算公司费用
 def calc_union_fee(value) :
     print('----[1]start : calc_union_fee ......')
     sql = 'SELECT product_id FROM np_cdr_' + cycleid
     sql += ' WHERE np_id = \'' + value + '\''
     sql += ' AND ISNULL(service_qty) GROUP BY product_id;'
     results = ()
     total_count = 0
     with billing() as billingDB :
         billingDB.execute(sql)
         results = billingDB.fetchall()
         total_count = billingDB.rowcount
     sql_const = ''' UPDATE np_cdr_%s
         SET fee_unit = %s, fee_rate = %s,
         service_qty = IF(fee_unit = 0, 0, (FLOOR((duration + fee_unit - 1) / fee_unit))),
         service_fee = IF(fee_unit = 0, 0, (FLOOR((duration + fee_unit - 1) / fee_unit) * fee_rate))
         WHERE np_id = %s AND product_id = %s AND reserve_00 = %s; '''
     param = []
     count = 0
     sum_count = 0
     if total_count > 0 and results :
         for record in results :
             curProductID = str(record[0])
             param.append(collect_union_fee(value, curProductID, ', '长话'))
             param.append(collect_union_fee(value, curProductID, ', '市话'))
             count += 1
             #print('total : %d | current : %d | product : %s' % (total_count, sum_count, curProductID))
             if 3000 == count :
                 #print('fee_param 3000 : ', param)
                 sum_count += count
                 with billing() as billingDB :
                     billingDB.executemany(sql_const, param)
                 print('------calc union fee progress | total : %d | current : %d ' % (total_count, sum_count))
                 count = 0
                 param.clear()
         if count > 0 :
             #print('fee_param less 3000 : ', param)
             sum_count += count
             with billing() as billingDB :
                 billingDB.executemany(sql_const, param)
             print('------calc union fee progress | total : %d | current : %d ' % (total_count, sum_count))
             count = 0
             param.clear()

     print('----[1]end : calc_union_fee ......')

 def do_by_npid(state) :
     # 恢复统计前原状态值
     if 2 == state :
         sql = 'UPDATE acct_offlinefile SET state = 2 WHERE cycle_id = \'' + cycleid + '\';'
         with billing() as billingDB :
             billingDB.execute(sql)
             effectrow = billingDB.rowcount
             if effectrow > 0 :
                 print('----OK. success to update set state = 2 where all np_id')
         return
     # 执行过程
     sql = 'SELECT np_id, np_name FROM acct_offlinefile WHERE cycle_id = \'' + cycleid + '\' AND state = ' + str(state - 1) + ' GROUP BY np_id;'
     results = ()
     np_count = 0
     with billing() as billingDB :
         billingDB.execute(sql)
         results = billingDB.fetchall()
         np_count = billingDB.rowcount
     if results :
         count = 0
         for np in results :
             count += 1
             print('--Total : %d || No. %d || np_id : %s || np_name : %s' % (np_count, count, np[0], np[1]))
             if 3 == state :
                 analysis_cdr_calltype(np[0])
             elif 4 == state :
                 calc_union_fee(np[0])
             else :
                 amount_sum_fee(np[0])
             sql = 'UPDATE acct_offlinefile SET state = ' + str(state) + ' WHERE np_id = \'' + np[0] + '\';'
             with billing() as billingDB :
                 billingDB.execute(sql)
                 if billingDB.rowcount > 0 :
                     print('----[OK]success to finish update set state = ' + str(state) + ' where np_name = %s' % np[1])

 def analysis_calltype_by_npid() :
     do_by_npid(3)

 def calc_np_cdr_fee_by_npid() :
     do_by_npid(4)

 def amount_sum_fee_by_npid() :
     do_by_npid(5)

 def restore_initial_state() :
     do_by_npid(2)

 def amount_sum_fee(np_id) :
     print('----[1]start : amount sum fee ......')
     # 查询所有账号
     sql = 'SELECT product_id FROM np_cdr_' + cycleid + ' WHERE np_id = \'' + np_id +  '\' GROUP BY product_id;'
     results = ()
     with billing() as billingDB :
         billingDB.execute(sql)
         results = billingDB.fetchall()
     count = 0
     sumCount = 0
     productFeeDict = {}
     sql_const = 'insert into compare_fee_' + cycleid + tableSuffix + ' (product_id, np_id, np_sum_duration, np_total_times, np_sum_qty, np_sum_fee, '
     sql_const += 'un_sum_duration, un_total_times, un_sum_fee, sum_np_fee, sum_original_fee) values '

     if results :
         for record in results :
             curProductID = str(record[0])
             count += 1
             sumCount += 1
             cur_np_sum_fee = 0
             cur_np_sum_original_fee = 0
             productFeeDict[curProductID] = '\'' + np_id + '\''

             sql = 'SELECT product_id, SUM(duration) AS np_sum_duration, COUNT(id) AS np_sum_cdrCnt, SUM(service_qty) AS np_service_qty,'
             sql += ' SUM(service_fee) AS np_service_fee, SUM(np_fee) AS np_sum_fee, SUM(original_fee) AS np_sum_original_fee '
             sql += ' FROM np_cdr_' + cycleid + ' WHERE np_id = \'' + np_id + '\' AND product_id = \'' + curProductID + '\';'
             np_results = ()
             with billing() as billingDB :
                 billingDB.execute(sql)
                 np_results = billingDB.fetchall()
             if np_results :
                 for record in np_results :
                     productFeeDict[curProductID] = productFeeDict.get(curProductID) + ',' + str(record[1]) + ',' + str(record[2]) + ',' + str(record[3]) + ',' + str(record[4])
                     cur_np_sum_fee = str(record[5])
                     cur_np_sum_original_fee = str(record[6])

             sql = 'SELECT product_id, SUM(total_duration) AS un_sum_duration, SUM(total_times) AS un_sum_cdrCnt,'
             sql += ' SUM(total_fee) AS un_sum_fee FROM dat_bill_' + cycleid + ' WHERE call_type & 4097 > 0 AND product_id = \'' + curProductID + '\';'
             un_results = ()
             with billing() as billingDB :
                 billingDB.execute(sql)
                 un_results = billingDB.fetchall()
             if un_results :
                 for record in un_results :
                     value = {'duration':0, 'cdrCnt':0, 'sumfee':0}
                     if record[0] :
                        value['duration'] = record[1]
                        value['cdrCnt'] = record[2]
                        value['sumfee'] = record[3]
                     productFeeDict[curProductID] = productFeeDict.get(curProductID) + ',' + str(value['duration']) + ',' + str(value['cdrCnt']) + ',' + str(value['sumfee'])

             productFeeDict[curProductID] = productFeeDict.get(curProductID) + ',' + cur_np_sum_fee + ',' + cur_np_sum_original_fee
             # 写入表中
             if 5000 == count :
                 # 写入表中
                 sql = sql_const
                 for key, value in productFeeDict.items() :
                     sql += ' (\'' + key + '\',' + value + '),'
                 sql = sql[:-1]
                 sql = sql.replace('None', 'NULL')
                 with billing() as billingDB :
                     billingDB.execute(sql)
                     print("------success insert into product_id count :: %d" % sumCount)
                 count = 0
                 productFeeDict.clear()
         # 最后一次写入
         if count > 0 :
             sql = sql_const
             for key, value in productFeeDict.items() :
                 sql += ' (\'' + key + '\',' + value + '),'
             sql = sql[:-1]
             sql = sql.replace('None', 'NULL')
             with billing() as billingDB :
                 billingDB.execute(sql)
                 print('------success insert into product_id sumCount :: %d ' % sumCount)
             count = 0
             productFeeDict.clear()
         print('----[1]end : amount sum fee ......')

 def set_bigger_flag() :
     print('--[1]start : set bigger flag ......')
     with billing() as billingDB :
         sql = 'UPDATE compare_fee_' + cycleid + tableSuffix + ' SET reserve_00 = IF(np_sum_fee > un_sum_fee, 1, 0);'
         billingDB.execute(sql)
     sql = 'SELECT count(id) AS biggerCnt FROM compare_fee_' + cycleid + tableSuffix + ' WHERE reserve_00 = 1;'
     with billing() as billingDB :
         billingDB.execute(sql)
         results = billingDB.fetchall()
         if results :
             for record in results :
                 print('----[npcdr_union_sum_fee > uncdr_sum_fee] total count : %d' % record[0])
     print('--[2]end : set bigger flag ......')

 def pickUpNpCdr() :
     sql = 'SELECT product_id FROM compare_fee_' + cycleid + tableSuffix + ' WHERE reserve_00 = 1;'
     with billing() as billingDB :
         billingDB.execute(sql)
         results = billingDB.fetchall()
         productid_count = 0
         sum_npsdr_count = 0
         calc_productid_count = 0
         if results :
             calc_productid_count = len(results)
             for record in results :
                 cur_productId = str(record[0])
                 sql = ''' INSERT INTO view_dat_sdr_%s(call_id, calling_id, called_id, origin_calling_id, origin_called_id,
                     custom_id, contract_id, product_id, call_type, access_time, ring_time,
                     bridge_time, start_time, end_time, source, record_file, extension,
                     transfer_num, voice_mail, satisfaction_result, last_dtmf, gen_time, ani_code,
                     dnis_code, ani_desc, dnis_desc, result, duration, service_qty,
                     service_fee, discount_fee, fee_rate, fee_unit, np_id, np_qty,
                     np_fee, np_rate, np_unit, flownode_name, flownode_id, cdr_state, insert_time) select
                     CONCAT('npcdr', id), calling_id, called_id, '', '',
                     '', '', product_id, '1', start_time, null,
                     null, start_time, end_time, source, '', '',
                     '', '', '', '', insert_time, ani_code,
                     '', '', '', 1, duration, service_qty,
                     service_fee, 0, fee_rate, fee_unit, np_id, np_qty,
                     np_fee, np_rate, np_unit, '', '', '01', sysdate()
                     from np_cdr_%s where product_id = '%s' '''
                 sql = sql % (cycleid, cycleid, cur_productId)
                 billingDB.execute(sql)
                 productid_count += 1
                 sum_npsdr_count += billingDB.rowcount
                 print('NO. %d | curProductId : %s | npSdrCnt : %d' % (productid_count, cur_productId, billingDB.rowcount))
         else :
             print('pickUp select productids is empty.')
             return

         sql = 'SELECT COUNT(id) AS cnt FROM np_cdr_' + cycleid
         sql += ' WHERE product_id IN (SELECT product_id FROM compare_fee_' + cycleid + tableSuffix + ' WHERE reserve_00 = 1);'
         billingDB.execute(sql)
         results = billingDB.fetchall()
         calc_total_npcdrCnt = 0
         if results :
             for record in results :
                 calc_total_npcdrCnt = record[0]
         print('------- OK. Finish pickup npcdr success. Result --------')
         result_id_count = 'real_productid_count : ' + str(calc_productid_count)
         result_id_count += ' || pickup_productid_count : ' + str(productid_count) + ' || result : '
         if calc_productid_count == productid_count :
             print(result_id_count + 'True')
         else :
             print(result_id_count + 'False')

         result_cdr_count = 'real_total_npcdrCnt : ' + str(calc_total_npcdrCnt)
         result_cdr_count += ' || pickup_npcdr_count : ' + str(sum_npsdr_count) + ' || result : '
         if calc_total_npcdrCnt == sum_npsdr_count :
             print(result_cdr_count + 'True')
         else :
             print(result_cdr_count + 'False')
         print('-----------------------End------------------------------')

 def set_cdr_valid_flag() :
     print('--start set valid flag .....')
     sql = 'SELECT product_id FROM compare_fee_' + cycleid + tableSuffix + ' WHERE reserve_00 = 1;'
     with billing() as billingDB :
         billingDB.execute(sql)
         results = billingDB.fetchall()
         if results :
             for record in results :
                 cur_productId = str(record[0])
                 sql = 'UPDATE view_dat_sdr_' + cycleid + ' SET reserve_00 = 1 WHERE call_id NOT LIKE \'%npcdr%\' AND product_id = \'' + cur_productId + '\';'
                 billingDB.execute(sql)
                 sql = 'UPDATE view_dat_sdr_' + cycleid + ' SET reserve_00 = 2 WHERE call_id LIKE \'%npcdr%\' AND product_id = \'' + cur_productId + '\';'
                 billingDB.execute(sql)
         sql = 'UPDATE view_dat_sdr_' + cycleid + ' SET reserve_00 = 0 WHERE ISNULL(reserve_00);'
         billingDB.execute(sql)
     print('--end set valid flag ......')

 if __name__ == '__main__' :
     if isRunning() :
         print('只允许运行一个实例')
         sys.exit(0)
     else :
         stdout_backup = None
         log_file = None
         if isCreateLog :
             # make a copy of original stdout route# make a copy of original stdout route
             stdout_backup = sys.stdout
             # define the log file that receives your log info
             log_file = open('message.log', 'w')
             # redirect print output to log file
             sys.stdout = log_file
         begin = datetime.datetime.now()

         do_step_dict = {'step1': True, 'step2': True, 'step3': True, 'step4': True, 'step5': True, 'step6': False, 'step7': False}
         # 1.
         if do_step_dict['step1'] :
             analysis_calltype_by_npid()
         # 2.
         if do_step_dict['step2'] :
             calc_np_cdr_fee_by_npid()
         # 3.
         if do_step_dict['step3'] :
             create_table_compareFee()
         # 4.
         if do_step_dict['step4'] :
             amount_sum_fee_by_npid()
         # 5.
         if do_step_dict['step5'] :
             set_bigger_flag()
         # 6.
         if do_step_dict['step6'] :
             pickUpNpCdr()
         # 7.
         if do_step_dict['step7'] :
             set_cdr_valid_flag()

         end = datetime.datetime.now()
         print('cost total time(s) :: %d ' % ((end - begin).seconds))

         if isCreateLog :
             # close open log file
             log_file.close()
             # restore the output to initial pattern
             sys.stdout = stdout_backup

应用的接口，可以通过代码中一一查看。

Good Good Study, Day Day Up.

顺序 选择 循环 总结