# encoding: utf-8
import requests
import time
from Crypto.Cipher import AES
import base64
import hashlib
import json
from queue import Queue
import threading
import xlrd
import yaml
import logging.config
import os
import csv class TokenHandler(object): appid = 'appidappidappid'
secret = 'secretsecretsecretsecret' def __init__(self):
pass def get_token(self):
url = 'http://1.1.1.1/api/token'
current_time = str(int(time.time()))
params = {
'appid' : TokenHandler.appid,
'secret': TokenHandler.secret,
'time' : current_time,
'sign' : ''
}
list_sign_text = []
#请求参数除sign外按key正序排序,并拼接成key=value形式的字符串
for sorted_key in sorted(params.keys()):
if sorted_key != 'sign':
list_sign_text.append(sorted_key + '=' + params.get(sorted_key))
#两个key=value之间用||连接
sign_text = '||'.join(list_sign_text)
logging.info('sign_text: ' + sign_text)
#给sign_text做sha256,然后再放回params里面
sha256_x = hashlib.sha256()
sha256_x.update(sign_text.encode('utf-8'))
sign_text_sha = sha256_x.hexdigest()
logging.info(sign_text_sha)
params['sign'] = sign_text_sha
logging.info('params: ' + str(params))
#发送请求
res = requests.post(url=url,params=params)
response_text = res.content.decode('unicode_escape')
logging.info(response_text)
res_dict = json.loads(response_text)
token = res_dict.get('data').get('token')
logging.info(token)
return token def logout(self):
pass class QueryData(object):
"""
user_info_queue队列的消费者,
result_queue队列的生产者
当user_info_queue队列为空时往result_queue队列塞入结束标记
completed_user用于存放已处理过的用户,如果已处理则跳过
"""
appid = 'appidappidappid'
secret = 'secretsecretsecretsecret'
appkey = 'appkeyappkeyappkey'
iv = 'appkeyappkeyappkey'
token = ''
completed_user = []
# user_info_queue = Queue()
# result_queue = Queue() def __init__(self):
pass def do_query(self):
while not Queues.user_info_queue.empty():
user_info_dict = Queues.user_info_queue.get()
name = user_info_dict.get('name')
phone = user_info_dict.get('phone')
idcard = user_info_dict.get('idcard')
if idcard not in QueryData.completed_user:
url = 'http://1.1.1.1/api/package/b'
current_time = str(int(time.time()))
params = {
'token': QueryData.token,
'name': Encryptor.encrypt(name,QueryData.appkey,QueryData.iv),
'phone': Encryptor.encrypt(phone,QueryData.appkey,QueryData.iv),
'id_card':Encryptor.encrypt(idcard,QueryData.appkey,QueryData.iv),
'order_id': 'a123',
'time': current_time,
'sign': ''
}
list_sign_text = []
# 请求参数除sign和token外按key正序排序,并拼接成key=value形式的字符串
for sorted_key in sorted(params.keys()):
if sorted_key != 'sign' and sorted_key != 'token':
list_sign_text.append(sorted_key + '=' + params.get(sorted_key))
# 两个key=value之间用||连接
sign_text = '||'.join(list_sign_text)
logging.info('sign_text: ' + sign_text)
sha256_x = hashlib.sha256()
sha256_x.update(sign_text.encode('utf-8'))
sign_text_sha = sha256_x.hexdigest()
logging.info(sign_text_sha)
params['sign'] = sign_text_sha
logging.info('params: ' + str(params))
res = requests.post(url=url, params=params)
response_text = res.content.decode('unicode_escape')
logging.info(response_text)
res_dict = json.loads(response_text)
#token过期刷新token
if res_dict['errno'] == 106:
QueryData.token = TokenHandler().get_token()
if res_dict['errno'] == 0:
cipher_data = res_dict.get('data')
plaintext_data = Encryptor.decrypt(cipher_data,QueryData.appkey,QueryData.iv)
res_plaintext_data_dict = plaintext_data.decode("unicode_escape")
logging.info(res_plaintext_data_dict)
res_plaintext_data_dict['idcard'] =idcard
#将结果组装成字典塞到result队列
logging.info(res_plaintext_data_dict)
Queues.result_queue.put(res_plaintext_data_dict)
else:
logging.info(("请注意,这个身份证号查询失败了",idcard))
error_dict_res = {}
error_dict_res['idcard'] = idcard
error_dict_res['response'] = res_dict
err_headers = list(error_dict_res.keys())
logging.info(("headers", err_headers))
if not os.path.exists('./error_idcard.csv'):
with open('error_idcard.csv', mode='x',encoding='utf-8', newline='') as error_idcard_csvfile_cw:
dict_writer = csv.DictWriter(error_idcard_csvfile_cw, err_headers)
dict_writer.writeheader()
with open('error_idcard.csv', mode='a',encoding='utf-8', newline='') as error_idcard_csvfile_a:
dict_writer = csv.DictWriter(error_idcard_csvfile_a, err_headers)
dict_writer.writerow(error_dict_res)
else:
logging.info(idcard + "已经查询过了,结果在result.csv中,跳过。。。")
logging.info('user_info_queue is empty!')
#如果user_info队列为空则塞个标志进去
Queues.result_queue.put({'is_finished':True}) class ResultWriter(object):
"""
result_queue消费者,负责将结果写入文件,并统计结束标记,如果结束标记大于等于生产者个数则结束文件写入
"""
# result_queue = Queue()
def __init__(self):
pass def do_write(self):
finished_thread_counter = 0
sample_data = {"idcard": "", "is_high_risk_user": "无",
"last_visit_dt": "2018-09-21 16:12:20", "30d_overdue_cnt": "无", "his_overdue_amt": "较高",
"last_overdue_dt": "2018-09-21", "last_overdue_amt": "较低", "curr_overdue_amt": "无",
"curr_overdue_days": "无", "first_overdue_dt": "2018-08-17", "first_overdue_amt": "高",
"last_repay_tm": "2018-09-21 16:24:27", "repay_times": "高", "curr_debt_product_cnt": "无",
"total_in_order_cnt": "高", "total_in_order_amt": "极高"}
headers = list(sample_data.keys())
logging.info(("headers", headers))
if not os.path.exists('./result.csv'):
with open('result.csv', mode='x',encoding='utf-8', newline='') as result_csvfile_x:
dict_writer = csv.DictWriter(result_csvfile_x, headers)
dict_writer.writeheader()
with open('result.csv', mode='a',encoding='utf-8', newline='') as ResultWriter_a:
dict_writer = csv.DictWriter(ResultWriter_a, headers)
while 1:
if Queues.result_queue.empty():
time.sleep(1)
else:
res_dict = Queues.result_queue.get()
if 'is_finished' in res_dict:
finished_thread_counter += 1
else:
dict_writer.writerow(res_dict)
logging.info('finished_thread_counter:' + str(finished_thread_counter))
if finished_thread_counter >= 10:
break class Encryptor(object):
@staticmethod
def encrypt(text, key, iv):
if type(text) is str:
text = text.encode(encoding='utf_8', errors='strict')
if type(key) is str:
key = key.encode(encoding='utf_8', errors='strict')
if type(iv) is str:
iv = iv.encode(encoding='utf_8', errors='strict')
if len(text) % 16:
text += (chr(16 - len(text) % 16) * (16 - len(text) % 16)).encode(encoding='utf_8', errors='strict')
text = base64.b64encode(s=AES.new(key, mode=AES.MODE_CBC, IV=iv).encrypt(text),
altchars=None).decode(encoding='utf_8', errors='strict')
return text @staticmethod
def decrypt(cipher_text, key, iv):
if type(key) is str:
key = key.encode(encoding='utf-8', errors='strict')
if type(iv) is str:
iv = iv.encode(encoding='utf-8', errors='strict')
if type(cipher_text) is str:
cipher_text_bytes = base64.b64decode(cipher_text)
# todo aes解密
plaintext_bytes = AES.new(key, mode=AES.MODE_CBC, IV=iv).decrypt(cipher_text_bytes)
# todo 去填充字节
for i in range(1, 17):
plaintext_bytes = plaintext_bytes.rstrip(chr(i).encode(encoding='utf-8', errors='strict'))
plaintext = plaintext_bytes.decode(encoding='utf-8', errors='strict')
return plaintext class QueryThread(threading.Thread):
def __init__(self,name):
threading.Thread.__init__(self)
self.name = name def run(self):
logging.info(self.name + 'start...')
QueryData().do_query()
logging.info(self.name + 'completed!') class WriteResultThread(threading.Thread):
def __init__(self,name):
threading.Thread.__init__(self)
self.name = name def run(self):
logging.info(self.name + 'start...')
ResultWriter().do_write()
logging.info(self.name + 'completed!') class Queues(object):
user_info_queue = Queue()
result_queue = Queue() def setup_logging(default_path = "log_config.yaml",default_level = logging.INFO,env_key = "LOG_CFG"):
path = default_path
value = os.getenv(env_key,None)
if value:
path = value
if os.path.exists(path):
with open(path,"r") as f:
config = yaml.load(f)
logging.config.dictConfig(config)
else:
logging.basicConfig(level = default_level) if __name__ == '__main__':
"""运行前先安装pycryptodome==3.7.0
pip install pycryptodome
"""
setup_logging()
test = TokenHandler()
QueryData.token = test.get_token()
user_info_queue = Queue(maxsize=10001)
result_queue = Queue(maxsize=10001) #todo 吧excel中数据读出来组装成字典放到队列user_info_queue中
book = xlrd.open_workbook('testdata_1.xlsx')
sheet = book.sheet_by_index(0) # 根据顺序获取sheet
for i in range(1,sheet.nrows ):
user_info_dict = {}
if len(sheet.cell(i,0).value) > 0 :
user_info_dict['name'] = sheet.cell(i,0).value.strip()
user_info_dict['idcard'] = sheet.cell(i,1).value.strip()
user_info_dict['phone'] = str(int(sheet.cell(i,2).value)).strip()
logging.info(user_info_dict)
user_info_queue.put(user_info_dict) #todo 把result.csv中的idcard内容读取出来放到QueryData.completed_user
sample_data = {"idcard": "", "is_high_risk_user": "无",
"last_visit_dt": "2018-09-21 16:12:20", "30d_overdue_cnt": "无", "his_overdue_amt": "较高",
"last_overdue_dt": "2018-09-21", "last_overdue_amt": "较低", "curr_overdue_amt": "无",
"curr_overdue_days": "无", "first_overdue_dt": "2018-08-17", "first_overdue_amt": "高",
"last_repay_tm": "2018-09-21 16:24:27", "repay_times": "高", "curr_debt_product_cnt": "无",
"total_in_order_cnt": "高", "total_in_order_amt": "极高"}
headers = list(sample_data.keys())
logging.info(("headers", headers))
if not os.path.exists('./result.csv'):
with open('result.csv', mode='x', encoding='utf-8', newline='') as result_csvfile_x:
dict_writer = csv.DictWriter(result_csvfile_x, headers)
dict_writer.writeheader()
with open('result.csv') as f:
reader = csv.DictReader(f)
list_completed_user_idcard = []
for row in reader:
if 'idcard' in row:
logging.info(('get completed_user idcard',row['idcard']))
list_completed_user_idcard.append(row['idcard'])
logging.info(('these idcard will skiped ', list_completed_user_idcard))
QueryData.completed_user = list_completed_user_idcard Queues.user_info_queue = user_info_queue
Queues.result_queue = result_queue
for i in range(10):
thread_q = QueryThread("queryThread" + str(i))
thread_q.start() thread_w = WriteResultThread("writeResuleThread" )
thread_w.start()
version: 1
disable_existing_loggers: False
formatters:
simple:
format: "%(asctime)s - %(name)s - %(levelname)s - %(threadName)s - %(lineno)d - %(message)s"
handlers:
console:
class: logging.StreamHandler
level: DEBUG
formatter: simple
stream: ext://sys.stdout
info_file_handler:
class: logging.handlers.RotatingFileHandler
level: INFO
formatter: simple
filename: info.log
maxBytes: 10485760
backupCount: 20
encoding: utf8
error_file_handler:
class: logging.handlers.RotatingFileHandler
level: ERROR
formatter: simple
filename: errors.log
maxBytes: 10485760
backupCount: 20
encoding: utf8
loggers:
my_module:
level: ERROR
handlers: [info_file_handler]
propagate: no
root:
level: INFO
handlers: [console,info_file_handler,error_file_handler]

log_config.yaml

python各模块组合实例的更多相关文章

  1. Python Paramiko模块与MySQL数据库操作

    Paramiko模块批量管理:通过调用ssh协议进行远程机器的批量命令执行. 要使用paramiko模块那就必须先安装这个第三方模块,仅需要在本地上安装相应的软件(python以及PyCrypto), ...

  2. python中argparse模块用法实例详解

    python中argparse模块用法实例详解 这篇文章主要介绍了python中argparse模块用法,以实例形式较为详细的分析了argparse模块解析命令行参数的使用技巧,需要的朋友可以参考下 ...

  3. python 全栈开发,Day19(组合,组合实例,初识面向对象小结,初识继承)

    一.组合 表示的一种什么有什么的关系 先来说一下,__init__的作用 class Dog: def __init__(self, name, kind, hp, ad): self.name = ...

  4. Python MySQLdb模块连接操作mysql数据库实例_python

    mysql是一个优秀的开源数据库,它现在的应用非常的广泛,因此很有必要简单的介绍一下用python操作mysql数据库的方法.python操作数据库需要安装一个第三方的模块,在http://mysql ...

  5. 【python库模块】Python subprocess模块功能与常见用法实例详解

    前言 这篇文章主要介绍了Python subprocess模块功能与常见用法,结合实例形式详细分析了subprocess模块功能.常用函数相关使用技巧. 参考 1. Python subprocess ...

  6. python使用cPickle模块序列化实例

    python使用cPickle模块序列化实例 这篇文章主要介绍了python使用cPickle模块序列化的方法,是一个非常实用的技巧,本文实例讲述了python使用cPickle模块序列化的方法,分享 ...

  7. [ Python入门教程 ] Python中日期时间datetime模块使用实例

    Python中datetime模块提供强大易用的日期处理功能,用于记录程序操作或修改时间.时间计算.日志时间显示等功能.datatime模块重新封装了time模块,提供的类包括date.time.da ...

  8. Python 日志模块实例

    python 打印对象的所有属性值: def prn_obj(obj):     print '\n'.join(['%s:%s' % item for item in obj.__dict__.it ...

  9. Day05 - Python 常用模块

    1. 模块简介 模块就是一个保存了 Python 代码的文件.模块能定义函数,类和变量.模块里也能包含可执行的代码. 模块也是 Python 对象,具有随机的名字属性用来绑定或引用. 下例是个简单的模 ...

随机推荐

  1. vuex2.0 基本使用(2) --- mutation 和 action

    我们的项目非常简单,当点击+1按钮的时候,count 加1,点击-1按钮的时候,count 减1. 1, mutation The only way to actually change state ...

  2. ubuntun与qt下载地址

    http://mirrors.melbourne.co.uk/ubuntu-releases/ http://download.qt.io/archive/qt/5.4/5.4.0/ 使用u盘安装ub ...

  3. html input 禁止输入中文

    <input type="text" class="tel" onkeyup="value=value.replace(/[\u4e00-\u9 ...

  4. JSON 解析 (二)—— Jackson的使用

    Jackson是基于Java语言的一种JSON和Java对象的数据处理工具.功能上简单易用,性能上根据目前主流转换工具比较,Jackson相对比较高效. <dependency> < ...

  5. Google浏览器——AxureRP_for_chorme_0_6_2添加

    准备 链接:https://share.weiyun.com/5PVwSMA Google浏览器版本 步骤 压缩解压 首先把需要安装的第三方插件,后缀.crx 改成 .rar,然后解压,得到一个文件夹 ...

  6. Newtonsoft.Json 概述

    有时候,在前后台数据交互或者APP与后台交互的时候,我们通常会使用Json进行数据交互,为此会使用到Newtonsoft.Json.dll 这个类库,这个类库非微软官方,但是下载量已经超过了数十万次, ...

  7. Mining Station on the Sea HDU - 2448(费用流 || 最短路 && hc)

    Mining Station on the Sea Time Limit: 5000/1000 MS (Java/Others)    Memory Limit: 32768/32768 K (Jav ...

  8. web 压力测试工具

    最近有收到任务,测试新服务器的性能. 花了很长时间做搜索,也整理了一些资料.以下是收集到一些简单易用的分析工具.推荐给大家使用. WebBenchhttp://www.ha97.com/4623.ht ...

  9. css颜色表示法

    css颜色值主要有三种表示方法: 1.颜色名表示,比如:red 红色,gold 金色 2.rgb表示,比如:rgb(255,0,0)表示红色 3.16进制数值表示,比如:#ff0000 表示红色,这种 ...

  10. Distinct Values(2018hdu多校第一场)

    给你一段长度为n的区间,然后在给你m个小区间,要求这m个小区间里的每个人都不能重复,请你输出字典序最小的方案. 我们可以开一个suf数组,表示我到我后面的不出现重复数字的区间至少需要到达的位置.所以对 ...