import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST, next_day_tag = 1900, '天配额超限，限制访问', 'DB_KEY_EXHAUST', '000003'

db = 'py_bdspider_status.db'

db = '%s\\%s' % (curPath, db)

def db_chk_one_exist(key):

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'SELECT key FROM baidu_map_key_used WHERE key="%s"' % (key)

    r = 0

    res = c.execute(sql).fetchone()

    if res is not None:

        r = 1

    conn.close

    return r

# def db_init_key_table():

#     conn = sqlite3.connect(db)

#     c = conn.cursor()

#     k_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

#     with open(k_file, 'r', encoding='utf-8') as pf:

#         for i in pf:

#             if len(i) < 4:

#                 continue

#             author, key = i.replace(' ', '').replace('\n', '').replace('\t', '').split(';')

#             r = db_chk_one_exist(key)

#             if r == 0:

#                 localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

#                 sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

#                     author, key, localtime_, 0)

#                 c.execute(sql)

#                 conn.commit()

#     conn.close()

#     pf.close()

#

#

# db_init_key_table()

def db_recovery_bdkeynum():

    if time.strftime("%H%M%S", time.localtime()) == next_day_tag:

        conn = sqlite3.connect(db)

        c = conn.cursor()

        localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

        sql = 'UPDATE baidu_map_key_used SET today_used = 0 ,update_time=%s  ' % (localtime_)

        c.execute(sql)

        conn.commit()

        conn.close()

    return

def db_get_one_effective():

    db_recovery_bdkeynum()

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ORDER BY today_used ASC' % (MAX_USED_TIMES)

    res, r = c.execute(sql).fetchone(), ''

    if res is None:

        r = DB_KEY_EXHAUST

    else:

        r = res[0]

    conn.close()

    return r

def db_update_one_today_used(key):

    conn = sqlite3.connect(db)

    c = conn.cursor()

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

    conn.close()

dir_, dir_exception, requested_file_list = 'baidu_map_uid', 'baidu_map_uid_exception', []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

ex_l = ['Proxy Error', 'APP IP校验失败', 'APP不存在，AK有误请检查再重试', 'The requested URL could not be retrieved',

        'Address already in use', '天配额超限，限制访问', 'Parameter Invalid']

def rm_invalid_file(file_postfix='.txt'):

    filepath = '%s\\%s\\' % (curPath, dir_)

    file_l = os.listdir(filepath)

    for i in file_l:

        fdir_o = os.path.join('%s%s' % (filepath, i))

        file_postfix_ = os.path.splitext(fdir_o)[1]

        if file_postfix_ == file_postfix:

            with open(fdir_o, 'r', encoding='utf-8') as ft:

                jstr = ''

                for i in ft:

                    jstr = '%s%s' % (jstr, i)

            ft.close()

            for ex in ex_l:

                if jstr.find(ex) > -1:

                    statinfo = os.stat(fdir_o)

                    strftime_st_ctime = time.strftime("%y%m%d%H%M%S", time.localtime(statinfo.st_ctime))

                    try:

                        os.remove(fdir_o)

                        print('remove', ex, strftime_st_ctime, fdir_o)

                    except Exception:

                        print('multiprocess--multithreading--', fdir_o)

rm_invalid_file()

def gen_requested_file_list(file_postfix='.html'):

    filepath = '%s\\%s' % (curPath, dir_)

    pathDir = os.listdir(filepath)

    for allDir in pathDir:

        child = os.path.join('%s%s' % (filepath, allDir))

        requested_file = child.split(dir_)[1].split('&')[0].split(file_postfix)[0]

        if requested_file not in requested_file_list:

            requested_file_list.append(requested_file)

file_postfix_l = ['.txt']

for i in file_postfix_l:

    gen_requested_file_list(i)

def gen_file_data(fname_source, file_type='.xlsx'):

    fname_open = '%s\\%s' % (curPath, fname_source)

    excel_ = '%s%s' % (fname_open, file_type)

    book = xlrd.open_workbook(excel_, on_demand=True)

    sheet = book.sheet_by_index(0)

    data = [[str(c.value) for c in sheet.row(i)] for i in range(sheet.nrows)]

    book.release_resources()

    del book

    return data

request_dic, target_type_list, target_type_except_list = {}, [], []

fname_source = '【SOURCE】采集员新增任务133598条-楼宇归集-互异百度uid数51700'

data_selfadd = gen_file_data(fname_source)

def replace_illeagl_tag(str_):

    l = [' ', '\n', '\t']

    for i in l:

        str_ = str_.replace(i, '')

    return str_

# 碧海富通城三期(3栋) ok

# =碧海富通城-三期(3栋) ok

replace_to_empty_l = [' ', '|', '\t', '\n', '/', '?', '？', '·', '.']

def gen_bd_query_origin_name(name_):

    for i in replace_to_empty_l:

        name_ = name_.replace(i, '')

    return name_.replace('（', '(').replace('）', ')').replace('?', '').replace('？', '')

for l in data_selfadd:

    dbid, area_code, type_, city, district, uid, name_, address, street, request_name, submit_time = l

    # if city != '深圳市':

    #     continue

    # if len(uid.replace(' ', '')) > 0:

    #     continue

    request_name = gen_bd_query_origin_name(name_)

    request_name_chk = '%s%s%s' % (city, district, request_name)

    if request_name_chk in requested_file_list:

        continue

    if city not in request_dic:

        request_dic[city] = {}

    if district not in request_dic[city]:

        request_dic[city][district] = {}

        request_dic[city][district] = []

    if request_name not in request_dic[city][district]:

        request_dic[city][district].append(request_name)

del data_selfadd

fname_source = '【TEAM】41876条JMTool官方数据百度POIuid_添加率0.9388_170830171339'

data_jmtool = gen_file_data(fname_source)

for l in data_jmtool:

    dbid, area_code, name_, request_name, type_, city, district, addr, street, bd_status, bd_message, bd_res_str, city_bd, district_bd, business_bd, cityid_bd, name_bd, uid, lat_bd, lng_bd, compute_res, name_ratio_res, combine_ratio_res, uid_href = l

    # if len(uid.replace(' ', '')) > 0:

    #     continue

    # if city != '深圳市':

    #     continue

    request_name = gen_bd_query_origin_name(name_)

    request_name_chk = '%s%s%s' % (city, district, request_name)

    if request_name_chk in requested_file_list:

        continue

    if city not in request_dic:

        request_dic[city] = {}

    if district not in request_dic[city]:

        request_dic[city][district] = {}

        request_dic[city][district] = []

    if request_name not in request_dic[city][district]:

        request_dic[city][district].append(request_name)

del data_jmtool

write_res_file_dir = '%s\\%s\\' % (curPath, dir_)

def write_res_file(input_, str_, dir_=write_res_file_dir, file_postfix='.txt'):

    for ex in ex_l:

        if str_.find(ex) > -1:

            global ak, url_

            print('EXCEPTION-', ex, 'AK-', ak, 'URL-', url_)

            return

    fname = '%s%s%s' % (dir_, input_, file_postfix)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    ft.close()

    print('ok', threading.get_ident(), input_)

class MyThread(threading.Thread):

    def __init__(self, func, args, name):

        threading.Thread.__init__(self)

        self.name, self.func, self.args = name, func, args

    def run(self):

        self.func(self.args)

# http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

def fun_(city):

    for district in request_dic[city]:

        for request_name in request_dic[city][district]:

            request_name_chk = '%s%s%s' % (city, district, request_name)

            # gen_requested_file_list('.txt')

            if request_name_chk in requested_file_list:

                continue

            ak = db_get_one_effective()

            if ak == DB_KEY_EXHAUST:

                print(DB_KEY_EXHAUST)

                break

            else:

                url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

            try:

                bd_res_json_str = requests.get(url_).text

                db_update_one_today_used(ak)

                write_res_file(request_name_chk, bd_res_json_str)

            except Exception:

                bd_res_json_str = '请求百度-异常'

                write_res_file(request_name_chk, bd_res_json_str, requested_file_dir_exception_str)

                print(request_name_chk, bd_res_json_str)

try:

    start_loop, stop_loop = int(sys.argv[1]), int(sys.argv[2])

except Exception:

    start_loop, stop_loop = -1, 200

def main():

    threads_list, nloop = [], 0

    request_dic_city_l = sorted(request_dic, reverse=False)

    for city in request_dic_city_l:

        nloop += 1

        if nloop < start_loop or nloop > stop_loop:

            continue

        thread_instance = MyThread(fun_, (city), fun_.__name__)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

if __name__ == '__main__':

    main()

　　

import time

import sys

import os

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

dir_, dir_exception, requested_file_list = 'baidu_map_uid', 'baidu_map_uid_exception', []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

ex_l = ['Proxy Error', 'APP IP校验失败', 'APP不存在，AK有误请检查再重试', 'The requested URL could not be retrieved',

        'Address already in use', '天配额超限，限制访问', 'Parameter Invalid']

def rm_invalid_file(file_postfix='.txt'):

    filepath = '%s\\%s\\' % (curPath, dir_)

    file_l = os.listdir(filepath)

    for i in file_l:

        fdir_o = os.path.join('%s%s' % (filepath, i))

        file_postfix_ = os.path.splitext(fdir_o)[1]

        if file_postfix_ == file_postfix:

            with open(fdir_o, 'r', encoding='utf-8') as ft:

                jstr = ''

                for i in ft:

                    jstr = '%s%s' % (jstr, i)

            ft.close()

            for ex in ex_l:

                if jstr.find(ex) > -1:

                    statinfo = os.stat(fdir_o)

                    strftime_st_ctime = time.strftime("%y%m%d%H%M%S",time.localtime(statinfo.st_ctime))

                    os.remove(fdir_o)

                    print('remove', ex,strftime_st_ctime, fdir_o)

rm_invalid_file()

　　

import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST, next_day_tag = 1900, '天配额超限，限制访问', 'DB_KEY_EXHAUST', '000003'

db = 'py_bdspider_status.db'

db = '%s\\%s' % (curPath, db)

def db_chk_one_exist(key):

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'SELECT key FROM baidu_map_key_used WHERE key="%s"' % (key)

    r = 0

    res = c.execute(sql).fetchone()

    if res is not None:

        r = 1

    conn.close

    return r

# def db_init_key_table():

#     conn = sqlite3.connect(db)

#     c = conn.cursor()

#     k_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

#     with open(k_file, 'r', encoding='utf-8') as pf:

#         for i in pf:

#             if len(i) < 4:

#                 continue

#             author, key = i.replace(' ', '').replace('\n', '').replace('\t', '').split(';')

#             r = db_chk_one_exist(key)

#             if r == 0:

#                 localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

#                 sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

#                     author, key, localtime_, 0)

#                 c.execute(sql)

#                 conn.commit()

#     conn.close()

#     pf.close()

#

#

# db_init_key_table()

def db_recovery_bdkeynum():

    if time.strftime("%H%M%S", time.localtime()) == next_day_tag:

        conn = sqlite3.connect(db)

        c = conn.cursor()

        localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

        sql = 'UPDATE baidu_map_key_used SET today_used = 0 ,update_time=%s  ' % (localtime_)

        c.execute(sql)

        conn.commit()

        conn.close()

    return

def db_get_one_effective():

    db_recovery_bdkeynum()

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ORDER BY today_used ASC' % (MAX_USED_TIMES)

    res, r = c.execute(sql).fetchone(), ''

    if res is None:

        r = DB_KEY_EXHAUST

    else:

        r = res[0]

    conn.close()

    return r

def db_update_one_today_used(key):

    conn = sqlite3.connect(db)

    c = conn.cursor()

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

    conn.close()

dir_, dir_exception, requested_file_list = 'baidu_map_uid', 'baidu_map_uid_exception', []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

ex_l = ['Proxy Error', 'APP IP校验失败', 'APP不存在，AK有误请检查再重试', 'The requested URL could not be retrieved',

        'Address already in use', '天配额超限，限制访问', 'Parameter Invalid']

def rm_invalid_file(file_postfix='.txt'):

    filepath = '%s\\%s\\' % (curPath, dir_)

    file_l = os.listdir(filepath)

    for i in file_l:

        fdir_o = os.path.join('%s%s' % (filepath, i))

        file_postfix_ = os.path.splitext(fdir_o)[1]

        if file_postfix_ == file_postfix:

            with open(fdir_o, 'r', encoding='utf-8') as ft:

                jstr = ''

                for i in ft:

                    jstr = '%s%s' % (jstr, i)

            ft.close()

            for ex in ex_l:

                if jstr.find(ex) > -1:

                    statinfo = os.stat(fdir_o)

                    strftime_st_ctime = time.strftime("%y%m%d%H%M%S",time.localtime(statinfo.st_ctime))

                    os.remove(fdir_o)

                    print('remove', ex,strftime_st_ctime, fdir_o)

rm_invalid_file()

def gen_requested_file_list(file_postfix='.html'):

    filepath = '%s\\%s' % (curPath, dir_)

    pathDir = os.listdir(filepath)

    for allDir in pathDir:

        child = os.path.join('%s%s' % (filepath, allDir))

        requested_file = child.split(dir_)[1].split('&')[0].split(file_postfix)[0]

        if requested_file not in requested_file_list:

            requested_file_list.append(requested_file)

file_postfix_l = ['.txt']

for i in file_postfix_l:

    gen_requested_file_list(i)

def gen_file_data(fname_source, file_type='.xlsx'):

    fname_open = '%s\\%s' % (curPath, fname_source)

    excel_ = '%s%s' % (fname_open, file_type)

    book = xlrd.open_workbook(excel_, on_demand=True)

    sheet = book.sheet_by_index(0)

    data = [[str(c.value) for c in sheet.row(i)] for i in range(sheet.nrows)]

    book.release_resources()

    del book

    return data

request_dic, target_type_list, target_type_except_list = {}, [], []

fname_source = '【SOURCE】采集员新增任务133598条-楼宇归集-互异百度uid数51700'

data_selfadd = gen_file_data(fname_source)

def replace_illeagl_tag(str_):

    l = [' ', '\n', '\t']

    for i in l:

        str_ = str_.replace(i, '')

    return str_

# 碧海富通城三期(3栋) ok

# =碧海富通城-三期(3栋) ok

replace_to_empty_l = [' ', '|', '\t', '\n', '/', '?', '？', '·', '.']

def gen_bd_query_origin_name(name_):

    for i in replace_to_empty_l:

        name_ = name_.replace(i, '')

    return name_.replace('（', '(').replace('）', ')').replace('?', '').replace('？', '')

for l in data_selfadd:

    dbid, area_code, type_, city, district, uid, name_, address, street, request_name, submit_time = l

    # if city != '深圳市':

    #     continue

    # if len(uid.replace(' ', '')) > 0:

    #     continue

    request_name = gen_bd_query_origin_name(name_)

    request_name_chk = '%s%s%s' % (city, district, request_name)

    if request_name_chk in requested_file_list:

        continue

    if city not in request_dic:

        request_dic[city] = {}

    if district not in request_dic[city]:

        request_dic[city][district] = {}

        request_dic[city][district] = []

    if request_name not in request_dic[city][district]:

        request_dic[city][district].append(request_name)

del data_selfadd

fname_source = '【TEAM】41876条JMTool官方数据百度POIuid_添加率0.9388_170830171339'

data_jmtool = gen_file_data(fname_source)

for l in data_jmtool:

    dbid, area_code, name_, request_name, type_, city, district, addr, street, bd_status, bd_message, bd_res_str, city_bd, district_bd, business_bd, cityid_bd, name_bd, uid, lat_bd, lng_bd, compute_res, name_ratio_res, combine_ratio_res, uid_href = l

    # if len(uid.replace(' ', '')) > 0:

    #     continue

    # if city != '深圳市':

    #     continue

    request_name = gen_bd_query_origin_name(name_)

    request_name_chk = '%s%s%s' % (city, district, request_name)

    if request_name_chk in requested_file_list:

        continue

    if city not in request_dic:

        request_dic[city] = {}

    if district not in request_dic[city]:

        request_dic[city][district] = {}

        request_dic[city][district] = []

    if request_name not in request_dic[city][district]:

        request_dic[city][district].append(request_name)

del data_jmtool

write_res_file_dir = '%s\\%s\\' % (curPath, dir_)

def write_res_file(input_, str_, dir_=write_res_file_dir, file_postfix='.txt'):

    for ex in ex_l:

        if str_.find(ex) > -1:

            global ak, url_

            print('EXCEPTION-', ex, 'AK-', ak, 'URL-', url_)

            return

    fname = '%s%s%s' % (dir_, input_, file_postfix)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    ft.close()

    print('ok', threading.get_ident(), input_)

class MyThread(threading.Thread):

    def __init__(self, func, args, name):

        threading.Thread.__init__(self)

        self.name, self.func, self.args = name, func, args

    def run(self):

        self.func(self.args)

# http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

def fun_(city):

    for district in request_dic[city]:

        for request_name in request_dic[city][district]:

            request_name_chk = '%s%s%s' % (city, district, request_name)

            # gen_requested_file_list('.txt')

            if request_name_chk in requested_file_list:

                continue

            ak = db_get_one_effective()

            if ak == DB_KEY_EXHAUST:

                print(DB_KEY_EXHAUST)

                break

            else:

                url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

            try:

                bd_res_json_str = requests.get(url_).text

                db_update_one_today_used(ak)

                write_res_file(request_name_chk, bd_res_json_str)

            except Exception:

                bd_res_json_str = '请求百度-异常'

                write_res_file(request_name_chk, bd_res_json_str, requested_file_dir_exception_str)

                print(request_name_chk, bd_res_json_str)

try:

    start_loop, stop_loop = int(sys.argv[1]), int(sys.argv[2])

except Exception:

    start_loop, stop_loop = -1, 200

def main():

    threads_list, nloop = [], 0

    request_dic_city_l = sorted(request_dic, reverse=False)

    for city in request_dic_city_l:

        nloop += 1

        if nloop < start_loop or nloop > stop_loop:

            continue

        thread_instance = MyThread(fun_, (city), fun_.__name__)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

if __name__ == '__main__':

    main()

　　

rm_invalid_file的更多相关文章

随机推荐

  1. 一分钟学awk够用

    [转载于58同城沈剑] 1.什么是AWK(1)Aho.Weinberger.Kernighan三位发明者名字首字母:(2)一个行文本处理工具: 2.AWK基本原理2.1原理:逐行处理文件中的数据 2. ...

  2. IQueryable接口与IEnumberable 区别

    总结一下: IEnumerable<T> 泛型类在调用自己的SKip 和 Take 等扩展方法之前数据就已经加载在本地内存里了,而IQueryable<T> 是将Skip ,t ...

  3. [转] docker rmi命令-删除image

    原文:http://www.simapple.com/341.html ---------------------------------------------------------------- ...

  4. 2017.7.14 使用case when和group by将多条数据合并成一行，并且根据某些列的合并值做条件判断来生成最终值

    参考来自:http://bbs.csdn.net/topics/390737006 1.效果演示 (1)不做处理 (2)合并多列,并对后四列的值做并集处理 2.SQL语句 (1)不做处理 SELECT ...

  5. Json序列化为对象方法

    /// <summary>/// json 序列化为对象/// </summary>/// <typeparam name="T">对象类型&l ...

  6. ubuntu 12.04上安装wn725n无线网卡驱动

    系统版本:$uname -a Linux yuliang-Veriton-M670G 3.11.0-17-generic #31~precise1-Ubuntu SMP Tue Feb 4 21:25 ...

  7. SQL 关键字 'USER' 附近有语法错误怎么办

    如下图所示,我想要访问我的Database1.mdf的user这张表,提示如下错误   user在SQL Server中是系统保留字,将user修改为[user]就可以了.但是直接在VS中是无法修改的 ...

  8. LeetCode题目:Generate Parentheses

    原题地址:https://leetcode.com/problems/generate-parentheses/ 解决方法:回溯法 class Solution { private: vector&l ...

  9. ubuntu下设置静态ip

    直接修改系统配置文件ubuntu的网络配置文件是:/etc/network/interfacesubuntu命令行修改网络配置方法前面auto  eth?,让网卡开机自动挂载. 为网卡配置静态IP地址 ...

  10. trac 的安装设置

    Trac的安装设置 原网页:http://wiki.ubuntu.org.cn/Trac%E7%9A%84%E5%AE%89%E8%A3%85%E8%AE%BE%E7%BD%AE 出自Ubuntu中文 ...