SQLite objects created in a thread can only be used in that same thread.The object was created in thread id 62332 and this is thread id 53032

sqlite支持的并发访问数？？

import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限，限制访问', 'DB_KEY_EXHAUST'

db = 'py_bdspider_status.db'

def db_init_key_table():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'DELETE  FROM  baidu_map_key_used'

    c.execute(sql)

    conn.commit()

    pcity_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

    with open(pcity_file, 'r', encoding='utf-8') as pf:

        c_ = 0

        for i in pf:

            if len(i) < 4:

                continue

            author, key = i.replace('\n', '').split('\t')

            localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

            sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

                author, key, localtime_, 0)

            c.execute(sql)

    conn.commit()

    conn.close()

# db_init_key_table()

def db_get_one_effective():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)

    res = c.execute(sql).fetchone()

    if res is None:

        return DB_KEY_EXHAUST

    else:

        return res[0]

    conn.close

def db_update_one_today_used(key):

    conn = sqlite3.connect(db)

    c = conn.cursor()

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

    conn.close()

dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'

requested_file_list = []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

def chk_if_requested_file():

    for f in requested_file_dir:

        to_in = f.split('.txt')[0]

        if to_in not in requested_file_list:

            requested_file_list.append(to_in)

def write_requested_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    print('ok', threading.get_ident(), request_name)

def write_requested_exception_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'

def fun_():

    fname_open = '%s\\%s' % (curPath, fname_source)

    FEXCEL = '%s%s' % (fname_open, '.xlsx')

    data = xlrd.open_workbook(FEXCEL)

    table = data.sheets()[0]

    nrows, ncols = table.nrows, table.ncols

    # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

    base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

    for i in range(1, nrows):

        l = table.row_values(i)

        dbid, area_code, name_, request_name, type_, city, district, addr, street = l

        request_name_chk = '%s%s%s' % (city, district, request_name)

        chk_if_requested_file()

        if request_name_chk in requested_file_list:

            continue

        ak = db_get_one_effective()

        if ak == DB_KEY_EXHAUST:

            print(DB_KEY_EXHAUST)

            break

        else:

            url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

        try:

            bd_res_json_str = requests.get(url_).text

            db_update_one_today_used(ak)

            write_requested_res(request_name_chk, bd_res_json_str)

        except Exception:

            bd_res_json_str = '请求百度-异常'

            write_requested_exception_res(request_name_chk, bd_res_json_str)

            print(bd_res_json_str)

class MyThread(threading.Thread):

    def __init__(self, func):

        threading.Thread.__init__(self)

        self.func = func

    def run(self):

        self.func()

thread_sum = 16

def main():

    threads_list = []

    for nloop in range(0, thread_sum, 1):

        thread_instance = MyThread(fun_)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

if __name__ == '__main__':

    main()

　　

import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限，限制访问', 'DB_KEY_EXHAUST'

db = 'py_bdspider_status.db'

def db_init_key_table():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'DELETE  FROM  baidu_map_key_used'

    c.execute(sql)

    conn.commit()

    pcity_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

    with open(pcity_file, 'r', encoding='utf-8') as pf:

        c_ = 0

        for i in pf:

            if len(i) < 4:

                continue

            author, key = i.replace('\n', '').split('\t')

            localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

            sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

                author, key, localtime_, 0)

            c.execute(sql)

    conn.commit()

    conn.close()

# db_init_key_table()

def db_get_one_effective():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)

    res = c.execute(sql).fetchone()

    if res is None:

        return DB_KEY_EXHAUST

    else:

        return res[0]

    conn.close

def db_update_one_today_used(key):

    conn = sqlite3.connect(db)

    c = conn.cursor()

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

    conn.close()

dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'

requested_file_list = []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

def chk_if_requested_file():

    for f in requested_file_dir:

        to_in = f.split('.txt')[0]

        if to_in not in requested_file_list:

            requested_file_list.append(to_in)

def write_requested_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    print('ok', threading.get_ident(), request_name)

def write_requested_exception_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'

def fun_():

    fname_open = '%s\\%s' % (curPath, fname_source)

    FEXCEL = '%s%s' % (fname_open, '.xlsx')

    data = xlrd.open_workbook(FEXCEL)

    table = data.sheets()[0]

    nrows, ncols = table.nrows, table.ncols

    # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

    base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

    i_counter = 0

    for i in range(1, nrows):

        l = table.row_values(i)

        i_counter += 1

        dbid, area_code, name_, request_name, type_, city, district, addr, street = l

        request_name_chk = '%s%s%s' % (city, district, request_name)

        chk_if_requested_file()

        if request_name_chk in requested_file_list:

            continue

        ak = db_get_one_effective()

        if ak == DB_KEY_EXHAUST:

            print(DB_KEY_EXHAUST)

            break

        else:

            url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

        try:

            bd_res_json_str = requests.get(url_).text

            db_update_one_today_used(ak)

            write_requested_res(request_name_chk, bd_res_json_str)

        except Exception:

            bd_res_json_str = '请求百度-异常'

            write_requested_exception_res(request_name_chk, bd_res_json_str)

            print(bd_res_json_str)

class MyThread(threading.Thread):

    def __init__(self, func):

        threading.Thread.__init__(self)

        self.func = func

    def run(self):

        self.func()

thread_sum = 100

def main():

    threads_list = []

    for nloop in range(0, thread_sum, 1):

        thread_instance = MyThread(fun_)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

if __name__ == '__main__':

    main()

　　

import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

db = 'py_bdspider_status.db'

conn = sqlite3.connect(db)

c = conn.cursor()

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限，限制访问', 'DB_KEY_EXHAUST'

def db_init_key_table():

    sql = 'DELETE  FROM  baidu_map_key_used'

    c.execute(sql)

    conn.commit()

    pcity_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

    with open(pcity_file, 'r', encoding='utf-8') as pf:

        c_ = 0

        for i in pf:

            if len(i) < 4:

                continue

            author, key = i.replace('\n', '').split('\t')

            localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

            sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

                author, key, localtime_, 0)

            c.execute(sql)

    conn.commit()

# db_init_key_table()

def db_get_one_effective():

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)

    res = c.execute(sql).fetchone()

    if res is None:

        return DB_KEY_EXHAUST

    else:

        return res[0]

def db_update_one_today_used(key):

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'

requested_file_list = []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

def chk_if_requested_file():

    for f in requested_file_dir:

        to_in = f.split('.txt')[0]

        if to_in not in requested_file_list:

            requested_file_list.append(to_in)

def write_requested_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    print('ok', threading.get_ident(), request_name)

def write_requested_exception_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'

def fun_():

    fname_open = '%s\\%s' % (curPath, fname_source)

    FEXCEL = '%s%s' % (fname_open, '.xlsx')

    data = xlrd.open_workbook(FEXCEL)

    table = data.sheets()[0]

    nrows, ncols = table.nrows, table.ncols

    # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

    base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

    i_counter = 0

    for i in range(1, nrows):

        l = table.row_values(i)

        i_counter += 1

        print(i_counter)

        if i_counter > 60:

            break

        dbid, area_code, name_, request_name, type_, city, district, addr, street = l

        request_name_chk = '%s%s%s' % (city, district, request_name)

        chk_if_requested_file()

        if request_name_chk in requested_file_list:

            continue

        ak = db_get_one_effective()

        if ak == DB_KEY_EXHAUST:

            print(DB_KEY_EXHAUST)

            break

        else:

            url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

            print(url_)

        try:

            bd_res_json_str = requests.get(url_).text

            db_update_one_today_used(ak)

            write_requested_res(request_name_chk, bd_res_json_str)

        except Exception:

            bd_res_json_str = '请求百度-异常'

            write_requested_exception_res(request_name_chk, bd_res_json_str)

            print(bd_res_json_str)

class MyThread(threading.Thread):

    def __init__(self, func):

        threading.Thread.__init__(self)

        self.func = func

    def run(self):

        self.func()

thread_sum = 4

def main():

    threads_list = []

    for nloop in range(0, thread_sum, 1):

        thread_instance = MyThread(fun_)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

    conn.close()

if __name__ == '__main__':

    main()

　　

SQLite支持的并发访问数的更多相关文章

  1. 查看apache当前并发访问数和进程数

    1.查看apache当前并发访问数: netstat -an | grep ESTABLISHED | wc -l 对比httpd.conf中MaxClients的数字差距多少. 2.查看有多少个进程 ...

  2. 高效解决「SQLite」数据库并发访问安全问题，只这一篇就够了

    Concurrent database access 本文译自:https://dmytrodanylyk.com/articles/concurrent-database/ 对于 Android D ...

  3. Socket支持多用户并发访问的解决办法

    //创建线程池,池中具有(cpu个数*50)条线程 ExecutorService executorService = Executors.newFixedThreadPool(Runtime.get ...

  4. 查看 Apache并发请求数及其TCP连接状态

    查看 Apache并发请求数及其TCP连接状态 (2011-06-27 15:08:36) 服务器上的一些统计数据: 1)统计80端口连接数 netstat -nat|grep -i "80 ...

  5. 查看 并发请求数及其TCP连接状态【转】

    服务器上的一些统计数据: 1)统计80端口连接数netstat -nat|grep -i "80"|wc -l 2)统计httpd协议连接数ps -ef|grep httpd|wc ...

  6. 查看 并发请求数及其TCP连接状态

    服务器上的一些统计数据: 1)统计80端口连接数netstat -nat|grep -i "80"|wc -l 2)统计httpd协议连接数ps -ef|grep httpd|wc ...

  7. 查看 Apache并发请求数及其TCP连接状态【转】

    查看 Apache并发请求数及其TCP连接状态 (2011-06-27 15:08:36) 服务器上的一些统计数据: 1)统计80端口连接数netstat -nat|grep -i "80& ...

  8. Java并发工具类（三）：控制并发线程数的Semaphore

    作用 Semaphore(信号量)是用来控制同时访问特定资源的线程数量,它通过协调各个线程,以保证合理的使用公共资源. 简介 Semaphore也是一个线程同步的辅助类,可以维护当前访问自身的线程个数 ...

  9. 查看http的并发请求数及其TCP连接状态

    统计80端口的连接数据 netstat -nat | grep -i "80" | wc -l 统计httpd协议连接数 ps -ef | grep httpd | wc -l 统 ...

随机推荐

  1. EffectiveJava（1） 构造器和静态工厂方法

    构造器和静态工厂方法 **构造器是大家创建类时的构造方法,即使不显式声明,它也会在类内部隐式声明,使我们可以通过类名New一个实例. 静态方法是构造器的另一种表现形式** 主题要点:何时以及如何创建对 ...

  2. 《深入理解Android 卷III》第七章 深入理解SystemUI

    <深入理解Android 卷III>即将公布,作者是张大伟.此书填补了深入理解Android Framework卷中的一个主要空白,即Android Framework中和UI相关的部分. ...

  3. Cygwin-安装和配置ssh服务

    Cygwin介绍: Cygwin是一个在windows平台上执行的类UNIX模拟环境.它对于学习UNIX/Linux操作环境,或者从UNIX到Windows的应用程序移植,或者进行某些特殊的开发工作, ...

  4. Exposing the Outlook Password Secrets

    Exposing the Outlook Password Secrets - www.SecurityXploded.comhttp://securityxploded.com/outlookpas ...

  5. 【Excle数据透视表】如何得到数据透视表中某个汇总行的明细数据

    例如: 现在想得到"北京 汇总"的明细数据,该怎么处理呢? 步骤 右键数据透视表任意单元格→数据透视表选项→启用显示明细数据→确定→单击"北京 汇总"行最后一个 ...

  6. JUnit编写单元测试代码注意点小结

    用eclipse编写单元测试的时候,可以直接选中某个类,然后右键new新疆一个junit case,界面如下图1所示: 图1:新建test case 选 择图1中的JUnit Test Case,然后 ...

  7. windows下简单配置squid反向代理服务器

    下载windwosNT版本的squid下载地址: http://squid.acmeconsulting.it/download/squid-2.6.STABLE13-bin.zip 1．把squid ...

  8. MySQL优化时可以设置的几个参数

    back_log:back_log值指出在MySQL暂时停止回答新请求之前的短时间内多少个请求可以被存在堆栈中.也就是说,如果MySql的连接数据达到max_connections时,新来的请求将会被 ...

  9. ctrip-apollo windows环境部署

    https://blog.csdn.net/u010286334/article/details/78389484转载过来的方法,尝试了没有成功,不晓得哪里错了,无法打包portal访问 1.下载源码 ...

  10. Linux 安装json神器 jq

    wget -O jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 chmod +x ./jq cp jq /u ...