SQLite objects created in a thread can only be used in that same thread.The object was created in thread id 62332 and this is thread id 53032

sqlite支持的并发访问数？？

import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限，限制访问', 'DB_KEY_EXHAUST'

db = 'py_bdspider_status.db'

def db_init_key_table():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'DELETE  FROM  baidu_map_key_used'

    c.execute(sql)

    conn.commit()

    pcity_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

    with open(pcity_file, 'r', encoding='utf-8') as pf:

        c_ = 0

        for i in pf:

            if len(i) < 4:

                continue

            author, key = i.replace('\n', '').split('\t')

            localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

            sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

                author, key, localtime_, 0)

            c.execute(sql)

    conn.commit()

    conn.close()

# db_init_key_table()

def db_get_one_effective():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)

    res = c.execute(sql).fetchone()

    if res is None:

        return DB_KEY_EXHAUST

    else:

        return res[0]

    conn.close

def db_update_one_today_used(key):

    conn = sqlite3.connect(db)

    c = conn.cursor()

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

    conn.close()

dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'

requested_file_list = []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

def chk_if_requested_file():

    for f in requested_file_dir:

        to_in = f.split('.txt')[0]

        if to_in not in requested_file_list:

            requested_file_list.append(to_in)

def write_requested_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    print('ok', threading.get_ident(), request_name)

def write_requested_exception_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'

def fun_():

    fname_open = '%s\\%s' % (curPath, fname_source)

    FEXCEL = '%s%s' % (fname_open, '.xlsx')

    data = xlrd.open_workbook(FEXCEL)

    table = data.sheets()[0]

    nrows, ncols = table.nrows, table.ncols

    # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

    base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

    for i in range(1, nrows):

        l = table.row_values(i)

        dbid, area_code, name_, request_name, type_, city, district, addr, street = l

        request_name_chk = '%s%s%s' % (city, district, request_name)

        chk_if_requested_file()

        if request_name_chk in requested_file_list:

            continue

        ak = db_get_one_effective()

        if ak == DB_KEY_EXHAUST:

            print(DB_KEY_EXHAUST)

            break

        else:

            url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

        try:

            bd_res_json_str = requests.get(url_).text

            db_update_one_today_used(ak)

            write_requested_res(request_name_chk, bd_res_json_str)

        except Exception:

            bd_res_json_str = '请求百度-异常'

            write_requested_exception_res(request_name_chk, bd_res_json_str)

            print(bd_res_json_str)

class MyThread(threading.Thread):

    def __init__(self, func):

        threading.Thread.__init__(self)

        self.func = func

    def run(self):

        self.func()

thread_sum = 16

def main():

    threads_list = []

    for nloop in range(0, thread_sum, 1):

        thread_instance = MyThread(fun_)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

if __name__ == '__main__':

    main()

　　

import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限，限制访问', 'DB_KEY_EXHAUST'

db = 'py_bdspider_status.db'

def db_init_key_table():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'DELETE  FROM  baidu_map_key_used'

    c.execute(sql)

    conn.commit()

    pcity_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

    with open(pcity_file, 'r', encoding='utf-8') as pf:

        c_ = 0

        for i in pf:

            if len(i) < 4:

                continue

            author, key = i.replace('\n', '').split('\t')

            localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

            sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

                author, key, localtime_, 0)

            c.execute(sql)

    conn.commit()

    conn.close()

# db_init_key_table()

def db_get_one_effective():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)

    res = c.execute(sql).fetchone()

    if res is None:

        return DB_KEY_EXHAUST

    else:

        return res[0]

    conn.close

def db_update_one_today_used(key):

    conn = sqlite3.connect(db)

    c = conn.cursor()

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

    conn.close()

dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'

requested_file_list = []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

def chk_if_requested_file():

    for f in requested_file_dir:

        to_in = f.split('.txt')[0]

        if to_in not in requested_file_list:

            requested_file_list.append(to_in)

def write_requested_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    print('ok', threading.get_ident(), request_name)

def write_requested_exception_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'

def fun_():

    fname_open = '%s\\%s' % (curPath, fname_source)

    FEXCEL = '%s%s' % (fname_open, '.xlsx')

    data = xlrd.open_workbook(FEXCEL)

    table = data.sheets()[0]

    nrows, ncols = table.nrows, table.ncols

    # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

    base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

    i_counter = 0

    for i in range(1, nrows):

        l = table.row_values(i)

        i_counter += 1

        dbid, area_code, name_, request_name, type_, city, district, addr, street = l

        request_name_chk = '%s%s%s' % (city, district, request_name)

        chk_if_requested_file()

        if request_name_chk in requested_file_list:

            continue

        ak = db_get_one_effective()

        if ak == DB_KEY_EXHAUST:

            print(DB_KEY_EXHAUST)

            break

        else:

            url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

        try:

            bd_res_json_str = requests.get(url_).text

            db_update_one_today_used(ak)

            write_requested_res(request_name_chk, bd_res_json_str)

        except Exception:

            bd_res_json_str = '请求百度-异常'

            write_requested_exception_res(request_name_chk, bd_res_json_str)

            print(bd_res_json_str)

class MyThread(threading.Thread):

    def __init__(self, func):

        threading.Thread.__init__(self)

        self.func = func

    def run(self):

        self.func()

thread_sum = 100

def main():

    threads_list = []

    for nloop in range(0, thread_sum, 1):

        thread_instance = MyThread(fun_)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

if __name__ == '__main__':

    main()

　　

import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

db = 'py_bdspider_status.db'

conn = sqlite3.connect(db)

c = conn.cursor()

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限，限制访问', 'DB_KEY_EXHAUST'

def db_init_key_table():

    sql = 'DELETE  FROM  baidu_map_key_used'

    c.execute(sql)

    conn.commit()

    pcity_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

    with open(pcity_file, 'r', encoding='utf-8') as pf:

        c_ = 0

        for i in pf:

            if len(i) < 4:

                continue

            author, key = i.replace('\n', '').split('\t')

            localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

            sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

                author, key, localtime_, 0)

            c.execute(sql)

    conn.commit()

# db_init_key_table()

def db_get_one_effective():

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)

    res = c.execute(sql).fetchone()

    if res is None:

        return DB_KEY_EXHAUST

    else:

        return res[0]

def db_update_one_today_used(key):

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'

requested_file_list = []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

def chk_if_requested_file():

    for f in requested_file_dir:

        to_in = f.split('.txt')[0]

        if to_in not in requested_file_list:

            requested_file_list.append(to_in)

def write_requested_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    print('ok', threading.get_ident(), request_name)

def write_requested_exception_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'

def fun_():

    fname_open = '%s\\%s' % (curPath, fname_source)

    FEXCEL = '%s%s' % (fname_open, '.xlsx')

    data = xlrd.open_workbook(FEXCEL)

    table = data.sheets()[0]

    nrows, ncols = table.nrows, table.ncols

    # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

    base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

    i_counter = 0

    for i in range(1, nrows):

        l = table.row_values(i)

        i_counter += 1

        print(i_counter)

        if i_counter > 60:

            break

        dbid, area_code, name_, request_name, type_, city, district, addr, street = l

        request_name_chk = '%s%s%s' % (city, district, request_name)

        chk_if_requested_file()

        if request_name_chk in requested_file_list:

            continue

        ak = db_get_one_effective()

        if ak == DB_KEY_EXHAUST:

            print(DB_KEY_EXHAUST)

            break

        else:

            url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

            print(url_)

        try:

            bd_res_json_str = requests.get(url_).text

            db_update_one_today_used(ak)

            write_requested_res(request_name_chk, bd_res_json_str)

        except Exception:

            bd_res_json_str = '请求百度-异常'

            write_requested_exception_res(request_name_chk, bd_res_json_str)

            print(bd_res_json_str)

class MyThread(threading.Thread):

    def __init__(self, func):

        threading.Thread.__init__(self)

        self.func = func

    def run(self):

        self.func()

thread_sum = 4

def main():

    threads_list = []

    for nloop in range(0, thread_sum, 1):

        thread_instance = MyThread(fun_)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

    conn.close()

if __name__ == '__main__':

    main()

　　

SQLite支持的并发访问数的更多相关文章

  1. 查看apache当前并发访问数和进程数

    1.查看apache当前并发访问数: netstat -an | grep ESTABLISHED | wc -l 对比httpd.conf中MaxClients的数字差距多少. 2.查看有多少个进程 ...

  2. 高效解决「SQLite」数据库并发访问安全问题，只这一篇就够了

    Concurrent database access 本文译自:https://dmytrodanylyk.com/articles/concurrent-database/ 对于 Android D ...

  3. Socket支持多用户并发访问的解决办法

    //创建线程池,池中具有(cpu个数*50)条线程 ExecutorService executorService = Executors.newFixedThreadPool(Runtime.get ...

  4. 查看 Apache并发请求数及其TCP连接状态

    查看 Apache并发请求数及其TCP连接状态 (2011-06-27 15:08:36) 服务器上的一些统计数据: 1)统计80端口连接数 netstat -nat|grep -i "80 ...

  5. 查看 并发请求数及其TCP连接状态【转】

    服务器上的一些统计数据: 1)统计80端口连接数netstat -nat|grep -i "80"|wc -l 2)统计httpd协议连接数ps -ef|grep httpd|wc ...

  6. 查看 并发请求数及其TCP连接状态

    服务器上的一些统计数据: 1)统计80端口连接数netstat -nat|grep -i "80"|wc -l 2)统计httpd协议连接数ps -ef|grep httpd|wc ...

  7. 查看 Apache并发请求数及其TCP连接状态【转】

    查看 Apache并发请求数及其TCP连接状态 (2011-06-27 15:08:36) 服务器上的一些统计数据: 1)统计80端口连接数netstat -nat|grep -i "80& ...

  8. Java并发工具类（三）：控制并发线程数的Semaphore

    作用 Semaphore(信号量)是用来控制同时访问特定资源的线程数量,它通过协调各个线程,以保证合理的使用公共资源. 简介 Semaphore也是一个线程同步的辅助类,可以维护当前访问自身的线程个数 ...

  9. 查看http的并发请求数及其TCP连接状态

    统计80端口的连接数据 netstat -nat | grep -i "80" | wc -l 统计httpd协议连接数 ps -ef | grep httpd | wc -l 统 ...

随机推荐

  1. linux：ping不通www.baidu.com

    如果某台Linux服务器ping不通域名, 如下提示: [root@localhost ~]# ping www.baidu.com ping: unknown host www.baidu.com ...

  2. ubuntu 12.04上安装wn725n无线网卡驱动

    系统版本:$uname -a Linux yuliang-Veriton-M670G 3.11.0-17-generic #31~precise1-Ubuntu SMP Tue Feb 4 21:25 ...

  3. WCF 404.3 MIME 映射错误

    WCF部署在IIS下,报错如下: HTTP 错误 404.3 - Not Found由于扩展配置问题而无法提供您请求的页面.如果该页面是脚本,请添加处理程序.如果应下载文件,请添加 MIME 映射. ...

  4. Linux下实现文件双向同步

    修改的地方时:私钥和公约处. 其次 vi /root/.unison/default.prf 这个文件在这个位置 root = /tmp/test root = ssh://root@10.10.3. ...

  5. <p>在我们的实际软件项目中，管理团队事实上比写代码或者实现一个客户的需求更为的有挑战性。由于编程实际上是和机器打交道，而和机器打交道，仅仅要你符合机器预定的逻辑，</p>

    在我们的实际软件项目中,管理团队事实上比写代码或者实现一个客户的需求更为的有挑战性. 由于编程实际上是和机器打交道.而和机器打交道,仅仅要你符合机器预定的逻辑, 一步步迈向解决这个问题的道路上一点都不 ...

  6. android应用中插入admob广告

    Step One  登陆admob,注册用户 直接登陆http://www.admob.com/,用google的账号登陆 Step Two 登陆admob后,在站点和应用程序选项中 选择并添加and ...

  7. LeetCode LinkList 23. Merge k Sorted Lists

    这两天一直也没有顾上记录一下自己做过的题目,回头看看,感觉忘的好快,今天做了一个hard,刚开始觉得挺难得,想了两种方法,一种是每次都从k个list中选取最小的一个,为空的直接跳过,再就是每次合并其中 ...

  8. matlab-非线性方程求根函数及函数曲线绘制

    Matlab中提供了很多求解非线性方程(y=f(x))的函数,刚開始使用,真的很困惑.全部.这里依据matlab的help文档对这些函数做一些小小的总结 fsolve函数 用来求解非线性方程组:F(x ...

  9. lvs+keepalived+bind实现负载均衡高可用智能dns

    整体架构: 1.IP地址规划: Dns1:172.28.0.54 Dns2:172.28.0.55 Dr服务器主:172.28.0.57 Dr服务器从:172.28.0.67 Vip:172.28.0 ...

  10. nginx內建模块使用

    目录 nginx內建模块使用 1. 內建模块的引入 1.1 查看安装信息 1.2 重新指定配置信息 2. 內建模块的使用 2.1 http_stub_status_module 2.2 http_ra ...