SQLite objects created in a thread can only be used in that same thread.The object was created in thread id 62332 and this is thread id 53032

sqlite支持的并发访问数？？

import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限，限制访问', 'DB_KEY_EXHAUST'

db = 'py_bdspider_status.db'

def db_init_key_table():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'DELETE  FROM  baidu_map_key_used'

    c.execute(sql)

    conn.commit()

    pcity_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

    with open(pcity_file, 'r', encoding='utf-8') as pf:

        c_ = 0

        for i in pf:

            if len(i) < 4:

                continue

            author, key = i.replace('\n', '').split('\t')

            localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

            sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

                author, key, localtime_, 0)

            c.execute(sql)

    conn.commit()

    conn.close()

# db_init_key_table()

def db_get_one_effective():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)

    res = c.execute(sql).fetchone()

    if res is None:

        return DB_KEY_EXHAUST

    else:

        return res[0]

    conn.close

def db_update_one_today_used(key):

    conn = sqlite3.connect(db)

    c = conn.cursor()

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

    conn.close()

dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'

requested_file_list = []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

def chk_if_requested_file():

    for f in requested_file_dir:

        to_in = f.split('.txt')[0]

        if to_in not in requested_file_list:

            requested_file_list.append(to_in)

def write_requested_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    print('ok', threading.get_ident(), request_name)

def write_requested_exception_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'

def fun_():

    fname_open = '%s\\%s' % (curPath, fname_source)

    FEXCEL = '%s%s' % (fname_open, '.xlsx')

    data = xlrd.open_workbook(FEXCEL)

    table = data.sheets()[0]

    nrows, ncols = table.nrows, table.ncols

    # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

    base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

    for i in range(1, nrows):

        l = table.row_values(i)

        dbid, area_code, name_, request_name, type_, city, district, addr, street = l

        request_name_chk = '%s%s%s' % (city, district, request_name)

        chk_if_requested_file()

        if request_name_chk in requested_file_list:

            continue

        ak = db_get_one_effective()

        if ak == DB_KEY_EXHAUST:

            print(DB_KEY_EXHAUST)

            break

        else:

            url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

        try:

            bd_res_json_str = requests.get(url_).text

            db_update_one_today_used(ak)

            write_requested_res(request_name_chk, bd_res_json_str)

        except Exception:

            bd_res_json_str = '请求百度-异常'

            write_requested_exception_res(request_name_chk, bd_res_json_str)

            print(bd_res_json_str)

class MyThread(threading.Thread):

    def __init__(self, func):

        threading.Thread.__init__(self)

        self.func = func

    def run(self):

        self.func()

thread_sum = 16

def main():

    threads_list = []

    for nloop in range(0, thread_sum, 1):

        thread_instance = MyThread(fun_)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

if __name__ == '__main__':

    main()

　　

import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限，限制访问', 'DB_KEY_EXHAUST'

db = 'py_bdspider_status.db'

def db_init_key_table():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'DELETE  FROM  baidu_map_key_used'

    c.execute(sql)

    conn.commit()

    pcity_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

    with open(pcity_file, 'r', encoding='utf-8') as pf:

        c_ = 0

        for i in pf:

            if len(i) < 4:

                continue

            author, key = i.replace('\n', '').split('\t')

            localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

            sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

                author, key, localtime_, 0)

            c.execute(sql)

    conn.commit()

    conn.close()

# db_init_key_table()

def db_get_one_effective():

    conn = sqlite3.connect(db)

    c = conn.cursor()

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)

    res = c.execute(sql).fetchone()

    if res is None:

        return DB_KEY_EXHAUST

    else:

        return res[0]

    conn.close

def db_update_one_today_used(key):

    conn = sqlite3.connect(db)

    c = conn.cursor()

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

    conn.close()

dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'

requested_file_list = []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

def chk_if_requested_file():

    for f in requested_file_dir:

        to_in = f.split('.txt')[0]

        if to_in not in requested_file_list:

            requested_file_list.append(to_in)

def write_requested_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    print('ok', threading.get_ident(), request_name)

def write_requested_exception_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'

def fun_():

    fname_open = '%s\\%s' % (curPath, fname_source)

    FEXCEL = '%s%s' % (fname_open, '.xlsx')

    data = xlrd.open_workbook(FEXCEL)

    table = data.sheets()[0]

    nrows, ncols = table.nrows, table.ncols

    # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

    base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

    i_counter = 0

    for i in range(1, nrows):

        l = table.row_values(i)

        i_counter += 1

        dbid, area_code, name_, request_name, type_, city, district, addr, street = l

        request_name_chk = '%s%s%s' % (city, district, request_name)

        chk_if_requested_file()

        if request_name_chk in requested_file_list:

            continue

        ak = db_get_one_effective()

        if ak == DB_KEY_EXHAUST:

            print(DB_KEY_EXHAUST)

            break

        else:

            url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

        try:

            bd_res_json_str = requests.get(url_).text

            db_update_one_today_used(ak)

            write_requested_res(request_name_chk, bd_res_json_str)

        except Exception:

            bd_res_json_str = '请求百度-异常'

            write_requested_exception_res(request_name_chk, bd_res_json_str)

            print(bd_res_json_str)

class MyThread(threading.Thread):

    def __init__(self, func):

        threading.Thread.__init__(self)

        self.func = func

    def run(self):

        self.func()

thread_sum = 100

def main():

    threads_list = []

    for nloop in range(0, thread_sum, 1):

        thread_instance = MyThread(fun_)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

if __name__ == '__main__':

    main()

　　

import xlrd

import time

import sys

import os

import requests

import sqlite3

import threading

curPath = os.path.abspath(os.path.dirname(__file__))

rootPath = os.path.split(curPath)[0]

sys.path.append(rootPath)

db = 'py_bdspider_status.db'

conn = sqlite3.connect(db)

c = conn.cursor()

MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限，限制访问', 'DB_KEY_EXHAUST'

def db_init_key_table():

    sql = 'DELETE  FROM  baidu_map_key_used'

    c.execute(sql)

    conn.commit()

    pcity_file = '%s\\%s' % (curPath, 'bdmap_key.txt')

    with open(pcity_file, 'r', encoding='utf-8') as pf:

        c_ = 0

        for i in pf:

            if len(i) < 4:

                continue

            author, key = i.replace('\n', '').split('\t')

            localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

            sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (

                author, key, localtime_, 0)

            c.execute(sql)

    conn.commit()

# db_init_key_table()

def db_get_one_effective():

    sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)

    res = c.execute(sql).fetchone()

    if res is None:

        return DB_KEY_EXHAUST

    else:

        return res[0]

def db_update_one_today_used(key):

    localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())

    sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (

        localtime_, key)

    c.execute(sql)

    conn.commit()

dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'

requested_file_list = []

requested_file_dir_str, requested_file_dir_exception_str = '%s\\%s\\' % (curPath, dir_), '%s\\%s\\' % (

    curPath, dir_exception)

requested_file_dir = os.listdir(requested_file_dir_str)

def chk_if_requested_file():

    for f in requested_file_dir:

        to_in = f.split('.txt')[0]

        if to_in not in requested_file_list:

            requested_file_list.append(to_in)

def write_requested_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

    print('ok', threading.get_ident(), request_name)

def write_requested_exception_res(request_name, str_, type_='.txt'):

    fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)

    with open(fname, 'w', encoding='utf-8') as ft:

        ft.write(str_)

fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'

def fun_():

    fname_open = '%s\\%s' % (curPath, fname_source)

    FEXCEL = '%s%s' % (fname_open, '.xlsx')

    data = xlrd.open_workbook(FEXCEL)

    table = data.sheets()[0]

    nrows, ncols = table.nrows, table.ncols

    # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1

    base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'

    i_counter = 0

    for i in range(1, nrows):

        l = table.row_values(i)

        i_counter += 1

        print(i_counter)

        if i_counter > 60:

            break

        dbid, area_code, name_, request_name, type_, city, district, addr, street = l

        request_name_chk = '%s%s%s' % (city, district, request_name)

        chk_if_requested_file()

        if request_name_chk in requested_file_list:

            continue

        ak = db_get_one_effective()

        if ak == DB_KEY_EXHAUST:

            print(DB_KEY_EXHAUST)

            break

        else:

            url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)

            print(url_)

        try:

            bd_res_json_str = requests.get(url_).text

            db_update_one_today_used(ak)

            write_requested_res(request_name_chk, bd_res_json_str)

        except Exception:

            bd_res_json_str = '请求百度-异常'

            write_requested_exception_res(request_name_chk, bd_res_json_str)

            print(bd_res_json_str)

class MyThread(threading.Thread):

    def __init__(self, func):

        threading.Thread.__init__(self)

        self.func = func

    def run(self):

        self.func()

thread_sum = 4

def main():

    threads_list = []

    for nloop in range(0, thread_sum, 1):

        thread_instance = MyThread(fun_)

        threads_list.append(thread_instance)

    for t in threads_list:

        t.setDaemon = False

        t.start()

    for t in threads_list:

        t.join()

    conn.close()

if __name__ == '__main__':

    main()

　　

SQLite支持的并发访问数的更多相关文章

  1. 查看apache当前并发访问数和进程数

    1.查看apache当前并发访问数: netstat -an | grep ESTABLISHED | wc -l 对比httpd.conf中MaxClients的数字差距多少. 2.查看有多少个进程 ...

  2. 高效解决「SQLite」数据库并发访问安全问题，只这一篇就够了

    Concurrent database access 本文译自:https://dmytrodanylyk.com/articles/concurrent-database/ 对于 Android D ...

  3. Socket支持多用户并发访问的解决办法

    //创建线程池,池中具有(cpu个数*50)条线程 ExecutorService executorService = Executors.newFixedThreadPool(Runtime.get ...

  4. 查看 Apache并发请求数及其TCP连接状态

    查看 Apache并发请求数及其TCP连接状态 (2011-06-27 15:08:36) 服务器上的一些统计数据: 1)统计80端口连接数 netstat -nat|grep -i "80 ...

  5. 查看 并发请求数及其TCP连接状态【转】

    服务器上的一些统计数据: 1)统计80端口连接数netstat -nat|grep -i "80"|wc -l 2)统计httpd协议连接数ps -ef|grep httpd|wc ...

  6. 查看 并发请求数及其TCP连接状态

    服务器上的一些统计数据: 1)统计80端口连接数netstat -nat|grep -i "80"|wc -l 2)统计httpd协议连接数ps -ef|grep httpd|wc ...

  7. 查看 Apache并发请求数及其TCP连接状态【转】

    查看 Apache并发请求数及其TCP连接状态 (2011-06-27 15:08:36) 服务器上的一些统计数据: 1)统计80端口连接数netstat -nat|grep -i "80& ...

  8. Java并发工具类（三）：控制并发线程数的Semaphore

    作用 Semaphore(信号量)是用来控制同时访问特定资源的线程数量,它通过协调各个线程,以保证合理的使用公共资源. 简介 Semaphore也是一个线程同步的辅助类,可以维护当前访问自身的线程个数 ...

  9. 查看http的并发请求数及其TCP连接状态

    统计80端口的连接数据 netstat -nat | grep -i "80" | wc -l 统计httpd协议连接数 ps -ef | grep httpd | wc -l 统 ...

随机推荐

  1. ElastcSearch的Mapping映射建立

    根据oracle的字段来建立ElasticSearch的Mapping public class Start { private static Logger log = LoggerFactory.g ...

  2. 转：敏捷方式scrum 方案

    http://www.cnblogs.com/taven/archive/2010/10/17/1853386.html 现在敏捷开发是越来越火了,人人都在谈敏捷,人人都在学习Scrum和XP... ...

  3. C++ 利用文件流复制文件

    bool CopyFile(const std::string &src, const std::string &dest) { std::ifstream fin(src.c_str ...

  4. 渐进式 JPEG （Progressive JPEG）来提升用户体验

    1.概述 jpg格式分为:Baseline JPEG(标准型)和Progressive JPEG(渐进式).两种格式有相同尺寸以及图像数据,扩展名也是相同的,唯一的区别是二者显示的方式不同. Base ...

  5. Nginx:解析HTTP配置的流程

    参考资料:深入理解Nginx(陶辉) 书中有详细的讲解,这里只用本人的理解梳理一下该流程. 一点提议:对于像我这样的新手,面对暂时看不懂章节,建议先往下看一下(可能就会有新的理解或灵感),而不要死磕在 ...

  6. Linux下画原理图和PCB

    Linux下画原理图和PCB Windows下大名鼎鼎的Allegro和经典的Protel 99SE都是不支持Linux操作系统的.做Linux驱动开发免不了要看一下原理图和PCB. 一般的做法有三种 ...

  7. Qemu线程池介绍

    有时我们希望把一部分工作通过创建线程的方式异步执行,这样我们可以在执行任务的同时,继续执行其他任务.但是如果这种需求比较多的话,频繁的创建和销毁线程带来很大的性能损耗.如果我们能创建一个或一些线程,然 ...

  8. IntelliJ IDEA 、genymotion模拟器、Android开发环境搭建

    首先打开IDEA,看到该界面,如果没有该界面,请在User/用户名/IntelliJIDEAProjects/下删除所有项目文件夹.然后重启IDEA即可看到 接着开始配置jdk和sdk 然后在Proj ...

  9. 关于.cap文件分析

    CAP文件是比较通用的一种文件格式,基本上大多数抓包软件都支持以此格式将捕获的网络数据包存储下来.    需要说明的是,CAP文件存储下来的,一般都是网络数据帧.不同网络传输协议下的帧格式是有差异的, ...

  10. .net之GridView、DataList、DetailsView（二）

    GridView自带分页功能,效率不高,适用于小数据量的显示. protected void GridView1_PageIndexChanging(object sender, GridViewPa ...