import sys
import os curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath) from selenium import webdriver
from selenium.webdriver.chrome.options import Options
# from selenium.webdriver.firefox.options import Options
import time
from time import sleep
import math
import random
import sys
import threading
import urllib.parse
import xlrd
import sys
import os
import sqlite3 MAX_TIME = 600 def py_stop_update_db():
# db = 'py_bdspider_status.db'
# db = '%s\\%s' % (curPath, db)
# conn = sqlite3.connect(db)
# pyname = os.path.basename(__file__).split('.py')[0]
# sql_ = '%s%s%s' % ('UPDATE pystatus_table SET pystatus =2 WHERE pyname="', pyname, '"')
# print(sql_)
# conn.execute(sql_)
# conn.commit()
# conn.close()
return def chk_time(browser, start_time):
if time.time() - start_time > MAX_TIME:
py_stop_update_db()
browser.delete_all_cookies()
browser.quit()
return dir_html = 'baidu_map_html_firstpage_pc_not_shop'
filepath = '%s\\%s' % (curPath, dir_html)
requested_file_list = []
pathDir = os.listdir(filepath)
for allDir in pathDir:
child = os.path.join('%s%s' % (filepath, allDir))
requested_file = child.split(dir_html)[1].split('&')[0].split('.html')[0]
requested_file_list.append(requested_file) tag_jmtool_list = ['(', '(', '-'] def extract_name(name_):
for i in tag_jmtool_list:
name_ = name_.split(i)[0]
return name_ pcity_list = []
pcity_file = '%s\\%s' % (curPath, '省会城市.txt')
with open(pcity_file, 'r', encoding='utf-8') as pf:
c_ = 0
for i in pf:
c_ += 1
if c_ == 3:
c_ = 0
pcity_list.append(i.replace(' ', '').replace('\n', '') + '市')
pcity_sorted_list = sorted(pcity_list) target_type_list = ['住宅小区', '写字楼']
# target_type_list = ['住宅小区']
target_type_list = ['专科医院']
target_type_list = ['商场']
requested_type_counter = 0
# 商场 4705 酒店 24915 专科医院 2513 商圈 334
target_dic = {}
# target_city_list = ['北京市', '上海市', '深圳市', '广州市']
target_city_list = ['深圳市', '广州市']
target_city_list = ['深圳市']
target_city_list = ['北京市', '上海市']
target_city_list = ['北京市', '上海市', '深圳市', '广州市']
target_city_list = ['北京市', '上海市']
target_city_list = ['深圳市', '广州市']
target_city_list = ['北京市']
target_city_list = ['北京市', '上海市', '深圳市', '广州市']
target_city_list = pcity_sorted_list[21:28]
#pcity_sorted_list[7:14]
#target_city_list = pcity_sorted_list # target_city_list = ['杭州市']
file_name = 'JMTool任务_csv_py_wholeCSV' FEXCEL = '%s\\%s%s' % (curPath, file_name, '.xlsx')
data = xlrd.open_workbook(FEXCEL)
table = data.sheets()[0]
nrows, ncols = table.nrows, table.ncols
res_dic, counter_ = {}, 0
for i in range(0, nrows):
l = table.row_values(i)
dbid, area_code, ref_area_type_code, city, district, address, city_street, name_, emp_, emp_1 = l
if city not in target_city_list:
continue
# if city not in target_city_list:
# target_city_list.append(city)
type_ = ref_area_type_code
if type_ not in target_type_list:
continue
name_ = name_.replace('?', '')
name_reduction = extract_name(name_)
if len(name_reduction) < 3:
name_reduction = name_
if city not in target_dic:
target_dic[city] = {}
if district not in target_dic[city]:
target_dic[city][district] = {}
if type_ not in target_dic[city][district]:
target_dic[city][district][type_] = {}
if name_reduction not in target_dic[city][district]:
target_dic[city][district][type_][name_reduction] = {}
target_dic[city][district][type_][name_reduction]['name_reduction_list'] = []
target_dic[city][district][type_][name_reduction]['history_list'] = []
try:
target_dic[city][district][type_][name_reduction]['name_reduction_list'].append(name_)
target_dic[city][district][type_][name_reduction]['history_list'].append(l)
except Exception:
print(Exception) write_res_html_dir = '%s\\%s\\' % (curPath, dir_html) def write_res_html(browser, dir_=write_res_html_dir):
close_alert(browser)
current_url_ = urllib.parse.unquote(browser.current_url)
try:
input_ = current_url_.split('&wd=')[1].split('/?')[0]
except Exception:
print('Exception-', __file__, sys._getframe().f_lineno, current_url_)
return
current_url_ = '%s%s%s' % ('<!--', browser.current_url, '-->')
page_source = '%s%s' % (current_url_, browser.page_source)
# localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
# file_name = '%s%s%s%s' % (dir_, input_, localtime_, '.html')
file_name = '%s%s%s' % (dir_, input_, '.html')
fo = open(file_name, 'w', encoding='utf-8')
fo.write(page_source)
fo.closed
print(os.path.basename(__file__), 'OK-writed-', sys._getframe().f_lineno, '') def gen_random_letter():
return chr(random.randint(97, 122)) def gen_random_num():
return random.randint(0, 10) def gen_sougo_pid():
res_ = ''
for i in range(1, 17, 1):
if i in [1, 3, 4, 15]:
res_ = '%s%s' % (res_, gen_random_letter())
else:
res_ = '%s%s' % (res_, gen_random_num())
return res_ def close_alert(browser, attitude='accept'):
return # executable_path_str = '%s\\%s' % (curPath, 'geckodriver.exe')
# browser = webdriver.Firefox(executable_path=executable_path_str) def mobile_mobile_pages_html(browser, input_):
start_time = time.time()
chk_time(browser, start_time)
sleep(3)
url_ = '%s%s' % ('http://map.baidu.com/?s=s%26wd%3D', input_)
browser.get(url_)
write_res_html(browser) class MyThread(threading.Thread):
def __init__(self, func, args, name):
threading.Thread.__init__(self)
self.name, self.func, self.args = name, func, args def run(self):
self.func(self.args) def thread_city(city):
global requested_type_counter
for district in target_dic[city]:
for type_ in target_dic[city][district]:
for name_reduction in target_dic[city][district][type_]:
for name_ in target_dic[city][district][type_][name_reduction]['name_reduction_list']:
input_ = '%s%s%s' % (city, district, name_)
if input_ in requested_file_list:
requested_type_counter += 1
print('requested_type_counter=', requested_type_counter, input_)
else:
# executable_path_str = '%s\\%s' % (curPath, 'chromedriver.exe')
# browser = webdriver.Chrome(executable_path=executable_path_str) executable_path_str = '%s\\%s' % (curPath, 'geckodriver.exe')
browser = webdriver.Firefox(executable_path=executable_path_str)
mobile_mobile_pages_html(browser, input_) threads_list = []
for city in target_dic:
thread_instance = MyThread(thread_city, (city), thread_city.__name__)
threads_list.append(thread_instance)
for t in threads_list:
t.setDaemon = False
t.start()
for t in threads_list:
t.join() # browser.delete_all_cookies()
# browser.quit()

  

Is this its limit?的更多相关文章

  1. mysql limit分页查询优化写法

    在mysql中进行分页查询时,一般会使用limit查询,而且通常查询中都会使用orderby排 序.但是在表数据量比较大的时候,例如查询语句片段limit 10000, 20,数据库会读取10020条 ...

  2. SQL中TOP,LIMIT,ROWNUM的用法

    SQL SERVER/MS Access的Select Top的用法: Select TOP number|percent table_columname FROM tablename MySQL/O ...

  3. mysql 中的LIMIT用法

    select * from table_name LIMIT 起始偏移量,数量 (1)起始偏移量为0:代表没有偏移,即从第1行开始. (2)数量为-1:代表是无穷,即偏移量之后所有的行. (3)LIM ...

  4. laravel 框架使用总结 limit

    后台开发就是数据的各种处理很多时候需要做到分页,但是在laravel中使用limit做分页的时候会出现问题,偏移量和每页的条数放进去好像不好使了 下面推荐给大家一种在laravel框架中非常好用的写法 ...

  5. [软件推荐]快速文件复制工具(Limit Copy) V4.0 绿色版

    快速文件复制工具(Limit Copy)绿色版是一款智能变频超快复制绿色软件. 快速文件复制工具(Limit Copy)功能比较完善,除了文件复制还可以智能变频,直接把要复制的文件拖入窗口即可,无需手 ...

  6. java.lang.OutOfMemoryError:GC overhead limit exceeded填坑心得

    我遇到这样的问题,本地部署时抛出异常java.lang.OutOfMemoryError:GC overhead limit exceeded导致服务起不来,查看日志发现加载了太多资源到内存,本地的性 ...

  7. TNS-12540: TNS:internal limit restriction exceeded

    应用程序以及客户端工具(Toad.PL/SQL Developer等)出现突然连接不上数据库服务器的情况,监听日志listener.log里面出现了TSN-12518与TSN-12540错误,如下所示 ...

  8. -bash: ulimit: pipe size: cannot modify limit: Invalid argument

    从root账号切换到oracle账号时,出现了"-bash: ulimit: pipe size: cannot modify limit: Invalid argument"提示 ...

  9. [MySQL性能优化系列]LIMIT语句优化

    1. 背景 假设有如下SQL语句: SELECT * FROM table1 LIMIT offset, rows 这是一条典型的LIMIT语句,常见的使用场景是,某些查询返回的内容特别多,而客户端处 ...

  10. migration integer limit option

    https://gist.github.com/stream7/1069589 :limit Numeric Type Column Size Max value 1 tinyint 1 byte 1 ...

随机推荐

  1. spring-hadoop-samples

    官方的spring-hadoop-samples的demo 写的还是挺好的,值得学习. 官网地址: http://projects.spring.io/spring-hadoop/#quick-sta ...

  2. Angular 学习笔记——filter

    <!DOCTYPE html> <html lang="en" ng-app="myApp"> <head> <met ...

  3. ionic准备之angular基础——dom操作相关(6)

    <!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8&quo ...

  4. java web 分页实现

       分页实现的效果:    ///////// /////////////////////////////////////////////////////// /////////////////// ...

  5. _.pick lodash

    http://lodash.think2011.net/pick _.pick(object, [props]) 创建一个从 object 中选中的属性的对象. 参数 object (Object) ...

  6. 跟踪运行时错误 vue

    如果在组件渲染时出现运行错误,错误将会被传递至全局 Vue.config.errorHandler 配置函数 (如果已设置).利用这个钩子函数来配合错误跟踪服务是个不错的主意.比如 Sentry,它为 ...

  7. 扩展Unity3D编辑器的脚本模板

      近期在学习Shader时感觉Shader语言參数众多.语法诡异,假设每次都从头開始写Shader一定是一件痛苦的事情.假设能够在本地定义好一组标准的Shader模板,这样当我们须要实现某些效果相似 ...

  8. 转 拉姆达表达式,委托、匿名方法、Lambda表达式的演进

    总结:Lambda表达式的语法:(参数列表=>执行语句) 无参数格式 :()=>{执行语句} 有参数格式:x=> x % 2 == 0 1.假设给我们一个泛型对象List<T& ...

  9. PHP设置会话(Session)超时过期时间实现登录时间限制

    最近某个PHP项目用到了限制登录时间的功能,比如用户登录系统60分钟后如果没有操作就自动退出,我搜索了网络收集了有以下方法可供参考. 第一种方法即设置php.ini配置文件,设置session.gc_ ...

  10. Angular 资料大集合

    https://angular.cn/   Angular 的中文网 http://www.apjs.net/#dir1    Angular 的中文网 http://www.ngnice.com/  ...