单线程爬虫：

import re

import requests

import time

url_EB = 'http://www.amazon.com/gp/search/other/ref=sr_sa_p_4?me=A22XNR713HGDVG&rh=n%3A9063592011%2Ck%3Aprojector&bbn=9063592011&keywords=projector&pickerToList=brandtextbin&ie=UTF8&qid=1461902521'

headers_EB = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.86 Safari/537.36'}

url_AML = '''https://www.amazon.com/gp/search/other/ref=sr_sa_p_4?me=A3UJI9WWE6PRP5&rh=i%3Amerchant-items

&pickerToList=brandtextbin&ie=UTF8&qid=1461899728'''

headers_AML ={'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.86 Safari/537.36'}

url_DL= 'https://www.amazon.com/gp/search/other/ref=sr_sa_p_4?me=AS7ZU4MN0FPOY&rh=i%3Amerchant-items&pickerToList=brandtextbin&ie=UTF8&qid=1461901862'

headers_DL = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.86 Safari/537.36'}

name = {'a':'ExclusiveBulbs',

        'b':'Amazing Lamps',

        'c':'Dynamic Lamps'}

# listing_count = re.findall('<span class="narrowValue">(.*?)</span',data.text)

# f = dict(map(lambda x,y:[x,y],store_name,listing_count))

#

# for k,v in f.items():

#     print(k,v)

def foo_one(url,headers,name):

    print('--------------------------开始爬去{0}at{1}---------------------------'.format(name,time.ctime()))

    response = requests.get(url,headers=headers)

    store_name = re.findall('<span class="refinementLink">(.*?)</span><span class="narrowValue">(.*?)</span',response.text)

    for i in store_name:

        print(i)

    print('--------------------------爬去完毕at{}----------------------------'.format(time.ctime()))

    time.sleep(1)

if __name__ == '__main__':

    foo_one(url_EB,headers_EB,name['a'])

    foo_one(url_AML,headers_AML,name['b'])

    foo_one(url_DL,headers_DL,name['c'])

输出：00:25:33开始，00:26:02结束 耗时29秒

--------------------------开始爬去ExclusiveBulbsatSat Apr 30 00:25:33 2016---------------------------

('A.Shine', ' (97)')

('AmpacElectronics', ' (1,644)')

('AuraBeam', ' (33,084)')

('AWO', ' (1,206)')

('Battery1inc', ' (694)')

('Comoze Lamps', ' (6,172)')

('Compatible Lamp', ' (317)')

('Corgi Lamps', ' (2,124)')

('CTLAMP', ' (3,499)')

('Dell', ' (191)')

('Diamond Lamps', ' (966)')

('Dynamic', ' (4)')

('Eiki', ' (460)')

('ePharos', ' (2,592)')

('Epson', ' (1,456)')

('EREPLACEMENT', ' (115)')

('eReplacements', ' (814)')

('eWo's', ' (120)')

('eWorldlamp', ' (354)')

('FI Lamps', ' (5,707)')

('FL Projector Lamp For Mitsubishi', ' (1)')

('For Epson', ' (3)')

('Generic', ' (9,769)')

('Good Lamp', ' (819)')

('HCDZ', ' (2,746)')

('Hitachi', ' (935)')

('IET Lamps', ' (2,144)')

('InFocus', ' (44)')

('JVC', ' (326)')

('KCL', ' (3,781)')

('Lampedia', ' (618)')

('Lutema', ' (1,956)')

('Mitsubishi', ' (1,006)')

('Mogobe', ' (1,335)')

('MyProjectorLamps', ' (473)')

('NEC', ' (446)')

('Nec Computers', ' (13)')

('Optoma', ' (956)')

('Osram Sylvania', ' (78)')

('Panasonic', ' (820)')

('Philips', ' (7,502)')

('Powerwarehouse', ' (9,971)')

('Projector Lamps World', ' (112)')

('Pureglare', ' (369)')

('Samsung', ' (1,078)')

('Sharp', ' (426)')

('Shopforbattery', ' (2,510)')

('SMART BOARD', ' (66)')

('Sony', ' (990)')

('TVLampsforless', ' (14)')

('Unknown', ' (722)')

--------------------------爬去完毕atSat Apr 30 00:25:57 2016----------------------------

--------------------------开始爬去Amazing LampsatSat Apr 30 00:25:58 2016---------------------------

('AWO', ' (1)')

('Comoze Lamps', ' (2)')

('DNGO', ' (8)')

('Electrified', ' (9)')

('ELECTRIFIED', ' (10)')

('Electrified Discounters', ' (5)')

('ELECTRIFIED LAMPS', ' (1,177)')

('ELECTRIFIED PRINTHEAD', ' (24)')

('ELECTRIFIED PRINTHEADS', ' (2)')

('FI Lamps', ' (2)')

('Generic', ' (34)')

('GloWatt', ' (1)')

('KCL', ' (1)')

('OEM', ' (1)')

('Powerwarehouse', ' (7)')

('SKU', ' (5)')

('Top Lamp', ' (1)')

('Unknown', ' (1)')

('USOM', ' (3)')

--------------------------爬去完毕atSat Apr 30 00:26:00 2016----------------------------

--------------------------开始爬去Dynamic LampsatSat Apr 30 00:26:01 2016---------------------------

('Battery1inc', ' (85)')

('BenQ', ' (237)')

('Buslink', ' (31)')

('Calumet', ' (2)')

('Comoze Lamps', ' (405)')

('CTLAMP', ' (615)')

('Dell', ' (82)')

('Divine Lighting', ' (36)')

('DNGO', ' (63)')

('Dynamic', ' (4)')

('Eiko', ' (140)')

('Electrified', ' (2)')

('ELECTRIFIED LAMPS', ' (24)')

('Electronix Xpress', ' (418)')

('ePharos', ' (502)')

('Epson', ' (631)')

('eReplacements', ' (119)')

('FI Lamps', ' (505)')

('FL Projector Lamp For Mitsubishi', ' (1)')

('G-lamps', ' (43)')

('GE', ' (248)')

('GE Lighting', ' (152)')

('General Electric', ' (53)')

('Generic', ' (1,671)')

('Genie', ' (101)')

('GLAMPS', ' (2)')

('Impact', ' (7)')

('Industrial Lighting Solutions', ' (9)')

('KCL', ' (280)')

('Kodak', ' (1)')

('Lampedia', ' (63)')

('M-Wave', ' (830)')

('Mitsubishi', ' (406)')

('Mitsubishi DLP TV Bulbs', ' (29)')

('Mocpinc', ' (10)')

('MyProjectorLamps', ' (344)')

('Nec', ' (19)')

('Optoma', ' (161)')

('Osram', ' (1,295)')

('Panasonic', ' (245)')

('Philips', ' (988)')

('Powerwarehouse', ' (239)')

('Projector Lamps World', ' (45)')

('Pureglare', ' (107)')

('Samsung', ' (323)')

('ShopJimmy', ' (3)')

('Sony', ' (141)')

('Sylvania', ' (115)')

('Technical Precision', ' (10)')

('Unknown', ' (167)')

('Welch Allyn Compatible', ' (1)')

--------------------------爬去完毕atSat Apr 30 00:26:02 2016----------------------------

多线程：00：32:37开始00:32:39结束 耗时2秒

import re

import requests

import threading

import time

from time import ctime,sleep

url_EB = 'http://www.amazon.com/gp/search/other/ref=sr_sa_p_4?me=A22XNR713HGDVG&rh=n%3A9063592011%2Ck%3Aprojector&bbn=9063592011&keywords=projector&pickerToList=brandtextbin&ie=UTF8&qid=1461902521'

headers_EB = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.86 Safari/537.36'}

url_AML = '''https://www.amazon.com/gp/search/other/ref=sr_sa_p_4?me=A3UJI9WWE6PRP5&rh=i%3Amerchant-items

&pickerToList=brandtextbin&ie=UTF8&qid=1461899728'''

headers_AML ={'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.86 Safari/537.36'}

url_DL= 'https://www.amazon.com/gp/search/other/ref=sr_sa_p_4?me=AS7ZU4MN0FPOY&rh=i%3Amerchant-items&pickerToList=brandtextbin&ie=UTF8&qid=1461901862'

headers_DL = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.86 Safari/537.36'}

name = {'a':'ExclusiveBulbs',

        'b':'Amazing Lamps',

        'c':'Dynamic Lamps'}

# listing_count = re.findall('<span class="narrowValue">(.*?)</span',data.text)

# f = dict(map(lambda x,y:[x,y],store_name,listing_count))

#

# for k,v in f.items():

#     print(k,v)

def foo_one(url,headers,name):

    print('--------------------------开始爬去{0}at{1}---------------------------'.format(name,time.ctime()))

    response = requests.get(url,headers=headers)

    store_name = re.findall('<span class="refinementLink">(.*?)</span><span class="narrowValue">(.*?)</span',response.text)

    for i in store_name:

        print(i)

    print('--------------------------爬去完毕{0}at{1}----------------------------'.format(name,time.ctime()))

threads = []

t1 = threading.Thread(target=foo_one,args=(url_EB,headers_EB,name['a']))

threads.append(t1)

t2 = threading.Thread(target=foo_one,args=(url_AML,headers_AML,name['b']))

threads.append(t2)

t3 = threading.Thread(target=foo_one,args=(url_DL,headers_DL,name['c']))

threads.append(t3)

if __name__ == '__main__':

    for t in threads:

        t.setDaemon(True)

        t.start()

    t.join()

    print ("all over %s" %ctime())

输出：

--------------------------开始爬去ExclusiveBulbsatSat Apr 30 00:32:37 2016---------------------------

--------------------------开始爬去Amazing LampsatSat Apr 30 00:32:37 2016---------------------------

--------------------------开始爬去Dynamic LampsatSat Apr 30 00:32:37 2016---------------------------

('A.Shine', ' (97)')

('AmpacElectronics', ' (1,645)')

('AuraBeam', ' (33,088)')

('AWO', ' (1,209)')

('Battery1inc', ' (694)')

('Comoze Lamps', ' (6,172)')

('Compatible Lamp', ' (317)')

('Corgi Lamps', ' (2,123)')

('CTLAMP', ' (3,501)')

('Dell', ' (191)')

('Diamond Lamps', ' (966)')

('Dynamic', ' (4)')

('Eiki', ' (457)')

('ePharos', ' (2,592)')

('Epson', ' (1,456)')

('EREPLACEMENT', ' (115)')

('eReplacements', ' (813)')

('eWo's', ' (120)')

('eWorldlamp', ' (354)')

('FI Lamps', ' (5,710)')

('FL Projector Lamp For Mitsubishi', ' (1)')

('For Epson', ' (3)')

('Generic', ' (9,771)')

('Good Lamp', ' (819)')

('HCDZ', ' (2,748)')

('Hitachi', ' (935)')

('IET Lamps', ' (2,137)')

('InFocus', ' (44)')

('JVC', ' (326)')

('KCL', ' (3,783)')

('Lampedia', ' (618)')

('Lutema', ' (1,955)')

('Mitsubishi', ' (1,006)')

('Mogobe', ' (1,336)')

('MyProjectorLamps', ' (473)')

('NEC', ' (450)')

('Nec Computers', ' (13)')

('Optoma', ' (956)')

('Osram Sylvania', ' (78)')

('Panasonic', ' (820)')

('Philips', ' (7,502)')

('Powerwarehouse', ' (9,972)')

('Projector Lamps World', ' (112)')

('Pureglare', ' (369)')

('Samsung', ' (1,078)')

('Sharp', ' (426)')

('Shopforbattery', ' (2,511)')

('SMART BOARD', ' (66)')

('Sony', ' (990)')

('TVLampsforless', ' (14)')

('Unknown', ' (722)')

--------------------------爬去完毕ExclusiveBulbsatSat Apr 30 00:32:38 2016----------------------------

('Battery1inc', ' (85)')

('BenQ', ' (237)')

('Buslink', ' (31)')

('Calumet', ' (2)')

('Comoze Lamps', ' (405)')

('CTLAMP', ' (615)')

('Dell', ' (82)')

('Divine Lighting', ' (36)')

('DNGO', ' (63)')

('Dynamic', ' (4)')

('Eiko', ' (140)')

('Electrified', ' (2)')

('ELECTRIFIED LAMPS', ' (24)')

('Electronix Xpress', ' (418)')

('ePharos', ' (502)')

('Epson', ' (631)')

('eReplacements', ' (119)')

('FI Lamps', ' (505)')

('FL Projector Lamp For Mitsubishi', ' (1)')

('G-lamps', ' (43)')

('GE', ' (248)')

('GE Lighting', ' (152)')

('General Electric', ' (53)')

('Generic', ' (1,671)')

('Genie', ' (101)')

('GLAMPS', ' (2)')

('Impact', ' (7)')

('Industrial Lighting Solutions', ' (9)')

('KCL', ' (280)')

('Kodak', ' (1)')

('Lampedia', ' (63)')

('M-Wave', ' (830)')

('Mitsubishi', ' (406)')

('Mitsubishi DLP TV Bulbs', ' (29)')

('Mocpinc', ' (10)')

('MyProjectorLamps', ' (344)')

('Nec', ' (19)')

('Optoma', ' (161)')

('Osram', ' (1,295)')

('Panasonic', ' (245)')

('Philips', ' (988)')

('Powerwarehouse', ' (239)')

('Projector Lamps World', ' (45)')

('Pureglare', ' (107)')

('Samsung', ' (323)')

('ShopJimmy', ' (3)')

('Sony', ' (141)')

('Sylvania', ' (115)')

('Technical Precision', ' (10)')

('Unknown', ' (167)')

('Welch Allyn Compatible', ' (1)')

--------------------------爬去完毕Dynamic LampsatSat Apr 30 00:32:39 2016----------------------------

all over Sat Apr 30 00:32:39 2016

　　

单线程爬虫VS多线程爬虫的效率对比的更多相关文章

  1. 【Python爬虫实战】多线程爬虫---糗事百科段子爬取

    多线程爬虫:即程序中的某些程序段并行执行,合理地设置多线程,可以让爬虫效率更高糗事百科段子普通爬虫和多线程爬虫分析该网址链接得出:https://www.qiushibaike.com/8hr/pag ...

  2. Java 多线程爬虫及分布式爬虫架构探索

    这是 Java 爬虫系列博文的第五篇,在上一篇 Java 爬虫服务器被屏蔽,不要慌,咱们换一台服务器 中,我们简单的聊反爬虫策略和反反爬虫方法,主要针对的是 IP 被封及其对应办法.前面几篇文章我们把 ...

  3. Java 多线程爬虫及分布式爬虫架构

    这是 Java 爬虫系列博文的第五篇,在上一篇 Java 爬虫服务器被屏蔽,不要慌,咱们换一台服务器 中,我们简单的聊反爬虫策略和反反爬虫方法,主要针对的是 IP 被封及其对应办法.前面几篇文章我们把 ...

  4. python多线程爬虫设计及实现示例

    爬虫的基本步骤分为:获取,解析,存储.假设这里获取和存储为io密集型(访问网络和数据存储),解析为cpu密集型.那么在设计多线程爬虫时主要有两种方案:第一种方案是一个线程完成三个步骤,然后运行多个线程 ...

  5. 抓包分析、多线程爬虫及xpath学习

    1.抓包分析 1.1 Fiddler安装及基本操作 由于很多网站采用的是HTTPS协议,而fiddler默认不支持HTTPS,先通过设置使fiddler能抓取HTTPS网站,过程可参考(https:/ ...

  6. python爬虫之多线程、多进程+代码示例

    python爬虫之多线程.多进程 使用多进程.多线程编写爬虫的代码能有效的提高爬虫爬取目标网站的效率. 一.什么是进程和线程 引用廖雪峰的官方网站关于进程和线程的讲解: 进程:对于操作系统来说,一个任 ...

  7. c#中@标志的作用 C#通过序列化实现深表复制 细说并发编程-TPL 大数据量下DataTable To List效率对比 【转载】C#工具类：实现文件操作File的工具类 异步多线程 Async .net 多线程 Thread ThreadPool Task .Net 反射学习

    c#中@标志的作用   参考微软官方文档-特殊字符@,地址 https://docs.microsoft.com/zh-cn/dotnet/csharp/language-reference/toke ...

  8. 【python3两小时快速入门】入门笔记03：简单爬虫+多线程爬虫

    作用,之间将目标网页保存金本地 1.爬虫代码修改自网络,目前运行平稳,博主需要的是精准爬取,数据量并不大,暂未加多线程. 2.分割策略是通过查询条件进行分类,循环启动多条线程. 1.单线程简单爬虫(第 ...

  9. Python爬虫进阶 | 多线程

    一.简介 为了提高爬虫程序效率,由于python解释器GIL,导致同一进程中即使有多个线程,实际上也只会有一个线程在运行,但通过request.get发送请求获取响应时有阻塞,所以采用了多线程依然可以 ...

随机推荐

  1. GSM模块_GPRS数据传输机制和原理

    通信专业术语 BSS--基站子系统,通过无线接口与移动台直接联系,负责在一定区域内和移动台通信.(GSM) BTS--基站收发台,可以看作一复杂的无线调制器,BSS的主要部分,每个分配有若干信道.(G ...

  2. grep和map计算两个集合交集、并集、补集

    #!/usr/bin/perl use strict; ######################################## 用grep 和map 获取两个列表的交集并集.补集###### ...

  3. dedecms上传图片相对路径改成绝对路径方法

    很多朋友使用dedecms的时候都用了二级域名的功能,所以造成很多文章中图片不显示的问题. 解决方案如下: 1. 进入dede后台"系统"-"系统基本参数"-& ...

  4. Python3 range()函数

    Python3 range() 函数用法  Python3 内置函数 Python3 range() 函数返回的是一个可迭代对象(类型是对象),而不是列表类型, 所以打印的时候不会打印列表. Pyth ...

  5. arm linux c++11编译

    include(CheckCXXCompilerFlag) CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11 ...

  6. 客户端与服务端Post报文构造请求及Http Post与Get请求方法

    客户端 namespace Client.Controllers { public class HomeController : Controller { // // GET: /Home/ publ ...

  7. Linux LVM逻辑卷配置过程详解（创建、扩展、缩减、删除、卸载、快照创建）(未完)

    转:http://blog.csdn.net/xuanfeng407/article/details/51465472

  8. Google I/O 2014 大会总结 Android开发新方向

    昨天晚上,Google I/O 2014大会召开,会上主要展示了下面几个部分的创新内容: Android L 操作系统 首先是界面,谷歌又一次设计了一套 UI 规范.并称之为"Materia ...

  9. CenterOS卸载和安装MYSQL

    1.首先在命令行输入mysql,看一下本地计算机上是否有mysql. 2.卸载mysql服务: 首先查看安装的rpm的包:rpm –qa |grep mysql    对之前的服务进行删除.rpm – ...

  10. Java Resource路径小结

    首先一点很重要,Java中不存在标准的相对路径,各种相对路径取资源的方式都是基于某种规则转化为绝对路劲 然后一点也很重要,绝对不要直接使用绝对路径,否则死得很难看 基于以上两点,总结Resource路 ...