《python网络数据采集》笔记1

第一部分-创建爬虫

1.urllib

1)urllib.request

request.urlopen(url)

request.urlretrieve 可以根据文件的 URL 下载文件

2)urllib.parse

3)urllib.error

2.异常：

try...except...else...

常见异常：HTTPError,AttributeError,

3.BeautifulSoup

from bs4 import BeautifulSoup

bsObj=BeautifulSoup(html,'lxml')

1)

print(bsObj.text)

print(bsObj.html)

print(bsObj.p.a)

...

2)

findAll(tag, attributes, recursive, text, limit, keywords)     #返回一个ResultSet

find(tag, attributes, recursive, text, keywords)   #返回一个Tag

借助它们,你可以通过标签的不同属性轻松地过滤 HTML 页面，查找需要的标签组或单个标签

例：

.findAll({"h1","h2","h3","h4","h5","h6"})

.findAll("span", {"class":{"green", "red"}})

.findAll(id="text")   同 .findAll("", {"id":"text"})

.findAll(src=True) 有src属性的标签

3)

get_text() 会把你正在处理的 HTML 文档中所有的标签都清除，然后返回一个只包含文字的str

4)返回类型NavigatorString

.children （所有子标签）

.next_sibling（      下一个兄弟标签 ）.next_siblings（所有之后的兄弟标签）

.previous_sibling（上一个兄弟标签）.previous（所有之前的兄弟标签）

.parent （直接父标签 ）.parents（所有父标签）、

5）

.attrs      获取标签所有属性(dict)

.attrs['src']      获取src值

6)正则表达式

7)lambda表达式

#获取有两个属性的标签：

bsObj.findAll(lambda tag: len(tag.attrs) == 2)

4.Scrapy

//TODO

5.JSON

把 JSON 转换成字典，

JSON 数组转换成列表，

JSON 字符串转换成 Python 字符串。

常用函数：loads,get

6.存储数据

1)下载

from urllib.request import urlretrieve

urlretrieve(resourceLocation,fileName)

2）CSV（Comma-Separated Values）

import csv

csvFile=open("test.csv","w+")

try:

writer=csv.writer(csvFile)

writer.writerow(('青山隐隐水迢迢 秋尽江南草未凋','24桥明月夜'))

for i in range(1,5):

writer.writerow((i,i+2,i*2))

finally:

csvFile.close()

3）MySQL

import pymysql

#获取连接 获取光标

conn=pymysql.connect(host='localhost',user='root',passwd=None)

cur=conn.cursor()

#执行SQL语句

cur.execute('use ssm01')

cur.execute('select * from user')

print(cur.fetchone())#获取一条数据

#关闭资源

cur.close()

coon.close()

4）Email

//TODO

7.读取文档

1)读取txt

from urllib.request import urlopen

txt=urlopen('http://www.pythonscraping.com/pages/warandpeace/chapter1.txt')

print(txt.read())

2)读取csv

#从网上直接把文件读成一个字符串，然后转换成一个 StringIO 对象，使它具有文件的属性。

from urllib.request import urlopen

from io import StringIO

import csv

data = urlopen('http://pythonscraping.com/files/MontyPythonAlbums.csv').read().decode('utf-8')

dataFile=StringIO(data)

csvFile=csv.reader(dataFile)

for row in csvFile:

print(row)

3）读取PDF

#PDFMiner3K

#把任意 PDF 读成字符串，然后用 StringIO 转换成文件对象

from urllib.request import urlopen

from pdfminer.pdfinterp import PDFResourceManager, process_pdf

from pdfminer.converter import TextConverter

from pdfminer.layout import LAParams

from io import StringIO

def readPDF(pdfFile):

rsrcmgr = PDFResourceManager()

retstr = StringIO()

laparams = LAParams()

device = TextConverter(rsrcmgr, retstr, laparams=laparams)

process_pdf(rsrcmgr, device, pdfFile)

device.close()

content = retstr.getvalue()

retstr.close()

return content

pdfFile = urlopen("http://pythonscraping.com/pages/warandpeace/chapter1.pdf")

outputString = readPDF(pdfFile)

print(outputString)

pdfFile.close()

//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

例

3-1.网络数据采集

#从 http://oreilly.com 开始，然后随机地从一个外链跳到另一个外链。

from urllib.request import urlopen

from bs4 import BeautifulSoup

import re

import datetime

import random

pages = set()

random.seed(datetime.datetime.now())

# 获取页面所有内链的列表

def getInternalLinks(bsObj, includeUrl):

internalLinks = []

# 找出所有以"/"开头的链接

for link in bsObj.findAll("a", href=re.compile("^(/|.*"+includeUrl+")")):

if link.attrs['href'] is not None:

if link.attrs['href'] not in internalLinks:

internalLinks.append(link.attrs['href'])

return internalLinks

# 获取页面所有外链的列表

def getExternalLinks(bsObj, excludeUrl):

externalLinks = []

# 找出所有以"http"或"www"开头且不包含当前URL的链接

for link in bsObj.findAll("a",href=re.compile("^(http|www)((?!"+excludeUrl+").)*$")):

if link.attrs['href'] is not None:

if link.attrs['href'] not in externalLinks:

externalLinks.append(link.attrs['href'])

return externalLinks

def splitAddress(address):

addressParts = address.replace("http://", "").split("/")

return addressParts

def getRandomExternalLink(startingPage):

html = urlopen(startingPage)

bsObj = BeautifulSoup(html,'lxml')

externalLinks = getExternalLinks(bsObj, splitAddress(startingPage)[0])

if len(externalLinks) == 0:

internalLinks = getInternalLinks(startingPage)

return getNextExternalLink(internalLinks[random.randint(0,len(internalLinks)-1)])

else:

return externalLinks[random.randint(0, len(externalLinks)-1)]

def followExternalOnly(startingSite):

externalLink = getRandomExternalLink("http://oreilly.com")

print("随机外链是："+externalLink)

followExternalOnly(externalLink)

followExternalOnly("http://oreilly.com")

5-1.JSON

import json

jsonString='{\

"arrayOfNums":[{"number":0},{"number":1},{"number":2}],\

"arrayOfFruits":[{"fruit":"apple"},{"fruit":"banana"},{"fruit":"pear"}]\

}'

jsonObj=json.loads(jsonString)

print(jsonObj.get("arrayOfFruits")[2].get("fruit"))

6-1.把 http://pythonscraping.com 的所有图片下载下来

from urllib.request import urlretrieve

from urllib.request import urlopen

from bs4 import BeautifulSoup

def pageSrc(url):

html=urlopen(url)

bsObj=BeautifulSoup(html,'lxml')

srcList=bsObj.findAll("img",src=True)

urlList=[]

for i in srcList:

urlList.append(i['src'])

return urlList

def getInternalLinks(bsObj,includeUrl):

internalLinks = []

# 找出所有以"/"开头的链接

for link in bsObj.findAll("a", href=re.compile("^(/|.*"+includeUrl+")")):

if link.attrs['href'] is not None:

if link.attrs['href'] not in internalLinks:

internalLinks.append(link.attrs['href'])

return internalLinks

def allimgs(url):

#找到该页面所有的img src

srcset=set()

for i in pageSrc(url):

if i not in srcset:

print(i)

srcset.add(i)

name=i.split('/').pop()

urlretrieve(i,name)

#找到该页面的所有内链

html=urlopen(url)

bsObj=BeautifulSoup(html,'lxml')

for i in getInternalLinks(bsObj,url):

newUrl=url+i

for j in pageSrc(newUrl):

if j not in srcset:

srcset.add(i)

print(j)

name=j.split('/').pop()

urlretrieve(j,name)

url="http://pythonscraping.com"

allimgs(url)

6-2.存储到CSV

#获取 HTML 表格并写入 CSV 文件

import csv

from urllib.request import urlopen

from bs4 import BeautifulSoup

html = urlopen("http://en.wikipedia.org/wiki/Comparison_of_text_editors")

bsObj = BeautifulSoup(html,'lxml')

# 主对比表格是当前页面上的第一个表格

table = bsObj.findAll("table",{"class":"wikitable"})[0]

rows = table.findAll("tr")

csvFile = open("editors.csv", 'wt', newline='',encoding='utf-8')

writer = csv.writer(csvFile)

try:

for row in rows:

csvRow = []

for cell in row.findAll(['td', 'th']):

csvRow.append(cell.get_text()[:-1])

print(csvRow)

writer.writerow(csvRow)

finally:

csvFile.close()

6-3.存储到mysql

#存储维基百科数据

from urllib.request import urlopen

from bs4 import BeautifulSoup

import re

import datetime

import random

import pymysql

conn = pymysql.connect(host='127.0.0.1',user='root', passwd=None, charset='utf8')

cur = conn.cursor()

cur.execute("USE ssm01")

cur.execute("CREATE TABLE pages(title varchar(200),content varchar(3000))")

random.seed(datetime.datetime.now())

#存储到数据库

def store(title, content):

cur.execute("INSERT INTO pages (title, content) VALUES (\"%s\",\"%s\")", (title, content))

cur.connection.commit()

#找到数据 存储到数据库

def getLinks(articleUrl):

html = urlopen("http://en.wikipedia.org"+articleUrl)

bsObj = BeautifulSoup(html,'lxml')

title = bsObj.find("h1").get_text()

content = bsObj.find("div", {"id":"mw-content-text"}).find("p").get_text()

store(title, content)

return bsObj.find("div", {"id":"bodyContent"}).findAll("a",href=re.compile("^(/wiki/)((?!:).)*$"))

links = getLinks("/wiki/Kevin_Bacon")

try:

while len(links) > 0:

newArticle = links[random.randint(0, len(links)-1)].attrs["href"]

print(newArticle)

links = getLinks(newArticle)

finally:

cur.close()

conn.close()