requsets模块和beautifulsoup模块

2.requests模块方法

requests是基于Python开发的HTTP库，使用Requests可以轻而易举的完成浏览器可有的任何操作。

request.get()
request.post()
request.put()
以上方法均是在此方法的基础上构建requests.request(method, url, **kwargs)
- method 包括 psot、get、put等等
- **kwargs 包括常用参数
  - url = ‘’,
  - params = {'k1':'v1','k2','v2'}, # get方法仅限的参数传递方式
  - cookies = {},
  - headers = {}, # 请求头
  - data = {}, # post等请求参数传递
  - json = {}, # json数据参数

2.1 requests.get

requests.get(

    url='xxx',

    params={'k1':'v1','nid':888},

    cookies={},

    headers={},

)

# http://www.baidu.com?k1=v2&nid=888

2.2 requests.post

requests.post(

    url='xxx',

    params={'k1':'v1','nid':888},

    cookies={},

    # data

    headers={'content-type': 'application/x-www-form-urlencoded'},

    data={},

    # json

    # headers={'content-type': 'application/json'},

    # json={}

)

其他参数

auth身份验证

def param_auth():

    from requests.auth import HTTPBasicAuth, HTTPDigestAuth

    ret = requests.get('https://api.github.com/user', auth=HTTPBasicAuth('wupeiqi', 'sdfasdfasdf'))

    print(ret.text)

    # ret = requests.get('http://192.168.1.1',

    # auth=HTTPBasicAuth('admin', 'admin'))

    # ret.encoding = 'gbk'

    # print(ret.text)

    # ret = requests.get('http://httpbin.org/digest-auth/auth/user/pass', auth=HTTPDigestAuth('user', 'pass'))

    # print(ret)

allow_redirects 重定向（控制是否url跳转）

def param_allow_redirects():

    ret = requests.get('http://127.0.0.1:8000/test/', allow_redirects=False)

    print(ret.text)

stream 流（true相应内容按流式下载）

def param_stream():

    ret = requests.get('http://127.0.0.1:8000/test/', stream=True)

    print(ret.content)

    ret.close()

    # from contextlib import closing

    # with closing(requests.get('http://httpbin.org/get', stream=True)) as r:

    # # 在此处理响应。

    # for i in r.iter_content():

    # print(i)

cert 是否携带证书(证书名)

requests.get('http://httpbin.org/get',cert="xxxx.pem")

session

此处的session并非之前所学session，

此处的session，是个容器，携带所有请求头、体等等,

所以，我们每次requests请求时，都需要cookies等手动添加请求中，

利用session可以自动携带cookies、session等信息发送请求

session = requests.Session()

session.post(url,data={}) # 省略cookies=cookie

# response_ = requests.post(url,data={},cookies=cookie)

3.BeautifulSoup模块方法

BeautifulSoup是一个模块，该模块用于接收一个HTML或XML字符串，然后将其进行格式化，之后遍可以使用他提供的方法进行快速查找指定元素，从而使得在HTML或XML中查找指定元素变得简单。

from bs4.Element import Tag

1，name 标签名

# tag = soup.find('a')

# name = tag.name # 获取

# print(name)

# tag.name = 'span' # 设置

# print(soup)

2，attr 属性标签

# tag = soup.find('a')

# attrs = tag.attrs    # 获取

# print(attrs)

# tag.attrs = {'ik':123} # 设置

# tag.attrs['id'] = 'iiiii' # 设置

# print(soup)

3，children,所有子标签

# body = soup.find('body')

# v = body.children

4, children,所有后代标签

# body = soup.find('body')

# v = body.descendants

5, clear,将标签的所有子标签全部清空（保留标签名）

# tag = soup.find('body')

# tag.clear()

# print(soup)

10, find,获取匹配的第一个标签

# tag = soup.find('a')

# print(tag)

# tag = soup.find(name='a', attrs={'class': 'sister'}, recursive=True, text='Lacie')

# tag = soup.find(name='a', class_='sister', recursive=True, text='Lacie')

# print(tag)

11, find_all,获取匹配的所有标签

# tags = soup.find_all('a')

# print(tags)

# tags = soup.find_all('a',limit=1)

# print(tags)

# tags = soup.find_all(name='a', attrs={'class': 'sister'}, recursive=True, text='Lacie')

# # tags = soup.find(name='a', class_='sister', recursive=True, text='Lacie')

# print(tags)

# ####### 列表 #######

# v = soup.find_all(name=['a','div'])

# print(v)

# v = soup.find_all(class_=['sister0', 'sister'])

# print(v)

# v = soup.find_all(text=['Tillie'])

# print(v, type(v[0]))

# v = soup.find_all(id=['link1','link2'])

# print(v)

# v = soup.find_all(href=['link1','link2'])

# print(v)

# ####### 正则 #######

import re

# rep = re.compile('p')

# rep = re.compile('^p')

# v = soup.find_all(name=rep)

# print(v)

# rep = re.compile('sister.*')

# v = soup.find_all(class_=rep)

# print(v)

# rep = re.compile('http://www.oldboy.com/static/.*')

# v = soup.find_all(href=rep)

# print(v)

# ####### 方法筛选 #######

# def func(tag):

# return tag.has_attr('class') and tag.has_attr('id')

# v = soup.find_all(name=func)

# print(v)

# ## get,获取标签属性

# tag = soup.find('a')

# v = tag.get('id')

# print(v)

12, has_attr,检查标签是否具有该属性

# tag = soup.find('a')

# v = tag.has_attr('id')

# print(v)

13, get_text,获取标签内部文本内容

# tag = soup.find('a')

# v = tag.get_text('id')

# print(v)

16, 当前的关联标签

# soup.next

# soup.next_element

# soup.next_elements

# soup.next_sibling

# soup.next_siblings

#

# tag.previous

# tag.previous_element

# tag.previous_elements

# tag.previous_sibling

# tag.previous_siblings

#

# tag.parent

# tag.parents

17, 查找某标签的关联标签

# tag.find_next(...)

# tag.find_all_next(...)

# tag.find_next_sibling(...)

# tag.find_next_siblings(...)

# tag.find_previous(...)

# tag.find_all_previous(...)

# tag.find_previous_sibling(...)

# tag.find_previous_siblings(...)

# tag.find_parent(...)

# tag.find_parents(...)

# 参数同find_all

20, append在当前标签内部追加一个标签

# tag = soup.find('body')

# tag.append(soup.find('a'))

# print(soup)

#

# from bs4.element import Tag

# obj = Tag(name='i',attrs={'id': 'it'})

# obj.string = '我是一个新来的'

# tag = soup.find('body')

# tag.append(obj)

# print(soup)

21, insert在当前标签内部指定位置插入一个标签

# from bs4.element import Tag

# obj = Tag(name='i', attrs={'id': 'it'})

# obj.string = '我是一个新来的'

# tag = soup.find('body')

# tag.insert(2, obj)

# print(soup)

22, insert_after,insert_before 在当前标签后面或前面插入

# from bs4.element import Tag

# obj = Tag(name='i', attrs={'id': 'it'})

# obj.string = '我是一个新来的'

# tag = soup.find('body')

# # tag.insert_before(obj)

# tag.insert_after(obj)

# print(soup)

23, replace_with 在当前标签替换为指定标签

# from bs4.element import Tag

# obj = Tag(name='i', attrs={'id': 'it'})

# obj.string = '我是一个新来的'

# tag = soup.find('div')

# tag.replace_with(obj)

# print(soup)

24, 创建标签之间的关系

# tag = soup.find('div')

# a = soup.find('a')

# tag.setup(previous_sibling=a)

# print(tag.previous_sibling)