爬虫系列之requests

爬取百度内容：

 import requests
 url = "https://www.baidu.com"

 if __name__ == '__main__':
     try:
         kv = {'user-agent': 'Mozilla/5.0'}
         r = requests.get(url, headers=kv)
         r.raise_for_status()  #返回状态值，如果不是200，则抛出异常
         r.encoding = r.apparent_encoding
         print(r.text)
         #print(r.request.headers)
     except:
         print("爬虫失败")

在URL中填上http://www.baidu.com/s?wd=keyword，keyword就是我们要百度搜索的内容，在requests中有params参数，可以把参数追加到URL中。

 import requests
 url = "http://www.baidu.com/s"
 keyword = "python"

 if __name__ == '__main__':
     try:
         kv = {'user-agent': 'Mozilla/5.0'}
         wd = {'wd': keyword}
         r = requests.get(url, headers=kv, params=wd)
         print(r.request.url)
         r.raise_for_status()
         r.encoding = r.apparent_encoding
         print(len(r.text))
     except:
         print("爬虫失败")

爬取图片

 import requests
 import os
 url = "http://image.nationalgeographic.com.cn/2017/0211/20170211061910157.jpg"

 kv = {'header': 'Mozilla/5.0'}
 root = "D://pic_save//"
 path = root + url.split('/')[-1]

 if __name__ == '__main__':
     try:
         if not os.path.exists(root):
             os.mkdir(root)
         if not os.path.exists(path):
             r = requests.get(url, headers=kv)
             print(r.status_code)
             with open (path, 'wb') as f:
                 f.write(r.content)
                 print("文件已保存成功")
         else:
             print("文件已存在")
     except:
         ("爬虫失败")