第一个python抓取单网页的例子

#!/usr/bin/env python
# coding=utf-8

import requests
from bs4 import BeautifulSoup
import pymysql

import sys, io
sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') # Change default encoding to utf8

print('连接到mysql服务器...')
db = pymysql.connect("localhost","root","root","python")
print('连接上了!')
cursor = db.cursor()

hdrs = {'User-Agent':'Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)'}

url = "http://www.xxx.com/tezgcmp/1303.html"

r = requests.get(url, headers = hdrs)
soup = BeautifulSoup(r.content.decode('gbk', 'ignore'), 'lxml')

title=soup.find("h1")
title=title.string.encode("utf-8")

intro=soup.select(".intro")
rintro=intro[0].string.encode("utf-8")

content=soup.select(".content")
rcontent=content[0].encode("utf-8")

insert = ("INSERT INTO article(title,intro,content)" "VALUES(%s,%s,%s)")
data = (title, rintro, rcontent)
cursor.execute(insert, data)
db.commit()

print('爬取数据并插入mysql数据库完成...')

备注:页面内容有h1,intro,content的

顺便写点心得

BeautifulSoup中的 find_all() 的返回是个list
find()返回的是单个值
.select() 返回的也是list
如果去第一个元素请用[0]这种方式