【url ---lib___】笔趣阁(抓取斗罗大陆完整)和(三寸天堂)

 # coding=gbk  #因为在黑屏下执行，所以代码会使用GBK
 url='http://www.biquge.info/10_10218/'
 UA={"User-Agent":"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"}
 UA1={"User-Agent":"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
     'Host':'www.xxbiquge.com',
     'Referer':'https://www.xxbiquge.com/2_2278/'}
 import time,lxml,pymysql
 from lxml import etree
 from urllib.request import Request
 from urllib.request import urlopen
 import os,sys,io
 sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030') 

 def source(url):#获取源
    global UA
    text=urlopen(Request(url,None,UA),timeout=5)
    return text.read()

 def respon(text):#解析章
    global url
    seletor=etree.HTML(text)
    url1=seletor.xpath("//*[@id='list']/dl/dd/a/@href")
    return url1

 def spider(url):#解析内容spider('http://www.biquge.info/10_10218/5002106.html')
    global UA1
    for i in url:
       i='https://www.xxbiquge.com'+i
       a=urlopen(Request(i,None,UA1),timeout=5).read()
       seletor=etree.HTML(a)
       text=seletor.xpath('//*[@id="content"]/text()')#内容
       c=''
       for aa in text:
           c=c+aa

       text1=seletor.xpath('//html/head/title/text()')[0].split('-')[0]#章节名
       #print(i,type(i),text1,type(text1))
       mysqlw(c,i,text1)
       time.sleep(3)

 #c=os.path.join(os.path.abspath(os.path.dirname(__name__)),'2.html')
 #with open(c,'r') as f:
 #   a=f.read()

 def mysqlw(text,url,chapter):#写内容
    b1=time.time()
    b=pymysql.connect('localhost',port=3306,user='root',passwd='liu',db='test',charset='utf8')
    cur=b.cursor()
    print(url,chapter,'w')

    #for i in cur.fetchall():
        #pass
    sql="""insert into douludalu(souce,html,chapter) values('%s','%s','%s')"""%(text,url,chapter)
    print(sql)
    try:
        cur.execute(sql)
        b.commit()
        print("插入成功")
    except Exception as e:
        print(e)
        b.rollback()
    b.close()
    print("关闭",'耗时',time.time()-b1)

 def mysqlr(text):#读内容
    b=pymysql.connect('localhost',port=3306,user='root',passwd='liu',db='test',charset='utf8')
    cur=b.cursor()
    sql='select * from douludalu where html="%s%s%%s"'%(',text,')
    cur.execute(sql)
    print(sql)
    for i in cur.fetchall():
        a=i[0]
        b=i[3]
        print(a,b)

 #a='2唐三已经挥出了八千余锤，铁坨不断的变小，已经不到最初时三分'
 #mysqlw(a,'1.html','第一章')
 def main():
    a=source('https://www.xxbiquge.com/2_2278/')
    b=respon(a)
    spider(b)
 #mysqlr('https://www.xxbiquge.com/2_2278/1036550.html')
 main()

——————————————————————————————————————————————————————————————————

三寸天堂

 # coding=gbk
 url='http://www.biquge.info/10_10218/'
 UA={"User-Agent":"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"}
 UA1={"User-Agent":"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
     'Host':'www.biquge.com.tw',
     'Referer':'http://www.biquge.com.tw/14_14055/',
     'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'}
 import time,lxml,pymysql,threading
 from lxml import etree
 from urllib.request import Request
 from urllib.request import urlopen
 import os,sys,io
 sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030') 

 def source(url):#获取源
    global UA
    text=urlopen(Request(url,None,UA),timeout=5)
    return text.read()

 def respon(text):#解析章
    global url
    seletor=etree.HTML(text)
    url1=seletor.xpath("//*[@id='list']/dl/dd/a/@href")
    return url1

 def spider(url):#解析内容spider('http://www.biquge.info/10_10218/5002106.html')
    global UA1
    i='http://www.biquge.com.tw/'+url
    print(i)
    a=urlopen(Request(i,None,UA1),timeout=5).read()
    if a is None:
        pass
    else:
        seletor=etree.HTML(a)
        text=seletor.xpath('//*[@id="content"]/text()')#内容
        c=''
        for aa in text:
            c=c+aa

        text1=seletor.xpath('//html/head/title/text()')[0]#章节名
    print(text1)
    #print(i,type(i),text1,type(text1))
    mysqlw(c,i,text1)
    time.sleep(3)

 def mysqlw(text,url,chapter):#写内容
    b1=time.time()
    b=pymysql.connect('localhost',port=3306,user='root',passwd='liu',db='test',charset='utf8')
    cur=b.cursor()
    print(url,chapter,'')
    #for i in cur.fetchall():
        #pass
    sql="""insert into suibian(souce,html,chapter) values('%s','%s','%s')"""%(text,url,chapter)
    try:
        cur.execute(sql)
        b.commit()
        print("插入成功")
    except Exception as e:
        print(e)
        b.rollback()
    b.close()
    print("关闭",'耗时',time.time()-b1)

 def mysqlr(text):#读内容
    b1=True
    b=pymysql.connect('localhost',port=3306,user='root',passwd='liu',db='test',charset='utf8')
    cur=b.cursor()
    sql='select * from douludalu where html="%s%s%%s"'%(',text,')
    cur.execute(sql)
    print(sql)
    for i in cur.fetchall():
        a=i[0]
        b=i[3]
        print(a,b)
    if i[3] is None:
        b1=False

 def main():
    print(threading.current_thread().name)
    cc=time.time()
    print('开始时间%s'%cc)
    a=source('http://www.biquge.com.tw/14_14055/')
    b=respon(a)
    for i in b:
        #print(i)
        spider(i)
    ctime=time.time()-cc
    print('完成耗时%s'%ctime)

 #c=os.path.join(os.path.abspath(os.path.dirname(__name__)),'1.html')
 #with open(c,'r') as f:
 #   a=f.read()
 main()

特别需要注意的是UA在Request中传值会出现错误，这时需要耐心来把问题解决

容易出现的错误【

　　1，协议中，referer错误，host错误

　　2，网页xpath错误，目测此网站的网页还是比较规则的

】