爬虫_古诗文网(队列，多线程，锁，正则，xpath)

 import requests
 from queue import Queue
 import threading
 from lxml import etree
 import re
 import csv

 class Producer(threading.Thread):
     headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'}
     def __init__(self, page_queue, poem_queue, *args, **kwargs):
         super(Producer, self).__init__(*args, **kwargs)
         self.page_queue = page_queue
         self.poem_queue = poem_queue

     def run(self):
         while True:
             if self.page_queue.empty():
                 break
             url = self.page_queue.get()
             self.parse_html(url)

     def parse_html(self, url):
         # poems = []
         headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'}
         response = requests.get(url, headers=headers)
         response.raise_for_status()
         html = response.text
         html_element = etree.HTML(html)
         titles = html_element.xpath('//div[@class="cont"]//b/text()')
         contents = html_element.xpath('//div[@class="contson"]')
         hrefs = html_element.xpath('//div[@class="cont"]/p[1]/a/@href')
         for index, content in enumerate(contents):
             title = titles[index]
             content = etree.tostring(content, encoding='utf-8').decode('utf-8')
             content = re.sub(r'<.*?>|\n|', '', content)
             content = re.sub(r'\u3000\u3000', '', content)
             content = content.strip()
             href = hrefs[index]
             self.poem_queue.put((title, content, href))

 class Consumer(threading.Thread):

     def __init__(self, poem_queue, writer, gLock, *args, **kwargs):
         super(Consumer, self).__init__(*args, **kwargs)
         self.writer = writer
         self.poem_queue = poem_queue
         self.lock = gLock

     def run(self):
         while True:
             try:
                 title, content, href = self.poem_queue.get(timeout=20)
                 self.lock.acquire()
                 self.writer.writerow((title, content, href))
                 self.lock.release()
             except:
                 break

 def main():
     page_queue = Queue(100)
     poem_queue = Queue(500)
     gLock = threading.Lock()
     fp = open('poem.csv', 'a',newline='', encoding='utf-8')
     writer = csv.writer(fp)
     writer.writerow(('title', 'content', 'href'))

     for x in range(1, 100):
         url = 'https://www.gushiwen.org/shiwen/default.aspx?page=%d&type=0&id=0' % x
         page_queue.put(url)

     for x in range(5):
         t = Producer(page_queue, poem_queue)
         t.start()

     for x in range(5):
         t = Consumer(poem_queue, writer, gLock)
         t.start()

 if __name__ == '__main__':
     main()

运行结果