aiohttp爬虫的模板，类的形式

 import asyncio
 import aiohttp
 import async_timeout
 from lxml import html
 from timeit import default_timer as timer

 from db import DBData

 class Crawler:
     def __init__(self, **kwargs):
         self.domains = kwargs["domains"]
         self.max_depth = kwargs["max_depth"]
         self.max_retries = 3
         self.max_workers = 10
         self.Q = asyncio.Queue()
         self.db_Q = asyncio.Queue()
         self.cache = set()
         self.count = 0
         self.loop = asyncio.get_event_loop()
         self.db_data = DBData()

         # Clear
         self.db_data.clear_crawler()

     async def get(self, url, timeout):
         with async_timeout.timeout(timeout):
             async with self.session.get(url) as response:
                 return await response.text()

     async def extract_urls(self, url, timeout=10):
         tree = html.fromstring(await self.get(url, timeout))
         # Search only in domains
         return {p for p in tree.xpath("//a/@href")}
                 # if any(domain in p for domain in self.domains)}

     async def worker(self):
         while True:
             url, depth, retries = await self.Q.get()
             if url in self.cache:
                 self.db_Q.put_nowait(url)
                 self.Q.task_done()
                 continue
             try:
                 new_urls = await self.extract_urls(url)
             except Exception as e:
                 if retries <= self.max_retries:
                     self.Q.put_nowait((url, depth, retries + 1))
                 else:
                     print("Error in %s: %s" % (url, repr(e)))
             else:
                 self.cache.add(url)
                 self.count += 1
                 self.db_Q.put_nowait(url)
                 print("Depth: %s Retry: %s Visited: %s" % (depth, retries, url))
                 if depth+1 <= self.max_depth:
                     for x in new_urls:
                         self.Q.put_nowait((x, depth + 1, retries))
             self.Q.task_done()

     async def run(self):
         async with aiohttp.ClientSession(loop=self.loop) as session:
             self.session = session
             workers = [self.worker() for _ in range(self.max_workers)]
             workers += [self.write_to_db() for _ in range(self.max_workers)]
             tasks = [self.loop.create_task(x) for x in workers]
             await asyncio.sleep(5)
             await self.Q.join()
             await self.db_Q.join()
             for task in tasks:
                 task.cancel()

     def start(self):
         for domain in self.domains:
             print("Crawling %s start..." % domain)

             self.Q.put_nowait((domain, 0, 0))
             start_time = timer()
             self.loop.run_until_complete(asyncio.gather(self.run()))
             self.loop.close()
             runtime = timer() - start_time

             print("Crawling %s end. Exec time: %s. Requests: %s" % (
                 domain, runtime, self.count))

     async def write_to_db(self):
         while True:
             address = await self.db_Q.get()
             if await self.db_data.check_url(address) is None:
                 self.db_data.add_url(address)
                 print("Write to DB: %s" % address)
             self.db_Q.task_done()

 if __name__ == "__main__":
     options = {
         "domains": ["https://www.yahoo.com/news/"],
         "max_depth": 1
     }
     c = Crawler(**options)
     c.start()