网DAI之家简单爬取

用requests和bs做个简单的爬取网DAI之家的例子。

只做笔记用。

#!/usr/bin/python3

import requests
from bs4 import BeautifulSoup
import re
import xlwt

class wdzj_spider:
    pingTaiInfo = []

    def request(self, url):
        headers = {'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}
        return requests.get(url, headers=headers)

    def saveToExcel(self, infoList, fileName='resultme.xls', sheetName='www.wdzj.com'):

        # 对于要写入的每一个元素，先找他的title在哪列，写入到对应的列去
        rowNo = 1
        excelTitle = {}

        book = xlwt.Workbook()  # 打开一个excel
        if book is None:
            print("创建文件{0}失败".format(fileName))
            return None
        sheet = book.add_sheet(sheetName)  # 根据顺序获取sheet
        if sheet is None:
            print("创建表单{0}失败".format(sheetName))
            return None

        for info in infoList:
            for item in info.items():
                if str(item[0]) in excelTitle:
                    colNo = excelTitle[item[0]]
                else:
                    colNo = len(excelTitle)
                    excelTitle[str(item[0])] = colNo
                sheet.write(rowNo, colNo, str(item[1]))
            rowNo = rowNo + 1
        for m in excelTitle.items():
            sheet.write(0, m[1], m[0])
        book.save(fileName)
        return rowNo

    def getDataplaneFromPage(self, Link):
        mainHtml = self.request(Link)
        dataInfo = {}
        #默认编码有问题，需手动转码
        txtUTF8 = str(mainHtml.content, 'utf-8')
        mainBS = BeautifulSoup(txtUTF8, "lxml",from_encoding="utf8")
        lis = mainBS.find_all('li', attrs={'class':'normal'})
        for tr in lis:
            divs = tr.find_all('div')
            dataInfo[divs[1].text.strip()] = divs[0].text.strip()
        # print("数据={0}".format(dataInfo))
        return dataInfo

    def getGongshangFromPage(self, Link):
        mainHtml = self.request(Link)
        gongshangInfo = {}
        #默认编码有问题，需手动转码
        txtUTF8 = str(mainHtml.content, 'utf-8')
        mainBS = BeautifulSoup(txtUTF8, "lxml",from_encoding="utf8")
        trs = mainBS.find('div', attrs={'class':'lcen'}).find_all('tr')
        for tr in trs:
            tdTitles = tr.find_all('td', attrs={'class':'t'})
            tds = tr.find_all('td')
            index = 1
            for td in tdTitles:
                gongshangInfo[td.text] = tds[index].text.strip('    \n\r')
                index = index + 2
        # print("工商信息={0}".format(gongshangInfo))
        return gongshangInfo

    def getLinkFromPage(self, pingtaiName, pingtaiLink):
        shujuInfo = {}
        lianxifangshiInfo = {}
        pingtaiHtml = self.request(pingtaiLink)
        #默认编码有问题，需手动转码
        txtUTF8 = str(pingtaiHtml.content, 'utf-8')
        mainBS = BeautifulSoup(txtUTF8, "lxml",from_encoding="utf8")
        briefText = mainBS.find("div", class_="cen-zk").get_text().strip('    \n\r')
        briefInfo={"P2P平台名称":pingtaiName, "简介":briefText}
        # print("简介={0}".format(briefText))
        gongshangLink = 'https://'+mainBS.find('a', text='工商/备案')['href'].lstrip('/')
        dataA = mainBS.find('div', attrs={'class':'common-header-nav'}).find('a', text='数据')

        lianxifangshiTitles =mainBS.find('div', attrs={'class':"da-lxfs zzfwbox"}).find_all('div', class_='l')
        lianxifangshiContents = mainBS.find('div', attrs={'class': "da-lxfs zzfwbox"}).find_all('div', class_='r')

        for i in range(0, len(lianxifangshiTitles)):
            lianxifangshiInfo[lianxifangshiTitles[i].get_text().strip('    \n\r')] = lianxifangshiContents[i].get_text().strip('    \n\r')
        # print("联系方式={0}".format(lianxifangshiInfo))

        if dataA:
            dataLink =  'https://' + dataA['href'].lstrip('/')
            shujuInfo = self.getDataplaneFromPage(dataLink)

        gongshangInfo = self.getGongshangFromPage(gongshangLink)
        self.pingTaiInfo.append({**briefInfo, **gongshangInfo, **shujuInfo, **lianxifangshiInfo})

    def getAllPage(self):
        startUrl = 'https://www.wdzj.com/dangan/search?filter=&sort=1&currentPage=1'
        mainHtml = self.request(startUrl)
        pageStr = BeautifulSoup(mainHtml.text, "lxml").find("span", class_="all").text
        searchObj = re.search(r'1/([0-9]+)', pageStr, re.M | re.I)
        pageCount = searchObj.group(1)

        startUrl = 'https://www.wdzj.com/dangan/search?filter=&sort=1&currentPage='
        baseUrl = 'https://www.wdzj.com'

        print("总页数：{0}".format(pageCount))
        for i in range(1, int(pageCount)+1):
        # for i in range(1, 2):

            urlPage = startUrl + str(i)
            pageHtml = self.request(urlPage)
            pageStrs = BeautifulSoup(pageHtml.text, "lxml").find('ul',attrs={'class':'terraceList'}).find_all('h2')

            print("---------------------------------")
            print("开始爬取第{0}页，共存在{1}个平台数据".format(i, len(pageStrs)))
            for p in pageStrs:
                a = p.find('a')
                self.getLinkFromPage(a.get_text(), baseUrl+a['href'])
                print("#", end='',flush=True)
            print("\n结束爬取第{0}页，共爬取{1}个平台数据".format(i,len(pageStrs)))
        self.saveToExcel(self.pingTaiInfo,fileName='p2p.xls')

if __name__ == '__main__':
    w = wdzj_spider()
    w.getAllPage()

做下修改

 #!/usr/bin/python3

 import requests
 from bs4 import BeautifulSoup
 import re
 import xlwt

 class wdzj_spider:
     pingTaiInfo = []
     book=None
     sheet=None
     excelTitle = {}
     rowNo = 1

     def request(self, url):
         headers = {'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}
         return requests.get(url, headers=headers)

     def openExcel(self, fileName='resultme.xls',sheetName='www.wdzj.com'):
         self.book = xlwt.Workbook()  # 打开一个excel
         if self.book is None:
             print("创建文件{0}失败".format(fileName))
             return None
         self.sheet = self.book.add_sheet(sheetName)  # 根据顺序获取sheet
         if self.sheet is None:
             print("创建表单{0}失败".format(sheetName))
             return None

     def closeExcel(self, fileName='resultme.xls'):
         for m in self.excelTitle.items():
             self.sheet.write(0, m[1], m[0])
         self.book.save(fileName)

     def saveToExcel(self, infoList, fileName='resultme.xls'):
         # 对于要写入的每一个元素，先找他的title在哪列，写入到对应的列去
         for info in infoList:
             for item in info.items():
                 if str(item[0]) in self.excelTitle:
                     colNo = self.excelTitle[item[0]]
                 else:
                     colNo = len(self.excelTitle)
                     self.excelTitle[str(item[0])] = colNo
                 self.sheet.write(self.rowNo, colNo, str(item[1]))
             self.rowNo = self.rowNo + 1
         self.book.save(fileName)
         return self.rowNo

     def getDataplaneFromPage(self, Link):
         mainHtml = self.request(Link)
         dataInfo = {}
         #默认编码有问题，需手动转码
         txtUTF8 = str(mainHtml.content, 'utf-8')
         mainBS = BeautifulSoup(txtUTF8, "lxml",from_encoding="utf8")
         lis = mainBS.find_all('li', attrs={'class':'normal'})
         for tr in lis:
             divs = tr.find_all('div')
             dataInfo[divs[1].text.strip()] = divs[0].text.strip()
         # print("数据={0}".format(dataInfo))
         return dataInfo

     def getGongshangFromPage(self, Link):
         mainHtml = self.request(Link)
         gongshangInfo = {}
         #默认编码有问题，需手动转码
         txtUTF8 = str(mainHtml.content, 'utf-8')
         mainBS = BeautifulSoup(txtUTF8, "lxml",from_encoding="utf8")
         trs = mainBS.find('div', attrs={'class':'lcen'}).find_all('tr')
         for tr in trs:
             tdTitles = tr.find_all('td', attrs={'class':'t'})
             tds = tr.find_all('td')
             index = 1
             for td in tdTitles:
                 gongshangInfo[td.text] = tds[index].text.strip('    \n\r')
                 index = index + 2
         # print("工商信息={0}".format(gongshangInfo))
         return gongshangInfo

     def getLinkFromPage(self, pingtaiName, pingtaiLink):
         shujuInfo = {}
         lianxifangshiInfo = {}
         pingtaiHtml = self.request(pingtaiLink)
         #默认编码有问题，需手动转码
         txtUTF8 = str(pingtaiHtml.content, 'utf-8')
         mainBS = BeautifulSoup(txtUTF8, "lxml",from_encoding="utf8")
         briefText = mainBS.find("div", class_="cen-zk").get_text().strip('    \n\r')
         briefInfo={"P2P平台名称":pingtaiName, "简介":briefText}
         # print("简介={0}".format(briefInfo))
         gongshangLink = 'https://'+mainBS.find('a', text='工商/备案')['href'].lstrip('/')
         dataA = mainBS.find('div', attrs={'class':'common-header-nav'}).find('a', text='数据')

         lianxifangshiBox = mainBS.find('div', attrs={'class':"da-lxfs zzfwbox"})
         lianxifangshiTitles=[]
         lianxifangshiContents=[]
         if lianxifangshiBox:
             lianxifangshiTitles =lianxifangshiBox.find_all('div', class_='l')
             lianxifangshiContents = lianxifangshiBox.find_all('div', class_='r')

         for i in range(0, len(lianxifangshiTitles)):
             lianxifangshiInfo[lianxifangshiTitles[i].get_text().strip('    \n\r')] = lianxifangshiContents[i].get_text().strip('    \n\r')
         # print("联系方式={0}".format(lianxifangshiInfo))

         if dataA:
             dataLink =  'https://' + dataA['href'].lstrip('/')
             shujuInfo = self.getDataplaneFromPage(dataLink)

         gongshangInfo = self.getGongshangFromPage(gongshangLink)
         self.pingTaiInfo.append({**briefInfo, **gongshangInfo, **shujuInfo, **lianxifangshiInfo})

     def getAllPage(self):
         startUrl = 'https://www.wdzj.com/dangan/search?filter=&sort=1&currentPage=1'
         mainHtml = self.request(startUrl)
         pageStr = BeautifulSoup(mainHtml.text, "lxml").find("span", class_="all").text
         searchObj = re.search(r'1/([0-9]+)', pageStr, re.M | re.I)
         pageCount = searchObj.group(1)

         startUrl = 'https://www.wdzj.com/dangan/search?filter=&sort=1&currentPage='
         baseUrl = 'https://www.wdzj.com'

         print("总页数：{0}".format(pageCount))
         fileName = 'p2p.xls'
         self.openExcel(fileName)
         for i in range(1, int(pageCount)+1):
         # for i in range(1, 2):
             urlPage = startUrl + str(i)
             pageHtml = self.request(urlPage)
             pageStrs = BeautifulSoup(pageHtml.text, "lxml").find('ul',attrs={'class':'terraceList'}).find_all('h2')

             print("---------------------------------")
             print("开始爬取第{0}页，共存在{1}个平台数据".format(i, len(pageStrs)))
             count=0
             for p in pageStrs:
                 a = p.find('a')
                 try:
                     self.getLinkFromPage(a.get_text(), baseUrl+a['href'])
                     count = count + 1
                 except:
                     print("爬取第{0}个数据，名称<{1}>失败".format(count+1, a.get_text()))
                 print("#", end='',flush=True)
             print("\n结束爬取第{0}页，共爬取{1}个平台数据".format(i,count))
             self.saveToExcel(self.pingTaiInfo, fileName=fileName)
             self.pingTaiInfo.clear()
         self.closeExcel(fileName)

 if __name__ == '__main__':
     w = wdzj_spider()
     w.getAllPage()