lxml_time_代理

 import requests
 from pyquery import PyQuery as pq
 import json
 import jsonpath
 from lxml import etree
 import os
 import re
 import time

 html = '''
 <div>
     <ul>
          <li class="item-0">first item</li>
          <li class="item-1"><a href="link2.html">second item</a></li>
          <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li>
          <li class="item-1 active"><a href="link4.html">fourth item</a></li>
          <li class="item-0"><a href="link5.html">fifth item</a></li>
      </ul>
 </div>
 '''

 # html = requests.get('http://news.4399.com/gonglue/lscs/kptj/').content.decode('gbk')
 num = 0
 # def pq方法(url):
 #     global num
 #     html=requests.get(url).content.decode('gbk')
 #     doc = pq(html)
 #     items = doc('#dq_list > li').items()
 #     # print(doc)
 #     # print(type(doc))
 #     for item in items:
 #         url=item.find('img').attr('lz_src')
 #         num+=1
 #         print(str(num),url)
 #         url_content=requests.get(url).content
 #         name = item.find('.kp-name').text()

 #         with open('e:/py3/002/'+'{:0>4}'.format(str(num))+name+'.jpg','wb') as file:
 #             file.write(url_content)
 #         # print(url,name)

 def transformCodec(re_data):#ascii (gbk) 转 unicode
     try:
         re_data = re_data.decode('gbk')
     except Exception as error:
         print (error)
         print ('delete illegal string,try again...')

         pos = re.findall(r'decodebytesinposition([\d]+)-([\d]+):illegal',str(error).replace(' ',''))
         if len(pos)==1:
             re_data = re_data[0:int(pos[0][0])]+re_data[int(pos[0][1]):]
             re_data = transformCodec(re_data)
             return re_data
     return re_data

 def lxml方法(url):
     global num
     header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6756.400 QQBrowser/10.3.2473.400'}
     content=requests.get(url,headers=header).content
     html=content.decode('utf-8')
     # print(html.status_code)
     # print(content)
     # print(html)
     r=etree.HTML(html)
     # items=r.xpath("//div[@class='box10-content']//ul[@id='dq_list']/li/a/img/@lz_src")
     items=r.xpath("//div[@id='list']/table//tr")
     # print(items)
     for item in items:
         dl_ip=item.xpath("./td[1]/text()")
         dl_port=item.xpath("./td[2]/text()")
         dl_name=item.xpath("./td[5]/text()")
         num+=1
         dl_ip=dl_ip[0]+":" if len(dl_ip)>=1 else ''
         dl_port=dl_port[0]+"#" if len(dl_port)>=1 else ''
         dl_name=dl_name[0] if len(dl_name)>=1 else ''

         # print(len(dl_ip))
         # print(dl_ip)
         # print(r'{}{}{}'.format(dl_ip,dl_port,dl_name))
         with open("proxy.txt",'a',encoding='utf-8') as file:
             file.write('{}{}{}\n'.format(dl_ip,dl_port,dl_name))
         # lzcontent=requests.get(lzsrc).content
         # with open('e:/py3/004/'+'{:0>4}'.format(str(num))+'_'+kpname+'.jpg','wb')as file:
         #     file.write(lzcontent)

 if __name__ == '__main__':
     with open("proxy.txt", 'w', encoding='utf-8') as file:
             file.write(str(time.localtime()[0])+'_'+str(time.localtime()[1])+'_'+str(time.localtime()[2])+'_采集：\n')
     # url='https://www.kuaidaili.com/free/inha/1/'
     for i in range(1,11):
         print('第'+str(i)+'次：\n')
         url2 = r'https://www.kuaidaili.com/free/inha/'+str(i)+r'/'
         print(url2)
         lxml方法(url2)
         time.sleep(5)

     # header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6756.400 QQBrowser/10.3.2473.400'}
     # pq方法()
     # print(str(time.localtime()[0])+'_'+str(time.localtime()[1])+'_'+str(time.localtime()[2]))
     print(str(num)+' ok!')

     # 创建目录
     '''
     for dirnum in range(1,100):
         dirnum2='{:0>3}'.format(str(dirnum))
         mkpath="e:\\py3\\{}\\".format(dirnum2)
         print(mkpath)
         print('已存在！') if os.path.exists(mkpath) else os.makedirs(mkpath)
     '''