使用python爬去国家民政最新的省份代码的程序，requests,beautifulsoup,lxml

使用的python3.6

民政网站，不同年份数据可能页面结构不一致，这点踩了很多坑，这也是代码越写越长的原因。

如果以后此段代码不可用，希望再仔细学习下 页面结构是否发生了变更。

 # -*- coding: utf-8 -*-
 """
 Created on Wed Jul 10 14:40:41 2019

 @author: Administrator
 """

 import pandas as pd
 import requests
 from bs4 import BeautifulSoup
 import time 

 url1 = 'http://www.mca.gov.cn/article/sj/xzqh//1980/'
 headers = {'content-type': 'application/json',
                'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'}

 # 1. 获取所有链接========================================================================
 def f1(url1):
     '2018-1980年中华人民共和国行政区划代码 的所有链接'
     #requests发出请求，设置url，header参数
     response = requests.get(url1, headers=headers, timeout=200, verify=False)
     soup = BeautifulSoup(response.text,'lxml') #将网页源码返回为BeautifulSoup类型
     _tmp1 = soup.select('td.arlisttd')
     end_1 = []
     for i in _tmp1:
         _a = i.select('a')[0].get('href')
         _b = i.select('a')[0].get('title')[:4]
         end_1.append(['http://www.mca.gov.cn'+_a,_b])
     return end_1

 end_2=[]
 for i in ['','?2','?3']:
     end_2 = end_2+f1(url1+i)

 def f2(url1='http://www.mca.gov.cn/article/sj/xzqh/2019/'):
     '2019年中华人民共和国行政区划代码'
     response = requests.get(url1, headers=headers, timeout=200, verify=False)
     soup = BeautifulSoup(response.text,'lxml')
     _tmp1 = soup.select('td.arlisttd')
     end_1 = []
     for i in _tmp1:
         _a = i.select('a')[0].get('href')
         _b = i.select('a')[0].get('title')[:7]
         end_1.append(['http://www.mca.gov.cn'+_a,_b])
     return end_1

 end_2 = end_2+f2()

 # 2. 获取数据========================================================================
 def f3(url1='http://www.mca.gov.cn/article/sj/xzqh/1980/201903/20190300014989.shtml'):
     #url1='http://www.mca.gov.cn/article/sj/xzqh/1980/201507/20150715854922.shtml'
     #url1='http://www.mca.gov.cn/article/sj/xzqh/1980/201507/20150715854918.shtml'
     #
     response = requests.get(url1, headers=headers, timeout=200, verify=False)
     soup = BeautifulSoup(response.text,'lxml')
     _txt = soup.select('script')[4].get_text().strip().replace('window.location.href="','').strip('";')
     if _txt[-4:]=='html':
         print('script!')
         url2 = _txt
     else:
         _tmp1 = soup.select('div.artext > div > p > a')
         if len(_tmp1)==0:
             _tmp1 = soup.select('div#zoom > a')
         url2 = _tmp1[0].get('href')
     print(url2)
     #return url2
     #url2='http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220856.html'
     time.sleep(0.5)
     response = requests.get(url2, headers=headers, timeout=200, verify=False)
     #将网页源码返回为BeautifulSoup类型
     soup = BeautifulSoup(response.text,'lxml')
     _tmp1 = soup.select('table > tr[height="19"]')
     end_1 = []
     if len(_tmp1)>5:
         for i in _tmp1:
             _a = i.select('td')[1].get_text().strip()
             if len(_a)>15: #部分数据页面，最后一行是备注。
                 continue
             else:
                 _b = i.select('td')[2].get_text().strip()
                 end_1.append([_a,_b])
     else:
         _tmp1 = soup.select('table > tr[height="20"]')
         for i in _tmp1:
             _a = i.select('td')[0].get_text().strip()
             if len(_a)>15 or _a=='行政区划代码': #部分数据页面，最后一行是备注。
                 continue
             else:
                 _b = i.select('td')[1].get_text().strip()
                 end_1.append([_a,_b])

     return end_1

 #循环对每个链接 获取数据
 end_3=[];#end_4=[]
 for j in range(len(end_2)):
     item = end_2[j]
     if ''  in item[1] or ''  in item[1]:
         print(j,item[0],item[1])
         tmp2 = f3(item[0])
         print('.')
         end_3.extend([[item[1]]+i for i in tmp2])
         #end_4.append(tmp2)
         time.sleep(0.1)

 df_result = pd.DataFrame(end_3)
 #pd.DataFrame(end_4).to_excel('所有连接.xlsx',index=False)
 df_result.to_excel('地区编码.xlsx',index=False)

 '''
 #\3 2019年5月份县以上行政区划代码_3852 > table > tbody > tr:nth-child(4)
 #list_content > div.list_right > div > ul > table > tbody > tr:nth-child(1) > td.arlisttd > a
 '''