爬一下国家统计局行政区划代码C#

目前NBS上有2015-2018四个年度的代码信息，写一个控制台程序爬一下县级行政区下的代码。

使用HttpWebRequest+HttpWebResponse获取html，使用HtmlAgilityPack类库解析HTML。

使用POST请求，请求头带Cookie信息，否则会被反爬机制挡死，返回“请开启JavaScript并刷新该页”。

县级URL Request获取数据的同时记录Response的Cookie信息，在请求镇级数据时，请求头发送此cookie。

“省-地-县-乡 ”与“省-县（地）-乡” 的URL长度不同，根据长度判断URL正确性时需注意，也许还有其他可能，暂未发现。

主方法

  class Program
     {
         static void Main(string[] args)
         {
             Console.ForegroundColor = ConsoleColor.Magenta;
             Console.WriteLine("\r\n----获取县级行政区乡、村二级区划代码");
             Console.WriteLine("----数据年份有：");
             Console.ResetColor();
             Cursor.WriteAt("A、2018", , );
             Cursor.WriteAt("B、2017", , );
             Cursor.WriteAt("C、2016", , );
             Cursor.WriteAt("D、2015", , );
             Input: Console.ForegroundColor = ConsoleColor.Magenta;
             Console.WriteLine();
             Console.WriteLine("----请输入一个年份代码（回车提交）：");
             Console.ResetColor();
             char chr = Convert.ToChar( Console.ReadLine().ToLower()[]);
             if ((int)chr >=  &&(int)chr <= )
             {
                 string year = string.Empty;
                 switch (chr)
                 {
                     case 'a':
                         year = ""; break;
                     case 'b':
                         year = ""; break;
                     case 'c':
                         year = ""; break;
                     default:
                         year = ""; break;
                 }
                 System.Diagnostics.Process.Start($"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/{year}");
                 Console.ForegroundColor = ConsoleColor.Magenta;
                 Console.WriteLine("浏览器已加载区划代码起始页，请进入县级行政单位页面，复制url，粘贴到下面（回车提交）：");
             }
             else
                 goto Input;
             Console.ResetColor();
             string cityurl = Console.ReadLine();
             if (cityurl.Length != && cityurl.Length!=)
             {
                 Console.ForegroundColor = ConsoleColor.Magenta;
                 Console.WriteLine("url有误，请确认是县级行政单位页面，重新复制链接，粘贴到下面：");
                 Console.ResetColor();
                 cityurl = Console.ReadLine();
             }
             try
             {
                 Console.ForegroundColor = ConsoleColor.Magenta;
                 Func<object, List<TownInfo>> func = new Func<object, List<TownInfo>>(GetTownInfos);
                 Task<List<TownInfo>> task = new Task<List<TownInfo>>(func, cityurl);
                 task.Start();
                 task.Wait();
                 if (task.Status == TaskStatus.RanToCompletion && task.Result.Count > )
                 {

                     List<VillageInfo> villageInfos = new List<VillageInfo>();
                     foreach (var item in task.Result)
                     {
                         //把乡镇信息写入村级列表，实现乡镇信息输出
                         VillageInfo villageInfo_town = new VillageInfo(item.Code, "", item.Name);
                         villageInfos.Add(villageInfo_town);
                         Func<object, List<VillageInfo>> func1 = new Func<object, List<VillageInfo>>(GetVillageInfos);
                         Task<List<VillageInfo>> task1 = new Task<List<VillageInfo>>(func1, item.Href);
                         task1.Start();
                         task1.Wait();
                         if (task1.Status == TaskStatus.RanToCompletion)
                         {
                             villageInfos.AddRange(task1.Result);
                         }
                     }
                     foreach (var item1 in villageInfos)
                     {
                         Console.WriteLine($"{item1.Name.Trim()}\t{item1.Cls.Trim()}\t{item1.Code.Trim()}");
                     }
                 }
                 else
                 { Console.WriteLine("乡镇列表获取失败！"); }

             }
             catch (Exception)
             {
                 throw new Exception("");
             }
             Console.ReadKey();
         }
         static string cookies = "AD_RS_COOKIE=20082854; wzws_cid=453a2d88181321410de83ba7eedaba3a141eb61ee7488027b6ab07a66054605e99e886827afa72708ce170398ea2fdfeec55455a7c0be8e779694026255f2166";
         //获取乡镇级信息列表
         static List<TownInfo> GetTownInfos(object cityurl)
         {
             List<TownInfo> townInfos = new List<TownInfo>();
             HttpGetHelper httpGetHelper = new HttpGetHelper() { Url =(string) cityurl, ContentType = "text/html; charset=gb2312", Encode = Encoding.GetEncoding(),RequestMethod="post"};
             //HtmlAgilityPack类库解析HTML
             HtmlDocument document = new HtmlDocument();
             document.LoadHtml(httpGetHelper.GetHtml(,ref cookies));
             //string html = httpGetHelper.GetHtml(ref cookies);
             //路径里"//"表示从根节点开始查找，两个斜杠‘//’表示查找所有childnodes；一个斜杠'/'表示只查找第一层的childnodes（即不查找grandchild）；点斜杠"./"表示从当前结点而不是根结点开始查找
             HtmlNodeCollection htmlNodes = document.DocumentNode.SelectNodes("//tr[@class='towntr']");
             foreach (var node in htmlNodes)
             {
                 HtmlNodeCollection htmlNodes1 = node.SelectNodes("./td");
                 HtmlNode htmlNodeHref = node.SelectSingleNode(".//a[@href]");
                 HtmlAttribute htmlAttribute = htmlNodeHref.Attributes["href"];
                 TownInfo townInfo = new TownInfo(htmlNodes1[].InnerText, htmlNodes1[].InnerText,
                     (cityurl as string).Substring(, (cityurl as string).LastIndexOf('/') + ) + htmlAttribute.Value);
                 townInfos.Add(townInfo);
             }
             return townInfos;
         }
         //获取村级信息列表
         static List<VillageInfo> GetVillageInfos(object townurl)
         {
             List<VillageInfo> villageInfos = new List<VillageInfo>();
             HttpGetHelper httpGetHelper = new HttpGetHelper() { Url = (string)townurl, ContentType = "text/html; charset=gb2312", Encode = Encoding.GetEncoding(), RequestMethod = "post"};
             HtmlDocument document = new HtmlDocument();
             document.LoadHtml(httpGetHelper.GetHtml(,ref cookies));
             //string html = httpGetHelper.GetHtml(ref cookies);
             HtmlNodeCollection htmlNodes = document.DocumentNode.SelectNodes("//tr[@class='villagetr']");
             foreach (var node in htmlNodes)
             {
                 HtmlNodeCollection htmlNodes1 = node.SelectNodes(".//td");
                 VillageInfo villageInfo = new VillageInfo(htmlNodes1[].InnerText,htmlNodes1[].InnerText,htmlNodes1[].InnerText);
                 villageInfos.Add(villageInfo);
             }
             return villageInfos;
         }
     }

辅助类/结构

   internal class Cursor
     {
         const int origRow = ;
         const int origCol = ;
         public static void WriteAt(string s, int c, int r)
         {
             Console.SetCursorPosition(origCol + c, origRow + r);
             Console.Write(s);
         }
     }
     //乡镇信息结构 编码、名称、超链
     struct TownInfo
     {
         string code;
         public string Code{ get { return code; } }
         string name;
         public string Name{get { return name; } }
         string href;
         public string Href { get { return href; } }
         public TownInfo (string code,string name,string href)
         {
             this.code = code;
             this.name = name;
             this.href = href;
         }
     }
     //村信息结构 编码、城乡划分类，名称
     struct VillageInfo
     {
         string code;
         public string Code{ get { return code; } }
         string  cls;
         public string Cls{ get { return cls; } }
         string name;
         public string Name{ get { return name; } }
         public VillageInfo(string code,string cls,string name)
         {
             this.code = code;
             this.cls = cls;
             this.name = name;
         }
     }

获取HTML

     public class HttpGetHelper
     {
         string url = string.Empty;
         public string Url
         {
             set { url = value; }
         }

         int timeOut=*;
         public int Timeout
         {
             set { timeOut = value; }
         }

         string contentType= "text/html;charset=utf-8";
         public string ContentType
         {
             set { contentType = value; }
         }

         string userAgent= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 ";
         public string UserAgent
         {
             set { userAgent = value; }
         }

         Encoding encode=Encoding.UTF8;
         public Encoding Encode
         {
             set { encode = value; }
         }
         string request_Method = "get";
         public string RequestMethod
         {
             set { request_Method = value; }
         }
         /// <summary>
         /// get html content
         /// </summary>
         /// <param name="cls">town=1;village=2</param>
         /// <param name="cookies">if cls=1 then ref cookies</param>
         /// <returns></returns>
         public string GetHtml(int cls,ref string cookies)
         {
             string html = string.Empty;
             try
             {
                 if (url!=string.Empty)
                 {
                     HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
                     request.Timeout = this.timeOut;
                     request.ContentType = this.contentType;
                     request.UserAgent = this.userAgent;
                     request.Headers.Add(HttpRequestHeader.Cookie, cookies);
                     request.Method = request_Method;
                     using (HttpWebResponse response =request.GetResponse()as HttpWebResponse)
                     {
                         if (response.StatusCode==HttpStatusCode.OK)
                         {//如果是县级url，则记录cookie
                             if (cls==)
                             {
                                 CookieCollection cookieCollection = response.Cookies;
                                 foreach (Cookie item in cookieCollection)
                                 {
                                     cookies = item.Name + "=" + item.Value + ";";
                                 }
                                 cookies.Remove(cookies.Length - );
                             }

                             using (StreamReader streamReader = new StreamReader(response.GetResponseStream(), encode))
                             {
                                 html = streamReader.ReadToEnd();
                                 streamReader.Close();
                             }
                         }
                     }
                 }
             }
             catch (Exception)
             {
                 throw new Exception($"GetHtml失败，url:{url}");
             }
             return html;
         }
     }