解析ASPX网页__doPostBack分页的网页table数据

由于急于上线的功能要去客服系统里抓取数据进行验证，客服方面又没有时间开发EDI接口给到我，所以用了本办法：爬人家web系统上的数据进行分析。

由于客服的web系统用ASP.Net的__doPostBack控件进行数据分页。__doPostBack是通过__EVENTTARGET,__EVENTARGUMENT两个隐藏控件向服务端发送控制信息的。

这里我要分析的页面概况如下：

这里有个导出按钮，直接模拟导出按钮获取数据。模拟点击页面来获取我们要解析需要的参数：__VIEWSTATE、__EVENTVALIDATION

设计一个demo

 private void button1_Click(object sender, EventArgs e)
        {
            string strViewState = System.Web.HttpUtility.UrlEncode("/wEPDwUJOTEyODUyNDU3D2QWAgIBDw8WAh4EVGV4dAUMUGFsbGV0IExpc3Q6ZGQYAQUPZ3ZzeXNzdGF0dXNsaXN0DzwrAAwBCAIOZOMeKZkMrWymOLPLpoGaeUA09JXcMmBeiapHUaN/Gi/F");
            string strEventValidation = System.Web.HttpUtility.UrlEncode("/wEdAA0xoT8jJS8wSLBSnnngxJ11LlhQ8m9UI3ZtBAB4T5ifli9v5Ab4i1r+ooL3Ki4G1V2mwGQwvscKG/slAoCqDXMdx+s83MyrFGOK+Bhp33qS53KOlIwqJuEKsQlYBBWX4thggho5YkoND4EeSEP5w92hCXHcK3jw5s4JUgUUp9F6PJLbP8X7bsfD4kS7SIahZoNk3kkOw9nHzhwCbKD2SjMJqipXHjkN/zBo3hFqKyPYi8LcXcQFP+NtnRoXzZkJPyDN+DvxnwFeFeJ9MIBWR693WMHe4rn0nQ4UPheoLGOgfsnsDvJMKKjfZWVoPlnQzPA=");
            StringBuilder url = new StringBuilder();
            url.Append("&__VIEWSTATE=" + strViewState);
            url.Append("&__EVENTVALIDATION=" + strEventValidation);
            url.Append("&Button1=导出");
            byte[] data = System.Text.Encoding.ASCII.GetBytes(url.ToString());
            Uri uri = new Uri(textBox1.Text.ToString());
            System.Net.HttpWebRequest request = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(uri);
            request.Method = "post";
            request.ContentType = "application/x-www-form-urlencoded";
            //request.ContentType = "application/json";
            request.ContentLength = data.Length;
            Stream requestStream = request.GetRequestStream();
            requestStream.Write(data, , data.Length);
            requestStream.Close();
            System.Net.HttpWebResponse response = (System.Net.HttpWebResponse)request.GetResponse();
            Stream responseStream = response.GetResponseStream();
            StreamReader readStream = new StreamReader(responseStream, System.Text.Encoding.Default);
            string html = readStream.ReadToEnd().ToString();
            readStream.Close();
            var strReg = @"<table[^>]*>[\s\S]*</table>";
            List<string> result = new List<string>();
            MatchCollection mc = Regex.Matches(html, strReg);
            DataTable dt = new DataTable();
            dt.Columns.Add("ID", typeof(string));                    //序号
            dt.Columns.Add("SN", typeof(string));                    //系统序列号
            dt.Columns.Add("WO", typeof(string));                    //工单号
            dt.Columns.Add("NextStation", typeof(string));           //下一站别
            dt.Columns.Add("RepairHeld", typeof(string));            //不良否
            dt.Columns.Add("SKUNo", typeof(string));                 //成品料号
            dt.Columns.Add("Line", typeof(string));                  //线别
            dt.Columns.Add("LastEditTime", typeof(string));          //最后时间
            dt.Columns.Add("Carton", typeof(string));                //卡通号
            dt.Columns.Add("PalletNo", typeof(string));              //栈板号
            dt.Columns.Add("Closed", typeof(string));                //完成否
            dt.Columns.Add("ShipOrderNo", typeof(string));           //出货通知单号
            dt.Columns.Add("ContainerNo", typeof(string));           //货柜号
            dt.Columns.Add("TruckLoaded", typeof(string));           //出货否
            string strhead = string.Empty;
            Match m = mc[];
            int rowindex = ;
            foreach (Match mtr in Regex.Matches(m.Value, "(?is)(?<=<tr>).+?(?=</tr>)"))
            {
                if (rowindex == ) { rowindex++; continue; }
                DataRow dr = dt.NewRow();
                MatchCollection tds = Regex.Matches(mtr.Value, "(?is)(?<=<td>).+?(?=</td>)");
                MatchCollection tds2 = Regex.Matches(mtr.Value, "(?is)(?<=target=\"_blank\">).+?(?=</a></td>)");
                MatchCollection tds6 = Regex.Matches(mtr.Value, "(?is)(?<=@\">).+?(?=</td>)");
                MatchCollection ids = Regex.Matches(tds[].Value, "(?is)(?<=\">).+?(?=</span>)");
                dr["ID"] = FormatString(ids[].Value);
                dr["SN"] = FormatString(tds2[].Value);
                dr["WO"] = FormatString(tds[].Value);
                dr["NextStation"] = FormatString(tds[].Value);
                dr["RepairHeld"] = FormatString(tds[].Value);
                dr["SKUNo"] = FormatString(tds6[].Value);
                dr["Line"] = FormatString(tds[].Value);
                dr["LastEditTime"] = FormatString(tds[].Value);
                dr["Carton"] = FormatString(tds[].Value);
                dr["PalletNo"] = FormatString(tds[].Value);
                dr["Closed"] = FormatString(tds[].Value);
                dr["ShipOrderNo"] = FormatString(tds[].Value);
                dr["ContainerNo"] = FormatString(tds[].Value);
                dr["TruckLoaded"] = FormatString(tds[].Value);
                dt.Rows.Add(dr);
                rowindex++;
            }
            gridControl1.DataSource = dt;
        }
        /// <summary>
        /// 页面加载
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void Form1_Load(object sender, EventArgs e)
        {
            this.textBox1.Text = @"http://172.16.1.13/WIPDetail.aspx?location=PA-1112069";
        }
        private string FormatString(string str)
        {
            str = str.Replace("\r\n", "").Replace("&nbsp;", "").Trim();
            return str;
        }

正则表达式分析解析出来的HTML文本即可。

其实针对上述既要分页又是隐藏控件分页的table不太好爬数据，如果解析不分页的table的话没那么复杂，可以直接获取整个页面的html元素然后进行解析

列如我需要知道如下页面的table表里面的数据：

首先根据URL，WebRequest请求该页面，IO流读取整个页面的HTML元素，首先正则定位table，然后遍历这些元素，按照<tr>,<th>解析

WebRequest request = WebRequest.Create("http://172.16.1.13/FGIMA.aspx?t=ssn&sno=" + str_Pallet);
                WebResponse response = request.GetResponse();
                StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("utf-8"));
                var html = reader.ReadToEnd();
                var strReg = @"(?is)(?<=<table>).+?(?=</table>)";
                List<string> result = new List<string>();
                MatchCollection mc = Regex.Matches(html, strReg);
                DataTable dt = new DataTable();

                string strhead = string.Empty;
                foreach (Match m in mc)
                {
                    int rowindex = ;
                    foreach (Match mtr in Regex.Matches(m.Value, "(?is)(?<=<tr>).+?(?=</tr>)"))
                    {
                        if (rowindex == )
                        {
                            strhead = mtr.Value;
                            foreach (Match mtdh in Regex.Matches(mtr.Value, "(?is)(?<=<th>).+?(?=</th>)"))
                            {
                                if (!dt.Columns.Contains(mtdh.Value.Replace("\r\n", "").Trim()))
                                {
                                    dt.Columns.Add(mtdh.Value.Replace("\r\n", "").Trim());
                                }
                            }
                        }
                        else
                        {
                            DataRow dr = dt.NewRow();
                            if (mtr.Value.Contains("累计数量")) continue;
                            if (mtr.Value.Contains("没有找到相关数据"))
                            {
                                return dt;
                            }
                            MatchCollection mtdds = Regex.Matches(mtr.Value, "(?is)(?<=<td>).+?(?=</td>)");
                            int colindex = ;
                            foreach (Match mtdh in Regex.Matches(strhead, "(?is)(?<=<th>).+?(?=</th>)"))
                            {
                                if (mtdh.Value.Contains("机器序列号"))
                                {
                                    foreach (Match mtdhdd in Regex.Matches(mtr.Value, "(?is)(?<=target=\"_blank\">).+?(?=</a>)"))
                                    {
                                        dr["机器序列号"] = mtdhdd.Value.Replace("\r\n", "").Trim();
                                    }
                                }
                                else
                                    dr[mtdh.Value.Replace("\r\n", "").Trim()] = mtdds[colindex].Value.Replace("\r\n", "").Trim();

                                colindex++;
                            }
                            dt.Rows.Add(dr);
                        }
                        rowindex++;
                    }
                }
                if (dt.Columns.Contains("机器序列号"))
                {
                    dt.Columns["机器序列号"].ColumnName = "SN";
                }
                if (dt.Columns.Contains("工单号码"))
                {
                    dt.Columns["工单号码"].ColumnName = "WorkorderNo";
                }
                if (dt.Columns.Contains("成品料号"))
                {
                    dt.Columns["成品料号"].ColumnName = "PartNo";
                }
                if (dt.Columns.Contains("栈板号码"))
                {
                    dt.Columns["栈板号码"].ColumnName = "Pallet";
                }
                reader.Close();
                reader.Dispose();
                response.Close();
                return dt;