asp.net 网页抓取内容

网页抓取代码

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
//
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Text;

namespace WSYL.Web.Common
{
    public static class GetSteamShipInfo
    {
        public static string GetWebSite(string steamshipname,int itype)
        {
            if (steamshipname == null || steamshipname.Trim() == "")
                return null;
            //step1: get html from url
            string urlToCrawl = @"网址";
            //generate http request
            HttpWebRequest req = (HttpWebRequest)WebRequest.Create(urlToCrawl);
            //use GET method to get url's html
            req.Method = "GET";
            //use request to get response
            HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
            // 二〇一五年八月十二日 18:14:45 需要增加判断网页解析超时问题 防止网页假死
            // string htmlCharset = "UTF-8";
            string htmlCharset = "utf-8";
            //use songtaste's html's charset GB2312 to decode html
            //otherwise will return messy code
            Encoding htmlEncoding = Encoding.GetEncoding(htmlCharset);
            StreamReader sr = new StreamReader(resp.GetResponseStream(), htmlEncoding);
            //read out the returned html
            string respHtml = sr.ReadToEnd();
            //第三种获取内容
            //Match TitleMatch = Regex.Match(rtbExtractedHtml.Text.ToString(), "<td width=\"30%\">([^<]*)</td>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
//需要获取的代码开始和结尾内容
            Match TitleMatch2 = Regex.Match(respHtml.ToString(), "<td align=\"left\" bgcolor=\"#EEEEEE\">([^<]*)</td>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            // txbExtractedInfo.Text = TitleMatch2.Groups[1].Value+"/"+ TitleMatch2.Groups[2].Value;
            if (TitleMatch2.Groups[1].Value.Length == 0 || TitleMatch2.Groups[1].Value=="")
               return respHtml = "";
            if(itype==0)
            {
                respHtml = TitleMatch2.Groups[1].Value.ToString();
            }
             if(itype==1)
            {
                respHtml = StripHtml(TitleMatch2.NextMatch().Value.ToString());
            }
             if (itype == 2)
             {
                 respHtml = TitleMatch2.Groups[1].Value + "/" + StripHtml(TitleMatch2.NextMatch().Value.ToString());
             }
            return  respHtml;
        }
        /// <summary>
        /// 去除html标签和空格有些例外会使得去除不干净，所以建议连续两次转化。这样将Html标签转化为了空格。太多连续的空格会影响之后对字符串的操作
        /// </summary>
        /// <param name="strHtml">标签内容</param>
        /// <returns></returns>
        private static string StripHtml(string strHtml)
        {
            Regex objRegExp = new Regex("<(.|\n)+?>");
            string strOutput = objRegExp.Replace(strHtml, "");
            strOutput = strOutput.Replace("<", "&lt;");
            strOutput = strOutput.Replace(">", "&gt;");
            //把所有空格变为一个空格
            Regex r = new Regex(@"\s+");
            strOutput = r.Replace(strOutput, " ");
            return strOutput.Trim();
        }
    }
}