C# WebClient Get获取网页内容

//不知道怎么删除，只好留着

1. Get方式：

WebClient web = new WebClient();
var html = web.DownloadString(url);

2. Post方式

         /// <summary>
         ///
         /// </summary>
         /// <param name="web"></param>
         /// <param name="url"></param>
         /// <param name="queryString">格式： paramname=value@name2=value2</param>
         /// <param name="clearHeads"></param>
         /// <returns></returns>
         public static string Post(this MyWebClient web, string url, string queryString, bool clearHeads=false)
         {
             string postString = queryString;// WebUtility.UrlEncode( queryString);//这里即为传递的参数，可以用工具抓包分析，也可以自己分析，主要是form里面每一个name都要加进来
             byte[] postData = Encoding.UTF8.GetBytes(postString);//编码，尤其是汉字，事先要看下抓取网页的编码方式
             web.RequestConentLength = postData.Length;
             if (clearHeads)
             {
                 web.Headers.Clear();
                 web.Headers.Add("Content-Type", "application/x-www-form-urlencoded");//采取POST方式必须加的header，如果改为GET方式的话就去掉这句话即可
             }

             byte[] responseData = web.UploadData(url, "POST", postData);//得到返回字符流
             string srcString = Encoding.UTF8.GetString(responseData);//解码
             return srcString;
         }

3. Headers设置

                     web.Headers.Add(HttpRequestHeader.Accept, "*/*");
                     web.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
                     web.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
                     //web.Headers.Add(HttpRequestHeader.Connection, "keep-alive");
                     web.Headers.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                     web.Headers.Add(HttpRequestHeader.Host, "wenshu.court.gov.cn");
                     web.Headers.Add("Origin", "http://wenshu.court.gov.cn");
                     //web.Headers.Add("Proxy-Connection", "keep-alive");
                     web.Headers.Add(HttpRequestHeader.UserAgent, userAgent);
                     web.Headers.Add("X-Requested-With", "XMLHttpRequest");
                     web.Headers.Add(HttpRequestHeader.Referer, WebUtility.UrlEncode(Referer1));

4.Cookie、超时等高可用基类

     public class MyWebClient : WebClient
     {
         public CookieContainer Cookies ;

         public MyWebClient(CookieContainer cookieContainer)
         {
             this.Cookies = cookieContainer;
         }

         public int TimeoutSeconds { get; set; } = ;

         public WebRequest Request { get; set; }

         public int RequestConentLength;

         protected override WebRequest GetWebRequest(Uri address)
         {
             HttpWebRequest request = base.GetWebRequest(address) as HttpWebRequest;

             if (request != null)
             {
                 request.Method = "Post";
                 request.CookieContainer = Cookies;
                 request.Timeout =  * TimeoutSeconds;
                 request.ContentLength = RequestConentLength;
             }

             Request = request;
             return request;
         }

         public WebResponse Response { get; set; }

         protected override WebResponse GetWebResponse(WebRequest request)
         {
             this.Response = base.GetWebResponse(request);
             return this.Response;
         }

         public string GetCookieValue(string cookieName)
         {
             var cookies = this.Cookies.GetCookies(this.Request.RequestUri);
             var ck = cookies[cookieName];
             return ck?.Value;
         }
     }

特别注意，一个浏览器对于网站多次请求的情况，要创建多个WebClient对象，但是它们应该共用一个CookieContainer。 而当写爬虫，模拟多个浏览器会话时，又不应全部使用同样的CookieContainer对象，以避免会话冲突。

附赠一个C#调用JS脚本的代码：

         public string CallJs(string jsCall , string jsFunctions)
         {
             Type obj = Type.GetTypeFromProgID("ScriptControl");
             if (obj == null) return null;
             object ScriptControl = Activator.CreateInstance(obj);
             obj.InvokeMember("Language", BindingFlags.SetProperty, null, ScriptControl, new object[] { "JavaScript" });
             //string js = "function time(a, b, msg){ var sum = a + b; return new Date().getTime() + ': ' + msg + ' = ' + sum }";
             obj.InvokeMember("AddCode", BindingFlags.InvokeMethod, null, ScriptControl, new object[] { jsFunctions });

             //return obj.InvokeMember("Eval", BindingFlags.InvokeMethod, null, ScriptControl, new object[] { "time(3, 5, '3 + 5')" }).ToString();
             return obj.InvokeMember("Eval", BindingFlags.InvokeMethod, null, ScriptControl, new object[] { jsCall }).ToString();
         }

使用示例：

string js = "function jsfunction(parm){ return parm + "abc"; }";
string val = CallJs($"jsfunction('{csvar}')", js.ToString());