HttpWebRequest 模拟浏览器访问网站

最近抓网页时报错：

要么返回 The remote server returned an error: (442)
要么返回： 非法访问，您的行为已被WAF系统记录！

想了想，就当是人家加了抓网页的东西，于是改了一下方法 加上Request.Header 之类的东西就行了。

具体加什么，咱们可以先用 fildder 抓一下包就可以了如：

 

GET http://www.baidu.com/ HTTP/1.1
Host: www.baidu.com
Connection: keep-alive
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
Accept-Encoding: gzip, deflate
Accept-Language: zh-CN,zh;q=0.9

　　

 public static string GetHtml()
         {
             string url = "http://www.baidu.com";
             string Html = string.Empty;//初始化新的webRequst
             HttpWebRequest Request = (HttpWebRequest)WebRequest.Create(url);
             Request.Timeout = ;
             Request.ReadWriteTimeout = ;
          //   Request.ImpersonationLevel = TokenImpersonationLevel.Anonymous;

             Request.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
           //  Request.Headers.Add("Accept-Encoding", "gzip, deflate");

             Request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
             Request.KeepAlive = true;
             Request.ProtocolVersion = HttpVersion.Version11;
             Request.Method = "GET";
             Request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
             Request.Host = "www.baidu.com";
             //Request.Accept = "text/json,*/*;q=0.5";
             //Request.Headers.Add("Accept-Charset", "utf-8;q=0.7,*;q=0.7");
             //Request.Headers.Add("Accept-Encoding", "gzip, deflate, x-gzip, identity; q=0.9");
             Request.UserAgent = @"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36";
             Request.Referer = url;
             Request.IfModifiedSince = DateTime.UtcNow;

             HttpWebResponse htmlResponse = (HttpWebResponse)Request.GetResponse();
             //从Internet资源返回数据流
              Stream htmlStream = htmlResponse.GetResponseStream();
            // Stream htmlStream = new System.IO.Compression.GZipStream(htmlResponse.GetResponseStream(), System.IO.Compression.CompressionMode.Decompress);
             //读取数据流
             StreamReader weatherStreamReader = new StreamReader(htmlStream, Encoding.GetEncoding("gb2312"));
             //读取数据
             Html = weatherStreamReader.ReadToEnd();
             weatherStreamReader.Close();
             htmlStream.Close();
             htmlResponse.Close();
             //针对不同的网站查看html源文件
             return Html;
         }      

再加一段PHP的代码： 在不修改本页面utf-8编码的情况下如何让抓取的gb2312页面不乱码。

$headers = array();
$headers[] = 'X-Apple-Tz: 0';
$headers[] = 'X-Apple-Store-Front: 143444,12';
$headers[] = 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
$headers[] = 'Accept-Encoding: gzip, deflate';
$headers[] = 'Accept-Language: en-US,en;q=0.5';
$headers[] = 'Cache-Control: no-cache';
$headers[] = 'Content-Type: application/x-www-form-urlencoded; charset=gb2312';//utf-8
$headers[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36';

$dat = cUrlGetData($url, $post_fields, $headers);
        function cUrlGetData($url, $post_fields = null, $headers = null) {
           $ch = curl_init();
            $timeout = 50000;
            curl_setopt($ch, CURLOPT_URL, $url);
            if ($post_fields && !empty($post_fields)) {
                curl_setopt($ch, CURLOPT_POST, 1);
                curl_setopt($ch, CURLOPT_POSTFIELDS, $post_fields);
            }
            if ($headers && !empty($headers)) {
                curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
            }
            curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
            curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
            curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
            curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
            curl_setopt($ch, CURLOPT_ENCODING, 'gzip,deflate');//这个是解释gzip内容.................
            $data = curl_exec($ch);
            if (curl_errno($ch)) {
                echo 'Error:' . curl_error($ch);
            }
            curl_close($ch);
            return $data;
        }

        //php脚本开始
     /*POST请求远程内容函数*/
     function ppost($url,$data,$ref){ // 模拟提交数据函数
        $curl = curl_init(); // 启动一个CURL会话
        curl_setopt($curl, CURLOPT_URL, $url); // 要访问的地址
        curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, 0); // 对认证证书来源的检查
        curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 1); // 从证书中检查SSL加密算法是否存在
        curl_setopt($curl, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']); // 模拟用户使用的浏览器
        curl_setopt($curl, CURLOPT_FOLLOWLOCATION, 1); // 使用自动跳转
        curl_setopt($curl, CURLOPT_REFERER, $ref);
        curl_setopt($curl, CURLOPT_POST, 1); // 发送一个常规的Post请求
        curl_setopt($curl, CURLOPT_POSTFIELDS, $data); // Post提交的数据包
        curl_setopt($curl, CURLOPT_COOKIEFILE,$GLOBALS ['cookie_file']); // 读取上面所储存的Cookie信息
        curl_setopt($curl, CURLOPT_COOKIEJAR, $GLOBALS['cookie_file']); // 存放Cookie信息的文件名称

        curl_setopt($curl, CURLOPT_HTTPHEADER,array('Accept-Encoding: gzip, deflate'));
        curl_setopt($curl, CURLOPT_ENCODING, 'gzip,deflate');//这个是解释gzip内容.................
        curl_setopt($curl, CURLOPT_TIMEOUT, 30); // 设置超时限制防止死循环
        curl_setopt($curl, CURLOPT_HEADER, 0); // 显示返回的Header区域内容
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); // 获取的信息以文件流的形式返回
        $tmpInfo = curl_exec($curl); // 执行操作
        if (curl_errno($curl)) {
           echo 'Errno'.curl_error($curl);
        }
        curl_close($curl); // 关键CURL会话
        return $tmpInfo; // 返回数据
    }