c# 查询本机可用的代理ip
现在有很多网站都提供免费的代理ip,但是你会发现很多网站显示的可以用的 ,在自己电脑上是用不了,写个小代码提取出自己电脑上可以用的代理,没什么技术含量,只是为了记录一下
string strUrl = "https://www.xicidaili.com/nt/";
HttpHelper httpProxy = new HttpHelper();
for (int i = ; i <= ; i++)
{
string strHtml = httpProxy.Get_Request(strUrl + i, httpProxy.cookie, "www.xicidaili.com", "https://www.xicidaili.com/nt/", blnHttps: true, strAccept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", strUserAgent: "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36");
HtmlAgilityPack.HtmlDocument docList = new HtmlAgilityPack.HtmlDocument();
docList.LoadHtml(strHtml);
var trListNodes = docList.DocumentNode.SelectNodes("//*[@id=\"ip_list\"]/tr");
for (int j = ; j < trListNodes.Count; j++)
{
var trNode = trListNodes[j];
string strIp = trNode.SelectSingleNode("td[2]").InnerText.Trim();
string strPort = trNode.SelectSingleNode("td[3]").InnerText.Trim();
int intPort = int.Parse(strPort);
try
{
WebProxy proxyObject = new WebProxy(strIp, intPort);// port为端口号 整数型
HttpWebRequest Req = WebRequest.Create("http://www.bookschina.com") as HttpWebRequest;
Req.Proxy = proxyObject; //设置代理
Req.Timeout = ; //超时
DateTime dt = DateTime.Now;
var Resp = (HttpWebResponse)Req.GetResponse();
Encoding bin = Encoding.GetEncoding("gb2312");
StreamReader sr = new StreamReader(Resp.GetResponseStream(), bin);
string str = sr.ReadToEnd();
sr.Close();
sr.Dispose();
var time = (DateTime.Now - dt).TotalMilliseconds;
Console.WriteLine(strIp + " " + intPort + " " + time);
}
catch
{ }
} }
HttpHelper
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks; namespace TestProxy
{
public class HttpHelper
{
public CookieContainer cookie;
public HttpHelper()
{
cookie = new CookieContainer();
}
private static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{
return true; //总是接受
}
public string Get_Request(
string strUrl,
CookieContainer _cookie = null,
string strHost = "",
string strRefer = "",
string strOrigin = "",
Dictionary<string, string> lstHeads = null,
string strEncoding = "utf-8",
string strContentType = "",
string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
bool blnAllowAutoRedirect = true,
int intTimeout = * ,
bool blnHttps = false,
System.Net.WebProxy proxy = null)
{
HttpWebRequest request;
HttpWebResponse response;
request = (HttpWebRequest)WebRequest.Create(strUrl);
request.Accept = strAccept;
request.Timeout = intTimeout;
request.Method = "GET";
request.Credentials = CredentialCache.DefaultCredentials;
request.UserAgent = strUserAgent;
request.AllowAutoRedirect = blnAllowAutoRedirect;
if (blnHttps)
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; }
if (!string.IsNullOrEmpty(strContentType))
{
request.ContentType = strContentType;
}
if (_cookie != null)
{
request.CookieContainer = _cookie;
}
if (!string.IsNullOrEmpty(strHost))
{
request.Host = strHost;
}
if (!string.IsNullOrEmpty(strRefer))
{
request.Referer = strRefer;
}
if (!string.IsNullOrEmpty(strOrigin))
{
request.Headers.Add("Origin", strOrigin);
}
if (lstHeads != null && lstHeads.Count > )
{
foreach (var item in lstHeads)
{
request.Headers.Add(item.Key, item.Value);
}
} if (proxy != null)
request.Proxy = proxy;
response = (HttpWebResponse)request.GetResponse();
var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));
string strResult = sr.ReadToEnd();
sr.Close();
request.Abort();
response.Close();
return strResult; } public string POST_Request(
string strUrl,
string postDataStr,
CookieContainer _cookie = null,
string strHost = "",
string strRefer = "",
string strOrigin = "",
Dictionary<string, string> lstHeads = null,
string strEncoding = "utf-8",
string strContentType = "",
string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
bool blnAllowAutoRedirect = true,
int intTimeout = * ,
bool blnHttps = false,
System.Net.WebProxy proxy = null)
{
HttpWebRequest request;
HttpWebResponse response;
request = (HttpWebRequest)WebRequest.Create(strUrl);
request.Accept = strAccept;
request.Timeout = intTimeout;
request.Method = "POST";
request.Host = strHost;
request.UserAgent = strUserAgent;
if (_cookie != null)
{
request.CookieContainer = _cookie;
}
if (blnHttps)
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; }
request.AllowAutoRedirect = blnAllowAutoRedirect;
if (!string.IsNullOrEmpty(strContentType))
{
request.ContentType = strContentType;
}
if (!string.IsNullOrEmpty(strOrigin))
{
request.Headers.Add("Origin", strOrigin);
}
if (!string.IsNullOrEmpty(strRefer))
{
request.Referer = strRefer;
}
if (!string.IsNullOrEmpty(strHost))
{
request.Host = strHost;
}
if (lstHeads != null && lstHeads.Count > )
{
foreach (var item in lstHeads)
{
request.Headers.Add(item.Key, item.Value);
}
}
if (!string.IsNullOrEmpty(postDataStr))
{
request.ContentLength = postDataStr.Length;
Stream myRequestStream = request.GetRequestStream();
StreamWriter myStreamWriter = new StreamWriter(myRequestStream);
myStreamWriter.Write(postDataStr);
myStreamWriter.Close();
}
if (proxy != null)
request.Proxy = proxy;
response = (HttpWebResponse)request.GetResponse();
var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));
string strResult = sr.ReadToEnd();
sr.Close();
request.Abort();
response.Close();
return strResult;
} public string DownloadFile(
string strURLAddress,
string strPath,
CookieContainer _cookie = null,
string strHost = "",
string strRefer = "",
string strOrigin = "",
Dictionary<string, string> lstHeads = null,
string strAccept = "",
string strUserAgent = "",
bool blnHttps = false,
System.Net.WebProxy proxy = null)
{
try
{
// 设置参数
HttpWebRequest request = WebRequest.Create(strURLAddress) as HttpWebRequest;
if (!string.IsNullOrEmpty(strAccept))
{
request.Accept = strAccept;
}
if (blnHttps)
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; }
if (!string.IsNullOrEmpty(strUserAgent))
{
request.UserAgent = strUserAgent;
}
if (_cookie != null)
{
request.CookieContainer = _cookie;
}
if (!string.IsNullOrEmpty(strOrigin))
{
request.Headers.Add("Origin", strOrigin);
}
if (!string.IsNullOrEmpty(strRefer))
{
request.Referer = strRefer;
}
if (!string.IsNullOrEmpty(strHost))
{
request.Host = strHost;
}
if (lstHeads != null && lstHeads.Count > )
{
foreach (var item in lstHeads)
{
request.Headers.Add(item.Key, item.Value);
}
}
if (proxy != null)
request.Proxy = proxy;
request.Timeout = ;
//发送请求并获取相应回应数据
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
string strReceivePath = string.Empty; //直到request.GetResponse()程序才开始向目标网页发送Post请求
Stream responseStream = response.GetResponseStream();
//创建本地文件写入流
Stream stream = new FileStream(strPath, FileMode.Create);
byte[] bArr = new byte[];
int size = responseStream.Read(bArr, , (int)bArr.Length);
while (size > )
{
stream.Write(bArr, , size);
stream.Flush();
size = responseStream.Read(bArr, , (int)bArr.Length);
}
stream.Close();
responseStream.Close();
return strPath;
}
catch (Exception ex)
{
return "";
}
} }
}
这个需要.net 4.5

c# 查询本机可用的代理ip的更多相关文章
- python检验代理ip是否可用、代理ip检验
python检验代理ip是否可用.代理ip检验 安装相关模块: pip install requests 验证代理IP是否可用脚本: import random import telnetlib im ...
- 利用Python爬取可用的代理IP
前言 就以最近发现的一个免费代理IP网站为例:http://www.xicidaili.com/nn/.在使用的时候发现很多IP都用不了. 所以用Python写了个脚本,该脚本可以把能用的代理IP检测 ...
- 把你的Centos设置成代理ip服务器
前言:最近在公司做爬虫相关的工作,做过数据抓取的都知道,写程序抓取数据的过程并不像平常我们用浏览器打开网页那么简单!大多数的网站为了自己站点的性能和数据安全都设置了各种反爬策略.最常见的就是添加验证码 ...
- 基于后端和爬虫创建的代理ip池
搭建免费的代理ip池 需要解决的问题: 使用什么方式存储ip 文件存储 缺点: 打开文件修改文件操作较麻烦 mysql 缺点: 查询速度较慢 mongodb 缺点: 查询速度较慢. 没有查重功能 re ...
- 【python3】如何建立爬虫代理ip池
一.为什么需要建立爬虫代理ip池 在众多的网站防爬措施中,有一种是根据ip的访问频率进行限制的,在某段时间内,当某个ip的访问量达到一定的阀值时,该ip会被拉黑.在一段时间内被禁止访问. 这种时候,可 ...
- springboot实现java代理IP池 Proxy Pool,提供可用率达到95%以上的代理IP
一.背景 前段时间,写java爬虫来爬网易云音乐的评论.不料,爬了一段时间后ip被封禁了.由此,想到了使用ip代理,但是找了很多的ip代理网站,很少有可以用的代理ip.于是,抱着边学习的心态,自己开发 ...
- Python3.x:免费代理ip的批量获取并入库
Python3.x:免费代理ip的批量获取并入库 一.简介 网络爬虫的世界,向来都是一场精彩的攻防战.现在许多网站的反爬虫机制在不断的完善,其中最令人头疼的,莫过于直接封锁你的ip.但是道高一尺魔高一 ...
- Python3.x:代理ip刷评分
Python3.x:代理ip刷评分 声明:仅供为学习材料,不允许用作商业用途: 一,功能: 针对某网站对企业自动刷评分: 网站:https://best.zhaopin.com/ 二,步骤: 1,获取 ...
- Python3.x:代理ip刷点赞
Python3.x:代理ip刷点赞 声明:仅供为学习材料,不允许用作商业用途: 一,功能: 针对某网站对企业自动刷点赞: 网站:https://best.zhaopin.com/ 二,步骤: 1,获取 ...
随机推荐
- 【t003】string
Time Limit: 1 second Memory Limit: 50 MB [问题描述] 设有字符串X,我们称在X的头尾及中间插入任意多个空格后构成的新字符串为X的扩展串,如字符串X为" ...
- CodeBlocks环境搭建及创建第一个C++程序
某业界大牛推荐最佳的途径是从raytracing入门,所以本屌开始学习<Ray Tracing In One Weekend>. 该书是基于C++的.本屌从未学过C++.感觉告诉我,要先搭 ...
- Arcgis api for javascript学习笔记(4.5版本) - 点击多边形(Polygon)并高亮显示
在现在的 arcgis_js_v45_api 版本中并没有直接提供点击Polygon对象高亮显示.需要实现如下几个步骤: 1.点击地图时,获取Polygon的Graphic对象: 2.对获取到的Gra ...
- WinFrom - 打开一个窗体关闭另一个窗体
在写Winform程序的时候,经常会遇到“打开一个新的窗体然后关闭本窗体”之类的问题.最常见的就是登陆界面和主窗体了.而最常见的处理方法就是表单判断正确后 new MainForm().Show(); ...
- PAT 1051-1060 题解
浏览全部代码:请戳 本文谨代表个人思路,欢迎讨论;) 1051. Pop Sequence (25) 题意 给定 stack 的容量,给定数据的入栈顺序:从 1 开始的正整数序列,在允许随机的出栈操作 ...
- windows下安装ffmpeg
一.下载地址: 网址:https://ffmpeg.org/ 选择Windows版本:https://ffmpeg.org/download.html#build-windows 二.解压安装: 下载 ...
- WPF 实现繁花曲线
原文:WPF 实现繁花曲线 版权声明:本文为博主原创文章,未经博主允许不得转载. https://blog.csdn.net/nihang1234/article/details/83346919 X ...
- hudson搭建经验总结(三)
作者:朱金灿 来源:http://blog.csdn.net/clever101 在创建hudson账户和分配权限时出错,当单击save按钮时出现 HTTP Status 500 - type Exc ...
- linux 静态库和动态库(共享库)的制作与使用(注意覆盖问题)
一.linux操作系统支持的函数库分支 静态库:libxxx.a,在编译时就将库编译进可执行程序 优点:程序的运行环境中不需要外部的函数库 缺点:可执行程序大 动态库:又称共享库,libxxx.so, ...
- Java内部类——学习笔记
参考:http://blog.csdn.net/aaronsi/article/details/187322 和 http://openhome.cc/Gossip/JavaGossip-V1/Inn ...