现在有很多网站都提供免费的代理ip,但是你会发现很多网站显示的可以用的 ,在自己电脑上是用不了,写个小代码提取出自己电脑上可以用的代理,没什么技术含量,只是为了记录一下

 string strUrl = "https://www.xicidaili.com/nt/";
HttpHelper httpProxy = new HttpHelper();
for (int i = ; i <= ; i++)
{
string strHtml = httpProxy.Get_Request(strUrl + i, httpProxy.cookie, "www.xicidaili.com", "https://www.xicidaili.com/nt/", blnHttps: true, strAccept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", strUserAgent: "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36");
HtmlAgilityPack.HtmlDocument docList = new HtmlAgilityPack.HtmlDocument();
docList.LoadHtml(strHtml);
var trListNodes = docList.DocumentNode.SelectNodes("//*[@id=\"ip_list\"]/tr");
for (int j = ; j < trListNodes.Count; j++)
{
var trNode = trListNodes[j];
string strIp = trNode.SelectSingleNode("td[2]").InnerText.Trim();
string strPort = trNode.SelectSingleNode("td[3]").InnerText.Trim();
int intPort = int.Parse(strPort);
try
{
WebProxy proxyObject = new WebProxy(strIp, intPort);// port为端口号 整数型
HttpWebRequest Req = WebRequest.Create("http://www.bookschina.com") as HttpWebRequest;
Req.Proxy = proxyObject; //设置代理
Req.Timeout = ; //超时
DateTime dt = DateTime.Now;
var Resp = (HttpWebResponse)Req.GetResponse();
Encoding bin = Encoding.GetEncoding("gb2312");
StreamReader sr = new StreamReader(Resp.GetResponseStream(), bin);
string str = sr.ReadToEnd();
sr.Close();
sr.Dispose();
var time = (DateTime.Now - dt).TotalMilliseconds;
Console.WriteLine(strIp + " " + intPort + " " + time);
}
catch
{ }
} }

HttpHelper

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks; namespace TestProxy
{
public class HttpHelper
{
public CookieContainer cookie;
public HttpHelper()
{
cookie = new CookieContainer();
}
private static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{
return true; //总是接受
}
public string Get_Request(
string strUrl,
CookieContainer _cookie = null,
string strHost = "",
string strRefer = "",
string strOrigin = "",
Dictionary<string, string> lstHeads = null,
string strEncoding = "utf-8",
string strContentType = "",
string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
bool blnAllowAutoRedirect = true,
int intTimeout = * ,
bool blnHttps = false,
System.Net.WebProxy proxy = null)
{
HttpWebRequest request;
HttpWebResponse response;
request = (HttpWebRequest)WebRequest.Create(strUrl);
request.Accept = strAccept;
request.Timeout = intTimeout;
request.Method = "GET";
request.Credentials = CredentialCache.DefaultCredentials;
request.UserAgent = strUserAgent;
request.AllowAutoRedirect = blnAllowAutoRedirect;
if (blnHttps)
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; }
if (!string.IsNullOrEmpty(strContentType))
{
request.ContentType = strContentType;
}
if (_cookie != null)
{
request.CookieContainer = _cookie;
}
if (!string.IsNullOrEmpty(strHost))
{
request.Host = strHost;
}
if (!string.IsNullOrEmpty(strRefer))
{
request.Referer = strRefer;
}
if (!string.IsNullOrEmpty(strOrigin))
{
request.Headers.Add("Origin", strOrigin);
}
if (lstHeads != null && lstHeads.Count > )
{
foreach (var item in lstHeads)
{
request.Headers.Add(item.Key, item.Value);
}
} if (proxy != null)
request.Proxy = proxy;
response = (HttpWebResponse)request.GetResponse();
var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));
string strResult = sr.ReadToEnd();
sr.Close();
request.Abort();
response.Close();
return strResult; } public string POST_Request(
string strUrl,
string postDataStr,
CookieContainer _cookie = null,
string strHost = "",
string strRefer = "",
string strOrigin = "",
Dictionary<string, string> lstHeads = null,
string strEncoding = "utf-8",
string strContentType = "",
string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
bool blnAllowAutoRedirect = true,
int intTimeout = * ,
bool blnHttps = false,
System.Net.WebProxy proxy = null)
{
HttpWebRequest request;
HttpWebResponse response;
request = (HttpWebRequest)WebRequest.Create(strUrl);
request.Accept = strAccept;
request.Timeout = intTimeout;
request.Method = "POST";
request.Host = strHost;
request.UserAgent = strUserAgent;
if (_cookie != null)
{
request.CookieContainer = _cookie;
}
if (blnHttps)
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; }
request.AllowAutoRedirect = blnAllowAutoRedirect;
if (!string.IsNullOrEmpty(strContentType))
{
request.ContentType = strContentType;
}
if (!string.IsNullOrEmpty(strOrigin))
{
request.Headers.Add("Origin", strOrigin);
}
if (!string.IsNullOrEmpty(strRefer))
{
request.Referer = strRefer;
}
if (!string.IsNullOrEmpty(strHost))
{
request.Host = strHost;
}
if (lstHeads != null && lstHeads.Count > )
{
foreach (var item in lstHeads)
{
request.Headers.Add(item.Key, item.Value);
}
}
if (!string.IsNullOrEmpty(postDataStr))
{
request.ContentLength = postDataStr.Length;
Stream myRequestStream = request.GetRequestStream();
StreamWriter myStreamWriter = new StreamWriter(myRequestStream);
myStreamWriter.Write(postDataStr);
myStreamWriter.Close();
}
if (proxy != null)
request.Proxy = proxy;
response = (HttpWebResponse)request.GetResponse();
var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));
string strResult = sr.ReadToEnd();
sr.Close();
request.Abort();
response.Close();
return strResult;
} public string DownloadFile(
string strURLAddress,
string strPath,
CookieContainer _cookie = null,
string strHost = "",
string strRefer = "",
string strOrigin = "",
Dictionary<string, string> lstHeads = null,
string strAccept = "",
string strUserAgent = "",
bool blnHttps = false,
System.Net.WebProxy proxy = null)
{
try
{
// 设置参数
HttpWebRequest request = WebRequest.Create(strURLAddress) as HttpWebRequest;
if (!string.IsNullOrEmpty(strAccept))
{
request.Accept = strAccept;
}
if (blnHttps)
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; }
if (!string.IsNullOrEmpty(strUserAgent))
{
request.UserAgent = strUserAgent;
}
if (_cookie != null)
{
request.CookieContainer = _cookie;
}
if (!string.IsNullOrEmpty(strOrigin))
{
request.Headers.Add("Origin", strOrigin);
}
if (!string.IsNullOrEmpty(strRefer))
{
request.Referer = strRefer;
}
if (!string.IsNullOrEmpty(strHost))
{
request.Host = strHost;
}
if (lstHeads != null && lstHeads.Count > )
{
foreach (var item in lstHeads)
{
request.Headers.Add(item.Key, item.Value);
}
}
if (proxy != null)
request.Proxy = proxy;
request.Timeout = ;
//发送请求并获取相应回应数据
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
string strReceivePath = string.Empty; //直到request.GetResponse()程序才开始向目标网页发送Post请求
Stream responseStream = response.GetResponseStream();
//创建本地文件写入流
Stream stream = new FileStream(strPath, FileMode.Create);
byte[] bArr = new byte[];
int size = responseStream.Read(bArr, , (int)bArr.Length);
while (size > )
{
stream.Write(bArr, , size);
stream.Flush();
size = responseStream.Read(bArr, , (int)bArr.Length);
}
stream.Close();
responseStream.Close();
return strPath;
}
catch (Exception ex)
{
return "";
}
} }
}

这个需要.net 4.5

c# 查询本机可用的代理ip的更多相关文章

  1. python检验代理ip是否可用、代理ip检验

    python检验代理ip是否可用.代理ip检验 安装相关模块: pip install requests 验证代理IP是否可用脚本: import random import telnetlib im ...

  2. 利用Python爬取可用的代理IP

    前言 就以最近发现的一个免费代理IP网站为例:http://www.xicidaili.com/nn/.在使用的时候发现很多IP都用不了. 所以用Python写了个脚本,该脚本可以把能用的代理IP检测 ...

  3. 把你的Centos设置成代理ip服务器

    前言:最近在公司做爬虫相关的工作,做过数据抓取的都知道,写程序抓取数据的过程并不像平常我们用浏览器打开网页那么简单!大多数的网站为了自己站点的性能和数据安全都设置了各种反爬策略.最常见的就是添加验证码 ...

  4. 基于后端和爬虫创建的代理ip池

    搭建免费的代理ip池 需要解决的问题: 使用什么方式存储ip 文件存储 缺点: 打开文件修改文件操作较麻烦 mysql 缺点: 查询速度较慢 mongodb 缺点: 查询速度较慢. 没有查重功能 re ...

  5. 【python3】如何建立爬虫代理ip池

    一.为什么需要建立爬虫代理ip池 在众多的网站防爬措施中,有一种是根据ip的访问频率进行限制的,在某段时间内,当某个ip的访问量达到一定的阀值时,该ip会被拉黑.在一段时间内被禁止访问. 这种时候,可 ...

  6. springboot实现java代理IP池 Proxy Pool,提供可用率达到95%以上的代理IP

    一.背景 前段时间,写java爬虫来爬网易云音乐的评论.不料,爬了一段时间后ip被封禁了.由此,想到了使用ip代理,但是找了很多的ip代理网站,很少有可以用的代理ip.于是,抱着边学习的心态,自己开发 ...

  7. Python3.x:免费代理ip的批量获取并入库

    Python3.x:免费代理ip的批量获取并入库 一.简介 网络爬虫的世界,向来都是一场精彩的攻防战.现在许多网站的反爬虫机制在不断的完善,其中最令人头疼的,莫过于直接封锁你的ip.但是道高一尺魔高一 ...

  8. Python3.x:代理ip刷评分

    Python3.x:代理ip刷评分 声明:仅供为学习材料,不允许用作商业用途: 一,功能: 针对某网站对企业自动刷评分: 网站:https://best.zhaopin.com/ 二,步骤: 1,获取 ...

  9. Python3.x:代理ip刷点赞

    Python3.x:代理ip刷点赞 声明:仅供为学习材料,不允许用作商业用途: 一,功能: 针对某网站对企业自动刷点赞: 网站:https://best.zhaopin.com/ 二,步骤: 1,获取 ...

随机推荐

  1. hexo改造

    一直在思考网站分类的问题. 用hexo默认的分类,并不利于用户直观的感受到网站的内容意图,尤其是hexo首页进入后是最近发表的文章列表.然后考虑开启多个hexo服务,每个hexo服务是一个分类内容,如 ...

  2. win10 支持默认把触摸提升鼠标事件 打开 Pointer 消息

    原文:win10 支持默认把触摸提升鼠标事件 打开 Pointer 消息 在 WPF 经常需要重写一套触摸事件,没有UWP的Pointer那么好用. 如果一直都觉得 WPF 的触摸做的不好,或想解决 ...

  3. Network management system scheduling for low power and lossy networks

    In one embodiment, a network management system (NMS) determines an intent to initialize a request-re ...

  4. CUDA二维纹理内存+OpenCV图像滤波

    CUDA和OpenCV混合编程,使用CUDA的纹理内存,实现图像的二值化以及滤波功能. #include <cuda_runtime.h> #include <highgui/hig ...

  5. Linux运维完全小白入门指南

    前几天整理了一下自己入门时候搜集的资料,一边整理一边回忆. 那时候我还是个小白,用虚拟机装了个CentOS系统来玩,但是总也装不上,在论坛上求助也没人理.半天终于有个人说在某网站看过这个问题,我又找了 ...

  6. VS2010设置VC6的字体样式及背景色、选中字高亮

    习惯了VC6.0的fixedsys字体,用VS2010还真不习惯.把VS2010打造成经典的.熟悉的模样,也并非难事.网上有相应的文章,我再记录下来,主要是为了自己查找方便(刚刚重装了系统,一切从头再 ...

  7. WPF Bind 绑定

    原文:WPF Bind 绑定 版权声明:本文为博主原创文章,未经博主允许不得转载. https://blog.csdn.net/koloumi/article/details/74332515 用过W ...

  8. WPF中制作立体效果的文字或LOGO图形(续)

    原文:WPF中制作立体效果的文字或LOGO图形(续) 上篇"WPF中制作立体效果的文字或LOGO图形"(http://blog.csdn.net/johnsuna/archive/ ...

  9. TOP计划猿10最佳实践文章

    本文转自:EETproject教师专辑 http://forum.eet-cn.com/FORUM_POST_10011_1200263220_0.HTM?click_from=8800111934, ...

  10. python 教程 第二章、 类型

    第二章. 类型 常量 5,1.23,9.25e-3,’This is a string’,”It’s a string!” 1) 数 整数:2 长整数: 浮点数:3.23,52.3E-4 复数:-5+ ...