现在有很多网站都提供免费的代理ip,但是你会发现很多网站显示的可以用的 ,在自己电脑上是用不了,写个小代码提取出自己电脑上可以用的代理,没什么技术含量,只是为了记录一下

 string strUrl = "https://www.xicidaili.com/nt/";
HttpHelper httpProxy = new HttpHelper();
for (int i = ; i <= ; i++)
{
string strHtml = httpProxy.Get_Request(strUrl + i, httpProxy.cookie, "www.xicidaili.com", "https://www.xicidaili.com/nt/", blnHttps: true, strAccept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", strUserAgent: "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36");
HtmlAgilityPack.HtmlDocument docList = new HtmlAgilityPack.HtmlDocument();
docList.LoadHtml(strHtml);
var trListNodes = docList.DocumentNode.SelectNodes("//*[@id=\"ip_list\"]/tr");
for (int j = ; j < trListNodes.Count; j++)
{
var trNode = trListNodes[j];
string strIp = trNode.SelectSingleNode("td[2]").InnerText.Trim();
string strPort = trNode.SelectSingleNode("td[3]").InnerText.Trim();
int intPort = int.Parse(strPort);
try
{
WebProxy proxyObject = new WebProxy(strIp, intPort);// port为端口号 整数型
HttpWebRequest Req = WebRequest.Create("http://www.bookschina.com") as HttpWebRequest;
Req.Proxy = proxyObject; //设置代理
Req.Timeout = ; //超时
DateTime dt = DateTime.Now;
var Resp = (HttpWebResponse)Req.GetResponse();
Encoding bin = Encoding.GetEncoding("gb2312");
StreamReader sr = new StreamReader(Resp.GetResponseStream(), bin);
string str = sr.ReadToEnd();
sr.Close();
sr.Dispose();
var time = (DateTime.Now - dt).TotalMilliseconds;
Console.WriteLine(strIp + " " + intPort + " " + time);
}
catch
{ }
} }

HttpHelper

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks; namespace TestProxy
{
public class HttpHelper
{
public CookieContainer cookie;
public HttpHelper()
{
cookie = new CookieContainer();
}
private static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{
return true; //总是接受
}
public string Get_Request(
string strUrl,
CookieContainer _cookie = null,
string strHost = "",
string strRefer = "",
string strOrigin = "",
Dictionary<string, string> lstHeads = null,
string strEncoding = "utf-8",
string strContentType = "",
string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
bool blnAllowAutoRedirect = true,
int intTimeout = * ,
bool blnHttps = false,
System.Net.WebProxy proxy = null)
{
HttpWebRequest request;
HttpWebResponse response;
request = (HttpWebRequest)WebRequest.Create(strUrl);
request.Accept = strAccept;
request.Timeout = intTimeout;
request.Method = "GET";
request.Credentials = CredentialCache.DefaultCredentials;
request.UserAgent = strUserAgent;
request.AllowAutoRedirect = blnAllowAutoRedirect;
if (blnHttps)
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; }
if (!string.IsNullOrEmpty(strContentType))
{
request.ContentType = strContentType;
}
if (_cookie != null)
{
request.CookieContainer = _cookie;
}
if (!string.IsNullOrEmpty(strHost))
{
request.Host = strHost;
}
if (!string.IsNullOrEmpty(strRefer))
{
request.Referer = strRefer;
}
if (!string.IsNullOrEmpty(strOrigin))
{
request.Headers.Add("Origin", strOrigin);
}
if (lstHeads != null && lstHeads.Count > )
{
foreach (var item in lstHeads)
{
request.Headers.Add(item.Key, item.Value);
}
} if (proxy != null)
request.Proxy = proxy;
response = (HttpWebResponse)request.GetResponse();
var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));
string strResult = sr.ReadToEnd();
sr.Close();
request.Abort();
response.Close();
return strResult; } public string POST_Request(
string strUrl,
string postDataStr,
CookieContainer _cookie = null,
string strHost = "",
string strRefer = "",
string strOrigin = "",
Dictionary<string, string> lstHeads = null,
string strEncoding = "utf-8",
string strContentType = "",
string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
bool blnAllowAutoRedirect = true,
int intTimeout = * ,
bool blnHttps = false,
System.Net.WebProxy proxy = null)
{
HttpWebRequest request;
HttpWebResponse response;
request = (HttpWebRequest)WebRequest.Create(strUrl);
request.Accept = strAccept;
request.Timeout = intTimeout;
request.Method = "POST";
request.Host = strHost;
request.UserAgent = strUserAgent;
if (_cookie != null)
{
request.CookieContainer = _cookie;
}
if (blnHttps)
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; }
request.AllowAutoRedirect = blnAllowAutoRedirect;
if (!string.IsNullOrEmpty(strContentType))
{
request.ContentType = strContentType;
}
if (!string.IsNullOrEmpty(strOrigin))
{
request.Headers.Add("Origin", strOrigin);
}
if (!string.IsNullOrEmpty(strRefer))
{
request.Referer = strRefer;
}
if (!string.IsNullOrEmpty(strHost))
{
request.Host = strHost;
}
if (lstHeads != null && lstHeads.Count > )
{
foreach (var item in lstHeads)
{
request.Headers.Add(item.Key, item.Value);
}
}
if (!string.IsNullOrEmpty(postDataStr))
{
request.ContentLength = postDataStr.Length;
Stream myRequestStream = request.GetRequestStream();
StreamWriter myStreamWriter = new StreamWriter(myRequestStream);
myStreamWriter.Write(postDataStr);
myStreamWriter.Close();
}
if (proxy != null)
request.Proxy = proxy;
response = (HttpWebResponse)request.GetResponse();
var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));
string strResult = sr.ReadToEnd();
sr.Close();
request.Abort();
response.Close();
return strResult;
} public string DownloadFile(
string strURLAddress,
string strPath,
CookieContainer _cookie = null,
string strHost = "",
string strRefer = "",
string strOrigin = "",
Dictionary<string, string> lstHeads = null,
string strAccept = "",
string strUserAgent = "",
bool blnHttps = false,
System.Net.WebProxy proxy = null)
{
try
{
// 设置参数
HttpWebRequest request = WebRequest.Create(strURLAddress) as HttpWebRequest;
if (!string.IsNullOrEmpty(strAccept))
{
request.Accept = strAccept;
}
if (blnHttps)
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request.ProtocolVersion = HttpVersion.Version10; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; }
if (!string.IsNullOrEmpty(strUserAgent))
{
request.UserAgent = strUserAgent;
}
if (_cookie != null)
{
request.CookieContainer = _cookie;
}
if (!string.IsNullOrEmpty(strOrigin))
{
request.Headers.Add("Origin", strOrigin);
}
if (!string.IsNullOrEmpty(strRefer))
{
request.Referer = strRefer;
}
if (!string.IsNullOrEmpty(strHost))
{
request.Host = strHost;
}
if (lstHeads != null && lstHeads.Count > )
{
foreach (var item in lstHeads)
{
request.Headers.Add(item.Key, item.Value);
}
}
if (proxy != null)
request.Proxy = proxy;
request.Timeout = ;
//发送请求并获取相应回应数据
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
string strReceivePath = string.Empty; //直到request.GetResponse()程序才开始向目标网页发送Post请求
Stream responseStream = response.GetResponseStream();
//创建本地文件写入流
Stream stream = new FileStream(strPath, FileMode.Create);
byte[] bArr = new byte[];
int size = responseStream.Read(bArr, , (int)bArr.Length);
while (size > )
{
stream.Write(bArr, , size);
stream.Flush();
size = responseStream.Read(bArr, , (int)bArr.Length);
}
stream.Close();
responseStream.Close();
return strPath;
}
catch (Exception ex)
{
return "";
}
} }
}

这个需要.net 4.5

c# 查询本机可用的代理ip的更多相关文章

  1. python检验代理ip是否可用、代理ip检验

    python检验代理ip是否可用.代理ip检验 安装相关模块: pip install requests 验证代理IP是否可用脚本: import random import telnetlib im ...

  2. 利用Python爬取可用的代理IP

    前言 就以最近发现的一个免费代理IP网站为例:http://www.xicidaili.com/nn/.在使用的时候发现很多IP都用不了. 所以用Python写了个脚本,该脚本可以把能用的代理IP检测 ...

  3. 把你的Centos设置成代理ip服务器

    前言:最近在公司做爬虫相关的工作,做过数据抓取的都知道,写程序抓取数据的过程并不像平常我们用浏览器打开网页那么简单!大多数的网站为了自己站点的性能和数据安全都设置了各种反爬策略.最常见的就是添加验证码 ...

  4. 基于后端和爬虫创建的代理ip池

    搭建免费的代理ip池 需要解决的问题: 使用什么方式存储ip 文件存储 缺点: 打开文件修改文件操作较麻烦 mysql 缺点: 查询速度较慢 mongodb 缺点: 查询速度较慢. 没有查重功能 re ...

  5. 【python3】如何建立爬虫代理ip池

    一.为什么需要建立爬虫代理ip池 在众多的网站防爬措施中,有一种是根据ip的访问频率进行限制的,在某段时间内,当某个ip的访问量达到一定的阀值时,该ip会被拉黑.在一段时间内被禁止访问. 这种时候,可 ...

  6. springboot实现java代理IP池 Proxy Pool,提供可用率达到95%以上的代理IP

    一.背景 前段时间,写java爬虫来爬网易云音乐的评论.不料,爬了一段时间后ip被封禁了.由此,想到了使用ip代理,但是找了很多的ip代理网站,很少有可以用的代理ip.于是,抱着边学习的心态,自己开发 ...

  7. Python3.x:免费代理ip的批量获取并入库

    Python3.x:免费代理ip的批量获取并入库 一.简介 网络爬虫的世界,向来都是一场精彩的攻防战.现在许多网站的反爬虫机制在不断的完善,其中最令人头疼的,莫过于直接封锁你的ip.但是道高一尺魔高一 ...

  8. Python3.x:代理ip刷评分

    Python3.x:代理ip刷评分 声明:仅供为学习材料,不允许用作商业用途: 一,功能: 针对某网站对企业自动刷评分: 网站:https://best.zhaopin.com/ 二,步骤: 1,获取 ...

  9. Python3.x:代理ip刷点赞

    Python3.x:代理ip刷点赞 声明:仅供为学习材料,不允许用作商业用途: 一,功能: 针对某网站对企业自动刷点赞: 网站:https://best.zhaopin.com/ 二,步骤: 1,获取 ...

随机推荐

  1. A GUIDE TO UNDERSTANDINGDISCRETIONARY ACCESS CONTROL INTRUSTED SYSTEMS

    1. INTRODUCTION   The main goal of the National Computer Security Center is to encourage the widespr ...

  2. Cocos2d-x 脚本语言Lua基本语法

    Cocos2d-x 脚本语言Lua基本语法 前面一篇博客对Lua这门小巧的语言进行了简单的介绍.本篇博客来给大家略微讲一下Lua的语法.不会长篇累牍得把Lua的全部语法都讲一遍,这里通过下面几点来讲L ...

  3. java.lang.ExceptionInInitializerError异常分析

    今天在项目开发时遇到一个问题,整个项目是使用Spring等框架搭建起来的在运行项目时不报任何的异常信息,就是找不到某个类信息,各方查找该类确实是存在的,最后通过断点跟踪时在异常栈内发现java.lan ...

  4. docker部署netcore应用(二)

    基于第一章已经安装好了docker,这次将把netcore应用部署到docker容器中 开发工具vs2017,准备个DotNet Core的Console应用程序,测试一下 发布DockerTest项 ...

  5. 【27.48%】【codeforces 699D】 Fix a Tree

    time limit per test2 seconds memory limit per test256 megabytes inputstandard input outputstandard o ...

  6. delphi json uLkJSON

    delphi 7 json 做个笔记,留着以后用 --源码 unit Umain; interface uses Windows, Messages, SysUtils, Variants, Clas ...

  7. 在.net MVC项目中使用ajax进行数据验证

    1.首先要在网页引入应该引入的js文件 在这里回顾一下在模板页里面挖坑的技术 2.在html中使用html辅助方法 3.验证模型 4验证方法

  8. Windows搭建Eclipse+JDK+SDK的Android --安卓开发入门级

     一 相关下载 (1) java JDK下载: 进入该网页: http://java.sun.com/javase/downloads/index.jsp (或者直接点击下载)例如以下图: 选择 ...

  9. matlab 矢量化编程(四)—— 标量函数转化为能够处理矢量的函数

    1. 组合的矢量实现 nchoosek(n, k) 的第二个参数在 matlab 下是不支持矢量化的,必须是标量形式.但 matlab 下的 gamma 函数,却可支持,矢量形式,又因为,gamma ...

  10. python中的验证码识别库PyTesser

    PyTesser PyTesser is an Optical Character Recognition module for Python. It takes as input an image ...