HttpUrlConnection底层实现和关于java host绑定ip即时生效的设置及分析

最近有个需求需要对于获取URL页面进行host绑定并且立即生效，在java里面实现可以用代理服务器来实现：因为在测试环境下可能需要通过绑定来访问测试环境的应用
实现代码如下：

public static String getResponseText(String queryUrl,String host,String ip) { //queryUrl，完整的url，host和ip需要绑定的host和ip
InputStream is = null;
BufferedReader br = null;
StringBuffer res = new StringBuffer();
try {
HttpURLConnection httpUrlConn = null;
URL url = new URL(queryUrl);
if(ip!=null){
String str[] = ip.split("\\.");
byte[] b =new byte[str.length];
for(int i=0,len=str.length;i<len;i++){
b[i] = (byte)(Integer.parseInt(str[i],10));
}
Proxy proxy = new Proxy(Proxy.Type.HTTP,
new InetSocketAddress(InetAddress.getByAddress(b), 80)); //b是绑定的ip，生成proxy代理对象，因为http底层是socket实现，
httpUrlConn = (HttpURLConnection) url
.openConnection(proxy);
}else{
httpUrlConn = (HttpURLConnection) url
.openConnection();
}
httpUrlConn.setRequestMethod("GET");
httpUrlConn.setDoOutput(true);
httpUrlConn.setConnectTimeout(2000);
httpUrlConn.setReadTimeout(2000);
httpUrlConn.setDefaultUseCaches(false);
httpUrlConn.setUseCaches(false);

is = httpUrlConn.getInputStream();

那么底层对于proxy对象到底是怎么处理，底层的socket实现到底怎么样，带着这个疑惑看了下jdk的rt.jar对于这块的处理
httpUrlConn = (HttpURLConnection) url.openConnection(proxy)

java.net.URL类里面的openConnection方法：
public URLConnection openConnection(Proxy proxy){
…
return handler.openConnection(this, proxy); Handler是sun.net.www.protocol.http.Handler.java类，继承java.net. URLStreamHandler.java类，用来处理http连接请求响应的。
}

Handler的方法：
protected java.net.URLConnection openConnection(URL u, Proxy p)
throws IOException {
return new HttpURLConnection(u, p, this);
}

只是简单的生成sun.net.www.protocl.http.HttpURLConnection对象，并进行初始化
protected HttpURLConnection(URL u, Proxy p, Handler handler) {
super(u);
requests = new MessageHeader(); 请求头信息生成类
responses = new MessageHeader(); 响应头信息解析类
this.handler = handler;
instProxy = p; 代理服务器对象
cookieHandler = (CookieHandler)java.security.AccessController.doPrivileged(
new java.security.PrivilegedAction() {
public Object run() {
return CookieHandler.getDefault();
}
});
cacheHandler = (ResponseCache)java.security.AccessController.doPrivileged(
new java.security.PrivilegedAction() {
public Object run() {
return ResponseCache.getDefault();
}
});
}

最终在httpUrlConn.getInputStream();才进行socket连接，发送http请求，解析http响应信息。具体过程如下：

sun.net.www.protocl.http.HttpURLConnection.java的getInputStream方法：

public synchronized InputStream getInputStream() throws IOException {

...socket连接
connect();
...
ps = (PrintStream)http.getOutputStream(); 获得输出流，打开连接之后已经生成。

if (!streaming()) {
writeRequests(); 输出http请求头信息
}
...
http.parseHTTP(responses, pi, this); 解析响应信息
if(logger.isLoggable(Level.FINEST)) {
logger.fine(responses.toString());
}
inputStream = http.getInputStream(); 获得输入流
}

其中connect()调用方法链：
plainConnect(){
...
Proxy p = null;
if (sel != null) {
URI uri = sun.net.www.ParseUtil.toURI(url);
Iterator<Proxy> it = sel.select(uri).iterator();
while (it.hasNext()) {
p = it.next();
try {
if (!failedOnce) {
http = getNewHttpClient(url, p, connectTimeout);
...
}

getNewHttpClient(){
...
return HttpClient.New(url, p, connectTimeout, useCache);
...
}

下面跟进去最终建立socket连接的代码：
sun.net.www.http.HttpClient.java的openServer()方法建立socket连接：

protected synchronized void openServer() throws IOException {
...
if ((proxy != null) && (proxy.type() == Proxy.Type.HTTP)) {
sun.net.www.URLConnection.setProxiedHost(host);
if (security != null) {
security.checkConnect(host, port);
}
privilegedOpenServer((InetSocketAddress) proxy.address());最终socket连接的是设置的代理服务器的地址，
...
}

private synchronized void privilegedOpenServer(final InetSocketAddress server)
throws IOException
{
try {
java.security.AccessController.doPrivileged(
new java.security.PrivilegedExceptionAction() {
public Object run() throws IOException {
openServer(server.getHostName(), server.getPort()); 注意openserver函数 这里的server的getHostName是设置的代理服务器，（ip或者hostname，如果是host绑定设置的代理服务器的ip，那么这里getHostName出来的就是ip地址，可以去查看InetSocketAddress类的getHostName方法）
return null;
}
});
} catch (java.security.PrivilegedActionException pae) {
throw (IOException) pae.getException();
}
}

public void openServer(String server, int port) throws IOException {
serverSocket = doConnect(server, port); 生成的Socket连接对象
try {
serverOutput = new PrintStream(
new BufferedOutputStream(serverSocket.getOutputStream()),
false, encoding); 生成输出流，
} catch (UnsupportedEncodingException e) {
throw new InternalError(encoding+" encoding not found");
}
serverSocket.setTcpNoDelay(true);
}

protected Socket doConnect (String server, int port)
throws IOException, UnknownHostException {
Socket s;
if (proxy != null) {
if (proxy.type() == Proxy.Type.SOCKS) {
s = (Socket) AccessController.doPrivileged(
new PrivilegedAction() {
public Object run() {
return new Socket(proxy);
}});
} else
s = new Socket(Proxy.NO_PROXY);
} else
s = new Socket();
// Instance specific timeouts do have priority, that means
// connectTimeout & readTimeout (-1 means not set)
// Then global default timeouts
// Then no timeout.
if (connectTimeout >= 0) {
s.connect(new InetSocketAddress(server, port), connectTimeout);
} else {
if (defaultConnectTimeout > 0) {
s.connect(new InetSocketAddress(server, port), defaultConnectTimeout);//连接到代理服务器，看下面Socket类的connect方法代码
} else {
s.connect(new InetSocketAddress(server, port));
}
}
if (readTimeout >= 0)
s.setSoTimeout(readTimeout);
else if (defaultSoTimeout > 0) {
s.setSoTimeout(defaultSoTimeout);
}
return s;
}

上面的new InetSocketAddress(server, port)这里会涉及到java DNS cache的处理，

public InetSocketAddress(String hostname, int port) {
if (port < 0 || port > 0xFFFF) {
throw new IllegalArgumentException("port out of range:" + port);
}
if (hostname == null) {
throw new IllegalArgumentException("hostname can't be null");
}
try {
addr = InetAddress.getByName(hostname); //这里会有java DNS缓存的处理，先从缓存取hostname绑定的ip地址，如果取不到再通过OS的DNS cache机制去取，取不到再从DNS服务器上取。
} catch(UnknownHostException e) {
this.hostname = hostname;
addr = null;
}
this.port = port;
}

当然最终的Socket.java的connect方法
java.net.socket

public void connect(SocketAddress endpoint, int timeout) throws IOException {
if (endpoint == null)

if (timeout < 0)
throw new IllegalArgumentException("connect: timeout can't be negative");

if (isClosed())
throw new SocketException("Socket is closed");

if (!oldImpl && isConnected())
throw new SocketException("already connected");

if (!(endpoint instanceof InetSocketAddress))
throw new IllegalArgumentException("Unsupported address type");

InetSocketAddress epoint = (InetSocketAddress) endpoint;

SecurityManager security = System.getSecurityManager();
if (security != null) {
if (epoint.isUnresolved())
security.checkConnect(epoint.getHostName(),
epoint.getPort());
else
security.checkConnect(epoint.getAddress().getHostAddress(),
epoint.getPort());
}
if (!created)
createImpl(true);
if (!oldImpl)
impl.connect(epoint, timeout);
else if (timeout == 0) {
if (epoint.isUnresolved()) //如果没有设置SocketAddress的ip地址，则用域名去访问
impl.connect(epoint.getAddress().getHostName(),
epoint.getPort());
else
impl.connect(epoint.getAddress(), epoint.getPort()); 最终socket连接的是设置的SocketAddress的ip地址，
} else
throw new UnsupportedOperationException("SocketImpl.connect(addr, timeout)");
connected = true;
/*
* If the socket was not bound before the connect, it is now because
* the kernel will have picked an ephemeral port & a local address
*/
bound = true;
}

我们再看下通过socket来发送HTTP请求的处理代码，也就是sun.net.www.protocl.http.HttpURLConnection.java的getInputStream方法中调用的writeRequests()方法：
private void writeRequests() throws IOException { 这段代码就是封装http请求的头请求信息，通过socket发送出去
/* print all message headers in the MessageHeader
* onto the wire - all the ones we've set and any
* others that have been set
*/
// send any pre-emptive authentication
if (http.usingProxy) {
setPreemptiveProxyAuthentication(requests);
}
if (!setRequests) {

/* We're very particular about the order in which we
* set the request headers here. The order should not
* matter, but some careless CGI programs have been
* written to expect a very particular order of the
* standard headers. To name names, the order in which
* Navigator3.0 sends them. In particular, we make *sure*
* to send Content-type: <> and Content-length:<> second
* to last and last, respectively, in the case of a POST
* request.
*/
if (!failedOnce)
requests.prepend(method + " " + http.getURLFile()+" " +
httpVersion, null);
if (!getUseCaches()) {
requests.setIfNotSet ("Cache-Control", "no-cache");
requests.setIfNotSet ("Pragma", "no-cache");
}
requests.setIfNotSet("User-Agent", userAgent);
int port = url.getPort();
String host = url.getHost();
if (port != -1 && port != url.getDefaultPort()) {
host += ":" + String.valueOf(port);
}
requests.setIfNotSet("Host", host);
requests.setIfNotSet("Accept", acceptString);

/*
* For HTTP/1.1 the default behavior is to keep connections alive.
* However, we may be talking to a 1.0 server so we should set
* keep-alive just in case, except if we have encountered an error
* or if keep alive is disabled via a system property
*/

// Try keep-alive only on first attempt
if (!failedOnce && http.getHttpKeepAliveSet()) {
if (http.usingProxy) {
requests.setIfNotSet("Proxy-Connection", "keep-alive");
} else {
requests.setIfNotSet("Connection", "keep-alive");
}
} else {
/*
* RFC 2616 HTTP/1.1 section 14.10 says:
* HTTP/1.1 applications that do not support persistent
* connections MUST include the "close" connection option
* in every message
*/
requests.setIfNotSet("Connection", "close");
}
// Set modified since if necessary
long modTime = getIfModifiedSince();
if (modTime != 0 ) {
Date date = new Date(modTime);
//use the preferred date format according to RFC 2068(HTTP1.1),
// RFC 822 and RFC 1123
SimpleDateFormat fo =
new SimpleDateFormat ("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US);
fo.setTimeZone(TimeZone.getTimeZone("GMT"));
requests.setIfNotSet("If-Modified-Since", fo.format(date));
}
// check for preemptive authorization
AuthenticationInfo sauth = AuthenticationInfo.getServerAuth(url);
if (sauth != null && sauth.supportsPreemptiveAuthorization() ) {
// Sets "Authorization"
requests.setIfNotSet(sauth.getHeaderName(), sauth.getHeaderValue(url,method));
currentServerCredentials = sauth;
}

if (!method.equals("PUT") && (poster != null || streaming())) {
requests.setIfNotSet ("Content-type",
"application/x-www-form-urlencoded");
}

if (streaming()) {
if (chunkLength != -1) {
requests.set ("Transfer-Encoding", "chunked");
} else {
requests.set ("Content-Length", String.valueOf(fixedContentLength));
}
} else if (poster != null) {
/* add Content-Length & POST/PUT data */
synchronized (poster) {
/* close it, so no more data can be added */
poster.close();
requests.set("Content-Length",
String.valueOf(poster.size()));
}
}

// get applicable cookies based on the uri and request headers
// add them to the existing request headers
setCookieHeader();
…
}

再来看看把socket响应信息解析为http的响应信息的代码：
sun.net.www.http.HttpClient.java的parseHTTP方法：
private boolean parseHTTPHeader(MessageHeader responses, ProgressSource pi, HttpURLConnection httpuc)
throws IOException {
/* If "HTTP/*" is found in the beginning, return true. Let
* HttpURLConnection parse the mime header itself.
*
* If this isn't valid HTTP, then we don't try to parse a header
* out of the beginning of the response into the responses,
* and instead just queue up the output stream to it's very beginning.
* This seems most reasonable, and is what the NN browser does.
*/

keepAliveConnections = -1;
keepAliveTimeout = 0;

boolean ret = false;
byte[] b = new byte[8];

try {
int nread = 0;
serverInput.mark(10);
while (nread < 8) {
int r = serverInput.read(b, nread, 8 - nread);
if (r < 0) {
break;
}
nread += r;
}
String keep=null;
ret = b[0] == 'H' && b[1] == 'T'
&& b[2] == 'T' && b[3] == 'P' && b[4] == '/' &&
b[5] == '1' && b[6] == '.';
serverInput.reset();
if (ret) { // is valid HTTP - response started w/ "HTTP/1."
responses.parseHeader(serverInput);

// we've finished parsing http headers
// check if there are any applicable cookies to set (in cache)
if (cookieHandler != null) {
URI uri = ParseUtil.toURI(url);
// NOTE: That cast from Map shouldn't be necessary but
// a bug in javac is triggered under certain circumstances
// So we do put the cast in as a workaround until
// it is resolved.
if (uri != null)
cookieHandler.put(uri, (Map<java.lang.String,java.util.List<java.lang.String>>)responses.getHeaders());
}

/* decide if we're keeping alive:
* This is a bit tricky. There's a spec, but most current
* servers (10/1/96) that support this differ in dialects.
* If the server/client misunderstand each other, the
* protocol should fall back onto HTTP/1.0, no keep-alive.
*/
if (usingProxy) { // not likely a proxy will return this
keep = responses.findValue("Proxy-Connection");
}
if (keep == null) {
keep = responses.findValue("Connection");
}
if (keep != null && keep.toLowerCase().equals("keep-alive")) {
/* some servers, notably Apache1.1, send something like:
* "Keep-Alive: timeout=15, max=1" which we should respect.
*/
HeaderParser p = new HeaderParser(
responses.findValue("Keep-Alive"));
if (p != null) {
/* default should be larger in case of proxy */
keepAliveConnections = p.findInt("max", usingProxy?50:5);
keepAliveTimeout = p.findInt("timeout", usingProxy?60:5);
}
} else if (b[7] != '0') {
/*
* We're talking 1.1 or later. Keep persistent until
* the server says to close.
*/
if (keep != null) {
/*
* The only Connection token we understand is close.
* Paranoia: if there is any Connection header then
* treat as non-persistent.
*/
keepAliveConnections = 1;
} else {
keepAliveConnections = 5;
}
}
……
}

对于java.net包的http，ftp等各种协议的底层实现，可以参考rt.jar下面的几个包的代码：
sun.net.www.protocl下的几个包。

在http client中也可以设置代理：
HostConfiguration conf = new HostConfiguration();
conf.setHost(host);
conf.setProxy(ip, 80);
statusCode = httpclient.executeMethod(conf,getMethod);

httpclient自己也是基于socket封装的http处理的库。底层代理的实现是一样的。

另外一种不设置代理，通过反射修改InetAddress的cache也是ok的。但是这种方法非常不推荐，不要使用，因为对于proxy代理服务器概念了解不清楚，最开始还使用这种方法，
public static void jdkDnsNoCache(final String host, final String ip)
throws SecurityException, NoSuchFieldException,
IllegalArgumentException, IllegalAccessException {
if (StringUtils.isBlank(host)) {
return;
}
final Class clazz = java.net.InetAddress.class;
final Field cacheField = clazz.getDeclaredField("addressCache");
cacheField.setAccessible(true);
final Object o = cacheField.get(clazz);
Class clazz2 = o.getClass();
final Field cacheMapField = clazz2.getDeclaredField("cache");
cacheMapField.setAccessible(true);
final Map cacheMap = (Map) cacheMapField.get(o);
AccessController.doPrivileged(new PrivilegedAction() {
public Object run() {
try {
synchronized (o) {// 同步是必须的,因为o可能会有多个线程同时访问修改。
// cacheMap.clear();//这步比较关键，用于清除原来的缓存
// cacheMap.remove(host);
if (!StringUtils.isBlank(ip)) {
InetAddress inet = InetAddress.getByAddress(host,IPUtil.int2byte(ip));
InetAddress addressstart = InetAddress.getByName(host);
Object cacheEntry = cacheMap.get(host);
cacheMap.put(host,newCacheEntry(inet,cacheEntry));
// cacheMap.put(host,newCacheEntry(newInetAddress(host, ip)));
}else{
cacheMap.remove(host);
}
// System.out.println(getStaticProperty(
// "java.net.InetAddress", "addressCacheInit"));
// System.out.println(invokeStaticMethod("java.net.InetAddress","getCachedAddress",new
// Object[]{host}));
}
} catch (Throwable te) {
throw new RuntimeException(te);
}
return null;
}
});
final Map cacheMapafter = (Map) cacheMapField.get(o);
System.out.println(cacheMapafter);

}

关于java中对于DNS的缓存设置可以参考：
1.在${java_home}/jre/lib/secuiry/java.secuiry文件，修改下面为
networkaddress.cache.negative.ttl=0 DNS解析不成功的缓存时间
networkaddress.cache.ttl=0 DNS解析成功的缓存的时间
2.jvm启动时增加下面两个启动环境变量
-Dsun.net.inetaddr.ttl=0
-Dsun.net.inetaddr.negative.ttl=0

如果在java程序中使用，可以这么设置设置：
java.security.Security.setProperty("networkaddress.cache.ttl" , "0");
java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0");

还有几篇文档链接可以查看：
http://www.rgagnon.com/javadetails/java-0445.html
http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6247501

linux下关于ＯＳ　ＤＮＳ设置的几个文件是
/etc/resolve.conf
/etc/nscd.conf
/etc/nsswitch.conf

http://www.linuxfly.org/post/543/
http://linux.die.net/man/5/nscd.conf
http://www.linuxhomenetworking.com/wiki/index.php/Quick_HOWTO_:_Ch18_:_Configuring_DNS
http://linux.die.net/man/5/nscd.conf