通过java.net.URLConnection发送HTTP请求(原生、爬虫)

1. 运用原生Java Api发送简单的Get请求、Post请求
2. 简单封装
3. 简单测试

　　如何通过Java发送HTTP请求，通俗点讲，如何通过Java（模拟浏览器）发送HTTP请求。Java有原生的API可用于发送HTTP请求，即java.net.URL、java.net.URLConnection，这些API很好用、很常用，但不够简便；所以，也流行有许多Java HTTP请求的framework，如，Apache的HttpClient。此文主要以ava原生的方式使用。

1. 运用原生Java Api发送简单的Get请求、Post请求

　　HTTP请求粗分为两种，一种是GET请求，一种是POST请求。（详细的请见：Hypertext Transfer Protocol -- HTTP/1.1 - Method Definitions）。

　　使用Java发送这两种请求的代码大同小异，只是一些参数设置的不同。步骤如下：

通过统一资源定位器（java.net.URL）获取连接器（java.net.URLConnection）
设置请求的参数
发送请求
以输入流的形式获取返回内容
关闭输入流

　　简单的Get请求示例如下：

import java.io.BufferedReader;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.HttpURLConnection;

import java.net.URL;

import java.net.URLConnection;

public class HttpGetRequest {

    /**

     * Main

     * @param args

     * @throws Exception

     */

    public static void main(String[] args) throws Exception {

        System.out.println(doGet());

    }

    /**

     * Get Request

     * @return

     * @throws Exception

     */

    public static String doGet() throws Exception {

        URL localURL = new URL("http://localhost:8080/OneHttpServer/");

        URLConnection connection = localURL.openConnection();

        HttpURLConnection httpURLConnection = (HttpURLConnection)connection;

        httpURLConnection.setRequestProperty("Accept-Charset", "utf-8");

        httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");

        InputStream inputStream = null;

        InputStreamReader inputStreamReader = null;

        BufferedReader reader = null;

        StringBuffer resultBuffer = new StringBuffer();

        String tempLine = null;

        if (httpURLConnection.getResponseCode() >= 300) {

            throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());

        }

        try {

            inputStream = httpURLConnection.getInputStream();

            inputStreamReader = new InputStreamReader(inputStream);

            reader = new BufferedReader(inputStreamReader);

            while ((tempLine = reader.readLine()) != null) {

                resultBuffer.append(tempLine);

            }

        } finally {

            if (reader != null) {

                reader.close();

            }

            if (inputStreamReader != null) {

                inputStreamReader.close();

            }

            if (inputStream != null) {

                inputStream.close();

            }

        }

        return resultBuffer.toString();

    }

}

　　简单的Post请求示例如下：

import java.io.BufferedReader;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.OutputStream;

import java.io.OutputStreamWriter;

import java.net.HttpURLConnection;

import java.net.URL;

import java.net.URLConnection;

public class HttpPostRequest {

    /**

     * Main

     * @param args

     * @throws Exception

     */

    public static void main(String[] args) throws Exception {

        System.out.println(doPost());

    }

    /**

     * Post Request

     * @return

     * @throws Exception

     */

    public static String doPost() throws Exception {

        String parameterData = "username=nickhuang&blog=http://www.cnblogs.com/nick-huang/";

        URL localURL = new URL("http://localhost:8080/OneHttpServer/");

        URLConnection connection = localURL.openConnection();

        HttpURLConnection httpURLConnection = (HttpURLConnection)connection;

        httpURLConnection.setDoOutput(true);

        httpURLConnection.setRequestMethod("POST");

        httpURLConnection.setRequestProperty("Accept-Charset", "utf-8");

        httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");

        httpURLConnection.setRequestProperty("Content-Length", String.valueOf(parameterData.length()));

        OutputStream outputStream = null;

        OutputStreamWriter outputStreamWriter = null;

        InputStream inputStream = null;

        InputStreamReader inputStreamReader = null;

        BufferedReader reader = null;

        StringBuffer resultBuffer = new StringBuffer();

        String tempLine = null;

        try {

            outputStream = httpURLConnection.getOutputStream();

            outputStreamWriter = new OutputStreamWriter(outputStream);

            outputStreamWriter.write(parameterData.toString());

            outputStreamWriter.flush();

            if (httpURLConnection.getResponseCode() >= 300) {

                throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());

            }

            inputStream = httpURLConnection.getInputStream();

            inputStreamReader = new InputStreamReader(inputStream);

            reader = new BufferedReader(inputStreamReader);

            while ((tempLine = reader.readLine()) != null) {

                resultBuffer.append(tempLine);

            }

        } finally {

            if (outputStreamWriter != null) {

                outputStreamWriter.close();

            }

            if (outputStream != null) {

                outputStream.close();

            }

            if (reader != null) {

                reader.close();

            }

            if (inputStreamReader != null) {

                inputStreamReader.close();

            }

            if (inputStream != null) {

                inputStream.close();

            }

        }

        return resultBuffer.toString();

    }

}

2. 简单封装

　　如果项目中有多处地方使用HTTP请求，我们适当对其进行封装，

可以大大减少代码量（不需每次都写一大段原生的请求Source code）
也可以使配置更灵活、方便（全局设置一些项目特有的配置，比如已商榷的time out时间、已确定的Proxy Server，避免以后改动繁琐）

　　以下简单封装成HttpRequestor，以便使用：

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.OutputStream;

import java.io.OutputStreamWriter;

import java.net.HttpURLConnection;

import java.net.InetSocketAddress;

import java.net.Proxy;

import java.net.URL;

import java.net.URLConnection;

import java.util.Iterator;

import java.util.Map;

public class HttpRequestor {

    private String charset = "utf-8";

    private Integer connectTimeout = null;

    private Integer socketTimeout = null;

    private String proxyHost = null;

    private Integer proxyPort = null;

    /**

     * Do GET request

     * @param url

     * @return

     * @throws Exception

     * @throws IOException

     */

    public String doGet(String url) throws Exception {

        URL localURL = new URL(url);

        URLConnection connection = openConnection(localURL);

        HttpURLConnection httpURLConnection = (HttpURLConnection)connection;

        httpURLConnection.setRequestProperty("Accept-Charset", charset);

        httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");

        InputStream inputStream = null;

        InputStreamReader inputStreamReader = null;

        BufferedReader reader = null;

        StringBuffer resultBuffer = new StringBuffer();

        String tempLine = null;

        if (httpURLConnection.getResponseCode() >= 300) {

            throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());

        }

        try {

            inputStream = httpURLConnection.getInputStream();

            inputStreamReader = new InputStreamReader(inputStream);

            reader = new BufferedReader(inputStreamReader);

            while ((tempLine = reader.readLine()) != null) {

                resultBuffer.append(tempLine);

            }

        } finally {

            if (reader != null) {

                reader.close();

            }

            if (inputStreamReader != null) {

                inputStreamReader.close();

            }

            if (inputStream != null) {

                inputStream.close();

            }

        }

        return resultBuffer.toString();

    }

    /**

     * Do POST request

     * @param url

     * @param parameterMap

     * @return

     * @throws Exception

     */

    public String doPost(String url, Map parameterMap) throws Exception {

        /* Translate parameter map to parameter date string */

        StringBuffer parameterBuffer = new StringBuffer();

        if (parameterMap != null) {

            Iterator iterator = parameterMap.keySet().iterator();

            String key = null;

            String value = null;

            while (iterator.hasNext()) {

                key = (String)iterator.next();

                if (parameterMap.get(key) != null) {

                    value = (String)parameterMap.get(key);

                } else {

                    value = "";

                }

                parameterBuffer.append(key).append("=").append(value);

                if (iterator.hasNext()) {

                    parameterBuffer.append("&");

                }

            }

        }

        System.out.println("POST parameter : " + parameterBuffer.toString());

        URL localURL = new URL(url);

        URLConnection connection = openConnection(localURL);

        HttpURLConnection httpURLConnection = (HttpURLConnection)connection;

        httpURLConnection.setDoOutput(true);

        httpURLConnection.setRequestMethod("POST");

        httpURLConnection.setRequestProperty("Accept-Charset", charset);

        httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");

        httpURLConnection.setRequestProperty("Content-Length", String.valueOf(parameterBuffer.length()));

        OutputStream outputStream = null;

        OutputStreamWriter outputStreamWriter = null;

        InputStream inputStream = null;

        InputStreamReader inputStreamReader = null;

        BufferedReader reader = null;

        StringBuffer resultBuffer = new StringBuffer();

        String tempLine = null;

        try {

            outputStream = httpURLConnection.getOutputStream();

            outputStreamWriter = new OutputStreamWriter(outputStream);

            outputStreamWriter.write(parameterBuffer.toString());

            outputStreamWriter.flush();

            if (httpURLConnection.getResponseCode() >= 300) {

                throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());

            }

            inputStream = httpURLConnection.getInputStream();

            inputStreamReader = new InputStreamReader(inputStream);

            reader = new BufferedReader(inputStreamReader);

            while ((tempLine = reader.readLine()) != null) {

                resultBuffer.append(tempLine);

            }

        } finally {

            if (outputStreamWriter != null) {

                outputStreamWriter.close();

            }

            if (outputStream != null) {

                outputStream.close();

            }

            if (reader != null) {

                reader.close();

            }

            if (inputStreamReader != null) {

                inputStreamReader.close();

            }

            if (inputStream != null) {

                inputStream.close();

            }

        }

        return resultBuffer.toString();

    }

    private URLConnection openConnection(URL localURL) throws IOException {

        URLConnection connection;

        if (proxyHost != null && proxyPort != null) {

            Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));

            connection = localURL.openConnection(proxy);

        } else {

            connection = localURL.openConnection();

        }

        return connection;

    }

    /**

     * Render request according setting

     * @param request

     */

    private void renderRequest(URLConnection connection) {

        if (connectTimeout != null) {

            connection.setConnectTimeout(connectTimeout);

        }

        if (socketTimeout != null) {

            connection.setReadTimeout(socketTimeout);

        }

    }

    /*

     * Getter & Setter

     */

    public Integer getConnectTimeout() {

        return connectTimeout;

    }

    public void setConnectTimeout(Integer connectTimeout) {

        this.connectTimeout = connectTimeout;

    }

    public Integer getSocketTimeout() {

        return socketTimeout;

    }

    public void setSocketTimeout(Integer socketTimeout) {

        this.socketTimeout = socketTimeout;

    }

    public String getProxyHost() {

        return proxyHost;

    }

    public void setProxyHost(String proxyHost) {

        this.proxyHost = proxyHost;

    }

    public Integer getProxyPort() {

        return proxyPort;

    }

    public void setProxyPort(Integer proxyPort) {

        this.proxyPort = proxyPort;

    }

    public String getCharset() {

        return charset;

    }

    public void setCharset(String charset) {

        this.charset = charset;

    }

}

　　写一个调用的测试类：

import java.util.HashMap;

import java.util.Map;

public class Call {

    public static void main(String[] args) throws Exception {

        /* Post Request */

        Map dataMap = new HashMap();

        dataMap.put("username", "Nick Huang");

        dataMap.put("blog", "IT");

        System.out.println(new HttpRequestor().doPost("http://localhost:8080/OneHttpServer/", dataMap));

        /* Get Request */

        System.out.println(new HttpRequestor().doGet("http://localhost:8080/OneHttpServer/"));

    }

}

3. 简单测试

　　以上的请求地址都是http://localhost:8080/，这是自己的一个用于测试的Web Application，就一个简单的Servlet和web.xml。毕竟需要测试请求参数是否能正常接收，处理超时的情况。

　　此处使用了注解:@WebServlet("/LoginServlet")：

import java.io.IOException;

import javax.servlet.ServletException;

import javax.servlet.annotation.WebServlet;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

/**

 * Servlet implementation class LoginServlet

 */

@WebServlet("/LoginServlet")

public class LoginServlet extends HttpServlet {

    private static final long serialVersionUID = 1L;

    /**

     * @see HttpServlet#HttpServlet()

     */

    public LoginServlet() {

        super();

        // TODO Auto-generated constructor stub

    }

    /**

     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)

     */

    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        // TODO Auto-generated method stub

        System.out.println("111");

    }

    /**

     * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)

     */

    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        // TODO Auto-generated method stub

         String username = request.getParameter("username");

            String blog = request.getParameter("blog");

            System.out.println(username);

            System.out.println(blog);

            response.setContentType("text/plain; charset=UTF-8");

            response.setCharacterEncoding("UTF-8");

//响应流

            response.getWriter().write("It is ok!");

    }

}

　　也可以在web.xml配置Servlet，web.xml：

<?xml version="1.0" encoding="UTF-8"?>

<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

    xmlns="http://java.sun.com/xml/ns/javaee" xmlns:web="http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"

    xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"

    id="WebApp_ID" version="2.5">

    <display-name>OneHttpServer</display-name>

    <welcome-file-list>

        <welcome-file>LoginServlet</welcome-file>

    </welcome-file-list>

    <servlet>

        <description></description>

        <display-name>LoginServlet</display-name>

        <servlet-name>LoginServlet</servlet-name>

        <servlet-class>LoginServlet</servlet-class>

    </servlet>

    <servlet-mapping>

        <servlet-name>LoginServlet</servlet-name>

        <url-pattern>/LoginServlet</url-pattern>

    </servlet-mapping>

</web-app>

　　测试类Test.java：

import java.util.HashMap;

import java.util.Map;

public class Test {

    public static void main(String[] args) {

        /* Post Request */

        Map dataMap = new HashMap();

        dataMap.put("username", "Nick Huang");

        dataMap.put("blog", "IT");

        try {

            System.out

                    .println(new HttpRequestor().doPost(

                            "http://localhost:8081/TestProgect2/LoginServlet",

                            dataMap));

            // System.out.println(new

            // HttpRequestor().doGet("http://localhost:8081/TestProgect2/LoginServlet"));

        } catch (Exception e) {

            // TODO Auto-generated catch block

            e.printStackTrace();

        }

        /* Get Request */

    }

}

　　运行Test.java便可在控制台得到结果。注意：请记得把LoginServlet要在加载到tomcat服务器中，这样才可以通过连接http://localhost:8080/LoginServlet访问到，否则会报连接失败异常。

　　对于接收LoginServlet.java类返回的流,在真实环境中可能是异步接收返回的流的,当一台机器A向另外一台机器B发送请求,B接收到A的信息,将信息处理后,需要将处理结果返回即响应返回给A,A会接收到B返回的信息，可以另外建一个Servlet类HttpListenerServlet.java来专门异步接收返回的信息：

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import javax.servlet.ServletException;

import javax.servlet.annotation.WebServlet;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

/**

 * Servlet implementation class HttpListenerServlet

 */

@WebServlet("/HttpListenerServlet")

public class HttpListenerServlet extends HttpServlet {

    private static final long serialVersionUID = 1L;

    /**

     * @see HttpServlet#HttpServlet()

     */

    public HttpListenerServlet() {

        super();

        // TODO Auto-generated constructor stub

    }

    /**

     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)

     */

    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        // TODO Auto-generated method stub

    }

    /**

     * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)

     */

    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        // TODO Auto-generated method stub

         InputStream inputStream = null;

         inputStream = request.getInputStream();

         InputStreamReader   inputStreamReader = new InputStreamReader(inputStream);

         BufferedReader reader = new BufferedReader(inputStreamReader);

         String tempLine = null;

         StringBuffer resultBuffer=new StringBuffer();

         while ((tempLine = reader.readLine()) != null) {

             resultBuffer.append(tempLine);

         }

         System.out.println(resultBuffer.toString());

    }

}