旧瓶新酒-获取网络资源即爬取下载页面内容（图片、html、css、js等）

这个java获取网络资源以前也写过不少

最近用到又重新写了一个，apache.commons.io中的例子就非常好，但是无法对请求进行详细设置

于是大部分照搬，局部替换以设置请求头

如需更加复杂的设置，可以考虑使用同为apche的httpComponents

---

**
```java
package boot.example;

import org.apache.commons.io.FileUtils;

import org.apache.commons.io.IOUtils;

import java.io.*;

import java.net.HttpURLConnection;

import java.net.URL;

/**

• Created by wq on 2017/6/6.
  
  */
  
  public class Download {
  
  public static void main(String[] args) {
  
  Download download = new Download();
  
  String url = "http://img1.3lian.com/2015/w7/85/d/21.jpg";
  
  String path1 = "E:\1.jpg";
  
  String path2 = "E:\2.jpg";
  
  String path3 = "E:\3.jpg";
  
  String url2="http://www.baidu.com";
  
  try {
  
  download.apacheCommonsIoDownload(url, path1);
  
  } catch (Exception e) {
  
  e.printStackTrace();
  
  }
  
  try {
  
  download.pureJavaNetDownload(url, path2);
  
  } catch (Exception e) {
  
  e.printStackTrace();
  
  }
  
  try {
  
  download.mixedDownload(url, path3);
  
  } catch (Exception e) {
  
  e.printStackTrace();
  
  }
  
  try{
  
  download.getContentAsString(url2);
  
  }catch (Exception e){
  
  e.printStackTrace();
  
  }
  
  }

  private void apacheCommonsIoDownload(String urlstr, String path) throws Exception {
  
  apacheCommonsIoDownload(urlstr, new File(path));
  
  }

  private void apacheCommonsIoDownload(String urlstr, File file) throws Exception {
  
  FileUtils.copyURLToFile(new URL(urlstr), file);
  
  }

  private void pureJavaNetDownload(String urlstr, String path) throws Exception {
  
  pureJavaNetDownload(urlstr, new File(path));
  
  }

  //无需依赖
  
  private void pureJavaNetDownload(String urlstr, File file) throws Exception {
  
  URL url = new URL(urlstr);
  
  HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
  
  httpURLConnection.setRequestMethod("GET");
  
  //有的网站屏蔽程序抓取 添加User-Agent头信息以避免403
  
  httpURLConnection.setRequestProperty("User-Agent", "Mozilla/4.0");
  
  httpURLConnection.setConnectTimeout(10000);
  
  httpURLConnection.setReadTimeout(10000);
  
  // httpURLConnection.set...更多请求设置
  
  httpURLConnection.connect();
  
  InputStream is = httpURLConnection.getInputStream();
  
  // 不需要设置可以直接下面 也就是org.apache.commons.io.FileUtils中copyURLToFile(URL source, File destination)的写法
  
  // InputStream is=url.openStream();
  
  try {
  
  FileOutputStream fos = new FileOutputStream(file);
  
  try {
  
  // 照搬org.apache.commons.io.IOUtils
  
  // IOUtils.copy(InputStream input, OutputStream output) 开始
  
  byte[] buffer = new byte[1024 * 4];
  
  int n;
  
  while (-1 != (n = is.read(buffer))) {
  
  fos.write(buffer, 0, n);
  
  }
  
  // IOUtils.copy(InputStream input, OutputStream output) 结束
  
  } finally {
  
  try {
  
  if (is != null) {
  
  fos.close();
  
  }
  
  } catch (IOException ioe) {
  
  // ignore
  
  }
  
  }
  
  } finally {
  
  try {
  
  if (is != null) {
  
  is.close();
  
  }
  
  } catch (IOException ioe) {
  
  // ignore
  
  }
  
  }
  
  }

  private void mixedDownload(String urlstr, String path) throws Exception {
  
  mixedDownload(urlstr, new File(path));
  
  }

  //使用IOUtils减少代码量 弃用FileUtils以对请求进行详细设置 推荐
  
  private void mixedDownload(String urlstr, File file) throws Exception {
  
  URL url = new URL(urlstr);
  
  HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
  
  httpURLConnection.setRequestMethod("GET");
  
  httpURLConnection.setRequestProperty("User-Agent", "Mozilla/4.0");
  
  httpURLConnection.setConnectTimeout(10000);
  
  httpURLConnection.setReadTimeout(10000);
  
  httpURLConnection.connect();
  
  InputStream is = httpURLConnection.getInputStream();
  
  try {
  
  FileOutputStream output = FileUtils.openOutputStream(file);
  
  try {
  
  IOUtils.copy(is, output);
  
  } finally {
  
  IOUtils.closeQuietly(output);
  
  }
  
  } finally {
  
  IOUtils.closeQuietly(is);
  
  }
  
  }

  private void getContentAsString(String urlstr) throws Exception {
  
  URL url = new URL(urlstr);
  
  InputStream is=url.openStream();
  
  ByteArrayOutputStream bos=new ByteArrayOutputStream();
  
  IOUtils.copy(is, bos);
  
  System.out.println(bos.toString());
  
  }
  
  }