Java 抓取网页中的内容【持续更新】

背景：前几天复习Java的时候看到URL类，当时就想写个小程序试试，迫于考试没有动手，今天写了下，感觉还不错

内容1. 抓取网页中的URL

知识点：Java URL+ 正则表达式

 import java.io.BufferedReader;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 public class URLReader {
     public static void main(String[] args) throws Exception {
         System.out.println("开始！");
         Pattern pattern = Pattern.compile("http(s)?://([\\w-]+\\.)+[\\w-]+(/[\\w- ./?%&=]*)?");
         URL baidu = new URL("http://www.cnblogs.com/A--Q/");
         BufferedReader br = new BufferedReader(new InputStreamReader(baidu.openStream(), "utf-8"));
         String inputLine;
         while ((inputLine = br.readLine()) != null) {
             Matcher matcher = pattern.matcher(inputLine);
             while (matcher.find()) {
                 System.out.println(matcher.group(0));
             }
         }
         br.close();
         System.out.println("程序执行结束！");
     }
 }

效果：

内容2. 抓取网页中的图片

 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 public class CatchImage {

     private static final String URL = "http://www.cnblogs.com/A--Q/p/5170713.html";
     private static final String ECODING = "UTF-8";
     private static final String IMGURL_REG = "<img.*src=(.*?)[^>]*?>";
     private static final String IMGSRC_REG = "http:\"?(.*?)(\"|>|\\s+)";

     public static void main(String[] args) throws Exception {
         System.out.println("start");
         CatchImage cm = new CatchImage();
         String HTML = cm.getHTML(URL);
         List<String> imgUrl = cm.getImageUrl(HTML);
         List<String> imgSrc = cm.getImageSrc(imgUrl);
         cm.Download(imgSrc);
         System.out.println("END");
     }

     private String getHTML(String url) throws Exception {
         URL uri = new URL(url);
         URLConnection connection = uri.openConnection();
         InputStream in = connection.getInputStream();
         byte[] buf = new byte[1024];
         int length = 0;
         StringBuffer sb = new StringBuffer();
         while ((length = in.read(buf, 0, buf.length)) > 0) {
             sb.append(new String(buf, ECODING));
         }
         in.close();
         return sb.toString();
     }

     private List<String> getImageUrl(String HTML) {
         Matcher matcher = Pattern.compile(IMGURL_REG).matcher(HTML);
         List<String> listImgUrl = new ArrayList<String>();
         while (matcher.find()) {
             listImgUrl.add(matcher.group());
         }
         return listImgUrl;
     }

     private List<String> getImageSrc(List<String> listImageUrl) {
         List<String> listImgSrc = new ArrayList<String>();
         for (String image : listImageUrl) {
             Matcher matcher = Pattern.compile(IMGSRC_REG).matcher(image);
             while (matcher.find()) {
                 listImgSrc.add(matcher.group().substring(0, matcher.group().length() - 1));
             }
         }
         return listImgSrc;
     }

     private void Download(List<String> listImgSrc) {
         try {
             for (String url : listImgSrc) {
                 String imageName = url.substring(url.lastIndexOf("/") + 1, url.length());
                 URL uri = new URL(url);
                 InputStream in = uri.openStream();
                 FileOutputStream fo = new FileOutputStream(new File(imageName));
                 byte[] buf = new byte[1024];
                 int length = 0;
                 System.out.println("开始下载:" + url);
                 while ((length = in.read(buf, 0, buf.length)) != -1) {
                     fo.write(buf, 0, length);
                 }
                 in.close();
                 fo.close();
                 System.out.println(imageName + "下载完成");
             }
         } catch (Exception e) {
             System.out.println("下载失败");
         }
     }

 }