java实现抓取某公司官网新闻

　　做项目时，并没有合作公司的获取新闻的接口，但是项目又急着上线，所以总监就让我来做一个简单的抓取，现将主要的工具类NewsUtil.java贴出来供大家参考。

NewsUtil.java

 package org.news.util;

 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.ArrayList;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 /**
  * 抓取新闻内容的辅助类
  * @author geenkDC
  * @time 2015-07-28 15:15:04
  */
 public class NewsUtil {
     /**
      * 通过提交的URL来抓取出新闻的链接
      * @param url
      * @return
      * @throws Exception
      */
     public static ArrayList<String> findUrlByUrl(String url) throws Exception
     {
         URL url0=new URL(url);
         ArrayList<String> urlList=new ArrayList<String>();
         URLConnection con;
         BufferedReader br=null;
         try {
             con = url0.openConnection();
             InputStream in=con.getInputStream();
             br=new BufferedReader(new InputStreamReader(in));
             String str="";
             while((str=br.readLine())!=null)
             {
                 urlList.addAll(findUrl(str));
             }
         } catch (IOException e) {
             throw new RuntimeException("URL读写错误:"+e.getMessage());
         }
         if(br!=null)
         {
             try {
                 br.close();
             } catch (IOException e) {
                 throw new RuntimeException("URL流关闭异常:"+e.getMessage());
             }
         }
         return urlList;
     }

     /**抓取新闻URL的真正实现类
      * @param str
      * @return
      */
     public static ArrayList<String> findUrl(String str)
     {
         ArrayList<String> urlList=new ArrayList<String>();
         //匹配新闻的URL
         String regex="http://[a-zA-Z0-9_\\.:\\d/?=&%]+\\.jhtml";
         Pattern p=Pattern.compile(regex);
         Matcher m=p.matcher(str);
         //找符合正则匹配的字串
         while(m.find())
         {
             String subStr=m.group().substring(m.group().lastIndexOf("/")+1, m.group().lastIndexOf(".jhtml"));

             try {
                 if (subStr.matches("[0-9]*")) {
                     urlList.add(m.group());

                 }
             } catch (Exception e) {
                 throw new RuntimeException("匹配新闻URL出错:"+e.getMessage());
             }
         }
         return urlList;
     }

     /**
      * 根据URL找到其的新闻内容
      * @param url
      * @return
      * @throws Exception
      */
     public static ArrayList<String> findContentByUrl(String url) throws Exception {
         URL url1=new URL(url);
         ArrayList<String> conList=new ArrayList<String>();
         URLConnection con;
         BufferedReader br=null;
         try {
             con = url1.openConnection();
             InputStream in=con.getInputStream();
             InputStreamReader isr=new InputStreamReader(in, "utf-8");
             br=new BufferedReader(isr);
             String str="";
             StringBuffer sb=new StringBuffer();
             while((str=br.readLine())!=null)
             {
                 sb.append(str);
             }
             conList.addAll(findContent(sb.toString()));
         } catch (IOException e) {
             throw new RuntimeException("URL读写错误:"+e.getMessage());
         }
         if(br!=null)
         {
             try {
                 br.close();
             } catch (IOException e) {
                 throw new RuntimeException("URL流关闭异常:"+e.getMessage());
             }
         }
         return conList;
     }

     /**
      * 抓取新闻内容的真正实现类
      * @param str
      * @return
      */
     public static ArrayList<String> findContent(String str) {
         ArrayList<String> strList=new ArrayList<String>();
         //匹配新闻内容div
         String regex="<div class=\"con_box\">([\\s\\S]*)</div>([\\s\\S]*)<div class=\"left_con\">";
         Pattern p=Pattern.compile(regex);
         Matcher m=p.matcher(str);
         //找符合正则匹配的字串
         while(m.find())
         {
             try {
                 strList.add(new String(m.group()));
             } catch (Exception e) {
                 throw new RuntimeException("抓取新闻内容出错:"+e.getMessage());
             }
         }
         return strList;
     }
 }

功能简单说明：

　　只要输入网站首页的url，程序会自动获取匹配的新闻条目的url,再根据每个新闻条目的url抓取该新闻的左右内容。