java正则 读取html 获取标题/超链接/链接文本/内容

java正则 读取html 获取标题/超链接/链接文本/内容

参考链接：http://yijianfengvip.blog.163.com/blog/static/17527343220114278593064/

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 
 * @author 
 * 
 */
public class WebContent
{
 /**
  * 读取一个网页全部内容
  */

///此方法 读取小量数据网站 速度较快

  public static String getOneHtml(String urlString)throws Exception{
   InputStreamReader in = new InputStreamReader(new URL(urlString).openStream());
         // read contents into string buffer
         StringBuilder input = new StringBuilder();
         int ch;
         while ((ch = in.read()) != -1) input.append((char) ch);
         //System.out.println(input);
   return input.toString();
  }

///此方法 读取大量数据网站 速度较快 但是不能太大。 例如sina.com.cn  本方法  运行十次耗时  7688毫秒  getOneHtml 耗时 41016毫秒

 public static String getAllHtml(String urlString)throws Exception{
 //  ConnectionManager manager = Parser.getConnectionManager();
 //  Cookie cookie = new Cookie("ASP.NET_SessionId","szpgjr45ucsacir0d1p4g255");
 //  Cookie cookie1 = new Cookie("SID", "111017");
 //  Cookie cookie2 = new Cookie("LID", "");
 //  manager.setCookie(cookie, "acad.cnki.net");
 //  manager.setCookie(cookie1, "acad.cnki.net");
 //  manager.setCookie(cookie2, "acad.cnki.net");
   //Parser parser = new Parser("");
   Parser parser = new Parser(urlString);
   parser.setEncoding("utf8");
   String filterStr = "html";
   NodeFilter innerFilter = new TagNameFilter(filterStr);
   NodeList nodes = parser.parse(innerFilter);
   return nodes.toHtml();
   //System.out.println(nodes.toHtml());
  }

  /**
   *
   * @param s
   * @return 获得网页标题
   */
  public String getTitle(final String s)
  {
   String regex;
   String title = "";
   final List<String> list = new ArrayList<String>();
   regex = "<title>.*?</title>";
   final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
   final Matcher ma = pa.matcher(s);
   while (ma.find())
   {
    list.add(ma.group());
   }
   for (int i = 0; i < list.size(); i++)
   {
    title = title + list.get(i);
   }
   return outTag(title);
  }

  /**
   *
   * @param s
   * @return 获得链接
   */
  public List<String> getLink(final String s)
  {
   String regex;
   final List<String> list = new ArrayList<String>();
   regex = "<a[^>]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))[^>]*>(.*?)</a>";
   final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
   final Matcher ma = pa.matcher(s);
   while (ma.find())
   {
    list.add(ma.group());
   }
   return list;
  }

  /**
   *
   * @param s
   * @return 获得脚本代码
   */
  public List<String> getScript(final String s)
  {
   String regex;
   final List<String> list = new ArrayList<String>();
   regex = "<script.*?</script>";
   final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
   final Matcher ma = pa.matcher(s);
   while (ma.find())
   {
    list.add(ma.group());
   }
   return list;
  }

  /**
   *
   * @param s
   * @return 获得CSS
   */
  public List<String> getCSS(final String s)
  {
   String regex;
   final List<String> list = new ArrayList<String>();
   regex = "<style.*?</style>";
   final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
   final Matcher ma = pa.matcher(s);
   while (ma.find())
   {
    list.add(ma.group());
   }
   return list;
  }

  /**
   *
   * @param s
   * @return 去掉标记
   */
  public String outTag(final String s)
  {
   return s.replaceAll("<.*?>", "");
  }

  /**
   *
   * @param s
   * @return 获取雅虎知识堂文章标题及内容
   */
  public HashMap<String, String> getFromYahoo(final String s)
  {
   final HashMap<String, String> hm = new HashMap<String, String>();
   final StringBuffer sb = new StringBuffer();
   String html = "";
   System.out.println("\n------------------开始读取网页(" + s + ")--------------------");
   try
   {
    html = getOneHtml(s);
   }
   catch (final Exception e)
   {
    e.getMessage();
   }
   // System.out.println(html);
   System.out.println("------------------读取网页(" + s + ")结束--------------------\n");
   System.out.println("------------------分析(" + s + ")结果如下--------------------\n");
   String title = outTag(getTitle(html));
   title = title.replaceAll("_雅虎知识堂", "");
   // Pattern pa=Pattern.compile("<div
   // class=\"original\">(.*?)((\r\n)*)(.*?)((\r\n)*)(.*?)</div>",Pattern.DOTALL);
   final Pattern pa = Pattern.compile("<div class=\"original\">(.*?)</p></div>", Pattern.DOTALL);
   final Matcher ma = pa.matcher(html);
   while (ma.find())
   {
    sb.append(ma.group());
   }
   String temp = sb.toString();
   temp = temp.replaceAll("(<br>)+?", "\n");// 转化换行
   temp = temp.replaceAll("<p><em>.*?</em></p>", "");// 去图片注释
   hm.put("title", title);
   hm.put("original", outTag(temp));
   return hm;

  }

  /**
   *
   * @param args
   *            测试一组网页，针对雅虎知识堂
   */
  public static void main(final String args[])
  {
   String url = "";
   final List<String> list = new ArrayList<String>();
   System.out.print("输入URL，一行一个，输入结束后输入 go 程序开始运行:   \n");
   /*
    * http://ks.cn.yahoo.com/question/1307121201133.html
    * http://ks.cn.yahoo.com/question/1307121101907.html
    * http://ks.cn.yahoo.com/question/1307121101907_2.html
    * http://ks.cn.yahoo.com/question/1307121101907_3.html
    * http://ks.cn.yahoo.com/question/1307121101907_4.html
    * http://ks.cn.yahoo.com/question/1307121101907_5.html
    * http://ks.cn.yahoo.com/question/1307121101907_6.html
    * http://ks.cn.yahoo.com/question/1307121101907_7.html
    * http://ks.cn.yahoo.com/question/1307121101907_8.html
    */
   final BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
   try
   {
    while (!(url = br.readLine()).equals("go"))
    {
     list.add(url);
    }
   }
   catch (final Exception e)
   {
    e.getMessage();
   }
   final WebContent wc = new WebContent();
   HashMap<String, String> hm = new HashMap<String, String>();
   for (int i = 0; i < list.size(); i++)
   {
    hm = wc.getFromYahoo(list.get(i));
    System.out.println("标题： " + hm.get("title"));
    System.out.println("内容： \n" + hm.get("original"));
   }
   /*
    * String htmlurl[] = {
    * "http://ks.cn.yahoo.com/question/1307121201133.html",
    * "http://ks.cn.yahoo.com/question/1307121101907.html",
    * "http://ks.cn.yahoo.com/question/1307121101907_2.html",
    * "http://ks.cn.yahoo.com/question/1307121101907_3.html",
    * "http://ks.cn.yahoo.com/question/1307121101907_4.html",
    * "http://ks.cn.yahoo.com/question/1307121101907_5.html",
    * "http://ks.cn.yahoo.com/question/1307121101907_6.html",
    * "http://ks.cn.yahoo.com/question/1307121101907_7.html",
    * "http://ks.cn.yahoo.com/question/1307121101907_8.html" }; WebContent
    * wc = new WebContent(); HashMap<String, String> hm = new HashMap<String,
    * String>(); for (int i = 0; i < htmlurl.length; i++) { hm =
    * wc.getFromYahoo(htmlurl[i]); System.out.println("标题： " +
    * hm.get("title")); System.out.println("内容： \n" + hm.get("original")); }
    */
   /*
    * String html=""; String link=""; String sscript=""; String content="";
    * System.out.println(htmlurl+" 开始读取网页内容：");
    * html=wc.getOneHtml(htmlurl); System.out.println(htmlurl+"
    * 读取完毕开始分析……"); html=html.replaceAll("(<script.*?)((\r\n)*)(.*?)((\r\n)*)(.*?)(</script>)","
    * ");//去除脚本 html=html.replaceAll("(<style.*?)((\r\n)*)(.*?)((\r\n)*)(.*?)(</style>)","
    * ");//去掉CSS html=html.replaceAll("<title>.*?</title>"," ");//除去页面标题
    * html=html.replaceAll("<a[^>]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))[^>]*>(.*?)</a>","
    * ");//去掉链接 html=html.replaceAll("(\\s){2,}?"," ");//除去多余空格
    * html=wc.outTag(html);//多余标记 System.out.println(html);
    */

   /*
    * String s[]=html.split(" +"); for(int i=0;i<s.length;i++){
    * content=(content.length()>s[i].length())?content:s[i]; }
    * System.out.println(content);
    */

   // System.out.println(htmlurl+"网页内容结束");
   /*
    * System.out.println(htmlurl+"网页脚本开始："); List
    * script=wc.getScript(html); for(int i=0;i<script.size();i++){
    * System.out.println(script.get(i)); }
    * System.out.println(htmlurl+"网页脚本结束：");
    *
    * System.out.println(htmlurl+"CSS开始："); List css=wc.getCSS(html);
    * for(int i=0;i<css.size();i++){ System.out.println(css.get(i)); }
    * System.out.println(htmlurl+"CSS结束：");
    *
    * System.out.println(htmlurl+"全部链接内容开始："); List list=wc.getLink(html);
    * for(int i=0;i<list.size();i++){ link=list.get(i).toString(); }
    * System.out.println(htmlurl+"全部链接内容结束：");
    *
    * System.out.println("内容"); System.out.println(wc.outTag(html));
    */
  }
 }