crawler_基于块儿统计正文抽取_改进版

在线查看效果：http://tool.haoshuju.cn/

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;

/**
 * <p>
 * 在线性时间内抽取主题类（新闻、博客等）网页的正文。 采用了<b>基于行块分布函数</b>的方法，为保持通用性没有针对特定网站编写规则。
 * </p>
 *
 * @author Chen Xin(xchen@ir.hit.edu.cn) Created on 2009-1-11 Updated on
 *         2010-08-09
 * @note cphmvp 优化性能提速三倍，优化列表页中含其它标签识别 2014-6-5 11:53:33
 */
public class TextExtract {

    private List<String> lines;
    private final static int blocksWidth = 3;
    private int threshold;
    private String html;
    private boolean flag;
    private int start;
    private int end;
    private StringBuilder text;
    private ArrayList<Integer> indexDistribution;

    public TextExtract() {
        lines = new ArrayList<String>();
        indexDistribution = new ArrayList<Integer>();
        text = new StringBuilder();
        flag = false;
        /* 当待抽取的网页正文中遇到成块的新闻标题未剔除时，只要增大此阈值即可。 */
        /* 阈值增大，准确率提升，召回率下降；值变小，噪声会大，但可以保证抽到只有一句话的正文 */
        threshold = -1;
    }

    /**
     * 抽取网页正文，不判断该网页是否是目录型。即已知传入的肯定是可以抽取正文的主题类网页。
     *
     * @param _html
     *            网页HTML字符串
     *
     * @return 网页正文string
     */
    public String parse(String _html) {
        return parse(_html, false);
    }

    /**
     * 判断传入HTML，若是主题类网页，则抽取正文；否则输出<b>"unkown"</b>。
     *
     * @param _html
     *            网页HTML字符串
     * @param _flag
     *            true进行主题类判断, 省略此参数则默认为false
     *
     * @return 网页正文string<br/>
     * @note 2014年6月4日11:55:51 修复图片标签空格处理 cphmvp
     */
    public String parse(String _html, boolean _flag) {
        flag = _flag;
        html = _html;
        html = preProcess(html);
        // System.out.println(html);
        return getText().replaceAll("imgsrc=", "img src=");
    }

    private static int FREQUENT_URL = 30;
    private static Pattern links = Pattern
            .compile(
                    "<[aA]\\s+[Hh][Rr][Ee][Ff]=[\"|\']?([^>\"\' ]+)[\"|\']?\\s*[^>]*>([^>]+)</a>(\\s*.{0,"
                            + FREQUENT_URL
                            + "}\\s*<a\\s+href=[\"|\']?([^>\"\' ]+)[\"|\']?\\s*[^>]*>([^>]+)</[aA]>){2,100}",
                    Pattern.DOTALL);

    private static String preProcess(String source) {

        source = source.replaceAll("(?is)<!DOCTYPE.*?>", "");
        source = source.replaceAll("(?is)<!--.*?-->", ""); // remove html
                                                            // comment
        source = source.replaceAll("(?is)<script.*?>.*?</script>", ""); // remove
                                                                        // javascript
        source = source.replaceAll("(?is)<style.*?>.*?</style>", ""); // remove
                                                                        // css
        source = source.replaceAll("&.{2,5};|&#.{2,5};", " "); // remove special
                                                                // char

        // 剔除连续成片的超链接文本（认为是，广告或噪音）,超链接多藏于span中
        source = source.replaceAll("<[sS][pP][aA][nN].*?>", "");
        source = source.replaceAll("</[sS][pP][aA][nN]>", "");

        // int len = source.length();
        // while ((source = links.matcher(source).replaceAll("")).length() !=
        // len) {
        // len = source.length();
        // }
        // ;
        // 提升性能三倍 cphmvp
        source = source.replaceAll(" ", "");
        // [\\s\\S]{0,30} 用以表示a之间的间隙容忍度
        String regex = "<[a|A][^>]*?>[^>]+</[a|A]>(?:\\s*[\\s\\S]{0,30}\\s*<[a|A][^>]*?>[^>]+</[a|A]>){2,100}";
        source = source.replaceAll(regex, "");
        // continue;

        // source = links.matcher(source).replaceAll("");

        // 防止html中在<>中包括大于号的判断
        source = source.replaceAll("<[^>'\"]*['\"].*['\"].*?>", "");

        source = source.replaceAll("<.*?>", "");
        source = source.replaceAll("<.*?>", "");
        source = source.replaceAll("\r\n", "\n");

        return source;

    }

    private String getText() {
        lines = Arrays.asList(html.split("\n"));
        indexDistribution.clear();

        int empty = 0;// 空行的数量
        for (int i = 0; i < lines.size() - blocksWidth; i++) {

            if (lines.get(i).length() == 0) {
                empty++;
            }

            int wordsNum = 0;
            for (int j = i; j < i + blocksWidth; j++) {
                lines.set(j, lines.get(j).replaceAll("\\s+", ""));
                wordsNum += lines.get(j).length();
            }
            indexDistribution.add(wordsNum);
            // System.out.println(wordsNum);
        }
        int sum = 0;

        for (int i = 0; i < indexDistribution.size(); i++) {
            sum += indexDistribution.get(i);
        }
        // 正文全部script情况，抽取不到正文
        if (indexDistribution.size() == 0)
            return "";
        threshold = Math.min(100, (sum / indexDistribution.size()) << (empty
                / (lines.size() - empty) >>> 1));
        threshold = Math.max(50, threshold);

        start = -1;
        end = -1;
        boolean boolstart = false, boolend = false;
        boolean firstMatch = true;// 前面的标题块往往比较小，应该减小与它匹配的阈值
        text.setLength(0);

        StringBuilder buffer = new StringBuilder();
        for (int i = 0; i < indexDistribution.size() - 1; i++) {

            if (firstMatch && !boolstart) {
                if (indexDistribution.get(i) > (threshold / 2) && !boolstart) {
                    if (indexDistribution.get(i + 1).intValue() != 0
                            || indexDistribution.get(i + 2).intValue() != 0) {
                        firstMatch = false;
                        boolstart = true;
                        start = i;
                        continue;
                    }
                }

            }
            if (indexDistribution.get(i) > threshold && !boolstart) {
                if (indexDistribution.get(i + 1).intValue() != 0
                        || indexDistribution.get(i + 2).intValue() != 0
                        || indexDistribution.get(i + 3).intValue() != 0) {
                    boolstart = true;
                    start = i;
                    continue;
                }
            }
            if (boolstart) {
                if (indexDistribution.get(i).intValue() == 0
                        || indexDistribution.get(i + 1).intValue() == 0) {
                    end = i;
                    boolend = true;
                }
            }

            if (boolend) {
                buffer.setLength(0);
                // System.out.println(start+1 + "\t\t" + end+1);
                for (int ii = start; ii <= end; ii++) {
                    if (lines.get(ii).length() < 5)
                        continue;
                    buffer.append(lines.get(ii) + "\n");
                }
                String str = buffer.toString();
                // System.out.println(str);
                if (str.contains("Copyright") || str.contains("版权所有"))
                    continue;
                text.append(str);
                boolstart = boolend = false;
            }
        }

        if (start > end) {
            buffer.setLength(0);
            int size_1 = lines.size() - 1;
            for (int ii = start; ii <= size_1; ii++) {
                if (lines.get(ii).length() < 5)
                    continue;
                buffer.append(lines.get(ii) + "\n");
            }
            String str = buffer.toString();
            // System.out.println(str);
            if ((!str.contains("Copyright")) || (!str.contains("版权所有"))) {
                text.append(str);
            }
        }

        return text.toString();
    }

    public static void main(String[] args) {
        System.out.println("===============");
        String s = "<img  class='fit-image' onload='javascript:if(this.width>498)this.width=498;' />hello";
        // source = source.replaceAll("<[^'\"]*['\"].*['\"].*?>", "");
        System.out.println(TextExtract.preProcess(s));
    }
}