使用DFA算法对敏感词进行过滤

项目目录结构如下：

其中resources资源目录中：

stopwd.txt ：停顿词，匹配时间直接过滤。

wd.txt：敏感词库。

1、WordFilter敏感词过滤类：

 package com.skyer.sensitivewdfilter;

 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

 /**
  * 思路： 创建一个FilterSet，枚举了0~65535的所有char是否是某个敏感词开头的状态
  *
  * 判断是否是 敏感词开头 | | 是 不是 获取头节点 OK--下一个字 然后逐级遍历，DFA算法
  */
 public class WordFilter {

     private static final FilterSet set = new FilterSet(); // 存储首字
     private static final Map<Integer, WordNode> nodes = new HashMap<Integer, WordNode>(1024, 1); // 存储节点
     private static final Set<Integer> stopwdSet = new HashSet<Integer>(); // 停顿词
     private static final char SIGN = '*'; // 敏感词过滤替换

     static {
         try {
             long a = System.nanoTime();
             init();
             a = System.nanoTime() - a;
             System.out.println("加载时间 : " + a + "ns");
             System.out.println("加载时间 : " + a / 1000000 + "ms");
         } catch (Exception e) {
             throw new RuntimeException("初始化过滤器失败");
         }
     }

     private static void init() {
         // 获取敏感词
         addSensitiveWord(readWordFromFile("wd.txt"));
         addStopWord(readWordFromFile("stopwd.txt"));
     }

     /**
      * 增加敏感词
      */
     private static List<String> readWordFromFile(String path) {
         List<String> words;
         BufferedReader br = null;
         try {
             br = new BufferedReader(new InputStreamReader(WordFilter.class.getClassLoader().getResourceAsStream(path)));
             words = new ArrayList<String>(1200);
             for (String buf = ""; (buf = br.readLine()) != null;) {
                 if (buf == null || buf.trim().equals(""))
                     continue;
                 words.add(buf);
             }
         } catch (Exception e) {
             throw new RuntimeException(e);
         } finally {
             try {
                 if (br != null)
                     br.close();
             } catch (IOException e) {
             }
         }
         return words;
     }

     /**
      * 增加停顿词
      */
     private static void addStopWord(final List<String> words) {
         if (words != null && words.size() > 0) {
             char[] chs;
             for (String curr : words) {
                 chs = curr.toCharArray();
                 for (char c : chs) {
                     stopwdSet.add(charConvert(c));
                 }
             }
         }
     }

     /**
      * 添加DFA节点
      */
     private static void addSensitiveWord(final List<String> words) {
         if (words != null && words.size() > 0) {
             char[] chs;
             int fchar;
             int lastIndex;
             WordNode fnode; // 首字母节点
             for (String curr : words) {
                 chs = curr.toCharArray();
                 fchar = charConvert(chs[0]);
                 if (!set.contains(fchar)) {// 没有首字定义
                     set.add(fchar);// 首字标志位 可重复add
                     fnode = new WordNode(fchar, chs.length == 1);
                     nodes.put(fchar, fnode);
                 } else {
                     fnode = nodes.get(fchar);
                     if (!fnode.isLast() && chs.length == 1)
                         fnode.setLast(true);
                 }
                 lastIndex = chs.length - 1;
                 for (int i = 1; i < chs.length; i++) {
                     fnode = fnode.addIfNoExist(charConvert(chs[i]), i == lastIndex);
                 }
             }
         }
     }

     /**
      * 过滤判断 将敏感词转化为成屏蔽词
      */
     public static final String doFilter(final String src) {
         char[] chs = src.toCharArray();
         int length = chs.length;
         int currc;
         int k;
         WordNode node;
         for (int i = 0; i < length; i++) {
             currc = charConvert(chs[i]);
             if (!set.contains(currc)) {
                 continue;
             }
             node = nodes.get(currc);
             if (node == null)
                 continue;
             boolean couldMark = false;
             int markNum = -1;
             if (node.isLast()) {
                 couldMark = true;
                 markNum = 0;
             }
             k = i;
             for (; ++k < length;) {
                 int temp = charConvert(chs[k]);
                 if (stopwdSet.contains(temp))
                     continue;
                 node = node.querySub(temp);
                 if (node == null)
                     break;
                 if (node.isLast()) {
                     couldMark = true;
                     markNum = k - i;
                 }
             }
             if (couldMark) {
                 for (k = 0; k <= markNum; k++) {
                     chs[k + i] = SIGN;
                 }
                 i = i + markNum;
             }
         }

         return new String(chs);
     }

     /**
      * 是否包含敏感词
      */
     public static final boolean isContains(final String src) {
         char[] chs = src.toCharArray();
         int length = chs.length;
         int currc;
         int k;
         WordNode node;
         for (int i = 0; i < length; i++) {
             currc = charConvert(chs[i]);
             if (!set.contains(currc)) {
                 continue;
             }
             node = nodes.get(currc);
             if (node == null)
                 continue;
             boolean couldMark = false;
             if (node.isLast()) {
                 couldMark = true;
             }
             k = i;
             for (; ++k < length;) {
                 int temp = charConvert(chs[k]);
                 if (stopwdSet.contains(temp))
                     continue;
                 node = node.querySub(temp);
                 if (node == null)
                     break;
                 if (node.isLast()) {
                     couldMark = true;
                 }
             }
             if (couldMark) {
                 return true;
             }
         }

         return false;
     }

     /**
      * 大写转化为小写 全角转化为半角
      */
     private static int charConvert(char src) {
         int r = BCConvert.qj2bj(src);
         return (r >= 'A' && r <= 'Z') ? r + 32 : r;
     }

 }

WordFilter.java

其中：

isContains ：是否包含敏感词

doFilter：过滤敏感词

2、WordNode敏感词节点：

 package com.skyer.sensitivewdfilter;

 import java.util.LinkedList;
 import java.util.List;

 public class WordNode {

     private int value; // 节点名称

     private List<WordNode> subNodes; // 子节点

     private boolean isLast; // 默认false

     public WordNode(int value) {
         this.value = value;
     }

     public WordNode(int value, boolean isLast) {
         this.value = value;
         this.isLast = isLast;
     }

     /**
      * @return 就是传入的subNode
      */
     private WordNode addSubNode(final WordNode subNode) {
         if (subNodes == null)
             subNodes = new LinkedList<WordNode>();
         subNodes.add(subNode);
         return subNode;
     }

     /**
      * 有就直接返回该子节点， 没有就创建添加并返回该子节点
      */
     public WordNode addIfNoExist(final int value, final boolean isLast) {
         if (subNodes == null) {
             return addSubNode(new WordNode(value, isLast));
         }
         for (WordNode subNode : subNodes) {
             if (subNode.value == value) {
                 if (!subNode.isLast && isLast)
                     subNode.isLast = true;
                 return subNode;
             }
         }
         return addSubNode(new WordNode(value, isLast));
     }

     public WordNode querySub(final int value) {
         if (subNodes == null) {
             return null;
         }
         for (WordNode subNode : subNodes) {
             if (subNode.value == value)
                 return subNode;
         }
         return null;
     }

     public boolean isLast() {
         return isLast;
     }

     public void setLast(boolean isLast) {
         this.isLast = isLast;
     }

     @Override
     public int hashCode() {
         return value;
     }

 }

WordNode.java

3、测试类：

 package com.skyer.test;

 import org.junit.Test;

 import com.skyer.sensitivewdfilter.WordFilter;

 public class TestSensitivewd {

     @Test
     public void TestFilter() {
         String s = ""; // 这里写你要过滤的句子（我这里不能写，否则会给博客园屏蔽掉）
         System.out.println("解析问题： " + s);
         System.out.println("解析字数 : " + s.length());
         String re;
         long nano = System.nanoTime();
         re = WordFilter.doFilter(s);
         nano = (System.nanoTime() - nano);
         System.out.println("解析时间 : " + nano + "ns");
         System.out.println("解析时间 : " + nano / 1000000 + "ms");
         System.out.println(re);
         System.out.println();

         nano = System.nanoTime();
         System.out.println("是否包含敏感词： " + WordFilter.isContains(s));
         nano = (System.nanoTime() - nano);
         System.out.println("解析时间 : " + nano + "ns");
         System.out.println("解析时间 : " + nano / 1000000 + "ms");
     }

 }

TestSensitivewd.java

4、测试结果：

原文参考：http://blog.csdn.net/fengshizty/article/details/52373005

DFA知识：http://www.cnblogs.com/naaoveGIS/archive/2016/10/14/5960352.html