ikanalyzer分词，计算信息熵排序分词结果

因需求，现需分词接口，故记录之。

1、需要依赖：

 <!-- https://mvnrepository.com/artifact/com.janeluo/ikanalyzer -->
         <dependency>
             <groupId>com.janeluo</groupId>
             <artifactId>ikanalyzer</artifactId>
             <version>2012_u6</version>
         </dependency>

maven依赖

2、完整代码如下：

 public JSONArray entropy(String content, Integer quantity) throws Exception {
         List<String> words = extract(DelHtmlTagUtil.delHTMLTag(content), quantity);
         JSONArray array = calculateWordEntropy(words);
         return array;
     }

 /**
      * 传入String类型的文章，智能提取单词放入list
      *
      * @param content  传入分词的内容
      * @param quantity 截取关键字在几个单词以上的数量，默认为1
      * @return
      */
     private List<String> extract(String content, Integer quantity) throws IOException {
         List<String> list = Lists.newArrayList();
         StringReader reader = new StringReader(content);
         IKSegmenter ik = new IKSegmenter(reader, true);
         Lexeme lex = null;
         while ((lex = ik.next()) != null) {
             //String typeString = lex.getLexemeTypeString();  词语类型
             String word = lex.getLexemeText();
             if (word.length() > quantity) {//判断截取关键字在几个单词以上的数量
                 list.add(word);
             }
         }
         return list;
     }

     private JSONArray calculateWordEntropy(List<String> words) throws Exception{

         int length = words.size();
         ArrayList<String[]> wordList = new ArrayList<String[]>();
         // 将分好的词每3个一组存到数组中
         for (int i = 0; i < length; i++) {

             String[] wordSeg = new String[3];
             if (i == 0) {
                 wordSeg[0] = "null";
                 wordSeg[1] = words.get(i);
                 wordSeg[2] = words.get(i + 1);
             } else if (i == length - 1) {
                 wordSeg[0] = words.get(i - 1);
                 wordSeg[1] = words.get(i);
                 wordSeg[2] = "null";
             } else {
                 wordSeg[0] = words.get(i - 1);
                 wordSeg[1] = words.get(i);
                 wordSeg[2] = words.get(i + 1);
             }

             wordList.add(wordSeg);

         }
         // 去除重复的词
         List<String> lists = Lists.newArrayList();
         for (int l = 0; l < length; l++) {
             lists.add(words.get(l));
         }
         List<String> tempList = Lists.newArrayList();
         for (String str : lists) {
             if (!(tempList.contains(str))) {
                 tempList.add(str);
             }
         }
         String[] wordClean = new String[tempList.size()];
         for (int m = 0; m < tempList.size(); m++) {
             wordClean[m] = tempList.get(m);
         }
         // 统计每个词的词频
         int[] frequent = new int[wordClean.length];
         for (int j = 0; j < wordClean.length; j++) {
             int count = 0;
             for (int k = 0; k < words.size(); k++) {
                 if (wordClean[j].equals(words.get(k))) {
                     count++;
                 }
             }
             frequent[j] = count;
         }
         // 将三元组中中间的那个词相同的存到一个list中，然后计算该词的信息熵
         double[] allEntropy = new double[wordClean.length];
         for (int n = 0; n < wordClean.length; n++) {
             ArrayList<String[]> wordSegList = new ArrayList<String[]>();
             int count = 1;
             for (int p = 0; p < wordList.size(); p++) {
                 String[] wordSegStr = wordList.get(p);
                 if (wordSegStr[1].equals(wordClean[n])) {
                     count++;
                     wordSegList.add(wordSegStr);
                 }
             }
             String[] leftword = new String[wordSegList.size()];
             String[] rightword = new String[wordSegList.size()];
             // 计算左信息熵
             for (int i = 0; i < wordSegList.size(); i++) {
                 String[] left = wordSegList.get(i);
                 leftword[i] = left[0];
             }
             // 去除左边重复的词
             List<String> listsLeft = new ArrayList<String>();
             for (int l = 0; l < leftword.length; l++) {
                 listsLeft.add(leftword[l]);
             }
             List<String> tempListLeft = new ArrayList<String>();
             for (String str : listsLeft) {
                 if (!(tempListLeft.contains(str))) {
                     tempListLeft.add(str);
                 }
             }
             String[] leftWordClean = new String[tempListLeft.size()];
             for (int m = 0; m < tempListLeft.size(); m++) {
                 leftWordClean[m] = tempListLeft.get(m);
             }
             // 统计左边每个词的词频
             int[] leftFrequent = new int[leftWordClean.length];
             for (int j = 0; j < leftWordClean.length; j++) {
                 int leftcount = 0;
                 for (int k = 0; k < leftword.length; k++) {
                     if (leftWordClean[j].equals(leftword[k])) {
                         leftcount++;
                     }
                 }
                 leftFrequent[j] = leftcount;
             }
             // 计算左熵值
             double leftEntropy = 0;
             for (int i = 0; i < leftFrequent.length; i++) {
                 double a = (double) leftFrequent[i] / count;
                 double b = Math.log((double) leftFrequent[i] / count);
                 leftEntropy += -a * b;
                 // leftEntropy +=
                 // (-(double)(leftFrequent[i]/count))*Math.log((double)(leftFrequent[i]/count));
             }
             // 计算右信息熵
             for (int i = 0; i < wordSegList.size(); i++) {
                 String[] right = wordSegList.get(i);
                 rightword[i] = right[2];
             }
             // 去除右边重复的词
             List<String> listsRight = new ArrayList<String>();
             for (int l = 0; l < rightword.length; l++) {
                 listsRight.add(rightword[l]);
             }
             List<String> tempListRight = new ArrayList<String>();
             for (String str : listsRight) {
                 if (!(tempListRight.contains(str))) {
                     tempListRight.add(str);
                 }
             }
             String[] rightWordClean = new String[tempListRight.size()];
             for (int m = 0; m < tempListRight.size(); m++) {
                 rightWordClean[m] = tempListRight.get(m);
             }
             // 统计右边每个词的词频
             int[] rightFrequent = new int[rightWordClean.length];
             for (int j = 0; j < rightWordClean.length; j++) {
                 int rightcount = 0;
                 for (int k = 0; k < rightword.length; k++) {
                     if (rightWordClean[j].equals(rightword[k])) {
                         rightcount++;
                     }
                 }
                 rightFrequent[j] = rightcount;
             }
             // 计算右熵值
             double rightEntropy = 0.0;
             for (int i = 0; i < rightFrequent.length; i++) {
                 double a = (double) rightFrequent[i] / count;
                 double b = Math.log((double) rightFrequent[i] / count);
                 rightEntropy += -a * b;
                 // rightEntropy +=
                 // (-(double)(rightFrequent[i]/count))*Math.log((double)(rightFrequent[i]/count));
             }
             // 计算词的总信息熵
             double wordEntropy = leftEntropy + rightEntropy;
             allEntropy[n] = wordEntropy;

         }
         JSONArray list = new JSONArray();
         for (int i = 0; i < allEntropy.length; i++) {
             JSONObject obj = new JSONObject();
             obj.put("name", wordClean[i]);
             obj.put("entropy", allEntropy[i]);
             list.add(obj);
         }
         Collections.sort(list, (o1, o2) -> {
             Double d1 = ((JSONObject) o1).getDouble("entropy");
             Double d2 = ((JSONObject) o2).getDouble("entropy");
             return d2.compareTo(d1);
         });

         return list;
     }

处理代理