Lucene 评分机制二 Payload

这里使用的Lucene4.7.0和Lucene3.X稍有不同

有下面三段内容,我想对船一系列的搜索进行加分

　　bike car jeep truck bus boat

　　train car ship boat van subway

　　car plane taxi boat vessel railway

• 定义自定义的MyAnalyzer,实现对字段的有效载荷进行赋值

 package com.pera.lucene.score.payload;

 import java.io.Reader;

 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.analysis.payloads.PayloadEncoder;
 import org.apache.lucene.util.Version;

 public class MyAnalyzer extends Analyzer
 {

     private PayloadEncoder encoder;

     MyAnalyzer(PayloadEncoder encoder)
     {
         this.encoder = encoder;
     }

     @Override
     protected TokenStreamComponents createComponents(String fieldName, Reader reader)
     {
         // 用来解析空格分隔的各个类别
         Tokenizer source = new WhitespaceTokenizer(Version.LUCENE_47, reader);
         // 自定义的Filter,用来获取字段的Payload值
         MyTokenFilter filter = new MyTokenFilter(source, encoder);

         return new TokenStreamComponents(source, filter);
     }

 }

• 自定义TokenFilter来达到取得字段的PayLoad值或通过字段对PayLoad值进行分析赋值

 package com.pera.lucene.score.payload;

 import java.io.IOException;

 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.payloads.PayloadEncoder;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;

 public class MyTokenFilter extends TokenFilter
 {
     private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
     private final PayloadAttribute payAtt = addAttribute(PayloadAttribute.class);
     private final PayloadEncoder encoder;

     public MyTokenFilter(TokenStream input, PayloadEncoder encoder)
     {
         super(input);
         this.encoder = encoder;
     }

     @Override
     public boolean incrementToken() throws IOException
     {
         if (input.incrementToken())
         {
             String term = termAtt.toString();
             if (App.scoreMap.containsKey(term))
             {
                 payAtt.setPayload(encoder.encode(App.scoreMap.get(term).toCharArray()));
             } else
             {
                 payAtt.setPayload(null);
             }
             return true;
         } else
             return false;
     }

 }

     public static ImmutableMap<String, String> scoreMap = ImmutableMap.of("boat", "5f", "ship", "20f", "vessel", "100f");

• 自定义PayloadSimilarity继承DefaultSimilarity 重载scorePayload方法,在检索时获得之前设置的PayLoad值

 package com.pera.lucene.score.payload;

 import org.apache.lucene.analysis.payloads.PayloadHelper;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.util.BytesRef;

 public class PayloadSimilarity extends DefaultSimilarity
 {
     @Override
     public float scorePayload(int doc, int start, int end, BytesRef payload)
     {
         return PayloadHelper.decodeFloat(payload.bytes);
     }
 }

• 建立索引 需要将之前定义的Analyzer和PayloadSimilarity设置到Config中

 package com.pera.lucene.score.payload;

 import java.io.File;
 import java.io.IOException;
 import java.util.Date;

 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.payloads.FloatEncoder;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.Version;

 public class Indexing
 {
     public void indexPayload() throws IOException
     {
         Directory dir = FSDirectory.open(new File(App.indexPath));
         Analyzer analyzer = new MyAnalyzer(new FloatEncoder());
         Similarity similarity = new PayloadSimilarity();

         IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_47, analyzer);
         iwc.setOpenMode(OpenMode.CREATE).setSimilarity(similarity);
         Date start = new Date();
         System.out.println("Indexing to directory '" + App.indexPath + "'...");
         IndexWriter writer = new IndexWriter(dir, iwc);
         Document doc = new Document();
         doc.add(new TextField("tools", "bike car jeep truck bus boat", Store.YES));
         writer.addDocument(doc);

         doc = new Document();
         doc.add(new TextField("tools", "train car ship boat van subway", Store.YES));
         writer.addDocument(doc);

         doc = new Document();
         doc.add(new TextField("tools", "car plane taxi boat vessel railway", Store.YES));
         writer.addDocument(doc);

         writer.close();

         Date end = new Date();
         System.out.println(end.getTime() - start.getTime() + " total milliseconds");
     }
 }

• 进行检索 检索时要将PayloadSimilarity设置到searcher中

 package com.pera.lucene.score.payload;

 import java.io.File;
 import java.io.IOException;

 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.payloads.AveragePayloadFunction;
 import org.apache.lucene.search.payloads.PayloadTermQuery;
 import org.apache.lucene.store.FSDirectory;

 public class Searching
 {

     public void searchPayload() throws IOException, ParseException
     {
         IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(App.indexPath)));
         IndexSearcher searcher = new IndexSearcher(reader);

         BooleanQuery bq = new BooleanQuery();

         PayloadTermQuery ptq1 = new PayloadTermQuery(new Term("tools", "ship"), new AveragePayloadFunction());
         PayloadTermQuery ptq2 = new PayloadTermQuery(new Term("tools", "boat"), new AveragePayloadFunction());
         PayloadTermQuery ptq3 = new PayloadTermQuery(new Term("tools", "vessel"), new AveragePayloadFunction());

         bq.add(ptq1, Occur.SHOULD);
         bq.add(ptq2, Occur.SHOULD);
         bq.add(ptq3, Occur.SHOULD);

         // 设置自定义的PayloadSimilarity
         searcher.setSimilarity(new PayloadSimilarity());
         TopDocs results = searcher.search(bq, 10);
         ScoreDoc[] hits = results.scoreDocs;

         int numTotalHits = results.totalHits;
         System.out.println(numTotalHits + " total matching documents");

         for (int i = 0; i < hits.length; i++)
         {
             int docId = hits[i].doc; // 文档编号
             float lucene_score = hits[i].score;
             String tools = searcher.doc(docId).get("tools");
             System.out.println("DocId:" + docId + "\tLucene Score:" + lucene_score + "\tTools:" + tools);
             Explanation explanation = searcher.explain(bq, docId);
             System.out.println(explanation.toString());
         }
     }
 }

• 检索结果 可以看到Doc2的排序由于有了PayLoad值排名得到了提升

3 total matching documents
DocId:2    Lucene Score:16.750757    Tools:car plane taxi boat vessel railway
16.750757 = (MATCH) product of:
  25.126135 = (MATCH) sum of:
    0.3186112 = (MATCH) btq, product of:
      0.06372224 = weight(tools:boat in 2) [PayloadSimilarity], result of:
        0.06372224 = score(doc=2,freq=0.5 = phraseFreq=0.5
), product of:
          0.33736566 = queryWeight, product of:
            0.71231794 = idf(docFreq=3, maxDocs=3)
            0.4736167 = queryNorm
          0.18888181 = fieldWeight in 2, product of:
            0.70710677 = tf(freq=0.5), with freq of:
              0.5 = phraseFreq=0.5
            0.71231794 = idf(docFreq=3, maxDocs=3)
            0.375 = fieldNorm(doc=2)
      5.0 = AveragePayloadFunction.docScore()
    24.807524 = (MATCH) btq, product of:
      0.24807523 = weight(tools:vessel in 2) [PayloadSimilarity], result of:
        0.24807523 = score(doc=2,freq=0.5 = phraseFreq=0.5
), product of:
          0.66565174 = queryWeight, product of:
            1.4054651 = idf(docFreq=1, maxDocs=3)
            0.4736167 = queryNorm
          0.37268022 = fieldWeight in 2, product of:
            0.70710677 = tf(freq=0.5), with freq of:
              0.5 = phraseFreq=0.5
            1.4054651 = idf(docFreq=1, maxDocs=3)
            0.375 = fieldNorm(doc=2)
      100.0 = AveragePayloadFunction.docScore()
  0.6666667 = coord(2/3)

DocId:1    Lucene Score:3.5200772    Tools:train car ship boat van subway
3.5200772 = (MATCH) product of:
  5.2801156 = (MATCH) sum of:
    4.9615045 = (MATCH) btq, product of:
      0.24807523 = weight(tools:ship in 1) [PayloadSimilarity], result of:
        0.24807523 = score(doc=1,freq=0.5 = phraseFreq=0.5
), product of:
          0.66565174 = queryWeight, product of:
            1.4054651 = idf(docFreq=1, maxDocs=3)
            0.4736167 = queryNorm
          0.37268022 = fieldWeight in 1, product of:
            0.70710677 = tf(freq=0.5), with freq of:
              0.5 = phraseFreq=0.5
            1.4054651 = idf(docFreq=1, maxDocs=3)
            0.375 = fieldNorm(doc=1)
      20.0 = AveragePayloadFunction.docScore()
    0.3186112 = (MATCH) btq, product of:
      0.06372224 = weight(tools:boat in 1) [PayloadSimilarity], result of:
        0.06372224 = score(doc=1,freq=0.5 = phraseFreq=0.5
), product of:
          0.33736566 = queryWeight, product of:
            0.71231794 = idf(docFreq=3, maxDocs=3)
            0.4736167 = queryNorm
          0.18888181 = fieldWeight in 1, product of:
            0.70710677 = tf(freq=0.5), with freq of:
              0.5 = phraseFreq=0.5
            0.71231794 = idf(docFreq=3, maxDocs=3)
            0.375 = fieldNorm(doc=1)
      5.0 = AveragePayloadFunction.docScore()
  0.6666667 = coord(2/3)

DocId:0    Lucene Score:0.106203735    Tools:bike car jeep truck bus boat
0.106203735 = (MATCH) product of:
  0.3186112 = (MATCH) sum of:
    0.3186112 = (MATCH) btq, product of:
      0.06372224 = weight(tools:boat in 0) [PayloadSimilarity], result of:
        0.06372224 = score(doc=0,freq=0.5 = phraseFreq=0.5
), product of:
          0.33736566 = queryWeight, product of:
            0.71231794 = idf(docFreq=3, maxDocs=3)
            0.4736167 = queryNorm
          0.18888181 = fieldWeight in 0, product of:
            0.70710677 = tf(freq=0.5), with freq of:
              0.5 = phraseFreq=0.5
            0.71231794 = idf(docFreq=3, maxDocs=3)
            0.375 = fieldNorm(doc=0)
      5.0 = AveragePayloadFunction.docScore()
  0.33333334 = coord(1/3)