全文检索(二)-基于lucene4.10的增删改查

今天 用lucene完毕了 一个简单的web应用。提取了早期编写的一个測试类。 首先简单介绍下lucene几个经常使用包;

lucene 包的组成结构：对于外部应用来说索引模块(index)和检索模块(search)是基本的外部应用入口



org.apache.Lucene.search/ 搜索入口 

org.apache.Lucene.index/ 索引入口 

org.apache.Lucene.analysis/ 语言分析器 

org.apache.Lucene.queryParser/ 查询分析器 

org.apache.Lucene.document/ 存储结构 

org.apache.Lucene.store/  底层IO/存储结构 

org.apache.Lucene.util/ 一些公用的数据结构

话不多说，直接上代码（这是早期封装的一个測试类。封装的还算比較完好，有兴趣的朋友能够在此基础上继续完好）：

package com.lucene.util;

import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.List;

import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleFragmenter;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.lucene.IKAnalyzer;

import com.message.base.search.SearchBean;

/**
 * lucene 4.10.1
 *
 * @creatTime 2014-10-28
 * @author 胡慧超
 *
 */
public class HhcIndexTools {

	private final static Logger logger = Logger.getLogger(HhcIndexTools.class);
	private static String indexPath = "E://lucene//index";

	public static void main(String[] args) {
		try {
//			createIndex();
//			searchIndex("码农");
//			query();
//			deleteIndex(null);
			forceDeleteIndex();
			query();
			highlighterSearch();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	/**
	 * 创建索引
	 */
	public static void createIndex() {
		// 最细粒切分算法--true的话是 智能切分
		Analyzer analyzer = new IKAnalyzer(false);
		Document doc = null;
		IndexWriter indexWriter = null;
		try {
			indexWriter = getIndexWriter(analyzer);
			// 加入索引
			doc = new Document();
			doc.add(new StringField("id", "1", Store.YES));
			doc.add(new TextField("title", "标题：開始", Store.YES));
			doc.add(new TextField("content", "内容：我如今是个码农", Store.YES));
			indexWriter.addDocument(doc);
			doc = new Document();
			doc.add(new StringField("id", "2", Store.YES));
			doc.add(new TextField("title", "标题：结束", Store.YES));
			doc.add(new TextField("content", "内容:我如今是个lucene开发project师的专家",
					Store.YES));
			indexWriter.addDocument(doc);
			indexWriter.commit();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			logger.info("索引器发送异常");
		} finally {
			try {
				destroyWriter(indexWriter);
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}

	/**
	 * 搜索文档
	 *
	 * @param keyword
	 */
	@SuppressWarnings("deprecation")
	public static void searchIndex(String keyword) {
		IndexReader indexReader = null;
		IndexSearcher indexSearcher = null;
		try {
			// 1.创建Directory 在硬盘上的F:/luence/index下建立索引
			Directory dir = FSDirectory.open(new File(indexPath));
			// 2.创建IndexReader
			indexReader = IndexReader.open(dir);
			// 实例化搜索器
			indexSearcher = new IndexSearcher(indexReader);

			// 使用QueryParser查询分析器构造Query对象
			QueryParser parse = new QueryParser(Version.LUCENE_4_10_1,
					"content", new IKAnalyzer(false));
			// 搜索包括keyword关键字的文档
			Query query = parse.parse(keyword.trim());

			// 使用lucene构造搜索引擎的时候，假设要针对多个域进行一次性查询
			// 这样的方法的优点就是能够加权给字段的控制
			// 在这四个域中检索
			String[] fields = { "phoneType", "name", "category", "price" };
			Query querys = new MultiFieldQueryParser(Version.LATEST, fields,
					new IKAnalyzer(false)).parse(keyword.trim());

			TopDocs results = indexSearcher.search(query, 1000);
			// 6.依据TopDocs获取ScoreDoc对象
			ScoreDoc[] score = results.scoreDocs;
			if (score.length > 0) {
				logger.info("查询结果数：" + score.length);
				System.out.println("查询结果数：" + score.length);
				for (int i = 0; i < score.length; i++) {
					// 7.依据Seacher和ScoreDoc对象获取详细的Document对象
					Document doc = indexSearcher.doc(score[i].doc);
					// 8.依据Document对象获取须要的值
					System.out.println(doc.toString());
					System.out.println(doc.get("title") + "["
							+ doc.get("content") + "]");
				}
			} else {
			}
		} catch (Exception e) {
			// TODO: handle exception
			logger.info("查询结果为空！");
		} finally {
			if (indexReader != null) {
				try {
					indexReader.close();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
		}

	}

	/**
	 * 对搜索返回的前n条结果进行分页显示
	 *
	 * @param keyWord
	 *            查询关键词
	 * @param pageSize
	 *            每页显示记录数
	 * @param currentPage
	 *            当前页
	 * @throws ParseException
	 */
	@SuppressWarnings("deprecation")
	public void paginationQuery(String keyWord, int pageSize, int currentPage)
			throws IOException, ParseException {
		String[] fields = { "title", "content" };
		QueryParser queryParser = new MultiFieldQueryParser(Version.LATEST,
				fields, new IKAnalyzer());
		Query query = queryParser.parse(keyWord.trim());

		IndexReader indexReader = IndexReader.open(FSDirectory.open(new File(
				indexPath)));
		IndexSearcher indexSearcher = new IndexSearcher(indexReader);

		// TopDocs 搜索返回的结果
		TopDocs topDocs = indexSearcher.search(query, 100);// 仅仅返回前100条记录
		TopDocs all = indexSearcher.search(new MatchAllDocsQuery(), 100);
		// int totalCount = topDocs.totalHits; // 搜索结果总数量
		ScoreDoc[] scoreDocs = topDocs.scoreDocs; // 搜索返回的结果集合

		// 查询起始记录位置
		int begin = pageSize * (currentPage - 1);
		// 查询终止记录位置
		int end = Math.min(begin + pageSize, scoreDocs.length);

		// 进行分页查询
		for (int i = begin; i < end; i++) {
			int docID = scoreDocs[i].doc;
			System.out.println("docID=" + docID);
			Document doc = indexSearcher.doc(docID);
			String title = doc.get("title");
			System.out.println("title is : " + title);
		}
		indexReader.close();
	}

	@SuppressWarnings("deprecation")
	public static void highlighterSearch() throws IOException, ParseException, InvalidTokenOffsetsException {
		IndexReader reader = IndexReader.open(FSDirectory.open(new File(
				indexPath)));
		IndexSearcher searcher = new IndexSearcher(reader);

		// String []fields={"title","content"};
		// QueryParser parser=new MultiFieldQueryParser(Version.LATEST, fields,
		// new IKAnalyzer());
		// Query query=parser.parse("");

		Term term = new Term("content", "lucene");
		TermQuery query = new TermQuery(term);

		TopDocs topdocs = searcher.search(query, Integer.MAX_VALUE);
		ScoreDoc[] scoreDoc = topdocs.scoreDocs;
		System.out.println("查询结果总数:" + topdocs.totalHits);
		System.out.println("最大的评分:" + topdocs.getMaxScore());

		for(int i=0;i<scoreDoc.length;i++){
			int docid=scoreDoc[i].doc;
			Document document=searcher.doc(docid);
			System.out.println("============文件【"+(i+1)+"】=========");
			System.out.println("检索关键字："+term.toString());
			String content=document.get("content");

			//高亮展示 SimpleHTMLFormatter //
			SimpleHTMLFormatter  formatter=new SimpleHTMLFormatter("<font color='red'>", "</font>");
			Highlighter highlighter=new Highlighter(formatter, new QueryScorer(query));
	        highlighter.setTextFragmenter(new SimpleFragmenter(content.length())); 

	        if(!"".equals(content)){
	        	TokenStream tokenstream=new IKAnalyzer().tokenStream(content, new StringReader(content));
	        	String highLightText = highlighter.getBestFragment(tokenstream,content);
	            System.out.println("高亮显示第 " + (i + 1) + " 条检索结果例如以下所看到的：");
	            System.out.println(highLightText);
	            /*End:结束关键字高亮*/
	            System.out.println("文件内容:"+content);
	            System.out.println("匹配相关度："+scoreDoc[i].score);
	        }
		}
	}

	/**
	 * 获取indexWriter对象---获取索引器
	 *
	 * @param dir
	 * @param analyer
	 * @return
	 * @throws IOException
	 */
	private static IndexWriter getIndexWriter(Analyzer analyzer)
			throws IOException {
		File indexFile = new File(indexPath);
		if (!indexFile.exists())
			indexFile.mkdir();// 索引库不存在 则新建一个
		Directory directory = FSDirectory.open(indexFile);
		// Directory directory = new RAMDirectory(); //在内存中建立索引

		IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_10_1,
				analyzer);
		LogMergePolicy mergePolicy = new LogDocMergePolicy();
		// 索引基本配置
		// 设置segment加入文档(Document)时的合并频率
		// 值较小,建立索引的速度就较慢
		// 值较大,建立索引的速度就较快,>10适合批量建立索引
		mergePolicy.setMergeFactor(30);
		// 设置segment最大合并文档(Document)数
		// 值较小有利于追加索引的速度
		// 值较大,适合批量建立索引和更快的搜索
		mergePolicy.setMaxMergeDocs(5000);
		conf.setMaxBufferedDocs(10000);
		conf.setMergePolicy(mergePolicy);
		conf.setRAMBufferSizeMB(64);

		conf.setOpenMode(OpenMode.CREATE_OR_APPEND);
		if (IndexWriter.isLocked(directory)) {// ?
			IndexWriter.unlock(directory);
		}
		IndexWriter indexWriter = new IndexWriter(directory, conf);
		return indexWriter;
	}

	/**
	 * 销毁writer
	 *
	 * @param writer
	 * @throws IOException
	 */
	private static void destroyWriter(IndexWriter indexWriter)
			throws IOException {
		if (indexWriter != null) {
			indexWriter.close();
		}
	}

	/**
	 * 批量删除
	 *
	 * @param list
	 * @throws IOException
	 */
	public static void deleteIndexs(List<SearchBean> list) throws IOException {
		if (list == null || list.size() > 0) {
			logger.debug("beans is null");
			return;
		}
		for (SearchBean bean : list) {
			deleteIndex(bean);
		}
	}

	/**
	 * 删除单个索引 --不会立马删除，生成.del文件
	 *
	 * @param bean
	 * @throws IOException
	 */
	private static void deleteIndex(SearchBean bean) throws IOException {
		// if(bean==null){
		// logger.debug("Get search bean is empty!");
		// return;
		// }
		IndexWriter indexWriter = getIndexWriter(new IKAnalyzer());
		// 參数是一个选项，能够是一个Query,也能够是一个term,term是一个精确查找的值
		// 这里删除id=1的文档，还会留在”回收站“。xxx.del
		indexWriter.deleteDocuments(new Term("id", "1"));
		destroyWriter(indexWriter);
	}

	/**
	 * 查询文档
	 */
	@SuppressWarnings("deprecation")
	public static void query() {
		// 1.创建Directory 在硬盘上的F:/luence/index下建立索引
		try {
			IndexReader indexReader = IndexReader.open(FSDirectory
					.open(new File(indexPath)));
			System.out.println("存储的文档数:" + indexReader.numDocs());
			System.out.println("总存储量:" + indexReader.maxDoc());
			System.out.println("被删除的文档：" + indexReader.numDeletedDocs());
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	/**
	 * 回滚回收站
	 *
	 * @throws IOException
	 */
	public void recoveryIndexByIsDelete() throws IOException {
		IndexWriter indexWriter = getIndexWriter(new IKAnalyzer());
		indexWriter.rollback();
		destroyWriter(indexWriter);
	}

	/**
	 * 清空回收站 在版本号3.6之后，已经没有了unDeleteAll()方法了
	 *
	 * @throws IOException
	 */
	public static void forceDeleteIndex() throws IOException {
		IndexWriter indexWriter = getIndexWriter(new IKAnalyzer());
		indexWriter.forceMergeDeletes();
		destroyWriter(indexWriter);
	}

	/**
	 * 更新索引
	 *
	 * @throws IOException
	 */
	public void update() throws IOException {
		IndexWriter indexWriter = new IndexWriter(FSDirectory.open(new File(
				indexPath)), new IndexWriterConfig(Version.LATEST,
				new IKAnalyzer(true)));

		Document document = new Document();

		document.add(new Field("id", "10", Field.Store.YES,
				Field.Index.NOT_ANALYZED_NO_NORMS));
		document.add(new Field("email", "9481629991", Field.Store.YES,
				Field.Index.NOT_ANALYZED));
		document.add(new Field("name", "小米", Field.Store.YES,
				Field.Index.NOT_ANALYZED_NO_NORMS));
		document.add(new Field("content", "小米好", Field.Store.NO,
				Field.Index.ANALYZED));

		// 这里的更新，从方法上能够看出，它实际上时将旧的删除，然后加入一个新文档的进去，将匹配到term的文档删除，然后就新的document加入进去
		indexWriter.updateDocument(new Term("id", "1"), document);

		indexWriter.close();
	}
}

另外附带高亮的项目代码

	/**
	 * 高亮的公共方法
	 * @param text  --查询内容
	 * @param query --查询query
	 * @param field --查询域
	 * @return
	 */
	private String highligher(String text,Query query,String field) {
		try {
			QueryScorer scorer = new QueryScorer(query);
			Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
			Formatter formatter = new SimpleHTMLFormatter("<span class='lighter'>","</span>");
			Highlighter lighter = new Highlighter(formatter,scorer);
			lighter.setTextFragmenter(fragmenter);
			String ht = lighter.getBestFragment(LuceneContext.getInstance().getAnalyzer(),
					field,text);
			if(ht==null) {
				if(text.length()>=200) {
					text = text.substring(0, 200);
					text=text+"....";
				}
				return text;
			}
			else return ht.trim();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InvalidTokenOffsetsException e) {
			e.printStackTrace();
		}
		return text;
	}