IKAnalyzer LUCENE.4.9 中文分词的高亮显示

import java.io.File; import java.io.IOException; import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.highlight.Highlighter; import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; import org.apache.lucene.search.highlight.QueryScorer; import org.apache.lucene.search.highlight.Scorer; import org.apache.lucene.search.highlight.SimpleFragmenter; import org.apache.lucene.search.highlight.SimpleHTMLFormatter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Version; import org.wltea.analyzer.lucene.IKAnalyzer; public class IKAnalyzerDemo1 { /** * @param args * @throws IOException */ public static void main(String[] args) throws IOException, ParseException, InvalidTokenOffsetsException { // TODO Auto-generated method stub //建立索引 String text1 = "IK Analyzer是一个结合词典分词和文法分词的中文分词开源工具包。它使用了全新的正向迭代最细粒度切了解形势分析—公共安全问题的凸显了解形势分析—公共安全问题的凸显" + "分算法。"; String text2 = "中文分词工具包可以和lucene是一起使用的了解形势分析—公共安全问题的凸显了解形势分析—公共安全问题的凸显了解形势分析—公共安全问题的凸显了解形势分析—公共安全问题的凸显"; String text3 = "中文分词,你妹"; String fieldName = "contents"; Analyzer analyzer = new IKAnalyzer(); File myFilePath = new File("C:\\results\\index"); Directory directory = FSDirectory.open(myFilePath); //RAMDirectory directory = new RAMDirectory(); IndexWriterConfig writerConfig = new IndexWriterConfig(Version.LUCENE_4_9, analyzer); IndexWriter indexWriter = new IndexWriter(directory, writerConfig); Document document1 = new Document(); //IndexableField field1 = new IndexableField("ID", "1", Field.Store.YES, Field.Index.NOT_ANALYZED); //document1.add(null); document1.add(new StringField("ID", "1", Field.Store.YES)); document1.add(new TextField(fieldName, text1, Field.Store.YES)); indexWriter.addDocument(document1); Document document2 = new Document(); document2.add(new StringField("ID", "2", Field.Store.YES)); document2.add(new TextField(fieldName, text2, Field.Store.YES)); indexWriter.addDocument(document2); Document document3 = new Document(); document3.add(new StringField("ID", "3", Field.Store.YES)); document3.add(new TextField(fieldName, text3, Field.Store.YES)); indexWriter.addDocument(document3); indexWriter.close(); //搜索 IndexReader indexReader = DirectoryReader.open(directory); // IndexReader.open(directory); IndexSearcher searcher = new IndexSearcher(indexReader); String request = "工具包 中文 分词"; System.out.println("request="+request); QueryParser parser = new QueryParser(Version.LUCENE_4_9, fieldName, analyzer); parser.setDefaultOperator(QueryParser.AND_OPERATOR); try { Query query = parser.parse(request); Scorer scorer = new QueryScorer(query); System.out.println("query="+query.toString().replaceAll("\\+contents:", "、")); TopDocs topDocs = searcher.search(query, 5); System.out.println("命中数:"+topDocs.totalHits); ScoreDoc[] docs = topDocs.scoreDocs; SimpleHTMLFormatter simpleHtmlFormatter = new SimpleHTMLFormatter("<B>", "</B>"); Highlighter highlighter = new Highlighter(simpleHtmlFormatter, scorer); highlighter.setTextFragmenter(new SimpleFragmenter(50)); for(ScoreDoc doc : docs){ Document d = searcher.doc(doc.doc); System.out.println("内容:"+d.get(fieldName)+"=id="+d.get("ID")); System.out.println(highlighter.getBestFragment(analyzer, fieldName, d.get(fieldName))); } } catch (ParseException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ if(indexReader != null){ try{ indexReader.close(); }catch (IOException e) { e.printStackTrace(); } } if(directory != null){ try{ directory.close(); }catch (Exception e) { e.printStackTrace(); } } } } }