package org.elasticsearch.search.highlight;import com.google.common.collect.Maps;
import org.apache.lucene.search.highlight.*;
import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
import org.apache.lucene.search.vectorhighlight.CustomFieldQuery;
import org.apache.lucene.search.vectorhighlight.FieldQuery;
import org.apache.lucene.search.vectorhighlight.SimpleBoundaryScanner;
import org.apache.lucene.search.vectorhighlight.FieldQuery.Phrase;
import org.apache.lucene.util.BytesRefHash;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.internal.SearchContext;import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;/*** * @author jkuang.nj**/
public class FastPlainHighlighter implements Highlighter
{private static final String CACHE_KEY = "highlight-fast";public static final char mark = 0;private static final SimpleBoundaryScanner DEFAULT_BOUNDARY_SCANNER = new SimpleBoundaryScanner();@Overridepublic HighlightField highlight(HighlighterContext highlighterContext){SearchContextHighlight.Field field = highlighterContext.field;SearchContext context = highlighterContext.context;FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;FieldMapper mapper = highlighterContext.mapper;Encoder encoder = field.fieldOptions().encoder().equals("html") ? HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;if (!hitContext.cache().containsKey(CACHE_KEY)){hitContext.cache().put(CACHE_KEY, new HighlighterEntry());}HighlighterEntry cache = (HighlighterEntry) hitContext.cache().get(CACHE_KEY);try{FieldQuery fieldQuery;if (field.fieldOptions().requireFieldMatch()){if (cache.fieldMatchFieldQuery == null){cache.fieldMatchFieldQuery = new CustomFieldQuery(highlighterContext.query, hitContext.topLevelReader(), true,field.fieldOptions().requireFieldMatch());}fieldQuery = cache.fieldMatchFieldQuery;}else{if (cache.noFieldMatchFieldQuery == null){cache.noFieldMatchFieldQuery = new CustomFieldQuery(highlighterContext.query, hitContext.topLevelReader(), true,field.fieldOptions().requireFieldMatch());}fieldQuery = cache.noFieldMatchFieldQuery;}if (!cache.analysises.containsKey(field.field())){cache.setPhrases(field.field(), fieldQuery.getPhrases(field.field()));cache.setWords(field.field(), fieldQuery.getTermSet(field.field()));}FastHighlighter entry = cache.mappers.get(mapper);if (entry == null){BoundaryScanner boundaryScanner = DEFAULT_BOUNDARY_SCANNER;if (field.fieldOptions().boundaryMaxScan() != SimpleBoundaryScanner.DEFAULT_MAX_SCAN|| field.fieldOptions().boundaryChars() != SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS){boundaryScanner = new SimpleBoundaryScanner(field.fieldOptions().boundaryMaxScan(), field.fieldOptions().boundaryChars());}CustomFieldQuery.highlightFilters.set(field.fieldOptions().highlightFilter());entry = new FastHighlighter(encoder, boundaryScanner);cache.mappers.put(mapper, entry);}String[] fragments;int numberOfFragments = field.fieldOptions().numberOfFragments() == 0 ? 1 : field.fieldOptions().numberOfFragments();int fragmentCharSize = field.fieldOptions().numberOfFragments() == 0 ? 50 : field.fieldOptions().fragmentCharSize();List textsToHighlight = null;try{textsToHighlight = HighlightUtils.loadFieldValues(field, mapper, context, hitContext);StringBuilder buffer = new StringBuilder();for (Object textToHighlight : textsToHighlight){String text = textToHighlight.toString();buffer.append(text).append(" ");}fragments = entry.getBestBestFragments(cache.analysises.get(field.field()), cache.phrases.get(field.field()), buffer,numberOfFragments, fragmentCharSize, field.fieldOptions().preTags(), field.fieldOptions().postTags());}catch (Exception e){e.printStackTrace();if (ExceptionsHelper.unwrap(e, BytesRefHash.MaxBytesLengthExceededException.class) != null){return null;}else{throw new FetchPhaseExecutionException(context, "Failed to highlight field [" + highlighterContext.fieldName + "]", e);}}if (fragments != null && fragments.length > 0){return new HighlightField(highlighterContext.fieldName, Text.convertFromStringArray(fragments));}int noMatchSize = highlighterContext.field.fieldOptions().noMatchSize();if (noMatchSize > 0 && textsToHighlight.size() > 0){String fieldContents = textsToHighlight.get(0).toString();return new HighlightField(highlighterContext.fieldName,new Text[] { new Text(fieldContents.substring(0, Math.min(fragmentCharSize, fieldContents.length()))) });}return null;}catch (Exception e){throw new FetchPhaseExecutionException(context, "Failed to highlight field [" + highlighterContext.fieldName + "]", e);}}@Overridepublic boolean canHighlight(FieldMapper fieldMapper){return true;}private class HighlighterEntry{public FieldQuery noFieldMatchFieldQuery;public FieldQuery fieldMatchFieldQuery;public Map> phrases = new HashMap<>();public Mapmappers = Maps.newHashMap();public Mapanalysises = new HashMap<>();public void setPhrases(String field, Setphrases) { if(!this.phrases.containsKey(field)){ this.phrases.put(field, phrases); } } public void setWords(String field, Setwords) { if (!analysises.containsKey(field)) { TreeAnalysis analysis = new TreeAnalysis(); if (words != null && words.size() > 0) { for (String word : words) { analysis.add(word); } } analysises.put(field, analysis); } } } static class FragmentScore implements Comparable{ int point = 0; int distance = 0; Listterms = new ArrayList<>(); HashSetset = new HashSet<>(); StringBuffer buffer = new StringBuffer(); public FragmentScore(int distance) { this.distance = distance; } public void updateScore(Setphrases) { for (Phrase phrase : phrases) { if (buffer.indexOf(phrase.toString()) >= 0) { this.point += 5 * phrase.list.size(); } } } public boolean add(Term term) { if (terms.size() == 0 || term.pos - terms.get(0).pos <= distance) { if (terms.size() == 0) { buffer.append(term.word); } else { int dis = term.pos - terms.get(terms.size() - 1).pos; buffer.append(mark).append(dis).append(mark); buffer.append(term.word); } terms.add(term); this.point += term.length(); if (set.size() > 0) { if (!set.contains(term.word)) { this.point += 2; set.add(term.word); } if (term.pos - terms.get(terms.size() - 1).pos == 1) { this.point += 2; } } return true; } return false; } @Override public int compareTo(FragmentScore o) { return -(this.point - o.point); } } public class FastHighlighter { BoundaryScanner boundaryScanner; Encoder encoder; public FastHighlighter(Encoder encoder, BoundaryScanner boundaryScanner) { this.encoder = encoder; this.boundaryScanner = boundaryScanner; } public String[] getBestBestFragments(TreeAnalysis analyzer, Setphrases, StringBuilder buffer, int maxNumFragments, int fragmentSize, String[] preTags, String[] postTags) { ListfragmentScores; if (maxNumFragments <= 1) { fragmentScores = getBestFragments(analyzer, phrases, buffer.toString(), fragmentSize); } else { fragmentScores = getBestFragments(analyzer, buffer.toString(), maxNumFragments, fragmentSize); } return toString(buffer, fragmentSize, fragmentScores, preTags, postTags); } public String[] toString(StringBuilder buffer, int fragmentSize, ListfragmentScores, String[] preTags, String[] postTags) { Listlist = new ArrayList<>(); for (FragmentScore score : fragmentScores) { Listterms = score.terms; Term head = terms.get(0); Term tail = terms.get(terms.size() - 1); int start = boundaryScanner.findStartOffset(buffer, head.startoffset); int end = boundaryScanner.findEndOffset(buffer, tail.endoffset()); if (fragmentScores.size() == 1 && buffer.length() <= fragmentSize) { start = 0; end = buffer.length(); } else if (fragmentSize - (tail.endoffset() - head.startoffset) > (fragmentSize / 10)) { int size = fragmentSize - (tail.endoffset() - head.startoffset); if (head.startoffset < (size * 3 / 10)) { start = 0; } else { start = boundaryScanner.findStartOffset(buffer, head.startoffset); } if (buffer.length() - start < fragmentSize) { end = buffer.length(); } else { end = boundaryScanner.findEndOffset(buffer, Math.max(start + fragmentSize, tail.endoffset())); } } StringBuffer result = new StringBuffer(); for (int i = 0; i < terms.size(); i++) { Term term = terms.get(i); result.append(buffer.substring(start, term.startoffset)); result.append(getTag(preTags, i)); result.append(encoder.encodeText(buffer.substring(term.startoffset, term.endoffset()))); result.append(getTag(postTags, i)); start = term.endoffset(); } result.append(buffer.substring(start, end)); list.add(result.toString()); } return list.toArray(new String[0]); } public final ListgetBestFragments(TreeAnalysis analyzer, Setphrases, String text, int fragmentSize) { if (analyzer == null) { return new ArrayList<>(); } Listfragments = new ArrayList(); FragmentScore fragmentScore = null; Listterms = analyzer.find(text); for (int i = 0, j = 0; i < terms.size(); i++) { FragmentScore fScore = new FragmentScore(fragmentSize); for (j = i; j < terms.size(); j++) { if (!fScore.add(terms.get(j))) { break; } } fScore.updateScore(phrases); if (fragmentScore == null || fragmentScore.compareTo(fScore) >= 0) { fragmentScore = fScore; } if (j >= terms.size()) { break; } } if (fragmentScore != null) { fragments.add(fragmentScore); } return fragments; } public final ListgetBestFragments(TreeAnalysis analyzer, String text, int maxNumFragments, int fragmentSize) { if (analyzer == null) { return null; } Listterms = analyzer.find(text); Listfragments = new ArrayList(); FragmentScore fScore = new FragmentScore(fragmentSize); for (int i = 0; i < terms.size(); i++) { if (!fScore.add(terms.get(i))) { fragments.add(fScore); fScore = new FragmentScore(fragmentSize); fScore.add(terms.get(i)); } } fragments.add(fScore); Collections.sort(fragments); while (fragments.size() > maxNumFragments) { fragments.remove(fragments.size() - 1); } return fragments; } protected String getTag(String[] tags, int num) { int n = num % tags.length; return tags[n]; } } public static class Term { String word; int startoffset, pos; public Term(int startoffset, int pos, String word) { this.startoffset = startoffset; this.pos = pos; this.word = word; } public int endoffset() { return this.startoffset+word.length(); } public int length() { return word.length(); } public String toString() { return "start:" + startoffset + " pos:" + pos+" word:"+word; } } public static class TreeAnalysis { private TNode root = new TNode((char) 0, false); boolean[] nodes = new boolean[64 * 1024]; static final char ch0 ='\uFF00'; static final char ch1 ='\uFF5F'; public Listfind(String str) { int start = 0; int length = str.length(); str = str.toLowerCase(); char[] values = str.toCharArray(); Listterms = new ArrayList<>(); int sumpos = 0; while (start < length) { char ch = values[start]; //全椒字符串换为半角字符 ch= (char) (ch > ch0 && ch < ch1 ? ch - 65248 :ch); if (!nodes[ch]) { start++; continue; } else { int pos = root.find(values, start, -1); if (pos >= start) { terms.add(new Term(start, start - sumpos + terms.size(), str.substring(start, pos + 1))); sumpos += pos + 1 - start; start = pos + 1; } else { start++; } } } return terms; } public void add(String str) { if (str == null || str.length() == 0) { return; } str = str.toLowerCase(); nodes[(int)str.charAt(0)] = true; root.insert(str, 0); } private static class TNode implements Comparable{ // 标记当前节点是否是一个词的终止字符 boolean mark; // 当前节点的字符 char value; // 子节点 TNode[] nodes; int nodesize; public TNode(char ch, boolean mark) { this.value = ch; this.mark = mark; } public int find(char[] chs, int nextPos, int leafoffset) { if (nextPos >= chs.length) { return -1; } int size = 0; char ch = chs[nextPos]; //全椒字符串换为半角字符 ch= (char) (ch > ch0 && ch < ch1 ? ch - 65248 :ch); while (size < this.nodesize && nodes[size++].value < ch); int pos = nodes[size - 1].value == ch ? size - 1 : -1; // int pos = index(chs[nextPos]); if (pos >= 0) { if (nodes[pos].mark) { leafoffset = nextPos; if (nodes[pos].nodesize == 0) { return nextPos; } } int next = nodes[pos].find(chs, nextPos + 1, leafoffset); return next > leafoffset ? next : leafoffset; } else { return -1; } } /*public int index(char ch) { if (this.nodesize < 5) { int size = 0; while (size < this.nodesize && nodes[size++].value < ch) ; return nodes[size - 1].value == ch ? size - 1 : -1; } else { return indexOf(nodes, this.nodesize, ch, Type._index); } }*/ int indexOf(TNode[] nodes, int size, char node, Type type) { int fromIndex = 0; int toIndex = size - 1; while (fromIndex <= toIndex) { int mid = (fromIndex + toIndex) >> 1; int cmp = nodes[mid].compareTo(node);// this.comparator.compare(nodes[mid], // node); if (cmp < 0) fromIndex = mid + 1; else if (cmp > 0) toIndex = mid - 1; else return type == Type._insert ? -(mid + 1) : mid; // key // found } switch (type) { case _insert: return fromIndex; case _index: return -(fromIndex + 1); default: return toIndex; } } public void insert(String str, int pos) { char ch = str.charAt(pos); boolean isleaf = pos == str.length() - 1; if (this.nodesize == 0) { nodes = new TNode[1]; nodes[0] = new TNode(ch, isleaf); if (!isleaf) { nodes[0].insert(str, pos + 1); } this.nodesize++; } else { int _index = indexOf(nodes, nodesize, ch, Type._insert); if (_index >= 0) { int moved = this.nodesize - _index; if (this.nodesize == nodes.length) { nodes = Arrays.copyOf(nodes, nodes.length + 1); } if (moved > 0) { System.arraycopy(nodes, _index, nodes, _index + 1, moved); } nodes[_index] = new TNode(ch, isleaf); if (!isleaf) { nodes[_index].insert(str, pos + 1); } this.nodesize++; } else { if (isleaf) { nodes[0].mark = true; } else { nodes[-_index - 1].insert(str, pos + 1); } } } } @Override public int compareTo(TNode o) { if (this.value > o.value) { return 1; } else if (this.value < o.value) { return -1; } return 0; } public int compareTo(char o) { if (this.value > o) { return 1; } else if (this.value < o) { return -1; } return 0; } enum Type { _insert, _index } } } } /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.lucene.search.vectorhighlight; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.queries.CustomScoreQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo; /** * FieldQuery breaks down query object into terms/phrases and keeps * them in a QueryPhraseMap structure. */ public class FieldQuery { final boolean fieldMatch; // fieldMatch==true, Map// fieldMatch==false, MapMaprootMaps = new HashMap<>(); // fieldMatch==true, Map// fieldMatch==false, MapMap> termSetMap = new HashMap<>(); //存储短语 Map> phraseMap = new HashMap<>(); int termOrPhraseNumber; // used for colored tag support // The maximum number of different matching terms accumulated from any one MultiTermQuery private static final int MAX_MTQ_TERMS = 1024; public static class Phrase { public Listlist = new ArrayList<>(); StringBuffer buffer = new StringBuffer(); public void add(String word,int position) { if(list.size()==0){ buffer.append(word); }else{ int pos= position-list.get(list.size()-1).position; char z = 0; buffer.append(z).append(pos).append(z); buffer.append(word); } list.add(new Term(position, word)); } public String toString() { return buffer.toString(); } public static class Term{ public int position; public String word; public Term(int position, String word) { this.position = position; this.word = word; } } } protected FieldQuery( Query query, IndexReader reader, boolean phraseHighlight, boolean fieldMatch ) throws IOException { this.fieldMatch = fieldMatch; SetflatQueries = new LinkedHashSet<>(); flatten( query, reader, flatQueries, 1f ); saveTerms( flatQueries, reader ); CollectionexpandQueries = expand( flatQueries ); for( Query flatQuery : expandQueries ){ QueryPhraseMap rootMap = getRootMap( flatQuery ); rootMap.add( flatQuery, reader ); float boost = 1f; while (flatQuery instanceof BoostQuery) { BoostQuery bq = (BoostQuery) flatQuery; flatQuery = bq.getQuery(); boost *= bq.getBoost(); } if( !phraseHighlight && flatQuery instanceof PhraseQuery ){ PhraseQuery pq = (PhraseQuery)flatQuery; if( pq.getTerms().length > 1 ){ for( Term term : pq.getTerms() ) rootMap.addTerm( term, boost ); } } } } /** For backwards compatibility you can initialize FieldQuery without * an IndexReader, which is only required to support MultiTermQuery */ FieldQuery( Query query, boolean phraseHighlight, boolean fieldMatch ) throws IOException { this (query, null, phraseHighlight, fieldMatch); } void flatten( Query sourceQuery, IndexReader reader, CollectionflatQueries, float boost ) throws IOException{ while (true) { if (sourceQuery.getBoost() != 1f) { boost *= sourceQuery.getBoost(); sourceQuery = sourceQuery.clone(); sourceQuery.setBoost(1f); } else if (sourceQuery instanceof BoostQuery) { BoostQuery bq = (BoostQuery) sourceQuery; sourceQuery = bq.getQuery(); boost *= bq.getBoost(); } else { break; } } if( sourceQuery instanceof BooleanQuery ){ BooleanQuery bq = (BooleanQuery)sourceQuery; for( BooleanClause clause : bq ) { if( !clause.isProhibited() ) { flatten( clause.getQuery(), reader, flatQueries, boost ); } } } else if( sourceQuery instanceof DisjunctionMaxQuery ){ DisjunctionMaxQuery dmq = (DisjunctionMaxQuery)sourceQuery; for( Query query : dmq ){ flatten( query, reader, flatQueries, boost ); } } else if( sourceQuery instanceof TermQuery ){ if (boost != 1f) { sourceQuery = new BoostQuery(sourceQuery, boost); } if( !flatQueries.contains( sourceQuery ) ) flatQueries.add( sourceQuery ); } else if( sourceQuery instanceof PhraseQuery ){ PhraseQuery pq = (PhraseQuery)sourceQuery; if( pq.getTerms().length == 1 ) sourceQuery = new TermQuery( pq.getTerms()[0] ); if (boost != 1f) { sourceQuery = new BoostQuery(sourceQuery, boost); } flatQueries.add(sourceQuery); } else if (sourceQuery instanceof ConstantScoreQuery) { final Query q = ((ConstantScoreQuery) sourceQuery).getQuery(); if (q != null) { flatten( q, reader, flatQueries, boost); } } else if (sourceQuery instanceof FilteredQuery) { final Query q = ((FilteredQuery) sourceQuery).getQuery(); if (q != null) { flatten( q, reader, flatQueries, boost); } } else if (sourceQuery instanceof CustomScoreQuery) { final Query q = ((CustomScoreQuery) sourceQuery).getSubQuery(); if (q != null) { flatten( q, reader, flatQueries, boost); } } else if (reader != null) { Query query = sourceQuery; Query rewritten; if (sourceQuery instanceof MultiTermQuery) { rewritten = new MultiTermQuery.TopTermsScoringBooleanQueryRewrite(MAX_MTQ_TERMS).rewrite(reader, (MultiTermQuery) query); } else { rewritten = query.rewrite(reader); } if (rewritten != query) { // only rewrite once and then flatten again - the rewritten query could have a speacial treatment // if this method is overwritten in a subclass. flatten(rewritten, reader, flatQueries, boost); } // if the query is already rewritten we discard it } // else discard queries } /* * Create expandQueries from flatQueries. * * expandQueries := flatQueries + overlapped phrase queries * * ex1) flatQueries={a,b,c} * => expandQueries={a,b,c} * ex2) flatQueries={a,"b c","c d"} * => expandQueries={a,"b c","c d","b c d"} */ Collectionexpand( CollectionflatQueries ){ SetexpandQueries = new LinkedHashSet<>(); for( Iteratori = flatQueries.iterator(); i.hasNext(); ){ Query query = i.next(); i.remove(); expandQueries.add( query ); float queryBoost = 1f; while (query instanceof BoostQuery) { BoostQuery bq = (BoostQuery) query; queryBoost *= bq.getBoost(); query = bq.getQuery(); } if( !( query instanceof PhraseQuery ) ) continue; for( Iteratorj = flatQueries.iterator(); j.hasNext(); ){ Query qj = j.next(); float qjBoost = 1f; while (qj instanceof BoostQuery) { BoostQuery bq = (BoostQuery) qj; qjBoost *= bq.getBoost(); qj = bq.getQuery(); } if( !( qj instanceof PhraseQuery ) ) continue; checkOverlap( expandQueries, (PhraseQuery)query, queryBoost, (PhraseQuery)qj, qjBoost ); } } return expandQueries; } /* * Check if PhraseQuery A and B have overlapped part. * * ex1) A="a b", B="b c" => overlap; expandQueries={"a b c"} * ex2) A="b c", B="a b" => overlap; expandQueries={"a b c"} * ex3) A="a b", B="c d" => no overlap; expandQueries={} */ private void checkOverlap( CollectionexpandQueries, PhraseQuery a, float aBoost, PhraseQuery b, float bBoost ){ if( a.getSlop() != b.getSlop() ) return; Term[] ats = a.getTerms(); Term[] bts = b.getTerms(); if( fieldMatch && !ats[0].field().equals( bts[0].field() ) ) return; checkOverlap( expandQueries, ats, bts, a.getSlop(), aBoost); checkOverlap( expandQueries, bts, ats, b.getSlop(), bBoost ); } /* * Check if src and dest have overlapped part and if it is, create PhraseQueries and add expandQueries. * * ex1) src="a b", dest="c d" => no overlap * ex2) src="a b", dest="a b c" => no overlap * ex3) src="a b", dest="b c" => overlap; expandQueries={"a b c"} * ex4) src="a b c", dest="b c d" => overlap; expandQueries={"a b c d"} * ex5) src="a b c", dest="b c" => no overlap * ex6) src="a b c", dest="b" => no overlap * ex7) src="a a a a", dest="a a a" => overlap; * expandQueries={"a a a a a","a a a a a a"} * ex8) src="a b c d", dest="b c" => no overlap */ private void checkOverlap( CollectionexpandQueries, Term[] src, Term[] dest, int slop, float boost ){ // beginning from 1 (not 0) is safe because that the PhraseQuery has multiple terms // is guaranteed in flatten() method (if PhraseQuery has only one term, flatten() // converts PhraseQuery to TermQuery) for( int i = 1; i < src.length; i++ ){ boolean overlap = true; for( int j = i; j < src.length; j++ ){ if( ( j - i ) < dest.length && !src[j].text().equals( dest[j-i].text() ) ){ overlap = false; break; } } if( overlap && src.length - i < dest.length ){ PhraseQuery.Builder pqBuilder = new PhraseQuery.Builder(); for( Term srcTerm : src ) pqBuilder.add( srcTerm ); for( int k = src.length - i; k < dest.length; k++ ){ pqBuilder.add( new Term( src[0].field(), dest[k].text() ) ); } pqBuilder.setSlop( slop ); Query pq = pqBuilder.build(); if (boost != 1f) { pq = new BoostQuery(pq, 1f); } if(!expandQueries.contains( pq ) ) expandQueries.add( pq ); } } } QueryPhraseMap getRootMap( Query query ){ String key = getKey( query ); QueryPhraseMap map = rootMaps.get( key ); if( map == null ){ map = new QueryPhraseMap( this ); rootMaps.put( key, map ); } return map; } /* * Return 'key' string. 'key' is the field name of the Query. * If not fieldMatch, 'key' will be null. */ private String getKey( Query query ){ if( !fieldMatch ) return null; while (query instanceof BoostQuery) { query = ((BoostQuery) query).getQuery(); } if( query instanceof TermQuery ) return ((TermQuery)query).getTerm().field(); else if ( query instanceof PhraseQuery ){ PhraseQuery pq = (PhraseQuery)query; Term[] terms = pq.getTerms(); return terms[0].field(); } else if (query instanceof MultiTermQuery) { return ((MultiTermQuery)query).getField(); } else throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." ); } /* * Save the set of terms in the queries to termSetMap. * * ex1) q=name:john * - fieldMatch==true * termSetMap=Map<"name",Set<"john">> * - fieldMatch==false * termSetMap=Map<"john">> * * ex2) q=name:john title:manager * - fieldMatch==true * termSetMap=Map<"name",Set<"john">, * "title",Set<"manager">> * - fieldMatch==false * termSetMap=Map<"john","manager">> * * ex3) q=name:"john lennon" * - fieldMatch==true * termSetMap=Map<"name",Set<"john","lennon">> * - fieldMatch==false * termSetMap=Map<"john","lennon">> */ void saveTerms( CollectionflatQueries, IndexReader reader ) throws IOException{ for( Query query : flatQueries ){ while (query instanceof BoostQuery) { query = ((BoostQuery) query).getQuery(); } Setterms = getTerms( query ); SettermSet = getTermSet( query ); if( query instanceof TermQuery ){ termSet.add( ((TermQuery)query).getTerm().text() ); } else if( query instanceof PhraseQuery ){ int[] positions=((PhraseQuery)query).getPositions(); Term[] terms2 =((PhraseQuery)query).getTerms(); Phrase phrase = new Phrase(); for (int i = 0; i < terms2.length; i++) { phrase.add(terms2[i].text(), positions[i]); termSet.add( terms2[i].text() ); } if(terms2.length > 1){ terms.add(phrase); } } else if (query instanceof MultiTermQuery && reader != null) { BooleanQuery mtqTerms = (BooleanQuery) query.rewrite(reader); for (BooleanClause clause : mtqTerms) { termSet.add (((TermQuery) clause.getQuery()).getTerm().text()); } } else throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." ); } } private SetgetTermSet( Query query ){ String key = getKey( query ); Setset = termSetMap.get( key ); if( set == null ){ set = new HashSet<>(); termSetMap.put( key, set ); } return set; } //用fastHighlighter private SetgetTerms( Query query ){ String key = getKey( query ); Setset = phraseMap.get( key ); if( set == null ){ set = new HashSet<>(); phraseMap.put( key, set ); } return set; } public SetgetTermSet( String field ){ return termSetMap.get( fieldMatch ? field : null ); } /** * 短语不进行分词 * @param field * @return */ public SetgetPhrases( String field ){ return phraseMap.get( fieldMatch ? field : null ); } /** * * @return QueryPhraseMap */ public QueryPhraseMap getFieldTermMap( String fieldName, String term ){ QueryPhraseMap rootMap = getRootMap( fieldName ); return rootMap == null ? null : rootMap.subMap.get( term ); } /** * * @return QueryPhraseMap */ public QueryPhraseMap searchPhrase( String fieldName, final ListphraseCandidate ){ QueryPhraseMap root = getRootMap( fieldName ); if( root == null ) return null; return root.searchPhrase( phraseCandidate ); } private QueryPhraseMap getRootMap( String fieldName ){ return rootMaps.get( fieldMatch ? fieldName : null ); } public int nextTermOrPhraseNumber(){ return termOrPhraseNumber++; } /** * Internal structure of a query for highlighting: represents * a nested query structure */ public static class QueryPhraseMap { boolean terminal; int slop; // valid if terminal == true and phraseHighlight == true float boost; // valid if terminal == true int termOrPhraseNumber; // valid if terminal == true FieldQuery fieldQuery; MapsubMap = new HashMap<>(); public QueryPhraseMap( FieldQuery fieldQuery ){ this.fieldQuery = fieldQuery; } void addTerm( Term term, float boost ){ QueryPhraseMap map = getOrNewMap( subMap, term.text() ); map.markTerminal( boost ); } private QueryPhraseMap getOrNewMap( MapsubMap, String term ){ QueryPhraseMap map = subMap.get( term ); if( map == null ){ map = new QueryPhraseMap( fieldQuery ); subMap.put( term, map ); } return map; } void add( Query query, IndexReader reader ) { float boost = 1f; while (query instanceof BoostQuery) { BoostQuery bq = (BoostQuery) query; query = bq.getQuery(); boost = bq.getBoost(); } if( query instanceof TermQuery ){ addTerm( ((TermQuery)query).getTerm(), boost ); } else if( query instanceof PhraseQuery ){ PhraseQuery pq = (PhraseQuery)query; Term[] terms = pq.getTerms(); Mapmap = subMap; QueryPhraseMap qpm = null; for( Term term : terms ){ qpm = getOrNewMap( map, term.text() ); map = qpm.subMap; } qpm.markTerminal( pq.getSlop(), boost ); } else throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." ); } public QueryPhraseMap getTermMap( String term ){ return subMap.get( term ); } private void markTerminal( float boost ){ markTerminal( 0, boost ); } private void markTerminal( int slop, float boost ){ this.terminal = true; this.slop = slop; this.boost = boost; this.termOrPhraseNumber = fieldQuery.nextTermOrPhraseNumber(); } public boolean isTerminal(){ return terminal; } public int getSlop(){ return slop; } public float getBoost(){ return boost; } public int getTermOrPhraseNumber(){ return termOrPhraseNumber; } public QueryPhraseMap searchPhrase( final ListphraseCandidate ){ QueryPhraseMap currMap = this; for( TermInfo ti : phraseCandidate ){ currMap = currMap.subMap.get( ti.getText() ); if( currMap == null ) return null; } return currMap.isValidTermOrPhrase( phraseCandidate ) ? currMap : null; } public boolean isValidTermOrPhrase( final ListphraseCandidate ){ // check terminal if( !terminal ) return false; // if the candidate is a term, it is valid if( phraseCandidate.size() == 1 ) return true; // else check whether the candidate is valid phrase // compare position-gaps between terms to slop int pos = phraseCandidate.get( 0 ).getPosition(); for( int i = 1; i < phraseCandidate.size(); i++ ){ int nextPos = phraseCandidate.get( i ).getPosition(); if( Math.abs( nextPos - pos - 1 ) > slop ) return false; pos = nextPos; } return true; } } }