// 查找满足搜索条件的文档，此函数线程安全
func (engine *Engine) Search(request types.SearchRequest) (output types.SearchResponse) {if !engine.initialized {log.Fatal("必须先初始化引擎")}var rankOptions types.RankOptionsif request.RankOptions == nil {rankOptions = *engine.initOptions.DefaultRankOptions} else {rankOptions = *request.RankOptions}if rankOptions.ScoringCriteria == nil {rankOptions.ScoringCriteria = engine.initOptions.DefaultRankOptions.ScoringCriteria}// 收集关键词tokens := []string{}if request.Text != "" {querySegments := engine.segmenter.Segment([]byte(request.Text))for _, s := range querySegments {token := s.Token().Text()if !engine.stopTokens.IsStopToken(token) {tokens = append(tokens, s.Token().Text())}}} else {for _, t := range request.Tokens {tokens = append(tokens, t)}}// 建立排序器返回的通信通道rankerReturnChannel := make(chan rankerReturnRequest, engine.initOptions.NumShards)// 生成查找请求lookupRequest := indexerLookupRequest{countDocsOnly:       request.CountDocsOnly,tokens:              tokens,labels:              request.Labels,docIds:              request.DocIds,options:             rankOptions,rankerReturnChannel: rankerReturnChannel,orderless:           request.Orderless,}// 向索引器发送查找请求for shard := 0; shard < engine.initOptions.NumShards; shard++ {engine.indexerLookupChannels[shard] <- lookupRequest}// 从通信通道读取排序器的输出numDocs := 0rankOutput := types.ScoredDocuments{}timeout := request.TimeoutisTimeout := falseif timeout <= 0 {// 不设置超时for shard := 0; shard < engine.initOptions.NumShards; shard++ {rankerOutput := <-rankerReturnChannelif !request.CountDocsOnly {for _, doc := range rankerOutput.docs {rankOutput = append(rankOutput, doc)}}numDocs += rankerOutput.numDocs}} else {// 设置超时deadline := time.Now().Add(time.Millisecond * time.Duration(request.Timeout))for shard := 0; shard < engine.initOptions.NumShards; shard++ {select {case rankerOutput := <-rankerReturnChannel:if !request.CountDocsOnly {for _, doc := range rankerOutput.docs {rankOutput = append(rankOutput, doc)}}numDocs += rankerOutput.numDocscase <-time.After(deadline.Sub(time.Now())):isTimeout = truebreak}}}// 再排序if !request.CountDocsOnly && !request.Orderless {if rankOptions.ReverseOrder {sort.Sort(sort.Reverse(rankOutput))} else {sort.Sort(rankOutput)}}// 准备输出output.Tokens = tokens// 仅当CountDocsOnly为false时才充填output.Docsif !request.CountDocsOnly {if request.Orderless {// 无序状态无需对Offset截断output.Docs = rankOutput} else {var start, end intif rankOptions.MaxOutputs == 0 {start = utils.MinInt(rankOptions.OutputOffset, len(rankOutput))end = len(rankOutput)} else {start = utils.MinInt(rankOptions.OutputOffset, len(rankOutput))end = utils.MinInt(start+rankOptions.MaxOutputs, len(rankOutput))}output.Docs = rankOutput[start:end]}}output.NumDocs = numDocsoutput.Timeout = isTimeoutreturn
}

func (engine *Engine) indexerLookupWorker(shard int) {for {request := <-engine.indexerLookupChannels[shard] // 关键var docs []types.IndexedDocumentvar numDocs int        if request.docIds == nil {      docs, numDocs = engine.indexers[shard].Lookup(request.tokens, request.labels, nil, request.countDocsOnly)} else {               docs, numDocs = engine.indexers[shard].Lookup(request.tokens, request.labels, request.docIds, request.countDocsOnly)}if request.countDocsOnly {      request.rankerReturnChannel <- rankerReturnRequest{numDocs: numDocs}continue           }if len(docs) == 0 {    request.rankerReturnChannel <- rankerReturnRequest{}continue           }                      if request.orderless { var outputDocs []types.ScoredDocumentfor _, d := range docs {        outputDocs = append(outputDocs, types.ScoredDocument{DocId: d.DocId,                 TokenSnippetLocations: d.TokenSnippetLocations,TokenLocations:        d.TokenLocations})}request.rankerReturnChannel <- rankerReturnRequest{docs:    outputDocs,numDocs: len(outputDocs),}continue}rankerRequest := rankerRankRequest{countDocsOnly:       request.countDocsOnly,docs:                docs,options:             request.options,rankerReturnChannel: request.rankerReturnChannel,}engine.rankerRankChannels[shard] <- rankerRequest}
}

// 查找包含全部搜索键(AND操作)的文档
// 当docIds不为nil时仅从docIds指定的文档中查找
func (indexer *Indexer) Lookup(tokens []string, labels []string, docIds map[uint64]bool, countDocsOnly bool) (docs []types.IndexedDocument, numDocs int) {if indexer.initialized == false {log.Fatal("索引器尚未初始化")}indexer.DocInfosShard.RLock()defer indexer.DocInfosShard.RUnlock()if indexer.DocInfosShard.NumDocuments == 0 {return}numDocs = 0// 合并关键词和标签为搜索键keywords := make([]string, len(tokens)+len(labels))copy(keywords, tokens)copy(keywords[len(tokens):], labels)indexer.InvertedIndexShard.RLock()table := make([]*types.KeywordIndices, len(keywords))for i, keyword := range keywords {indices, found := indexer.InvertedIndexShard.InvertedIndex[keyword]if !found {// 当反向索引表中无此搜索键时直接返回
            indexer.InvertedIndexShard.RUnlock()return} else {// 否则加入反向表中table[i] = indices}// 当没有找到时直接返回if len(table) == 0 {indexer.InvertedIndexShard.RUnlock()return}// 归并查找各个搜索键出现文档的交集// 从后向前查保证先输出DocId较大文档indexPointers := make([]int, len(table))for iTable := 0; iTable < len(table); iTable++ {indexPointers[iTable] = indexer.getIndexLength(table[iTable]) - 1}// 平均文本关键词长度，用于计算BM25avgDocLength := indexer.InvertedIndexShard.TotalTokenLength / float32(indexer.DocInfosShard.NumDocuments)indexer.InvertedIndexShard.RUnlock()for ; indexPointers[0] >= 0; indexPointers[0]-- {// 以第一个搜索键出现的文档作为基准，并遍历其他搜索键搜索同一文档baseDocId := indexer.getDocId(table[0], indexPointers[0])// 全局范围查找目标文档是否存在if _, ok := indexer.DocInfosShard.DocInfos[baseDocId]; !ok {// if !IsDocExist(baseDocId) {// 文档信息中不存在反向索引文档时，跳过// 该情况由不对称删除操作所造成continue}if docIds != nil {_, found := docIds[baseDocId]if !found {continue}}iTable := 1found := truefor ; iTable < len(table); iTable++ {// 二分法比简单的顺序归并效率高，也有更高效率的算法，// 但顺序归并也许是更好的选择，考虑到将来需要用链表重新实现// 以避免反向表添加新文档时的写锁。// TODO: 进一步研究不同求交集算法的速度和可扩展性。position, foundBaseDocId := indexer.searchIndex(table[iTable],0, indexPointers[iTable], baseDocId)if foundBaseDocId {indexPointers[iTable] = position} else {if position == 0 {// 该搜索键中所有的文档ID都比baseDocId大，因此已经没有// 继续查找的必要。return} else {// 继续下一indexPointers[0]的查找indexPointers[iTable] = position - 1found = falsebreak}}}if found {indexedDoc := types.IndexedDocument{}// 当为LocationsIndex时计算关键词紧邻距离if indexer.initOptions.IndexType == types.LocationsIndex {// 计算有多少关键词是带有距离信息的numTokensWithLocations := 0for i, t := range table[:len(tokens)] {if len(t.Locations[indexPointers[i]]) > 0 {numTokensWithLocations++}}if numTokensWithLocations != len(tokens) {if !countDocsOnly {docs = append(docs, types.IndexedDocument{DocId: baseDocId,})}numDocs++break}// 计算搜索键在文档中的紧邻距离tokenProximity, tokenLocations := computeTokenProximity(table[:len(tokens)], indexPointers, tokens)indexedDoc.TokenProximity = int32(tokenProximity)indexedDoc.TokenSnippetLocations = tokenLocations// 添加TokenLocationsindexedDoc.TokenLocations = make([][]int, len(tokens))for i, t := range table[:len(tokens)] {indexedDoc.TokenLocations[i] = t.Locations[indexPointers[i]]}}// 当为LocationsIndex或者FrequenciesIndex时计算BM25if indexer.initOptions.IndexType == types.LocationsIndex ||indexer.initOptions.IndexType == types.FrequenciesIndex {bm25 := float32(0)d := indexer.DocInfosShard.DocInfos[baseDocId].TokenLengthsfor i, t := range table[:len(tokens)] {var frequency float32if indexer.initOptions.IndexType == types.LocationsIndex {frequency = float32(len(t.Locations[indexPointers[i]]))} else {frequency = t.Frequencies[indexPointers[i]]}// 计算BM25if len(t.DocIds) > 0 && frequency > 0 && indexer.initOptions.BM25Parameters != nil && avgDocLength != 0 {// 带平滑的idfidf := float32(math.Log2(float64(indexer.DocInfosShard.NumDocuments)/float64(len(t.DocIds)) + 1))k1 := indexer.initOptions.BM25Parameters.K1b := indexer.initOptions.BM25Parameters.Bbm25 += idf * frequency * (k1 + 1) / (frequency + k1*(1-b+b*d/avgDocLength))}}indexedDoc.BM25 = float32(bm25)}indexedDoc.DocId = baseDocIdif !countDocsOnly {docs = append(docs, indexedDoc)}numDocs++}}return
}