// 将文档加入索引
//
// 输入参数：
//  docId   标识文档编号，必须唯一
//  data    见DocumentIndexData注释
//
// 注意：
//      1. 这个函数是线程安全的，请尽可能并发调用以提高索引速度
//  2. 这个函数调用是非同步的，也就是说在函数返回时有可能文档还没有加入索引中，因此
//         如果立刻调用Search可能无法查询到这个文档。强制刷新索引请调用FlushIndex函数。
func (engine *Engine) IndexDocument(docId uint64, data types.DocumentIndexData) {if !engine.initialized {log.Fatal("必须先初始化引擎")}atomic.AddUint64(&engine.numIndexingRequests, 1)shard := int(murmur.Murmur3([]byte(fmt.Sprint("%d", docId))) % uint32(engine.initOptions.NumShards))engine.segmenterChannel <- segmenterRequest{docId: docId, shard: shard, data: data}
}

    // 建立分词器使用的通信通道segmenterChannel chan segmenterRequest

func (engine *Engine) segmenterWorker() {for {request := <-engine.segmenterChannel //关键tokensMap := make(map[string][]int)numTokens := 0if !engine.initOptions.NotUsingSegmenter && request.data.Content != "" {// 当文档正文不为空时，优先从内容分词中得到关键词segments := engine.segmenter.Segment([]byte(request.data.Content))for _, segment := range segments {token := segment.Token().Text()if !engine.stopTokens.IsStopToken(token) {tokensMap[token] = append(tokensMap[token], segment.Start())}}numTokens = len(segments)} else {// 否则载入用户输入的关键词for _, t := range request.data.Tokens {if !engine.stopTokens.IsStopToken(t.Text) {tokensMap[t.Text] = t.Locations}}numTokens = len(request.data.Tokens)}// 加入非分词的文档标签for _, label := range request.data.Labels {if !engine.initOptions.NotUsingSegmenter {if !engine.stopTokens.IsStopToken(label) {tokensMap[label] = []int{}}} else {tokensMap[label] = []int{}}}indexerRequest := indexerAddDocumentRequest{document: &types.DocumentIndex{DocId:       request.docId,TokenLength: float32(numTokens),Keywords:    make([]types.KeywordIndex, len(tokensMap)),},}iTokens := 0for k, v := range tokensMap {indexerRequest.document.Keywords[iTokens] = types.KeywordIndex{Text: k,// 非分词标注的词频设置为0，不参与tf-idf计算
                Frequency: float32(len(v)),Starts:    v}iTokens++}var dealDocInfoChan = make(chan bool, 1)indexerRequest.dealDocInfoChan = dealDocInfoChanengine.indexerAddDocumentChannels[request.shard] <- indexerRequestrankerRequest := rankerAddDocRequest{docId:           request.docId,fields:          request.data.Fields,dealDocInfoChan: dealDocInfoChan,}engine.rankerAddDocChannels[request.shard] <- rankerRequest}
}

    // 启动分词器for iThread := 0; iThread < options.NumSegmenterThreads; iThread++ {go engine.segmenterWorker()}

func (engine *Engine) indexerAddDocumentWorker(shard int) {for {request := <-engine.indexerAddDocumentChannels[shard] //关键addInvertedIndex := engine.indexers[shard].AddDocument(request.document, request.dealDocInfoChan)// saveif engine.initOptions.UsePersistentStorage {for k, v := range addInvertedIndex {engine.persistentStorageIndexDocumentChannels[shard] <- persistentStorageIndexDocumentRequest{typ:            "index",keyword:        k,keywordIndices: v,}}}atomic.AddUint64(&engine.numTokenIndexAdded,uint64(len(request.document.Keywords)))atomic.AddUint64(&engine.numDocumentsIndexed, 1)}
}

        // 启动索引器和排序器for shard := 0; shard < options.NumShards; shard++ {go engine.indexerAddDocumentWorker(shard) //关键go engine.indexerRemoveDocWorker(shard)go engine.rankerAddDocWorker(shard)go engine.rankerRemoveDocWorker(shard)for i := 0; i < options.NumIndexerThreadsPerShard; i++ {go engine.indexerLookupWorker(shard)}for i := 0; i < options.NumRankerThreadsPerShard; i++ {go engine.rankerRankWorker(shard)}}

// 向反向索引表中加入一个文档
func (indexer *Indexer) AddDocument(document *types.DocumentIndex, dealDocInfoChan chan<- bool) (addInvertedIndex map[string]*types.KeywordIndices) {if indexer.initialized == false {log.Fatal("索引器尚未初始化")}indexer.InvertedIndexShard.Lock()defer indexer.InvertedIndexShard.Unlock()// 更新文档总数及关键词总长度
    indexer.DocInfosShard.Lock()if _, found := indexer.DocInfosShard.DocInfos[document.DocId]; !found {indexer.DocInfosShard.DocInfos[document.DocId] = new(types.DocInfo)indexer.DocInfosShard.NumDocuments++}if document.TokenLength != 0 {originalLength := indexer.DocInfosShard.DocInfos[document.DocId].TokenLengthsindexer.DocInfosShard.DocInfos[document.DocId].TokenLengths = float32(document.TokenLength)indexer.InvertedIndexShard.TotalTokenLength += document.TokenLength - originalLength}indexer.DocInfosShard.Unlock()close(dealDocInfoChan)// docIdIsNew := truefoundKeyword := falseaddInvertedIndex = make(map[string]*types.KeywordIndices)for _, keyword := range document.Keywords {addInvertedIndex[keyword.Text], foundKeyword = indexer.InvertedIndexShard.InvertedIndex[keyword.Text]if !foundKeyword {addInvertedIndex[keyword.Text] = new(types.KeywordIndices)}indices := addInvertedIndex[keyword.Text]if !foundKeyword {// 如果没找到该搜索键则加入switch indexer.initOptions.IndexType {case types.LocationsIndex:indices.Locations = [][]int{keyword.Starts}case types.FrequenciesIndex:indices.Frequencies = []float32{keyword.Frequency}}indices.DocIds = []uint64{document.DocId}indexer.InvertedIndexShard.InvertedIndex[keyword.Text] = indicescontinue}// 查找应该插入的位置position, found := indexer.searchIndex(indices, 0, indexer.getIndexLength(indices)-1, document.DocId)if found {// docIdIsNew = false// 覆盖已有的索引项switch indexer.initOptions.IndexType {case types.LocationsIndex:indices.Locations[position] = keyword.Startscase types.FrequenciesIndex:indices.Frequencies[position] = keyword.Frequency}continue}// 当索引不存在时，插入新索引项switch indexer.initOptions.IndexType {case types.LocationsIndex:indices.Locations = append(indices.Locations, []int{})copy(indices.Locations[position+1:], indices.Locations[position:])indices.Locations[position] = keyword.Startscase types.FrequenciesIndex:indices.Frequencies = append(indices.Frequencies, float32(0))copy(indices.Frequencies[position+1:], indices.Frequencies[position:])indices.Frequencies[position] = keyword.Frequency}indices.DocIds = append(indices.DocIds, 0)copy(indices.DocIds[position+1:], indices.DocIds[position:])indices.DocIds[position] = document.DocId}return
}

// 二分法查找indices中某文档的索引项
// 第一个返回参数为找到的位置或需要插入的位置
// 第二个返回参数标明是否找到
func (indexer *Indexer) searchIndex( indices *types.KeywordIndices, start int, end int, docId uint64) (int, bool) {// 特殊情况if indexer.getIndexLength(indices) == start {return start, false}if docId < indexer.getDocId(indices, start) {return start, false} else if docId == indexer.getDocId(indices, start) {return start, true}if docId > indexer.getDocId(indices, end) {return end + 1, false} else if docId == indexer.getDocId(indices, end) {return end, true}// 二分var middle intfor end-start > 1 {middle = (start + end) / 2      if docId == indexer.getDocId(indices, middle) {return middle, true} else if docId > indexer.getDocId(indices, middle) {start = middle     } else {end = middle}}return end, false
}