bleve搜索引擎源码分析之索引——mapping真复杂啊
接下来看看下面index部分的源码实现:
data := struct {Name stringDes string}{Name: "hello world this is bone",Des: "this is a good time",}// index some dataindex.Index("id", data)
其中,
index.Index("id", data)
实现代码:
// Index adds the specified index operation to the // batch. NOTE: the bleve Index is not updated // until the batch is executed. func (b *Batch) Index(id string, data interface{}) error {if id == "" {return ErrorEmptyID}doc := document.NewDocument(id)err := b.index.Mapping().MapDocument(doc, data)if err != nil {return err}b.internal.Update(doc)return nil }
根据mapping来映射文档,
b.index.Mapping().MapDocument(doc, data)
该代码的实现:
func (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{}) error {docType := im.determineType(data)docMapping := im.mappingForType(docType)walkContext := im.newWalkContext(doc, docMapping)if docMapping.Enabled {docMapping.walkDocument(data, []string{}, []uint64{}, walkContext)// see if the _all field was disabledallMapping := docMapping.documentMappingForPath("_all")if allMapping == nil || (allMapping.Enabled != false) {field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors)doc.AddField(field)}}return nil }
func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes []uint64, context *walkContext) {// allow default "json" tag to be overridenstructTagKey := dm.StructTagKeyif structTagKey == "" {structTagKey = "json"}val := reflect.ValueOf(data)typ := val.Type()switch typ.Kind() {case reflect.Map:// FIXME can add support for other map keys in the futureif typ.Key().Kind() == reflect.String {for _, key := range val.MapKeys() {fieldName := key.String()fieldVal := val.MapIndex(key).Interface()dm.processProperty(fieldVal, append(path, fieldName), indexes, context)}}case reflect.Struct:for i := 0; i < val.NumField(); i++ {field := typ.Field(i)fieldName := field.Name// anonymous fields of type struct can elide the type nameif field.Anonymous && field.Type.Kind() == reflect.Struct {fieldName = ""}// if the field has a name under the specified tag, prefer thattag := field.Tag.Get(structTagKey)tagFieldName := parseTagName(tag)if tagFieldName == "-" {continue}// allow tag to set field name to empty, only if anonymousif field.Tag != "" && (tagFieldName != "" || field.Anonymous) {fieldName = tagFieldName}if val.Field(i).CanInterface() {fieldVal := val.Field(i).Interface()newpath := pathif fieldName != "" {newpath = append(path, fieldName)}dm.processProperty(fieldVal, newpath, indexes, context)}}case reflect.Slice, reflect.Array:for i := 0; i < val.Len(); i++ {if val.Index(i).CanInterface() {fieldVal := val.Index(i).Interface()dm.processProperty(fieldVal, path, append(indexes, uint64(i)), context)}}case reflect.Ptr:ptrElem := val.Elem()if ptrElem.IsValid() && ptrElem.CanInterface() {dm.processProperty(ptrElem.Interface(), path, indexes, context)}case reflect.String:dm.processProperty(val.String(), path, indexes, context)case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:dm.processProperty(float64(val.Int()), path, indexes, context)case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:dm.processProperty(float64(val.Uint()), path, indexes, context)case reflect.Float32, reflect.Float64:dm.processProperty(float64(val.Float()), path, indexes, context)case reflect.Bool:dm.processProperty(val.Bool(), path, indexes, context)}}
func (dm *DocumentMapping) processProperty(property interface{}, path []string, indexes []uint64, context *walkContext) {pathString := encodePath(path)// look to see if there is a mapping for this fieldsubDocMapping := dm.documentMappingForPath(pathString)closestDocMapping := dm.closestDocMapping(pathString)// check to see if we even need to do further processingif subDocMapping != nil && !subDocMapping.Enabled {return}propertyValue := reflect.ValueOf(property)if !propertyValue.IsValid() {// cannot do anything with the zero valuereturn}propertyType := propertyValue.Type()switch propertyType.Kind() {case reflect.String:propertyValueString := propertyValue.String()if subDocMapping != nil {// index by explicit mappingfor _, fieldMapping := range subDocMapping.Fields {fieldMapping.processString(propertyValueString, pathString, path, indexes, context)}} else if closestDocMapping.Dynamic {// automatic indexing behavior// first see if it can be parsed by the default date parserdateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser)if dateTimeParser != nil {parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString)if err != nil {// index as textfieldMapping := newTextFieldMappingDynamic(context.im)fieldMapping.processString(propertyValueString, pathString, path, indexes, context)} else {// index as datetimefieldMapping := newDateTimeFieldMappingDynamic(context.im)fieldMapping.processTime(parsedDateTime, pathString, path, indexes, context)}}case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:dm.processProperty(float64(propertyValue.Int()), path, indexes, context)returncase reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:dm.processProperty(float64(propertyValue.Uint()), path, indexes, context)returncase reflect.Float64, reflect.Float32:propertyValFloat := propertyValue.Float()if subDocMapping != nil {// index by explicit mappingfor _, fieldMapping := range subDocMapping.Fields {fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context)}} else if closestDocMapping.Dynamic {// automatic indexing behaviorfieldMapping := newNumericFieldMappingDynamic(context.im)fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context)}case reflect.Bool:propertyValBool := propertyValue.Bool()if subDocMapping != nil {// index by explicit mappingfor _, fieldMapping := range subDocMapping.Fields {fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context)}} else if closestDocMapping.Dynamic {// automatic indexing behaviorfieldMapping := newBooleanFieldMappingDynamic(context.im)fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context)}case reflect.Struct:switch property := property.(type) {case time.Time:// don't descend into the time structif subDocMapping != nil {// index by explicit mappingfor _, fieldMapping := range subDocMapping.Fields {fieldMapping.processTime(property, pathString, path, indexes, context)}} else if closestDocMapping.Dynamic {fieldMapping := newDateTimeFieldMappingDynamic(context.im)fieldMapping.processTime(property, pathString, path, indexes, context)}default:dm.walkDocument(property, path, indexes, context)}default:dm.walkDocument(property, path, indexes, context)} }
分词的部分终于来了!
func (fm *FieldMapping) processString(propertyValueString string, pathString string, path []string, indexes []uint64, context *walkContext) {fieldName := getFieldName(pathString, path, fm)options := fm.Options()if fm.Type == "text" { analyzer := fm.analyzerForField(path, context)field := document.NewTextFieldCustom(fieldName, indexes, []byte(propertyValueString), options, analyzer)context.doc.AddField(field) if !fm.IncludeInAll { context.excludedFromAll = append(context.excludedFromAll, fieldName)}} else if fm.Type == "datetime" { dateTimeFormat := context.im.DefaultDateTimeParserif fm.DateFormat != "" { dateTimeFormat = fm.DateFormat }dateTimeParser := context.im.DateTimeParserNamed(dateTimeFormat)if dateTimeParser != nil { parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString)if err == nil {fm.processTime(parsedDateTime, pathString, path, indexes, context)} }} }func (fm *FieldMapping) processFloat64(propertyValFloat float64, pathString string, path []string, indexes []uint64, context *walkContext) {fieldName := getFieldName(pathString, path, fm)if fm.Type == "number" {options := fm.Options()field := document.NewNumericFieldWithIndexingOptions(fieldName, indexes, propertyValFloat, options)context.doc.AddField(field)if !fm.IncludeInAll {context.excludedFromAll = append(context.excludedFromAll, fieldName)}} }
转载于:https://www.cnblogs.com/bonelee/p/6675628.html
bleve搜索引擎源码分析之索引——mapping真复杂啊相关推荐
- bleve搜索引擎源码分析之索引——mapping和lucene一样,也有_all
例子: package mainimport ("fmt""github.com/blevesearch/bleve" )func main() {// ope ...
- wukong引擎源码分析之索引——part 1 倒排列表本质是有序数组存储
searcher.IndexDocument(0, types.DocumentIndexData{Content: "此次百度收购将成中国互联网最大并购"}) engine.go ...
- 悟空分词与mysql结合_悟空分词的搜索和排序源码分析之——索引
转自:http://blog.codeg.cn/2016/02/02/wukong-source-code-reading/ 索引过程分析 下面我们来分析索引过程. // 将文档加入索引 // // ...
- wukong引擎源码分析之索引——part 3 文档评分 无非就是将docid对应的fields信息存储起来,为搜索结果rank评分用...
之前的文章分析过,接受索引请求处理的代码在segmenter_worker.go里: func (engine *Engine) segmenterWorker() {for {request := ...
- wukong引擎源码分析之索引——part 2 持久化 直接set(key,docID数组)在kv存储里...
前面说过,接收indexerRequest的代码在index_worker.go里: func (engine *Engine) indexerAddDocumentWorker(shard int) ...
- postgreSQL源码分析——索引的建立与使用——GIST索引(2)
2021SC@SDUSC 本篇博客主要讲解GiST索引创建以及删除的相关函数 这里写目录标题 GIST创建 相关数据结构 GISTBuildState GISTInsertStack gistbuil ...
- postgreSQL源码分析——索引的建立与使用——Hash索引(2)
2021SC@SDUSC 目录 Hash索引创建 hashbuild函数 _hash_init函数 Hash索引的插入 hashinsert函数 _hash_doinsert函数 总结 Hash索引创 ...
- v64.06 鸿蒙内核源码分析(索引节点) | 谁是文件系统最重要的概念 | 百篇博客分析OpenHarmony源码
子曰:"君子博学于文,约之以礼,亦可以弗畔矣夫!" <论语>:颜渊篇 百篇博客系列篇.本篇为: v64.xx 鸿蒙内核源码分析(索引节点篇) | 谁是文件系统最重要的概 ...
- mysql源码分析——索引的数据结构
引子 说几句题外话,在京被困三个月之久,不能回家,所以这个源码分析就中断了.之所以在家搞这个数据库的源码分析,主要是在家环境齐全,公司的电脑老旧不堪.意外事件往往打断正常的习惯和运行轨迹,但这却是正常 ...
最新文章
- JDBC连接Oracle数据库测试
- Java 高级算法——数组中查询重复的数字
- Centos普通用户提权至ROOT
- .Net Web开发技术栈
- ECV2020开赛!识别火焰/头盔/后厨老鼠/摔倒,30万奖金,4万张数据集,等你来战!...
- copy所有的java文件到硬盘_将d:\java目录下的所有.java文件复制到d:\jad目录下,并将原来文件的扩展名从.java改为.jad...
- android传递布局到下个页面,浅入浅出Android(017):当前Activity向下一个Activity切换,并传递数据...
- 医学图像处理期末复习(二)
- java builder pool_每周10道Java面试题:String, String Pool, StringBuilder
- 千博HTML5自适应企业网站系统 v2021 Build0622
- 【安卓开发】AS神奇的报错:Cannot find AVD system path. Please define ANDROID_SDK_ROOT
- ModuleNotFoundError: No module named ‘keras.api‘
- 金属激光切割机行业调研报告 - 市场现状分析与发展前景预测
- 《Java程序员职场全功略:从小工到专家》连载四:IT人不容易
- 前端学习 第二弹: JavaScript中的一些函数与对象(1)
- 深圳无车日:吕锐锋搭公交 卓钦锐徒步走
- 网管必知:Windows常用网络命令详解
- xp php环境搭建 iis,Windows XP环境下快速配置IIS+PHP详细过程
- Ableton Live 10 mac 破解版永久激活方法
- unity动画系统之两个动画片段之间连线设置
热门文章
- mysql创建用户并授登录权限_mysql创建用户并授予权限
- nbns协议_网络协议详解1 - NBNS
- python录制视频和声音_python录制系统声音
- mysql 自动重启 计划_解决MYSQL死机,定时重启MYSQL,wdcp计划任务设定方法,
- ssm路径访问不到_ssm整合!!!
- windows2012挂linux盘阵,磁盘阵(IPSAN)挂载Windows和Linux测试过程.doc
- 基于颜色特征的图像匹配MATLAB,基于颜色特征的图像检索系统 这是个MATLAB程序 - 下载 - 搜珍网...
- php中堆和栈的使用
- python【数据结构与算法】二分模板
- 7-29 修理牧场 (25 分)