laravel使用elasticsearch

laravel 中安装扩展包

composer安装elasticsearch扩展包

Elasticsearch 官方提供了 Composer 包，在引入时需要注意要指定版本，因为不同版本的 Elasticsearch 的 API 略有不同，因为我用的是 7.12.x，因此需使用 ~7.12.x 来指定包版本。

composer require elasticsearch/elasticsearch "7.12.x" --ignore-platform-reqs

laravel 配置 es

config/database.php

'elasticsearch' => [// Elasticsearch 支持多台服务器负载均衡，因此这里是一个数组'hosts' => explode(',',env('ES_HOSTS')),
]

.env 配置

ES_HOSTS=172.17.0.8

初始化 Elasticsearch 对象，并注入到 Laravel 容器中：

App/Providers/AppServiceProvider.php

注释：在laravel容器中自定义一个名为es的服务对象，通过ESClientBuilder以及配置文件中的信息连接
到es，我们可以通过app(‘es’)->info()查看连接之后的es对象信息。

<?php
namespace App\Providers;
use Illuminate\Support\ServiceProvider;
use Elasticsearch\ClientBuilder as ESClientBuilder;class AppServiceProvider extends ServiceProvider
{/*** Register any application services.** @return void*/public function register(){// 注册一个名为 es 的单例$this->app->singleton('es',function (){// 从配置文件读取 Elasticsearch 服务器列表$builder = ESClientBuilder::create()->setHosts(config('database.elasticsearch.hosts'));// 如果是开发环境if (app()->environment()==='local'){// 配置日志，Elasticsearch 的请求和返回数据将打印到日志文件中，方便我们调试$builder->setLogger(app('log')->driver());}return $builder->build();});}/*** Bootstrap any application services.** @return void*/public function boot(){//}
}
?>

注册完成后，进行测试

php artisan tinker>>>app('es')->info();

结果如下：

创建索引

PUT /products/
{"mappings": {"properties": {"name":{"type": "text","analyzer": "ik_smart"},"long_name":{"type": "text","analyzer": "ik_smart"},"brand_id":{"type": "integer"},"category_id":{"type":"integer"},"category":{"type": "keyword"},"category_path":{"type": "keyword"},"shop_id":{"type":"integer"},"price":{"type":"scaled_float","scaling_factor":100},"sold_count":{"type":"integer"},"review_count":{"type":"integer"},"status":{"type":"integer"},"create_time" : {"type" : "date"},"last_time" : {"type" : "date"},"skus":{"type": "nested","properties": {"name":{"type":"text","analyzer": "ik_smart"},"price":{"type":"scaled_float","scaling_factor":100}}},"attributes":{"type": "nested","properties": {"name": { "type": "keyword" },"value": { "type": "keyword"}}}}}
}

laravel中使用es

通过laravel把数据从mysql同步到es中

这里只是演示如何把数据从mysql同步进入es中

在之前的logstash操作中，通过jdbc的方式从mysql同步数据到es中，这次试用laravel的方式同步数据到es中

1 . 创建索引

注：这里存在一个时间字段 create_time 格式为 date 的问题，laravel从mysql获取到的时间格式插入到es中会出现无法解析的情况，需要对时间格式进行 “format”: “yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis” 的操作

PUT /test/
{"mappings": {"properties": {"id":{"type": "integer"},"name":{"type": "text","analyzer": "ik_smart"},"long_name":{"type": "text","analyzer": "ik_smart"},"brand_id":{"type": "integer"},"shop_id":{"type":"integer"},"price":{"type":"scaled_float","scaling_factor":100},"sold_count":{"type":"integer"},"review_count":{"type":"integer"},"status":{"type":"integer"},"create_time" : {"type" : "date","format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"},"last_time" : {"type" : "date","format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"},"skus":{"type": "nested","properties": {"name":{"type":"text","analyzer": "ik_smart"},"price":{"type":"scaled_float","scaling_factor":100}}},"attributes":{"type": "nested","properties": {"name": { "type": "keyword" },"value": { "type": "keyword"}}}}}
}

2 . 创建代码脚本

php artisan make:command Elasticsearch/SyncProducts

app/Console/Commands/Elasticsearch/SyncProducts.php

<?phpnamespace App\Console\Commands\Elasticsearch;use App\Models\Product;
use Illuminate\Console\Command;class SyncProducts extends Command
{/*** The name and signature of the console command.** @var string*/protected $signature = 'es:sync-products';/*** The console command description.** @var string*/protected $description = '将商品数据同步到 Elasticsearch';/*** Create a new command instance.** @return void*/public function __construct(){parent::__construct();}/*** Execute the console command.** @return mixed*/public function handle(){// 获取 Elasticsearch 对象$es = app('es');$sql = 'id,name,long_name,brand_id,three_category_id,shop_id,price,sold_count,review_count,status,create_time,last_time';Product::query()->selectRaw($sql)// 使用 chunkById 避免一次性加载过多数据->chunkById(100, function ($products) use ($es) {$this->info(sprintf('正在同步 ID 范围为 %s 至 %s 的商品', $products->first()->id, $products->last()->id));// 初始化请求体$req = ['body' => []];// 遍历商品foreach ($products as $product) {// 将商品模型转为 Elasticsearch 所用的数组$data = $product->toESArray($product->id,$product->three_category_id);$req['body'][] = ['index' => ['_index' => 'test','_id'    => $data['id'],],];$req['body'][] = $data;}try {// 使用 bulk 方法批量创建$es->bulk($req);} catch (\Exception $e) {$this->info($e->getMessage());}});$this->info('同步完成');}
}

3 . 创建Product的Model Product.php、进行数据过滤

<?phpnamespace App\Models;use Illuminate\Support\Arr;
use Illuminate\Database\Eloquent\Model;class Product extends Model
{public $table = "products";/*** 格式化为es的数据* @param $product_id* @param $category_id* @return array*/public function toESArray($product_id,$category_id){// 只取出需要的字段$arr = Arr::only($this->toArray(), ['id','name','long_name','brand_id','shop_id','price','sold_count','review_count','status','create_time','last_time']);$productSkus = ProductSkus::query()->selectRaw('name,price')->where('product_id',$product_id)->get()->toArray();// skus在索引中是一个二维数组， 这里只取出需要的 SKU 字段$arr['skus'] = $productSkus;$sql = "lmrs_at.name as name,lmrs_at_val.name as value";$attributes = Attributes::query()->selectRaw($sql)->from('attributes as at')->leftJoin('attribute_values as at_val','at.id','at_val.attribute_id')->where('at.category_id',$category_id)->get()->toArray();// attributes 在索引中是一个二维数组， 这里只取出需要的商品属性字段$arr['attributes'] = $attributes;return $arr;}
}

我们在写入商品数据的时候用的是 bulk() 方法，这是 Elasticsearch 提供的一个批量操作接口。设想一下假如我们系统里有数百万条商品，如果每条商品都单独请求一次 Elasticsearch 的 API，那就是数百万次的请求，性能肯定是很差的，而 bulk() 方法可以让我们用一次 API 请求完成一批操作，从而减少请求次数的数量级，提高整体性能。

4 . 运行脚本

php artisan es:sync-products

插入数据后

laravel中es的查询操作

1 . 添加工具类 PublicService.php

<?php
namespace App\Services;use App\Models\ProductCategory;class PublicService
{/*** 多维数组转换一维数组* @param $input* @param $flatten* @return array*/public static function getDataByEs($input){$total = $input['hits']['total']['value'];$input = $input['hits']['hits'];$data  = collect($input)->pluck('_source');return [$data,$total];}/*** 获取分类信息* @param $category* @return \Illuminate\Database\Eloquent\Builder|\Illuminate\Database\Eloquent\Model|object|null*/public static function category($category){$category_array =   explode(',',$category);$category_id    =   array_pop($category_array);return ProductCategory::query()->where('id',$category_id)->first();}
}

2 . 封装一个用于商品搜索的 ElasticSeachService.php

<?php
namespace App\Services;class ElasticsearchService
{/*** 搜索结构体* @var array*/protected $params = ['type'  => '_doc','body'  => ['query' => ['bool' => ['filter'=> [],'must'  => []]]]];/*** 通过构造函数进行索引初始化* ElasticsearchService constructor.* @param $index*/public function __construct($index){//要搜索的索引名称$this->params['index'] = $index;return $this;}/*** 根据字段-条件搜索* @param $type .搜索类型：term精准搜索，match分词器模糊查询，prefix字段前缀* @param $key  .搜索的字段名称* @param $value.搜索字段值* @return $this*/public function  queryByFilter($type,$key,$value){$this->params['body']['query']['bool']['filter'][] = [$type => [$key => $value]];return $this;}/*** 关键词按照权重进行搜索* @param $keyWords* @return $this*/public function keyWords($keyWords){//如果不是数据则转换为数组$keyWords = is_array($keyWords) ? $keyWords : [$keyWords];foreach ($keyWords as $key => $value){$this->queryByMust($value);}return $this;}/*** 根据权重进行多字段搜索* @param $seach 搜索的值* @return $this*/public function  queryByMust($seach){$this->params['body']['query']['bool']['must'][] = ['multi_match' => ['query' => $seach,'fields'=> ['long_name^3','category^2']]];return $this;}/*** 获取指定字段* @param $keyWords  一维数组* @return $this*/public function source($keyWords){$keyWords = is_array($keyWords) ? $keyWords : [$keyWords];$this->params['body']['_source'] = $keyWords;return $this;}/*** 设置分页* @param $page* @param $pageSize* @return $this*/public function paginate($page,$pageSize){$this->params['body']['from'] = ($page - 1) * $pageSize;$this->params['body']['size'] = $pageSize;return $this;}/*** 排序* @param $filed        .排序字段* @param $direction    .排序值* @return $this*/public function orderBy($filed,$direction){if(!isset($this->params['body']['sort'])){$this->params['body']['sort'] = [];}$this->params['body']['sort'][] = [$filed => $direction];return  $this;}/*** 聚合查询 商品属性筛选的条件* @param $name     属性名称* @param $value    属性值* @return $this*/public function attributeFilter($name,$value){//attributes 为 索引中的  attributes  字段$this->params['body']['query']['bool']['filter'] = ['nested' => ['path' => 'attributes','query' => ['bool' => ['must' => [['term' => ['attributes.name' => $name]],['term' => ['attributes.value' => $value]]]]]]];return $this;}/*** 聚合查询(方式2) 商品属性筛选的条件* @param $name* @param $value* @return $this*//*public function attributeFilter($name,$value){$this->params['body']['query']['bool']['filter'][] = ['nested' => ['path'  => 'attributes','query' => [['term' => ['attributes.name' => $name]],['term' => ['attributes.value' => $value]]],],];return $this;}*//*** 返回结构体* @return array*/public function getParams(){return $this->params;}/*** 多维数组转换一维数组* @param $input* @param $flatten* @return array*/public static function getDataByEs($input){$total = $input['hits']['total']['value'];$input = $input['hits']['hits'];$data  = collect($input)->pluck('_source');return [$data,$total];}}

3 . 创建控制器，在控制器中调用 ProductController.php

<?phpnamespace App\Http\Controllers\Api\V1;use App\Http\Controllers\Controller;
use App\Models\Product;
use App\Models\ProductSkus;
use App\Services\PublicService;
use Illuminate\Http\Request;
use App\Http\Controllers\ApiResponse;
use App\Services\ElasticsearchService;class ProductController extends Controller
{use ApiResponse;/*** 商品首页信息* @param Request $request* @return \Illuminate\Http\JsonResponse*/public function index(Request $request){try{$page = $request->input('page');if(!$page) return $this->failed('页数不能为空');$search      = $request->input('search');$order       = $request->input('order');$attributes  = $request->input("attributes");$category_id = $request->input('category_id');$shop_id     = $request->input('shop_id',1);$pageSize    = $request->input('pageSize',20);$esService = new ElasticsearchService('products');//封装es类，增加商品状态为上架条件的分页查询结构$builder = $esService->queryByFilter('term','shop_id',$shop_id)->paginate($page,$pageSize);//分类搜索if ($category_id){//获取分类信息$category = PublicService::category($category_id);//根据字段-条件搜索if($category && $category->is_directory){$builder->queryByFilter('prefix','category_path',$category->path.$category->id.'-');}else{$builder->queryByFilter('term','category_id',$category->id);}}//关键词按照权重进行搜索if($search){$keywords = array_filter(explode(' ',$search));$builder->keyWords($keywords);}//排序if($order){if (preg_match('/^(.+)_(asc|desc)$/',$order,$m)){//只有当前三个字段才可以进行排序搜索if (in_array($m[1],['price','sold_count','review_count'])){$builder->orderBy($m[1],$m[2]);}}}$attributeFilter = [];//根据商品类型搜索if($attributes){$attrArray = explode("|",$attributes);foreach ($attrArray as $attr){list($name,$value) = explode(":",$attr);$attributeFilter[$name] = $value;$builder->attributeFilter($name,$value);}}//获取的字段$builder->source(['id','name','long_name','shop_id','skus','attributes','create_time']);//执行es搜索$restful = app('es')->search($builder->getParams());//多维数组转换一维数组list($data,$total) = $builder->getDataByEs($restful);return $this->success(compact('data','total','page'));}catch (\Exception $e){return $this->failed($e->getMessage().':'.$e->getLine());}}/*** 获取商品详情信息* @param Request $request* @return \Illuminate\Http\JsonResponse*/public function productInfo(Request $request){try{$product_id = $request->input('product_id');if(!$product_id) return $this->failed('缺少商品id');//通过id从es中获取数据$es = new ElasticsearchService('products');$builder = $es->queryByFilter('term','status',1)->queryByFilter('term','id',$product_id);$builder->source(['id','name','long_name','shop_id','create_time']);$restful = app('es')->search($builder->getParams());list($data,$total) = $es->getDataByEs($restful);return $this->success(compact('data','total'));}catch (\Exception $e){return $this->failed($e->getMessage());}}/*** 创建数据* @param Request $request* @return \Illuminate\Http\JsonResponse*/public function createProduct(Request $request){try{$input = $request->all();$shop_id = $input['shop_id'];$input['one_category_id']   =   425;$input['two_category_id']   =   438;$input['three_category_id'] =   440;$info = Product::query()->create($input);$id = $info->id;$sql = 'product_id,attribute_id,name,num,price,status,shop_id';$skus = ProductSkus::query()->selectRaw($sql)->where('product_id',18)->get()->each(function ($query) use ($id,$shop_id){$query->product_id = $id;$query->shop_id = $shop_id;});$skus = $skus->toArray();//添加新属性ProductSkus::query()->insert($skus);//格式化为es的数据$esArray = $info->toESArray($info->id,$info->three_category_id);//对于es来说，已有数据，则是编辑，没有则是新增app('es')->index(['id' => $esArray['id'], 'index' => 'test', 'body' => $esArray]);return $this->success('添加成功成功');}catch (\Exception $e){return $this->failed($e->getMessage());}}/*** 修改商品信息* @param Request $request* @return \Illuminate\Http\JsonResponse*/public function editProductInfo(Request $request){try{$product_id = $request->input('product_id');if(!$product_id) return $this->failed('缺少商品id');$name = $request->input('name');$long_name = $request->input('long_name');$price = $request->input('price');$sql = 'id,name,long_name,brand_id,shop_id,price,sold_count,review_count,status,create_time,last_time,three_category_id';$info = Product::query()->selectRaw($sql)->where('id',$product_id)->first();if(!$info) return $this->failed('暂无当前商品信息');//先修改mysql中的数据，在把数据同步到es中：如果有用logstash，则无需次操作，logstash会自动同步数据到es中$info->name = $name;$info->long_name = $long_name;$info->price = $price;$info->save();//格式化为es的数据$esArray = $info->toESArray($info->id,$info->three_category_id);//对于es来说，已有数据，则是编辑，没有则是新增app('es')->index(['id' => $esArray['id'], 'index' => 'test', 'body' => $esArray]);return $this->success('编辑成功');}catch (\Exception $e){return $this->failed($e->getMessage());}}
}

4 . 问题：laravel数据修改后对es数据影响的问题

在es中不能单独修改一个字段的数据，需要整条数据修改，因为es中无法修改一个字段的信息数据，es会删除掉原有的数据，重新生成只有修改时录入的数据。

解决方案： 在修改完数据库的数据后，需要重新生成当前es所需要的数据(在Model中封装好的toESArray方法)，然后重新写入，或者通过第三方数据监听，如（go-mysql-es 、logstash）实时监听并同步修改数据。

.
.
.
$esArray = Product::find($product_id)->toESArray();//对于es来说，已有数据，则是编辑，没有则是新增
app('es')->index(['id' => $arr['id'], 'index' => 'test', 'body' => $esArray]);
.
.
.

es 录入的 $arr 数据格式为：

array:13 ["id" => 1"name" => "HUAWEI Mate Book 11""long_name" => "HUAWEI Mate Book 11 16GB 512GB 触屏 集显""brand_id" => 11"shop_id" => 1"price" => "5299.00""sold_count" => 111"review_count" => 1111"status" => 2"create_time" => "2021-05-25 15:12:09""last_time" => "2021-05-25 15:12:14""skus" => array:6 [0 => array:2 ["name" => "皓月银 I5/16GB/512GB 触屏 集成显卡 官方标配""price" => "6299.00"]1 => array:2 ["name" => "皓月银 I7/16GB/512GB 触屏 集成显卡 官方标配""price" => "6599.00"]]"attributes" => array:12 [0 => array:2 ["name" => "颜色""value" => "星际蓝"]1 => array:2 ["name" => "颜色""value" => "冰霜银"]]
]