ElasticSearch的scroll分批查询

项目集成RestHighLeverConfig

application.yml添加配置参数

#ES集群rest接口
rest-elasticsearch:ipAndPorts: 192.168.8.23:9200,192.168.8.24:9200,192.168.8.25:9200schema: httpconnectTimeOut: 1000socketTimeOut: 30000connectionRequestTimeOut: 500maxConnectNum: 100maxConnectPerRoute: 100username: elasticpassword: 123456

配置类代码

@Configuration
public class ESRestHighLevelConfig {private static final Logger LOGGER  = LoggerFactory.getLogger(ESRestHighLevelConfig.class);private static final String CLUSTER_NODES_SPLIT_SYMBOL = ",";private static final String HOST_PORT_SPLIT_SYMBOL = ":";@Value("${rest-elasticsearch.ipAndPorts}")private String ipAndPorts;@Value("${rest-elasticsearch.schema}")private String schema;@Value("${rest-elasticsearch.connectTimeOut}")private int connectTimeOut;@Value("${rest-elasticsearch.socketTimeOut}")private int socketTimeOut;@Value("${rest-elasticsearch.connectionRequestTimeOut}")private int connectionRequestTimeOut;@Value("${rest-elasticsearch.maxConnectNum}")private int maxConnectNum;@Value("${rest-elasticsearch.maxConnectPerRoute}")private int maxConnectPerRoute;@Value("${rest-elasticsearch.username}")private String username;@Value("${rest-elasticsearch.password}")private String password;@Bean(name = "restHighLevelClient")public RestHighLevelClient restClient() {return new RestHighLevelClient(initRestClientBuilder());}/*** 初始化ES配置信息*/private RestClientBuilder  initRestClientBuilder() {LOGGER.info("elasticsearch  初始化...");final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));String[] clusterNodeArray = ipAndPorts.trim().split(CLUSTER_NODES_SPLIT_SYMBOL);HttpHost[] httpHosts = new HttpHost[clusterNodeArray.length];for (int i = 0; i < clusterNodeArray.length;i++) {String[] clusterNodeInfoArray = clusterNodeArray[i].trim().split(HOST_PORT_SPLIT_SYMBOL);httpHosts[i] = new HttpHost(clusterNodeInfoArray[0],Integer.parseInt(clusterNodeInfoArray[1]),schema);}RestClientBuilder restClientBuilder = RestClient.builder(httpHosts).setHttpClientConfigCallback(httpClientBuilder -> {//httpclient的连接数配置httpClientBuilder.setMaxConnTotal(maxConnectNum);httpClientBuilder.setMaxConnPerRoute(maxConnectPerRoute);//设置认证信息return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);}).setRequestConfigCallback(requestConfigBuilder -> {requestConfigBuilder.setConnectTimeout(connectTimeOut);requestConfigBuilder.setSocketTimeout(socketTimeOut);requestConfigBuilder.setConnectionRequestTimeout(connectionRequestTimeOut);return requestConfigBuilder;});LOGGER.info("elasticsearch  初始化成功...");return restClientBuilder;}}

业务层

@Controller
@RequestMapping("testEsScroll")
public class TestEsController {@AutowiredESRestHighLevelConfig esRestHighLevelConfig;@RequestMapping(value = "scroll")@ResponseBodypublic void scroll(){RestHighLevelClient client = esRestHighLevelConfig.restClient();//初始化scrollfinal Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1));SearchRequest searchRequest = new SearchRequest();searchRequest.indices("poc"); //指定索引名称searchRequest.scroll(scroll);SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();//设置查询条件，如果要查询全部就用 // QueryBuilder mathAll = QueryBuilders.matchAllQuery();//多条件查询QueryBuilder q1 = QueryBuilders.termQuery("channelCode", "onlineBank");QueryBuilder q2 = QueryBuilders.rangeQuery("transTime").gte(1565937193000L).lte(1566973993000L);BoolQueryBuilder must = QueryBuilders.boolQuery().must(q1).must(q2);QueryBuilder mathAll = QueryBuilders.matchAllQuery();searchSourceBuilder.query(must);//searchSourceBuilder.query(QueryBuilders.termQuery("channelCode","onlineBank")); //term精准查询//searchSourceBuilder.query(QueryBuilders.rangeQuery("age").gte(20)); // 范围查询//searchSourceBuilder.query(QueryBuilders.rangeQuery("age").from(10).to(20)); //闭区间，范围查询//searchSourceBuilder.query(QueryBuilders.rangeQuery("age").from(15,false).to(20,false)); //开区间，范围查询searchSourceBuilder.size(10);  //每次返回的数据条数searchRequest.source(searchSourceBuilder);SearchResponse searchResponse = null;try {searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);} catch (IOException e) {e.printStackTrace();}String scrollId = searchResponse.getScrollId();  //首次查询返回的油标scrollIDSearchHit[] searchHits = searchResponse.getHits().getHits();  //获取结果数据for (SearchHit searchHit : searchHits) {System.out.println(searchHit.getSourceAsString());}//遍历搜索命中的数据，直到没有数据int count = 10;while (searchHits != null && searchHits.length > 0){SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);scrollRequest.scroll(scroll);try {searchResponse = client.scroll(scrollRequest,RequestOptions.DEFAULT);} catch (IOException e) {e.printStackTrace();}scrollId = searchResponse.getScrollId(); //每次查询后更新scrollIdsearchHits = searchResponse.getHits().getHits();if(searchHits != null && searchHits.length > 0){System.out.println("---------下一页---------");for (SearchHit searchHit : searchHits) {System.out.println(searchHit.getSourceAsString());}}count = count + searchHits.length;}//清除滚屏ClearScrollRequest clearScrollRequest = new ClearScrollRequest();clearScrollRequest.addScrollId(scrollId);ClearScrollResponse clearScrollResponse = null;try {clearScrollResponse = client.clearScroll(clearScrollRequest,RequestOptions.DEFAULT);} catch (IOException e) {e.printStackTrace();}boolean succeeded = clearScrollResponse.isSucceeded();System.out.println("succeeded: "+succeeded+",  共接收交易数据:"+count+" 条");}
}

ElasticSearch的scroll分批查询相关推荐

java使用ElasticSearch的scroll查询，高效的解决es查询数量的限制。
java使用ElasticSearch的scroll查询,高效的解决es查询数量的限制. 一.为什么要使用ES的scroll (1)首先我们要明白es的查询机制:ES的搜索是分2个阶段进行的,即Que ...
ElasticSearch两种分页查询
在存在大量数据时,一般我们进行查询都需要进行分页查询.例如:我们指定页码.并指定每页显示多少条数据,然后Elasticsearch返回对应页码的数据. 1.使用from和size来进行分页在执行查询 ...
2021年大数据ELK（十六）：Elasticsearch SQL（职位查询案例）
全网最详细的大数据ELK文章系列,强烈建议收藏加关注! 新文章都已经列出历史文章目录,帮助大家回顾前面的知识重点. 目录职位查询案例一.查询职位索引库中的一条数据二.将SQL转换为DSL 三.职 ...
java操作es聚合操作并显示其他字段_java使用elasticsearch分组进行聚合查询过程解析...
这篇文章主要介绍了java使用elasticsearch分组进行聚合查询过程解析,文中通过示例代码介绍的非常详细,对大家的学习或者工作具有一定的参考学习价值,需要的朋友可以参考下 java连接elas ...
java分批查询oracle数据库_数据库的分批查询问题
在开发某个要操作数据库的项目时候,我突然想到如何数据库的数据量太大,一般项目设计的时候,会一次读取所有的数据到内存中,这有可能造成内存的溢出,或者系统效率的降低.那么我们如何设计一个开发方案来解决这中 ...
【Elasticsearch】ElasticSearch里面的偏好查询
1.概述转载:ElasticSearch里面的偏好查询在es查询的时候我们可以控制Preference,来完成特定shard或节点上的数据查询,默认情况下查询是随机的. 假如现在我们有一份索引5个 ...
【Elasticsearch】改进布尔查询的搜索相关性
1.概述翻译:Improving search relevance with boolean queries 有人翻译:Elasticsearch:使用布尔查询提高搜索的相关性当你在Elastic ...
【Elasticsearch】Elasticsearch Span Query跨度查询
1.概述转载:Elasticsearch Span Query跨度查询 ES基于Lucene开发,因此也继承了Lucene的一些多样化的查询,比如本篇说的Span Query跨度查询,就是基于Luc ...
初识ElasticSearch(2) -文档查询之match查询 | 分词器
1. 分词器: 2. match查询: 2.1. 数据准备 - 创建带分词器的索引映射 2.2. 数据准备 - 添加文档 2.3. 数据准备 - 查看文本分词 2.4. 查询 - 映射有分词器的字段查 ...

ElasticSearch的scroll分批查询

项目集成RestHighLeverConfig

ElasticSearch的scroll分批查询相关推荐

最新文章

热门文章