【搜索引擎】Smartsys部署ES的准备工作

1. 扩展表空间

扩展SMARTSYS_HISTORY和SMARTSYS_HISTORY表空间，使当前使用率降至40%或以下。

2. 将现有历史事项表重命名

ALTER TABLE scada_event rename to scada_event_bak;
ALTER INDEX SCADA_EVENT_INDEX rename to SCADA_EVENT_INDEX_BAK;
ALTER INDEX SCADA_EVENT_INDEX2 rename to SCADA_EVENT_INDEX2_BAK;

3. 创建新的带自增字段的历史事项表

CREATE TABLE "SMARTSYS"."scada_event"
(
"ID" BIGINT IDENTITY(1,1) CLUSTER PRIMARY KEY,
"KEY_ID_TAG" VARCHAR(100) DEFAULT 'dummy' NOT NULL,
"TAG_ID" BIGINT NOT NULL,
"OCCUR_TIME" TIMESTAMP(0),
"MILLI_SECOND" INTEGER NOT NULL,
"SOE_TIME" TIMESTAMP(0),
"SOE_MSECOND" INTEGER,
"DOMAIN_ID" INTEGER,
"STATION_ID" BIGINT NOT NULL,
"REGION_ID" INTEGER,
"INFER_NAME" VARCHAR(64),
"ALARM_PRIORITY" INTEGER,
"STATUS" INTEGER,
"CONTENT" VARCHAR(256),
"CONFIRM_TIME" TIMESTAMP(0),
"CONFIRM_NODE_NAME" VARCHAR(64),
"ALARM_TYPE" INTEGER NOT NULL,
"APP_ID" INTEGER,
"CONTAINER1_ID" BIGINT,
"CONTAINER2_ID" BIGINT,
"DEV_ID" BIGINT,
"CONFIRM_USER_NAME" VARCHAR(64),
"CONTEXT" INTEGER,
"SYSID" INTEGER DEFAULT 1 NOT NULL,
"AREA_RESP" INTEGER DEFAULT (-1) NOT NULL,
"OBJ_ID" BIGINT,
"REASON_CODE" INTEGER,
"DICTIONARY" CHARACTER VARYING(512),
"VALUE" DOUBLE) STORAGE(ON "SMART_HISTORY", CLUSTERBTR);
CREATE  INDEX "SCADA_EVENT_INDEX" ON "SMARTSYS"."scada_event"("DEV_ID" ASC,"OCCUR_TIME" ASC,"OBJ_ID" ASC,"STATION_ID" ASC,"CONTAINER1_ID" ASC,"ALARM_TYPE" ASC,"STATUS" ASC) STORAGE(ON "SMART_HISTORY_INDEX", CLUSTERBTR) ;
CREATE  INDEX "SCADA_EVENT_INDEX2" ON "SMARTSYS"."scada_event"("OCCUR_TIME" ASC,"ALARM_TYPE" ASC,"STATUS" ASC,"STATION_ID" ASC,"CONTAINER1_ID" ASC,"CONTAINER2_ID" ASC) STORAGE(ON "SMART_HISTORY_INDEX", CLUSTERBTR) ;

经过对金仓和达梦数据库的测试，在执行步骤1和2的时候不需要停Smartsys系统，切换过程持续几秒钟，事项可能会存在及少量的丢失，但不会影响切换结果。

4. 根据需要将部分旧的事项数据从SCADA_EVENT_BAK中，分批导入ES。

--1. 分批次迁移历史数据，分批的目的在于将每次的数据提交量控制在5000 0000条以下，以防止临时缓冲区不足。
insert into SCADA_EVENT(KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,
CONFIRM_NODE_NAME,ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE")
select KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,CONFIRM_NODE_NAME,
ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE" from scada_event_bak
where occur_time >= '2019-08-01';
commit;insert into SCADA_EVENT(KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,
CONFIRM_NODE_NAME,ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE")
select KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,CONFIRM_NODE_NAME,
ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE" from scada_event_bak
where occur_time >= '2019-07-01' and occur_time < '2019-08-01';
commit;insert into SCADA_EVENT(KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,
CONFIRM_NODE_NAME,ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE")
select KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,CONFIRM_NODE_NAME,
ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE" from scada_event_bak
where occur_time >= '2019-06-01' and occur_time < '2019-07-01';
commit;insert into SCADA_EVENT(KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,
CONFIRM_NODE_NAME,ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE")
select KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,CONFIRM_NODE_NAME,
ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE" from scada_event_bak
where occur_time >= '2019-05-01' and occur_time < '2019-06-01';
commit;insert into SCADA_EVENT(KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,
CONFIRM_NODE_NAME,ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE")
select KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,CONFIRM_NODE_NAME,
ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE" from scada_event_bak
where occur_time >= '2019-04-01' and occur_time < '2019-05-01';
commit;insert into SCADA_EVENT(KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,
CONFIRM_NODE_NAME,ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE")
select KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,CONFIRM_NODE_NAME,
ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE" from scada_event_bak
where occur_time >= '2019-03-01' and occur_time < '2019-04-01';
commit;insert into SCADA_EVENT(KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,
CONFIRM_NODE_NAME,ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE")
select KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,CONFIRM_NODE_NAME,
ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE" from scada_event_bak
where occur_time >= '2019-02-01' and occur_time < '2019-03-01';
commit;insert into SCADA_EVENT(KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,
CONFIRM_NODE_NAME,ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE")
select KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,CONFIRM_NODE_NAME,
ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE" from scada_event_bak
where occur_time >= '2019-01-01' and occur_time < '2019-02-01';
commit;insert into SCADA_EVENT(KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,
CONFIRM_NODE_NAME,ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE")
select KEY_ID_TAG,TAG_ID,OCCUR_TIME,MILLI_SECOND,SOE_TIME,SOE_MSECOND,DOMAIN_ID,STATION_ID,REGION_ID,INFER_NAME,ALARM_PRIORITY,STATUS,CONTENT,CONFIRM_TIME,CONFIRM_NODE_NAME,
ALARM_TYPE,APP_ID,CONTAINER1_ID,CONTAINER2_ID,DEV_ID,CONFIRM_USER_NAME,"CONTEXT",SYSID,AREA_RESP,OBJ_ID,REASON_CODE,"DICTIONARY","VALUE" from scada_event_bak
where occur_time < '2019-01-01';
commit;

这一步骤需要花费较长时间，根据实测数据显示，插入效率大约为：4000 0000条/小时。完成后需要检查执行日志（打印信息），确认每条语句都已经执行成功，并提交完成。

5. 运行ES迁移程序，将新SCADA_EVENT的内容迁移至ES中，并保持该迁移程序持续运行，按既定策略定期完成数据从SCADA_EVENT到ES的同步。

【搜索引擎】Smartsys部署ES的准备工作相关推荐

centos7 如何安装部署k8s_五步教你如何使用k8s快速部署ES
前言今天小编打算用五步教大家如何使用k8s快速部署ES,有兴趣的小伙伴可以了解一下~ 由于是使用本地存储,所以需要先创建pv 1.创建存储类 local-elasticsearch.yaml kin ...
教你在Kubernetes中快速部署ES集群
摘要:ES集群是进行大数据存储和分析,快速检索的利器,本文简述了ES的集群架构,并提供了在Kubernetes中快速部署ES集群的样例:对ES集群的监控运维工具进行了介绍,并提供了部分问题定位经验,最 ...
docker部署es和kibana遇到的坑
使用docker部署es和kibana遇到的坑 docker容器部署ES和kibana需要注意的点: 1.es和kibana的镜像版本必须一致.我这次用的是7.7.1版本 2.es非常占用内存,因此在 ...
使用Docker快速安装部署ES和Kibana并配置IK中文分词器以及自定义分词拓展词库
使用Docker快速安装部署ES和Kibana的前提:首先需要确保已经安装了Docker环境如果没有安装Docker的话,可以参考上一篇的内容:Linux上安装Docker 有了Docker环境后, ...
CentOS安装Elasticsearch_IK分词器拼音分词器_部署kibana_部署es集群
CentOS安装Elasticsearch_IK分词器_部署kibana_部署es集群一.部署单点es ①:创建网络因为我们还需要部署kibana容器,因此需要让es和kibana容器互联.这里先 ...
docker 部署 ES
docker 部署 ES 前言部署 ES 1.下载 ES 镜像 2.运行 ES 3.url 地址访问解决开启 ES 导致主机卡顿修改 ES 默认占用内存大小前言不熟悉 docker 基本命令 ...
Centos7 x86服务器一键部署ES集群/单机shell
背景由于日常的实施工作中经常用到单机或者集群的es,每次都手动部署感觉很麻烦,于是抽空写了一个一键部署脚本,通过简单指定集群名称.节点ip.端口等几个配置参数后自动部署es单机或者集群数据库安装脚 ...
一、搜索引擎篇-揭开es神秘的面纱
一.es是什么? elasticsearch是一个开源的分布式.RESTful 风格的搜索和数据分析引擎,它的底层是开源库lucene. 二.lucene是什么? 最受欢迎的java开源全文搜索引擎开 ...
【Solr】——搜索引擎的部署及使用
最近在做毕业设计,一个电商APP,遇到搜索商品的问题,以往做的数据库模糊查询感觉不是那么的高大上,于是决定使用Solr搜索引擎,在此给同样想法的小伙伴一点借鉴,望共勉. 一.Solr部署 1.Solr ...