本篇只是DWT层，其他内容请关注我的博客！在<项目>专栏里！！！

本篇文章参考尚硅谷大数据项目写成！

一、DWT层

1.设备主题宽表

2.会员主题宽表

3.商品主题宽表

4.活动主题宽表

5.地区主题宽表

二、数据导入脚本

一、DWT层

1.设备主题宽表

1）建表语句

create external table dwt_uv_topic
(`mid_id` string comment '设备id',`brand` string comment '手机品牌',`model` string comment '手机型号',`login_date_first` string  comment '首次活跃时间',`login_date_last` string  comment '末次活跃时间',`login_day_count` bigint comment '当日活跃次数',`login_count` bigint comment '累积活跃天数'
) COMMENT '设备主题宽表'
stored as parquet
location '/warehouse/gmall/dwt/dwt_uv_topic'
tblproperties ("parquet.compression"="lzo");

2）数据导入

insert overwrite table dwt_uv_topic
selectnvl(new.mid_id,old.mid_id),nvl(new.model,old.model),nvl(new.brand,old.brand),if(old.mid_id is null,'2022-05-20',old.login_date_first),if(new.mid_id is not null,'2022-05-20',old.login_date_last),if(new.mid_id is not null, new.login_count,0),nvl(old.login_count,0)+if(new.login_count>0,1,0)
from
(select*from dwt_uv_topic
)old
full outer join
(select*from dws_uv_detail_daycountwhere dt='2022-05-20'
)new
on old.mid_id=new.mid_id;

3）查询数据

select * from dwt_uv_topic limit 5;

2.会员主题宽表

1）建表语句

create external table dwt_user_topic
(user_id string  comment '用户id',login_date_first string  comment '首次登录时间',login_date_last string  comment '末次登录时间',login_count bigint comment '累积登录天数',login_last_30d_count bigint comment '最近30日登录天数',order_date_first string  comment '首次下单时间',order_date_last string  comment '末次下单时间',order_count bigint comment '累积下单次数',order_amount decimal(16,2) comment '累积下单金额',order_last_30d_count bigint comment '最近30日下单次数',order_last_30d_amount bigint comment '最近30日下单金额',payment_date_first string  comment '首次支付时间',payment_date_last string  comment '末次支付时间',payment_count decimal(16,2) comment '累积支付次数',payment_amount decimal(16,2) comment '累积支付金额',payment_last_30d_count decimal(16,2) comment '最近30日支付次数',payment_last_30d_amount decimal(16,2) comment '最近30日支付金额'
)COMMENT '会员主题宽表'
stored as parquet
location '/warehouse/gmall/dwt/dwt_user_topic/'
tblproperties ("parquet.compression"="lzo");

2）数据导入

insert overwrite table dwt_user_topic
selectnvl(new.user_id,old.user_id),if(old.login_date_first is null and new.login_count>0,'2022-05-20',old.login_date_first),if(new.login_count>0,'2022-05-20',old.login_date_last),nvl(old.login_count,0)+if(new.login_count>0,1,0),nvl(new.login_last_30d_count,0),if(old.order_date_first is null and new.order_count>0,'2022-05-20',old.order_date_first),if(new.order_count>0,'2022-05-20',old.order_date_last),nvl(old.order_count,0)+nvl(new.order_count,0),nvl(old.order_amount,0)+nvl(new.order_amount,0),nvl(new.order_last_30d_count,0),nvl(new.order_last_30d_amount,0),if(old.payment_date_first is null and new.payment_count>0,'2022-05-20',old.payment_date_first),if(new.payment_count>0,'2022-05-20',old.payment_date_last),nvl(old.payment_count,0)+nvl(new.payment_count,0),nvl(old.payment_amount,0)+nvl(new.payment_amount,0),nvl(new.payment_last_30d_count,0),nvl(new.payment_last_30d_amount,0)
from
dwt_user_topic old
full outer join
(selectuser_id,sum(if(dt='2022-05-20',login_count,0)) login_count,sum(if(dt='2022-05-20',order_count,0)) order_count,sum(if(dt='2022-05-20',order_amount,0)) order_amount,sum(if(dt='2022-05-20',payment_count,0)) payment_count,sum(if(dt='2022-05-20',payment_amount,0)) payment_amount,sum(if(login_count>0,1,0)) login_last_30d_count,sum(order_count) order_last_30d_count,sum(order_amount) order_last_30d_amount,sum(payment_count) payment_last_30d_count,sum(payment_amount) payment_last_30d_amountfrom dws_user_action_daycountwhere dt>=date_add( '2022-05-20',-30)group by user_id
)new
on old.user_id=new.user_id;

3）查询数据

select * from dwt_user_topic limit 5;

3.商品主题宽表

1）建表语句

create external table dwt_sku_topic
(sku_id string comment 'sku_id',spu_id string comment 'spu_id',order_last_30d_count bigint comment '最近30日被下单次数',order_last_30d_num bigint comment '最近30日被下单件数',order_last_30d_amount decimal(16,2)  comment '最近30日被下单金额',order_count bigint comment '累积被下单次数',order_num bigint comment '累积被下单件数',order_amount decimal(16,2) comment '累积被下单金额',payment_last_30d_count   bigint  comment '最近30日被支付次数',payment_last_30d_num bigint comment '最近30日被支付件数',payment_last_30d_amount  decimal(16,2) comment '最近30日被支付金额',payment_count   bigint  comment '累积被支付次数',payment_num bigint comment '累积被支付件数',payment_amount  decimal(16,2) comment '累积被支付金额',refund_last_30d_count bigint comment '最近三十日退款次数',refund_last_30d_num bigint comment '最近三十日退款件数',refund_last_30d_amount decimal(16,2) comment '最近三十日退款金额',refund_count bigint comment '累积退款次数',refund_num bigint comment '累积退款件数',refund_amount decimal(16,2) comment '累积退款金额',cart_last_30d_count bigint comment '最近30日被加入购物车次数',cart_count bigint comment '累积被加入购物车次数',favor_last_30d_count bigint comment '最近30日被收藏次数',favor_count bigint comment '累积被收藏次数',appraise_last_30d_good_count bigint comment '最近30日好评数',appraise_last_30d_mid_count bigint comment '最近30日中评数',appraise_last_30d_bad_count bigint comment '最近30日差评数',appraise_last_30d_default_count bigint comment '最近30日默认评价数',appraise_good_count bigint comment '累积好评数',appraise_mid_count bigint comment '累积中评数',appraise_bad_count bigint comment '累积差评数',appraise_default_count bigint comment '累积默认评价数')COMMENT '商品主题宽表'
stored as parquet
location '/warehouse/gmall/dwt/dwt_sku_topic/'
tblproperties ("parquet.compression"="lzo");

2）数据导入

insert overwrite table dwt_sku_topic
select nvl(new.sku_id,old.sku_id),sku_info.spu_id,nvl(new.order_count30,0),nvl(new.order_num30,0),nvl(new.order_amount30,0),nvl(old.order_count,0) + nvl(new.order_count,0),nvl(old.order_num,0) + nvl(new.order_num,0),nvl(old.order_amount,0) + nvl(new.order_amount,0),nvl(new.payment_count30,0),nvl(new.payment_num30,0),nvl(new.payment_amount30,0),nvl(old.payment_count,0) + nvl(new.payment_count,0),nvl(old.payment_num,0) + nvl(new.payment_count,0),nvl(old.payment_amount,0) + nvl(new.payment_count,0),nvl(new.refund_count30,0),nvl(new.refund_num30,0),nvl(new.refund_amount30,0),nvl(old.refund_count,0) + nvl(new.refund_count,0),nvl(old.refund_num,0) + nvl(new.refund_num,0),nvl(old.refund_amount,0) + nvl(new.refund_amount,0),nvl(new.cart_count30,0),nvl(old.cart_count,0) + nvl(new.cart_count,0),nvl(new.favor_count30,0),nvl(old.favor_count,0) + nvl(new.favor_count,0),nvl(new.appraise_good_count30,0),nvl(new.appraise_mid_count30,0),nvl(new.appraise_bad_count30,0),nvl(new.appraise_default_count30,0)  ,nvl(old.appraise_good_count,0) + nvl(new.appraise_good_count,0),nvl(old.appraise_mid_count,0) + nvl(new.appraise_mid_count,0),nvl(old.appraise_bad_count,0) + nvl(new.appraise_bad_count,0),nvl(old.appraise_default_count,0) + nvl(new.appraise_default_count,0)
from
dwt_sku_topic old
full outer join
(select sku_id,sum(if(dt='2022-05-20', order_count,0 )) order_count,sum(if(dt='2022-05-20',order_num ,0 ))  order_num, sum(if(dt='2022-05-20',order_amount,0 )) order_amount ,sum(if(dt='2022-05-20',payment_count,0 )) payment_count,sum(if(dt='2022-05-20',payment_num,0 )) payment_num,sum(if(dt='2022-05-20',payment_amount,0 )) payment_amount,sum(if(dt='2022-05-20',refund_count,0 )) refund_count,sum(if(dt='2022-05-20',refund_num,0 )) refund_num,sum(if(dt='2022-05-20',refund_amount,0 )) refund_amount,  sum(if(dt='2022-05-20',cart_count,0 )) cart_count,sum(if(dt='2022-05-20',favor_count,0 )) favor_count,sum(if(dt='2022-05-20',appraise_good_count,0 )) appraise_good_count,  sum(if(dt='2022-05-20',appraise_mid_count,0 ) ) appraise_mid_count ,sum(if(dt='2022-05-20',appraise_bad_count,0 )) appraise_bad_count,  sum(if(dt='2022-05-20',appraise_default_count,0 )) appraise_default_count,sum(order_count) order_count30 ,sum(order_num) order_num30,sum(order_amount) order_amount30,sum(payment_count) payment_count30,sum(payment_num) payment_num30,sum(payment_amount) payment_amount30,sum(refund_count) refund_count30,sum(refund_num) refund_num30,sum(refund_amount) refund_amount30,sum(cart_count) cart_count30,sum(favor_count) favor_count30,sum(appraise_good_count) appraise_good_count30,sum(appraise_mid_count) appraise_mid_count30,sum(appraise_bad_count) appraise_bad_count30,sum(appraise_default_count) appraise_default_count30 from dws_sku_action_daycountwhere dt >= date_add ('2022-05-20', -30)group by sku_id
)new
on new.sku_id = old.sku_id
left join
(select * from dwd_dim_sku_info where dt='2022-05-20') sku_info
on nvl(new.sku_id,old.sku_id)= sku_info.id;

3）查询数据

select * from dwt_sku_topic limit 5;

4.活动主题宽表

1）建表语句

create external table dwt_activity_topic(`id` string COMMENT '编号',`activity_name` string  COMMENT '活动名称',`activity_type` string  COMMENT '活动类型',`start_time` string  COMMENT '开始时间',`end_time` string  COMMENT '结束时间',`create_time` string  COMMENT '创建时间',`display_day_count` bigint COMMENT '当日曝光次数',`order_day_count` bigint COMMENT '当日下单次数',`order_day_amount` decimal(20,2) COMMENT '当日下单金额',`payment_day_count` bigint COMMENT '当日支付次数',`payment_day_amount` decimal(20,2) COMMENT '当日支付金额',`display_count` bigint COMMENT '累积曝光次数',`order_count` bigint COMMENT '累积下单次数',`order_amount` decimal(20,2) COMMENT '累积下单金额',`payment_count` bigint COMMENT '累积支付次数',`payment_amount` decimal(20,2) COMMENT '累积支付金额'
) COMMENT '活动主题宽表'
stored as parquet
location '/warehouse/gmall/dwt/dwt_activity_topic/'
tblproperties ("parquet.compression"="lzo");

2）数据导入

insert overwrite table dwt_activity_topic
selectnvl(new.id,old.id),nvl(new.activity_name,old.activity_name),nvl(new.activity_type,old.activity_type),nvl(new.start_time,old.start_time),nvl(new.end_time,old.end_time),nvl(new.create_time,old.create_time),nvl(new.display_count,0),nvl(new.order_count,0),nvl(new.order_amount,0.0),nvl(new.payment_count,0),nvl(new.payment_amount,0.0),nvl(new.display_count,0)+nvl(old.display_count,0),nvl(new.order_count,0)+nvl(old.order_count,0),nvl(new.order_amount,0.0)+nvl(old.order_amount,0.0),nvl(new.payment_count,0)+nvl(old.payment_count,0),nvl(new.payment_amount,0.0)+nvl(old.payment_amount,0.0)
from
(select*from dwt_activity_topic
)old
full outer join
(select*from dws_activity_info_daycountwhere dt='2022-05-20'
)new
on old.id=new.id;

3）查询数据

select * from dwt_activity_topic limit 5;

5.地区主题宽表

1）建表语句

create external table dwt_area_topic(`id` bigint COMMENT '编号',`province_name` string COMMENT '省份名称',`area_code` string COMMENT '地区编码',`iso_code` string COMMENT 'iso编码',`region_id` string COMMENT '地区ID',`region_name` string COMMENT '地区名称',`login_day_count` string COMMENT '当天活跃设备数',`login_last_30d_count` string COMMENT '最近30天活跃设备数',`order_day_count` bigint COMMENT '当天下单次数',`order_day_amount` decimal(16,2) COMMENT '当天下单金额',`order_last_30d_count` bigint COMMENT '最近30天下单次数',`order_last_30d_amount` decimal(16,2) COMMENT '最近30天下单金额',`payment_day_count` bigint COMMENT '当天支付次数',`payment_day_amount` decimal(16,2) COMMENT '当天支付金额',`payment_last_30d_count` bigint COMMENT '最近30天支付次数',`payment_last_30d_amount` decimal(16,2) COMMENT '最近30天支付金额'
) COMMENT '地区主题宽表'
stored as parquet
location '/warehouse/gmall/dwt/dwt_area_topic/'
tblproperties ("parquet.compression"="lzo");

2）数据导入

insert overwrite table dwt_area_topic
selectnvl(old.id,new.id),nvl(old.province_name,new.province_name),nvl(old.area_code,new.area_code),nvl(old.iso_code,new.iso_code),nvl(old.region_id,new.region_id),nvl(old.region_name,new.region_name),nvl(new.login_day_count,0),nvl(new.login_last_30d_count,0),nvl(new.order_day_count,0),nvl(new.order_day_amount,0.0),nvl(new.order_last_30d_count,0),nvl(new.order_last_30d_amount,0.0),nvl(new.payment_day_count,0),nvl(new.payment_day_amount,0.0),nvl(new.payment_last_30d_count,0),nvl(new.payment_last_30d_amount,0.0)
from
(select*from dwt_area_topic
)old
full outer join
(selectid,province_name,area_code,iso_code,region_id,region_name,sum(if(dt='2022-05-20',login_count,0)) login_day_count,sum(if(dt='2022-05-20',order_count,0)) order_day_count,sum(if(dt='2022-05-20',order_amount,0.0)) order_day_amount,sum(if(dt='2022-05-20',payment_count,0)) payment_day_count,sum(if(dt='2022-05-20',payment_amount,0.0)) payment_day_amount,sum(login_count) login_last_30d_count,sum(order_count) order_last_30d_count,sum(order_amount) order_last_30d_amount,sum(payment_count) payment_last_30d_count,sum(payment_amount) payment_last_30d_amountfrom dws_area_stats_daycountwhere dt>=date_add('2022-05-20',-30)group by id,province_name,area_code,iso_code,region_id,region_name
)new
on old.id=new.id;

3）查询数据

select * from dwt_area_topic limit 5;

二、数据导入脚本

vim dws_to_dwt.sh

#!/bin/bashAPP=default
hive=/training/hive/bin/hive# 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
if [ -n "$1" ] ;thendo_date=$1
else do_date=`date -d "-1 day" +%F`
fisql="
set mapreduce.job.queuename=default;
insert overwrite table ${APP}.dwt_uv_topic
selectnvl(new.mid_id,old.mid_id),nvl(new.model,old.model),nvl(new.brand,old.brand),if(old.mid_id is null,'$do_date',old.login_date_first),if(new.mid_id is not null,'$do_date',old.login_date_last),if(new.mid_id is not null, new.login_count,0),nvl(old.login_count,0)+if(new.login_count>0,1,0)
from
(select*from ${APP}.dwt_uv_topic
)old
full outer join
(select*from ${APP}.dws_uv_detail_daycountwhere dt='$do_date'
)new
on old.mid_id=new.mid_id;insert overwrite table ${APP}.dwt_user_topic
selectnvl(new.user_id,old.user_id),if(old.login_date_first is null and new.login_count>0,'$do_date',old.login_date_first),if(new.login_count>0,'$do_date',old.login_date_last),nvl(old.login_count,0)+if(new.login_count>0,1,0),nvl(new.login_last_30d_count,0),if(old.order_date_first is null and new.order_count>0,'$do_date',old.order_date_first),if(new.order_count>0,'$do_date',old.order_date_last),nvl(old.order_count,0)+nvl(new.order_count,0),nvl(old.order_amount,0)+nvl(new.order_amount,0),nvl(new.order_last_30d_count,0),nvl(new.order_last_30d_amount,0),if(old.payment_date_first is null and new.payment_count>0,'$do_date',old.payment_date_first),if(new.payment_count>0,'$do_date',old.payment_date_last),nvl(old.payment_count,0)+nvl(new.payment_count,0),nvl(old.payment_amount,0)+nvl(new.payment_amount,0),nvl(new.payment_last_30d_count,0),nvl(new.payment_last_30d_amount,0)
from
${APP}.dwt_user_topic old
full outer join
(selectuser_id,sum(if(dt='$do_date',login_count,0)) login_count,sum(if(dt='$do_date',order_count,0)) order_count,sum(if(dt='$do_date',order_amount,0)) order_amount,sum(if(dt='$do_date',payment_count,0)) payment_count,sum(if(dt='$do_date',payment_amount,0)) payment_amount,sum(if(login_count>0,1,0)) login_last_30d_count,sum(order_count) order_last_30d_count,sum(order_amount) order_last_30d_amount,sum(payment_count) payment_last_30d_count,sum(payment_amount) payment_last_30d_amountfrom ${APP}.dws_user_action_daycountwhere dt>=date_add( '$do_date',-30)group by user_id
)new
on old.user_id=new.user_id;insert overwrite table ${APP}.dwt_sku_topic
select nvl(new.sku_id,old.sku_id),sku_info.spu_id,nvl(new.order_count30,0),nvl(new.order_num30,0),nvl(new.order_amount30,0),nvl(old.order_count,0) + nvl(new.order_count,0),nvl(old.order_num,0) + nvl(new.order_num,0),nvl(old.order_amount,0) + nvl(new.order_amount,0),nvl(new.payment_count30,0),nvl(new.payment_num30,0),nvl(new.payment_amount30,0),nvl(old.payment_count,0) + nvl(new.payment_count,0),nvl(old.payment_num,0) + nvl(new.payment_num,0),nvl(old.payment_amount,0) + nvl(new.payment_amount,0),nvl(new.refund_count30,0),nvl(new.refund_num30,0),nvl(new.refund_amount30,0),nvl(old.refund_count,0) + nvl(new.refund_count,0),nvl(old.refund_num,0) + nvl(new.refund_num,0),nvl(old.refund_amount,0) + nvl(new.refund_amount,0),nvl(new.cart_count30,0),nvl(old.cart_count,0) + nvl(new.cart_count,0),nvl(new.favor_count30,0),nvl(old.favor_count,0) + nvl(new.favor_count,0),nvl(new.appraise_good_count30,0),nvl(new.appraise_mid_count30,0),nvl(new.appraise_bad_count30,0),nvl(new.appraise_default_count30,0)  ,nvl(old.appraise_good_count,0) + nvl(new.appraise_good_count,0),nvl(old.appraise_mid_count,0) + nvl(new.appraise_mid_count,0),nvl(old.appraise_bad_count,0) + nvl(new.appraise_bad_count,0),nvl(old.appraise_default_count,0) + nvl(new.appraise_default_count,0)
from
(selectsku_id,spu_id,order_last_30d_count,order_last_30d_num,order_last_30d_amount,order_count,order_num,order_amount  ,payment_last_30d_count,payment_last_30d_num,payment_last_30d_amount,payment_count,payment_num,payment_amount,refund_last_30d_count,refund_last_30d_num,refund_last_30d_amount,refund_count,refund_num,refund_amount,cart_last_30d_count,cart_count,favor_last_30d_count,favor_count,appraise_last_30d_good_count,appraise_last_30d_mid_count,appraise_last_30d_bad_count,appraise_last_30d_default_count,appraise_good_count,appraise_mid_count,appraise_bad_count,appraise_default_count from ${APP}.dwt_sku_topic
)old
full outer join
(select sku_id,sum(if(dt='$do_date', order_count,0 )) order_count,sum(if(dt='$do_date',order_num ,0 ))  order_num, sum(if(dt='$do_date',order_amount,0 )) order_amount ,sum(if(dt='$do_date',payment_count,0 )) payment_count,sum(if(dt='$do_date',payment_num,0 )) payment_num,sum(if(dt='$do_date',payment_amount,0 )) payment_amount,sum(if(dt='$do_date',refund_count,0 )) refund_count,sum(if(dt='$do_date',refund_num,0 )) refund_num,sum(if(dt='$do_date',refund_amount,0 )) refund_amount,  sum(if(dt='$do_date',cart_count,0 )) cart_count,sum(if(dt='$do_date',favor_count,0 )) favor_count,sum(if(dt='$do_date',appraise_good_count,0 )) appraise_good_count,  sum(if(dt='$do_date',appraise_mid_count,0 ) ) appraise_mid_count ,sum(if(dt='$do_date',appraise_bad_count,0 )) appraise_bad_count,  sum(if(dt='$do_date',appraise_default_count,0 )) appraise_default_count,sum(order_count) order_count30 ,sum(order_num) order_num30,sum(order_amount) order_amount30,sum(payment_count) payment_count30,sum(payment_num) payment_num30,sum(payment_amount) payment_amount30,sum(refund_count) refund_count30,sum(refund_num) refund_num30,sum(refund_amount) refund_amount30,sum(cart_count) cart_count30,sum(favor_count) favor_count30,sum(appraise_good_count) appraise_good_count30,sum(appraise_mid_count) appraise_mid_count30,sum(appraise_bad_count) appraise_bad_count30,sum(appraise_default_count) appraise_default_count30 from ${APP}.dws_sku_action_daycountwhere dt >= date_add ('$do_date', -30)group by sku_id
)new
on new.sku_id = old.sku_id
left join
(select * from ${APP}.dwd_dim_sku_info where dt='$do_date') sku_info
on nvl(new.sku_id,old.sku_id)= sku_info.id;insert overwrite table ${APP}.dwt_activity_topic
selectnvl(new.id,old.id),nvl(new.activity_name,old.activity_name),nvl(new.activity_type,old.activity_type),nvl(new.start_time,old.start_time),nvl(new.end_time,old.end_time),nvl(new.create_time,old.create_time),nvl(new.display_count,0),nvl(new.order_count,0),nvl(new.order_amount,0.0),nvl(new.payment_count,0),nvl(new.payment_amount,0.0),nvl(new.display_count,0)+nvl(old.display_count,0),nvl(new.order_count,0)+nvl(old.order_count,0),nvl(new.order_amount,0.0)+nvl(old.order_amount,0.0),nvl(new.payment_count,0)+nvl(old.payment_count,0),nvl(new.payment_amount,0.0)+nvl(old.payment_amount,0.0)
from
(select*from ${APP}.dwt_activity_topic
)old
full outer join
(select*from ${APP}.dws_activity_info_daycountwhere dt='$do_date'
)new
on old.id=new.id;insert overwrite table ${APP}.dwt_area_topic
selectnvl(old.id,new.id),nvl(old.province_name,new.province_name),nvl(old.area_code,new.area_code),nvl(old.iso_code,new.iso_code),nvl(old.region_id,new.region_id),nvl(old.region_name,new.region_name),nvl(new.login_day_count,0),nvl(new.login_last_30d_count,0),nvl(new.order_day_count,0),nvl(new.order_day_amount,0.0),nvl(new.order_last_30d_count,0),nvl(new.order_last_30d_amount,0.0),nvl(new.payment_day_count,0),nvl(new.payment_day_amount,0.0),nvl(new.payment_last_30d_count,0),nvl(new.payment_last_30d_amount,0.0)
from
(select*from ${APP}.dwt_area_topic
)old
full outer join
(selectid,province_name,area_code,iso_code,region_id,region_name,sum(if(dt='$do_date',login_count,0)) login_day_count,sum(if(dt='$do_date',order_count,0)) order_day_count,sum(if(dt='$do_date',order_amount,0.0)) order_day_amount,sum(if(dt='$do_date',payment_count,0)) payment_day_count,sum(if(dt='$do_date',payment_amount,0.0)) payment_day_amount,sum(login_count) login_last_30d_count,sum(order_count) order_last_30d_count,sum(order_amount) order_last_30d_amount,sum(payment_count) payment_last_30d_count,sum(payment_amount) payment_last_30d_amountfrom ${APP}.dws_area_stats_daycountwhere dt>=date_add('$do_date',-30)group by id,province_name,area_code,iso_code,region_id,region_name
)new
on old.id=new.id;
"$hive -e "$sql"

2）增加脚本执行权限：chmod 777 dws_to_dwt.sh

3）执行脚本导入数据： dws_to_dwt.sh 2022-05-21

4）查看导入数据

select * from dwt_uv_topic limit 5;
select * from dwt_user_topic limit 5;
select * from dwt_sku_topic limit 5;
select * from dwt_activity_topic limit 5;
select * from dwt_area_topic limit 5;

作者水平低，如有错误，恳请指正！谢谢！！！！！

本篇文章参考尚硅谷大数据项目写成！！！

数据仓库搭建DWT层相关推荐

数据仓库搭建DWD层
本篇只是DWD层,其他内容请关注我的博客!在<项目>专栏里!!! 本篇文章参考尚硅谷大数据项目写成! 目录一.用户行为日志 1.1日志格式 1.2get_json_object函数使用 ...
数据仓库搭建ODS层
其他内容请关注我的博客!在<项目>专栏里!!! 目录一.用户行为数据 1.1创建日志表 1.2ODS层加载数据脚本二.业务数据 2.1hive建表 2.2ODS层加载数据脚本一.用户 ...
数据仓库搭建DWS层
本篇只是DWS层,其他内容请关注我的博客!在<项目>专栏里!!! 本篇文章参考尚硅谷大数据项目写成! 一.业务术语 1)用户用户以设备为判断标准,在移动统计中,每个独立设备认为是一个独立 ...
数据仓库搭建ADS层
本篇只是ADS层,其他内容请关注我的博客!在<项目>专栏里!!! 本篇文章参考尚硅谷大数据项目写成! 目录搭建ADS层一.设备主题 1.1活跃设备数(日.周.月) 1.2 每日新增设备 ...
层数据仓库_小尝试：基于指标体系的数据仓库搭建和数据可视化
关于作者:小姬,某知名互联网公司产品专家,对数据采集.生产.加工有所了解,期望多和大家交流数据知识,以数据作为提出好问题的基础,挖掘商业价值. 0x00 前言我将整理文章分享数据工作中的经验,因为业 ...
10.数据仓库搭建之DWD层搭建
数据仓库搭建之DWD层搭建我们在设计项目中DWD层时,需要注意以下几点: 1)DWD层的设计依据维度建模理论,该层存储维度模型当中的事实表. 2)DWD层的数据存储格式为ORC列式存储结合snapp ...
11.数据仓库搭建之DWS层搭建
数据仓库搭建之DWS层搭建在搭建该层时,我们需要注意的是: 1)本层的设计主要参考指标体系 2)DWS层数据的数据存储格式为orc列式存储+snappy压缩. 3)DWS层表名的命名规范为:dws ...
数据仓库之电商数仓-- 3.3、电商数据仓库系统(DWT层)
目录八.数仓搭建-DWT层 8.1 访客主题 8.2 用户主题 8.3 商品主题 8.4 优惠券主题 8.5 活动主题 8.6 地区主题 8.7 DWT层首日数据导入脚本 8.8 DWT层每日数据导 ...
6.数据仓库搭建之数据仓库设计
数据仓库搭建之数据仓库设计 1.数据仓库的分层规划本项目的分层规划如下图所示: 对于原始数据层(ODS):该层我们存放的是未经处理的原始数据,结构上与源系统保持一致,这是数据仓库的数据准备区. 对于 ...

数据仓库搭建DWT层

一、DWT层

1.设备主题宽表

2.会员主题宽表

3.商品主题宽表

4.活动主题宽表

5.地区主题宽表

二、数据导入脚本

数据仓库搭建DWT层相关推荐

最新文章

热门文章