hive难点以及例子SXT

一、炸裂函数注意点

1、只取炸裂字段

select
explode(字段名)
from
表名

2、原表中的字段和炸裂字段

select
表中字段,
列的别名
from
表名
lateral view explode(字段名) 炸裂开表的别名 as 列的别名

二、partition by 与 distribute by

partition by name order by orderdate 与distribute by name sort by orderdate 结果是一样的。

三、窗口函数

current row :当前行
n perceding :往前n行数据
n follewing :往后n行数据
unbonded : 起点 ，unbounded perceding 表示从前面的起点 unbounded following :表示到后面的终点

ntile(n):把有序分区中的行分发到指定数据的组中，各个组有编号，编号从1开始，对于每一行，ntile返回此行所属的组的编号。理解： 把数据根据顺序分N 组，可以用于取百分子多少的数据

排名函数是窗口函数

rank():1134
dense_rank():1123
row_number():1234

distinct的优化

select
shop,
count( distinct id) uv
from
visit
group by shop
;--优化用group by 实现去重select
shop count(id) uv
from
(
select
shop,
id
from visit
group by shop,id
) t1
group by shop
;

三、例题

格式化日期

date_format(regexp_replace(data_dt,'/','-'),'yyyy-MM');

例子1

[外链图片转存失败(img-6cmp9Zps-1569033300770)(C:\Users\12847\Desktop\NOTES\Apart03Hadoop\imgs\例子1.png)]

--数据准备  visitu01 2017/1/21 5
u02 2017/1/23 6
u03 2017/1/22 8
u04 2017/1/20 3
u01 2017/1/23 6
u01 2017/2/21 8
u02 2017/1/23 6
u01 2017/2/22 4
-- 创建表
create table visit1(
userId string,
visitDate string,
visitCount int)
row format delimited fields terminated by " "
;
-- 查询过程
1、格式化日期
select
userId,
date_format(regexp_replace(visitDate,'/','-'),'yyyy-MM') visiteDate,
visitCount
from
visit1
;2、统计出用户每个月的访问次数
select
userId,
visitDate,
sum(visitCount) count
from
(select
userId,
date_format(regexp_replace(visitDate,'/','-'),'yyyy-MM') visitDate,
visitCount
from
visit1)t1
group by userId,visitDate
;3、根据distridute by userId visitDate sort by visitDate 来进行 得到select
userId,
visitDate,
count,
sum(count) over(distribute by userId sort by visitDate asc) visit_count
from
(select
userId,
visitDate,
sum(visitCount) count
from
(select
userId,
date_format(regexp_replace(visitDate,'/','-'),'yyyy-MM') visitDate,
visitCount
from
visit1)t1
group by userId,visitDate)t2
;

结果：

[外链图片转存失败(img-m2LLGXsV-1569033300772)(C:\Users\12847\Desktop\NOTES\Apart03Hadoop\imgs\例子l结果.png)]

例子2

[外链图片转存失败(img-oYyIIlQe-1569033300773)(C:\Users\12847\Desktop\NOTES\Apart03Hadoop\imgs\例子2.png)]

--数据
u1 a
u2 b
u1 b
b1 a
u3 c
u4 b
u1 a
u2 c
u5 b
u4 b
u6 c
u2 c
u1 b
u2 a
u2 a
u3 a
u5 a
u5 a
u5 a
--建表
create table visit2 (
user_id string,
shop string)
row format delimited fields terminated by ' '
;
-----查询过程
问题一
1、通过group by 来去重
select
shop,
user_id
from
visit2
group by shop,user_id
;
2、访客数统计
select
shop,
count(*) visit_count
from
(
select
shop,
user_id
from
visit2
group by shop,user_id
)t1
group by shop
;

结果

[外链图片转存失败(img-eSCAtFpC-1569033300773)(C:\Users\12847\Desktop\NOTES\Apart03Hadoop\imgs\例子2结果.png)]

问题二

1、每个店铺每个访客的访问次数
select
shop,
user_id,
count(*) visit_count
from visit2
group by shop,user_id
;
2、标记排序
select
shop,
user_id,
visit_count,
row_number() over(distribute by shop sort by visit_count desc) rank
from
(
select
shop,
user_id,
count(*) visit_count
from visit2
group by shop,user_id
)t1
;
3、得出访问top3
select
shop,
user_id,
visit_count
from
(
select
shop,
user_id,
visit_count,
row_number() over(distribute by shop sort by visit_count desc) rank
from
(
select
shop,
user_id,
count(*) visit_count
from visit2
group by shop,user_id
)t1
)t2
where rank<4
;

[外链图片转存失败(img-VjoxJGD9-1569033300774)(C:\Users\12847\Desktop\NOTES\Apart03Hadoop\imgs\例子3结果.png)]

例子3

问题一

[外链图片转存失败(img-E5TXASwk-1569033300775)(C:\Users\12847\Desktop\NOTES\Apart03Hadoop\imgs\例子31.png)]

补充：申领了一颗胡杨

--数据准备及建表
plant_carbon表
plant_id plant_name low_carbon
p001 梭梭树 17
p002 沙柳 19
p003 樟子树 146
p004 胡杨 215user_low_carbon表
user_id data_dt carbon
u-001 2017/1/2 10
u-001 2017/1/3 13
u-002 2017/1/3 16
u-002 2017/1/6 10
u-0012 2017/1/8 10
u-002 2017/1/9 10
...--查询
1、统计用户在2017年10月1日之前总的减少碳的排放量
select
user_id,
sum(carbon) carbon_sum
from
user_low_carbon
where data_fromat(regexp_replace(visitDate,'/','-'),'yyyy-MM')<'2017-10'
group by user_id
;
t12、取出换购胡杨树和沙柳的碳排放量
select
low_carbon p1
from plant_carbon
where plant_id='p004'
;
t2
select
low_carbon p2
from plant_carbon
where plant_id='p002'
;
t3
3、
select3、计算每个人审理沙柳的棵树
select
user_id,
floor((carbon_sum-p1)/p2) plant_count
from
t1,t2,t2
;
t44、按照申领沙柳排序 并取前10名,比他后一名多领几颗沙柳
select
user_id,
plant_count,
lead(plant_count,1,'9999-99-99')over(order by plant_count desc) lead_plant_count
from t4
limit 10
;
t55、求相差的棵树select
user_id,
plant_count,
(plant_count-lead_plant_count) plant_count_diff
from
t5
order by plant_count desc
------优化
在统计用户在2017年10月1日之前总的减少碳的排放量 排序desc ，limit 11;order by carbon_sum
limit 11

问题二:思路一

[外链图片转存失败(img-8VfVgaZp-1569033300775)(C:\Users\12847\Desktop\NOTES\Apart03Hadoop\imgs\例子32.png)]

--数据准备及建表
plant_carbon表
plant_id plant_name low_carbon
p001 梭梭树 17
p002 沙柳 19
p003 樟子树 146
p004 胡杨 215user_low_carbon表
user_id data_dt carbon
u-001 2017/1/2 10
u-001 2017/1/3 13
u-002 2017/1/3 16
u-002 2017/1/6 10
u-0012 2017/1/8 10
u-002 2017/1/9 10
...1、过滤出2017年且单日低碳排放量超过100g,格式化输出日期
select
user_id,
dete_format(regexp_replace(data_dt,'/','-'),'yyyy-MM-dd') data_dt，
sum(carbon) sum_carbon
from user_low_carbon
where substring(data_dt,1,4)='2017'
group by  user_id,data_dt
hiving sum_carbon>=100
;
t12、将前2行数据，以及后两行数据的日期放置当前行
select
user_id,
data_dt,
lag(data_dt,2,'1970-01-01') over(partition by user_id order by data_dt) lag2,
lag(data_dt,1,'1970-01-01') over(partition by user_id order by data_dt) lag1,
lead(data_dt,1,'1970-01-01') over(partition by user_id order by data_dt) lead1,
lead(data_dt,2,'1970-01-01') over(partition by user_id order by data_dt) lead2
from  t1
;
t23、计算当前日期跟前后两行时间的差值
select
user_id,
data_dt,
datadiff(data_dt,lag2) lag2_diff,
datadiff(data_dt,lag1) lag1_diff,
datadiff(data_dt,lead1) lead1_diff,
datadiff(data_dt,lead2) lead2_diff
from t2
;
t34、过滤出连续三天超过100g的用户
select
user_id,
data_dt
from t3
where
(lag2_diff=2 and lag1_diff=1)or(lag1_diff=1 and lead1_diff=-1)or(lead1_diff=-1 and lead2_diff=-2);t45、关联原表
select
user.user_id,
user.data_dt,
user.carbon
from
t4
join user_low_carbon user
on
t4.user_id = user.user_id and t4.data_dt = data_format(regexp_replace(user.data_dt,'/','-'),'yyyy-MM-dd') ;

问题二: 思路二

1、过滤出2017年且单日低碳排放量超过100g,格式化输出日期
select
user_id,
dete_format(regexp_replace(data_dt,'/','-'),'yyyy-MM-dd') data_dt，
sum(carbon) sum_carbon
from user_low_carbon
where substring(data_dt,1,4)='2017'
group by  user_id,data_dt
hiving sum_carbon>=100
;
t12、按照日期进行排序并且给每一天数据进行标记
select
user_id,
data_dt,
rank() over(partition user_id oder by data_dt) rank
from  t1
;t23、将日期减去当前的rank值
select
user_id,
data_dt,
date_sub(data_dt,rank) data_sub_rank
from t2
;
t33、对user_id 和 data_sub_rank 分组求count(*)>3select
user_id,
data_dt,
count(*) over(distribute by user_id,data_sub_rank sort by data_dt) data_dt_count
from t3
having data_dt_count>=3;
这里写的有问题:hving 必须与group by 连用，where 又在分组函数之前执行，所以得先做一个视图再进行判断

问题二: 思路三 MR

mapper 输出：(key:user+date, value:一行) 保证数据 同一个用户，按时间排序
groping:user  分组根据：user
reduce()：一个用户一个reduce
values:
{date = 1970-01-01---给初始值list = new ArrayList()values.for(if(date==1970-01-01){list.add(value);date=value.dt;}else{if(value.dt-dt==1){list.add(value);date=value.dt;}else{if(list.size>=3){context.wirte();list.clear;list.add(value);date=value.dt;}else{list.clear;list.add(value);date=value.dt;}}})if(list.size>=3){context.wirte();\}
}

groping:user 分组根据：user
reduce()：一个用户一个reduce
values:
{
date = 1970-01-01—给初始值
list = new ArrayList()
values.for(
if(date1970-01-01){
list.add(value);
date=value.dt;
}else{
if(value.dt-dt1){
list.add(value);
date=value.dt;
}else{
if(list.size>=3){
context.wirte();
list.clear;
list.add(value);
date=value.dt;
}else{
list.clear;
list.add(value);
date=value.dt;
}
}
}
)
if(list.size>=3){
context.wirte();
}
}