/usr/local/kafka/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list 192.168.1.210:9092 -topic test --time -2 （--time -1为查看kafka最大偏移量）
test:2:0
test:1:0
test:0:0

1). 首先查看存放在zookeeper上面的offset
/usr/local/zookeeper/bin/zkCli.sh -server 192.168.1.210:2181
#] get /consumers/test_group/offsets/test/0
16051516
cZxid = 0x10050df5c
ctime = Wed Sep 19 16:15:50 CST 2018
mZxid = 0x100691806
mtime = Thu Nov 08 11:30:08 CST 2018
pZxid = 0x10050df5c
cversion = 0
dataVersion = 6433
aclVersion = 0
ephemeralOwner = 0x0
dataLength = 8
numChildren = 0
2). 把导入到Hbase的sparkstreaming任务kill掉，然后把hbase的数据全部导入到HDFS上
3). 然后编写导入到HDFS上面的spark代码，设置相同的group_id，则会读取之前存放在zookeeper中的offset

我这边保存的文件以日期为命名格式（每天凌晨执行一次，在sparkstreaming里面设置）
hdfs dfs -ls /data/res/20181108
drwxr-xr-x   - hadoop    hadoop          0 2018-11-08 00:00 /data/res/20181108/part-00000
drwxr-xr-x   - hadoop    hadoop          0 2018-11-08 00:00 /data/res/20181108/part-00001
drwxr-xr-x   - hadoop    hadoop          0 2018-11-08 00:00 /data/res/20181108/part-00002

]$ cat hdfs2hive.sh
#! /bin/shrm -f /data/hadoop/data/*hdfs="/usr/local/hadoop/bin/hdfs"
hive="/usr/local/hive/bin/hive"yesterday=`date +"%Y%m%d" -d "-1 days"`
today=`date +"%Y%m%d"`## 合并hdfs文件夹到本地
hdfs_dir="/data/soda_yx/$yesterday"res=`$hdfs dfs -ls $hdfs_dir`if [ -n "$res" ];then$hdfs dfs -getmerge $hdfs_dir "/data/hadoop/data/data/res.data"if [ $? -eq 0 ];thenecho "merge to local file is success."fi
fi## 按天过滤出文件
dir="/data/hadoop/data"
`cat $dir/res.data |awk -F"\t" '{if($36=="'"$yesterday"'") print $0}' > $dir/$yesterday`
`cat $dir/res.data |awk -F"\t" '{if($36=="'"$today"'") print $0}' > $dir/$today`
if [ $? -eq 0 ];thenecho "filter file is success."
fi## 插入数据到hive中去
if [ -n "$dir/$yesterday" ];then$hive -e "LOAD DATA LOCAL INPATH '$local_file' INTO TABLE xxx.soda_report partition(dt='$yesterday')"if [ $? -eq 0 ];thenecho "Import local data to hive is success."fi
fiif [ -n "$dir/$today" ];then$hive -e "LOAD DATA LOCAL INPATH '$dir/$today' INTO TABLE xxx.soda_report partition(dt='$today')"if [ $? -eq 0 ];thenecho "Import local data to hive is success."fi
fi