MapReduce读写orc文件
2024-04-24 08:40:33
博客地址:http://www.fanlegefan.com
文章地址:http://www.fanlegefan.com/archives/mapreduceorc/
MapReduce 读取ORC格式文件
创建orc格式hive表
create table test_orc(name string,age int) stored as orc查看hive表结构
show create table test_orc
CREATE TABLE `test_orc`(`name` string, `age` int)
ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
LOCATION'hdfs://localhost:9000/user/work/warehouse/test_orc'
TBLPROPERTIES ('transient_lastDdlTime'='1502868725')插入测试数据
insert into table test_orc select name ,age from test limit 10;jar依赖
<dependency><groupId>org.apache.orc</groupId><artifactId>orc-core</artifactId><version>1.2.3</version>
</dependency>
<dependency><groupId>org.apache.orc</groupId><artifactId>orc-mapreduce</artifactId><version>1.1.0</version>
</dependency>
<dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-mapreduce-client-core</artifactId><version>2.6.0</version>
</dependency>MR读取ORC格式文件代码如下
package com.fan.hadoop.orc;import com.fan.hadoop.parquet.thrift.ParquetThriftWriterMR;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.orc.mapred.OrcStruct;
import org.apache.orc.mapreduce.OrcInputFormat;
import java.io.IOException;public class OrcReaderMR {public static class OrcMap extends Mapper<NullWritable,OrcStruct,Text,IntWritable> {// Assume the ORC file has type: struct<s:string,i:int>public void map(NullWritable key, OrcStruct value,Context output) throws IOException, InterruptedException {// take the first field as the key and the second field as the valueoutput.write((Text) value.getFieldValue(0),(IntWritable) value.getFieldValue(1));}}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();Job job = Job.getInstance(conf);job.setJarByClass(ParquetThriftWriterMR.class);job.setJobName("parquetthrfit");String in = "hdfs://localhost:9000/user/work/warehouse/test_orc";String out = "hdfs://localhost:9000/test/orc";job.setMapperClass(OrcMap.class);OrcInputFormat.addInputPath(job, new Path(in));job.setInputFormatClass(OrcInputFormat.class);job.setNumReduceTasks(0);job.setOutputFormatClass(TextOutputFormat.class);FileOutputFormat.setOutputPath(job, new Path(out));job.waitForCompletion(true);}}
查看生成文件
hadoop dfs -cat /test/orc/part-m-00000kafka 14
tensflow 98
hadoop 34
hbase 68
flume 57
kafka 99
kafka 28
flume 24
tensflow 35
flume 44MR写ORC格式文件
package com.fan.hadoop.orc;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.orc.OrcConf;
import org.apache.orc.TypeDescription;
import org.apache.orc.mapred.OrcStruct;
import org.apache.orc.mapreduce.OrcOutputFormat;
import java.io.IOException;public class OrcWriterMR {public static class OrcWriterMapperextends Mapper<LongWritable,Text,NullWritable,OrcStruct> {private TypeDescription schema =TypeDescription.fromString("struct<name:string,age:int>");private OrcStruct pair = (OrcStruct) OrcStruct.createValue(schema);private final NullWritable nada = NullWritable.get();private Text name = new Text();private IntWritable age = new IntWritable();public void map(LongWritable key, Text value,Context output) throws IOException, InterruptedException {if(!"".equals(value.toString())){String[] arr = value.toString().split("\t");name.set(arr[0]);age.set(Integer.valueOf(arr[1]));pair.setFieldValue(0, name);pair.setFieldValue(1,age);output.write(nada, pair);}}}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();OrcConf.MAPRED_OUTPUT_SCHEMA.setString(conf,"struct<name:string,age:int>");Job job = Job.getInstance(conf);job.setJarByClass(OrcWriterMR.class);job.setJobName("OrcWriterMR");String in = "hdfs://localhost:9000/user/work/warehouse/test/ddd.txt";String out = "hdfs://localhost:9000/test/orc2";job.setMapperClass(OrcWriterMapper.class);job.setInputFormatClass(TextInputFormat.class);job.setNumReduceTasks(0);job.setOutputFormatClass(OrcOutputFormat.class);FileInputFormat.addInputPath(job, new Path(in));OrcOutputFormat.setOutputPath(job, new Path(out));job.waitForCompletion(true);}
}
查看生成文件
#### 生成orc文件hadoop dfs -ls /test/orc2-rw-r--r-- 3 work supergroup 0 2017-08-16 17:45 /test/orc2/_SUCCESS
-rw-r--r-- 3 work supergroup 6314874 2017-08-16 17:45 /test/orc2/part-m-00000.orc将数据放到hive表路径下
hadoop fs -cp /test/orc2/part-m-00000.orc /user/work/warehouse/test_orc/在hive表中查看数据
hive> select * from test_orc limit 13;
OK
kafka 14
tensflow 98
hadoop 34
hbase 68
flume 57
kafka 99
kafka 28
flume 24
tensflow 35
flume 44
flume 44
tensflow 35
flume 24
Time taken: 0.045 seconds, Fetched: 13 row(s)
MapReduce读写orc文件相关推荐
- java读写orc文件_java读取hive的orc文件
hive API2 hive udf 读取HDFS的orc文件 package hive_udf_province; import org.apache.hadoop.conf.Configurati ...
- MapReduce基础开发之十读写ORC File
1.ORC File Orc是Hive特有的一种列式存储的文件格式,它有着非常高的压缩比和读取效率,因此很快取代了之前的RCFile,成为Hive中非常常用的一种文件格式. 2.编译ORC Jar包 ...
- MapReduce 读取ORC格式文件
1.创建orc格式hive表: create table test_orc(name string,age int) stored as orc 2.查看表结构: show create table ...
- spark SQL读取ORC文件从Driver启动到开始执行Task(或stage)间隔时间太长（计算Partition时间太长）且产出orc单个文件中stripe个数太多问题解决方案...
1.背景: 控制上游文件个数每天7000个,每个文件大小小于256M,50亿条+,orc格式.查看每个文件的stripe个数,500个左右,查询命令:hdfs fsck viewfs://hadoop ...
- 【MapReduce】MapReduce读写MySQL数据
MapReduce读写MySQL数据 数据 代码实现 自定义类来接收源数据 自定义类型来存储结果数据 Mapper阶段 Reducer阶段 Driver阶段 上传运行 打包 上传集群运行 使用MapR ...
- 利用Hudi Bootstrap转化现有Hive表的parquet/orc文件为Hudi表
前些天发现了一个巨牛的人工智能学习网站,通俗易懂,风趣幽默,忍不住分享一下给大家.点击跳转到网站:https://www.captainai.net/dongkelun 前言 在最开始学习Hudi源码 ...
- 大数据：Hive - ORC 文件存储格式
一.ORC File文件结构 ORC的全称是(Optimized Row Columnar),ORC文件格式是一种Hadoop生态圈中的列式存储格式,它的产生早在2013年初,最初产生自Apache ...
- Hive - ORC 文件存储格式详细解析
一.ORC File文件结构 ORC的全称是(Optimized Row Columnar),ORC文件格式是一种Hadoop生态圈中的列式存储格式,它的产生早在2013年初,最初产生自Apache ...
- 利用pandas读写HDF5文件
一.简介 HDF5(Hierarchical Data Formal)是用于存储大规模数值数据的较为理想的存储格式,文件后缀名为h5,存储读取速度非常快,且可在文件内部按照明确的层次存储数据,同一个H ...
最新文章
- [译文]Domain Driven Design Reference（三）—— 模型驱动设计的构建模块
- 编译与解释实践（1）－flex and bison 配置安装
- ARKit从入门到精通（3）-ARKit自定义实现
- devtools的ctrl加r_Chrome DevTools调试技巧
- oracle apex 发布应用,基于Oracle APEX的快速Web应用开发
- keras提取模型中的某一层_Tensorflow笔记：高级封装——Keras
- android网络游戏开发实战pdf_python爬虫开发与项目实战PDF高清文档下载
- 小米线刷 mysql_小米6刷机(线刷)第三方ROM——LineageOS
- 机器学习-神经网络NN
- 极光推送入门教程-后端
- 服务器抓取MIUI ota信息,官方科普｜关于MIUI系统OTA
- iOS 常用第三方开源框架介绍
- matlab中怎么输入特殊符号
- bzoj 1814 Fornula 1
- git fetch 理解
- 阿里生鲜布局重要抓手！它如何解决全球生鲜商家痛点？
- clang diagnostic用例
- 对抗拖延症最直接有效的方法
- ABB机器人基础培训资料整理与总结
- web.Response