MapReduce解决乘用车辆和商用车辆的销售数据分析
2024-05-19 10:26:54
首先还是看下我们的需求
然后拿到我们的数据
可以看到我们的数据里面还有很多空值,是还没清洗的脏数据,一会我们处理的时候需要将其处理掉.
一.统计车辆不同用途的数量分布
package hadoop.MapReduce.car.Use;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;public class usecount {public static class UseMapper extends Mapper<LongWritable, Text,Text, IntWritable> {@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String[] lines = value.toString().split("\t");if (null != lines && lines.length > 10 && !lines[10].isEmpty()) {context.write(new Text(lines[10]), new IntWritable(1));}}}public static class UseReduce extends Reducer<Text,IntWritable,Text,IntWritable>{@Overrideprotected void reduce(Text key, Iterable<IntWritable> values, Context context) throwsIOException, InterruptedException {int count = 0;for (IntWritable value:values){count += value.get();}context.write(key,new IntWritable(count));}
}public static class UseDriver{public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {Configuration conf = new Configuration();Job job = new Job(conf);job.setJarByClass(UseDriver.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);job.setMapperClass(UseMapper.class);job.setReducerClass(UseReduce.class);FileInputFormat.addInputPath(job,new Path("D:\\a\\cars.txt"));FileOutputFormat.setOutputPath(job,new Path("D:\\a\\a1\\1"));job.waitForCompletion(true);}
}
}
可以看到我们这边用lines[10].isEmpty()去清洗了空格,若不想用这种方法还可以用try/catch将其抛出异常,
一开始我们用的line[10] != null,后来发现不行,还是会将空格输出出来.
二.统计山西省2013年每个月的汽车销售数量的比例
package hadoop.MapReduce.car.BiLi;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;public class bilicount {
// static int all = 0;
public static class BiliMapper extends Mapper<LongWritable, Text,Text,LongWritable>{@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String[] lines = value.toString().split("\t");if (null != lines && lines.length > 1&& !lines[0].isEmpty()) {if (lines[0].equals("山西省") && lines[4].equals("2013")) {context.write(new Text(lines[1]), new LongWritable(1));
// all++;}}}
}public static class BiliReduce extends Reducer<Text,LongWritable,Text, DoubleWritable> {double all = 0;Map<String,Long> maps = new HashMap<String,Long>();@Overrideprotected void reduce(Text key, Iterable<LongWritable> values, Context context) throwsIOException, InterruptedException {long count = (long)0;for (LongWritable value : values) {count += value.get();}all += count;maps.put(key.toString(),count);
// bili = count/all;
// context.write(key,new DoubleWritable(bili));}protected void cleanup(org.apache.hadoop.mapreduce.Reducer<Text,LongWritable,Text, DoubleWritable>.Context context) throws IOException, InterruptedException {Set<String> keySet = maps.keySet();for (String str : keySet) {long value = maps.get(str);double bili = value/all;context.write(new Text(str),new DoubleWritable(bili));}}
}public static class BiliDriver {public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {Configuration conf = new Configuration();Job job = new Job(conf);job.setJarByClass(BiliDriver.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(LongWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(DoubleWritable.class);job.setMapperClass(BiliMapper.class);job.setReducerClass(BiliReduce.class);FileInputFormat.addInputPath(job,new Path("D:\\a\\cars.txt"));FileOutputFormat.setOutputPath(job,new Path("D:\\a\\a2\\1"));job.waitForCompletion(true);}
}
}
可以看到统计总数,我们这边用了cleanup来释放,其实在map端加入一个静态全局变量来计算也是一个好办法,比起这个更加简单.
三.统计山西省2013年各市、区县的汽车销售的数量分布
package hadoop.MapReduce.car.FenBu;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;public class fenbu {
public static class FenbuMapper extends Mapper<LongWritable, Text,Text,IntWritable> {@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String[] lines = value.toString().split("\t");if (null != lines && lines.length > 4 && lines[2] != null && lines[3] != null) {if (lines[0].equals("山西省") && lines[4].equals("2013")) {context.write(new Text(lines[2]+"\t"+lines[3]),new IntWritable(1));}}}
}public static class FenbuReduce extends Reducer<Text,IntWritable,Text,IntWritable> {@Overrideprotected void reduce(Text key, Iterable<IntWritable> values, Context context)throws IOException, InterruptedException {int count = 0;for (IntWritable value:values){count += value.get();}context.write(key,new IntWritable(count));}
}public static class FenbuDriver{public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {Configuration conf = new Configuration();Job job = new Job(conf);job.setJarByClass(FenbuDriver.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);job.setMapperClass(FenbuMapper.class);job.setReducerClass(FenbuReduce.class);FileInputFormat.addInputPath(job,new Path("D:\\a\\cars.txt"));FileOutputFormat.setOutputPath(job,new Path("D:\\a\\a3"));job.waitForCompletion(true);}
}
}
可以看到这里用的lines[3] != null做得判断,这里还会输出空格,记得改为 !lines[3].isEmpty()
四.统计买车的男女比例
package hadoop.MapReduce.car.ManAndWoman;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;public class manwoman {
// static double people = 0;
public static class MWMapper extends Mapper<LongWritable, Text,Text, IntWritable> {@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String[] lines = value.toString().split("\t");if (null != lines && lines.length > 38 && !lines[38].isEmpty()) {context.write(new Text(lines[38]),new IntWritable(1));
// people++;}}
}public static class MWReduce extends Reducer<Text,IntWritable,Text, IntWritable> {@Overrideprotected void reduce(Text key, Iterable<IntWritable> values, Context context)throws IOException, InterruptedException {
// double bili;int count = 0;for (IntWritable value:values){count += value.get();}
// bili = count/people;context.write(key,new IntWritable(count));}
}public static class MWDriver{public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {Configuration conf = new Configuration();Job job = new Job(conf);job.setJarByClass(MWDriver.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);job.setMapperClass(MWMapper.class);job.setReducerClass(MWReduce.class);FileInputFormat.addInputPath(job,new Path("D:\\a\\cars.txt"));FileOutputFormat.setOutputPath(job,new Path("D:\\a\\a4\\3"));job.waitForCompletion(true);}
}
}
这边我是求的人数,题目求的比例,只要把注释的解开就行
五.统计车的所有权、车辆类型和品牌的数量分布
package hadoop.MapReduce.car.SuoYou;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;public class All {
public static class AMapper extends Mapper<LongWritable, Text,Text, IntWritable> {@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String[] lines = value.toString().split("\t");if (null != lines && lines.length > 9 && lines[9] != null) {context.write(new Text(lines[9]),new IntWritable(1));}}
}public static class AReduce extends Reducer<Text,IntWritable,Text,IntWritable> {@Overrideprotected void reduce(Text key, Iterable<IntWritable> values, Context context)throws IOException, InterruptedException {int count = 0;for (IntWritable value:values){count += value.get();}context.write(key,new IntWritable(count));}
}public static class ADriver{public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {Configuration conf = new Configuration();Job job = new Job(conf);job.setJarByClass(ADriver.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);job.setMapperClass(AMapper.class);job.setReducerClass(AReduce.class);FileInputFormat.addInputPath(job,new Path("D:\\a\\cars.txt"));FileOutputFormat.setOutputPath(job,new Path("D:\\a\\a5\\4"));job.waitForCompletion(true);}
}
}
六.统计不同品牌的车在每个月的销售量分布
package hadoop.MapReduce.car.PinPai;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;public class PP {
public static class PPMapper extends Mapper<LongWritable, Text,Text, IntWritable> {@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String[] lines = value.toString().split("\t");if (null != lines && lines.length > 7 && lines[1] != null && lines[7] != null) {context.write(new Text(lines[1]+"\t"+lines[7]),new IntWritable(1));}}
}public static class PPReduce extends Reducer<Text,IntWritable,Text,IntWritable> {@Overrideprotected void reduce(Text key, Iterable<IntWritable> values, Context context)throws IOException, InterruptedException {int count = 0;for (IntWritable value:values){count += value.get();}context.write(key,new IntWritable(count));}
}public static class PPDriver{public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {Configuration conf = new Configuration();Job job = new Job(conf);job.setJarByClass(PPDriver.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);job.setMapperClass(PPMapper.class);job.setReducerClass(PPReduce.class);FileInputFormat.addInputPath(job,new Path("D:\\a\\cars.txt"));FileOutputFormat.setOutputPath(job,new Path("D:\\a\\a6"));job.waitForCompletion(true);}
}
}
七.通过不同种类（品牌）车辆销售情况，来统计发动机 型号和燃油种类
package hadoop.MapReduce.car.FaDong;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;public class FA {
public static class FAMapper extends Mapper<LongWritable, Text,Text, IntWritable> {@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String[] lines = value.toString().split("\t");if (null != lines && lines.length > 15 && !lines[8].isEmpty() && !lines[12].isEmpty() && !lines[15].isEmpty()) {context.write(new Text(lines[8] + "\t" + lines[12] + "\t" + lines[15]), new IntWritable(1));}}
}public static class FAReduce extends Reducer<Text,IntWritable,Text,IntWritable> {@Overrideprotected void reduce(Text key, Iterable<IntWritable> values, Context context)throws IOException, InterruptedException {int count = 0;for (IntWritable value:values){count += value.get();}context.write(key,new IntWritable(count));}
}public static class FADriver{public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {Configuration conf = new Configuration();Job job = new Job(conf);job.setJarByClass(FADriver.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);job.setMapperClass(FAMapper.class);job.setReducerClass(FAReduce.class);FileInputFormat.addInputPath(job,new Path("D:\\a\\cars.txt"));FileOutputFormat.setOutputPath(job,new Path("D:\\a\\a8"));job.waitForCompletion(true);}
}
}
MapReduce解决乘用车辆和商用车辆的销售数据分析相关推荐
- MapReduce项目案例4——乘用车辆和商用车辆销售数据分析
项目介绍 1.数据概况 本数据为上牌汽车的销售数据,分为乘用车辆和商用车辆 数据包含销售相关数据与汽车具体参数 2.数据项包括 省0,月1,市2,区县3,年4,车辆型号5,制造商6,品牌7,车辆类型8 ...
- 2022-2028全球与中国商用车辆HMI解决方案市场现状及未来发展趋势
辰宇信息咨询市场调研公司最近发布-<2022-2028全球与中国商用车辆HMI解决方案市场调研报告> 内容摘要 本文重点分析在全球及中国有重要角色的企业,分析这些企业商用车辆HMI解决方案 ...
- 车载以太网 - 车辆信息和车辆声明 - 09
到今天这一篇,对于ISO 13400 - 2的DoIP协议介绍就完事了,对DoIP协议的规范介绍基本已经全部包括了,无论是我们作为软件开发人员还是测试人员,了解这部分内容基本就能够满足这块基础的开发要 ...
- Pig、Hive、MapReduce 解决分组 Top K 问题
2019独角兽企业重金招聘Python工程师标准>>> 问题: 有如下数据文件 city.txt (id, city, value) cat city.txt 1 wh 500 2 ...
- 自动驾驶车辆控制（车辆运动学模型）
本文应配合b站up主"忠厚老实的老王"的自动驾驶控制算法系列视频食用. 文章目录 1. 两个车辆运动学模型 1.1 三个坐标系 1.2符号定义 1.3车辆运动学模型 1.4车辆动力 ...
- 车辆管理java_java车辆管理系统
//车辆管理系统 package javaapplication27; import javax.swing.*; import java.io.*; import java.awt.*; impor ...
- 车辆提示请防止车辆滑动 即车辆驻车异常
车型:宝马X1 F49 宝马X2 F48 一系F52 二系F45/F46 MINI车型通病 故障描述: 车辆停车挂到P档 按电子手刹 提示请防止车辆滑动 车机显示屏提示变速箱故障 故障排查 使用解码器 ...
- 论文(2):智能车辆、自主车辆、地面无人平台、机器人、自动化、智能驾驶等相关英文术语整理
# 注:研究方向为感知定位,术语可能不能包括所有小方向 # 术语逐步增加中 # 英文很菜,不要喷呀 (1)地图相关 地图构建:map building ,Mapping 大场景.大规模.大范围:lar ...
- 车辆工程跨考计算机网络安全,车辆工程考研-车辆工程研究生考研科目-就业前景-跨考教育...
车辆工程是机械工程一级学科所属的一个二级学科,属工科门类.该专业原来是教育部专业目录外的一个专业,称"汽车工程",1999年列入新调整的教育部专业目录. 1.研究方向 目前,各大院 ...
最新文章
- C++随时输出到文件-outfile
- 使用Cython库包对python的py文件（源码）进行加密，把python的.py文件生成.so文件并调用
- 1.2句柄及 WinMain函数
- 【网络编程】之三、socket网络编程
- Centos 8 RHEL 8 破解root密码
- 太阳花图片_花是大自然给予人类的礼物 你知道石斛花的花语是什么吗
- 可方向导不一定连续的例子
- 联想电脑Fn热键驱动
- 正交幅度调制（QAM)
- 站群服务器找11火星软件
- Tkinter GUI设计中文文档
- 制作favicon图标
- 跨越生态裂谷 华为云Stack如何为企业智能化转型架桥铺路？
- 奇迹按键精灵挂机脚本_奇迹挂机捡东西脚本
- RSE2022/云检测：A hybrid generative adversarial network for weakly-supervised cloud detection 多光谱图像弱监督云检
- 手机5g什么时候普及_5g手机什么时候普及（买5G手机后悔了）
- 【数学基础】参数估计之最大后验估计（Maximum A Posteriori，MAP）
- [转]信息安全相关理论题(六)
- 北大2017计算机期末考试试题,北京大学2017年自主招生测试题汇总
- ADAMSSimulink 机器人动力学仿真入门（二）：ADAMS设置无人机连接、驱动、力与变量（代码已开源）