离线轻量级大数据平台Spark之MLib机器学习库SVM实例

支持向量机，因其英文名为support vector machine，故一般简称SVM，通俗来讲，它是一种二类分类模型，其基本模型定义为特征空间上的间隔最大的线性分类器，其学习策略便是间隔最大化，最终可转化为一个凸二次规划问题的求解。
http://www.dataguru.cn/thread-371987-1-1.html
参考该网站理解SVM基础数学原理。

依据距离计算分类的思想，适用于各维。

具体代码如下：

package sk.mlib;import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;import scala.Tuple2;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.mllib.classification.SVMModel;
import org.apache.spark.mllib.classification.SVMWithSGD;
import org.apache.spark.mllib.evaluation.BinaryClassificationMetrics;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.mllib.util.MLUtils;public class SVMWithSGDDemo {public static void main(String[] args) {SparkConf conf = new SparkConf().setAppName("JavaSVMWithSGDExample");SparkContext sc = new SparkContext(conf);// $example on$String path = "/tmp/svmdata.txt";JavaRDD<LabeledPoint> data = MLUtils.loadLibSVMFile(sc, path).toJavaRDD();// Split initial RDD into two... [60% training data, 40% testing data].JavaRDD<LabeledPoint> training = data.sample(false, 0.6, 11L);training.cache();JavaRDD<LabeledPoint> test = data.subtract(training);// Run training algorithm to build the model.int numIterations = 100;final SVMModel model = SVMWithSGD.train(training.rdd(), numIterations);// Clear the default threshold.model.clearThreshold();// Compute raw scores on the test set.JavaRDD<Tuple2<Object, Object>> scoreAndLabels = test.map(new Function<LabeledPoint, Tuple2<Object, Object>>() {public Tuple2<Object, Object> call(LabeledPoint p) {Double score = model.predict(p.features());return new Tuple2<Object, Object>(score, p.label());}});// Get evaluation metrics.BinaryClassificationMetrics metrics =new BinaryClassificationMetrics(JavaRDD.toRDD(scoreAndLabels));double auROC = metrics.areaUnderROC();System.out.println("Area under ROC = " + auROC);// Save and load modelmodel.save(sc, "/tmp/javaSVMWithSGDModel");SVMModel sameModel = SVMModel.load(sc, "/tmp/javaSVMWithSGDModel");//应用模型分类System.out.println("Prediction of (-0.857554,0.555556,1,1,0.555556,0.333333,1,0.777778,0.333333,-1 ):"+sameModel.predict(Vectors.dense(-0.857554,0.555556,1,1,0.555556,0.333333,1,0.777778,0.333333,-1 )));sc.stop();}
}
/*执行结果：Area under ROC = 0.9017094017094017Prediction of (-0.857554,0.555556,1,1,0.555556,0.333333,1,0.777778,0.333333,-1 ):3.238535993736797*/

输入的数据集：标签特征向量1：特征向量值1 特征向量2：特征向量值2 ... 特征向量n：特征向量值n

0 1:-0.860107 2:-0.111111 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.555556 9:-1 10:-1
0 1:-0.859671 2:-0.111111 3:-0.333333 4:-0.333333 5:-0.111111 6:0.333333 7:1 8:-0.555556 9:-0.777778 10:-1
0 1:-0.857807 2:-0.555556 3:-1 4:-1 5:-1 6:-0.777778 7:-0.777778 8:-0.555556 9:-1 10:-1
0 1:-0.85768 2:0.111111 3:0.555556 4:0.555556 5:-1 6:-0.555556 7:-0.333333 8:-0.555556 9:0.333333 10:-1
0 1:-0.857569 2:-0.333333 3:-1 4:-1 5:-0.555556 6:-0.777778 7:-1 8:-0.555556 9:-1 10:-1
1 1:-0.857554 2:0.555556 3:1 4:1 5:0.555556 6:0.333333 7:1 8:0.777778 9:0.333333 10:-1
0 1:-0.857408 2:-1 3:-1 4:-1 5:-1 6:-0.777778 7:1 8:-0.555556 9:-1 10:-1
0 1:-0.857339 2:-0.777778 3:-1 4:-0.777778 5:-1 6:-0.777778 7:-1 8:-0.555556 9:-1 10:-1
1 1:-0.855171 2:-0.777778 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-1 9:-1 10:-0.111111
0 1:-0.855171 2:-0.333333 3:-0.777778 4:-1 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-1
0 1:-0.854841 2:-1 3:-1 4:-1 5:-1 6:-1 7:-1 8:-0.555556 9:-1 10:-1
0 1:-0.854709 2:-0.777778 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-1
1 1:-0.853868 2:-0.111111 3:-0.555556 4:-0.555556 5:-0.555556 6:-0.777778 7:-0.555556 8:-0.333333 9:-0.333333 10:-1
0 1:-0.85354 2:-1 3:-1 4:-1 5:-1 6:-0.777778 7:-0.555556 8:-0.555556 9:-1 10:-1
1 1:-0.853454 2:0.555556 3:0.333333 4:-0.111111 5:1 6:0.333333 7:0.777778 8:-0.111111 9:-0.111111 10:-0.333333
1 1:-0.852997 2:0.333333 3:-0.333333 4:0.111111 5:-0.333333 6:0.111111 7:-1 8:-0.333333 9:-0.555556 10:-1
0 1:-0.852842 2:-0.333333 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-1
1 1:-0.852671 2:-0.333333 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.555556 9:-1 10:-1
0 1:-0.852543 2:1 3:0.333333 4:0.333333 5:0.111111 6:-0.333333 7:1 8:-0.333333 9:-1 10:-0.777778
0 1:-0.852536 2:0.111111 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.555556 9:-1 10:-1
1 1:-0.851958 2:0.333333 3:-0.555556 4:-0.777778 5:1 6:-0.111111 7:1 8:-0.111111 9:-0.333333 10:-0.333333
1 1:-0.851957 2:1 3:-0.111111 4:-0.111111 5:-0.555556 6:0.111111 7:0.333333 8:0.333333 9:1 10:-1
0 1:-0.85163 2:-0.555556 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-1
0 1:-0.851217 2:-1 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.555556 9:-1 10:-1
1 1:-0.850295 2:-0.111111 3:-0.777778 4:-0.555556 5:-0.333333 6:-0.777778 7:0.333333 8:-0.555556 9:0.111111 10:-1
0 1:-0.850198 2:-0.555556 3:-0.777778 4:-1 5:-1 6:-1 7:-1 8:-0.777778 9:-1 10:-1
0 1:-0.850107 2:-0.111111 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-1
0 1:-0.850038 2:-0.777778 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-1
0 1:-0.849517 2:-1 3:-1 4:-0.555556 5:-1 6:-0.777778 7:-1 8:-1 9:-1 10:-1
0 1:-0.849517 2:-0.555556 3:-1 4:-1 5:-1 6:-1 7:-1 8:-0.777778 9:-1 10:-1
0 1:-0.849393 2:-0.777778 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.555556 9:-1 10:-1
1 1:-0.849331 2:1 3:0.333333 4:0.333333 5:-0.555556 6:0.555556 7:-0.111111 8:0.333333 9:-0.333333 10:-0.555556
0 1:-0.848968 2:-0.777778 3:-1 4:-1 5:-0.777778 6:-0.777778 7:-1 8:-0.555556 9:-1 10:-1
0 1:-0.848891 2:-0.555556 3:-1 4:-0.777778 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-1
0 1:-0.848267 2:-0.777778 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-1
1 1:-0.848135 2:1 3:1 4:1 5:0.555556 6:0.111111 7:-1 8:0.555556 9:0.777778 10:-1
0 1:-0.847895 2:0.111111 3:-0.777778 4:-1 5:-1 6:-1 7:-1 8:0.333333 9:-1 10:-1
1 1:-0.847478 2:-0.111111 3:-0.333333 4:-0.333333 5:0.777778 6:-0.777778 7:1 8:-0.111111 9:0.111111 10:-1
1 1:-0.846481 2:-0.777778 3:-0.111111 4:-0.555556 5:-0.555556 6:0.111111 7:0.333333 8:0.333333 9:-0.111111 10:-1
1 1:-0.845249 2:1 3:-0.333333 4:-0.555556 5:-1 6:-0.555556 7:-0.555556 8:0.111111 9:-0.111111 10:-0.777778
1 1:-0.845097 2:0.111111 3:1 4:1 5:-0.777778 6:0.555556 7:1 8:0.333333 9:-0.555556 10:-0.555556
1 1:-0.844791 2:-0.111111 3:0.111111 4:-0.111111 5:0.111111 6:1 7:-1 8:-0.555556 9:-1 10:-1
1 1:-0.844637 2:1 3:1 4:1 5:-0.333333 6:0.555556 7:-1 8:0.555556 9:1 10:-1
0 1:-0.84462 2:-1 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-0.777778
1 1:-0.84439 2:-0.555556 3:0.333333 4:0.333333 5:-0.333333 6:-0.333333 7:0.777778 8:-0.333333 9:0.555556 10:-1
0 1:-0.844351 2:-1 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-1
0 1:-0.844265 2:-0.333333 3:-1 4:-1 5:-0.555556 6:-0.777778 7:-1 8:-0.555556 9:-1 10:-1
1 1:-0.844156 2:0.333333 3:0.555556 4:0.333333 5:-0.777778 6:-0.333333 7:0.555556 8:-0.555556 9:0.555556 10:-0.777778
1 1:-0.843926 2:0.777778 3:-0.111111 4:0.555556 5:-1 6:-0.777778 7:-0.555556 8:-0.777778 9:-1 10:-0.111111
1 1:-0.843914 2:-0.111111 3:-0.555556 4:-0.555556 5:-0.333333 6:-0.777778 7:-0.333333 8:-0.555556 9:-0.333333 10:-1
1 1:-0.843667 2:1 3:-0.555556 4:0.111111 5:-0.777778 6:-0.555556 7:-0.111111 8:-0.333333 9:1 10:-0.777778
1 1:-0.843607 2:-0.111111 3:-0.111111 4:-0.111111 5:0.555556 6:1 7:0.555556 8:0.333333 9:-0.555556 10:0.333333
1 1:-0.843604 2:1 3:-0.111111 4:-0.111111 5:0.111111 6:0.555556 7:0.555556 8:0.333333 9:-1 10:-1
1 1:-0.843496 2:1 3:0.111111 4:0.111111 5:-0.555556 6:-0.333333 7:-0.111111 8:-0.555556 9:0.111111 10:-1
1 1:-0.843352 2:0.555556 3:1 4:1 5:-1 6:-0.555556 7:0.111111 8:-0.555556 9:0.777778 10:-1
1 1:-0.843228 2:0.555556 3:-0.777778 4:-0.333333 5:-1 6:-0.111111 7:-1 8:-0.111111 9:-0.333333 10:-0.333333
1 1:-0.843162 2:-0.111111 3:-0.777778 4:-0.555556 5:-1 6:0.111111 7:1 8:-0.111111 9:-1 10:-1
1 1:-0.843099 2:0.777778 3:-0.111111 4:-0.111111 5:-0.777778 6:-0.777778 7:-0.777778 8:-0.111111 9:-1 10:-1
1 1:-0.842893 2:-0.111111 3:-0.555556 4:-0.111111 5:-0.111111 6:-0.555556 7:-0.555556 8:-0.333333 9:1 10:-1
0 1:-0.842892 2:-1 3:-1 4:-1 5:-1 6:-0.777778 7:-0.777778 8:-0.777778 9:-1 10:-1
1 1:-0.842769 2:0.777778 3:1 4:1 5:-1 6:1 7:0.555556 8:-0.555556 9:-0.555556 10:-1
1 1:-0.842766 2:0.111111 3:-0.555556 4:-0.333333 5:-1 6:-0.111111 7:-0.777778 8:-0.555556 9:0.777778 10:-1
0 1:-0.842757 2:-1 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.777778 9:-1 10:-1
1 1:-0.842637 2:1 3:-0.333333 4:-0.777778 5:-1 6:-0.555556 7:-0.777778 8:-0.333333 9:-0.555556 10:1
0 1:-0.842614 2:-0.333333 3:-1 4:-1 5:-1 6:-0.777778 7:-1 8:-0.555556 9:-1 10:-1

执行结果和样本集有关系。

离线轻量级大数据平台Spark之MLib机器学习库SVM实例相关推荐

离线轻量级大数据平台Spark之MLib机器学习库Word2Vec实例
Word2Vecword2vec能将文本中出现的词向量化,可以在捕捉语境信息的同时压缩数据规模.Word2Vec实际上是两种不同的方法:Continuous Bag of Words (CBOW) 和 ...
离线轻量级大数据平台Spark之MLib机器学习库TF-IDF实例
TF-IDF(termfrequency–inverse document frequency)是TF-IDF是一种统计方法,用以评估一字词对于一个文件集或一个语料库中的其中一份文件的重要程度.字词的 ...
离线轻量级大数据平台Spark之MLib机器学习库线性回归实例
1.线性回归线性回归是利用称为线性回归方程的函数对一个或多个自变量和因变量之间关系进行建模的一种回归分析方法,只有一个自变量的情况称为简单回归,大于一个自变量情况的叫做多元回归,在实际情况中大多数都 ...
离线轻量级大数据平台Spark之MLib机器学习库概念学习
Mlib机器学习库 1.1机器学习概念机器学习有很多定义,倾向于下面这个定义.机器学习是对能通过经验自动改进的计算机算法的研究.机器学习依赖数据经验并评估和优化算法所运行出的模型.机器学习算法尝试根 ...
离线轻量级大数据平台Spark之MLib机器学习库朴素贝叶斯实例
1.朴素贝叶斯介绍表示事件B已经发生的前提下,事件A发生的概率,叫做事件B发生下事件A的条件概率,公式为: 贝叶斯定理: 从已知P(A|B)获得P(B|A)值. 假设A和B代表两类互相影响的事件,如 ...
离线轻量级大数据平台Spark之MLib机器学习库聚类算法KMeans实例
1.KMeans算法所谓聚类问题,就是给定一个元素集合D,其中每个元素具有n个可观察属性,使用某种算法将D划分成k个子集,要求每个子集内部的元素之间相异度尽可能低,而不同子集的元素相异度尽可能高.其 ...
离线轻量级大数据平台Spark之MLib机器学习协同过滤ALS实例
1.协同过滤协同过滤(Collaborative Filtering,简称CF,WIKI上的定义是:简单来说是利用某个兴趣相投.拥有共同经验之群体的喜好来推荐感兴趣的资讯给使用者,个人透过合作的机制 ...
离线轻量级大数据平台Spark之单机部署及Java开发
1.Spark平台基本介绍 Spark 由加州大学伯克利分校 AMP 实验室 (Algorithms, Machines, and PeopleLab) 开发,可用来构建大型的.低延迟的数据分析应用程 ...
离线轻量级大数据平台Spark之JavaRDD关联join操作
对两个RDD进行关联操作,如: 1)文件post_data.txt包含:post_id\title\content 2)文件train.txt包含:dev_id\post_id\praise\time ...

离线轻量级大数据平台Spark之MLib机器学习库SVM实例

离线轻量级大数据平台Spark之MLib机器学习库SVM实例相关推荐

最新文章

热门文章