val orig_file=sc.textFile("train_nohead.tsv")//println(orig_file.first())val data_file=orig_file.map(_.split("\t")).map{r =>val trimmed =r.map(_.replace("\"",""))val lable=trimmed(r.length-1).toDoubleval feature=trimmed.slice(4,r.length-1).map(d => if(d=="?")0.0else d.toDouble)LabeledPoint(lable,Vectors.dense(feature))}/*特征标准化优化，似乎对决策数没啥影响*/val vectors=data_file.map(x =>x.features)val rows=new RowMatrix(vectors)println(rows.computeColumnSummaryStatistics().variance)//每列的方差val scaler=new StandardScaler(withMean=true,withStd=true).fit(vectors)//标准化val scaled_data=data_file.map(point => LabeledPoint(point.label,scaler.transform(point.features))).randomSplit(Array(0.7,0.3),11L)//固定seed为11L，确保每次每次实验能得到相同的结果val data_train=scaled_data(0)val data_test=scaled_data(1)

/*训练决策树模型*/val model_DT=DecisionTree.train(data_train,Algo.Classification,Entropy,maxTreeDepth)/*决策树的精确度*/val predectionAndLabeledDT=data_test.map { point =>val predectLabeled = if (model_DT.predict(point.features) > 0.5) 1.0 else 0.0(predectLabeled,point.label)}val metricsDT=new MulticlassMetrics(predectionAndLabeledDT)println(metricsDT.accuracy)//0.6273062730627307
/*决策树的PR曲线和AOC曲线*/val dtMetrics = Seq(model_DT).map{ model =>val scoreAndLabels = data_test.map { point =>val score = model.predict(point.features)(if (score > 0.5) 1.0 else 0.0, point.label)}val metrics = new BinaryClassificationMetrics(scoreAndLabels)(model.getClass.getSimpleName, metrics.areaUnderPR, metrics.areaUnderROC)}val allMetrics = dtMetricsallMetrics.foreach{ case (m, pr, roc) =>println(f"$m, Area under PR: ${pr * 100.0}%2.4f%%, Area under ROC: ${roc * 100.0}%2.4f%%")}
/*
DecisionTreeModel, Area under PR: 74.2335%, Area under ROC: 62.4326%
*/

/*调参函数*/def trainDTWithParams(input: RDD[LabeledPoint], maxDepth: Int,impurity: Impurity) = {DecisionTree.train(input, Algo.Classification, impurity, maxDepth)}

 /*改变深度*/val dtResultsEntropy = Seq(1, 2, 3, 4, 5, 10, 20).map { param =>val model = trainDTWithParams(data_train, param, Entropy)val scoreAndLabels = data_test.map { point =>val score = model.predict(point.features)(if (score > 0.5) 1.0 else 0.0, point.label)}val metrics = new BinaryClassificationMetrics(scoreAndLabels)(s"$param tree depth", metrics.areaUnderROC)}dtResultsEntropy.foreach { case (param, auc) => println(f"$param, " +f"AUC = ${auc * 100}%2.2f%%") }
/*
1 tree depth, AUC = 58.57%
2 tree depth, AUC = 60.69%
3 tree depth, AUC = 61.40%
4 tree depth, AUC = 61.30%
5 tree depth, AUC = 62.43%
10 tree depth, AUC = 62.26%
20 tree depth, AUC = 60.59%
*/

 /*改变纯度*/val dtResultsEntropy = Seq(Gini,Entropy).map { param =>val model = trainDTWithParams(data_train, 5, param)val scoreAndLabels = data_test.map { point =>val score = model.predict(point.features)(if (score > 0.5) 1.0 else 0.0, point.label)}val metrics = new BinaryClassificationMetrics(scoreAndLabels)(s"$param tree depth", metrics.areaUnderROC)}dtResultsEntropy.foreach { case (param, auc) => println(f"$param, " +f"AUC = ${auc * 100}%2.2f%%") }
/*
org.apache.spark.mllib.tree.impurity.Gini$@32d8e58d tree depth, AUC = 62.37%
org.apache.spark.mllib.tree.impurity.Entropy$@1ddba7a0 tree depth, AUC = 62.43%
*/