// Goodness of Fit test
def chiSqTest(observed: Vector, expected: Vector): ChiSqTestResult = {ChiSqTest.chiSquared(observed, expected)}
//Goodness of Fit test
def chiSqTest(observed: Vector): ChiSqTestResult = ChiSqTest.chiSquared(observed)//independence test
def chiSqTest(observed: Matrix): ChiSqTestResult = ChiSqTest.chiSquaredMatrix(observed)
//independence test
def chiSqTest(data: RDD[LabeledPoint]): Array[ChiSqTestResult] = {ChiSqTest.chiSquaredFeatures(data)
}

 /** Pearon's goodness of fit test on the input observed and expected counts/relative frequencies.* Uniform distribution is assumed when `expected` is not passed in.*/def chiSquared(observed: Vector,expected: Vector = Vectors.dense(Array[Double]()),methodName: String = PEARSON.name): ChiSqTestResult = {// Validate input argumentsval method = methodFromString(methodName)if (expected.size != 0 && observed.size != expected.size) {throw new IllegalArgumentException("observed and expected must be of the same size.")}val size = observed.sizeif (size > 1000) {logWarning("Chi-squared approximation may not be accurate due to low expected frequencies "+ s" as a result of a large number of categories: $size.")}val obsArr = observed.toArray// 假设expected值没有设置，默认取1.0 / sizeval expArr = if (expected.size == 0) Array.tabulate(size)(_ => 1.0 / size) else expected.toArray/ 假设expected、observed值都必需要大于1if (!obsArr.forall(_ >= 0.0)) {throw new IllegalArgumentException("Negative entries disallowed in the observed vector.")}if (expected.size != 0 && ! expArr.forall(_ >= 0.0)) {throw new IllegalArgumentException("Negative entries disallowed in the expected vector.")}// Determine the scaling factor for expectedval obsSum = obsArr.sumval expSum = if (expected.size == 0.0) 1.0 else expArr.sumval scale = if (math.abs(obsSum - expSum) < 1e-7) 1.0 else obsSum / expSum// compute chi-squared statisticval statistic = obsArr.zip(expArr).foldLeft(0.0) { case (stat, (obs, exp)) =>if (exp == 0.0) {if (obs == 0.0) {throw new IllegalArgumentException("Chi-squared statistic undefined for input vectors due"+ " to 0.0 values in both observed and expected.")} else {return new ChiSqTestResult(0.0, size - 1, Double.PositiveInfinity, PEARSON.name,NullHypothesis.goodnessOfFit.toString)}}// 计算(observed-expected)2/expectedif (scale == 1.0) {stat + method.chiSqFunc(obs, exp)} else {stat + method.chiSqFunc(obs, exp * scale)}}val df = size - 1val pValue = chiSquareComplemented(df, statistic)new ChiSqTestResult(pValue, df, statistic, PEARSON.name, NullHypothesis.goodnessOfFit.toString)}

版权声明：本文博客原创文章，博客，未经同意，不得转载。