1. // affairs：一年来婚外情的频率
  2. // gender：性别
  3. // age：年龄
  4. // yearsmarried：婚龄
  5. // children：是否有小孩
  6. // religiousness：宗教信仰程度（5分制，1分表示反对，5分表示非常信仰）
  7. // education：学历
  8. // occupation：职业（逆向编号的戈登7种分类）
  9. // rating：对婚姻的自我评分（5分制，1表示非常不幸福，5表示非常幸福）
  10. import org.apache.spark.sql.SparkSession
  11. import org.apache.spark.sql.Dataset
  12. import org.apache.spark.sql.Row
  13. import org.apache.spark.sql.DataFrame
  14. import org.apache.spark.sql.Column
  15. import org.apache.spark.sql.DataFrameReader
  16. import org.apache.spark.rdd.RDD
  17. import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
  18. import org.apache.spark.sql.Encoder
  19. import org.apache.spark.ml.linalg.Vectors
  20. import org.apache.spark.ml.feature.StringIndexer
  21. import org.apache.spark.ml.feature.OneHotEncoder
  22. import org.apache.spark.ml.feature.VectorAssembler
  23. import org.apache.spark.ml.feature.StandardScaler
  24. import org.apache.spark.ml.feature.PCA
  25. import org.apache.spark.ml.clustering.KMeans
  26. scala> val spark = SparkSession.builder().appName("Spark SQL basic example").config("spark.some.config.option", "some-value").getOrCreate()
  27. scala>
  28. scala> // For implicit conversions like converting RDDs to DataFrames
  29. scala> import spark.implicits._
  30. scala> val data: DataFrame = spark.read.format("csv").option("header", true).load("hdfs://ns1/datafile/wangxiao/Affairs.csv")
  31. data: org.apache.spark.sql.DataFrame = [affairs: string, gender: string ... 7 more fields]
  32. scala>
  33. scala> data.cache
  34. res0: data.type = [affairs: string, gender: string ... 7 more fields]
  35. scala>
  36. scala> data.limit(10).show()
  38. scala>
  39. scala> // 转换字符类型，将Double和String的字段分开放
  40. scala> val data1 = data.select(
  41. | data("affairs").cast("Double"),
  42. | data("age").cast("Double"),
  43. | data("yearsmarried").cast("Double"),
  44. | data("religiousness").cast("Double"),
  45. | data("education").cast("Double"),
  46. | data("occupation").cast("Double"),
  47. | data("rating").cast("Double"),
  48. | data("gender").cast("String"),
  49. | data("children").cast("String"))
  50. data1: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 7 more fields]
  51. scala>
  52. scala> data1.printSchema()
  53. root
  54. |-- affairs: double (nullable = true)
  55. |-- age: double (nullable = true)
  56. |-- yearsmarried: double (nullable = true)
  57. |-- religiousness: double (nullable = true)
  58. |-- education: double (nullable = true)
  59. |-- occupation: double (nullable = true)
  60. |-- rating: double (nullable = true)
  61. |-- gender: string (nullable = true)
  62. |-- children: string (nullable = true)
  63. scala> data1.limit(10).show
  65. scala>
  66. scala> val dataDF = data1
  67. dataDF: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 7 more fields]
  68. scala>
  69. scala> dataDF.cache()
  70. res4: dataDF.type = [affairs: double, age: double ... 7 more fields]
  71. scala>
  72. scala> //###################################
  73. scala> val indexer = new StringIndexer().setInputCol("gender").setOutputCol("genderIndex").fit(dataDF)
  74. indexer: org.apache.spark.ml.feature.StringIndexerModel = strIdx_19a888aff882
  75. scala> val indexed = indexer.transform(dataDF)
  76. indexed: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 8 more fields]
  77. scala> // OneHot编码，注意setDropLast设置为false
  78. scala> val encoder = new OneHotEncoder().setInputCol("genderIndex").setOutputCol("genderVec").setDropLast(false)
  79. encoder: org.apache.spark.ml.feature.OneHotEncoder = oneHot_f0f47e0b5b37
  80. scala> val encoded = encoder.transform(indexed)
  81. encoded: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 9 more fields]
  82. scala> encoded.show()
  84. scala>
  85. scala> val indexer1 = new StringIndexer().setInputCol("children").setOutputCol("childrenIndex").fit(encoded)
  86. indexer1: org.apache.spark.ml.feature.StringIndexerModel = strIdx_7e4d8c69b823
  87. scala> val indexed1 = indexer1.transform(encoded)
  88. indexed1: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 10 more fields]
  89. scala> val encoder1 = new OneHotEncoder().setInputCol("childrenIndex").setOutputCol("childrenVec").setDropLast(false)
  90. encoder1: org.apache.spark.ml.feature.OneHotEncoder = oneHot_9a8906781325
  91. scala> val encoded1 = encoder1.transform(indexed1)
  92. encoded1: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 11 more fields]
  93. scala> encoded1.show()
  95. scala>
  96. scala> val encodeDF: DataFrame = encoded1
  97. encodeDF: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 11 more fields]
  98. scala> encodeDF.show()
  100. scala> encodeDF.printSchema()
  101. root
  102. |-- affairs: double (nullable = true)
  103. |-- age: double (nullable = true)
  104. |-- yearsmarried: double (nullable = true)
  105. |-- religiousness: double (nullable = true)
  106. |-- education: double (nullable = true)
  107. |-- occupation: double (nullable = true)
  108. |-- rating: double (nullable = true)
  109. |-- gender: string (nullable = true)
  110. |-- children: string (nullable = true)
  111. |-- genderIndex: double (nullable = true)
  112. |-- genderVec: vector (nullable = true)
  113. |-- childrenIndex: double (nullable = true)
  114. |-- childrenVec: vector (nullable = true)
  115. scala>
  116. scala> //#################################
  117. scala> val assembler = new VectorAssembler().setInputCols(Array("affairs", "age", "yearsmarried", "religiousness", "education", "occupation", "rating", "genderVec", "childrenVec")).setOutputCol("features")
  118. assembler: org.apache.spark.ml.feature.VectorAssembler = vecAssembler_8ccd528981cd
  119. scala>
  120. scala> val vecDF: DataFrame = assembler.transform(encodeDF)
  121. vecDF: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 12 more fields]
  122. scala> vecDF.select("features").show
  123. 16/11/05 15:56:14 WARN Executor: 1 block locks were not released by TID = 11:
  124. [rdd_17_0]
  125. +--------------------+
  126. | features|
  127. +--------------------+
  128. |[0.0,37.0,10.0,3....|
  129. |[0.0,27.0,4.0,4.0...|
  130. |[0.0,32.0,15.0,1....|
  131. |[0.0,57.0,15.0,5....|
  132. |[0.0,22.0,0.75,2....|
  133. |[0.0,32.0,1.5,2.0...|
  134. |[0.0,22.0,0.75,2....|
  135. |[0.0,57.0,15.0,2....|
  136. |[0.0,32.0,15.0,4....|
  137. |[0.0,22.0,1.5,4.0...|
  138. |[0.0,37.0,15.0,2....|
  139. |[0.0,27.0,4.0,4.0...|
  140. |[0.0,47.0,15.0,5....|
  141. |[0.0,22.0,1.5,2.0...|
  142. |[0.0,27.0,4.0,4.0...|
  143. |[0.0,37.0,15.0,1....|
  144. |[0.0,37.0,15.0,2....|
  145. |[0.0,22.0,0.75,3....|
  146. |[0.0,22.0,1.5,2.0...|
  147. |[0.0,27.0,10.0,2....|
  148. +--------------------+
  149. only showing top 20 rows
  150. scala>
  151. scala> // 标准化--均值标准差
  152. scala> val scaler = new StandardScaler().setInputCol("features").setOutputCol("scaledFeatures").setWithStd(true).setWithMean(true)
  153. scaler: org.apache.spark.ml.feature.StandardScaler = stdScal_2e35fbc29084
  154. scala>
  155. scala> // Compute summary statistics by fitting the StandardScaler.
  156. scala> val scalerModel = scaler.fit(vecDF)
  157. scalerModel: org.apache.spark.ml.feature.StandardScalerModel = stdScal_2e35fbc29084
  158. scala>
  159. scala> // Normalize each feature to have unit standard deviation.
  160. scala> val scaledData: DataFrame = scalerModel.transform(vecDF)
  161. scaledData: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 13 more fields]
  162. scala> // scaledData:DataFrame = [features: vector, scaledFeatures: vector]
  163. scala>
  164. scala> scaledData.select("features", "scaledFeatures").show
  165. 16/11/05 15:56:20 WARN Executor: 1 block locks were not released by TID = 13:
  166. [rdd_17_0]
  167. +--------------------+--------------------+
  168. | features| scaledFeatures|
  169. +--------------------+--------------------+
  170. |[0.0,37.0,10.0,3....|[-0.4413500298573...|
  171. |[0.0,27.0,4.0,4.0...|[-0.4413500298573...|
  172. |[0.0,32.0,15.0,1....|[-0.4413500298573...|
  173. |[0.0,57.0,15.0,5....|[-0.4413500298573...|
  174. |[0.0,22.0,0.75,2....|[-0.4413500298573...|
  175. |[0.0,32.0,1.5,2.0...|[-0.4413500298573...|
  176. |[0.0,22.0,0.75,2....|[-0.4413500298573...|
  177. |[0.0,57.0,15.0,2....|[-0.4413500298573...|
  178. |[0.0,32.0,15.0,4....|[-0.4413500298573...|
  179. |[0.0,22.0,1.5,4.0...|[-0.4413500298573...|
  180. |[0.0,37.0,15.0,2....|[-0.4413500298573...|
  181. |[0.0,27.0,4.0,4.0...|[-0.4413500298573...|
  182. |[0.0,47.0,15.0,5....|[-0.4413500298573...|
  183. |[0.0,22.0,1.5,2.0...|[-0.4413500298573...|
  184. |[0.0,27.0,4.0,4.0...|[-0.4413500298573...|
  185. |[0.0,37.0,15.0,1....|[-0.4413500298573...|
  186. |[0.0,37.0,15.0,2....|[-0.4413500298573...|
  187. |[0.0,22.0,0.75,3....|[-0.4413500298573...|
  188. |[0.0,22.0,1.5,2.0...|[-0.4413500298573...|
  189. |[0.0,27.0,10.0,2....|[-0.4413500298573...|
  190. +--------------------+--------------------+
  191. only showing top 20 rows
  192. scala>
  193. scala> //##########################
  194. scala> // 主成分
  195. scala> val pca = new PCA().setInputCol("scaledFeatures").setOutputCol("pcaFeatures").setK(3).fit(scaledData)
  196. 16/11/05 15:56:21 WARN Executor: 1 block locks were not released by TID = 14:
  197. [rdd_17_0]
  198. 16/11/05 15:56:22 WARN Executor: 1 block locks were not released by TID = 15:
  199. [rdd_17_0]
  200. 16/11/05 15:56:24 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS
  201. 16/11/05 15:56:24 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS
  202. 16/11/05 15:56:25 WARN LAPACK: Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK
  203. 16/11/05 15:56:25 WARN LAPACK: Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK
  204. pca: org.apache.spark.ml.feature.PCAModel = pca_8569d580d6e4
  205. scala> pca.explainedVariance.values //解释变量方差
  206. res11: Array[Double] = Array(0.28779526464781313, 0.23798543640278289, 0.11742828783633019)
  207. scala> pca.pc //载荷（观测变量与主成分的相关系数）
  208. res12: org.apache.spark.ml.linalg.DenseMatrix =
  209. -0.12034310848156521 0.05153952289637974 0.6678769450480689
  210. -0.42860623714516627 0.05417889891307473 -0.05592377098140197
  211. -0.44404074412877986 0.1926596811059294 -0.017025575192258197
  212. -0.12233707317255231 0.08053139375662526 -0.5093149296300096
  213. -0.14664751606128462 -0.3872166556211308 -0.03406819489501708
  214. -0.145543746024348 -0.43054860653839705 0.07841454709046872
  215. 0.17703994181974803 -0.12792784984216296 -0.5173229755329072
  216. 0.2459668445061567 0.4915809641798787 0.010477548320795945
  217. -0.2459668445061567 -0.4915809641798787 -0.010477548320795945
  218. -0.44420980045271047 0.240652448514566 -0.089356723885704
  219. 0.4442098004527103 -0.24065244851456588 0.08935672388570405
  220. scala> pca.extractParamMap()
  221. res13: org.apache.spark.ml.param.ParamMap =
  222. {
  223. pca_8569d580d6e4-inputCol: scaledFeatures,
  224. pca_8569d580d6e4-k: 3,
  225. pca_8569d580d6e4-outputCol: pcaFeatures
  226. }
  227. scala> pca.params
  228. res14: Array[org.apache.spark.ml.param.Param[_]] = Array(pca_8569d580d6e4__inputCol, pca_8569d580d6e4__k, pca_8569d580d6e4__outputCol)
  229. scala>
  230. scala> val pcaDF: DataFrame = pca.transform(scaledData)
  231. pcaDF: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 14 more fields]
  232. scala> // pcaDF:DataFrame = [features: vector, scaledFeatures: vector,pcaFeatures: vector]
  233. scala> pcaDF.cache()
  234. res15: pcaDF.type = [affairs: double, age: double ... 14 more fields]
  235. scala>
  236. scala> pcaDF.printSchema()
  237. root
  238. |-- affairs: double (nullable = true)
  239. |-- age: double (nullable = true)
  240. |-- yearsmarried: double (nullable = true)
  241. |-- religiousness: double (nullable = true)
  242. |-- education: double (nullable = true)
  243. |-- occupation: double (nullable = true)
  244. |-- rating: double (nullable = true)
  245. |-- gender: string (nullable = true)
  246. |-- children: string (nullable = true)
  247. |-- genderIndex: double (nullable = true)
  248. |-- genderVec: vector (nullable = true)
  249. |-- childrenIndex: double (nullable = true)
  250. |-- childrenVec: vector (nullable = true)
  251. |-- features: vector (nullable = true)
  252. |-- scaledFeatures: vector (nullable = true)
  253. |-- pcaFeatures: vector (nullable = true)
  254. scala> pcaDF.select("features", "scaledFeatures", "pcaFeatures").show
  255. 16/11/05 15:56:36 WARN Executor: 1 block locks were not released by TID = 18:
  256. [rdd_64_0]
  257. +--------------------+--------------------+--------------------+
  258. | features           | scaledFeatures     | pcaFeatures        |
  259. +--------------------+--------------------+--------------------+
  260. |[0.0,37.0,10.0,3....|[-0.4413500298573...|[0.27828160409293...|
  261. |[0.0,27.0,4.0,4.0...|[-0.4413500298573...|[2.42147114101165...|
  262. |[0.0,32.0,15.0,1....|[-0.4413500298573...|[0.18301418047489...|
  263. |[0.0,57.0,15.0,5....|[-0.4413500298573...|[-2.9795960667914...|
  264. |[0.0,22.0,0.75,2....|[-0.4413500298573...|[1.79299133565688...|
  265. |[0.0,32.0,1.5,2.0...|[-0.4413500298573...|[2.65694237441759...|
  266. |[0.0,22.0,0.75,2....|[-0.4413500298573...|[3.48234503794570...|
  267. |[0.0,57.0,15.0,2....|[-0.4413500298573...|[-2.4215838062079...|
  268. |[0.0,32.0,15.0,4....|[-0.4413500298573...|[-0.6964555195741...|
  269. |[0.0,22.0,1.5,4.0...|[-0.4413500298573...|[2.18771069800414...|
  270. |[0.0,37.0,15.0,2....|[-0.4413500298573...|[-2.4259075891377...|
  271. |[0.0,27.0,4.0,4.0...|[-0.4413500298573...|[-0.7743038356008...|
  272. |[0.0,47.0,15.0,5....|[-0.4413500298573...|[-2.6176149267534...|
  273. |[0.0,22.0,1.5,2.0...|[-0.4413500298573...|[2.95788535193022...|
  274. |[0.0,27.0,4.0,4.0...|[-0.4413500298573...|[2.50146472861263...|
  275. |[0.0,37.0,15.0,1....|[-0.4413500298573...|[-0.5123817022008...|
  276. |[0.0,37.0,15.0,2....|[-0.4413500298573...|[-0.9191740114044...|
  277. |[0.0,22.0,0.75,3....|[-0.4413500298573...|[2.97391491782863...|
  278. |[0.0,22.0,1.5,2.0...|[-0.4413500298573...|[3.17940505267806...|
  279. |[0.0,27.0,10.0,2....|[-0.4413500298573...|[0.74585406839527...|
  280. +--------------------+--------------------+--------------------+
  281. only showing top 20 rows
  282. scala>
  283. scala> //#####################################
  284. scala>
  285. scala> // 注意最大迭代次數和轮廓系数
  286. scala> val KSSE = (2 to 10 by 1).par.toList.map { k =>
  287. | // 聚类
  288. | // Trains a k-means model.
  289. | val kmeans = new KMeans().setK(k).setSeed(1L).setFeaturesCol("pcaFeatures")
  290. | val model = kmeans.fit(pcaDF)
  291. |
  292. | // Evaluate clustering by computing Within Set Sum of Squared Errors.
  293. | val WSSSE = model.computeCost(pcaDF)
  294. |
  295. | (k, WSSSE)
  296. | }
  297. KSSE: List[(Int, Double)] = List((2,2876.20580405469), (3,1680.6647048004902), (4,1395.7184052948346), (5,1239.9362814229812), (6,999.2793106095127), (7,849.0071338527408), (8,737.8560221633246), (9,771.8211752483357), (10,655.7836351785677))
  298. scala>
  299. scala> KSSE.foreach(println)
  300. (2,2876.20580405469)
  301. (3,1680.6647048004902)
  302. (4,1395.7184052948346)
  303. (5,1239.9362814229812)
  304. (6,999.2793106095127)
  305. (7,849.0071338527408)
  306. (8,737.8560221633246)
  307. (9,771.8211752483357)
  308. (10,655.7836351785677)

来自 “ ITPUB博客 ” ，链接：http://blog.itpub.net/29070860/viewspace-2127855/，如需转载，请注明出处，否则将追究法律责任。

转载于:http://blog.itpub.net/29070860/viewspace-2127855/

Spark2-oneHot编码-标准化-主成分-聚类相关推荐

  1. 文献学习(part65)--稳健主成分聚类方法的构建及其比较研究

    学习笔记,仅供参考,有错必纠 关键词:主成分聚类分析:稳健统计量: 协方差矩阵:离群值 文章目录 稳健主成分聚类方法的构建及其比较研究 摘要 引言 传统主成分聚类方法及其不稳健性 传统主成分聚类方法的 ...

  2. R语言主成分PCA、因子分析、聚类对地区经济研究分析重庆市经济指标

    全文下载链接:http://tecdat.cn/?p=27515 建立重庆市经济指标发展体系,以重庆市一小时经济圈作为样本,运用因子分析方法进行实证分析,在借鉴了相关评价理论和评价方法的基础上,本文提 ...

  3. 数据分享|R语言主成分PCA、因子分析、聚类对地区经济研究分析重庆市经济指标...

    原文链接:http://tecdat.cn/?p=27515  建立重庆市经济指标发展体系,以重庆市一小时经济圈作为样本,运用因子分析方法进行实证分析,在借鉴了相关评价理论和评价方法的基础上,本文提取 ...

  4. 主成分分析，充分图，聚类，主成分回归——数据分析与R语言 Lecture 11

    主成分分析,充分图,聚类,主成分回归--数据分析与R语言 Lecture 11 主成分分析 例子:求相关矩阵特征值 例子:求主成分载荷 例子:画碎石图确定主成分 例子:主成分得分-相当于predict ...

  5. R语言主成分回归（PCR）、 多元线性回归特征降维分析光谱数据和汽车油耗、性能数据...

    原文链接:http://tecdat.cn/?p=24152 什么是PCR?(PCR = PCA + MLR)(点击文末"阅读原文"获取完整代码数据). • PCR是处理许多 x ...

  6. R语言实战笔记--第十四章 主成分和因子分析

    R语言实战笔记–第十四章 主成分和因子分析 标签(空格分隔): R语言 主成分分析 因子分析 原理及区别 主成分分析与因子分析很接近,其目的均是为了降维,以更简洁的数据去解释结果,但这两种方法其实是相 ...

  7. ggbiplot-最好看的PCA作图：样品PCA散点+分组椭圆+主成分丰度和相关

    写在前面 https://github.com/vqv/ggbiplot/blob/master/README.md 前几天在<宏基因组0>微信讨论群看到了有人发了一个上面链接,点开一看居 ...

  8. python numpy数组和one-hot编码相互转换

    a=[0,0,1,0,1,0,1]result=[] for i, x in enumerate(a):if x==1:result.append(i)print(result) python num ...

  9. 主成分分析二级指标权重_羡慕神仙权重？主成分与因子分析带你揭开权重的秘密...

    文末领取[世界500强面试题及评点50题] 01 主成分分析 1.主成分分析流程 原始数据标准化 计算标准化变量间的相关系数矩阵 计算相关系数矩阵的特征值和特征向量 计算主成分变量值 统计结果分析,提 ...

最新文章

  1. Java程序员技术培训需要培训哪些?
  2. 推荐系统炼丹笔记：阿里边缘计算+奉送20个推荐系统强特
  3. 提升【百度网盘】下载速度
  4. php asciii 回车换行,ubuntu下关于telnet俩个特殊ascii字符回车0x0d与换行0x0a
  5. java mockserver搭建_mockjs,json-server一起搭建前端通用的数据模拟框架教程
  6. 爬取http://ycb-benchmarks.s3-website-us-east-1.amazonaws.com/的链接并下载文件
  7. Tomcat配置和使用——详解
  8. “请求未在nginx中配置的域名时，给浏览器返回508错误码”配置示例
  9. 【并行计算-CUDA开发】CUDA shared memory bank 冲突
  10. github每次push时自动输入用户名密码
  11. Polycom高清视频会议桌面系统HDX 4000
  12. 淘宝商城事件：中小卖家缺失的互联网信任
  13. 栈和队列的共同处和不同处
  14. 软件著作权登记怎么查询
  15. java batter_android电池管理系统从上层的java到底层驱动的调用（转载）
  16. 掌握这个小技巧，让你的 C++ 编译速度提升 50 倍！
  17. 2013年MBA、MPA、MPAcc入学考试综合能力辅导教材
  18. 【中文】【吴恩达课后编程作业】Course 5 - 序列模型 - 第一周作业
  19. Qt OpenGL 旋转、平移、缩放
  20. 批处理命令 bat文件

热门文章

  1. 说说O2O(2)：O2O和二维码
  2. 程序员写简历时的技术词汇拼写规范备忘录！
  3. BUI+Springboot,问题总结
  4. c 微信项目开发多语言切换,微信小程序实现多国语言的切换
  5. matlab计算投影矩阵,如何在OpenCV和Matlab校准工具箱中形成投影矩阵？
  6. C++语言的define用法
  7. 解决usbisp不识别无法烧录Atmega328P，Arduino不识别问题
  8. JavaScript中实现首字母大写，小写
  9. python热力图颜色设置_【Python】绘制热力图seaborn.heatmap，cmap设置颜色的参数
  10. SQL学习笔记1：SQL语句可以分三类