# -*- coding: utf-8 -*-
"""
Created on Mon Feb 22 13:21:22 2016
K-NearestNeighbor
"""
import numpy as np
import operatorclass KNNClassifier():"""This is a Nearest Neighbor classifier. """#定义k的值def __init__(self, k=3):self._k = k#计算新样本与已知分类样本的距离并从小到大排列    def _calEDistance(self, inSample, dataset):m = dataset.shape[0]diffMat = np.tile(inSample, (m,1)) - datasetsqDiffMat = diffMat**2 #每个元素平方sqDistances = sqDiffMat.sum(axis = 1)  #求和distances = sqDistances ** 0.5 #开根号return distances.argsort()  #按距离的从小到达排列的下标值def _classify0(self, inX, dataSet, labels):k = self._kdataSetSize = dataSet.shape[0]                  diffMat = np.tile(inX, (dataSetSize,1)) - dataSet      sqDiffMat = diffMat**2sqDistances = sqDiffMat.sum(axis=1)                  distances = sqDistances**0.5sortedDistIndicies = distances.argsort()            classCount={}                                      for i in range(k):voteIlabel = labels[sortedDistIndicies[i]]classCount[voteIlabel] = classCount.get(voteIlabel,0) + 1sortedClassCount = sorted(classCount.iteritems(), key=operator.itemgetter(1), reverse=True)return sortedClassCount[0][0]#对一个样本进行分类def _classify(self, sample, train_X, train_y):#数据类型检测if isinstance(sample, np.ndarray) and isinstance(train_X, np.ndarray) \and isinstance(train_y, np.ndarray):passelse:try:sample = np.array(sample)train_X = np.array(train_X)train_y = np.array(train_y)except:raise TypeError("numpy.ndarray required for train_X and ..")sortedDistances = self._calEDistance(sample, train_X)classCount = {}for i in range(self._k):oneVote = train_y[sortedDistances[i]] #获取最近的第i个点的类别classCount[oneVote] = classCount.get(oneVote, 0) + 1sortedClassCount = sorted(classCount.iteritems(),\key=operator.itemgetter(1), reverse=True)#print "the sample :", sample, "is classified as",sortedClassCount[0][0]    return sortedClassCount[0][0]def classify(self, test_X, train_X, train_y):results = [] #数据类型检测if isinstance(test_X, np.ndarray) and isinstance(train_X, np.ndarray) \and isinstance(train_y, np.ndarray):passelse:try:test_X = np.array(test_X)train_X = np.array(train_X)train_y = np.array(train_y)except:raise TypeError("numpy.ndarray required for train_X and ..")d = len(np.shape(test_X))if d == 1:sample = test_Xresult = self._classify(sample, train_X, train_y)results.append(result)else:for i in range(len(test_X)):sample = test_X[i]result = self._classify(sample, train_X, train_y)results.append(result)return resultsif __name__=="__main__":train_X = [[1, 2, 0, 1, 0],[0, 1, 1, 0, 1],[1, 0, 0, 0, 1],[2, 1, 1, 0, 1],[1, 1, 0, 1, 1]]train_y = [1, 1, 0, 0, 0]clf = KNNClassifier(k = 3)sample = [[1,2,0,1,0],[1,2,0,1,1]]result = clf.classify(sample, train_X, train_y)

# -*- coding: utf-8 -*-
"""
Created on Mon Feb 22 13:21:22 2016
K-NearestNeighbor
"""
import numpy as np
import operatorclass KNNClassifier():"""This is a Nearest Neighbor classifier. """#定义k的值def __init__(self, k=3):self._k = k#计算新样本与已知分类样本的距离并从小到大排列    def _calEDistance(self, inSample, dataset):m = dataset.shape[0]diffMat = np.tile(inSample, (m,1)) - datasetsqDiffMat = diffMat**2 #每个元素平方sqDistances = sqDiffMat.sum(axis = 1)  #求和distances = sqDistances ** 0.5 #开根号return distances.argsort()  #按距离的从小到达排列的下标值def _classify0(self, inX, dataSet, labels):k = self._kdataSetSize = dataSet.shape[0]                  diffMat = np.tile(inX, (dataSetSize,1)) - dataSet      sqDiffMat = diffMat**2sqDistances = sqDiffMat.sum(axis=1)                  distances = sqDistances**0.5sortedDistIndicies = distances.argsort()            classCount={}                                      for i in range(k):voteIlabel = labels[sortedDistIndicies[i]]classCount[voteIlabel] = classCount.get(voteIlabel,0) + 1sortedClassCount = sorted(classCount.iteritems(), key=operator.itemgetter(1), reverse=True)return sortedClassCount[0][0]#对一个样本进行分类def _classify(self, sample, train_X, train_y):#数据类型检测if isinstance(sample, np.ndarray) and isinstance(train_X, np.ndarray) \and isinstance(train_y, np.ndarray):passelse:try:sample = np.array(sample)train_X = np.array(train_X)train_y = np.array(train_y)except:raise TypeError("numpy.ndarray required for train_X and ..")sortedDistances = self._calEDistance(sample, train_X)classCount = {}for i in range(self._k):oneVote = train_y[sortedDistances[i]] #获取最近的第i个点的类别classCount[oneVote] = classCount.get(oneVote, 0) + 1sortedClassCount = sorted(classCount.iteritems(),\key=operator.itemgetter(1), reverse=True)#print "the sample :", sample, "is classified as",sortedClassCount[0][0]    return sortedClassCount[0][0]def classify(self, test_X, train_X, train_y):results = [] #数据类型检测if isinstance(test_X, np.ndarray) and isinstance(train_X, np.ndarray) \and isinstance(train_y, np.ndarray):passelse:try:test_X = np.array(test_X)train_X = np.array(train_X)train_y = np.array(train_y)except:raise TypeError("numpy.ndarray required for train_X and ..")d = len(np.shape(test_X))if d == 1:sample = test_Xresult = self._classify(sample, train_X, train_y)results.append(result)else:for i in range(len(test_X)):sample = test_X[i]result = self._classify(sample, train_X, train_y)results.append(result)return resultsif __name__=="__main__":train_X = [[1, 2, 0, 1, 0],[0, 1, 1, 0, 1],[1, 0, 0, 0, 1],[2, 1, 1, 0, 1],[1, 1, 0, 1, 1]]train_y = [1, 1, 0, 0, 0]clf = KNNClassifier(k = 3)sample = [[1,2,0,1,0],[1,2,0,1,1]]result = clf.classify(sample, train_X, train_y)

# -*- coding: utf-8 -*-
"""
Created on Mon Nov 07 09:06:37 2016@author: yehx
"""
# -*- coding: utf-8 -*-
"""
Created on Sun Feb 21 12:17:10 2016
Decision Tree Source Code
@author: liudiwei
"""
import os
import numpy as npclass DecitionTree():"""This is a decision tree classifier. """def __init__(self, criteria='ID3'):self._tree = Noneif criteria == 'ID3' or criteria == 'C4.5':self._criteria = criteriaelse:raise Exception("criterion should be ID3 or C4.5")def _calEntropy(slef, y):'''功能：_calEntropy用于计算香农熵 e=-sum(pi*log pi)参数：其中y为数组array输出：信息熵entropy'''n = y.shape[0]  labelCounts = {}for label in y:if label not in labelCounts.keys():labelCounts[label] = 1else:labelCounts[label] += 1entropy = 0.0for key in labelCounts:prob = float(labelCounts[key])/nentropy -= prob * np.log2(prob)return entropydef _splitData(self, X, y, axis, cutoff):"""参数：X为特征,y为label,axis为某个特征的下标,cutoff是下标为axis特征取值值输出：返回数据集中特征下标为axis，特征值等于cutoff的子数据集先将特征列从样本矩阵里除去，然后将属性值为cutoff的数据归为一类"""ret = []featVec = X[:,axis]n = X.shape[1]      #特征个数#除去第axis列特征后的样本矩阵X = X[:,[i for i in range(n) if i!=axis]]for i in range(len(featVec)):if featVec[i] == cutoff:ret.append(i)return X[ret, :], y[ret]   def _chooseBestSplit(self, X, y):"""ID3 & C4.5参数：X为特征，y为label功能：根据信息增益或者信息增益率来获取最好的划分特征输出：返回最好划分特征的下标"""numFeat = X.shape[1]baseEntropy = self._calEntropy(y)bestSplit = 0.0best_idx  = -1for i in range(numFeat):featlist = X[:,i]   #得到第i个特征对应的特征列uniqueVals = set(featlist)curEntropy = 0.0splitInfo = 0.0for value in uniqueVals:sub_x, sub_y = self._splitData(X, y, i, value)prob = len(sub_y)/float(len(y))      #计算某个特征的某个值的概率curEntropy += prob * self._calEntropy(sub_y)    #迭代计算条件熵splitInfo -=  prob * np.log2(prob) #分裂信息，用于计算信息增益率IG = baseEntropy - curEntropyif self._criteria=="ID3":if IG > bestSplit:bestSplit = IGbest_idx = iif self._criteria=="C4.5":if splitInfo == 0.0:passIGR = IG/splitInfoif IGR > bestSplit:bestSplit = IGRbest_idx = ireturn best_idxdef _majorityCnt(self, labellist):"""参数:labellist是类标签，序列类型为list输出：返回labellist中出现次数最多的label"""labelCount={}for vote in labellist:if vote not in labelCount.keys(): labelCount[vote] = 0labelCount[vote] += 1sortedClassCount = sorted(labelCount.iteritems(), key=lambda x:x[1], \reverse=True)return sortedClassCount[0][0]def _createTree(self, X, y, featureIndex):"""参数:X为特征,y为label,featureIndex类型是元组，记录X特征在原始数据中的下标输出:根据当前的featureIndex创建一颗完整的树"""labelList = list(y)#如果所有的标签都一样（叶节点），直接返回标签if labelList.count(labelList[0]) == len(labelList): return labelList[0]#如果没有特征可以继续划分，那么将所有的label归为大多数的一类，并返回标签if len(featureIndex) == 0:return self._majorityCnt(labelList)#返回最佳分类特征的下标bestFeatIndex = self._chooseBestSplit(X,y)#返回最佳分类特征的索引bestFeatAxis = featureIndex[bestFeatIndex]featureIndex = list(featureIndex)#获得删除最佳分类特征索引后的列表
        featureIndex.remove(bestFeatAxis)featureIndex = tuple(featureIndex)myTree = {bestFeatAxis:{}}featValues = X[:, bestFeatIndex]uniqueVals = set(featValues)for value in uniqueVals:#对每个value递归地创建树sub_X, sub_y = self._splitData(X,y, bestFeatIndex, value)myTree[bestFeatAxis][value] = self._createTree(sub_X, sub_y, \featureIndex)return myTree  def fit(self, X, y):"""参数：X是特征，y是类标签注意事项：对数据X和y进行类型检测，保证其为array输出：self本身"""if isinstance(X, np.ndarray) and isinstance(y, np.ndarray):passelse:try:X = np.array(X)y = np.array(y)except:raise TypeError("numpy.ndarray required for X,y")featureIndex  = tuple(['x'+str(i) for i in range(X.shape[1])])self._tree = self._createTree(X,y,featureIndex)return self  #allow using: clf.fit().predict()def _classify(self, tree, sample):"""用训练好的模型对输入数据进行分类 注意：决策树的构建是一个递归的过程，用决策树分类也是一个递归的过程_classify()一次只能对一个样本（sample）分类"""featIndex = tree.keys()[0] #得到数的根节点值secondDict = tree[featIndex] #得到以featIndex为划分特征的结果axis=featIndex[1:] #得到根节点特征在原始数据中的下标key = sample[int(axis)] #获取待分类样本中下标为axis的值valueOfKey = secondDict[key] #获取secondDict中keys为key的value值if type(valueOfKey).__name__=='dict': #如果value为dict，则继续递归分类return self._classify(valueOfKey, sample)else: return valueOfKeydef predict(self, X):if self._tree==None:raise NotImplementedError("Estimator not fitted, call `fit` first")#对X的类型进行检测，判断其是否是数组if isinstance(X, np.ndarray): passelse: try:X = np.array(X)except:raise TypeError("numpy.ndarray required for X")if len(X.shape) == 1:return self._classify(self._tree, X)else:result = []for i in range(X.shape[0]):value = self._classify(self._tree, X[i])print str(i+1)+"-th sample is classfied as:", value result.append(value)return np.array(result)def show(self, outpdf):if self._tree==None:pass#plot the tree using matplotlibimport treePlottertreePlotter.createPlot(self._tree, outpdf)if __name__=="__main__":trainfile=r"data\train.txt"testfile=r"data\test.txt"import syssys.path.append(r"F:\CSU\Github\MachineLearning\lib")  import dataload as dloadtrain_x, train_y = dload.loadData(trainfile)test_x, test_y = dload.loadData(testfile)clf = DecitionTree(criteria="C4.5")clf.fit(train_x, train_y)result = clf.predict(test_x)    outpdf = r"tree.pdf"clf.show(outpdf)

# -*- coding: utf-8 -*-
"""
Created on Tue Nov 08 14:01:44 2016
K - means cluster
"""import numpy as npclass KMeansClassifier():"this is a k-means classifier"def __init__(self, k=3, initCent='random', max_iter=500 ):self._k = kself._initCent = initCentself._max_iter = max_iterself._clusterAssment = Noneself._labels = Noneself._sse = Nonedef _calEDist(self, arrA, arrB):"""功能：欧拉距离距离计算输入：两个一维数组"""return np.math.sqrt(sum(np.power(arrA-arrB, 2)))def _calMDist(self, arrA, arrB):"""功能：曼哈顿距离距离计算输入：两个一维数组"""return sum(np.abs(arrA-arrB))def _randCent(self, data_X, k):"""功能：随机选取k个质心输出：centroids #返回一个m*n的质心矩阵"""n = data_X.shape[1] #获取特征的维数centroids = np.empty((k,n))  #使用numpy生成一个k*n的矩阵，用于存储质心for j in range(n):minJ = min(data_X[:, j])rangeJ  = float(max(data_X[:, j] - minJ))#使用flatten拉平嵌套列表(nested list)centroids[:, j] = (minJ + rangeJ * np.random.rand(k, 1)).flatten()return centroids def fit(self, data_X):"""输入：一个m*n维的矩阵"""if not isinstance(data_X, np.ndarray) or \isinstance(data_X, np.matrixlib.defmatrix.matrix):try:data_X = np.asarray(data_X)except:raise TypeError("numpy.ndarray resuired for data_X")m = data_X.shape[0]  #获取样本的个数#一个m*2的二维矩阵，矩阵第一列存储样本点所属的族的索引值，#第二列存储该点与所属族的质心的平方误差self._clusterAssment = np.zeros((m,2)) if self._initCent == 'random':self._centroids = self._randCent(data_X, self._k)clusterChanged = Truefor _ in range(self._max_iter): #使用"_"主要是因为后面没有用到这个值clusterChanged = Falsefor i in range(m):   #将每个样本点分配到离它最近的质心所属的族minDist = np.inf #首先将minDist置为一个无穷大的数minIndex = -1    #将最近质心的下标置为-1for j in range(self._k): #次迭代用于寻找最近的质心arrA = self._centroids[j,:]arrB = data_X[i,:]distJI = self._calEDist(arrA, arrB) #计算误差值if distJI <</span> minDist:minDist = distJIminIndex = jif self._clusterAssment[i,0] !=minIndex:clusterChanged = Trueself._clusterAssment[i,:] = minIndex, minDist**2if not clusterChanged:#若所有样本点所属的族都不改变,则已收敛,结束迭代breakfor i in range(self._k):#更新质心，将每个族中的点的均值作为质心index_all = self._clusterAssment[:,0] #取出样本所属簇的索引值value = np.nonzero(index_all==i) #取出所有属于第i个簇的索引值ptsInClust = data_X[value[0]]    #取出属于第i个簇的所有样本点self._centroids[i,:] = np.mean(ptsInClust, axis=0) #计算均值
        self._labels = self._clusterAssment[:,0]self._sse = sum(self._clusterAssment[:,1])def predict(self, X):#根据聚类结果，预测新输入数据所属的族#类型检查if not isinstance(X,np.ndarray):try:X = np.asarray(X)except:raise TypeError("numpy.ndarray required for X")m = X.shape[0]#m代表样本数量preds = np.empty((m,))for i in range(m):#将每个样本点分配到离它最近的质心所属的族minDist = np.inffor j in range(self._k):distJI = self._calEDist(self._centroids[j,:], X[i,:])if distJI <</span> minDist:minDist = distJIpreds[i] = jreturn predsclass biKMeansClassifier():"this is a binary k-means classifier"def __init__(self, k=3):self._k = kself._centroids = Noneself._clusterAssment = Noneself._labels = Noneself._sse = Nonedef _calEDist(self, arrA, arrB):"""功能：欧拉距离距离计算输入：两个一维数组"""return np.math.sqrt(sum(np.power(arrA-arrB, 2)))def fit(self, X):m = X.shape[0]self._clusterAssment = np.zeros((m,2))centroid0 = np.mean(X, axis=0).tolist()centList =[centroid0]for j in range(m):#计算每个样本点与质心之间初始的平方误差self._clusterAssment[j,1] = self._calEDist(np.asarray(centroid0), \X[j,:])**2while (len(centList) <</span> self._k):lowestSSE = np.inf#尝试划分每一族,选取使得误差最小的那个族进行划分for i in range(len(centList)):index_all = self._clusterAssment[:,0] #取出样本所属簇的索引值value = np.nonzero(index_all==i) #取出所有属于第i个簇的索引值ptsInCurrCluster = X[value[0],:] #取出属于第i个簇的所有样本点clf = KMeansClassifier(k=2)clf.fit(ptsInCurrCluster)#划分该族后，所得到的质心、分配结果及误差矩阵centroidMat, splitClustAss = clf._centroids, clf._clusterAssmentsseSplit = sum(splitClustAss[:,1])index_all = self._clusterAssment[:,0] value = np.nonzero(index_all==i)sseNotSplit = sum(self._clusterAssment[value[0],1])if (sseSplit + sseNotSplit) <</span> lowestSSE:bestCentToSplit = ibestNewCents = centroidMatbestClustAss = splitClustAss.copy()lowestSSE = sseSplit + sseNotSplit#该族被划分成两个子族后,其中一个子族的索引变为原族的索引#另一个子族的索引变为len(centList),然后存入centListbestClustAss[np.nonzero(bestClustAss[:,0]==1)[0],0]=len(centList)bestClustAss[np.nonzero(bestClustAss[:,0]==0)[0],0]=bestCentToSplitcentList[bestCentToSplit] = bestNewCents[0,:].tolist()centList.append(bestNewCents[1,:].tolist())self._clusterAssment[np.nonzero(self._clusterAssment[:,0] == \bestCentToSplit)[0],:]= bestClustAss self._labels = self._clusterAssment[:,0] self._sse = sum(self._clusterAssment[:,1])self._centroids = np.asarray(centList)def predict(self, X):#根据聚类结果，预测新输入数据所属的族#类型检查if not isinstance(X,np.ndarray):try:X = np.asarray(X)except:raise TypeError("numpy.ndarray required for X")m = X.shape[0]#m代表样本数量preds = np.empty((m,))for i in range(m):#将每个样本点分配到离它最近的质心所属的族minDist = np.inffor j in range(self._k):distJI = self._calEDist(self._centroids[j,:],X[i,:])if distJI <</span> minDist:minDist = distJIpreds[i] = jreturn preds

# -*- coding: utf-8 -*-
"""
Created on Thu Nov 17 23:04:33 2016
获取股票的历史涨跌幅，并分别存为csv格式
@author: yehx
"""import numpy as np
import pandas as pd#按照市值从小到大的顺序活得100支股票的代码
df = get_fundamentals(query(fundamentals.eod_derivative_indicator.market_cap).order_by(fundamentals.eod_derivative_indicator.market_cap.asc()).limit(100),'2016-11-17', '1y')#分别对这一百只股票进行100支股票操作
#获取从2016.05.01到2016.11.17的涨跌幅数据
#选取记录大于40个的数据，去除次新股
#将文件名名为“股票代码.csv”
for stock in range(100):priceChangeRate = get_price_change_rate(df['market_cap'].columns[stock], '20160501', '20161117')if priceChangeRate is None:openDays = 0else:openDays = len(priceChangeRate)if openDays > 40:tempPrice = priceChangeRate[39:(openDays - 1)]for rate in range(len(tempPrice)):tempPrice[rate] = "%.3f" %tempPrice[rate]fileName = ''fileName = fileName.join(df['market_cap'].columns[i].split('.')) + '.csv'fileNametempPrice.to_csv(fileName)

# -*- coding: utf-8 -*-
"""
Created on Wed Nov 09 15:21:48 2016
Logistic回归分类
"""
import numpy as npclass LogisticRegressionClassifier():def __init__(self):self._alpha = None#定义一个sigmoid函数def _sigmoid(self, fx):return 1.0/(1 + np.exp(-fx))#alpha为步长（学习率）；maxCycles最大迭代次数def _gradDescent(self, featData, labelData, alpha, maxCycles):dataMat = np.mat(featData)                      #size: m*nlabelMat = np.mat(labelData).transpose()        #size: m*1m, n = np.shape(dataMat)weigh = np.ones((n, 1)) for i in range(maxCycles):hx = self._sigmoid(dataMat * weigh)error = labelMat - hx       #size:m*1weigh = weigh + alpha * dataMat.transpose() * error#根据误差修改回归系数return weigh#使用梯度下降方法训练模型，如果使用其它的寻参方法，此处可以做相应修改def fit(self, train_x, train_y, alpha=0.01, maxCycles=100):return self._gradDescent(train_x, train_y, alpha, maxCycles)#使用学习得到的参数进行分类def predict(self, test_X, test_y, weigh):dataMat = np.mat(test_X)labelMat = np.mat(test_y).transpose()  #使用transpose()转置hx = self._sigmoid(dataMat*weigh)  #size:m*1m = len(hx)error = 0.0for i in range(m):if int(hx[i]) > 0.5:print str(i+1)+'-th sample ', int(labelMat[i]), 'is classfied as: 1' if int(labelMat[i]) != 1:error += 1.0print "classify error."else:print str(i+1)+'-th sample ', int(labelMat[i]), 'is classfied as: 0' if int(labelMat[i]) != 0:error += 1.0print "classify error."error_rate = error/mprint "error rate is:", "%.4f" %error_ratereturn error_rate

# -*- coding: utf-8 -*-
"""
Created on Mon Nov 14 11:15:47 2016
Naive Bayes Clssification
"""
# -*- coding: utf-8 -*-
import numpy as npclass NaiveBayes:def __init__(self):self._creteria = "NB"def _createVocabList(self, dataList):"""创建一个词库向量"""vocabSet = set([])for line in dataList:print set(line)vocabSet = vocabSet | set(line)return list(vocabSet)#文档词集模型def _setOfWords2Vec(self, vocabList, inputSet):"""功能:根据给定的一行词，将每个词映射到此库向量中，出现则标记为1，不出现则为0"""outputVec = [0] * len(vocabList)for word in inputSet:if word in vocabList:outputVec[vocabList.index(word)] = 1else:print "the word:%s is not in my vocabulary!" % wordreturn outputVec# 修改 _setOfWordsVec  文档词袋模型def _bagOfWords2VecMN(self, vocabList, inputSet):"""功能：对每行词使用第二种统计策略，统计单个词的个数，然后映射到此库中输出：一个n维向量，n为词库的长度，每个取值为单词出现的次数"""returnVec = [0]*len(vocabList)for word in inputSet:if word in vocabList:returnVec[vocabList.index(word)] += 1 # 更新此处代码return returnVecdef _trainNB(self, trainMatrix, trainLabel):"""输入：训练矩阵和类别标签,格式为numpy矩阵格式功能：计算条件概率和类标签概率"""numTrainDocs = len(trainMatrix) #统计样本个数numWords = len(trainMatrix[0])  #统计特征个数，理论上是词库的长度pNeg = sum(trainLabel)/float(numTrainDocs) #计算负样本出现的概率
        p0Num = np.ones(numWords) #初始样本个数为1，防止条件概率为0，影响结果     p1Num = np.ones(numWords) #作用同上
       p0InAll = 2.0 #词库中只有两类，所以此处初始化为2(use laplace)p1InAll = 2.0 # 再单个文档和整个词库中更新正负样本数据for i in range(numTrainDocs):if trainLabel[i] == 1:p1Num += trainMatrix[i]p1InAll += sum(trainMatrix[i])else:p0Num += trainMatrix[i]p0InAll += sum(trainMatrix[i])print p1InAll#计算给定类别的条件下，词汇表中单词出现的概率#然后取log对数，解决条件概率乘积下溢p0Vect = np.log(p0Num/p0InAll) #计算类标签为0时的其它属性发生的条件概率p1Vect = np.log(p1Num/p1InAll)  #log函数默认以e为底  #p(ci|w=0)return p0Vect, p1Vect, pNegdef _classifyNB(self, vecSample, p0Vec, p1Vec, pNeg):"""使用朴素贝叶斯进行分类,返回结果为0/1"""prob_y0 = sum(vecSample * p0Vec) + np.log(1-pNeg)prob_y1 = sum(vecSample * p1Vec) + np.log(pNeg) #log是以e为底if prob_y0 <</span> prob_y1:return 1else:return 0# 测试NB算法def testingNB(self, testSample):listOPosts, listClasses = loadDataSet()myVocabList = self._createVocabList(listOPosts)
#        print myVocabListtrainMat=[]for postinDoc in listOPosts:trainMat.append(self._bagOfWords2VecMN(myVocabList, postinDoc))p0V,p1V,pAb = self._trainNB(np.array(trainMat), np.array(listClasses))print trainMatthisSample = np.array(self._bagOfWords2VecMN(myVocabList, testSample))result = self._classifyNB(thisSample, p0V, p1V, pAb)print testSample,'classified as: ', resultreturn result###############################################################################
def loadDataSet():wordsList=[['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],['my', 'dalmation', 'is', 'so', 'cute', ' and', 'I', 'love', 'him'],['stop', 'posting', 'stupid', 'worthless', 'garbage'],['mr', 'licks','ate','my', 'steak', 'how', 'to', 'stop', 'him'],['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]classLable = [0,1,0,1,0,1] # 0：good; 1:badreturn wordsList, classLableif __name__=="__main__":    clf = NaiveBayes()testEntry = [['love', 'my', 'girl', 'friend'],['stupid', 'garbage'],['Haha', 'I', 'really', "Love", "You"],['This', 'is', "my", "dog"]]clf.testingNB(testEntry[0])
#    for item in testEntry:
#        clf.testingNB(item)

# -*- coding: utf-8 -*-
"""
Created on Thu Nov 17 23:04:33 2016
获取股票的历史涨跌幅，先合并为DataFrame后存为csv格式
@author: yehx
"""import numpy as np
import pandas as pd#按照市值从小到大的顺序获得50支股票的代码
df = get_fundamentals(query(fundamentals.eod_derivative_indicator.market_cap).order_by(fundamentals.eod_derivative_indicator.market_cap.asc()).limit(50),'2016-11-17', '1y')
b1= {}
priceChangeRate_300 = get_price_change_rate('000300.XSHG', '20060101', '20161118')
df300 = pd.DataFrame(priceChangeRate_300)
lenReference =  len(priceChangeRate_300)
dfout = df300dflen = pd.DataFrame()
dflen['000300.XSHG'] = [lenReference]
#分别对这一百只股票进行50支股票操作
#获取从2006.01.01到2016.11.17的涨跌幅数据
#将数据存到DataFrame中
#DataFrame存为csv文件
for stock in range(50):priceChangeRate = get_price_change_rate(df['market_cap'].columns[stock], '20150101', '20161118')if priceChangeRate is None:openDays = 0else:openDays = len(priceChangeRate)dftempPrice = pd.DataFrame(priceChangeRate)tempArr = []for i in range(lenReference):          if df300.index[i] in list(dftempPrice.index):#保存为4位有效数字tempArr.append( "%.4f" %((dftempPrice.loc[str(df300.index[i])][0])))passelse:tempArr.append(float(0.0))fileName = ''fileName = fileName.join(df['market_cap'].columns[stock].split('.')) dfout[fileName] = tempArrdflen[fileName] = [len(priceChangeRate)]dfout = dfout.append(dflen)
dfout.to_csv('00050.csv')

# -*- coding: utf-8 -*-
"""
Created on Thu Nov 17 23:04:33 2016
csv格式股票历史涨跌幅数据处理
@author: yehx
"""import numpy as np
import pandas as pd
import random
import csv
import sys
reload(sys)
sys.setdefaultencoding('utf-8')'''- 加载csv格式数据
'''
def loadCSVfile1(datafile):filelist = []with open(datafile) as file:lines = csv.reader(file)for oneline in lines:filelist.append(oneline)filelist = np.array(filelist)return filelist#数据处理
#随机选取30个历史涨跌幅数据
#构建自己的数据库def dataProcess(dataArr, subLen):totLen, totWid = np.shape(data)print totLen, totWidlenArr = dataArr[totLen-1,2:totWid]columnCnt = 1dataOut = []for lenData in lenArr:columnCnt = columnCnt + 1N60 = int(lenData) / (2 * subLen)print N60if N60 > 0:randIndex = random.sample(range(totLen-int(lenData)-1,totLen-subLen), N60)for i in randIndex:dataOut.append(dataArr[i:(i+subLen),columnCnt])dataOut = np.array(dataOut)return dataOutif __name__=="__main__":datafile = "00100 (3).csv"data = loadCSVfile1(datafile) df = pd.DataFrame(data)m, n = np.shape(data)dataOut = dataProcess(data, 30)m, n = np.shape(dataOut)#保存处理结果csvfile = file('csvtest.csv', 'wb')writer = csv.writer(csvfile)writer.writerows(dataOut)csvfile.close()