def chooseBestFeatureToSplit(dataSet):#特征数量，-1是因为最后一列是类别标签numFeatures = len(dataSet[0]) - 1#计算数据集的原始香农熵baseEntropy = calcShannonEnt1(dataSet)bestInfoGain = 0.0                       #信息增益赋初值0bestFeature = -1                         #最优特征的索引值for i in range(numFeatures):#获取dataSet的第i个所有特征存到featList中featList = [example[i] for example in dataSet]#print(featList)  #每个特征的15项特征值列表#创建set集合{}，元素不可重复uniqueVals = set(featList)newEntropy = 0.0for value in uniqueVals:#subDataSet划分后的子集subDataSet = splitDataSet(dataSet,i,value)#计算子集的概率=子集个数/整个训练样本数prob = len(subDataSet)/float(len(dataSet))#计算香农熵newEntropy += prob * calcShannonEnt(subDataSet)#计算信息增益infoGain = baseEntropy - newEntropy#print("第%d个特征的增益为%.3f" %(i,infoGain))#C4.5算法：计算增益比（信息增益率）#infoGain2 = (baseEntropy - newEntropy)/baseEntropyif (infoGain >bestInfoGain):bestInfoGain = infoGain      #更新信息增益，找到最大的信息增益bestFeature = i              #记录信息增益最大的特征的索引值return bestFeature                   #返回信息增益最大的特征的索引值

2.2增益率

称为属性a的“固有值” [Quinlan, 1993]，属性a的可能取值数 目越多（即V越大），则IV(a)的值通常就越大。

增益率准则对可取值数目较少的属性有所偏好。

C4.5 [Quinlan, 1993]采用了一个启发式方法：先从候选划分属性中找出信 息增益高于平均水平的属性，再从中选取增益率最高的

实现代码：

def calcShannonEnt1(dataSet, method = 'none'):numEntries = len(dataSet)labelCount = {}for feature in dataSet:if method =='prob': #当参数为prob时转而计算增益率label =  featureelse:label = feature[-1]if label not in labelCount.keys():labelCount[label]=1else:labelCount[label]+=1shannonEnt = 0.0for key in labelCount:numLabels = labelCount[key]prob = numLabels/numEntriesshannonEnt -= prob*(log(prob,2))return shannonEnt
#增益率
def chooseBestFeatureToSplit2(dataSet): #使用增益率进行划分数据集numFeatures = len(dataSet[0]) -1 #最后一个位置的特征不算baseEntropy = calcShannonEnt(dataSet) #计算数据集的总信息熵bestInfoGain = 0.0bestFeature = -1for i in range(numFeatures):featList = [example[i] for example in dataSet]newEntropyProb = calcShannonEnt1(featList, method='prob') #计算内部增益率uniqueVals = set(featList)newEntropy = 0.0for value in uniqueVals:# 通过不同的特征值划分数据子集subDataSet = splitDataSet(dataSet, i, value)prob = len(subDataSet)/float(len(dataSet))newEntropy += prob *calcGini(subDataSet)newEntropy  = newEntropy*newEntropyProbinfoGain = baseEntropy - newEntropy #计算每个信息值的信息增益if(infoGain > bestInfoGain):bestInfoGain = infoGainbestFeature = ireturn bestFeature #返回信息增益的最佳索引

2.3基尼指数

定义：分类问题中，假设D有K个类，样本点属于第k类的概率为Pk, 则概率 分布的基尼值定义为：

Gini(D)越小，数据集D的纯度越高； 给定数据集D，属性a的基尼指数定义为：

在候选属性集合A中，选择那个使得划分后基尼指数最小的属性作为最有划分属性。

实现代码：

#基尼指数
def calcGini(dataset):feature = [example[-1] for example in dataset]uniqueFeat = set(feature)sumProb =0.0for feat in uniqueFeat:prob = feature.count(feat)/len(uniqueFeat)sumProb += prob*probsumProb = 1-sumProbreturn sumProb
def chooseBestFeatureToSplit3(dataSet): #使用基尼系数进行划分数据集numFeatures = len(dataSet[0]) -1 #最后一个位置的特征不算bestInfoGain = np.InfbestFeature = 0.0for i in range(numFeatures):featList = [example[i] for example in dataSet]uniqueVals = set(featList)newEntropy = 0.0for value in uniqueVals:# 通过不同的特征值划分数据子集subDataSet = splitDataSet(dataSet, i, value)prob = len(subDataSet)/float(len(dataSet))newEntropy += prob *calcGini(subDataSet)infoGain = newEntropyif(infoGain < bestInfoGain): # 选择最小的基尼系数作为划分依据bestInfoGain = infoGainbestFeature = ireturn bestFeature #返回决策属性的最佳索引

2.4数据集

关于购买手机的144条数据，六个属性分别为：颜色, 运行内存, 内存, 价格, 品牌,指纹解锁位置

用于验证的十条数据：

读入数据的代码：

import pandas as pdmyData = pd.read_excel('H:\Python project\shuju.xlsx',header = None)print(myData.head())myData = np.array(myData).tolist()for d in myData:for i in range(len(d)):d[i] = d[i].strip()Labels = ['颜色', '运行内存', '内存', '价格', '品牌','指纹解锁位置']

2.5创建决策树

#创建决策树
def createTree(dataSet,labels):#取分类标签classList = [example[-1] for example in dataSet]#特征可能存在多个属性，需要判断一下，如果类别完全相同则停止继续划分if classList.count(classList[0]) == len(classList):return classList[0]if len(dataSet[0]) == 1:return majorityCnt(classList)                 #遍历完所有特征时返回出现次数最多的类标签bestFeat = chooseBestFeatureToSplit3(dataSet)      #选择最优特征bestFeatLabel = labels[bestFeat]                  #最优特征的类标签myTree = {bestFeatLabel:{}}                       #根据最有特征的标签生成树#得到训练集中所有最优特征的属性值featValues = [example[bestFeat] for example in dataSet]#去掉重复的属性值uniqueVals = set(featValues)#遍历特征，创建决策树for value in uniqueVals:subLabels = labels[:]myTree[bestFeatLabel][value] = createTree(splitDataSet(dataSet, bestFeat, value),subLabels)return myTree

2.6保存和读取决策树

#存储决策树
def storeTree(inputTree, filename):import picklefw = open(filename, 'wb')pickle.dump(inputTree, fw)fw.close()
#读取决策树
def grabTree(filename):import picklefr = open(filename)return pickle.load(fr)  #决策树字典

2.7绘制决策树

#绘制决策树
import matplotlib.pyplot as plt
plt.rcParams['font.sans-serif']=['SimHei']decisionNode = dict(boxstyle="sawtooth", fc="0.8")   #设置结点格式
leafNode = dict(boxstyle="round4", fc="0.8")         #设置叶节点格式
arrow_args = dict(arrowstyle="<-")                   #设置箭头格式#获取决策树叶子结点数目
def getNumLeafs(myTree):numLeafs = 0                                     #初始化叶子结点#firstStr = myTree.keys()[0]firstStr = next(iter(myTree))                    #获取结点属性secondDict = myTree[firstStr]                    #获取下一组字典for key in secondDict.keys():#测试该结点是否为字典，如果不是字典，代表此结点为叶子结点if type(secondDict[key]).__name__=='dict':numLeafs += getNumLeafs(secondDict[key])else:   numLeafs +=1return numLeafs#获取决策树的深度
def getTreeDepth(myTree):maxDepth = 0                              #初始化决策树层数firstStr = next(iter(myTree))secondDict = myTree[firstStr]for key in secondDict.keys():if type(secondDict[key]).__name__=='dict':thisDepth = 1 + getTreeDepth(secondDict[key])else:   thisDepth = 1if thisDepth > maxDepth:maxDepth = thisDepth              #更新树的层数return maxDepth#绘制结点
def plotNode(nodeTxt, centerPt, parentPt, nodeType):createPlot.ax1.annotate(nodeTxt, xy=parentPt,  xycoords='axes fraction',xytext=centerPt, textcoords='axes fraction',va="center", ha="center", bbox=nodeType, arrowprops=arrow_args)#标注有向边属性值
def plotMidText(cntrPt, parentPt, txtString):xMid = (parentPt[0]-cntrPt[0])/2.0 + cntrPt[0]yMid = (parentPt[1]-cntrPt[1])/2.0 + cntrPt[1]createPlot.ax1.text(xMid, yMid, txtString, va="center", ha="center", rotation=30)def plotTree(myTree, parentPt, nodeTxt):#获取决策树叶结点数目，决定了树的宽度numLeafs = getNumLeafs(myTree)#获取决策树层数depth = getTreeDepth(myTree)#获取下一个字典firstStr = next(iter(myTree))#中心位置cntrPt = (plotTree.xOff + (1.0 + float(numLeafs))/2.0/plotTree.totalW, plotTree.yOff)#标注有向边属性值plotMidText(cntrPt, parentPt, nodeTxt)#绘制结点plotNode(firstStr, cntrPt, parentPt, decisionNode)#下一个字典，继续绘制子结点secondDict = myTree[firstStr]plotTree.yOff = plotTree.yOff - 1.0/plotTree.totalDfor key in secondDict.keys():# 测试该结点是否为字典，如果不是字典，代表此结点为叶子结点if type(secondDict[key]).__name__=='dict':#不是叶子结点，递归调用继续绘制plotTree(secondDict[key],cntrPt,str(key))else:      #是叶子结点则进行绘制，标注有向边plotTree.xOff = plotTree.xOff + 1.0/plotTree.totalWplotNode(secondDict[key], (plotTree.xOff, plotTree.yOff), cntrPt, leafNode)plotMidText((plotTree.xOff, plotTree.yOff), cntrPt, str(key))plotTree.yOff = plotTree.yOff + 1.0/plotTree.totalD#创建绘制面板
def createPlot(inTree):fig = plt.figure(1, facecolor='white')            #创建figfig.clf()                                         #清空figaxprops = dict(xticks=[], yticks=[])createPlot.ax1 = plt.subplot(111, frameon=False, **axprops)     #去掉x,y轴plotTree.totalW = float(getNumLeafs(inTree))      #决策树叶子结点数目plotTree.totalD = float(getTreeDepth(inTree))     #决策树层数plotTree.xOff = -0.5/plotTree.totalW; plotTree.yOff = 1.0;   #x偏移plotTree(inTree, (0.5,1.0), '')                   #绘制决策树plt.show()

2.8使用决策树进行分类

#使用决策树的分类函数
def classify(inputTree,featLabels,testVec):#firstStr = next(iter(inputTree))            #获取决策树结点firstStr = list(inputTree.keys())[0]#print(firstStr)secondDict = inputTree[firstStr]            #下一个字典featIndex = featLabels.index(firstStr)      #获取存储选择的最优特征标签的索引classLabel = -1for key in secondDict.keys():if testVec[featIndex] == key:# 测试该结点是否为字典，如果不是字典，代表此结点为叶子结点if type(secondDict[key]).__name__=='dict':classLabel = classify(secondDict[key],featLabels,testVec)else:classLabel = secondDict[key]# 标记classLabel为-1当循环过后若仍然为-1，表示未找到该数据对应的节点则我们返回他兄弟节点出现次数最多的类别if classLabel == -1:return (getLeafBestCls(inputTree))else:return classLabel#求该节点下所有叶子节点的列表
def getLeafscls(myTree, clsList):numLeafs = 0firstStr = list(myTree.keys())[0]secondDict = myTree[firstStr]for key in secondDict.keys():if type(secondDict[key]).__name__ == 'dict':clsList =getLeafscls(secondDict[key],clsList)else:clsList.append(secondDict[key])return clsList#返回出现次数最多的类别
def getLeafBestCls(myTree):clsList = []resultList = getLeafscls(myTree,clsList)return max(resultList,key = resultList.count)

2.9主函数

if __name__ == '__main__':import pandas as pdmyData = pd.read_excel('H:\Python project\shuju.xlsx',header = None)print(myData.head())myData = np.array(myData).tolist()for d in myData:for i in range(len(d)):d[i] = d[i].strip()Labels = ['颜色', '运行内存', '内存', '价格', '品牌','指纹解锁位置']myTree = createTree(myData, Labels)storeTree(storeTree, 'saveTree.txt')  #将决策树存储进入txt文件treePlotter.createPlot(myTree)print("预测结果:")print(classify(myTree, Labels, ['白色', '8G', '256G', '3000以下', '华为','正面解锁']))dataSet2 = [['白色', '8G', '256G', '3000以上', '小米', '侧面解锁'],['白色', '16G', '128G', '3000以下', '华为', '正面解锁'],['黑色', '8G', '128G', '3000以上', '小米', '反面解锁'],['蓝色', '8G', '256G', '3000以下', '华为', '反面解锁'],['黑色', '16G', '256G', '3000以上', '小米', '反面解锁'],['黑色', '8G', '256G', '3000以下', '小米', '侧面解锁'],['黑色', '16G', '128G', '3000以下', '小米', '侧面解锁'],['蓝色', '16G', '256G', '3000以上', '华为', '反面解锁'],['白色', '8G', '256G', '3000以下', '华为', '反面解锁']]for dataVet in dataSet2:print(classify(myTree,Labels,dataSet2))

使用信息增益方法生成的决策树：

对验证数据的验证结果：

正确率为60%

使用增益率方法生成的决策树：

对验证数据的验证结果：

正确率为60%

使用基尼指数方法生成的决策树：

对验证数据的验证结果：

正确率为60%

可能是代码原因或者是数据出现问题，导致三种方法的正确率都一样

信息增益的缺点：通过信息增益来选择特征会偏好取值较多的特征，一个极端情况的例子就是如果对于某个特征所有样本在此特征上的取值都不相同，这样通过这个特征计算得到的信息增益式最大的。

增益率的缺点：信息增益率朝着信息增益相反的方向发展，偏好于取值较少的特征。