决策树学习采用的是自顶向下的递归方法，其基本思想是以信息熵为度量构造一棵熵值下降最快的树，到叶子节点处的熵值为零，此时每个叶节点中的实例都属于同一类。ID3的原理是基于信息熵增益Gain达到最大，设原始问题的标签有正例和负例，p和n表示其相应的个数。则原始问题的信息熵为

# -*- coding: utf-8 -*-#计算给定数据集的香农熵
def calcShannonEnt(dataSet):  numEntries=len(dataSet)  #数据实例总数labelCounts={}  #对类别数量创建了一个数据字典，键值是最后一列的数值for featVec in dataSet:   #featVec表示特征集currentLabel=featVec[-1]    # currentLabel表示当前键值，featVec[-1]表示数据集中的最后一列#如果当前键值不存在，扩展字典将当前键值加入字典，设置当前键值表示的类别数量为0if currentLabel not in labelCounts.keys(): labelCounts[currentLabel]=0#如果当前键值存在，则类别数量累加labelCounts[currentLabel]+=1shannonEnt=0.0for key in labelCounts:prob=float(labelCounts[key])/numEntries #每个键值都记录了当前类别出现的次数shannonEnt -=prob*log(prob,2)return shannonEnt　

#创建数据集
def createDataSet():dataSet=[[1,1,'yes'],[1,1,'yes'],[0,1,'no'],[0,1,'no'],[0,1,'no']]labels=['no surfacing','flippers']return dataSet,labels

1 >>> import trees
2 >>> reload(trees)
3 <module 'trees' from 'E:\python excise\trees.pyc'>
4 >>> myDat,labels=trees.createDataSet()
5 >>> myDat
6 [[1, 1, 'yes'], [1, 1, 'yes'], [0, 1, 'no'], [0, 1, 'no'], [0, 1, 'no']]
7 >>> trees.calcShannonEnt(myDat)
8 0.9709505944546686
9 >>> 

>>> myDat[0][-1]='maybe'
>>> myDat
[[1, 1, 'maybe'], [1, 1, 'yes'], [0, 1, 'no'], [0, 1, 'no'], [0, 1, 'no']]
>>> trees.calcShannonEnt(myDat)
1.3709505944546687

#按照给定特征划分数据集
#dataSet:待划分的数据集，axis:划分数据集的特征，value:需要返回的特征的值
def splitDataSet(dataSet,axis,value):retDataSet=[]   #为了不修改原始数据dataSet，创建一个新的列表对象for featVec in dataSet:if featVec[axis]==value:     reducedFeatVec=featVec[:axis]   #获取从第0列到特征列的数据reducedFeatVec.extend(featVec[axis+1:])  #获取从特征列之后的数据retDataSet.append(reducedFeatVec) #目前reducedFeatVec表示除了特征列的数据return retDataSet

1 >>> reload(trees)
2 <module 'trees' from 'E:\python excise\trees.pyc'>
3 >>> myDat,labels=trees.createDataSet()
4 >>> myDat
5 [[1, 1, 'yes'], [1, 1, 'yes'], [0, 1, 'no'], [0, 1, 'no'], [0, 1, 'no']]
6 >>> trees.splitDataSet(myDat,0,1)
7 [[1, 'yes'], [1, 'yes']]
8 >>> trees.splitDataSet(myDat,0,0)
9 [[1, 'no'], [1, 'no'], [1, 'no']]

#选择最好的数据集划分方式
def chooseBestFeatureToSplit(dataSet):numFeatures=len(dataSet[0])-1        #减去类别那一列baseEntropy=calcShannonEnt(dataSet)   #计算整个数据集的原始香农熵bestInfoGain=0.0;bestFeature=-1  #现在最好的特征是数据集中的最后一列  #i=0，新熵，增益  #i=1，新熵，增益for i in range(numFeatures):    #循环遍历数据集中的所有特征featList=[example[i] for example in dataSet]  #获取第i个特征所有可能的取值，特征0一个列表，特征1一个列表...uniqueVals=set(featList)  #集合数据类型（set）与列表类型相似，不同之处仅在于集合类型中每个值互不相同newEntropy=0.0for value in uniqueVals:subDataSet=splitDataSet(dataSet,i,value)  #划分后的数据集prob=len(subDataSet)/float(len(dataSet))newEntropy+=prob*calcShannonEnt(subDataSet) #求划分完的数据集的熵infoGain=baseEntropy-newEntropyif(infoGain>bestInfoGain):bestInfoGain=infoGainbestFeature=i           return bestFeature

1 >>> reload(trees)
2 <module 'trees' from 'E:\python excise\trees.pyc'>
3 >>> myDat,labels=trees.createDataSet()
4 >>> trees.chooseBestFeatureToSplit(myDat)
5 0

#创建树的函数代码
def createTree(dataSet,labels):classList=[example[-1] for example in dataSet]if classList.count(classList[0])==len(classList):  #类别完全相同规则停止继续划分return classList[0]if len(dataSet[0])==1: #确认至少有数据集    return majorityCnt(classList)bestFeat=chooseBestFeatureToSplit(dataSet)bestFeatLabel=labels[bestFeat]myTree={bestFeatLabel:{}}del(labels[bestFeat])  #得到列表包含的所有属性featValues=[example[bestFeat] for example in dataSet]uniqueVals=set(featValues)for value in uniqueVals:subLabels=labels[:]myTree[bestFeatLabel][value]=createTree(splitDataSet(dataSet,bestFeat,value),subLabels)return myTree　

1 >>> reload(trees)
2 <module 'trees' from 'E:\python excise\trees.pyc'>
3 >>> myDat,labels=trees.createDataSet()
4 >>> myTree=trees.createTree(myDat,labels)
5 >>> myTree
6 {'no surfacing': {0: 'no', 1: 'yes'}}

def majorityCnt(classList):classCount={}for vote in classList:if vote not in classCount.keys():classCount[vote]=0classCount[vote]+=1sortedClassCount=sorted(classCount.iteritems(),key=operator.itemgetter(1),reverse=True)return sortedClassCount[0][0]

#测试算法：使用决策树执行分类
def classify(inputTree,featLabels,testVec):firstStr=inputTree.keys()[0]secondDict=inputTree[firstStr]featIndex=featLabels.index(firstStr)for key in secondDict.keys():if testVec[featIndex]==key:if type(secondDict[key]).__name__=='dict':classLabel=classify(secondDict[key],featLabels,testVec)else:classLabel=secondDict[key]return classLabel

1 >>> import trees
2 >>> myDat,labels=trees.createDataSet()
3 >>> labels
4 ['no surfacing', 'flippers']
5 >>> trees.classify(myTree,labels,[1,0])
6 'no'
7 >>> trees.classify(myTree,labels,[1,1])
8 'yes'

#使用算法：决策树的存储
def storeTree(inputTree,filename):import picklefw=open(filename,'w')pickle.dump(inputTree,fw)fw.close()
def grabTree(filename):import picklefr=open(filename)return pickle.load(fr)

1 >>> reload(trees)
2 >>><module 'tree' from 'trees.py'>
3 >>> trees.storeTree(myTree,'classifierStorage.txt')
4 >>> trees.grabTree('classifierStorage.txt')
5 {'no surfacing': {0: 'no', 1: {'flippers': {0: 'no', 1: 'yes'}}}}