利用AdaBoost方法构建多个弱分类器进行分类

1.AdaBoost 思想

补充：这里的若分类器之间有比较强的依赖关系;对于若依赖关系的分类器一般使用Bagging的方法

弱分类器是指分类效果要比随机猜测效果略好的分类器，我们可以通过构建多个弱分类器来进行最终抉择（俗话说，三个臭皮匠顶个诸葛亮大概就这意思）。首先我们给每个样例初始化一个权重，构成向量D，然后再更新D，更新规则如下：

当一个样例被分类器正确分类时，我们就减小它的权重

否则，增大它的权重

对于每个弱分类器，我们根据它对样例分类错误率来设置它的权重alpha,分类错误率越高，相应的alpha就会越小，如下所示

最终我们训练出多个弱分类器，通过加权分类结果，输出最终分类结果，如下图所示

2.实验过程

 # -*- coding: utf-8 -*-
 """
 Created on Wed Mar 29 16:57:37 2017

 @author: MyHome
 """
 import  numpy as np

 '''返回分类结果向量'''
 def stumpClassify(dataMatrix,dimen,threshVal,threshIneq):
     retArray = np.ones((np.shape(dataMatrix)[0],1))
     if threshIneq == "lt":
         retArray[dataMatrix[:,dimen] <= threshVal] = -1.0
     else:
         retArray[dataMatrix[:,dimen] > threshVal] = -1.0

     return retArray

 '''构造一个最佳决策树，返回决策树字典'''
 def buildStump(dataArr,classLabels,D):
     dataMatrix = np.mat(dataArr)
     labelMat = np.mat(classLabels).T
     m,n = dataMatrix.shape
     numSteps = 10.0
     bestStump = {}
     bestClassEst = np.mat(np.zeros((m,1)))
     minError = np.inf

     for i in xrange(n):
         rangeMin = dataMatrix[:,i].min()
         rangeMax = dataMatrix[:,i].max()
         stepSize = (rangeMax - rangeMin)/numSteps
         for j in xrange(-1,int(numSteps)+1):
             for inequal in ["lt","gt"]:
                 threshVal = (rangeMin + float(j)*stepSize)
                 #print threshVal
                 predictedVals = stumpClassify(dataMatrix,i,threshVal,inequal)
                 errArr = np.mat(np.ones((m,1)))
                 errArr[predictedVals==labelMat] = 0
                 weightedError = D.T*errArr

                 if weightedError < minError:
                     minError = weightedError
                     bestClassEst = predictedVals.copy()
                     bestStump["dim"] = i
                     bestStump["thresh"] = threshVal
                     bestStump["ineq"] = inequal

     return bestStump,minError,bestClassEst

 '''训练多个单层决策树分类器，构成一个数组'''
 def adaBoostTrainDS(dataArr,classLabels,numIt =40):
     weakClassArr = []
     m = np.shape(dataArr)[0]
     D = np.mat(np.ones((m,1))/m)
     aggClassEst = np.mat(np.zeros((m,1)))
     for i in range(numIt):
         bestStump,error,classEst = buildStump(dataArr,classLabels,D)
         #print "D:",D.T
         alpha = float(0.5*np.log((1.0-error)/max(error,1e-16)))
         bestStump["alpha"] = alpha
         weakClassArr.append(bestStump)
         #print "ClassEst:",classEst.T.shape
         expon = np.multiply(-1*alpha*np.mat(classLabels).T,classEst)
         #print expon
         D = np.multiply(D,np.exp(expon))
         D = D / D.sum()
         aggClassEst += alpha*classEst
         #print "aggClassEst: ",aggClassEst.T
         aggErrors = np.multiply(np.sign(aggClassEst)!= np.mat(classLabels).T,np.ones((m,1)))
         errorRate = aggErrors.sum()/m
         print "total error:",errorRate,"\n"
         if errorRate ==0.0:
             break
     return weakClassArr

 '''分类器'''
 def adaClassify(datToClass,classifierArr):
     dataMatrix = np.mat(datToClass)
     m = np.shape(dataMatrix)[0]
     aggClassEst = np.mat(np.zeros((m,1)))
     for i in range(len(classifierArr)):
         classEst = stumpClassify(dataMatrix,classifierArr[i]["dim"],\
         classifierArr[i]["thresh"],classifierArr[i]["ineq"])
         aggClassEst += classifierArr[i]["alpha"]*classEst
         #print aggClassEst
     return np.sign(aggClassEst)

 '''载入数据'''
 def loadDataSet(fileName):
     numFeat = len(open(fileName).readline().split("\t"))
     dataMat = []
     labelMat = []
     fr = open(fileName)
     for line in fr.readlines():
         lineArr = []
         curLine = line.strip().split("\t")
         for i in range(numFeat-1):
             lineArr.append(float(curLine[i]))
         dataMat.append(lineArr)
         labelMat.append(float(curLine[-1]))
     #print dataMat,labelMat
     return dataMat,labelMat

 if __name__ == "__main__":
     datArr,labelArr =  loadDataSet("horseColicTraining2.txt")

     classifierArray = adaBoostTrainDS(datArr,labelArr,10)
     testData,testY = loadDataSet("horseColicTest2.txt")
     predictionArr = adaClassify(testData,classifierArray)
     errorArr = np.mat(np.ones((len(testData),1)))
     FinalerrorRate = errorArr[predictionArr!= np.mat(testY).T].sum()/float(errorArr.shape[0])
     print "FinalerrorRate:",FinalerrorRate

3.实验结果

total error: 0.284280936455

total error: 0.284280936455

total error: 0.247491638796

total error: 0.247491638796

total error: 0.254180602007

total error: 0.240802675585

total error: 0.240802675585

total error: 0.220735785953

total error: 0.247491638796

total error: 0.230769230769

FinalerrorRate: 0.238805970149

4.实验总结

通过多个构建多个弱分类器，然后根据各个弱分类器的能力大小（即权重)来对分类结果进行加权求和，得出最终结果。只要数据集比较完整，这种方法还是很强大的，后续还可以尝试更多其他的分类器进行集成。