《机器学习实战》笔记——AdaBoost

笔记见备注

 # _*_ coding:utf-8 _*_
 from numpy import *
 # 简单数据集
 def loadSimpData():
     datMat = matrix([[1., 2.1],
                      [2., 1.1],
                      [1.3, 1.],
                      [1., 1.],
                      [2., 1.]])

     classLabels = [1.0, 1.0, -1.0, -1.0, 1.0]
     return datMat, classLabels

 # 7-1 单层决策树生成函数

 # lt=less than
 # 分类器的构建(单纯地将某一特征上的所有取值与输入的阈值进行比较，若制定lt为负，则特征值小于阈值的样本被标记为-1)
 # 相反而知，若指定gt为负，则特征值大于阈值的样本被标记为-1
 def stumpClassify(dataMatrix, dimen, threshVal, threshIneq):
     retArray = ones((shape(dataMatrix)[0],1))
     if threshIneq == 'lt':
         retArray[dataMatrix[:, dimen] <= threshVal] = -1.0
     else:
         retArray[dataMatrix[:, dimen] > threshVal] = -1.0
     return retArray

 # stumpClassify分类器的预测值收到了特征、阈值和阈值两边到底哪边为正标签哪边为父标签的影响
 # 所以有三重循环
 # 第一重：遍历每个特征
 # 第二重：对每个特征上依次设定不同的阈值
 # 第三重：每个特征的每个阈值设定以后 还要依次以小、大于阈值作为依据调用分类器。得出预测结果。将结果与真实结果对比，得出错误向量
 # 通过错误向量得出加权错误值之后与当前的最小错误值进行对比，迭代后得到最终的最小错误
 def buildStump(dataArr, classLabels, D):
     dataMatrix = mat(dataArr)
     labelMat = mat(classLabels).T
     m,n = shape(dataMatrix)
     numSteps = 10.0 # 也可以变大，使得阈值的精确度更高，但是会造成计算量的增大
     bestStump = {}
     bestClasEst = mat(zeros((m,1)))
     minError = inf
     for i in range(n):
         rangeMin = dataMatrix[:,i].min()
         rangeMax = dataMatrix[:,i].max()
         stepSize = (rangeMax - rangeMin)/numSteps
         for j in range(-1, int(numSteps)+1):    # 为什么从-1开始？？？？
             for inequal in ['lt','gt']:
                 threshVal = (rangeMin + float(j)*stepSize)
                 predictedVals = stumpClassify(dataMatrix, i, threshVal, inequal)
                 errArr = mat(ones((m, 1)))
                 errArr[predictedVals == labelMat] = 0
                 weightedError = D.T * errArr
                 # print ("split: dim %d, thresh %.2f, thresh ineqal: %s, the weighted error is %.3f"\
                 # %(i, threshVal, inequal, weightedError))
                 if weightedError < minError:
                     minError = weightedError
                     bestClasEst = predictedVals.copy()
                     bestStump['dim'] = i
                     bestStump['thresh'] = threshVal
                     bestStump['ineq'] = inequal
     return bestStump, minError, bestClasEst # 单层决策树建立出来之后，需要知道的是最好的决策树（是的预测值的加权错误最小的树）的、
     # 特征、阈值、约定的负标签方向；最小错误值；最好的预测结果（一个以样本个数为维度的向量）

 # 7-2 基于单层决策树的AdaBoost训练过程
 def adaBoostTrainDS(dataArr, classLabels, numIt=40):
     weakClassArr = []
     m = shape(dataArr)[0]
     D = mat(ones((m,1))/m)
     aggClassEst = mat(zeros((m,1)))
     for i in range(numIt):
         bestStump, error, classEst = buildStump(dataArr, classLabels, D)
         print ("D:", D.T)
         alpha = float(0.5*log((1.0-error)/max(error,1e-16)))    # 避免没有错误是发生溢出
         bestStump['alpha'] = alpha
         weakClassArr.append(bestStump)
         print ("classEst: ", classEst.T)
         expon = multiply(-1*alpha*mat(classLabels).T, classEst)
         D = multiply(D, exp(expon))
         D = D/D.sum()
         aggClassEst += alpha*classEst
         print ("aggClassEst: ",aggClassEst.T)
         aggErrors = multiply(sign(aggClassEst) != mat(classLabels).T, ones((m,1)))  #ones zeros的（）里面一定不能忘了是元祖，而不是两个数
         errorRate = aggErrors.sum()/m
         print ("total error: ", errorRate, "\n")
         if errorRate == 0.0: break
     return weakClassArr