机器学习实战第7章——利用adaboost元算法提高分类性能(代码片段)

import numpy as np
from math import log


def loadSimpData():
    datMat = np.matrix([
        [1. , 2.1],
        [2, 1.1],
        [1.3, 1.],
        [1., 1.],
        [2., 1.]
    ])
    classLabels = [1.0, 1.0, -1.0, -1.0, 1.0]
    return datMat, classLabels


def loadDataSet(fileName):      #general function to parse tab -delimited floats
    numFeat = len(open(fileName).readline().split(‘	‘)) #get number of fields
    dataMat = []; labelMat = []
    fr = open(fileName)
    for line in fr.readlines():
        lineArr =[]
        curLine = line.strip().split(‘	‘)
        for i in range(numFeat-1):
            lineArr.append(float(curLine[i]))
        dataMat.append(lineArr)
        labelMat.append(float(curLine[-1]))
    return dataMat,labelMat


"""
    函数说明：通过阈值比较对数据进行分类
    Parameters:
        dataMatrix -输入数据
        dimen -第几列
        threshVal -阈值
        threshIneq -不等号
    return：
        分类结果
"""
def stumpClassify(dataMatrix, dimen, threshVal, threshIneq):
    retArray = np.ones((np.shape(dataMatrix)[0],1))
    if threshIneq == ‘lt‘:
        retArray[dataMatrix[:,dimen] <= threshVal] = -1.0
    else:
        retArray[dataMatrix[:,dimen] <= threshVal] = 1.0
    return retArray


"""
    函数说明：建立单层决策树
    Parameters:
        dataArr     -输入数据集
        classLabels -分类
        D           -权重向量
    return:
        bestStump   -最佳单层决策树字典
        minError    -最小错误率
        bestClassEst    -最佳类别估计值
"""
def buildStump(dataArr, classLabels, D):
    dataMatrix = np.mat(dataArr)
    labelMat = np.mat(classLabels).T
    m, n = np.shape(dataMatrix)
    #numSteps:步数; bestStump:保存最佳单层决策树的相关信息; bestClassEst:最佳预测结果
    numSteps = 10.0; bestStump = ; bestClassEst = np.mat(np.zeros((m, 1)))
    minError = np.inf
    # 对每个特征进行遍历
    for i in range(n):
        rangeMin = dataMatrix[:,i].min(); rangeMax = dataMatrix[:,i].max();
        stepSize = (rangeMax - rangeMin) / numSteps
        # 遍历所有步长，阈值设置为整个取值范围之外也可以
        for j in range(-1, int(numSteps)+1):
            # 在大于和小于之间切换
            for inequal in [‘lt‘, ‘gt‘]:
                threshVal = (rangeMin + float(j) * stepSize)
                predictedVals = stumpClassify(dataMatrix, i, threshVal, inequal)
                errArr = np.mat(np.ones((m, 1)))
                errArr[predictedVals == labelMat] = 0
                weightError = D.T * errArr    #计算加权错误率
                print("split:dim %d,thresh %.2f,thresh inequal: %s,the weighted error is %.3f"%(i,threshVal,inequal,weightError))
                # 找到最小错误率的最佳单层决策树，并保存相关信息
                if weightError < minError:
                    minError = weightError
                    bestClassEst = predictedVals.copy()
                    bestStump[‘dim‘] = i
                    bestStump[‘thresh‘] = threshVal
                    bestStump[‘ineq‘] = inequal
    return bestStump, minError, bestClassEst


"""
    函数说明：基于单层决策树的AdaBoost训练过程
    Patameters:
        dataArr     -数据集
        classLabels -数据标签
        numIt       -迭代次数
    return:
        weakClassArr    -所有的最佳分类器决策树
        aggClassEst     -累计分类估计
"""
def adaBoostTrainDS(dataArr,classLabels,numIt=40):
    weakClassArr = []
    m = np.shape(dataArr)[0]
    D = np.mat(np.ones((m,1))/m)   #init D to all equal
    aggClassEst = np.mat(np.zeros((m,1)))
    for i in range(numIt):
        bestStump,error,classEst = buildStump(dataArr,classLabels,D)#build Stump
        print("D:",D.T)
        # alpha公式：alpha=1/2*ln((1-error)/error)
        # max(error, 1e-16)是为了防止error=0的情况
        alpha = float(0.5 * log((1.0 - error) / max(error, 1e-16)))
        bestStump[‘alpha‘] = alpha
        weakClassArr.append(bestStump)                  #保存最优单层决策树
        print("classEst: ",classEst.T)
        #更新D，D是一个概率分布向量，总和为1
        # D=(D*exp(-alpha*classLabels*classEst))/D.sum()
        # classEst是分类器预测的分类结果，classEst是
        # np.multiply是矩阵对应元素相乘，返回和矩阵大小一样，classLabels是m*1,classEst是1*m
        expon = np.multiply(-1 * alpha * np.mat(classLabels).T,classEst) #exponent for D calc, getting messy
        D = np.multiply(D, np.exp(expon))                              #Calc New D for next iteration
        D = D / D.sum()
        #更新累计估计类别
        #aggClassEst记录每个类别的累计估计值，最终分类函数为f(x) = sign(aggClassEst)
        aggClassEst += alpha*classEst
        print("aggClassEst: ",aggClassEst.T)
        #计算错误率
        aggErrors = np.multiply(np.sign(aggClassEst) != np.mat(classLabels).T,np.ones((m,1)))
        errorRate = aggErrors.sum() / m
        print("total error: ",errorRate)
        #误差为0时退出循环
        if errorRate == 0.0: break
    return weakClassArr,aggClassEst


"""
    函数说明：测试算法，AdaBoost分类函数
    Parameters: 
        datToClass      -待分类样例
        classifierArr   -多个弱分类器组成的数组
    return：
        类别
"""
def adaClassify(datToClass,classifierArr):
    dataMatrix = np.mat(datToClass)
    m = np.shape(dataMatrix)[0]
    aggClassEst = np.mat(np.zeros((m,1)))
    for i in range(len(classifierArr)):
        #基于每个分类器得到一个类别值
        print(classifierArr[i])
        classEst = stumpClassify(dataMatrix,classifierArr[i][‘dim‘],                                 classifierArr[i][‘thresh‘],                                 classifierArr[i][‘ineq‘])#call stump classify
        aggClassEst += classifierArr[i][‘alpha‘]*classEst
        print(aggClassEst)
    return np.sign(aggClassEst)

if __name__ == ‘__main__‘:
    datMat, classLabels = loadSimpData()
    D = np.mat(np.ones((5,1)) / 5)
    buildStump(datMat, classLabels, D)
    classifierArr, aggClassEst = adaBoostTrainDS(datMat, classLabels)
    print(adaClassify([0,0],classifierArr))
    print(adaClassify([[0,0],[1,0],[2,2]],classifierArr))
    #将第5章的马疝病数据应用到AdaBoost算法中
    datArr, labelArr = loadDataSet(‘horseColicTraining2.txt‘)
    classifierArr,aggClassEst = adaBoostTrainDS(datArr, labelArr, 10)
    testArr, testLableArr = loadDataSet(‘horseColicTest2.txt‘)
    prediction = adaClassify(testArr, classifierArr)
    errArr = np.mat(np.ones((67,1)))
    errRate = errArr[prediction!= np.mat(testLableArr).T].sum() / 67
    print(errRate)

"""
    函数说明：ROC曲线绘制及AUC计算函数
    Parameters:
        predStrengths   -分类器的预测强度,马疝病数据集中为1*299
        classLabels     -分类标签
"""
def plotROC(predStrengths, classLabels):
    import matplotlib.pyplot as plt
    # cur = (1.0,1.0) #绘制光标的位置
    cur = (0.0,0.0) #绘制光标的位置
    ySum = 0.0 #用于计算AUC的值
    numPosClas = sum(np.array(classLabels)==1.0)    #保存正例的数目
    yStep = 1/float(numPosClas); xStep = 1/float(len(classLabels)-numPosClas)   #y轴的步长、x轴的步长
    sortedIndicies = predStrengths.argsort()#获取排好序的索引，从小到大
    fig = plt.figure()
    fig.clf()
    ax = plt.subplot(111)
    #loop through all the values, drawing a line segment at each point
    for index in sortedIndicies.tolist()[0][::-1]:　　#改为从大到小
        if classLabels[index] == 1.0:   #标签为1修改真阳率
            delX = 0; delY = yStep;
        else:                           #标签不为1，修改假阳率
            delX = xStep; delY = 0;
            ySum += cur[1]              #保存纵坐标为之后计算面积
        #draw line from cur to (cur[0]-delX,cur[1]-delY)
        # ax.plot([cur[0],cur[0]-delX],[cur[1],cur[1]-delY], c=‘b‘)
        ax.plot([cur[0],cur[0]+delX],[cur[1],cur[1]+delY], c=‘b‘)
        # cur = (cur[0]-delX,cur[1]-delY)
        cur = (cur[0]+delX,cur[1]+delY)
    ax.plot([0,1],[0,1],‘b--‘)
    plt.xlabel(‘False positive rate‘); plt.ylabel(‘True positive rate‘)
    plt.title(‘ROC curve for AdaBoost horse colic detection system‘)
    ax.axis([0,1,0,1])
    plt.show()
    print("the Area Under the Curve is: ",ySum*xStep)   #AUC，小矩形面积之和


if __name__ == ‘__main__‘:
    datArr, labelArr = loadDataSet(‘horseColicTraining2.txt‘)
    classifierArr,aggClassEst = adaBoostTrainDS(datArr, labelArr, 50)
    #testArr, testLableArr = loadDataSet(‘horseColicTest2.txt‘)
    #prediction = adaClassify(testArr, classifierArr)
    #errArr = np.mat(np.ones((67,1)))
    #errRate = errArr[prediction!= np.mat(testLableArr).T].sum() / 67
    #print(errRate)
    plotROC(aggClassEst.T, labelArr)