Logistic回归python实现

2017-08-12

Logistic 回归，作为分类器：

分别用了梯度上升，牛顿法来最优化损失函数：

 # -*- coding: utf-8 -*-

 '''
 function: 实现Logistic回归，拟合直线，对数据进行分类；
             利用梯度上升，随机梯度上升，改进的随机梯度上升，牛顿法分别对损失函数优化；
             这里没有给出最后测试分类的函数；
 date: 2017.8.12
 '''

 from numpy import *

 #从文件加载处理数据
 def loadDataSet():
     dataMat = []
     labelMat = []
     fr = open('testSet.txt')
     for line in fr.readlines():
         lineArr = line.strip().split()
         dataMat.append([1.0, float(lineArr[0]), float(lineArr[1])])
         labelMat.append(int(lineArr[2]))
     return dataMat, labelMat

 #sigmoid function X: w1*x1+w2*x2+...+wn*xn
 def sigmoid(x):
     return 1 / (1 + exp(-x))

 #梯度上升求函数的最大值时取得的权重
 def gradAscent(dataMatIn, classLabels):
     dataMatrix = mat(dataMatIn)
     labelMat = mat(classLabels).transpose()   #将m维行向量转制为m维列向量
     m,n = shape(dataMatrix)
     alpha = 0.001 #设置梯度上升的步长
     maxCycles = 500 #最大迭代次数
     weights = ones((n,1))  #weights就是theta，n维列向量,二维数组
     for i in range(maxCycles):
         h = sigmoid(dataMatrix*weights)  #计算所有数据的分类概率，h是m维向量,这里实际上进行了300次乘法运算
         error = labelMat - h #计算(y-h(x))：误差
         weights = weights + alpha * dataMatrix.transpose()*error #对所有weights同时更新
     print('shape of weights', shape(weights))
     return weights

 #随机上升上升
 def stocGradAscent0(dataMatrix, classLabels):
     m,n = shape(dataMatrix)
     alpha = 0.01
     weights = ones(n, float) #n 维行向量
     for i in range(m):
         h = sigmoid(sum(dataMatrix[i] * weights))
         error = classLabels[i] - h
         weights = weights + alpha * error * dataMatrix[int(i)]
     return weights

 #改进的随机上升上升
 def stocGradAscent1(dataMatrix, classLabels, numIter = 550):
     m,n = shape(dataMatrix)
     weights = ones(n)
     dataIndex = []
     for j in range(numIter):
         for k in range(m):
             dataIndex.append(k)
         for i in range(m):
             alpha = 4 / (i + j + 1.0) + 0.01 #aloha随着迭代次数增多不断减小但是不为0，为了解决局部波动
             randIndex = int(random.uniform(0, len(dataIndex))) #随机选取一个数据进行更新参数
             h = sigmoid(dataMatrix[randIndex] * weights)
             error = classLabels[randIndex] - h
             weights = weights + alpha * error * dataMatrix[randIndex]
             del(dataIndex[randIndex]) #这里会不会对本来的dataMatrix有影响,不会，因为没有操作矩阵
     return weights

 '''
 functuion: 牛顿法更新theta,计算海森矩阵
 note: 这里一定要对 XT*X 结果求逆，不然分类效果无法直视
 '''
 def computeHessianMatrix(dataMatrix):
     hessianMatrix = mat(dataMatrix).transpose().dot(mat(dataMatrix))
     return hessianMatrix.I #矩阵求逆

 '''
 function: 牛顿法更新theta
 '''
 def newtonMethod(dataMat, labelMat, numIter = 10):
     m,n = shape(dataMat)
     dataMatrix = mat(dataMat)
     labelMat = mat(labelMat).transpose()
     weights = ones((n,1)) #参数向量
     hessianMatrix = computeHessianMatrix(dataMatrix)
     print('shape of hessian', shape(hessianMatrix))
     for k in range(numIter):
         h = sigmoid(dataMatrix * weights)
         error = (labelMat - h)
         weights = weights - (dataMatrix*hessianMatrix).transpose() * error
     return weights

 #画出决策边界
 def plotBestFit(weights):
     import matplotlib.pyplot as plt
     dataMat, labelMat = loadDataSet() #或取数据和标签
     dataArr = array(dataMat) #将二维列表转换为数组
     n = shape(dataArr)[0] #行数，代表数据的个数

     xcord1 = []; ycord1 = []
     xcord2 = []; ycord2 = []
     for i in range(n):
         if int(labelMat[i]) == 1:
             xcord1.append(dataArr[i,1]); ycord1.append(dataArr[i,2])
         else:
             xcord2.append(dataArr[i,1]); ycord2.append(dataArr[i,2])

     fig = plt.figure()
     ax = fig.add_subplot(111)
     ax.scatter(xcord1, ycord1, s = 30, c = 'red', marker = 's')
     ax.scatter(xcord2, ycord2, s = 30, c = 'green')
     x = arange(-3.0, 3.0, 0.1)
     print(len(x))
     print(len(weights))
     print(shape(weights))
     y = (-weights[0] -  weights[1]*x) / weights[2] #把x2堪称y,x0=1,所以就是x1,x2之间的函数
     print(len(y))
     ax.plot(x,y)
     plt.xlabel('X1'); plt.ylabel('X2')
     plt.show() #显示图像

 '''
 function: 对测试数据进行测试
 input: testDate 是一个向量，或者多个向量
 '''
 def logisticTest(weights, testData):
     m,n = shape(testData)
     typeLabel = []
     for i in range(m):
         result = sigmoid(sum(testData[i] * weights)) #得到一个册数数据的概率
         if result > 0.5:
             typeLabel.append(1)
         else:
             typeLabel.append(0)
     return typeLabel

 '''
 function: 计算分类的正确率
 input: calssLables 是测试数据的类别向量
 '''
 def getCorrectRate(classLabels, testLabel):
     correctRate = 0.0
     numOfRight = 0
     for i in range(len(testLabel)):
         if classLabels[i] == testLabel[i]:
             numOfRight += 1
     correctRate = numOfRight / float(len(testLabel))
     return correctRate

 #测试算法
 dataMat, labelMat = loadDataSet()
 print('shape of datamet, ', shape(dataMat))
 weights1 = gradAscent(dataMat, labelMat)
 weights2 = stocGradAscent0(array(dataMat), labelMat)
 weights3 = stocGradAscent1(dataMat, labelMat)

 weights4 = newtonMethod(dataMat, labelMat, 2000) #牛顿法

 #测试正确率
 typeLabel = logisticTest(weights4, [[1,1,0],[1,1,1]])
 print(typeLabel)
 correctRate = getCorrectRate([1,0],typeLabel)
 print('正确率： ' ,str(correctRate))

 #画图
 #plotBestFit(array(weights1)) #这里因为普通梯度上升返回的是一个矩阵，所以要转换为向量
 #plotBestFit(weights2)
 #plotBestFit(weights3)
 plotBestFit(array(weights4))

# -*- coding: utf-8 -*-
'''function: 实现Logistic回归，拟合直线，对数据进行分类；利用梯度上升，随机梯度上升，改进的随机梯度上升，牛顿法分别对损失函数优化；这里没有给出最后测试分类的函数；date: 2017.8.12'''
from numpy import *
#从文件加载处理数据def loadDataSet():dataMat = []labelMat = []fr = open('testSet.txt')for line in fr.readlines():lineArr = line.strip().split()dataMat.append([1.0, float(lineArr[0]), float(lineArr[1])])labelMat.append(int(lineArr[2]))return dataMat, labelMat
#sigmoid function X: w1*x1+w2*x2+...+wn*xndef sigmoid(x):return 1 / (1 + exp(-x))
#梯度上升求函数的最大值时取得的权重def gradAscent(dataMatIn, classLabels):dataMatrix = mat(dataMatIn)labelMat = mat(classLabels).transpose()   #将m维行向量转制为m维列向量m,n = shape(dataMatrix)alpha = 0.001 #设置梯度上升的步长maxCycles = 500 #最大迭代次数 weights = ones((n,1))  #weights就是theta，n维列向量,二维数组for i in range(maxCycles):h = sigmoid(dataMatrix*weights)  #计算所有数据的分类概率，h是m维向量,这里实际上进行了300次乘法运算error = labelMat - h #计算(y-h(x))：误差weights = weights + alpha * dataMatrix.transpose()*error #对所有weights同时更新print('shape of weights', shape(weights))return weights
#随机上升上升def stocGradAscent0(dataMatrix, classLabels):m,n = shape(dataMatrix)alpha = 0.01weights = ones(n, float) #n 维行向量for i in range(m):h = sigmoid(sum(dataMatrix[i] * weights))error = classLabels[i] - hweights = weights + alpha * error * dataMatrix[int(i)]return weights
#改进的随机上升上升def stocGradAscent1(dataMatrix, classLabels, numIter = 550):m,n = shape(dataMatrix)weights = ones(n)dataIndex = []for j in range(numIter):for k in range(m):dataIndex.append(k)for i in range(m):alpha = 4 / (i + j + 1.0) + 0.01 #aloha随着迭代次数增多不断减小但是不为0，为了解决局部波动randIndex = int(random.uniform(0, len(dataIndex))) #随机选取一个数据进行更新参数h = sigmoid(dataMatrix[randIndex] * weights)error = classLabels[randIndex] - hweights = weights + alpha * error * dataMatrix[randIndex]del(dataIndex[randIndex]) #这里会不会对本来的dataMatrix有影响,不会，因为没有操作矩阵return weights
'''functuion: 牛顿法更新theta,计算海森矩阵note: 这里一定要对 XT*X 结果求逆，不然分类效果无法直视'''def computeHessianMatrix(dataMatrix):hessianMatrix = mat(dataMatrix).transpose().dot(mat(dataMatrix))return hessianMatrix.I #矩阵求逆
'''function: 牛顿法更新theta'''def newtonMethod(dataMat, labelMat, numIter = 10):m,n = shape(dataMat)dataMatrix = mat(dataMat)labelMat = mat(labelMat).transpose()weights = ones((n,1)) #参数向量hessianMatrix = computeHessianMatrix(dataMatrix)print('shape of hessian', shape(hessianMatrix))for k in range(numIter):h = sigmoid(dataMatrix * weights)error = (labelMat - h)weights = weights - (dataMatrix*hessianMatrix).transpose() * error return weights
#画出决策边界def plotBestFit(weights):import matplotlib.pyplot as pltdataMat, labelMat = loadDataSet() #或取数据和标签dataArr = array(dataMat) #将二维列表转换为数组n = shape(dataArr)[0] #行数，代表数据的个数
xcord1 = []; ycord1 = []xcord2 = []; ycord2 = []for i in range(n):if int(labelMat[i]) == 1:xcord1.append(dataArr[i,1]); ycord1.append(dataArr[i,2])else:xcord2.append(dataArr[i,1]); ycord2.append(dataArr[i,2])
fig = plt.figure()ax = fig.add_subplot(111)ax.scatter(xcord1, ycord1, s = 30, c = 'red', marker = 's')ax.scatter(xcord2, ycord2, s = 30, c = 'green')x = arange(-3.0, 3.0, 0.1)print(len(x))print(len(weights))print(shape(weights))y = (-weights[0] -  weights[1]*x) / weights[2] #把x2堪称y,x0=1,所以就是x1,x2之间的函数print(len(y))ax.plot(x,y) plt.xlabel('X1'); plt.ylabel('X2')plt.show() #显示图像
'''function: 对测试数据进行测试input: testDate 是一个向量，或者多个向量'''def logisticTest(weights, testData):m,n = shape(testData)typeLabel = []for i in range(m): result = sigmoid(sum(testData[i] * weights)) #得到一个册数数据的概率if result > 0.5:typeLabel.append(1)else:typeLabel.append(0)return typeLabel
'''function: 计算分类的正确率input: calssLables 是测试数据的类别向量'''def getCorrectRate(classLabels, testLabel):correctRate = 0.0numOfRight = 0for i in range(len(testLabel)):if classLabels[i] == testLabel[i]:numOfRight += 1correctRate = numOfRight / float(len(testLabel))return correctRate
#测试算法dataMat, labelMat = loadDataSet()print('shape of datamet, ', shape(dataMat))weights1 = gradAscent(dataMat, labelMat)weights2 = stocGradAscent0(array(dataMat), labelMat)weights3 = stocGradAscent1(dataMat, labelMat)
weights4 = newtonMethod(dataMat, labelMat, 2000) #牛顿法

#测试正确率typeLabel = logisticTest(weights4, [[1,1,0],[1,1,1]])print(typeLabel)correctRate = getCorrectRate([1,0],typeLabel)print('正确率： ' ,str(correctRate))
#画图#plotBestFit(array(weights1)) #这里因为普通梯度上升返回的是一个矩阵，所以要转换为向量#plotBestFit(weights2)#plotBestFit(weights3)plotBestFit(array(weights4))