1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
| def plotBestFit(weakClassArr): #画出数据集和所有的基学习器
#import matplotlib.pyplot as plt
dataMat,labelMat=loadSimpData() #数据矩阵和标签向量
dataArr = array(dataMat) #转换成数组
n = shape(dataArr)[0]
xcord1 = []; ycord1 = [] #声明两个不同颜色的点的坐标
xcord2 = []; ycord2 = []
for i in range(n):
if int(labelMat[i])== 1:
xcord1.append(dataArr[i,0]); ycord1.append(dataArr[i,1])
else:
xcord2.append(dataArr[i,0]); ycord2.append(dataArr[i,1])
fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(xcord1, ycord1, s=30, c='red', marker='s')
ax.scatter(xcord2, ycord2, s=30, c='green')
print "weakClassArr[0]['thresh']",weakClassArr[0]['dim']
for j in range(len(weakClassArr)):
if(weakClassArr[j]['dim'] == 1):
x = arange(-0.0, 2.5, 0.1)
y = x*0+weakClassArr[j]['thresh']
ax.plot(x, y)
else:
y = array(arange(-0.0, 2.5, 0.1))
x = y*0+weakClassArr[j]['thresh']
ax.plot(x, y)
plt.xlabel('X1'); plt.ylabel('X2');
plt.show()
def stumpClassify(dataMatrix,dimen,threshVal,threshIneq): #通过阈值比较对数据进行分类
retArray = ones((shape(dataMatrix)[0],1)) #首先将返回的数组的全部元素设置为1
if threshIneq == 'lt':
retArray[dataMatrix[:,dimen] <= threshVal] = -1.0 #将满足<=不等式的元素设为-1
else:
retArray[dataMatrix[:,dimen] > threshVal] = -1.0 #将满足>不等式的元素设为-1
return retArray
def buildStump(dataArr,classLabels,D): #遍历stumpClassify()函数所有的可能输入值，并找到数据集上最佳的单层决策树
dataMatrix = mat(dataArr); labelMat = mat(classLabels).T
m,n = shape(dataMatrix) #m=5,n=2
numSteps = 10.0; bestStump = {}; bestClasEst = mat(zeros((m,1)))
minError = inf #初始误差总和，为无穷大
for i in range(n): #循环X和Y两个维度
rangeMin = dataMatrix[:,i].min(); rangeMax = dataMatrix[:,i].max(); #取得X和Y两个维度的最大值和最小值特征
stepSize = (rangeMax-rangeMin)/numSteps #步进长度
for j in range(-1,int(numSteps)+1): #从-1到10步进
for inequal in ['lt', 'gt']: #"lt"为满足<=不等式，"gt"为满足>不等式
threshVal = (rangeMin + float(j) * stepSize) #当前阈值
predictedVals = stumpClassify(dataMatrix,i,threshVal,inequal) #根据阈值和不等式，计算预测的分类
errArr = mat(ones((m,1)))
errArr[predictedVals == labelMat] = 0 #样本估计错误的标记为1
weightedError = D.T*errArr #通过权重和错误标记，计算泛化误差
#print "split: dim %d, thresh %.2f, thresh ineqal: %s, the weighted error is %.3f" % (i, threshVal, inequal, weightedError)
if weightedError < minError: #如果泛化误差是最小的
minError = weightedError
bestClasEst = predictedVals.copy() #保存最佳预测结果
bestStump['dim'] = i #保存维度、阈值、不等式符号
bestStump['thresh'] = threshVal
bestStump['ineq'] = inequal
return bestStump,minError,bestClasEst
def adaBoostTrainDS(dataArr,classLabels,numIt=40): #基于单层决策树的AdaBoost训练
weakClassArr = []
m = shape(dataArr)[0] #需要分类的数据量，m=5
D = mat(ones((m,1))/m) #D为权重向量，初始D1...D5的和等于1
aggClassEst = mat(zeros((m,1))) #基分类器的线性组合
for i in range(numIt):
#建立单层决策树，bestStump包括维度，不等式，阈值，error泛化误差，classEst是每个基分类器
bestStump,error,classEst = buildStump(dataArr,classLabels,D)
print "最佳决策树=",bestStump,"泛化误差=",error,"更新前的分类器预测结果=",classEst.T
#print "D:",D.T
alpha = float(0.5*log((1.0-error)/max(error,1e-16))) #根据泛化误差，计算基分类器的权重α值
bestStump['alpha'] = alpha #把权重α值添加到最佳决策树的列表中
print "最佳决策树=",bestStump
weakClassArr.append(bestStump) #保存单层最佳决策树参数到数组中
print "预测分类: ",classEst.T
expon = multiply(-1*alpha*mat(classLabels).T,classEst) #权重α×真实分类×预测分类，multiply为对应元素相乘，不是矩阵相乘
D = multiply(D,exp(expon)) #Calc New D for next iteration
D = D/D.sum() #更新D,D.sum()为规范化因子
#calc training error of all classifiers, if this is 0 quit for loop early (use break)
aggClassEst += alpha*classEst
print "更新后的分类器预测结果: ",aggClassEst.T
aggErrors = multiply(sign(aggClassEst) != mat(classLabels).T,ones((m,1))) #boolean值和1相乘
errorRate = aggErrors.sum()/m #计算错误率
print "total error: ",errorRate
if errorRate == 0.0: break
return weakClassArr,aggClassEst
def adaClassify(datToClass,classifierArr): #AdaBoost分类函数
dataMatrix = mat(datToClass) #输入[0,0]转换成[[0,0]]矩阵
m = shape(dataMatrix)[0]
aggClassEst = mat(zeros((m,1)))
for i in range(len(classifierArr)):
classEst = stumpClassify(dataMatrix,classifierArr[i]['dim'],\
classifierArr[i]['thresh'],\
classifierArr[i]['ineq']) #维度、阈值、不等式符号
aggClassEst += classifierArr[i]['alpha']*classEst #计算在每一个基分类器上的预测值的累加和
print "aggClassEst=",aggClassEst
return sign(aggClassEst)
|
，其中 
为基学习器，
为系数
，其中f(x)是正确的分类，等于-1或者1，H(x)是分类器的分类结果，等于-1或者1**
**
的偏导，得 
，并令其等于0，得
**
的定义为
，
是规范化因子
