RNN求解过程推导与实现

RNN求解过程推导与实现

RNN

LSTM

BPTT

matlab code

opencv code

BPTT,Back Propagation Through Time.

首先来看看怎么处理RNN。

RNN展开网络如下图

RNN展开结构.jpg

RNN节点结构.jpg

现令第t时刻的输入表示为,隐层节点的输出为,输出层的预测值,输入到隐层的权重矩阵,隐层自循环的权重矩阵,隐层到输出层的权重矩阵,对应的偏执向量分别表示为,输入层的某一个节点使用i标识，如,类似的隐层和输出层某一节点表示为。这里我们仅以三层网络为例。

那么首先正向计算

其中分别表示激活前对应的加权和，表示激活函数。

然后看看误差如何传递。

假设真实的输出应该是,那么误差可以定义为,是训练样本的index。整个网络的误差

我们将RNN再放大一些，看看细节

RNN节点内部连接.jpg

令则

矩阵向量化表示

所以梯度为：

其中是点乘符号，即对应元素乘。

代码实现：

我们可以注意到在计算梯度时需要用到的之前计算过的量，即需要保存的量包括，前向计算时隐层节点和输出节点的输出值，以及由时刻累积的。

人人都能用Python写出LSTM-RNN的代码！[你的神经网络学习最佳起步]这篇文章里使用python实现了基本的RNN过程。代码功能是模拟二进制相加过程中的依次进位过程，代码很容易明白。

这里改写成matlab代码

1. function error = binaryRNN( ) 
   

   ---

2. largestNumber=256; 
   

   ---

3. T=8; 
   

   ---

4. dic=dec2bin(0:largestNumber-1)-'0';% 将uint8表示成二进制数组，这是一个查找表 
   

   ---

5. %% 初始化参数 
   

   ---

6. eta=0.1;% 学习步长 
   

   ---

7. inputDim=2;% 输入维度 
   

   ---

8. hiddenDim=16; %隐层节点个数 
   

   ---

9. outputDim=1; % 输出层节点个数 
   

   ---

10. ---

11. W=rand(hiddenDim,outputDim)*2-1;% (-1,1)参数矩阵 
    

    ---

12. U=rand(hiddenDim,hiddenDim)*2-1;% (-1,1)参数矩阵 
    

    ---

13. V=rand(inputDim,hiddenDim)*2-1; % (-1,1)参数矩阵 
    

    ---

14. ---

15. delta_W=zeros(hiddenDim,outputDim); % 时刻间中间变量 
    

    ---

16. delta_U=zeros(hiddenDim,hiddenDim); 
    

    ---

17. delta_V=zeros(inputDim,hiddenDim); 
    

    ---

18. error=0; 
    

    ---

19. for p=1:10000 
    

    ---

20. aInt=randi(largestNumber/2); 
    

    ---

21. bInt=randi(largestNumber/2); 
    

    ---

22. a=dic(aInt+1,:); 
    

    ---

23. b=dic(bInt+1,:); 
    

    ---

24. cInt=aInt+bInt; 
    

    ---

25. c=dic(cInt+1,:); 
    

    ---

26. y=zeros(1,T); 
    

    ---

27. ---

28. preh=zeros(1,hiddenDim); 
    

    ---

29. hDic=zeros(T,hiddenDim); 
    

    ---

30. %% 前向计算 
    

    ---

31. for t=T:-1:1 % 注意应该从最低位计算，也就是二进制数组最右端开始计算 
    

    ---

32. x=[a(t),b(t)];  
    

    ---

33. h=sigmoid(x*V+preh*U); 
    

    ---

34. y(t)=sigmoid(h*W);  
    

    ---

35. hDic(t,:)=h; 
    

    ---

36. preh=h; 
    

    ---

37. end 
    

    ---

38. ---

39. err=y-c; 
    

    ---

40. error=error+norm(err,2)/2; 
    

    ---

41. next_delta_h=zeros(1,hiddenDim); 
    

    ---

42. %% 反馈 
    

    ---

43. for t=1:T 
    

    ---

44. delta_y = err(t).*sigmoidOutput2d(y(t)); 
    

    ---

45. delta_h=(delta_y*W'+next_delta_h*U').*sigmoidOutput2d(hDic(t,:)); 
    

    ---

46. ---

47. delta_W=delta_W+hDic(t,:)'*delta_y; 
    

    ---

48. if t<T 
    

    ---

49. delta_U=delta_U+hDic(t+1,:)'*delta_h; 
    

    ---

50. end 
    

    ---

51. delta_V=delta_V+[a(t),b(t)]'*delta_h; 
    

    ---

52. next_delta_h=delta_h;  
    

    ---

53. end 
    

    ---

54. % 梯度下降  
    

    ---

55. W=W-eta*delta_W; 
    

    ---

56. U=U-eta*delta_U; 
    

    ---

57. V=V-eta*delta_V; 
    

    ---

58. ---

59. delta_W=zeros(hiddenDim,outputDim); 
    

    ---

60. delta_U=zeros(hiddenDim,hiddenDim); 
    

    ---

61. delta_V=zeros(inputDim,hiddenDim); 
    

    ---

62. ---

63. if mod(p,1000)==0 
    

    ---

64. fprintf('Samples:%d\n',p); 
    

    ---

65. fprintf('True:%d\n',cInt); 
    

    ---

66. fprintf('Predict:%d\n',bin2dec(int2str(round(y)))); 
    

    ---

67. fprintf('Error:%f\n',norm(err,2)/2); 
    

    ---

68. end 
    

    ---

69. end 
    

    ---

70. end 
    

    ---

71. ---

72. function sx=sigmoid(x) 
    

    ---

73. sx=1./(1+exp(-x)); 
    

    ---

74. end 
    

    ---

75. ---

76. function dx=sigmoidOutput2d(output) 
    

    ---

77. dx=output.*(1-output); 
    

    ---

78. end 
    

    ---

为了更深入理解RNN过程，这里我想用OpenCV和C++实现自己的RNN，简单的单隐层网络。同样类似的功能，模拟多个十进制数的加法进位过程。

1. # include "highgui.h" 
   

   ---

2. # include "cv.h" 
   

   ---

3. # include <iostream> 
   

   ---

4. #include "math.h" 
   

   ---

5. #include<cstdlib> 
   

   ---

6. using namespace std; 
   

   ---

7. ---

8. # define random(x) ((rand()*rand())%x) //生成0-x的随机数 
   

   ---

9. ---

10. void Display(CvMat* mat) 
    

    ---

11. { 
    

    ---

12. cout << setiosflags(ios::fixed); 
    

    ---

13. for (int i = 0; i < mat->rows; i++) 
    

    ---

14. { 
    

    ---

15. for (int j = 0; j < mat->cols; j++) 
    

    ---

16. cout << cvmGet(mat, i, j) << " "; 
    

    ---

17. cout << endl; 
    

    ---

18. } 
    

    ---

19. ---

20. } 
    

    ---

21. ---

22. ---

23. // sigmoid 函数 
    

    ---

24. float sigmoid(float x) 
    

    ---

25. { 
    

    ---

26. return 1 / (1 + exp(-x)); 
    

    ---

27. } 
    

    ---

28. CvMat* sigmoidM(CvMat* mat) 
    

    ---

29. { 
    

    ---

30. CvMat*mat2 = cvCloneMat(mat); 
    

    ---

31. ---

32. for (int i = 0; i < mat2->rows; i++) 
    

    ---

33. { 
    

    ---

34. for (int j = 0; j < mat2->cols; j++) 
    

    ---

35. cvmSet(mat2, i, j, sigmoid(cvmGet(mat, i, j))); 
    

    ---

36. } 
    

    ---

37. return mat2; 
    

    ---

38. } 
    

    ---

39. //sigmoid 函数的导数 
    

    ---

40. float diffSigmoid(float x) 
    

    ---

41. { 
    

    ---

42. //注意，这里的x已经是sigmoid的结果 
    

    ---

43. return x*(1 - x); 
    

    ---

44. } 
    

    ---

45. CvMat* diffSigmoidM(CvMat* mat) 
    

    ---

46. { 
    

    ---

47. CvMat* mat2 = cvCloneMat(mat); 
    

    ---

48. ---

49. for (int i = 0; i < mat2->rows; i++) 
    

    ---

50. { 
    

    ---

51. for (int j = 0; j < mat2->cols; j++) 
    

    ---

52. { 
    

    ---

53. float t = cvmGet(mat, i, j); 
    

    ---

54. cvmSet(mat2, i, j, t*(1 - t)); 
    

    ---

55. } 
    

    ---

56. ---

57. } 
    

    ---

58. return mat2; 
    

    ---

59. ---

60. } 
    

    ---

61. ---

62. ---

63. /**************随机生成inputdim个整数，并求和****************** 
    

    ---

64. * inputdim 整数的个数 
    

    ---

65. * MAX 整数的最大范围 
    

    ---

66. * Sample 存放整数 
    

    ---

67. * 返回 整数和 
    

    ---

68. **************************************************************/ 
    

    ---

69. int sample(int inputdim, CvMat* Sample,int MAX) 
    

    ---

70. { 
    

    ---

71. int sum = 0; 
    

    ---

72. for (int i = 0; i < inputdim; i++) 
    

    ---

73. { 
    

    ---

74. int t = random(MAX); 
    

    ---

75. cvmSet(Sample, 0, i, t); 
    

    ---

76. sum += cvmGet(Sample,0,i); 
    

    ---

77. } 
    

    ---

78. return sum; 
    

    ---

79. } 
    

    ---

80. /********将整数拆分成10以内的数，作为每个时刻的输入************* 
    

    ---

81. * Sample 存放的整数 大小 1*inputdim 
    

    ---

82. * 返回 拆分后的输入数据 大小 inputdim*9 
    

    ---

83. ****************************************************************/ 
    

    ---

84. CvMat* splitM( CvMat*Sample) 
    

    ---

85. { 
    

    ---

86. CvMat* mat = cvCreateMat(Sample->cols, 8, CV_32F); 
    

    ---

87. cvSetZero(mat); 
    

    ---

88. for (int i = 0; i < mat->rows; i++) 
    

    ---

89. { 
    

    ---

90. int x = cvmGet(Sample,0,i); 
    

    ---

91. for (int j = 0; j < 8; ++j) 
    

    ---

92. { 
    

    ---

93. cvmSet(mat,i,j, x % 10); 
    

    ---

94. x = x / 10; 
    

    ---

95. } 
    

    ---

96. } 
    

    ---

97. return mat; 
    

    ---

98. } 
    

    ---

99. ---

100. /***************将数字数组整合成一个整数****************************** 
     

     ---

101. *mat 数字数组，即每个元素是十以内的整数，大小1*9 
     

     ---

102. *返回 整合后的整数 
     

     ---

103. *********************************************************************/ 
     

     ---

104. int merge(CvMat* mat) 
     

     ---

105. { 
     

     ---

106. double d = 0; 
     

     ---

107. for (int i = mat->cols; i >0; i--) 
     

     ---

108. { 
     

     ---

109. d = 10 * d + round(10*(cvmGet(mat,0,i-1))); 
     

     ---

110. ---

111. } 
     

     ---

112. return int(d); 
     

     ---

113. } 
     

     ---

114. /*****************将输出的数值拆分************************************** 
     

     ---

115. * y 输出的数值 
     

     ---

116. * 返回 长度为9的数组,这里转换成了0，1之间的数 
     

     ---

117. ***********************************************************************/ 
     

     ---

118. CvMat* split(int y) 
     

     ---

119. { 
     

     ---

120. CvMat* mat = cvCreateMat(1, 8, CV_32F); 
     

     ---

121. for (int i = 0; i < 8; i++) 
     

     ---

122. { 
     

     ---

123. cvmSet(mat,0,i, (y % 10) / 10.0); 
     

     ---

124. y = y / 10; 
     

     ---

125. } 
     

     ---

126. return mat; 
     

     ---

127. ---

128. } 
     

     ---

129. ---

130. /**********************产生随机矩阵****************************** 
     

     ---

131. * rows, cols, 矩阵的规模 
     

     ---

132. * a, b, 区间 
     

     ---

133. * 返回 返回[a,b]之间的随机矩阵 
     

     ---

134. *****************************************************************/ 
     

     ---

135. CvMat*randM(int rows,int cols, float a,float b) 
     

     ---

136. { 
     

     ---

137. CvMat* mat = cvCreateMat(rows, cols, CV_32FC1); 
     

     ---

138. float* ptr; 
     

     ---

139. for (int i = 0; i < mat->rows; i++) 
     

     ---

140. { 
     

     ---

141. for (int j = 0; j < mat->cols; j++) 
     

     ---

142. { 
     

     ---

143. cvmSet(mat, i, j, random(1000) / 1000.0*(b - a) + a); 
     

     ---

144. } 
     

     ---

145. } 
     

     ---

146. return mat; 
     

     ---

147. } 
     

     ---

148. ---

149. int main() 
     

     ---

150. { 
     

     ---

151. srand(time(NULL)); 
     

     ---

152. //首先，先定义网络 
     

     ---

153. int inputdim = 2;//不超过10 
     

     ---

154. int hiddendim = 16; 
     

     ---

155. int outputdim = 1; 
     

     ---

156. float eta = 0.1; 
     

     ---

157. int MAX = 100000000;//令整数最多八位 
     

     ---

158. //初始化参数矩阵 
     

     ---

159. CvMat* V = randM(inputdim, hiddendim,-1,1); 
     

     ---

160. CvMat* U = randM(hiddendim, hiddendim, -1, 1); 
     

     ---

161. CvMat* W = randM(hiddendim, outputdim, -1, 1); 
     

     ---

162. CvMat* bh = randM(1, hiddendim, -1, 1); 
     

     ---

163. CvMat* by = randM(1, outputdim, -1, 1);//偏置 
     

     ---

164. ---

165. CvMat*Sample = cvCreateMat(1, inputdim, CV_32F); 
     

     ---

166. cvSetZero(Sample); 
     

     ---

167. CvMat* delta_V = cvCloneMat(V); 
     

     ---

168. CvMat* delta_U = cvCloneMat(U); 
     

     ---

169. CvMat* delta_W = cvCloneMat(W); 
     

     ---

170. CvMat* delta_by = cvCloneMat(by); 
     

     ---

171. CvMat* delta_bh = cvCloneMat(bh); 
     

     ---

172. ---

173. //开始训练，训练集大小10000 
     

     ---

174. for (int p = 0; p < 20000; p++) 
     

     ---

175. { 
     

     ---

176. int sum = sample(inputdim,Sample,MAX); 
     

     ---

177. CvMat* sampleM = splitM(Sample);//每一行对应着一个整数的拆分，个位在前 
     

     ---

178. CvMat* d = split(sum);//真实结果拆分，每位存放的是除以10后的小数 
     

     ---

179. //正向计算 
     

     ---

180. CvMat* pre_h = cvCreateMat(1, hiddendim, CV_32F); 
     

     ---

181. cvSetZero(pre_h);//初始化最开始的h_{-1} 
     

     ---

182. CvMat* y = cvCreateMat(1, 8, CV_32F); 
     

     ---

183. cvSetZero(y);//定义输出量 
     

     ---

184. CvMat* h = cvCreateMat(8, hiddendim, CV_32F);//每一行存储一个时刻的隐变量输出 
     

     ---

185. ---

186. CvMat* temp1 = cvCreateMat(1, hiddendim, CV_32F); 
     

     ---

187. CvMat* temp2 = cvCreateMat(1, outputdim, CV_32F); 
     

     ---

188. CvMat* xt = cvCreateMatHeader(inputdim, 1, CV_32S); 
     

     ---

189. for (int t = 0; t < 8; t++) 
     

     ---

190. { 
     

     ---

191. cvGetCol(sampleM, xt, t);//获取第t时刻输入值 
     

     ---

192. cvGEMM(xt, V, 1,bh, 1, temp1, CV_GEMM_A_T); 
     

     ---

193. cvGEMM(pre_h, U, 1, temp1, 1, pre_h);// t时刻隐层输出 
     

     ---

194. pre_h = sigmoidM(pre_h); 
     

     ---

195. ---

196. cvGEMM(pre_h, W, 1, by, 1, temp2); 
     

     ---

197. float yvalue = sigmoid(cvmGet(temp2, 0, 0)); 
     

     ---

198. cvmSet(y, 0, t, yvalue);//t时刻的输出 
     

     ---

199. ---

200. //保存隐层输出 
     

     ---

201. for (int j = 0; j < hiddendim; j++) 
     

     ---

202. { 
     

     ---

203. cvmSet(h, t, j, cvmGet(pre_h, 0, j)); 
     

     ---

204. } 
     

     ---

205. } 
     

     ---

206. cvReleaseMat(&temp1); 
     

     ---

207. cvReleaseMat(&temp2); 
     

     ---

208. ---

209. //观察代码 
     

     ---

210. int oy = merge(y); 
     

     ---

211. CvMat* temp = cvCreateMat(1, 8, CV_32F); 
     

     ---

212. cvSub(y, d, temp); 
     

     ---

213. double error = 0.5*cvDotProduct(temp, temp); 
     

     ---

214. if ((p+1)%1000==0) 
     

     ---

215. { 
     

     ---

216. cout << "************************第" << p + 1 << "个样本***********" << endl; 
     

     ---

217. cout << "真实值:" << sum%MAX << endl; 
     

     ---

218. cout << "预测值:" << oy << endl; 
     

     ---

219. cout << "误差：" << error << endl; 
     

     ---

220. } 
     

     ---

221. //反向传递误差 
     

     ---

222. cvSetZero(delta_V); 
     

     ---

223. cvSetZero(delta_U); 
     

     ---

224. cvSetZero(delta_W); 
     

     ---

225. cvSetZero(delta_bh); 
     

     ---

226. cvSetZero(delta_by); 
     

     ---

227. ---

228. CvMat* delta_h = cvCreateMat(1, hiddendim, CV_32F); 
     

     ---

229. cvSetZero(delta_h); 
     

     ---

230. CvMat* delta_y = cvCreateMat(1, outputdim, CV_32F); 
     

     ---

231. cvSetZero(delta_y); 
     

     ---

232. CvMat* next_delta_h = cvCreateMat(1, hiddendim, CV_32F); 
     

     ---

233. cvSetZero(next_delta_h); 
     

     ---

234. ---

235. for (int t = 7; t > 0; t--) 
     

     ---

236. { 
     

     ---

237. cvmSet(delta_y, 0, 0, (cvmGet(y, 0, t) - cvmGet(d, 0, t))*diffSigmoid(cvmGet(y, 0, t))); 
     

     ---

238. cvGEMM(delta_y, W, 1, delta_h, 0, delta_h, CV_GEMM_B_T); 
     

     ---

239. cvGEMM(next_delta_h, U, 1, delta_h, 1, delta_h, CV_GEMM_B_T); 
     

     ---

240. cvMul(delta_h, diffSigmoidM(cvGetRow(h, temp, t)), delta_h); 
     

     ---

241. //更新delta_y,delta_h 
     

     ---

242. cvGEMM(cvGetRow(h, temp, t), delta_y, 1, delta_W, 1, delta_W, CV_GEMM_A_T); 
     

     ---

243. if (t>0) 
     

     ---

244. cvGEMM(cvGetRow(h, temp, t - 1), delta_h, 1, delta_U, 1, delta_U, CV_GEMM_A_T); 
     

     ---

245. cvGetCol(sampleM, xt, t); 
     

     ---

246. cvGEMM(xt, delta_h, 1, delta_V, 1, delta_V); 
     

     ---

247. cvAddWeighted(delta_by, 1, delta_y, 1, 0, delta_by); 
     

     ---

248. cvAddWeighted(delta_bh, 1, delta_h, 1, 0, delta_bh); 
     

     ---

249. ---

250. cvAddWeighted(delta_h, 1, next_delta_h, 0, 0, next_delta_h); 
     

     ---

251. ---

252. } 
     

     ---

253. cvAddWeighted(W, 1, delta_W, -eta, 0, W); 
     

     ---

254. cvAddWeighted(V, 1, delta_V, -eta, 0, V); 
     

     ---

255. cvAddWeighted(U, 1, delta_U, -eta, 0, U); 
     

     ---

256. ---

257. cvAddWeighted(by, 1, delta_by, -eta, 0, by); 
     

     ---

258. cvAddWeighted(bh, 1, delta_bh, -eta, 0, bh); 
     

     ---

259. ---

260. cvReleaseMat(&sampleM); 
     

     ---

261. cvReleaseMat(&d); 
     

     ---

262. cvReleaseMat(&pre_h); 
     

     ---

263. cvReleaseMat(&y); 
     

     ---

264. cvReleaseMat(&h); 
     

     ---

265. cvReleaseMat(&delta_h); 
     

     ---

266. cvReleaseMat(&delta_y); 
     

     ---

267. } 
     

     ---

268. cvReleaseMat(&U); 
     

     ---

269. cvReleaseMat(&V); 
     

     ---

270. cvReleaseMat(&W); 
     

     ---

271. cvReleaseMat(&by); 
     

     ---

272. cvReleaseMat(&bh); 
     

     ---

273. cvReleaseMat(&Sample); 
     

     ---

274. cvReleaseMat(&delta_V); 
     

     ---

275. cvReleaseMat(&delta_U); 
     

     ---

276. cvReleaseMat(&delta_W); 
     

     ---

277. cvReleaseMat(&delta_by); 
     

     ---

278. cvReleaseMat(&delta_bh); 
     

     ---

279. system("PAUSE"); 
     

     ---

280. return 0; 
     

     ---

281. } 
     

     ---

下面是代码结果，并没有完全一致。分析下主要原因可能是由于输出层是(0,1)的小数，但我们希望得到的是[0,10]的整数，而通过round或者强制类型转换总会带来较大误差，所以会出现预测值和真实值差别很大，这时候其实比较值的差异意义不大，应该对比每一位上数字的差异。

1479024804302.jpg

再下面是3个输入，32个隐层节点的结果

1479024912622.jpg

PS. 作为opencv新手，觉得matlab半小时搞定的东西，opencv要捣鼓两个小时。。。