模型搭建练习1_用numpy和tensor、variable实现前后向传播、实现激活函数

用numpy实现搭建一个简单的forward和backward

 import numpy as np
 N, D_in, H, D_out = 64, 1000, 100, 10
 x = np.random.randn(N, D_in)      # (64, 1000)
 y = np.random.randn(N, D_out)     # (64, 10)
 w1 = np.random.randn(D_in, H)     # (1000, 100)
 w2 = np.random.randn(H, D_out)    # (100, 10)
 learning_rate = 1e-6

 for t in range(2):
     # Forward pass: compute predicted y
     h = x.dot(w1)                  # (64, 100)
     h_relu = np.maximum(h, 0)      # (64, 100) 实现relu函数功能
     y_pred = h_relu.dot(w2)        # (64, 10)

     loss = np.square(y_pred - y).sum()      # sum()所有元素求和
     # Backprop to compute gradients of w1 and w2 with respect to loss
     grad_y_pred = 2.0 * (y_pred - y)
     grad_w2 = h_relu.T.dot(grad_y_pred)
     grad_h_relu = grad_y_pred.dot(w2.T)
     grad_h = grad_h_relu.copy()             # (64, 100)
     grad_h[h < 0] = 0               # 在h中负元素对应位置处grad_h中置0 -> 实现relu函数功能
     grad_w1 = x.T.dot(grad_h)       # .T是转置 (1000, 100)

     # Update weights
     w1 -= learning_rate * grad_w1           # (1000, 100)
     w2 -= learning_rate * grad_w2

用tensor实现搭建一个简单的forward和backward

 import torch

 dtype = torch.FloatTensor
 # dtype = torch.cuda.FloatTensor

 # N is batch size; D_in is input dimension;
 # H is hidden dimension; D_out is output dimension.
 N, D_in, H, D_out = 64, 1000, 100, 10

 x = torch.randn(N, D_in).type(dtype)
 y = torch.randn(N, D_out).type(dtype)

 # Randomly initialize weights
 w1 = torch.randn(D_in, H).type(dtype)
 w2 = torch.randn(H, D_out).type(dtype)

 learning_rate = 1e-6
 for t in range(500):
     # Forward pass: compute predicted y
     h = x.mm(w1)        # 与numpy对比，dot点乘
     h_relu = h.clamp(min=0)
     y_pred = h_relu.mm(w2)

     loss = (y_pred - y).pow(2).sum()
     # Backprop to compute gradients of w1 and w2 with respect to loss
     grad_y_pred = 2.0 * (y_pred - y)
     grad_w2 = h_relu.t().mm(grad_y_pred)
     grad_h_relu = grad_y_pred.mm(w2.t())
     grad_h = grad_h_relu.clone()
     grad_h[h < 0] = 0
     grad_w1 = x.t().mm(grad_h)

     # Update weights using gradient descent
     w1 -= learning_rate * grad_w1
     w2 -= learning_rate * grad_w2

用variable实现forward和backward

 # use PyTorch Variables and autograd to implement our two-layer network; 
# now we no longer need to manually implement the backward pass through the network

 import torch
 from torch.autograd import Variable

 dtype = torch.FloatTensor
 N, D_in, H, D_out = 64, 1000, 100, 10

 # Setting requires_grad=False indicates that we do not need to compute gradients with respect to these Variables during the backward pass.
 x = Variable(torch.randn(N, D_in).type(dtype), requires_grad=False)
 y = Variable(torch.randn(N, D_out).type(dtype), requires_grad=False)

 # Setting requires_grad=True indicates that we want to compute gradients with respect to these Variables during the backward pass.
 w1 = Variable(torch.randn(D_in, H).type(dtype), requires_grad=True)
 w2 = Variable(torch.randn(H, D_out).type(dtype), requires_grad=True)

 learning_rate = 1e-6
 for t in range(2):
     # Forward pass: we do not need to keep references to intermediate values since we are not implementing the backward pass by hand
     y_pred = x.mm(w1).clamp(min=0).mm(w2)

     # Now loss is a Variable of shape (1,) and loss.data is a Tensor of shape (1,); loss.data[0] is a scalar value holding the loss.
     loss = (y_pred - y).pow(2).sum()
     # print(loss)                 # [torch.FloatTensor of size 1]
     # print(loss.size())          # torch.Size([1])
     # print(loss.data)            # [torch.FloatTensor of size 1]
     print(loss.data[0])

     loss.backward()

     w1.data -= learning_rate * w1.grad.data
     w2.data -= learning_rate * w2.grad.data

     w1.grad.data.zero_()
     w2.grad.data.zero_()

用variable实现relu函数

 import torch
 from torch.autograd import Variable

 class MyReLU(torch.autograd.Function):
     def forward(self, input):
         self.save_for_backward(input)
         return input.clamp(min=0)

     def backward(self, grad_output):
         input, = self.saved_tensors
         grad_input = grad_output.clone()
         grad_input[input < 0] = 0
         return grad_input

 dtype = torch.FloatTensor
 N, D_in, H, D_out = 64, 1000, 100, 10

 x = Variable(torch.randn(N, D_in).type(dtype), requires_grad=False)
 y = Variable(torch.randn(N, D_out).type(dtype), requires_grad=False)

 w1 = Variable(torch.randn(D_in, H).type(dtype), requires_grad=True)
 w2 = Variable(torch.randn(H, D_out).type(dtype), requires_grad=True)

 learning_rate = 1e-6
 for t in range(2):
     relu = MyReLU()

     # Forward pass
     y_pred = relu(x.mm(w1)).mm(w2)

     loss = (y_pred - y).pow(2).sum()
     loss.backward()

     w1.data -= learning_rate * w1.grad.data
     w2.data -= learning_rate * w2.grad.data

     w1.grad.data.zero_()
     w2.grad.data.zero_()