CS231N Assigenment1 two_layer_net笔记

two_layer_net.ipynb

之前对 x.reshape(x.shape[0], -1)语句的输出结果理解一直有误：

1 x = [[1,4,7,2],[2,5,7,4]]
2 x = np.array(x)
3 x0 = x.reshape(x.shape[0], -1)
4 x1 = x.reshape(x.shape[1], -1)
5 print(x0)
6 print(x1)

的输出实际为

[[1 4 7 2]
 [2 5 7 4]]
[[1 4]
 [7 2]
 [2 5]
 [7 4]]

Affine layer: forward

# Test the affine_forward function

num_inputs = 2
input_shape = (4, 5, 6)
output_dim = 3

input_size = num_inputs * np.prod(input_shape)
#print(np.prod(input_shape)) #120
weight_size = output_dim * np.prod(input_shape)

x = np.linspace(-0.1, 0.5, num=input_size).reshape(num_inputs, *input_shape)
#prin(t(*input_shape)# 4 5 6
print(np.shape(x))# (2, 4, 5, 6)
w = np.linspace(-0.2, 0.3, num=weight_size).reshape(np.prod(input_shape), output_dim)
b = np.linspace(-0.3, 0.1, num=output_dim)

out, _ = affine_forward(x, w, b)
correct_out = np.array([[ 1.49834967,  1.70660132,  1.91485297],
                        [ 3.25553199,  3.5141327,   3.77273342]])

# Compare your output with ours. The error should be around e-9 or less.
print('Testing affine_forward function:')
print('difference: ', rel_error(out, correct_out))

要补充的函数为

def affine_forward(x, w, b):

    out = None

    x_vector = x.reshape(x.shape[0], -1)
    out = x_vector.dot(w)
    out += b

    return out, cache

Affine layer: backward

# Test the affine_forward function

num_inputs = 2
input_shape = (4, 5, 6)
output_dim = 3

input_size = num_inputs * np.prod(input_shape)
#print(np.prod(input_shape)) #120
weight_size = output_dim * np.prod(input_shape)

x = np.linspace(-0.1, 0.5, num=input_size).reshape(num_inputs, *input_shape)
#print(*input_shape)# 4 5 6
print(np.shape(x))# (2, 4, 5, 6)
w = np.linspace(-0.2, 0.3, num=weight_size).reshape(np.prod(input_shape), output_dim)
b = np.linspace(-0.3, 0.1, num=output_dim)

out, _ = affine_forward(x, w, b)
correct_out = np.array([[ 1.49834967,  1.70660132,  1.91485297],
                        [ 3.25553199,  3.5141327,   3.77273342]])

# Compare your output with ours. The error should be around e-9 or less.
print('Testing affine_forward function:')
print('difference: ', rel_error(out, correct_out))

其中函数为

def affine_backward(dout, cache):

    x, w, b = cache
    dx, dw, db = None, None, None

    # shape ：x(10*2*3) w(6*5) b(5) out(10*5)
    dx = np.dot(dout, w.T)  # (N, M) dot (D, M).T -> (N, D): (10, 6)
    dx = dx.reshape(x.shape)  # 将 dx 调整为与 x 相同的形状: (10*2*3)

    dw = np.dot(x.reshape(x.shape[0], -1).T, dout)  # (D, N) dot (N, M) -> (D, M) 6*10 dot 10*5 = 6*5

    db = np.sum(dout, axis=0)  # 沿着样本维度求和，得到 (M,) 形状的梯度

    return dx, dw, db

ReLU activation

正向

out = np.maximum(x,0)

反向的话刚开始理解错了，写成了

 dx=np.maximum(dx,0)

显然是错误的，应当判断x是否小于0而不是dx

dx = np.copy(dout)
dx[x <= 0] = 0

即可。

Sandwich layers

看一下layer_utis.py，看起来就是affine和relu封装在了一起，误差也在e-11 e-12这样

Loss layers: SVM & Softmax

svm

def svm_loss(x, y):
    loss, dx = None, None

    num_train = x.shape[0]
    scores = x - np.max(x, axis=1, keepdims=True)
    correct_class_scores = scores[np.arange(num_train), y]
    margins = np.maximum(0, scores - correct_class_scores[:, np.newaxis] + 1)
    margins[np.arange(num_train), y] = 0
    loss = np.sum(margins) / num_train

    num_pos = np.sum(margins > 0, axis=1)
    dx = np.zeros_like(x)
    dx[margins > 0] = 1
    dx[np.arange(num_train), y] -= num_pos
    dx /= num_train

    return loss, dx

造了个3,4的看一下过程

x:
[[ 4.17943411e-04  1.39710028e-03 -1.78590431e-03]
 [-7.08827734e-04 -7.47253161e-05 -7.75016769e-04]
 [-1.49797903e-04  1.86172902e-03 -1.42552930e-03]
 [-3.76356699e-04 -3.42275390e-04  2.94907637e-04]]

y:
[2 1 1 0]

scores:
[[-0.00097916  0.         -0.003183  ]
 [-0.0006341   0.         -0.00070029]
 [-0.00201153  0.         -0.00328726]
 [-0.00067126 -0.00063718  0.        ]]

correct class scores:
[-0.003183    0.          0.         -0.00067126]

margins:
[[1.00220385 1.003183   1.        ]
 [0.9993659  1.         0.99929971]
 [0.99798847 1.         0.99671274]
 [1.         1.00003408 1.00067126]]

margins:
[[1.00220385 1.003183   0.        ]
 [0.9993659  0.         0.99929971]
 [0.99798847 0.         0.99671274]
 [0.         1.00003408 1.00067126]]

num_pos:
[2 2 2 2]
dx：

[[1. 1. 0.]
 [1. 0. 1.]
 [1. 0. 1.]
 [0. 1. 1.]]

dx:
[[ 1.  1. -2.]
 [ 1. -2.  1.]
 [ 1. -2.  1.]
 [-2.  1.  1.]]

softmax

dx去掉了x转置dot（dscore）

def softmax_loss(x, y):

    loss, dx = None, None
*START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****

    num_train = x.shape[0]
    scores = x - np.max(x, axis=1, keepdims=True)
    exp_scores = np.exp(scores)
    #correct_class_scores = scores[np.arange(num_train), y]

    probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)

    loss = np.sum(-np.log(probs[np.arange(num_train ), y])) / num_train 

    # Compute the gradient
    dscores = probs
    dscores[np.arange(num_train ), y] -= 1
    dscores /= num_train 

    dx = dscores

    return loss, dx

Two-layer network

这一部分就看一下fc_net.py里面TwoLayerNet的类

假设输入维度是D，隐藏（层）维度H，分类数为C

结构是affine - relu - affine - softmax，不包含梯度下降。

模型参数存在字典self.params中

from builtins import range
from builtins import object
import numpy as np

from ..layers import *
from ..layer_utils import *

class TwoLayerNet(object):

    def __init__(
        self,
        input_dim=3 * 32 * 32,
        hidden_dim=100,
        num_classes=10,
        weight_scale=1e-3,
        reg=0.0,
    ):

        self.params = {}
        self.reg = reg

        self.params['W1'] = weight_scale * np.random.randn(input_dim,hidden_dim)
        self.params['b1'] = np.zeros(hidden_dim)

        self.params['W2'] = weight_scale * np.random.randn(hidden_dim, num_classes)
        self.params['b2'] = np.zeros(num_classes)

    def loss(self, X, y=None):

        scores = None

        hidden_layer = np.maximum(0, np.dot(X, self.params['W1']) + self.params['b1'])  # ReLU activation
        scores = np.dot(hidden_layer, self.params['W2']) + self.params['b2']

        # If y is None then we are in test mode so just return scores
        if y is None:
            return scores

        loss, grads = 0, {}

        num_train = X.shape[0]
        scores -= np.max(scores, axis=1, keepdims=True)  # for numerical stability
        softmax_scores = np.exp(scores) / np.sum(np.exp(scores), axis=1, keepdims=True)
        correct_class_scores = softmax_scores[range(num_train), y]
        data_loss = -np.log(correct_class_scores).mean()
        reg_loss = 0.5 * self.reg * (np.sum(self.params['W1'] ** 2) + np.sum(self.params['W2'] ** 2))
        loss = data_loss + reg_loss

        # Backward pass
        dscores = softmax_scores.copy()
        dscores[range(num_train), y] -= 1
        dscores /= num_train

        grads['W2'] = np.dot(hidden_layer.T, dscores) + self.reg * self.params['W2']
        grads['b2'] = np.sum(dscores, axis=0)

        dhidden = np.dot(dscores, self.params['W2'].T)
        dhidden[hidden_layer <= 0] = 0  # backpropagate through ReLU

        grads['W1'] = np.dot(X.T, dhidden) + self.reg * self.params['W1']
        grads['b1'] = np.sum(dhidden, axis=0)

        return loss, grads