正文共3862个字，4张图，预计阅读时间10分钟。

def batchnorm_backward(dout, cache):  
"""  
Backward pass for batch normalization.    
For this implementation, you should write out a computation graph for

batch normalization on paper and propagate gradients backward through  intermediate nodes.

Inputs:  
- dout: Upstream derivatives, of shape (N, D)  
- cache: Variable of intermediates from batchnorm_forward.

Returns a tuple of:  
- dx: Gradient with respect to inputs x, of shape (N, D)  
- dgamma: Gradient with respect to scale parameter gamma, of shape (D,)  
- dbeta: Gradient with respect to shift parameter beta, of shape (D,)  
"""  
dx, dgamma, dbeta = None, None, None

x, gamma, beta, var, miu, x_hat, eps = cache  
m = len(x)  
dx_hat = dout * gamma  
dvar = np.sum(dx_hat * (x-miu), axis=0) * -0.5 * (var + eps) ** (-1.5)  
dmiu = np.sum(dx_hat * (-1) / np.sqrt(var+eps), axis=0) + dvar * np.mean(-2 * (x - miu), axis=0)  
dx = dx_hat / np.sqrt(var + eps) + dvar * 2 * (x - miu) / m + dmiu / m  
dgamma = np.sum(dout * x_hat, axis=0)  
dbeta = np.sum(dout, axis=0)

return dx, dgamma, dbeta

def softmax_loss(x, y):  """  Computes the loss and gradient for softmax classification.  Inputs:  - x: Input data, of shape (N, C) where x[i, j] is the score for the jth class    for the ith input.  - y: Vector of labels, of shape (N,) where y[i] is the label for x[i] and    0 <= y[i] < C  Returns a tuple of:  - loss: Scalar giving the loss  - dx: Gradient of the loss with respect to x  """  probs = np.exp(x - np.max(x, axis=1, keepdims=True))  probs /= np.sum(probs, axis=1, keepdims=True)  N = x.shape[0]  loss = -np.sum(np.log(probs[np.arange(N), y])) / N  dx = probs.copy()  dx[np.arange(N), y] -= 1  dx /= N  return loss, dx

dx = probs.copy() dx[np.arange(N), y] -= 1

def relu_backward(dout, cache):  
"""  
Computes the backward pass for a layer of rectified linear units (ReLUs).  
Input:  
- dout: Upstream derivatives, of any shape  
- cache: Input x, of same shape as dout  Returns:  
- dx: Gradient with respect to x  
"""

dx, x = None, cache  dx = dout  dx[x <= 0] = 0

return dx