"""利用numpy实现一个两层的全连接网络网络结构是：input ->(w1) fc_h -> relu ->(w2) output数据是随机出的
"""
import numpy as np
#维度和大小参数定义
batch_size = 64
input_dim = 1000
output_dim = 10
hidden_dim = 100# 数据虚拟 (x,y)
# 每行是一条数据 输入是64*1000,1000表示有1000维度的特征 输出是64*100
# 训练完参数之后，若对一条数据forward,直接运用w1 w2参数即可
# 使用relu激活函数
x = np.random.randn(batch_size,input_dim)
y = np.random.randn(batch_size,output_dim)#定义要训练的参数 w1(1000*100) w2(100*10)
# 方便起见，不设bisa
w1 = np.random.randn(input_dim,hidden_dim)
w2 = np.random.randn(hidden_dim,output_dim)# lr
lr = 1e-06
#实现
for i in range(500):#迭代500次#前向传播h = x.dot(w1) #隐藏层h_relu = np.maximum(h,0) #relu激活函数y_hat = h_relu.dot(w2)#计算损失loss = np.square(y_hat - y).sum()#计算梯度y_hat_grad = 2.0*(y_hat-y)w2_grad = h_relu.T.dot(y_hat_grad)h_relu_grad = y_hat_grad.dot(w2.T)h_grad = h_relu_grad.copy()h_grad[h < 0] = 0w1_grad = x.T.dot(h_grad)#更新参数w1 = w1 - lr*w1_gradw2 = w2 - lr*w2_grad#print("epoch "+str(i)+" end......")
#print("参数w1:")
#print(w1)
#print("参数w1:")
#print(w2)

"""使用pytorch实现上面的二层神经网络
"""
# pytorch中
## 内积
# tensor.mm(tensor)
## 转置
# tensor.t()
## 乘方运算
# tensor.pow(n)
import torch
device = torch.device('cpu')
# device = torch.device('cuda') # Uncomment this to run on GPU# N is batch size; D_in is input dimension;
# H is hidden dimension; D_out is output dimension.
N, D_in, H, D_out = 64, 1000, 100, 10# Create random input and output data
x = torch.randn(N, D_in, device=device)
y = torch.randn(N, D_out, device=device)# Randomly initialize weights
w1 = torch.randn(D_in, H, device=device)
w2 = torch.randn(H, D_out, device=device)learning_rate = 1e-6
for t in range(500):# Forward pass: compute predicted yh = x.mm(w1)h_relu = h.clamp(min=0)y_pred = h_relu.mm(w2)# Compute and print loss; loss is a scalar, and is stored in a PyTorch Tensor# of shape (); we can get its value as a Python number with loss.item().loss = (y_pred - y).pow(2).sum()#print(t, loss.item())# Backprop to compute gradients of w1 and w2 with respect to lossgrad_y_pred = 2.0 * (y_pred - y)grad_w2 = h_relu.t().mm(grad_y_pred)grad_h_relu = grad_y_pred.mm(w2.t())grad_h = grad_h_relu.clone()grad_h[h < 0] = 0grad_w1 = x.t().mm(grad_h)# Update weights using gradient descentw1 -= learning_rate * grad_w1w2 -= learning_rate * grad_w2

"""使用pytorch的自动求导 重新实现
"""
import torchdevice = torch.device('cpu')
# device = torch.device('cuda') # Uncomment this to run on GPU# N is batch size; D_in is input dimension;
# H is hidden dimension; D_out is output dimension.
N, D_in, H, D_out = 64, 1000, 100, 10# Create random Tensors to hold input and outputs
x = torch.randn(N, D_in, device=device)
y = torch.randn(N, D_out, device=device)# Create random Tensors for weights; setting requires_grad=True means that we
# want to compute gradients for these Tensors during the backward pass.
w1 = torch.randn(D_in, H, device=device, requires_grad=True)
w2 = torch.randn(H, D_out, device=device, requires_grad=True)learning_rate = 1e-6
for t in range(500):# Forward pass: compute predicted y using operations on Tensors. Since w1 and# w2 have requires_grad=True, operations involving these Tensors will cause# PyTorch to build a computational graph, allowing automatic computation of# gradients. Since we are no longer implementing the backward pass by hand we# don't need to keep references to intermediate values.y_pred = x.mm(w1).clamp(min=0).mm(w2)# Compute and print loss. Loss is a Tensor of shape (), and loss.item()# is a Python number giving its value.loss = (y_pred - y).pow(2).sum()#print(t, loss.item())# Use autograd to compute the backward pass. This call will compute the# gradient of loss with respect to all Tensors with requires_grad=True.# After this call w1.grad and w2.grad will be Tensors holding the gradient# of the loss with respect to w1 and w2 respectively.loss.backward()# Update weights using gradient descent. For this step we just want to mutate# the values of w1 and w2 in-place; we don't want to build up a computational# graph for the update steps, so we use the torch.no_grad() context manager# to prevent PyTorch from building a computational graph for the updateswith torch.no_grad():w1 -= learning_rate * w1.gradw2 -= learning_rate * w2.grad# Manually zero the gradients after running the backward passw1.grad.zero_()w2.grad.zero_()

# 自己定义网络的一层实现# 定义自己Relu类，继承自Function函数
# 必须同时实现forward和backward
# 前向传播时，forward用的自己定义的这个
# 反向传播时，必然会再找自己实现的这个backward，如不写，则no implment errorclass MyReLU(torch.autograd.Function):"""We can implement our own custom autograd Functions by subclassingtorch.autograd.Function and implementing the forward and backward passeswhich operate on Tensors."""@staticmethoddef forward(ctx, x):"""In the forward pass we receive a context object and a Tensor containing theinput; we must return a Tensor containing the output, and we can use thecontext object to cache objects for use in the backward pass."""ctx.save_for_backward(x)return x.clamp(min=0)@staticmethoddef backward(ctx, grad_output):"""In the backward pass we receive the context object and a Tensor containingthe gradient of the loss with respect to the output produced during theforward pass. We can retrieve cached data from the context object, and mustcompute and return the gradient of the loss with respect to the input to theforward function."""x, = ctx.saved_tensorsgrad_x = grad_output.clone()grad_x[x < 0] = 0return grad_xdevice = torch.device('cpu')
# device = torch.device('cuda') # Uncomment this to run on GPU# N is batch size; D_in is input dimension;
# H is hidden dimension; D_out is output dimension.
N, D_in, H, D_out = 64, 1000, 100, 10# Create random Tensors to hold input and output
x = torch.randn(N, D_in, device=device)
y = torch.randn(N, D_out, device=device)# Create random Tensors for weights.
w1 = torch.randn(D_in, H, device=device, requires_grad=True)
w2 = torch.randn(H, D_out, device=device, requires_grad=True)learning_rate = 1e-6
for t in range(500):# Forward pass: compute predicted y using operations on Tensors; we call our# custom ReLU implementation using the MyReLU.apply functiony_pred = MyReLU.apply(x.mm(w1)).mm(w2)# Compute and print lossloss = (y_pred - y).pow(2).sum()#print(t, loss.item())# Use autograd to compute the backward pass.loss.backward()with torch.no_grad():# Update weights using gradient descentw1 -= learning_rate * w1.gradw2 -= learning_rate * w2.grad# Manually zero the gradients after running the backward passw1.grad.zero_()w2.grad.zero_()