ResNet网络总结

2015_ResNet_何凯明：

图：

网络描述：

ResNet的主要思想是在网络中增加了直连通道，即Highway Network的思想。此前的网络结构是性能输入做一个非线性变换，而Highway Network则允许保留之前网络层的一定比例的输出。

第二幅图中这两种结构分别针对ResNet34（左图）和ResNet50/101/152（右图），一般称整个结构为一个”building block“。其中右图又称为”bottleneck design”，目的一目了然，就是为了降低参数的数目，第一个1x1的卷积把256维channel降到64维，然后在最后通过1x1卷积恢复，整体上用的参数数目：1x1x256x64 + 3x3x64x64 + 1x1x64x256 = 69632，而不使用bottleneck的话就是两个3x3x256的卷积，参数数目: 3x3x256x256x2 = 1179648，差了16.94倍。对于常规ResNet，可以用于34层或者更少的网络中，对于Bottleneck Design的ResNet通常用于更深的如101这样的网络中，目的是减少计算和参数量（实用目的）

特点，优点：

（1）提出residual结构（残差结构），并搭建超深的网络结构（突破1000层）。

（2）使用batch normalization 加速训练（丢弃dropout）。

（3）学习结果对网络权重的波动变化更加敏感。

（4）相比传统的VGG网络，复杂度降低，所需的参数量下降。

（5）网络深度更深，不会出现梯度消失现象。

（6）解决了深层次的网络退化问题，在增加网络层数的过程中，training accuracy 逐渐趋于饱和，继续增加层数，training accuracy 就会出现下降的现象，而这种下降不是由过拟合造成的。

代码：

Pytorch1实现：

#nn.Sequential实现
class ResidualBlock(nn.Module):#实现子module：Residual Blockdef __init__(self, in_ch, out_ch, stride=1, shortcut=None):super(ResidualBlock,self).__init__()self.left = nn.Sequential(nn.Conv2d(in_ch,out_ch,3,stride,padding=1,bias=False),nn.BatchNorm2d(out_ch),nn.ReLU(inplace = True),#inplace = True原地操作nn.Conv2d(out_ch,out_ch,3,stride=1,padding=1,bias=False),nn.BatchNorm2d(out_ch))self.right = shortcutdef forward(self,x):out = self.left(x)residual = x if self.right is None else self.right(x)out += residualreturn F.relu(out)class ResNet34(nn.Module):#224x224x3#实现主module:ResNet34def __init__(self, num_classes=1):super(ResNet34,self).__init__()self.pre = nn.Sequential(nn.Conv2d(3,64,7,stride=2,padding=3,bias=False),# (224+2*p-)/2(向下取整)+1，size减半->112nn.BatchNorm2d(64),#112x112x64nn.ReLU(inplace = True),nn.MaxPool2d(3,2,1)#kernel_size=3, stride=2, padding=1)#56x56x64#重复的layer,分别有3,4,6,3个residual blockself.layer1 = self.make_layer(64,64,3)#56x56x64,layer1层输入输出一样，make_layer里，应该不用对shortcut进行处理，但是为了统一操作。。。self.layer2 = self.make_layer(64,128,4,stride=2)#第一个stride=2,剩下3个stride=1;28x28x128self.layer3 = self.make_layer(128,256,6,stride=2)#14x14x256self.layer4 = self.make_layer(256,512,3,stride=2)#7x7x512#分类用的全连接self.fc = nn.Linear(512,num_classes)def make_layer(self,in_ch,out_ch,block_num,stride=1):#当维度增加时，对shortcut进行option B的处理shortcut = nn.Sequential(#首个ResidualBlock需要进行option B处理nn.Conv2d(in_ch,out_ch,1,stride,bias=False),#1x1卷积用于增加维度；stride=2用于减半size；为简化不考虑偏差nn.BatchNorm2d(out_ch))layers = []layers.append(ResidualBlock(in_ch,out_ch,stride,shortcut))for i in range(1,block_num):layers.append(ResidualBlock(out_ch,out_ch))#后面的几个ResidualBlock,shortcut直接相加return nn.Sequential(*layers)def forward(self,x):    #224x224x3x = self.pre(x)     #56x56x64x = self.layer1(x)  #56x56x64x = self.layer2(x)  #28x28x128x = self.layer3(x)  #14x14x256x = self.layer4(x)  #7x7x512x = F.avg_pool2d(x,7)#1x1x512x = x.view(x.size(0),-1)#将输出拉伸为一行：1x512x = self.fc(x)    #1x1# nn.BCELoss:二分类用的交叉熵，用的时候需要在该层前面加上 Sigmoid 函数return nn.Sigmoid()(x)#1x1，将结果化为(0~1)之间

Pytorch实现：

#直接堆叠
class BasicBlock(nn.Module):expansion = 1def __init__(self, in_channel, out_channel, stride=1, downsample=None):super(BasicBlock, self).__init__()self.conv1 = nn.Conv2d(in_channels=in_channel, out_channels=out_channel,kernel_size=3, stride=stride, padding=1, bias=False)self.bn1 = nn.BatchNorm2d(out_channel)self.relu = nn.ReLU()self.conv2 = nn.Conv2d(in_channels=out_channel, out_channels=out_channel,kernel_size=3, stride=1, padding=1, bias=False)self.bn2 = nn.BatchNorm2d(out_channel)self.downsample = downsampledef forward(self, x):identity = xif self.downsample is not None:identity = self.downsample(x)out = self.conv1(x)out = self.bn1(out)out = self.relu(out)out = self.conv2(out)out = self.bn2(out)out += identityout = self.relu(out)return outclass Bottleneck(nn.Module):expansion = 4def __init__(self, in_channel, out_channel, stride=1, downsample=None):super(Bottleneck, self).__init__()self.conv1 = nn.Conv2d(in_channels=in_channel, out_channels=out_channel,kernel_size=1, stride=1, bias=False)  # squeeze channelsself.bn1 = nn.BatchNorm2d(out_channel)# -----------------------------------------self.conv2 = nn.Conv2d(in_channels=out_channel, out_channels=out_channel,kernel_size=3, stride=stride, bias=False, padding=1)self.bn2 = nn.BatchNorm2d(out_channel)# -----------------------------------------self.conv3 = nn.Conv2d(in_channels=out_channel, out_channels=out_channel*self.expansion,kernel_size=1, stride=1, bias=False)  # unsqueeze channelsself.bn3 = nn.BatchNorm2d(out_channel*self.expansion)self.relu = nn.ReLU(inplace=True)self.downsample = downsampledef forward(self, x):identity = xif self.downsample is not None:identity = self.downsample(x)out = self.conv1(x)out = self.bn1(out)out = self.relu(out)out = self.conv2(out)out = self.bn2(out)out = self.relu(out)out = self.conv3(out)out = self.bn3(out)out += identityout = self.relu(out)return outclass ResNet(nn.Module):def __init__(self, block, blocks_num, num_classes=1000, include_top=True):super(ResNet, self).__init__()self.include_top = include_topself.in_channel = 64self.conv1 = nn.Conv2d(3, self.in_channel, kernel_size=7, stride=2,padding=3, bias=False)self.bn1 = nn.BatchNorm2d(self.in_channel)self.relu = nn.ReLU(inplace=True)self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)self.layer1 = self._make_layer(block, 64, blocks_num[0])self.layer2 = self._make_layer(block, 128, blocks_num[1], stride=2)self.layer3 = self._make_layer(block, 256, blocks_num[2], stride=2)self.layer4 = self._make_layer(block, 512, blocks_num[3], stride=2)if self.include_top:self.avgpool = nn.AdaptiveAvgPool2d((1, 1))  # output size = (1, 1)self.fc = nn.Linear(512 * block.expansion, num_classes)for m in self.modules():if isinstance(m, nn.Conv2d):nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')def _make_layer(self, block, channel, block_num, stride=1):downsample = Noneif stride != 1 or self.in_channel != channel * block.expansion:downsample = nn.Sequential(nn.Conv2d(self.in_channel, channel * block.expansion, kernel_size=1, stride=stride, bias=False),nn.BatchNorm2d(channel * block.expansion))layers = []layers.append(block(self.in_channel, channel, downsample=downsample, stride=stride))self.in_channel = channel * block.expansionfor _ in range(1, block_num):layers.append(block(self.in_channel, channel))return nn.Sequential(*layers)def forward(self, x):x = self.conv1(x)x = self.bn1(x)x = self.relu(x)x = self.maxpool(x)x = self.layer1(x)x = self.layer2(x)x = self.layer3(x)x = self.layer4(x)if self.include_top:x = self.avgpool(x)x = torch.flatten(x, 1)x = self.fc(x)return xdef resnet34(num_classes=1000, include_top=True):return ResNet(BasicBlock, [3, 4, 6, 3], num_classes=num_classes, include_top=include_top)def resnet101(num_classes=1000, include_top=True):return ResNet(Bottleneck, [3, 4, 23, 3], num_classes=num_classes, include_top=include_top)