实验笔记之——covariance matrix pooling
2024-05-20 13:46:28
本博文为协方差池化的实验笔记。具体理论请参考博客《学习笔记之——covariance pooling》
实验python train.py -opt options/train/train_sr.json
先激活虚拟环境source activate pytorch
tensorboard --logdir tb_logger/ --port 6008
浏览器打开http://172.20.36.203:6008/#scalars
先给出代码
############################################################################
#high order
class M_RCAB(nn.Module):## Residual Channel Attention Block (RCAB)def __init__(self, nf, kernel_size=3, reduction=16, stride=1, dilation=1, groups=1, bias=True, \pad_type='zero', norm_type=None, act_type='relu', mode='CNA', res_scale=1):super(M_RCAB, self).__init__()self.res = sequential(conv_block(nf, nf, kernel_size, stride, dilation, groups, bias, pad_type, \norm_type, act_type, mode),conv_block(nf, nf, kernel_size, stride, dilation, groups, bias, pad_type, \norm_type, None, mode),M_CALayer(nf, reduction, stride, dilation, groups, bias, pad_type, norm_type, act_type, mode))self.res_scale = res_scaledef forward(self, x):res = self.res(x).mul(self.res_scale)return x + resclass M_CALayer(nn.Module):# Channel Attention (CA) Layerdef __init__(self, channel, reduction=16, stride=1, dilation=1, groups=1, \bias=True, pad_type='zero', norm_type=None, act_type='relu', mode='CNA'):super(M_CALayer, self).__init__()# feature channel downscale and upscale --> channel weightself.attention = sequential(conv_block(channel*channel, 4*channel, 1, stride, dilation, groups, bias, pad_type, \norm_type, act_type, mode),conv_block(4*channel, channel, 1, stride, dilation, groups, bias, pad_type, \norm_type, None, mode),nn.Sigmoid())def forward(self, x):features=x# covariance poolingbatchSize = features.data.shape[0]dim = features.data.shape[1]h = features.data.shape[2]w = features.data.shape[3]S = h*wfeatures = features.reshape(batchSize,dim,S)#channel wise, put the H,W togetherI_hat =(1./S)*torch.eye(S,S,device = features.device)+(-1./S/S)*torch.ones(S,S,device = features.device)I_hat = I_hat.view(1,S,S).repeat(batchSize,1,1).type(features.dtype)y = features.bmm(I_hat).bmm(features.transpose(1,2))#get the covariance map#reshape H=y.data.shape[1]W=y.data.shape[2]K=H*Wy=y.reshape(batchSize,K)y=y.reshape(batchSize,K,1,1)return x * self.attention(y)class M_ResidualGroupBlock(nn.Module):## Residual Group (RG)def __init__(self, nf, nb, kernel_size=3, reduction=16, stride=1, dilation=1, groups=1, bias=True, \pad_type='zero', norm_type=None, act_type='relu', mode='CNA', res_scale=1):super(M_ResidualGroupBlock, self).__init__()group = [M_RCAB(nf, kernel_size, reduction, stride, dilation, groups, bias, pad_type, \norm_type, act_type, mode, res_scale) for _ in range(nb)]conv = conv_block(nf, nf, kernel_size, stride, dilation, groups, bias, pad_type, \norm_type, None, mode)self.res = sequential(*group, conv)self.res_scale = res_scaledef forward(self, x):res = self.res(x).mul(self.res_scale)return x + res
理论部分
改进代码
############################################################################
#high order
class M_RCAB(nn.Module):## Residual Channel Attention Block (RCAB)def __init__(self, nf, kernel_size=3, reduction=16, stride=1, dilation=1, groups=1, bias=True, \pad_type='zero', norm_type=None, act_type='relu', mode='CNA', res_scale=1):super(M_RCAB, self).__init__()self.res = sequential(conv_block(nf, nf, kernel_size, stride, dilation, groups, bias, pad_type, \norm_type, act_type, mode),conv_block(nf, nf, kernel_size, stride, dilation, groups, bias, pad_type, \norm_type, None, mode),M_CALayer(nf, reduction, stride, dilation, groups, bias, pad_type, norm_type, act_type, mode))self.res_scale = res_scaledef forward(self, x):res = self.res(x).mul(self.res_scale)return x + resclass M_CALayer(nn.Module):# Channel Attention (CA) Layerdef __init__(self, channel, reduction=16, stride=1, dilation=1, groups=1, \bias=True, pad_type='zero', norm_type=None, act_type='relu', mode='CNA'):super(M_CALayer, self).__init__()# feature channel downscale and upscale --> channel weightself.covar_polling=MPNCOV(input_dim=channel)self.attention = sequential(nn.Conv2d(1, 4*channel, 32),conv_block(4*channel, channel, 1, stride, dilation, groups, bias, pad_type, \norm_type, act_type, mode),
# we perform row-wise convolution for the covariance matrix by regarding each row as a group in group convolution.
#Then we perform the second convolution and this time we use the sigmoid function as a nonlinear activationnn.Sigmoid())def forward(self, x):#Covariance poolingy=self.covar_polling(x)#reshapey=y.reshape(y.data.shape[0],1,y.data.shape[1],y.data.shape[2])return x * self.attention(y)class M_ResidualGroupBlock(nn.Module):## Residual Group (RG)def __init__(self, nf, nb, kernel_size=3, reduction=16, stride=1, dilation=1, groups=1, bias=True, \pad_type='zero', norm_type=None, act_type='relu', mode='CNA', res_scale=1):super(M_ResidualGroupBlock, self).__init__()group = [M_RCAB(nf, kernel_size, reduction, stride, dilation, groups, bias, pad_type, \norm_type, act_type, mode, res_scale) for _ in range(nb)]conv = conv_block(nf, nf, kernel_size, stride, dilation, groups, bias, pad_type, \norm_type, None, mode)self.res = sequential(*group, conv)self.res_scale = res_scaledef forward(self, x):res = self.res(x).mul(self.res_scale)return x + res##############################################################################################################
from torch.autograd import Functionclass MPNCOV(nn.Module):"""Matrix power normalized Covariance pooling (MPNCOV)implementation of fast MPN-COV (i.e.,iSQRT-COV)https://arxiv.org/abs/1712.01034Args:iterNum: #iteration of Newton-schulz methodis_sqrt: whether perform matrix square root or notis_vec: whether the output is a vector or notinput_dim: the #channel of input featuredimension_reduction: if None, it will not use 1x1 conv toreduce the #channel of feature.if 256 or others, the #channel of featurewill be reduced to 256 or others."""def __init__(self, iterNum=3, is_sqrt=True, is_vec=None, input_dim=2048, dimension_reduction=None):super(MPNCOV, self).__init__()self.iterNum=iterNumself.is_sqrt = is_sqrtself.is_vec = is_vecself.dr = dimension_reductionif self.dr is not None:self.conv_dr_block = nn.Sequential(nn.Conv2d(input_dim, self.dr, kernel_size=1, stride=1, bias=False),nn.BatchNorm2d(self.dr),nn.ReLU(inplace=True))output_dim = self.dr if self.dr else input_dimif self.is_vec:self.output_dim = int(output_dim*(output_dim+1)/2)else:self.output_dim = int(output_dim*output_dim)self._init_weight()def _init_weight(self):for m in self.modules():if isinstance(m, nn.Conv2d):nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')elif isinstance(m, nn.BatchNorm2d):nn.init.constant_(m.weight, 1)nn.init.constant_(m.bias, 0)def _cov_pool(self, x):return Covpool.apply(x)def _sqrtm(self, x):return Sqrtm.apply(x, self.iterNum)def _triuvec(self, x):return Triuvec.apply(x)def forward(self, x):if self.dr is not None:x = self.conv_dr_block(x)x = self._cov_pool(x)if self.is_sqrt:x = self._sqrtm(x)if self.is_vec:x = self._triuvec(x)return xclass Covpool(Function):@staticmethoddef forward(ctx, input):x = inputbatchSize = x.data.shape[0]dim = x.data.shape[1]h = x.data.shape[2]w = x.data.shape[3]M = h*wx = x.reshape(batchSize,dim,M)I_hat = (-1./M/M)*torch.ones(M,M,device = x.device) + (1./M)*torch.eye(M,M,device = x.device)I_hat = I_hat.view(1,M,M).repeat(batchSize,1,1).type(x.dtype)y = x.bmm(I_hat).bmm(x.transpose(1,2))ctx.save_for_backward(input,I_hat)return y@staticmethoddef backward(ctx, grad_output):input,I_hat = ctx.saved_tensorsx = inputbatchSize = x.data.shape[0]dim = x.data.shape[1]h = x.data.shape[2]w = x.data.shape[3]M = h*wx = x.reshape(batchSize,dim,M)grad_input = grad_output + grad_output.transpose(1,2)grad_input = grad_input.bmm(x).bmm(I_hat)grad_input = grad_input.reshape(batchSize,dim,h,w)return grad_inputclass Sqrtm(Function):@staticmethoddef forward(ctx, input, iterN):x = inputbatchSize = x.data.shape[0]dim = x.data.shape[1]dtype = x.dtypeI3 = 3.0*torch.eye(dim,dim,device = x.device).view(1, dim, dim).repeat(batchSize,1,1).type(dtype)normA = (1.0/3.0)*x.mul(I3).sum(dim=1).sum(dim=1)A = x.div(normA.view(batchSize,1,1).expand_as(x))Y = torch.zeros(batchSize, iterN, dim, dim, requires_grad = False, device = x.device).type(dtype)Z = torch.eye(dim,dim,device = x.device).view(1,dim,dim).repeat(batchSize,iterN,1,1).type(dtype)if iterN < 2:ZY = 0.5*(I3 - A)YZY = A.bmm(ZY)else:ZY = 0.5*(I3 - A)Y[:,0,:,:] = A.bmm(ZY)Z[:,0,:,:] = ZYfor i in range(1, iterN-1):ZY = 0.5*(I3 - Z[:,i-1,:,:].bmm(Y[:,i-1,:,:]))Y[:,i,:,:] = Y[:,i-1,:,:].bmm(ZY)Z[:,i,:,:] = ZY.bmm(Z[:,i-1,:,:])YZY = 0.5*Y[:,iterN-2,:,:].bmm(I3 - Z[:,iterN-2,:,:].bmm(Y[:,iterN-2,:,:]))y = YZY*torch.sqrt(normA).view(batchSize, 1, 1).expand_as(x)ctx.save_for_backward(input, A, YZY, normA, Y, Z)ctx.iterN = iterNreturn y@staticmethoddef backward(ctx, grad_output):input, A, ZY, normA, Y, Z = ctx.saved_tensorsiterN = ctx.iterNx = inputbatchSize = x.data.shape[0]dim = x.data.shape[1]dtype = x.dtypeder_postCom = grad_output*torch.sqrt(normA).view(batchSize, 1, 1).expand_as(x)der_postComAux = (grad_output*ZY).sum(dim=1).sum(dim=1).div(2*torch.sqrt(normA))I3 = 3.0*torch.eye(dim,dim,device = x.device).view(1, dim, dim).repeat(batchSize,1,1).type(dtype)if iterN < 2:der_NSiter = 0.5*(der_postCom.bmm(I3 - A) - A.bmm(der_postCom))else:dldY = 0.5*(der_postCom.bmm(I3 - Y[:,iterN-2,:,:].bmm(Z[:,iterN-2,:,:])) -Z[:,iterN-2,:,:].bmm(Y[:,iterN-2,:,:]).bmm(der_postCom))dldZ = -0.5*Y[:,iterN-2,:,:].bmm(der_postCom).bmm(Y[:,iterN-2,:,:])for i in range(iterN-3, -1, -1):YZ = I3 - Y[:,i,:,:].bmm(Z[:,i,:,:])ZY = Z[:,i,:,:].bmm(Y[:,i,:,:])dldY_ = 0.5*(dldY.bmm(YZ) -Z[:,i,:,:].bmm(dldZ).bmm(Z[:,i,:,:]) -ZY.bmm(dldY))dldZ_ = 0.5*(YZ.bmm(dldZ) -Y[:,i,:,:].bmm(dldY).bmm(Y[:,i,:,:]) -dldZ.bmm(ZY))dldY = dldY_dldZ = dldZ_der_NSiter = 0.5*(dldY.bmm(I3 - A) - dldZ - A.bmm(dldY))der_NSiter = der_NSiter.transpose(1, 2)grad_input = der_NSiter.div(normA.view(batchSize,1,1).expand_as(x))grad_aux = der_NSiter.mul(x).sum(dim=1).sum(dim=1)for i in range(batchSize):grad_input[i,:,:] += (der_postComAux[i] \- grad_aux[i] / (normA[i] * normA[i])) \*torch.ones(dim,device = x.device).diag().type(dtype)return grad_input, Noneclass Triuvec(Function):@staticmethoddef forward(ctx, input):x = inputbatchSize = x.data.shape[0]dim = x.data.shape[1]dtype = x.dtypex = x.reshape(batchSize, dim*dim)I = torch.ones(dim,dim).triu().reshape(dim*dim)index = I.nonzero()y = torch.zeros(batchSize,int(dim*(dim+1)/2),device = x.device).type(dtype)y = x[:,index]ctx.save_for_backward(input,index)return y@staticmethoddef backward(ctx, grad_output):input,index = ctx.saved_tensorsx = inputbatchSize = x.data.shape[0]dim = x.data.shape[1]dtype = x.dtypegrad_input = torch.zeros(batchSize,dim*dim,device = x.device,requires_grad=False).type(dtype)grad_input[:,index] = grad_outputgrad_input = grad_input.reshape(batchSize,dim,dim)return grad_inputdef CovpoolLayer(var):return Covpool.apply(var)def SqrtmLayer(var, iterN):return Sqrtm.apply(var, iterN)def TriuvecLayer(var):return Triuvec.apply(var)
改进2
class M_CALayer(nn.Module):# Channel Attention (CA) Layerdef __init__(self, channel, reduction=16, stride=1, dilation=1, groups=1, \bias=True, pad_type='zero', norm_type=None, act_type='relu', mode='CNA'):super(M_CALayer, self).__init__()# feature channel downscale and upscale --> channel weightself.covar_polling=MPNCOV(input_dim=channel)self.attention = sequential(nn.AdaptiveAvgPool2d(1),conv_block(1, channel // reduction, 1, stride, dilation, groups, bias, pad_type, \norm_type, act_type, mode),conv_block(channel // reduction, channel, 1, stride, dilation, groups, bias, pad_type, \norm_type, None, mode),nn.Sigmoid())# self.attention = sequential(
# nn.AdaptiveAvgPool2d(1),#output size
# nn.Conv2d(1, channel, 32),
# conv_block(channel, channel, 1, stride, dilation, groups, bias, pad_type, \
# norm_type, act_type, mode),
# # we perform row-wise convolution for the covariance matrix by regarding each row as a group in group convolution.
# #Then we perform the second convolution and this time we use the sigmoid function as a nonlinear activation
# nn.Sigmoid())def forward(self, x):#Covariance poolingy=self.covar_polling(x)#reshapey=y.reshape(y.data.shape[0],1,y.data.shape[1],y.data.shape[2])return x * self.attention(y)上面属于channel wise的，下面改为position-wise
实验
补充资料
（关于tensor.get_shape().as_list()）https://blog.csdn.net/gqixf/article/details/82769174
https://github.com/jiangtaoxie/fast-MPN-COV
关注github动态：
https://github.com/ZilinGao/Global-Second-order-Pooling-Convolutional-Networks
https://github.com/daitao/SAN
用一模一样的二阶结构，两篇CVPR
实验笔记之——covariance matrix pooling相关推荐
- 视频理解论文实验笔记2014-2022
视频理解论文实验笔记 看了李沐团队的视频,其中关于视频理解的串讲(上集 下集)讲的太好了,按照他的顺序看了这些论文,并做了重点针对实验部分的笔记 文章目录 视频理解论文实验笔记 2D Base cvp ...
- ROS实验笔记之——VINS-Mono在l515上的实现
之前博客<ROS实验笔记之--Intel Realsense l515激光相机的使用>实现了用l515运行RTABmap,本博文试试在l515上实现vins-mono 首先需要将vins- ...
- 实验笔记之——基于DWT的octave layer（DWT在pytorch中实现）
之前的博文<论文阅读笔记之--<Multi-level Wavelet-CNN for Image Restoration>及基于pytorch的复现>曾经研究过WMCNN.本 ...
- R语言使用psych包进行探索性因子分析EFA、使用cov2cor函数将原始数据的协方差矩阵将其转换为相关性矩阵（ covariance matrix into correlation matrix)
R语言使用psych包进行探索性因子分析EFA.使用cov2cor函数将原始数据的协方差矩阵将其转换为相关性矩阵( covariance matrix transform into correlati ...
- 深度学习入门教程UFLDL学习实验笔记三：主成分分析PCA与白化whitening
深度学习入门教程UFLDL学习实验笔记三:主成分分析PCA与白化whitening 主成分分析与白化是在做深度学习训练时最常见的两种预处理的方法,主成分分析是一种我们用的很多的降维的一种手段,通 ...
- 深度学习入门教程UFLDL学习实验笔记一：稀疏自编码器
深度学习入门教程UFLDL学习实验笔记一:稀疏自编码器 UFLDL即(unsupervised feature learning & deep learning).这是斯坦福网站上的一篇 ...
- 考oracle ocm,Oracle数据库OCM考试系列教程与总结_OCM考试实验笔记
oracle数据库OCM考试系列教程与总结_OCM考试实验笔记 以下包括OCM考试系列的文章,在以下各章节中基本的内容都已经涉及到,在这些文章中,将对之前的OCM考试系列文章进行汇总,对一些之前没提到 ...
- CS144 lab4 计算机网络实验 笔记
CS144 lab4 计算机网络实验 笔记 介绍 本实验中,我们将组合TCP sender和TCP receiver实现一个完整的TCP connection TCP是全双工连接,所以两方可以同时接收 ...
- 【Packet Tracer 实验笔记5】
[测试目的] 1.拓扑配置:交换机为各个VLAN提供统一的Trunk链路实现VLAN成员的跨交换机通信 2.实验目的:了解跨交换机VLAN的内部主机之间通信机制 3.测试方法:PC3 ----ping ...
最新文章
- 读后感与机翻《人类因果学习的分解:自下而上的联想学习和自上而下的图式推理》
- 【数道云大数据】大数据平台哪一个好用？武汉2019年大数据平台排行版？...
- 网页设计的12种颜色
- 数据挖掘-贝叶斯定理
- ASP.NET MVC5+EF6+EasyUI 后台管理系统（92）-打印EasyUI 的datagrid表格
- BAL数据集与BA优化
- 如何写博客(网摘)第一步:我是谁？
- ASP .NET基本概念
- 利用nginx重写url参数并跳转
- ORA-00911错误及解决方法（另附所有ora错误原因及解决方法 网址）
- js中子父级页面相互调用
- python做购物车代码大全-Python实现一个简单的购物车程序
- yolov5深度剖析+源码debug级讲解系列（二）backbone构建
- 数据库基础知识（1）－－数据库php连接
- linux pthread_cleanup_push 线程实现,线程清理（pthread_cleanup_push函数和pthread_cleanup_pop函数）...
- 一个微信公众号sdk(封装的比较全面)
- ‘字符型‘变量和‘字符串型‘变量
- 硬件知识：电源开关上的“1“和“0“分别是什么意思
- SkyWalking服务应用
- docker部署微服务项目