目标检测算法中ROI提取方法比较+源码分析

本文主要介绍ROI提取结构在目标检测框架中的作用，并结合源码，理解它的实现方式。包含的算法有：ROI-pooling，ROI-align，Deformable-psroi-pooling。
目前，主流的目标检测算法大致分为2种，one-stage和two-stage方法。

one-stage：典型代表为SSD，相当于two-stage中的rpn结构，先通过基本的特征提取网络如resnet或vggnet得到特征图，再通过5层的卷积得到目标位置和类别。这种方法计算速度快，但是精度较two-stage方法差一些
two-stage:典型代表为Faster-rcnn。其结构分为RPN(Region Proposal Network)和RCNN(Region Convolution Neural Network)两个部分。RPN的特征通过ROI-Pooling层传递到RCNN中。

本文介绍的方法，仅出现在two-stage的方法中。顾名思义，该层的作用就是将RPN中提取的位置，截取特征图中特征用于进一步的分类和定位。

ROI-Pooling

Roi-pooling是Faster-rcnn原版使用的特征提取方式，论文在此。这里用动图来说明roi-pooling的过程(动图来源)

从上图看出，ROI-Pooling层的输入有两个：RPN层得到的位置和特征提取网络得到的特征。参数有pooling结果的宽高。
好了，下面结合图片来理解ROI-pooling代码，代码来源是pytorch-fasterrcnn项目，由于是cuda代码，所以如果你看了另外支持自定义operation的框架（如caffe，mxnet等）的roipooling实现方式，就会发现它们是完全一致的。

代码的注释中，增加了上图实例中各个变量的实际值，以便于读者理解。

_global__ void ROIPoolForward(const int nthreads, const float* bottom_data,const float spatial_scale, const int height, const int width,const int channels, const int pooled_height, const int pooled_width,const float* bottom_rois, float* top_data, int* argmax_data)
{CUDA_KERNEL_LOOP(index, nthreads){//index是gpu并行时块的计数int pw = index % pooled_width;//pooled_width=2，用户设置的参数，控制pooling输出大小int ph = (index / pooled_width) % pooled_height;//pooled_height=2，用户设置的参数，控制pooling输出大小int c  = (index / pooled_width / pooled_height) % channels;int n  = index / pooled_width / pooled_height / channels;// bottom_rois += n * 5;int roi_batch_ind = bottom_rois[n * 5 + 0];int roi_start_w = round(bottom_rois[n * 5 + 1] * spatial_scale);//rsw=0,左上角点横坐标，来自RPNint roi_start_h = round(bottom_rois[n * 5 + 2] * spatial_scale);//rsh=3,左上角点纵坐标，来自RPNint roi_end_w = round(bottom_rois[n * 5 + 3] * spatial_scale);//rew=7,右下角点横坐标，来自RPNint roi_end_h = round(bottom_rois[n * 5 + 4] * spatial_scale);//reh=8,右下角点纵坐标，来自RPN// Force malformed ROIs to be 1x1int roi_width = fmaxf(roi_end_w - roi_start_w + 1, 1);int roi_height = fmaxf(roi_end_h - roi_start_h + 1, 1);float bin_size_h = (float)(roi_height) / (float)(pooled_height);float bin_size_w = (float)(roi_width) / (float)(pooled_width);int hstart = (int)(floor((float)(ph) * bin_size_h));int wstart = (int)(floor((float)(pw) * bin_size_w));int hend = (int)(ceil((float)(ph + 1) * bin_size_h));int wend = (int)(ceil((float)(pw + 1) * bin_size_w));// Add roi offsets and clip to input boundarieshstart = fminf(fmaxf(hstart + roi_start_h, 0), height);hend = fminf(fmaxf(hend + roi_start_h, 0), height);wstart = fminf(fmaxf(wstart + roi_start_w, 0), width);wend = fminf(fmaxf(wend + roi_start_w, 0), width);bool is_empty = (hend <= hstart) || (wend <= wstart);//当roi_width<pooled_width或roi_height<pooled_height时触发，此时bin_size<1// Define an empty pooling region to be zerofloat maxval = is_empty ? 0 : -FLT_MAX;// If nothing is pooled, argmax = -1 causes nothing to be backprop'dint maxidx = -1;int bottom_data_batch_offset = roi_batch_ind * channels * height * width;int bottom_data_offset = bottom_data_batch_offset + c * height * width;//max-pooling操作，不同的index对应的hstart,wstart,hend,wend不同for (int h = hstart; h < hend; ++h) {for (int w = wstart; w < wend; ++w) {int bottom_index = h * width + w;if (bottom_data[bottom_data_offset + bottom_index] > maxval) {maxval = bottom_data[bottom_data_offset + bottom_index];maxidx = bottom_data_offset + bottom_index;}}}top_data[index] = maxval;if (argmax_data != NULL)argmax_data[index] = maxidx;}
}

ROI-align

从上面ROI-pooling的实现过程不难看出，由于取整的影响，各个index方块中对应的宽高是不同的，有些是2，有些是3。而ROI-align做了一个小改动，使h, w可以是小数，并通过双线性内插取得各个像素值，消除了取整带来的误差。源码位置。
此时，特征图上的pooling方框变成了下面这样

    __global__ void ROIAlignForward(const int nthreads, const float* bottom_data, const float spatial_scale, const int height, const int width,const int channels, const int aligned_height, const int aligned_width, const float* bottom_rois, float* top_data) {CUDA_1D_KERNEL_LOOP(index, nthreads) {int pw = index % aligned_width;int ph = (index / aligned_width) % aligned_height;int c  = (index / aligned_width / aligned_height) % channels;int n  = index / aligned_width / aligned_height / channels;// bottom_rois += n * 5;float roi_batch_ind = bottom_rois[n * 5 + 0];float roi_start_w = bottom_rois[n * 5 + 1] * spatial_scale;float roi_start_h = bottom_rois[n * 5 + 2] * spatial_scale;float roi_end_w = bottom_rois[n * 5 + 3] * spatial_scale;float roi_end_h = bottom_rois[n * 5 + 4] * spatial_scale;// Force malformed ROIs to be 1x1float roi_width = fmaxf(roi_end_w - roi_start_w + 1., 0.);float roi_height = fmaxf(roi_end_h - roi_start_h + 1., 0.);float bin_size_h = roi_height / (aligned_height - 1.);float bin_size_w = roi_width / (aligned_width - 1.);//注意，此处的h,w变成了float型，避免取整带来的误差float h = (float)(ph) * bin_size_h + roi_start_h;float w = (float)(pw) * bin_size_w + roi_start_w;//保留了整数的hstart,hstart，便于下面计算取整带来的位置偏移到底是多少int hstart = fminf(floor(h), height - 2);int wstart = fminf(floor(w), width - 2);int img_start = roi_batch_ind * channels * height * width;// bilinear interpolationif (h < 0 || h >= height || w < 0 || w >= width) {top_data[index] = 0.;} else {//计算位置偏移，h是float型，hstart是int型float h_ratio = h - (float)(hstart);float w_ratio = w - (float)(wstart);int upleft = img_start + (c * height + hstart) * width + wstart;int upright = upleft + 1;int downleft = upleft + width;int downright = downleft + 1;//双线性内插top_data[index] = bottom_data[upleft] * (1. - h_ratio) * (1. - w_ratio)+ bottom_data[upright] * (1. - h_ratio) * w_ratio+ bottom_data[downleft] * h_ratio * (1. - w_ratio)+ bottom_data[downright] * h_ratio * w_ratio;}}}

有些细心的同学可能发现了上面的代码没有进行pooling操作，只有内插。那是因为caffe实现roi-align时，在后面又接了一个avg_pooling和max_pooling，以实现不同的roipooling操作，并且避免了重复开发。
代码位置

Deformable Roi pooling

可变形roi提取方法来源于论文Deformable Convolutional Networks,文章介绍了形变卷积的方法（增加offset）和所带来的好处。其中，Deformable Roi pooling就是一种由此衍生而来的思路。代码来源
由于要实现可变形，所以代码中加入了offset变量。

    template <typename DType>__global__ void DeformablePSROIPoolForwardKernel(const int count,const DType* bottom_data,const DType spatial_scale,const int channels,const int height, const int width,const int pooled_height, const int pooled_width,const DType* bottom_rois, const DType* bottom_trans,const bool no_trans,const DType trans_std,const int sample_per_part,const int output_dim,const int group_size,const int part_size,const int num_classes,const int channels_each_class,DType* top_data,DType* top_count) {CUDA_KERNEL_LOOP(index, count) {//常规套路// The output is in order (n, ctop, ph, pw)int pw = index % pooled_width;int ph = (index / pooled_width) % pooled_height;int ctop = (index / pooled_width / pooled_height) % output_dim;int n = index / pooled_width / pooled_height / output_dim;// [start, end) interval for spatial samplingconst DType* offset_bottom_rois = bottom_rois + n * 5;int roi_batch_ind = offset_bottom_rois[0];DType roi_start_w = static_cast<DType>(round(offset_bottom_rois[1])) * spatial_scale - 0.5;DType roi_start_h = static_cast<DType>(round(offset_bottom_rois[2])) * spatial_scale - 0.5;DType roi_end_w = static_cast<DType>(round(offset_bottom_rois[3]) + 1.) * spatial_scale - 0.5;DType roi_end_h = static_cast<DType>(round(offset_bottom_rois[4]) + 1.) * spatial_scale - 0.5;// Force too small ROIs to be 1x1DType roi_width = max(roi_end_w - roi_start_w, 0.1); //avoid 0DType roi_height = max(roi_end_h - roi_start_h, 0.1);// Compute w and h at bottomDType bin_size_h = roi_height / static_cast<DType>(pooled_height);DType bin_size_w = roi_width / static_cast<DType>(pooled_width);//一个采样点采样几次DType sub_bin_size_h = bin_size_h / static_cast<DType>(sample_per_part);DType sub_bin_size_w = bin_size_w / static_cast<DType>(sample_per_part);int part_h = floor(static_cast<DType>(ph) / pooled_height*part_size);int part_w = floor(static_cast<DType>(pw) / pooled_width*part_size);int class_id = ctop / channels_each_class;//传递bottom_trans,也就是采样点位移DType trans_x = no_trans ? static_cast<DType>(0) :bottom_trans[(((n * num_classes + class_id) * 2) * part_size + part_h)*part_size + part_w] * trans_std;DType trans_y = no_trans ? static_cast<DType>(0) :bottom_trans[(((n * num_classes + class_id) * 2 + 1) * part_size + part_h)*part_size + part_w] * trans_std;DType wstart = static_cast<DType>(pw)* bin_size_w+ roi_start_w;wstart += trans_x * roi_width;DType hstart = static_cast<DType>(ph) * bin_size_h+ roi_start_h;hstart += trans_y * roi_height;DType sum = 0;int count = 0;int gw = floor(static_cast<DType>(pw) * group_size / pooled_width);int gh = floor(static_cast<DType>(ph)* group_size / pooled_height);gw = min(max(gw, 0), group_size - 1);gh = min(max(gh, 0), group_size - 1);//内插采样（比roi-align更精确）const DType* offset_bottom_data = bottom_data + (roi_batch_ind * channels) * height * width;for (int ih = 0; ih < sample_per_part; ih++) {for (int iw = 0; iw < sample_per_part; iw++) {DType w = wstart + iw*sub_bin_size_w;DType h = hstart + ih*sub_bin_size_h;// bilinear interpolationif (w<-0.5 || w>width - 0.5 || h<-0.5 || h>height - 0.5) {continue;}w = min(max(w, 0.), width - 1.);h = min(max(h, 0.), height - 1.);int c = (ctop*group_size + gh)*group_size + gw;DType val = bilinear_interp(offset_bottom_data + c*height*width, w, h, width, height);sum += val;count++;}}top_data[index] = count == 0 ? static_cast<DType>(0) : sum / count;top_count[index] = count;}}

本文描述了各个ROI特征提取技术的流程，每一个新的技术都是对上一代方法缺点进行了改进。
最后，祝您身体健康，再见！

https://blog.csdn.net/jiongnima/article/details/80016683
https://towardsdatascience.com/review-dcn-deformable-convolutional-networks-2nd-runner-up-in-2017-coco-detection-object-14e488efce44