cascade DecodeBBox层

https://zhuanlan.zhihu.com/p/36095768

我的推断，第二第三阶段应该不是把所有anchor进行bounding box regression，然后再选取当前条件下的所有roi，而是第一阶段选取512个roi，然后把在第一阶段匹配好的roi送到第二、三阶段

layer {
  name: "proposals_2nd"
  type: "DecodeBBox"
  bottom: "bbox_pred"
  bottom: "rois"
  bottom: "match_gt_boxes"
  top: "proposals_2nd"
  bbox_reg_param {
    bbox_mean: 0 bbox_mean: 0 bbox_mean: 0 bbox_mean: 0
    bbox_std: 0.1 bbox_std: 0.1 bbox_std: 0.2 bbox_std: 0.2
  }
  propagate_down: 0
  propagate_down: 0
  propagate_down: 0
}

这段代码就证明了这个想法：rois来自于第一阶段proposal_info，这些rois也是在第一阶段做roi-pooling用来训练的。

个人感觉cascade的模型就是4张图提取512个roi进行训练,然后经过第一阶段训练后,把这512个roi经过回归精修然后去除回归后x1大于x2和y1大于y2的和回归后和gt的iou大于0.95的,这样roi可能就没有512个了.把这些输入给第二阶段的proposal_info_2nd,让这个层再去决定训练样本,这样大可能训练的数据是不足512,并且3个阶段其实都是训练的同一个批roi,也就是说第一阶段进去的那些roi,后面几个阶段实际上也在训练他们,而不是新出来的框DecodeBbox层的输入是bbox_pred,rois和match_gt_boxes.首先明确一点,rpn网络会输出很多proposals出来,ProposalTarget层将这些proposals和gt算iou,确定正负样本并选取1:3的比例,然后输出rois,rois就是拿来具体训练的从rpn中获得那部分预提取框.DecodeBbox层就是将这些原本的rois回归成更精准的框,也就是在原始的rois的坐标上增加经过训练得到的回归的值,这个是通过DecodeBBoxesWithPrior函数实现.DecodeBbox层分为大致3个步骤:1.回归得到更精准的rois  2.去掉回归后x1大于x2和y1大于y2的框  3.去掉回归后和gt的iou大于0.95的框
这部分的结果

#include <cfloat>
#include <vector>

#include "caffe/util/bbox_util.hpp"
#include "caffe/layers/decode_bbox_layer.hpp"

namespace caffe {

template <typename Dtype>
void DecodeBBoxLayer<Dtype>::LayerSetUp(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  // bbox mean and std
  BBoxRegParameter bbox_reg_param = this->layer_param_.bbox_reg_param();
  bbox_mean_.Reshape(,,,); bbox_std_.Reshape(,,,);
  if (bbox_reg_param.bbox_mean_size() >  && bbox_reg_param.bbox_std_size() > ) {
    int num_means = this->layer_param_.bbox_reg_param().bbox_mean_size();
    int num_stds = this->layer_param_.bbox_reg_param().bbox_std_size();
    CHECK_EQ(num_means,); CHECK_EQ(num_stds,);
    for (int i = ; i < ; i++) {
      bbox_mean_.mutable_cpu_data()[i] = bbox_reg_param.bbox_mean(i);
      bbox_std_.mutable_cpu_data()[i] = bbox_reg_param.bbox_std(i);
      CHECK_GT(bbox_std_.mutable_cpu_data()[i],);
    }
  } else {
    caffe_set(bbox_mean_.count(), Dtype(), bbox_mean_.mutable_cpu_data());
    caffe_set(bbox_std_.count(), Dtype(), bbox_std_.mutable_cpu_data());
  }
}

template <typename Dtype>
void DecodeBBoxLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top) {
  // bottom: bbox_blob, prior_blob, (match_gt_boxes)
  CHECK_EQ(bottom[]->num(),bottom[]->num());
  if (bottom.size()>=) {
    CHECK_EQ(bottom[]->num(),bottom[]->num());
    CHECK(this->phase_ == TRAIN);
  }
  CHECK_EQ(bottom[]->channels(),);
  CHECK_EQ(bottom[]->channels(),);
  bbox_pred_.ReshapeLike(*bottom[]);
  top[]->ReshapeLike(*bottom[]);
}

template <typename Dtype>
void DecodeBBoxLayer<Dtype>::Forward_cpu(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  const int num = bottom[]->num();
  const int bbox_dim = bottom[]->channels();
  const int prior_dim = bottom[]->channels();

  //decode prior box [img_id x1 y1 x2 y2]
  const Dtype* prior_data = bottom[]->cpu_data();
  vector<BBox> prior_bboxes;
  for (int i = ; i < num; i++) {
    BBox bbox;
    bbox.xmin = prior_data[i*prior_dim + ];
    bbox.ymin = prior_data[i*prior_dim + ];
    bbox.xmax = prior_data[i*prior_dim + ];
    bbox.ymax = prior_data[i*prior_dim + ];
    prior_bboxes.push_back(bbox);
  }

  // decode bbox predictions
  const Dtype* bbox_data = bottom[]->cpu_data();
  Dtype* bbox_pred_data = bbox_pred_.mutable_cpu_data();

  DecodeBBoxesWithPrior(bbox_data,prior_bboxes,bbox_dim,bbox_mean_.cpu_data(),
          bbox_std_.cpu_data(),bbox_pred_data);

  vector<bool> valid_bbox_flags(num,true);
  // screen out mal-boxes
  if (this->phase_ == TRAIN) {
    for (int i = ; i < num; i++) {
      const int base_index = i*bbox_dim+;
      if (bbox_pred_data[base_index] > bbox_pred_data[base_index+]
              || bbox_pred_data[base_index+] > bbox_pred_data[base_index+]) {
        valid_bbox_flags[i] = false;
      }
    }
  }
  // screen out high IoU boxes, to remove redundant gt boxes
  if (bottom.size()== && this->phase_ == TRAIN) {
    const Dtype* match_gt_boxes = bottom[]->cpu_data();
    const int gt_dim = bottom[]->channels();
    const float gt_iou_thr = this->layer_param_.decode_bbox_param().gt_iou_thr();
    for (int i = ; i < num; i++) {
      const float overlap = match_gt_boxes[i*gt_dim+gt_dim-];
      if (overlap >= gt_iou_thr) {
        valid_bbox_flags[i] = false;
      }
    }
  }

  vector<int> valid_bbox_ids;
  for (int i = ; i < num; i++) {
    if (valid_bbox_flags[i]) {
      valid_bbox_ids.push_back(i);
    }
  }
  const int keep_num = valid_bbox_ids.size();
  CHECK_GT(keep_num,);

  top[]->Reshape(keep_num, prior_dim, , );
  Dtype* decoded_bbox_data = top[]->mutable_cpu_data();
  for (int i = ; i < keep_num; i++) {
    const int keep_id = valid_bbox_ids[i];
    const int base_index = keep_id*bbox_dim+;
    decoded_bbox_data[i*prior_dim] =  prior_data[keep_id*prior_dim];
    decoded_bbox_data[i*prior_dim+] = bbox_pred_data[base_index];
    decoded_bbox_data[i*prior_dim+] = bbox_pred_data[base_index+];
    decoded_bbox_data[i*prior_dim+] = bbox_pred_data[base_index+];
    decoded_bbox_data[i*prior_dim+] = bbox_pred_data[base_index+];
  }
}

INSTANTIATE_CLASS(DecodeBBoxLayer);
REGISTER_LAYER_CLASS(DecodeBBox);

} // namespace caffe

DecodeBBoxesWithPrior函数在bbox_util.cpp里实现,完成的功能就是把bounding box regression的结果对输入的prior_bbox(其实就是faster中的输入的region proposal)进行回归获得更精确的框坐标,然后存储在pred_data

template <typename Dtype>
void DecodeBBoxesWithPrior(const Dtype* bbox_data, const vector<BBox> prior_bboxes,
        const int bbox_dim, const Dtype* means, const Dtype* stds,
        Dtype* pred_data) {
  const int num = prior_bboxes.size();
  const int cls_num = bbox_dim/;
  for (int i = ; i < num; i++) {
    Dtype pw, ph, cx, cy;
    pw = prior_bboxes[i].xmax-prior_bboxes[i].xmin+;
    ph = prior_bboxes[i].ymax-prior_bboxes[i].ymin+;
    cx = 0.5*(prior_bboxes[i].xmax+prior_bboxes[i].xmin);
    cy = 0.5*(prior_bboxes[i].ymax+prior_bboxes[i].ymin);
    for (int c = ; c < cls_num; c++) {
      Dtype bx, by, bw, bh;
      // bbox de-normalization
      bx = bbox_data[i*bbox_dim+*c]*stds[]+means[];
      by = bbox_data[i*bbox_dim+*c+]*stds[]+means[];
      bw = bbox_data[i*bbox_dim+*c+]*stds[]+means[];
      bh = bbox_data[i*bbox_dim+*c+]*stds[]+means[];

      Dtype tx, ty, tw, th;
      tx = bx*pw+cx; ty = by*ph+cy;
      tw = pw*exp(bw); th = ph*exp(bh);
      tx -= (tw-)/; ty -= (th-)/;
      pred_data[i*bbox_dim+*c] = tx;
      pred_data[i*bbox_dim+*c+] = ty;
      pred_data[i*bbox_dim+*c+] = tx+tw-;
      pred_data[i*bbox_dim+*c+] = ty+th-;
    }
  }
}