（原）堆叠hourglass网络

转载请注明出处：

https://www.cnblogs.com/darkknightzh/p/11486185.html

论文：

https://arxiv.org/abs/1603.06937

官方torch代码（没具体看）：

https://github.com/princeton-vl/pose-hg-demo

第三方pytorch代码（位于models/StackedHourGlass.py）：

https://github.com/Naman-ntc/Pytorch-Human-Pose-Estimation

1. 简介

该论文利用多尺度特征来识别姿态，如下图所示，每个子网络称为hourglass Network，是一个沙漏型的结构，多个这种结构堆叠起来，称作stacked hourglass。堆叠的方式，方便每个模块在整个图像上重新估计姿态和特征。如下图所示，输入图像通过全卷积网络fcn后，得到特征，而后通过多个堆叠的hourglass，得到最终的热图。

Hourglass如下图所示。其中每个方块均为下下图的残差模块。

Hourglass采用了中间监督（Intermediate Supervision）。每个hourglass均会有热图（蓝色）。训练阶段，将这些热图和真实热图计算损失MSE，并求和，得到损失；推断阶段，使用的是最后一个hourglass的热图。

2. stacked hourglass

堆叠hourglass结构如下图所示（nChannels=256,nStack=2,nModules=2,numReductions=4, nJoints=17）：

代码如下：

 class StackedHourGlass(nn.Module):
     """docstring for StackedHourGlass"""
     def __init__(self, nChannels, nStack, nModules, numReductions, nJoints):
         super(StackedHourGlass, self).__init__()
         self.nChannels = nChannels
         self.nStack = nStack
         self.nModules = nModules
         self.numReductions = numReductions
         self.nJoints = nJoints

         self.start = M.BnReluConv(3, 64, kernelSize = 7, stride = 2, padding = 3)  # BN+ReLU+conv

         self.res1 = M.Residual(64, 128) # 输入和输出不等，输入通过1*1conv结果和3*(BN+ReLU+conv)求和
         self.mp = nn.MaxPool2d(2, 2)
         self.res2 = M.Residual(128, 128) # 输入和输出相等，为x+3*(BN+ReLU+conv)
         self.res3 = M.Residual(128, self.nChannels) # 输入和输出相等，为x+3*(BN+ReLU+conv)；否则输入通过1*1conv结果和3*(BN+ReLU+conv)求和。

         _hourglass, _Residual, _lin1, _chantojoints, _lin2, _jointstochan = [],[],[],[],[],[]

         for _ in range(self.nStack):  # 堆叠个数
             _hourglass.append(Hourglass(self.nChannels, self.numReductions, self.nModules))
             _ResidualModules = []
             for _ in range(self.nModules):
                 _ResidualModules.append(M.Residual(self.nChannels, self.nChannels))   # 输入和输出相等，为x+3*(BN+ReLU+conv)
             _ResidualModules = nn.Sequential(*_ResidualModules)
             _Residual.append(_ResidualModules)   # self.nModules 个 3*(BN+ReLU+conv)
             _lin1.append(M.BnReluConv(self.nChannels, self.nChannels))       # BN+ReLU+conv
             _chantojoints.append(nn.Conv2d(self.nChannels, self.nJoints,1))  # 1*1 conv，维度变换
             _lin2.append(nn.Conv2d(self.nChannels, self.nChannels,1))        # 1*1 conv，维度不变
             _jointstochan.append(nn.Conv2d(self.nJoints,self.nChannels,1))   # 1*1 conv，维度变换

         self.hourglass = nn.ModuleList(_hourglass)
         self.Residual = nn.ModuleList(_Residual)
         self.lin1 = nn.ModuleList(_lin1)
         self.chantojoints = nn.ModuleList(_chantojoints)
         self.lin2 = nn.ModuleList(_lin2)
         self.jointstochan = nn.ModuleList(_jointstochan)

     def forward(self, x):
         x = self.start(x)
         x = self.res1(x)
         x = self.mp(x)
         x = self.res2(x)
         x = self.res3(x)
         out = []

         for i in range(self.nStack):
             x1 = self.hourglass[i](x)
             x1 = self.Residual[i](x1)
             x1 = self.lin1[i](x1)
             out.append(self.chantojoints[i](x1))
             x1 = self.lin2[i](x1)
             x = x + x1 + self.jointstochan[i](out[i])   # 特征求和

         return (out)

3. hourglass

hourglass在numReductions>1时，递归调用自己，结构如下：

代码如下：

 class Hourglass(nn.Module):
     """docstring for Hourglass"""
     def __init__(self, nChannels = 256, numReductions = 4, nModules = 2, poolKernel = (2,2), poolStride = (2,2), upSampleKernel = 2):
         super(Hourglass, self).__init__()
         self.numReductions = numReductions
         self.nModules = nModules
         self.nChannels = nChannels
         self.poolKernel = poolKernel
         self.poolStride = poolStride
         self.upSampleKernel = upSampleKernel

         """For the skip connection, a residual module (or sequence of residuaql modules)  """
         _skip = []
         for _ in range(self.nModules):
             _skip.append(M.Residual(self.nChannels, self.nChannels))  # 输入和输出相等，为x+3*(BN+ReLU+conv)
         self.skip = nn.Sequential(*_skip)

         """First pooling to go to smaller dimension then pass input through
         Residual Module or sequence of Modules then  and subsequent cases:
             either pass through Hourglass of numReductions-1 or pass through M.Residual Module or sequence of Modules """
         self.mp = nn.MaxPool2d(self.poolKernel, self.poolStride)

         _afterpool = []
         for _ in range(self.nModules):
             _afterpool.append(M.Residual(self.nChannels, self.nChannels))  # 输入和输出相等，为x+3*(BN+ReLU+conv)
         self.afterpool = nn.Sequential(*_afterpool)

         if (numReductions > 1):
             self.hg = Hourglass(self.nChannels, self.numReductions-1, self.nModules, self.poolKernel, self.poolStride)  # 嵌套调用本身
         else:
             _num1res = []
             for _ in range(self.nModules):
                 _num1res.append(M.Residual(self.nChannels,self.nChannels))  # 输入和输出相等，为x+3*(BN+ReLU+conv)
             self.num1res = nn.Sequential(*_num1res)  # doesnt seem that important ?

         """ Now another M.Residual Module or sequence of M.Residual Modules  """
         _lowres = []
         for _ in range(self.nModules):
             _lowres.append(M.Residual(self.nChannels,self.nChannels))   # 输入和输出相等，为x+3*(BN+ReLU+conv)
         self.lowres = nn.Sequential(*_lowres)

         """ Upsampling Layer (Can we change this??????) As per Newell's paper upsamping recommended  """
         self.up = myUpsample()#nn.Upsample(scale_factor = self.upSampleKernel)   # 将高和宽扩充为原来2倍，实现上采样

     def forward(self, x):
         out1 = x
         out1 = self.skip(out1)          # 输入和输出相等，为x+3*(BN+ReLU+conv)
         out2 = x
         out2 = self.mp(out2)            # 降维
         out2 = self.afterpool(out2)     # 输入和输出相等，为x+3*(BN+ReLU+conv)
         if self.numReductions>1:
             out2 = self.hg(out2)        # 嵌套调用本身
         else:
             out2 = self.num1res(out2)   # 输入和输出相等，为x+3*(BN+ReLU+conv)
         out2 = self.lowres(out2)        # 输入和输出相等，为x+3*(BN+ReLU+conv)
         out2 = self.up(out2)            # 升维

         return out2 + out1              # 求和

4. 上采样myUpsample

上采样代码如下：

 class myUpsample(nn.Module):
     def __init__(self):
         super(myUpsample, self).__init__()
         pass
     def forward(self, x):   # 将高和宽扩充为原来2倍，实现上采样
         return x[:, :, :, None, :, None].expand(-1, -1, -1, 2, -1, 2).reshape(x.size(0), x.size(1), x.size(2)*2, x.size(3)*2)

其中x为(N)(C)(H)(W)的矩阵，x[:, :, :, None, :, None]为(N)(C)(H)(1)(W)(1)的矩阵，expand之后变成(N)(C)(H)(2)(W)(2)的矩阵，最终reshape之后变成(N)(C)(2H) (2W)的矩阵，实现了将1个像素水平和垂直方向各扩充2倍，变成4个像素（4个像素值相同），完成了上采样。

5. 残差模块

残差模块结构如下：

代码如下：

 class Residual(nn.Module):
         """docstring for Residual"""  # 输入和输出相等，为x+3*(BN+ReLU+conv)；否则输入通过1*1conv结果和3*(BN+ReLU+conv)求和
         def __init__(self, inChannels, outChannels):
                 super(Residual, self).__init__()
                 self.inChannels = inChannels
                 self.outChannels = outChannels
                 self.cb = ConvBlock(inChannels, outChannels)      # 3 * (BN+ReLU+conv) 其中第一组降维，第二组不变，第三组升维
                 self.skip = SkipLayer(inChannels, outChannels)    # 输入和输出通道相等，则输出=输入，否则为1*1 conv

         def forward(self, x):
                 out = 0
                 out = out + self.cb(x)
                 out = out + self.skip(x)
                 return out

其中skiplayer代码如下：

 class SkipLayer(nn.Module):
         """docstring for SkipLayer"""  # 输入和输出通道相等，则输出=输入，否则为1*1 conv
         def __init__(self, inChannels, outChannels):
                 super(SkipLayer, self).__init__()
                 self.inChannels = inChannels
                 self.outChannels = outChannels
                 if (self.inChannels == self.outChannels):
                         self.conv = None
                 else:
                         self.conv = nn.Conv2d(self.inChannels, self.outChannels, 1)

         def forward(self, x):
                 if self.conv is not None:
                         x = self.conv(x)
                 return x

6. conv

 class BnReluConv(nn.Module):
         """docstring for BnReluConv"""    # BN+ReLU+conv
         def __init__(self, inChannels, outChannels, kernelSize = 1, stride = 1, padding = 0):
                 super(BnReluConv, self).__init__()
                 self.inChannels = inChannels
                 self.outChannels = outChannels
                 self.kernelSize = kernelSize
                 self.stride = stride
                 self.padding = padding

                 self.bn = nn.BatchNorm2d(self.inChannels)
                 self.conv = nn.Conv2d(self.inChannels, self.outChannels, self.kernelSize, self.stride, self.padding)
                 self.relu = nn.ReLU()

         def forward(self, x):
                 x = self.bn(x)
                 x = self.relu(x)
                 x = self.conv(x)
                 return x

7. ConvBlock

 class ConvBlock(nn.Module):
         """docstring for ConvBlock"""  # 3 * (BN+ReLU+conv) 其中第一组降维，第二组不变，第三组升维
         def __init__(self, inChannels, outChannels):
                 super(ConvBlock, self).__init__()
                 self.inChannels = inChannels
                 self.outChannels = outChannels
                 self.outChannelsby2 = outChannels//2

                 self.cbr1 = BnReluConv(self.inChannels, self.outChannelsby2, 1, 1, 0)        # BN+ReLU+conv
                 self.cbr2 = BnReluConv(self.outChannelsby2, self.outChannelsby2, 3, 1, 1)    # BN+ReLU+conv
                 self.cbr3 = BnReluConv(self.outChannelsby2, self.outChannels, 1, 1, 0)       # BN+ReLU+conv

         def forward(self, x):
                 x = self.cbr1(x)
                 x = self.cbr2(x)
                 x = self.cbr3(x)
                 return x