892 lines
33 KiB
Python
892 lines
33 KiB
Python
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""
|
|
This code is based on
|
|
https://github.com/HRNet/Lite-HRNet/blob/hrnet/models/backbones/litehrnet.py
|
|
"""
|
|
|
|
import paddle
|
|
import paddle.nn as nn
|
|
import paddle.nn.functional as F
|
|
|
|
from numbers import Integral
|
|
from paddle import ParamAttr
|
|
from paddle.regularizer import L2Decay
|
|
from paddle.nn.initializer import Normal, Constant
|
|
from ppdet.core.workspace import register
|
|
from ppdet.modeling.shape_spec import ShapeSpec
|
|
from ppdet.modeling.ops import channel_shuffle
|
|
from .. import layers as L
|
|
|
|
__all__ = ['LiteHRNet']
|
|
|
|
|
|
class ConvNormLayer(nn.Layer):
|
|
def __init__(self,
|
|
ch_in,
|
|
ch_out,
|
|
filter_size,
|
|
stride=1,
|
|
groups=1,
|
|
norm_type=None,
|
|
norm_groups=32,
|
|
norm_decay=0.,
|
|
freeze_norm=False,
|
|
act=None):
|
|
super(ConvNormLayer, self).__init__()
|
|
self.act = act
|
|
norm_lr = 0. if freeze_norm else 1.
|
|
if norm_type is not None:
|
|
assert norm_type in ['bn', 'sync_bn', 'gn'], \
|
|
"norm_type should be one of ['bn', 'sync_bn', 'gn'], but got {}".format(norm_type)
|
|
param_attr = ParamAttr(
|
|
initializer=Constant(1.0),
|
|
learning_rate=norm_lr,
|
|
regularizer=L2Decay(norm_decay), )
|
|
bias_attr = ParamAttr(
|
|
learning_rate=norm_lr, regularizer=L2Decay(norm_decay))
|
|
global_stats = True if freeze_norm else None
|
|
if norm_type in ['bn', 'sync_bn']:
|
|
self.norm = nn.BatchNorm2D(
|
|
ch_out,
|
|
weight_attr=param_attr,
|
|
bias_attr=bias_attr,
|
|
use_global_stats=global_stats, )
|
|
elif norm_type == 'gn':
|
|
self.norm = nn.GroupNorm(
|
|
num_groups=norm_groups,
|
|
num_channels=ch_out,
|
|
weight_attr=param_attr,
|
|
bias_attr=bias_attr)
|
|
norm_params = self.norm.parameters()
|
|
if freeze_norm:
|
|
for param in norm_params:
|
|
param.stop_gradient = True
|
|
conv_bias_attr = False
|
|
else:
|
|
conv_bias_attr = True
|
|
self.norm = None
|
|
|
|
self.conv = nn.Conv2D(
|
|
in_channels=ch_in,
|
|
out_channels=ch_out,
|
|
kernel_size=filter_size,
|
|
stride=stride,
|
|
padding=(filter_size - 1) // 2,
|
|
groups=groups,
|
|
weight_attr=ParamAttr(initializer=Normal(
|
|
mean=0., std=0.001)),
|
|
bias_attr=conv_bias_attr)
|
|
|
|
def forward(self, inputs):
|
|
out = self.conv(inputs)
|
|
if self.norm is not None:
|
|
out = self.norm(out)
|
|
|
|
if self.act == 'relu':
|
|
out = F.relu(out)
|
|
elif self.act == 'sigmoid':
|
|
out = F.sigmoid(out)
|
|
return out
|
|
|
|
|
|
class DepthWiseSeparableConvNormLayer(nn.Layer):
|
|
def __init__(self,
|
|
ch_in,
|
|
ch_out,
|
|
filter_size,
|
|
stride=1,
|
|
dw_norm_type=None,
|
|
pw_norm_type=None,
|
|
norm_decay=0.,
|
|
freeze_norm=False,
|
|
dw_act=None,
|
|
pw_act=None):
|
|
super(DepthWiseSeparableConvNormLayer, self).__init__()
|
|
self.depthwise_conv = ConvNormLayer(
|
|
ch_in=ch_in,
|
|
ch_out=ch_in,
|
|
filter_size=filter_size,
|
|
stride=stride,
|
|
groups=ch_in,
|
|
norm_type=dw_norm_type,
|
|
act=dw_act,
|
|
norm_decay=norm_decay,
|
|
freeze_norm=freeze_norm, )
|
|
self.pointwise_conv = ConvNormLayer(
|
|
ch_in=ch_in,
|
|
ch_out=ch_out,
|
|
filter_size=1,
|
|
stride=1,
|
|
norm_type=pw_norm_type,
|
|
act=pw_act,
|
|
norm_decay=norm_decay,
|
|
freeze_norm=freeze_norm, )
|
|
|
|
def forward(self, x):
|
|
x = self.depthwise_conv(x)
|
|
x = self.pointwise_conv(x)
|
|
return x
|
|
|
|
|
|
class CrossResolutionWeightingModule(nn.Layer):
|
|
def __init__(self,
|
|
channels,
|
|
ratio=16,
|
|
norm_type='bn',
|
|
freeze_norm=False,
|
|
norm_decay=0.):
|
|
super(CrossResolutionWeightingModule, self).__init__()
|
|
self.channels = channels
|
|
total_channel = sum(channels)
|
|
self.conv1 = ConvNormLayer(
|
|
ch_in=total_channel,
|
|
ch_out=total_channel // ratio,
|
|
filter_size=1,
|
|
stride=1,
|
|
norm_type=norm_type,
|
|
act='relu',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
self.conv2 = ConvNormLayer(
|
|
ch_in=total_channel // ratio,
|
|
ch_out=total_channel,
|
|
filter_size=1,
|
|
stride=1,
|
|
norm_type=norm_type,
|
|
act='sigmoid',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
|
|
def forward(self, x):
|
|
mini_size = x[-1].shape[-2:]
|
|
out = [F.adaptive_avg_pool2d(s, mini_size) for s in x[:-1]] + [x[-1]]
|
|
out = paddle.concat(out, 1)
|
|
out = self.conv1(out)
|
|
out = self.conv2(out)
|
|
out = paddle.split(out, self.channels, 1)
|
|
out = [
|
|
s * F.interpolate(
|
|
a, s.shape[-2:], mode='nearest') for s, a in zip(x, out)
|
|
]
|
|
return out
|
|
|
|
|
|
class SpatialWeightingModule(nn.Layer):
|
|
def __init__(self, in_channel, ratio=16, freeze_norm=False, norm_decay=0.):
|
|
super(SpatialWeightingModule, self).__init__()
|
|
self.global_avgpooling = nn.AdaptiveAvgPool2D(1)
|
|
self.conv1 = ConvNormLayer(
|
|
ch_in=in_channel,
|
|
ch_out=in_channel // ratio,
|
|
filter_size=1,
|
|
stride=1,
|
|
act='relu',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
self.conv2 = ConvNormLayer(
|
|
ch_in=in_channel // ratio,
|
|
ch_out=in_channel,
|
|
filter_size=1,
|
|
stride=1,
|
|
act='sigmoid',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
|
|
def forward(self, x):
|
|
out = self.global_avgpooling(x)
|
|
out = self.conv1(out)
|
|
out = self.conv2(out)
|
|
return x * out
|
|
|
|
|
|
class ConditionalChannelWeightingBlock(nn.Layer):
|
|
def __init__(self,
|
|
in_channels,
|
|
stride,
|
|
reduce_ratio,
|
|
norm_type='bn',
|
|
freeze_norm=False,
|
|
norm_decay=0.):
|
|
super(ConditionalChannelWeightingBlock, self).__init__()
|
|
assert stride in [1, 2]
|
|
branch_channels = [channel // 2 for channel in in_channels]
|
|
|
|
self.cross_resolution_weighting = CrossResolutionWeightingModule(
|
|
branch_channels,
|
|
ratio=reduce_ratio,
|
|
norm_type=norm_type,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
self.depthwise_convs = nn.LayerList([
|
|
ConvNormLayer(
|
|
channel,
|
|
channel,
|
|
filter_size=3,
|
|
stride=stride,
|
|
groups=channel,
|
|
norm_type=norm_type,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay) for channel in branch_channels
|
|
])
|
|
|
|
self.spatial_weighting = nn.LayerList([
|
|
SpatialWeightingModule(
|
|
channel,
|
|
ratio=4,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay) for channel in branch_channels
|
|
])
|
|
|
|
def forward(self, x):
|
|
x = [s.chunk(2, axis=1) for s in x]
|
|
x1 = [s[0] for s in x]
|
|
x2 = [s[1] for s in x]
|
|
|
|
x2 = self.cross_resolution_weighting(x2)
|
|
x2 = [dw(s) for s, dw in zip(x2, self.depthwise_convs)]
|
|
x2 = [sw(s) for s, sw in zip(x2, self.spatial_weighting)]
|
|
|
|
out = [paddle.concat([s1, s2], axis=1) for s1, s2 in zip(x1, x2)]
|
|
out = [channel_shuffle(s, groups=2) for s in out]
|
|
return out
|
|
|
|
|
|
class ShuffleUnit(nn.Layer):
|
|
def __init__(self,
|
|
in_channel,
|
|
out_channel,
|
|
stride,
|
|
norm_type='bn',
|
|
freeze_norm=False,
|
|
norm_decay=0.):
|
|
super(ShuffleUnit, self).__init__()
|
|
branch_channel = out_channel // 2
|
|
self.stride = stride
|
|
if self.stride == 1:
|
|
assert in_channel == branch_channel * 2, \
|
|
"when stride=1, in_channel {} should equal to branch_channel*2 {}".format(in_channel, branch_channel * 2)
|
|
if stride > 1:
|
|
self.branch1 = nn.Sequential(
|
|
ConvNormLayer(
|
|
ch_in=in_channel,
|
|
ch_out=in_channel,
|
|
filter_size=3,
|
|
stride=self.stride,
|
|
groups=in_channel,
|
|
norm_type=norm_type,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay),
|
|
ConvNormLayer(
|
|
ch_in=in_channel,
|
|
ch_out=branch_channel,
|
|
filter_size=1,
|
|
stride=1,
|
|
norm_type=norm_type,
|
|
act='relu',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay), )
|
|
self.branch2 = nn.Sequential(
|
|
ConvNormLayer(
|
|
ch_in=branch_channel if stride == 1 else in_channel,
|
|
ch_out=branch_channel,
|
|
filter_size=1,
|
|
stride=1,
|
|
norm_type=norm_type,
|
|
act='relu',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay),
|
|
ConvNormLayer(
|
|
ch_in=branch_channel,
|
|
ch_out=branch_channel,
|
|
filter_size=3,
|
|
stride=self.stride,
|
|
groups=branch_channel,
|
|
norm_type=norm_type,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay),
|
|
ConvNormLayer(
|
|
ch_in=branch_channel,
|
|
ch_out=branch_channel,
|
|
filter_size=1,
|
|
stride=1,
|
|
norm_type=norm_type,
|
|
act='relu',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay), )
|
|
|
|
def forward(self, x):
|
|
if self.stride > 1:
|
|
x1 = self.branch1(x)
|
|
x2 = self.branch2(x)
|
|
else:
|
|
x1, x2 = x.chunk(2, axis=1)
|
|
x2 = self.branch2(x2)
|
|
out = paddle.concat([x1, x2], axis=1)
|
|
out = channel_shuffle(out, groups=2)
|
|
return out
|
|
|
|
|
|
class IterativeHead(nn.Layer):
|
|
def __init__(self,
|
|
in_channels,
|
|
norm_type='bn',
|
|
freeze_norm=False,
|
|
norm_decay=0.):
|
|
super(IterativeHead, self).__init__()
|
|
num_branches = len(in_channels)
|
|
self.in_channels = in_channels[::-1]
|
|
|
|
projects = []
|
|
for i in range(num_branches):
|
|
if i != num_branches - 1:
|
|
projects.append(
|
|
DepthWiseSeparableConvNormLayer(
|
|
ch_in=self.in_channels[i],
|
|
ch_out=self.in_channels[i + 1],
|
|
filter_size=3,
|
|
stride=1,
|
|
dw_act=None,
|
|
pw_act='relu',
|
|
dw_norm_type=norm_type,
|
|
pw_norm_type=norm_type,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay))
|
|
else:
|
|
projects.append(
|
|
DepthWiseSeparableConvNormLayer(
|
|
ch_in=self.in_channels[i],
|
|
ch_out=self.in_channels[i],
|
|
filter_size=3,
|
|
stride=1,
|
|
dw_act=None,
|
|
pw_act='relu',
|
|
dw_norm_type=norm_type,
|
|
pw_norm_type=norm_type,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay))
|
|
self.projects = nn.LayerList(projects)
|
|
|
|
def forward(self, x):
|
|
x = x[::-1]
|
|
y = []
|
|
last_x = None
|
|
for i, s in enumerate(x):
|
|
if last_x is not None:
|
|
last_x = F.interpolate(
|
|
last_x,
|
|
size=s.shape[-2:],
|
|
mode='bilinear',
|
|
align_corners=True)
|
|
s = s + last_x
|
|
s = self.projects[i](s)
|
|
y.append(s)
|
|
last_x = s
|
|
|
|
return y[::-1]
|
|
|
|
|
|
class Stem(nn.Layer):
|
|
def __init__(self,
|
|
in_channel,
|
|
stem_channel,
|
|
out_channel,
|
|
expand_ratio,
|
|
norm_type='bn',
|
|
freeze_norm=False,
|
|
norm_decay=0.):
|
|
super(Stem, self).__init__()
|
|
self.conv1 = ConvNormLayer(
|
|
in_channel,
|
|
stem_channel,
|
|
filter_size=3,
|
|
stride=2,
|
|
norm_type=norm_type,
|
|
act='relu',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
mid_channel = int(round(stem_channel * expand_ratio))
|
|
branch_channel = stem_channel // 2
|
|
if stem_channel == out_channel:
|
|
inc_channel = out_channel - branch_channel
|
|
else:
|
|
inc_channel = out_channel - stem_channel
|
|
self.branch1 = nn.Sequential(
|
|
ConvNormLayer(
|
|
ch_in=branch_channel,
|
|
ch_out=branch_channel,
|
|
filter_size=3,
|
|
stride=2,
|
|
groups=branch_channel,
|
|
norm_type=norm_type,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay),
|
|
ConvNormLayer(
|
|
ch_in=branch_channel,
|
|
ch_out=inc_channel,
|
|
filter_size=1,
|
|
stride=1,
|
|
norm_type=norm_type,
|
|
act='relu',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay), )
|
|
self.expand_conv = ConvNormLayer(
|
|
ch_in=branch_channel,
|
|
ch_out=mid_channel,
|
|
filter_size=1,
|
|
stride=1,
|
|
norm_type=norm_type,
|
|
act='relu',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
self.depthwise_conv = ConvNormLayer(
|
|
ch_in=mid_channel,
|
|
ch_out=mid_channel,
|
|
filter_size=3,
|
|
stride=2,
|
|
groups=mid_channel,
|
|
norm_type=norm_type,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
self.linear_conv = ConvNormLayer(
|
|
ch_in=mid_channel,
|
|
ch_out=branch_channel
|
|
if stem_channel == out_channel else stem_channel,
|
|
filter_size=1,
|
|
stride=1,
|
|
norm_type=norm_type,
|
|
act='relu',
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
|
|
def forward(self, x):
|
|
x = self.conv1(x)
|
|
x1, x2 = x.chunk(2, axis=1)
|
|
x1 = self.branch1(x1)
|
|
x2 = self.expand_conv(x2)
|
|
x2 = self.depthwise_conv(x2)
|
|
x2 = self.linear_conv(x2)
|
|
out = paddle.concat([x1, x2], axis=1)
|
|
out = channel_shuffle(out, groups=2)
|
|
|
|
return out
|
|
|
|
|
|
class LiteHRNetModule(nn.Layer):
|
|
def __init__(self,
|
|
num_branches,
|
|
num_blocks,
|
|
in_channels,
|
|
reduce_ratio,
|
|
module_type,
|
|
multiscale_output=False,
|
|
with_fuse=True,
|
|
norm_type='bn',
|
|
freeze_norm=False,
|
|
norm_decay=0.):
|
|
super(LiteHRNetModule, self).__init__()
|
|
assert num_branches == len(in_channels),\
|
|
"num_branches {} should equal to num_in_channels {}".format(num_branches, len(in_channels))
|
|
assert module_type in [
|
|
'LITE', 'NAIVE'
|
|
], "module_type should be one of ['LITE', 'NAIVE']"
|
|
self.num_branches = num_branches
|
|
self.in_channels = in_channels
|
|
self.multiscale_output = multiscale_output
|
|
self.with_fuse = with_fuse
|
|
self.norm_type = 'bn'
|
|
self.module_type = module_type
|
|
|
|
if self.module_type == 'LITE':
|
|
self.layers = self._make_weighting_blocks(
|
|
num_blocks,
|
|
reduce_ratio,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
elif self.module_type == 'NAIVE':
|
|
self.layers = self._make_naive_branches(
|
|
num_branches,
|
|
num_blocks,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay)
|
|
|
|
if self.with_fuse:
|
|
self.fuse_layers = self._make_fuse_layers(
|
|
freeze_norm=freeze_norm, norm_decay=norm_decay)
|
|
self.relu = nn.ReLU()
|
|
|
|
def _make_weighting_blocks(self,
|
|
num_blocks,
|
|
reduce_ratio,
|
|
stride=1,
|
|
freeze_norm=False,
|
|
norm_decay=0.):
|
|
layers = []
|
|
for i in range(num_blocks):
|
|
layers.append(
|
|
ConditionalChannelWeightingBlock(
|
|
self.in_channels,
|
|
stride=stride,
|
|
reduce_ratio=reduce_ratio,
|
|
norm_type=self.norm_type,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay))
|
|
return nn.Sequential(*layers)
|
|
|
|
def _make_naive_branches(self,
|
|
num_branches,
|
|
num_blocks,
|
|
freeze_norm=False,
|
|
norm_decay=0.):
|
|
branches = []
|
|
for branch_idx in range(num_branches):
|
|
layers = []
|
|
for i in range(num_blocks):
|
|
layers.append(
|
|
ShuffleUnit(
|
|
self.in_channels[branch_idx],
|
|
self.in_channels[branch_idx],
|
|
stride=1,
|
|
norm_type=self.norm_type,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay))
|
|
branches.append(nn.Sequential(*layers))
|
|
return nn.LayerList(branches)
|
|
|
|
def _make_fuse_layers(self, freeze_norm=False, norm_decay=0.):
|
|
if self.num_branches == 1:
|
|
return None
|
|
fuse_layers = []
|
|
num_out_branches = self.num_branches if self.multiscale_output else 1
|
|
for i in range(num_out_branches):
|
|
fuse_layer = []
|
|
for j in range(self.num_branches):
|
|
if j > i:
|
|
fuse_layer.append(
|
|
nn.Sequential(
|
|
L.Conv2d(
|
|
self.in_channels[j],
|
|
self.in_channels[i],
|
|
kernel_size=1,
|
|
stride=1,
|
|
padding=0,
|
|
bias=False, ),
|
|
nn.BatchNorm2D(self.in_channels[i]),
|
|
nn.Upsample(
|
|
scale_factor=2**(j - i), mode='nearest')))
|
|
elif j == i:
|
|
fuse_layer.append(None)
|
|
else:
|
|
conv_downsamples = []
|
|
for k in range(i - j):
|
|
if k == i - j - 1:
|
|
conv_downsamples.append(
|
|
nn.Sequential(
|
|
L.Conv2d(
|
|
self.in_channels[j],
|
|
self.in_channels[j],
|
|
kernel_size=3,
|
|
stride=2,
|
|
padding=1,
|
|
groups=self.in_channels[j],
|
|
bias=False, ),
|
|
nn.BatchNorm2D(self.in_channels[j]),
|
|
L.Conv2d(
|
|
self.in_channels[j],
|
|
self.in_channels[i],
|
|
kernel_size=1,
|
|
stride=1,
|
|
padding=0,
|
|
bias=False, ),
|
|
nn.BatchNorm2D(self.in_channels[i])))
|
|
else:
|
|
conv_downsamples.append(
|
|
nn.Sequential(
|
|
L.Conv2d(
|
|
self.in_channels[j],
|
|
self.in_channels[j],
|
|
kernel_size=3,
|
|
stride=2,
|
|
padding=1,
|
|
groups=self.in_channels[j],
|
|
bias=False, ),
|
|
nn.BatchNorm2D(self.in_channels[j]),
|
|
L.Conv2d(
|
|
self.in_channels[j],
|
|
self.in_channels[j],
|
|
kernel_size=1,
|
|
stride=1,
|
|
padding=0,
|
|
bias=False, ),
|
|
nn.BatchNorm2D(self.in_channels[j]),
|
|
nn.ReLU()))
|
|
|
|
fuse_layer.append(nn.Sequential(*conv_downsamples))
|
|
fuse_layers.append(nn.LayerList(fuse_layer))
|
|
|
|
return nn.LayerList(fuse_layers)
|
|
|
|
def forward(self, x):
|
|
if self.num_branches == 1:
|
|
return [self.layers[0](x[0])]
|
|
if self.module_type == 'LITE':
|
|
out = self.layers(x)
|
|
elif self.module_type == 'NAIVE':
|
|
for i in range(self.num_branches):
|
|
x[i] = self.layers[i](x[i])
|
|
out = x
|
|
if self.with_fuse:
|
|
out_fuse = []
|
|
for i in range(len(self.fuse_layers)):
|
|
y = out[0] if i == 0 else self.fuse_layers[i][0](out[0])
|
|
for j in range(self.num_branches):
|
|
if j == 0:
|
|
y += y
|
|
elif i == j:
|
|
y += out[j]
|
|
else:
|
|
y += self.fuse_layers[i][j](out[j])
|
|
if i == 0:
|
|
out[i] = y
|
|
out_fuse.append(self.relu(y))
|
|
out = out_fuse
|
|
elif not self.multiscale_output:
|
|
out = [out[0]]
|
|
return out
|
|
|
|
|
|
@register
|
|
class LiteHRNet(nn.Layer):
|
|
"""
|
|
@inproceedings{Yulitehrnet21,
|
|
title={Lite-HRNet: A Lightweight High-Resolution Network},
|
|
author={Yu, Changqian and Xiao, Bin and Gao, Changxin and Yuan, Lu and Zhang, Lei and Sang, Nong and Wang, Jingdong},
|
|
booktitle={CVPR},year={2021}
|
|
}
|
|
Args:
|
|
network_type (str): the network_type should be one of ["lite_18", "lite_30", "naive", "wider_naive"],
|
|
"naive": Simply combining the shuffle block in ShuffleNet and the highresolution design pattern in HRNet.
|
|
"wider_naive": Naive network with wider channels in each block.
|
|
"lite_18": Lite-HRNet-18, which replaces the pointwise convolution in a shuffle block by conditional channel weighting.
|
|
"lite_30": Lite-HRNet-30, with more blocks compared with Lite-HRNet-18.
|
|
freeze_at (int): the stage to freeze
|
|
freeze_norm (bool): whether to freeze norm in HRNet
|
|
norm_decay (float): weight decay for normalization layer weights
|
|
return_idx (List): the stage to return
|
|
"""
|
|
|
|
def __init__(self,
|
|
network_type,
|
|
freeze_at=0,
|
|
freeze_norm=True,
|
|
norm_decay=0.,
|
|
return_idx=[0, 1, 2, 3]):
|
|
super(LiteHRNet, self).__init__()
|
|
if isinstance(return_idx, Integral):
|
|
return_idx = [return_idx]
|
|
assert network_type in ["lite_18", "lite_30", "naive", "wider_naive"], \
|
|
"the network_type should be one of [lite_18, lite_30, naive, wider_naive]"
|
|
assert len(return_idx) > 0, "need one or more return index"
|
|
self.freeze_at = freeze_at
|
|
self.freeze_norm = freeze_norm
|
|
self.norm_decay = norm_decay
|
|
self.return_idx = return_idx
|
|
self.norm_type = 'bn'
|
|
|
|
self.module_configs = {
|
|
"lite_18": {
|
|
"num_modules": [2, 4, 2],
|
|
"num_branches": [2, 3, 4],
|
|
"num_blocks": [2, 2, 2],
|
|
"module_type": ["LITE", "LITE", "LITE"],
|
|
"reduce_ratios": [8, 8, 8],
|
|
"num_channels": [[40, 80], [40, 80, 160], [40, 80, 160, 320]],
|
|
},
|
|
"lite_30": {
|
|
"num_modules": [3, 8, 3],
|
|
"num_branches": [2, 3, 4],
|
|
"num_blocks": [2, 2, 2],
|
|
"module_type": ["LITE", "LITE", "LITE"],
|
|
"reduce_ratios": [8, 8, 8],
|
|
"num_channels": [[40, 80], [40, 80, 160], [40, 80, 160, 320]],
|
|
},
|
|
"naive": {
|
|
"num_modules": [2, 4, 2],
|
|
"num_branches": [2, 3, 4],
|
|
"num_blocks": [2, 2, 2],
|
|
"module_type": ["NAIVE", "NAIVE", "NAIVE"],
|
|
"reduce_ratios": [1, 1, 1],
|
|
"num_channels": [[30, 60], [30, 60, 120], [30, 60, 120, 240]],
|
|
},
|
|
"wider_naive": {
|
|
"num_modules": [2, 4, 2],
|
|
"num_branches": [2, 3, 4],
|
|
"num_blocks": [2, 2, 2],
|
|
"module_type": ["NAIVE", "NAIVE", "NAIVE"],
|
|
"reduce_ratios": [1, 1, 1],
|
|
"num_channels": [[40, 80], [40, 80, 160], [40, 80, 160, 320]],
|
|
},
|
|
}
|
|
|
|
self.stages_config = self.module_configs[network_type]
|
|
|
|
self.stem = Stem(3, 32, 32, 1)
|
|
num_channels_pre_layer = [32]
|
|
for stage_idx in range(3):
|
|
num_channels = self.stages_config["num_channels"][stage_idx]
|
|
setattr(self, 'transition{}'.format(stage_idx),
|
|
self._make_transition_layer(num_channels_pre_layer,
|
|
num_channels, self.freeze_norm,
|
|
self.norm_decay))
|
|
stage, num_channels_pre_layer = self._make_stage(
|
|
self.stages_config, stage_idx, num_channels, True,
|
|
self.freeze_norm, self.norm_decay)
|
|
setattr(self, 'stage{}'.format(stage_idx), stage)
|
|
self.head_layer = IterativeHead(num_channels_pre_layer, 'bn',
|
|
self.freeze_norm, self.norm_decay)
|
|
|
|
def _make_transition_layer(self,
|
|
num_channels_pre_layer,
|
|
num_channels_cur_layer,
|
|
freeze_norm=False,
|
|
norm_decay=0.):
|
|
num_branches_pre = len(num_channels_pre_layer)
|
|
num_branches_cur = len(num_channels_cur_layer)
|
|
transition_layers = []
|
|
for i in range(num_branches_cur):
|
|
if i < num_branches_pre:
|
|
if num_channels_cur_layer[i] != num_channels_pre_layer[i]:
|
|
transition_layers.append(
|
|
nn.Sequential(
|
|
L.Conv2d(
|
|
num_channels_pre_layer[i],
|
|
num_channels_pre_layer[i],
|
|
kernel_size=3,
|
|
stride=1,
|
|
padding=1,
|
|
groups=num_channels_pre_layer[i],
|
|
bias=False),
|
|
nn.BatchNorm2D(num_channels_pre_layer[i]),
|
|
L.Conv2d(
|
|
num_channels_pre_layer[i],
|
|
num_channels_cur_layer[i],
|
|
kernel_size=1,
|
|
stride=1,
|
|
padding=0,
|
|
bias=False, ),
|
|
nn.BatchNorm2D(num_channels_cur_layer[i]),
|
|
nn.ReLU()))
|
|
else:
|
|
transition_layers.append(None)
|
|
else:
|
|
conv_downsamples = []
|
|
for j in range(i + 1 - num_branches_pre):
|
|
conv_downsamples.append(
|
|
nn.Sequential(
|
|
L.Conv2d(
|
|
num_channels_pre_layer[-1],
|
|
num_channels_pre_layer[-1],
|
|
groups=num_channels_pre_layer[-1],
|
|
kernel_size=3,
|
|
stride=2,
|
|
padding=1,
|
|
bias=False, ),
|
|
nn.BatchNorm2D(num_channels_pre_layer[-1]),
|
|
L.Conv2d(
|
|
num_channels_pre_layer[-1],
|
|
num_channels_cur_layer[i]
|
|
if j == i - num_branches_pre else
|
|
num_channels_pre_layer[-1],
|
|
kernel_size=1,
|
|
stride=1,
|
|
padding=0,
|
|
bias=False, ),
|
|
nn.BatchNorm2D(num_channels_cur_layer[i]
|
|
if j == i - num_branches_pre else
|
|
num_channels_pre_layer[-1]),
|
|
nn.ReLU()))
|
|
transition_layers.append(nn.Sequential(*conv_downsamples))
|
|
return nn.LayerList(transition_layers)
|
|
|
|
def _make_stage(self,
|
|
stages_config,
|
|
stage_idx,
|
|
in_channels,
|
|
multiscale_output,
|
|
freeze_norm=False,
|
|
norm_decay=0.):
|
|
num_modules = stages_config["num_modules"][stage_idx]
|
|
num_branches = stages_config["num_branches"][stage_idx]
|
|
num_blocks = stages_config["num_blocks"][stage_idx]
|
|
reduce_ratio = stages_config['reduce_ratios'][stage_idx]
|
|
module_type = stages_config['module_type'][stage_idx]
|
|
|
|
modules = []
|
|
for i in range(num_modules):
|
|
if not multiscale_output and i == num_modules - 1:
|
|
reset_multiscale_output = False
|
|
else:
|
|
reset_multiscale_output = True
|
|
modules.append(
|
|
LiteHRNetModule(
|
|
num_branches,
|
|
num_blocks,
|
|
in_channels,
|
|
reduce_ratio,
|
|
module_type,
|
|
multiscale_output=reset_multiscale_output,
|
|
with_fuse=True,
|
|
freeze_norm=freeze_norm,
|
|
norm_decay=norm_decay))
|
|
in_channels = modules[-1].in_channels
|
|
return nn.Sequential(*modules), in_channels
|
|
|
|
def forward(self, inputs):
|
|
x = inputs['image']
|
|
dims = x.shape
|
|
if len(dims) == 5:
|
|
x = paddle.reshape(x, (dims[0] * dims[1], dims[2], dims[3],
|
|
dims[4])) # [6, 3, 128, 96]
|
|
|
|
x = self.stem(x)
|
|
y_list = [x]
|
|
for stage_idx in range(3):
|
|
x_list = []
|
|
transition = getattr(self, 'transition{}'.format(stage_idx))
|
|
for j in range(self.stages_config["num_branches"][stage_idx]):
|
|
if transition[j] is not None:
|
|
if j >= len(y_list):
|
|
x_list.append(transition[j](y_list[-1]))
|
|
else:
|
|
x_list.append(transition[j](y_list[j]))
|
|
else:
|
|
x_list.append(y_list[j])
|
|
y_list = getattr(self, 'stage{}'.format(stage_idx))(x_list)
|
|
x = self.head_layer(y_list)
|
|
res = []
|
|
for i, layer in enumerate(x):
|
|
if i == self.freeze_at:
|
|
layer.stop_gradient = True
|
|
if i in self.return_idx:
|
|
res.append(layer)
|
|
return res
|
|
|
|
@property
|
|
def out_shape(self):
|
|
return [
|
|
ShapeSpec(
|
|
channels=self._out_channels[i], stride=self._out_strides[i])
|
|
for i in self.return_idx
|
|
]
|