1. 研究目的了解残差网络ResNet的原理和架构探究残差网络的优势分析残差网络的深度对模型性能的影响实践应用残差网络解决实际问题。2. 研究准备根据GPU安装pytorch版本实现GPU运行研究代码配置环境用来运行 Python、Jupyter Notebook和相关库等相关库。3. 研究内容启动jupyter notebook使用新增的pytorch环境新建ipynb文件为了检查环境配置是否合理输入import torch以及torch.cuda.is_available()若返回TRUE则说明研究环境配置正确若返回False但可以正确导入torch则说明pytorch配置成功但研究运行是在CPU进行的结果如下3.1 残差网络1使用jupyter notebook新增的pytorch环境新建ipynb文件完成基本数据操作的研究代码与练习结果如下import torch from torch import nn from torch.nn import functional as F from d2l import torch as d2l class Residual(nn.Module): #save def init(self, input_channels, num_channels, use_1x1convFalse, strides1): super().init() self.conv1 nn.Conv2d(input_channels, num_channels, kernel_size3, padding1, stridestrides) self.conv2 nn.Conv2d(num_channels, num_channels, kernel_size3, padding1) if use_1x1conv: self.conv3 nn.Conv2d(input_channels, num_channels, kernel_size1, stridestrides) else: self.conv3 None self.bn1 nn.BatchNorm2d(num_channels) self.bn2 nn.BatchNorm2d(num_channels) def forward(self, X): Y F.relu(self.bn1(self.conv1(X))) Y self.bn2(self.conv2(Y)) if self.conv3: X self.conv3(X) Y X return F.relu(Y) blk Residual(3,3) X torch.rand(4, 3, 6, 6) Y blk(X) Y.shapeblk Residual(3,6, use_1x1convTrue, strides2) blk(X).shapeResNet模型b1 nn.Sequential(nn.Conv2d(1, 64, kernel_size7, stride2, padding3), nn.BatchNorm2d(64), nn.ReLU(), nn.MaxPool2d(kernel_size3, stride2, padding1)) def resnet_block(input_channels, num_channels, num_residuals, first_blockFalse): blk [] for i in range(num_residuals): if i 0 and not first_block: blk.append(Residual(input_channels, num_channels, use_1x1convTrue, strides2)) else: blk.append(Residual(num_channels, num_channels)) return blk b2 nn.Sequential(*resnet_block(64, 64, 2, first_blockTrue)) b3 nn.Sequential(*resnet_block(64, 128, 2)) b4 nn.Sequential(*resnet_block(128, 256, 2)) b5 nn.Sequential(*resnet_block(256, 512, 2)) net nn.Sequential(b1, b2, b3, b4, b5, nn.AdaptiveAvgPool2d((1,1)), nn.Flatten(), nn.Linear(512, 10)) X torch.rand(size(1, 1, 224, 224)) for layer in net: X layer(X) print(layer.class.name,output shape:\t, X.shape)训练模型lr, num_epochs, batch_size 0.05, 10, 256 train_iter, test_iter d2l.load_data_fashion_mnist(batch_size, resize96) d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())3.2练习1.图7.4.1中的Inception块与残差块之间的主要区别是什么在删除了Inception块中的一些路径之后它们是如何相互关联的Inception块和残差块Residual block是两种不同的网络模块其主要区别在于它们的结构和连接方式。Inception块是由多个不同大小的卷积核和池化操作组成的它们在不同的分支中并行进行操作然后将它们的输出在通道维度上进行拼接。这种设计可以捕捉不同尺度和层次的特征并且具有较大的感受野从而提高网络的表达能力。残差块Residual block是通过引入跳跃连接skip connection来解决梯度消失问题的一种方式。在残差块中输入通过一个或多个卷积层后与原始输入进行相加操作。这种设计允许信息在网络中直接跳过一些层级使得网络能够更容易地学习残差原始输入与输出之间的差异从而加速训练和改善模型的收敛性。当从Inception块中删除一些路径时它们仍然与其他路径相互关联。删除路径后剩下的路径仍然可以在Inception块中共享信息并通过拼接或连接操作将它们的输出合并起来。这样可以减少模型的计算复杂度和参数量并且有助于防止过拟合。在残差网络ResNet中每个残差块通过跳跃连接将输入直接添加到输出中确保了信息的流动。这种结构使得残差网络能够更深地堆叠层级并且可以训练非常深的神经网络而不会导致梯度消失或退化问题。2.参考ResNet论文 (He et al., 2016)中的表1以实现不同的变体。根据ResNet论文中的表1我们可以实现ResNet的不同变体如ResNet-18、ResNet-34、ResNet-50、ResNet-101和ResNet-152。以下是这些变体的具体实现代码import torch from torch import nn from torch.nn import functional as F from d2l import torch as d2l class Residual(nn.Module): def init(self, input_channels, num_channels, use_1x1convFalse, strides1): super().init() self.conv1 nn.Conv2d(input_channels, num_channels, kernel_size3, padding1, stridestrides) self.conv2 nn.Conv2d(num_channels, num_channels, kernel_size3, padding1) if use_1x1conv: self.conv3 nn.Conv2d(input_channels, num_channels, kernel_size1, stridestrides) else: self.conv3 None self.bn1 nn.BatchNorm2d(num_channels) self.bn2 nn.BatchNorm2d(num_channels) def forward(self, X): Y F.relu(self.bn1(self.conv1(X))) Y self.bn2(self.conv2(Y)) if self.conv3: X self.conv3(X) Y X return F.relu(Y) def resnet_block(input_channels, num_channels, num_residuals, first_blockFalse): blk [] for i in range(num_residuals): if i 0 and not first_block: blk.append(Residual(input_channels, num_channels, use_1x1convTrue, strides2)) else: blk.append(Residual(num_channels, num_channels)) return blk class ResNet(nn.Module): def init(self, num_classes, block_sizes): super().init() self.b1 nn.Sequential(nn.Conv2d(1, 64, kernel_size7, stride2, padding3), nn.BatchNorm2d(64), nn.ReLU(), nn.MaxPool2d(kernel_size3, stride2, padding1)) self.b2 nn.Sequential(*resnet_block(64, 64, block_sizes[0], first_blockTrue)) self.b3 nn.Sequential(*resnet_block(64, 128, block_sizes[1])) self.b4 nn.Sequential(*resnet_block(128, 256, block_sizes[2])) self.b5 nn.Sequential(*resnet_block(256, 512, block_sizes[3])) self.avgpool nn.AdaptiveAvgPool2d((1, 1)) self.flatten nn.Flatten() self.fc nn.Linear(512, num_classes) def forward(self, X): X self.b1(X) X self.b2(X) X self.b3(X) X self.b4(X) X self.b5(X) X self.avgpool(X) X self.flatten(X) X self.fc(X) return X def resnet18(num_classes): return ResNet(num_classes, [2, 2, 2, 2]) def resnet34(num_classes): return ResNet(num_classes, [3, 4, 6, 3]) def resnet50(num_classes): return ResNet(num_classes, [3, 4, 6, 3]) def resnet101(num_classes): return ResNet(num_classes, [3, 4, 23, 3]) Usage example num_classes 10 # Number of output classes net resnet18(num_classes) # Choose the ResNet variant Training lr, num_epochs, batch_size 0.1, 10, 256 train_iter, test_iter d2l.load_data_fashion_mnist(batch_size, resize96) d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())3.对于更深层次的网络ResNet引入了“bottleneck”架构来降低模型复杂性。请试着去实现它。ResNet引入了“bottleneck”架构。在这个架构中每个残差块由一个1x1卷积层、一个3x3卷积层和一个1x1卷积层组成其中1x1卷积层用于减少维度和恢复维度。这样可以显著减少参数数量和计算量。import torch from torch import nn from torch.nn import functional as F from d2l import torch as d2l class Bottleneck(nn.Module): def init(self, input_channels, num_channels, use_1x1convFalse, strides1): super().init() self.conv1 nn.Conv2d(input_channels, num_channels, kernel_size1) self.conv2 nn.Conv2d(num_channels, num_channels, kernel_size3, padding1, stridestrides) self.conv3 nn.Conv2d(num_channels, num_channels * 4, kernel_size1) self.bn1 nn.BatchNorm2d(num_channels) self.bn2 nn.BatchNorm2d(num_channels) self.bn3 nn.BatchNorm2d(num_channels * 4) if use_1x1conv: self.conv4 nn.Conv2d(input_channels, num_channels * 4, kernel_size1, stridestrides) else: self.conv4 None def forward(self, X): Y F.relu(self.bn1(self.conv1(X))) Y F.relu(self.bn2(self.conv2(Y))) Y self.bn3(self.conv3(Y)) if self.conv4: X self.conv4(X) Y X return F.relu(Y) def bottleneck_block(input_channels, num_channels, num_residuals, first_blockFalse): blk [] for i in range(num_residuals): if i 0 and not first_block: blk.append(Bottleneck(input_channels, num_channels, use_1x1convTrue, strides2)) else: blk.append(Bottleneck(num_channels * 4, num_channels)) return blk class ResNet(nn.Module): def init(self, num_classes, block_sizes): super().init() self.b1 nn.Sequential(nn.Conv2d(1, 64, kernel_size7, stride2, padding3), nn.BatchNorm2d(64), nn.ReLU(), nn.MaxPool2d(kernel_size3, stride2, padding1)) self.b2 nn.Sequential(*bottleneck_block(64, 64, block_sizes[0], first_blockTrue)) self.b3 nn.Sequential(*bottleneck_block(256, 128, block_sizes[1])) self.b4 nn.Sequential(*bottleneck_block(512, 256, block_sizes[2])) self.b5 nn.Sequential(*bottleneck_block(1024, 512, block_sizes[3])) self.avgpool nn.AdaptiveAvgPool2d((1, 1)) self.flatten nn.Flatten() self.fc nn.Linear(2048, num_classes) def forward(self, X): X self.b1(X) X self.b2(X) X self.b3(X) X self.b4(X) X self.b5(X) X self.avgpool(X) X self.flatten(X) X self.fc(X) return X def resnet50(num_classes): return ResNet(num_classes, [3, 4, 6, 3]) def resnet101(num_classes): return ResNet(num_classes, [3, 4, 23, 3]) def resnet152(num_classes): return ResNet(num_classes, [3, 8, 36, 3]) Usage example num_classes 10 # Number of output classes net resnet50(num_classes) # Choose the ResNet variant Training lr, num_epochs, batch_size 0.1, 10, 256 train_iter, test_iter d2l.load_data_fashion_mnist(batch_size, resize96) d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())4.在ResNet的后续版本中作者将“卷积层、批量规范化层和激活层”架构更改为“批量规范化层、激活层和卷积层”架构。请尝试做这个改进。详见 (He et al., 2016)中的图1在ResNet的后续版本中作者将“卷积层、批量规范化层和激活层”架构更改为“批量规范化层、激活层和卷积层”架构。这种改进可以提高训练的稳定性和收敛速度。以下是将ResNet的层结构改为“批量规范化层、激活层和卷积层”架构的代码实现import torch from torch import nn from torch.nn import functional as F from d2l import torch as d2l class Bottleneck(nn.Module): def init(self, input_channels, num_channels, use_1x1convFalse, strides1): super().init() self.bn1 nn.BatchNorm2d(input_channels) self.conv1 nn.Conv2d(input_channels, num_channels, kernel_size1) self.bn2 nn.BatchNorm2d(num_channels) self.conv2 nn.Conv2d(num_channels, num_channels, kernel_size3, padding1, stridestrides) self.bn3 nn.BatchNorm2d(num_channels) self.conv3 nn.Conv2d(num_channels, num_channels * 4, kernel_size1) if use_1x1conv: self.conv4 nn.Conv2d(input_channels, num_channels * 4, kernel_size1, stridestrides) else: self.conv4 None def forward(self, X): Y F.relu(self.bn1(X)) Y self.conv1(Y) Y F.relu(self.bn2(Y)) Y self.conv2(Y) Y F.relu(self.bn3(Y)) Y self.conv3(Y) if self.conv4: X self.conv4(X) Y X return Y def bottleneck_block(input_channels, num_channels, num_residuals, first_blockFalse): blk [] for i in range(num_residuals): if i 0 and not first_block: blk.append(Bottleneck(input_channels, num_channels, use_1x1convTrue, strides2)) else: blk.append(Bottleneck(num_channels * 4, num_channels)) return blk class ResNet(nn.Module): def init(self, num_classes, block_sizes): super().init() self.b1 nn.Sequential(nn.Conv2d(1, 64, kernel_size7, stride2, padding3), nn.BatchNorm2d(64), nn.ReLU(), nn.MaxPool2d(kernel_size3, stride2, padding1)) self.b2 nn.Sequential(*bottleneck_block(64, 64, block_sizes[0], first_blockTrue)) self.b3 nn.Sequential(*bottleneck_block(256, 128, block_sizes[1])) self.b4 nn.Sequential(*bottleneck_block(512, 256, block_sizes[2])) self.b5 nn.Sequential(*bottleneck_block(1024, 512, block_sizes[3])) self.avgpool nn.AdaptiveAvgPool2d((1, 1)) self.flatten nn.Flatten() self.fc nn.Linear(2048, num_classes) def forward(self, X): X self.b1(X) X self.b2(X) X self.b3(X) X self.b4(X) X self.b5(X) X self.avgpool(X) X self.flatten(X) X self.fc(X) return X def resnet50(num_classes): return ResNet(num_classes, [3, 4, 6, 3]) def resnet101(num_classes): return ResNet(num_classes, [3, 4, 23, 3]) def resnet152(num_classes): return ResNet(num_classes, [3, 8, 36, 3]) Usage example num_classes 10 # Number of output classes net resnet50(num_classes) # Choose the ResNet variant Training lr, num_epochs, batch_size 0.1, 10, 256 train_iter, test_iter d2l.load_data_fashion_mnist(batch_size, resize96) d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())5.为什么即使函数类是嵌套的我们仍然要限制增加函数的复杂性呢限制函数的复杂性有几个原因可读性和可维护性随着函数的复杂性增加函数的代码可能变得冗长、难以理解和难以维护。函数的目标是封装特定的功能使代码更具可读性和可维护性。如果一个函数过于复杂它可能会变得难以理解导致困惑和错误。可重用性函数的目标之一是促进代码的重用。通过将代码封装在函数中可以在不同的上下文中多次使用。然而当函数变得过于复杂时其可重用性可能会下降。复杂的函数可能包含过多的逻辑和依赖关系使其难以在其他上下文中重用。可测试性函数的复杂性会增加测试的难度。当函数包含大量的逻辑和依赖关系时编写相应的测试用例和确保代码的正确性变得更加困难。通过限制函数的复杂性可以使函数更容易进行单元测试并提高代码的可靠性。4. 研究体会本次实验完成后我对残差网络ResNet的理解较之前更为深入主要收获如下1. 原理更清晰。残差网络借助跳跃连接和残差块有效解决了深层网络的梯度消失问题使训练过程更加稳定模型表达能力也得到增强。2. 优势更明显。与传统卷积网络相比残差网络结构更深能够提取更丰富的特征在复杂任务上表现更优训练收敛速度也更快。3. 深度需适度。实验结果表明网络深度增加可提升性能但过深会引发退化问题。因此需要在深度与性能之间寻求平衡选择合理的网络配置。4. 应用效果好。将残差网络应用于图像分类、目标检测等任务均取得了良好效果充分体现了其强大的性能优势。