cnn1.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "language_info": {
   "name": "python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "version": "3.7.6-final"
  },
  "orig_nbformat": 2,
  "file_extension": ".py",
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3,
  "kernelspec": {
   "name": "python37664bit5a6158c51997408fa24508b198f2fb91",
   "display_name": "Python 3.7.6 64-bit"
  }
 },
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional Neural Networks I - Basic Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross-correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def corr2d(X, K):\n",
    "    \"\"\"Compute 2D cross-correlation.\"\"\"\n",
    "    h, w = K.shape\n",
    "    Y = np.zeros((X.shape[0] - h + 1, X.shape[1] - w + 1))\n",
    "    for i in range(Y.shape[0]):\n",
    "        for j in range(Y.shape[1]):\n",
    "            Y[i, j] = (X[i: i + h, j: j + w] * K).sum()\n",
    "    return Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "(2, 2)\n"
    }
   ],
   "source": [
    "X = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]])\n",
    "K = np.array([[0, 1], [2, 3]])\n",
    "o = corr2d(X, K)\n",
    "print(o.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conv1d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "torch.Size([20, 33, 48])\n"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn  # 类用法（Conv1d，大写）\n",
    "\n",
    "m = nn.Conv1d(16, 33, kernel_size=3)\n",
    "input = torch.randn(20, 16, 50)\n",
    "output = m(input)\n",
    "print(output.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conv2d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "torch.Size([1, 1, 27, 27])\n"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn  # 类用法（Conv2d，大写）\n",
    "\n",
    "layer = nn.Conv2d(1, 1, kernel_size=2, stride=1, padding=0)\n",
    "x = torch.rand(1, 1, 28, 28)\n",
    "out = layer.forward(x)\n",
    "print(out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conv3d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "torch.Size([20, 33, 8, 8, 18])\n"
    }
   ],
   "source": [
    "# With square kernels and equal stride\n",
    "m = nn.Conv3d(16, 33, 3, stride=1)\n",
    "# non-square kernels and unequal stride and with padding\n",
    "# m = nn.Conv3d(16, 33, (3, 5, 2), stride=(2, 1, 1), padding=(4, 2, 0))\n",
    "input = torch.randn(20, 16, 10, 10, 20)\n",
    "output = m(input)\n",
    "print(output.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stride & Padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "torch.Size([1, 3, 28, 28])\ntorch.Size([1, 3, 14, 14])\ntorch.Size([1, 3, 14, 14])\n"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn  # 类用法（Conv2d，大写）\n",
    "\n",
    "x = torch.rand(1, 1, 28, 28)\n",
    "\n",
    "layer = nn.Conv2d(1, 3, kernel_size=3, stride=1, padding=1)\n",
    "print(layer.forward(x).shape)  # torch.Size([1, 3, 28, 28])\n",
    "\n",
    "\n",
    "layer = nn.Conv2d(1, 3, kernel_size=3, stride=2, padding=1)\n",
    "print(layer.forward(x).shape)  # torch.Size([1, 3, 14, 14])\n",
    "\n",
    "out = layer(x)\n",
    "print(out.shape)  # torch.Size([1, 3, 14, 14])\n",
    ""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "torch.Size([3, 1, 3, 3])\ntorch.Size([3])\n"
    }
   ],
   "source": [
    "print(layer.weight.shape)\n",
    "print(layer.bias.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "torch.Size([1, 16, 26, 26])\ntorch.Size([1, 16, 14, 14])\n"
    }
   ],
   "source": [
    "import torch\n",
    "from torch.nn import functional as F #  函数用法（conv2d 小写）\n",
    "\n",
    "\"\"\"手动定义卷积核(weight)和偏置\"\"\"\n",
    "w = torch.rand(16, 3, 5, 5)  # 16种3通道的5乘5卷积核\n",
    "b = torch.rand(16)  # 和卷积核种类数保持一致(不同通道共用一个bias)\n",
    "\n",
    "\"\"\"定义输入样本\"\"\"\n",
    "x = torch.randn(1, 3, 28, 28)  # 1张3通道的28乘28的图像\n",
    "\n",
    "\"\"\"2D卷积得到输出\"\"\"\n",
    "out = F.conv2d(x, w, b, stride=1, padding=1)  # 步长为1,外加1圈padding\n",
    "print(out.shape)\n",
    "\n",
    "out = F.conv2d(x, w, b, stride=2, padding=2)  # 步长为2,外加2圈padding\n",
    "print(out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1x1 Convolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "torch.Size([1, 10, 28, 28])\n"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn  # 类用法（Conv2d，大写）\n",
    "\n",
    "x = torch.rand(1, 1, 28, 28)\n",
    "layer = nn.Conv2d(1, 10, kernel_size=1, stride=1, padding=0)\n",
    "\n",
    "out = layer(x)\n",
    "print(out.shape)  # torch.Size([1, 3, 14, 14])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MaxPool2d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "torch.Size([20, 16, 50])\ntorch.Size([20, 7, 24])\ntorch.Size([20, 16, 50])\ntorch.Size([20, 8, 24])\n"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn  \n",
    "\n",
    "m = torch.nn.MaxPool2d(3, stride=2)\n",
    "input = torch.randn(20, 16, 50)\n",
    "print(input.shape)\n",
    "output = m(input)\n",
    "print(output.shape)\n",
    "\n",
    "m = torch.nn.MaxPool2d((2,4), stride=2)\n",
    "input = torch.randn(20, 16, 50)\n",
    "print(input.shape)\n",
    "output = m(input)\n",
    "print(output.shape)\n",
    ""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BatchNorm2d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "torch.Size([2, 3, 4, 4])\n"
    }
   ],
   "source": [
    "b1=torch.nn.BatchNorm2d(3)\n",
    "a=torch.randn(2,3,4,4)\n",
    " \n",
    "c=b1(a)\n",
    "print(c.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "torch.Size([5, 8])\n"
    }
   ],
   "source": [
    "m = nn.Dropout(p=0.2)\n",
    "input = torch.randn(5, 8)\n",
    "output = m(input)\n",
    "print(output.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "tensor([[ 1.9281, -0.6089, -1.0607, -0.1118,  0.7749,  1.6022,  0.0590, -1.8302],\n        [ 0.2979, -1.7233, -0.4147, -0.4191, -1.4895,  0.9159,  0.1905, -0.5955],\n        [-0.3762, -0.3812, -1.0487, -1.5566, -1.6676,  0.0213, -0.3471,  0.0858],\n        [ 0.3586,  0.2265,  2.2237,  0.4313,  1.3910, -1.6948, -1.8769,  0.2513],\n        [ 1.0944,  0.6405, -0.1162, -0.4766, -0.2789,  0.3491, -1.3027, -0.9096]])\n"
    }
   ],
   "source": [
    "print(input )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "tensor([[ 2.4102, -0.7611, -0.0000, -0.1398,  0.9686,  2.0027,  0.0738, -2.2878],\n        [ 0.3723, -2.1542, -0.5184, -0.0000, -1.8618,  1.1449,  0.2382, -0.7443],\n        [-0.4703, -0.0000, -0.0000, -1.9457, -2.0845,  0.0266, -0.4338,  0.0000],\n        [ 0.4482,  0.2831,  2.7796,  0.5392,  1.7388, -0.0000, -0.0000,  0.3142],\n        [ 1.3680,  0.8006, -0.1452, -0.5957, -0.3486,  0.4364, -1.6283, -1.1371]])\n"
    }
   ],
   "source": [
    "print(output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Walk Through"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Device configuration\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# Hyper parameters\n",
    "num_epochs = 5\n",
    "num_classes = 10\n",
    "batch_size = 100\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MNIST dataset\n",
    "train_dataset = torchvision.datasets.MNIST(root='./data/',\n",
    "                                           train=True, \n",
    "                                           transform=transforms.ToTensor(),\n",
    "                                           download=True)\n",
    "\n",
    "test_dataset = torchvision.datasets.MNIST(root='./data/',\n",
    "                                          train=False, \n",
    "                                          transform=transforms.ToTensor())\n",
    "\n",
    "# Data loader\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size, \n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size, \n",
    "                                          shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convolutional neural network (two convolutional layers)\n",
    "class ConvNet(nn.Module):\n",
    "    def __init__(self, num_classes=10):\n",
    "        super(ConvNet, self).__init__()\n",
    "        self.layer1 = nn.Sequential(\n",
    "            nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),\n",
    "            nn.BatchNorm2d(16),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2))\n",
    "        self.layer2 = nn.Sequential(\n",
    "            nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2))\n",
    "        self.fc = nn.Linear(7*7*32, num_classes)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = self.layer1(x)\n",
    "        out = self.layer2(out)\n",
    "        out = out.reshape(out.size(0), -1)\n",
    "        out = self.fc(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ConvNet(num_classes).to(device)\n",
    "\n",
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Epoch [1/5], Step [100/600], Loss: 0.1673\nEpoch [1/5], Step [200/600], Loss: 0.1539\nEpoch [1/5], Step [300/600], Loss: 0.0787\nEpoch [1/5], Step [400/600], Loss: 0.0651\nEpoch [1/5], Step [500/600], Loss: 0.0676\nEpoch [1/5], Step [600/600], Loss: 0.0505\nEpoch [2/5], Step [100/600], Loss: 0.0349\nEpoch [2/5], Step [200/600], Loss: 0.0910\nEpoch [2/5], Step [300/600], Loss: 0.0544\nEpoch [2/5], Step [400/600], Loss: 0.0086\nEpoch [2/5], Step [500/600], Loss: 0.0155\nEpoch [2/5], Step [600/600], Loss: 0.0587\nEpoch [3/5], Step [100/600], Loss: 0.0439\nEpoch [3/5], Step [200/600], Loss: 0.0226\nEpoch [3/5], Step [300/600], Loss: 0.0369\nEpoch [3/5], Step [400/600], Loss: 0.0339\nEpoch [3/5], Step [500/600], Loss: 0.0377\nEpoch [3/5], Step [600/600], Loss: 0.0824\nEpoch [4/5], Step [100/600], Loss: 0.0329\nEpoch [4/5], Step [200/600], Loss: 0.0223\nEpoch [4/5], Step [300/600], Loss: 0.0124\nEpoch [4/5], Step [400/600], Loss: 0.0182\nEpoch [4/5], Step [500/600], Loss: 0.0320\nEpoch [4/5], Step [600/600], Loss: 0.0722\nEpoch [5/5], Step [100/600], Loss: 0.0111\nEpoch [5/5], Step [200/600], Loss: 0.0355\nEpoch [5/5], Step [300/600], Loss: 0.0067\nEpoch [5/5], Step [400/600], Loss: 0.0225\nEpoch [5/5], Step [500/600], Loss: 0.0064\nEpoch [5/5], Step [600/600], Loss: 0.0039\n"
    }
   ],
   "source": [
    "# Train the model\n",
    "total_step = len(train_loader)\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        # Forward pass\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if (i+1) % 100 == 0:\n",
    "            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n",
    "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Test Accuracy of the model on the 10000 test images: 98.98 %\n"
    }
   ],
   "source": [
    "# Test the model\n",
    "model.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "\n",
    "    print('Test Accuracy of the model on the 10000 test images: {} %'.format(100 * correct / total))\n",
    "\n",
    "# Save the model checkpoint\n",
    "torch.save(model.state_dict(), 'model.ckpt')\n",
    "\n",
    "#(last nn.ipynb) Accuracy of the model on the 10000 test images: 82 %"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchsummary import summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "----------------------------------------------------------------\n        Layer (type)               Output Shape         Param #\n================================================================\n            Conv2d-1           [-1, 16, 28, 28]             416\n       BatchNorm2d-2           [-1, 16, 28, 28]              32\n              ReLU-3           [-1, 16, 28, 28]               0\n         MaxPool2d-4           [-1, 16, 14, 14]               0\n            Conv2d-5           [-1, 32, 14, 14]          12,832\n       BatchNorm2d-6           [-1, 32, 14, 14]              64\n              ReLU-7           [-1, 32, 14, 14]               0\n         MaxPool2d-8             [-1, 32, 7, 7]               0\n            Linear-9                   [-1, 10]          15,690\n================================================================\nTotal params: 29,034\nTrainable params: 29,034\nNon-trainable params: 0\n----------------------------------------------------------------\nInput size (MB): 0.00\nForward/backward pass size (MB): 0.47\nParams size (MB): 0.11\nEstimated Total Size (MB): 0.58\n----------------------------------------------------------------\n"
    }
   ],
   "source": [
    "summary(model, (1, 28, 28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "ConvNet(\n  (layer1): Sequential(\n    (0): Conv2d(1, 16, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n    (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (2): ReLU()\n    (3): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n  )\n  (layer2): Sequential(\n    (0): Conv2d(16, 32, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n    (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (2): ReLU()\n    (3): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n  )\n  (fc): Linear(in_features=1568, out_features=10, bias=True)\n)\n"
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}