mini_project_Kaan.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "5a1b1647",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "0b1cc8eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time \n",
    "import torch\n",
    "from torch import Tensor, empty\n",
    "from typing import Optional, Tuple, Union, List\n",
    "import helpers_functional as F\n",
    "from helpers_layer import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f98137d",
   "metadata": {},
   "source": [
    "### Testing of Conv2d Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "cab8ac30",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected.shape: torch.Size([1000, 10, 18, 18])\n",
      "Actual.shape: torch.Size([1000, 10, 18, 18])\n",
      "\n",
      "Computation Time Comparison:\n",
      "\tPytorch Conv2d: \t 0.02079010009765625\n",
      "\tImplemented Conv2d: \t 0.03800606727600098\n",
      "\tConv2d: \t\t 0.03204202651977539\n"
     ]
    }
   ],
   "source": [
    "N = 1000\n",
    "in_channels = 4\n",
    "out_channels = 10\n",
    "kernel_size = (2, 3)\n",
    "padding = (2,3)\n",
    "stride = 2\n",
    "\n",
    "conv_pytorch = torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)\n",
    "conv = Conv2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)\n",
    "x = torch.randn((N, in_channels, 32, 32))\n",
    "\n",
    "t1 = time.time()\n",
    "dummy = conv_pytorch(x)\n",
    "t2 = time.time()\n",
    "expected = conv(x)\n",
    "t3 = time.time()\n",
    "actual = F.conv2d(x, conv.weight, conv.bias, padding=padding, stride=stride)\n",
    "t4 = time.time()\n",
    "actual2 = F.conv2d(x, conv_pytorch.weight, conv_pytorch.bias, padding=padding, stride=stride)\n",
    "\n",
    "print(\"Expected.shape:\", expected.shape)\n",
    "print(\"Actual.shape:\", actual.shape)\n",
    "\n",
    "torch.testing.assert_allclose(actual, expected)\n",
    "torch.testing.assert_allclose(actual2, dummy)\n",
    "\n",
    "print(\"\\nComputation Time Comparison:\")\n",
    "print(\"\\tPytorch Conv2d: \\t\", t2-t1)\n",
    "print(\"\\tImplemented Conv2d: \\t\", t3-t2)\n",
    "print(\"\\tConv2d: \\t\\t\", t4-t3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee045c82",
   "metadata": {},
   "source": [
    "### Testing of ConvTranspose2d Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "0fc46d06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dummy.shape: torch.Size([100, 10, 39, 49])\n",
      "Expected.shape: torch.Size([100, 10, 39, 49])\n",
      "Actual.shape: torch.Size([100, 10, 39, 49])\n",
      "\n",
      "Computation Time Comparison:\n",
      "\tPytorch Conv2d: \t 0.1132659912109375\n",
      "\tImplemented Conv2d: \t 0.18238592147827148\n",
      "\tConv2d: \t\t 0.17034912109375\n"
     ]
    }
   ],
   "source": [
    "N = 100\n",
    "in_channels = 4\n",
    "out_channels = 10\n",
    "kernel_size = (6, 9)\n",
    "dilation = (2,2)\n",
    "padding = (2,0)\n",
    "output_padding = 1\n",
    "stride = 1\n",
    "\n",
    "convtranspose_pytorch = torch.nn.ConvTranspose2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, output_padding=output_padding, dilation=dilation)\n",
    "convtranspose = ConvTranspose2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, output_padding=output_padding, dilation=dilation)\n",
    "x = torch.randn((N, in_channels, 32, 32))\n",
    "\n",
    "t1 = time.time()\n",
    "dummy = convtranspose_pytorch(x)\n",
    "print(\"Dummy.shape:\", dummy.shape)\n",
    "t2 = time.time()\n",
    "expected = convtranspose(x)\n",
    "print(\"Expected.shape:\", expected.shape)\n",
    "t3 = time.time()\n",
    "actual = F.conv_transpose2d(x, convtranspose.weight, convtranspose.bias, padding=padding, stride=stride, dilation=dilation, output_padding=output_padding)\n",
    "print(\"Actual.shape:\", actual.shape)\n",
    "t4 = time.time()\n",
    "actual2 = F.conv_transpose2d(x, convtranspose_pytorch.weight, convtranspose_pytorch.bias, padding=padding, stride=stride, dilation=dilation, output_padding=output_padding)\n",
    "\n",
    "\n",
    "torch.testing.assert_allclose(actual, expected)\n",
    "torch.testing.assert_allclose(actual2, dummy)\n",
    "\n",
    "print(\"\\nComputation Time Comparison:\")\n",
    "print(\"\\tPytorch Conv2d: \\t\", t2-t1)\n",
    "print(\"\\tImplemented Conv2d: \\t\", t3-t2)\n",
    "print(\"\\tConv2d: \\t\\t\", t4-t3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a197d9e",
   "metadata": {},
   "source": [
    "### Testing of Backward of Conv2d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "fe0ecc08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected.shape: torch.Size([1000, 10, 18, 18])\n",
      "torch.Size([1000, 10, 18, 18])\n"
     ]
    }
   ],
   "source": [
    "N = 1000\n",
    "in_channels = 4\n",
    "out_channels = 10\n",
    "kernel_size = (2, 3)\n",
    "padding = (2,3)\n",
    "stride = 2\n",
    "\n",
    "conv_pytorch = torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)\n",
    "conv = Conv2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)\n",
    "x = torch.randn((N, in_channels, 32, 32))\n",
    "\n",
    "dummy = conv_pytorch(x)\n",
    "expected = conv(x)\n",
    "actual = F.conv2d(x, conv.weight, conv.bias, padding=padding, stride=stride)\n",
    "actual2 = F.conv2d(x, conv_pytorch.weight, conv_pytorch.bias, padding=padding, stride=stride)\n",
    "\n",
    "print(\"Expected.shape:\", expected.shape)\n",
    "\n",
    "torch.testing.assert_allclose(actual, expected)\n",
    "torch.testing.assert_allclose(actual2, dummy)\n",
    "\n",
    "grad_wrt_output = torch.ones_like(expected, requires_grad=True)\n",
    "print(grad_wrt_output.shape)\n",
    "\n",
    "#expected_bward_out = conv_pytorch.backward(grad_wrt_output)\n",
    "#print(\"Expected Backward.shape:\", expected_bward_out.shape)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "f43bd353",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size torch.Size([1, 2, 5, 5])\n",
      "torch.Size([1, 2, 5, 5])\n",
      "torch.Size([2, 4, 4, 4])\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "requested an input grad size of [8, 8], but valid sizes range from [6, 7] to [6, 7] (for a grad_output of torch.Size([5, 5]))",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[0;32mIn [54]\u001b[0m, in \u001b[0;36m<cell line: 30>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[38;5;28mprint\u001b[39m(res\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[0;32m---> 31\u001b[0m     grad \u001b[38;5;241m=\u001b[39m \u001b[43mconv_\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgrad_output\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     32\u001b[0m \u001b[38;5;28mprint\u001b[39m(grad\u001b[38;5;241m.\u001b[39msize())\n\u001b[1;32m     34\u001b[0m \u001b[38;5;28minput\u001b[39m\u001b[38;5;241m.\u001b[39mrequires_grad_(\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "File \u001b[0;32m~/Documents/GitHub/DeepLearning/helpers_layer.py:276\u001b[0m, in \u001b[0;36mConv2d.backward\u001b[0;34m(self, grad_output)\u001b[0m\n\u001b[1;32m    274\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput \u001b[38;5;241m==\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    275\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mForward is not implemented, so backward cannot be implemented.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 276\u001b[0m grad_input \u001b[38;5;241m=\u001b[39m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv2d_grad_input\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_shape\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minput\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_output\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgrad_output\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstride\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstride\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpadding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdilation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdilation\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroups\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    277\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgrad_weight \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39mconv2d_grad_weight(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mweight\u001b[38;5;241m.\u001b[39mshape, grad_output, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstride, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdilation, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups)\n\u001b[1;32m    278\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/Documents/GitHub/DeepLearning/helpers_functional.py:148\u001b[0m, in \u001b[0;36mconv2d_grad_input\u001b[0;34m(input_shape, weight, grad_output, stride, padding, dilation, groups)\u001b[0m\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m input_shape \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    146\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconv2d_grad_input requires specifying an input_size\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 148\u001b[0m grad_input_padding \u001b[38;5;241m=\u001b[39m \u001b[43m_grad_input_padding\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgrad_output\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minput_shape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstride\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkernel_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdilation\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    150\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m conv_transpose2d(\u001b[38;5;28minput\u001b[39m\u001b[38;5;241m=\u001b[39mgrad_output, weight\u001b[38;5;241m=\u001b[39mweight, bias\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, stride\u001b[38;5;241m=\u001b[39mstride, padding\u001b[38;5;241m=\u001b[39mpadding, output_padding\u001b[38;5;241m=\u001b[39mgrad_input_padding, groups\u001b[38;5;241m=\u001b[39mgroups, dilation\u001b[38;5;241m=\u001b[39mdilation)\n",
      "File \u001b[0;32m~/Documents/GitHub/DeepLearning/helpers_functional.py:133\u001b[0m, in \u001b[0;36m_grad_input_padding\u001b[0;34m(grad_output, input_size, stride, padding, kernel_size, dilation)\u001b[0m\n\u001b[1;32m    131\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m size, min_size, max_size \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(input_size, min_sizes, max_sizes):\n\u001b[1;32m    132\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m size \u001b[38;5;241m<\u001b[39m min_size \u001b[38;5;129;01mor\u001b[39;00m size \u001b[38;5;241m>\u001b[39m max_size:\n\u001b[0;32m--> 133\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    134\u001b[0m             (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequested an input grad size of \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, but valid sizes range \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    135\u001b[0m              \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m to \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m (for a grad_output of \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mformat(input_size, min_sizes, max_sizes, grad_output\u001b[38;5;241m.\u001b[39msize()[\u001b[38;5;241m2\u001b[39m:]))\n\u001b[1;32m    137\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mtuple\u001b[39m(input_size[d] \u001b[38;5;241m-\u001b[39m min_sizes[d] \u001b[38;5;28;01mfor\u001b[39;00m d \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(k))\n",
      "\u001b[0;31mValueError\u001b[0m: requested an input grad size of [8, 8], but valid sizes range from [6, 7] to [6, 7] (for a grad_output of torch.Size([5, 5]))"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    input = torch.normal(mean = torch.zeros(1,4,8,8),std = 1)\n",
    "\n",
    "    kernel = torch.tensor([4,4])\n",
    "    stride = 1\n",
    "    padding = 0\n",
    "    dilation= 1\n",
    "    groups = 1\n",
    "    in_channels = 4\n",
    "    out_channels = 10\n",
    "\n",
    "    conv = torch.nn.Conv2d(4, 2, kernel, stride=stride, padding=padding, dilation=dilation, groups=groups)\n",
    "    weight = conv.state_dict()[\"weight\"]\n",
    "    bias = conv.state_dict()[\"bias\"]\n",
    "\n",
    "    conv_ = Conv2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)\n",
    "\n",
    "    m = conv.forward(input) \n",
    "    print(\"size\",m.size())\n",
    "    m_ = conv_.forward(input)\n",
    "\n",
    "\n",
    "grad_output = torch.normal(mean=torch.zeros(m.size(0), m.size(1), m.size(-2), m.size(-1)), std= 1)\n",
    "print(grad_output.shape)\n",
    "\n",
    "res = torch.nn.functional.grad.conv2d_weight(input, weight.shape, grad_output, stride = stride, dilation=dilation)\n",
    "\n",
    "print(res.shape)\n",
    "\n",
    "with torch.no_grad():\n",
    "    grad = conv_.backward(grad_output)\n",
    "print(grad.size())\n",
    "\n",
    "input.requires_grad_(True)\n",
    "weight.requires_grad_(True)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a2f8002",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04050c5d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdf89563",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54c5882e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b85a66cc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9301745",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f14eb25",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abf11ddf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e2881f6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}