{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading training and test data\n",
    "\n",
    "import torch\n",
    "data_tr = torch.load(\"training-test-data/training_data.txt\")\n",
    "target_tr = torch.load(\"training-test-data/training_targets.txt\")\n",
    "data_test = torch.load(\"training-test-data/test_data.txt\")\n",
    "target_test = torch.load(\"training-test-data/test_targets.txt\")\n",
    "\n",
    "num_samples = 512 # number of training samples\n",
    "num_samples_test = 100 # number of test samples\n",
    "new_dim1 = 32 * 1 # first dimension\n",
    "new_dim2 = 32 * 1 # second dimension\n",
    "old_dim = 32 # CIFAR original dimension\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "K9RwhEkwwe35"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.colors import LogNorm\n",
    "import numpy as np\n",
    "\n",
    "# Two-layer linear convolutional neural network\n",
    "output_channels = 1\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, ker_size1, ker_size2, output_channels):\n",
    "        super(Net, self).__init__()\n",
    "        self.ker_size1 = ker_size1\n",
    "        self.ker_size2 = ker_size2\n",
    "        self.output_channels = output_channels\n",
    "        self.conv1 = nn.Conv2d(3, output_channels, kernel_size=(self.ker_size1, self.ker_size2), bias=False) \n",
    "        self.fc1 = nn.Linear(int(new_dim1 * new_dim2 * output_channels), 1, bias=False)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        y1 = F.pad(x, (0,self.ker_size2-1,0,self.ker_size1-1), mode='circular') # Circular padding \n",
    "        y1 = self.conv1(y1)\n",
    "        # y1 = F.relu(y1) # ReLU activations\n",
    "        y1 = y1.reshape(y1.size(0), -1)\n",
    "        y1 = self.fc1(y1) \n",
    "        return y1\n",
    "\n",
    "    def initialize(self, initialization_scale, ker_size1):\n",
    "        print(\"init\")\n",
    "        self.fc1.weight.data.mul_(initialization_scale)\n",
    "        self.conv1.weight.data.mul_(initialization_scale)\n",
    "\n",
    "\n",
    "output = torch.zeros((num_samples, 1))\n",
    "output = output.float()\n",
    "output_test = torch.zeros((num_samples_test, 1))\n",
    "output_test = output.float()\n",
    "\n",
    "\n",
    "# Batch gradient descent\n",
    "def train_minibatch(network, optimizer):\n",
    "    minibatch_size = 32\n",
    "    num_batch = int(num_samples/minibatch_size)\n",
    "    for i in range(num_batch):\n",
    "        network.train()\n",
    "        optimizer.zero_grad()\n",
    "        start_index = i * minibatch_size\n",
    "        end_index = start_index + minibatch_size\n",
    "        output = network(data_tr[start_index:end_index])\n",
    "        loss = torch.sum(torch.exp(-1 * torch.mul(output.flatten(), target_tr[start_index:end_index]))) / minibatch_size\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "# Evaluate training data loss\n",
    "def train_eval(network):\n",
    "    network.eval()\n",
    "    train_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        output = network(data_tr)\n",
    "        train_loss = torch.sum(torch.exp(-1 * torch.mul(output.flatten(), target_tr)))\n",
    "        pred = output.apply_(lambda x: 1 if x > 0 else -1)\n",
    "        correct += pred.eq(target_tr.data.view_as(pred)).sum()\n",
    "    train_loss /= num_samples\n",
    "    print('\\nTrain set: Avg. loss: {:.9f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "    train_loss, correct, num_samples,\n",
    "    100. * correct / num_samples))\n",
    "    return train_loss\n",
    "\n",
    "def test(network):\n",
    "    network.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        output_test = network(data_test)\n",
    "        test_loss = torch.sum(torch.exp(-1 * torch.mul(output_test.flatten(), target_test)))\n",
    "        pred = output_test.apply_(lambda x: 1 if x > 0 else -1)\n",
    "        correct += pred.eq(target_test.data.view_as(pred)).sum()\n",
    "    test_loss /= num_samples_test\n",
    "    accuracy = 100. * correct / num_samples_test\n",
    "    losses = test_loss\n",
    "    return (accuracy, losses)\n",
    "\n",
    "from matplotlib.colors import LogNorm\n",
    "\n",
    "\n",
    "# Get the information about beta\n",
    "def extract_info(network, show_photo): \n",
    "\n",
    "  # Compute beta for linear CNNs\n",
    "    betas = []\n",
    "    \n",
    "    print(\"new\")\n",
    "  \n",
    "  # Compute margin\n",
    "    with torch.no_grad():\n",
    "        network.eval()\n",
    "        output_np = np.ndarray.flatten(network(data_tr).data.numpy())\n",
    "        target_np = np.ndarray.flatten(target_tr.data.numpy())\n",
    "        margins = [target_np[i] * output_np[i] for i in range(num_samples)]\n",
    "        min_margin = min(margins) # get the minimum margin for any datapoint \n",
    "        \n",
    "    for c in range(3):\n",
    "        beta_test = np.zeros((new_dim1,new_dim2))\n",
    "        for i in range(new_dim1):\n",
    "            for j in range(new_dim2):\n",
    "                tempimg = torch.zeros((1,3,new_dim1, new_dim2))\n",
    "                tempimg[0,c,i,j]=1\n",
    "                beta_test[i,j] = network(tempimg)  \n",
    "        betas.append(beta_test / min_margin)\n",
    "\n",
    "    # Compute R(beta)\n",
    "    w1 = network.conv1.weight.detach().numpy()\n",
    "    w2 = network.fc1.weight.detach().numpy()\n",
    "    w1_norm_sq = np.sum(np.square(w1))\n",
    "    w2_norm_sq = np.sum(np.square(w2))\n",
    "    print(w1_norm_sq, w2_norm_sq)\n",
    "    Rbeta = (np.sum(np.square(w1)) + np.sum(np.square(w2))) * np.sqrt(new_dim1 * new_dim2)\n",
    "    Rbeta = Rbeta / min_margin\n",
    "\n",
    "    print(\"Rbeta: \" + str(Rbeta))\n",
    "\n",
    "    if show_photo:\n",
    "        sns.set_context(\"paper\",font_scale=3,rc={\"lines.linewidth\":1})\n",
    "        fig, axs = plt.subplots(2, 3, figsize=(20,10))\n",
    "        for c in range(3):\n",
    "            axs[0, c].imshow(np.absolute(betas[c]),cmap='gray')\n",
    "            axs[1, c].imshow(np.absolute(np.fft.fft2(betas[c], norm='ortho')), cmap='gray', norm=LogNorm(vmin=0.0001, vmax=0.08))\n",
    "        plt.show()\n",
    "    return (Rbeta, betas)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "a4c968872c9d49f6b7099e8d48c4913e",
      "796d5bfe0d704163ae96e7a9c347f279",
      "2534b3e353a6473a87917a08616b2aaa",
      "5506f04202b44e04afdb4b83ab772e2c",
      "f3d9ae4b61824662a1e781ed204b50b7",
      "56752e5dde164084b1760e22101742fe",
      "b74fc411c3b24dc8a16a104f6875911b",
      "b6350e661ba642c88cbf75a06bb402cd"
     ]
    },
    "id": "KmD9SySOwqkC",
    "outputId": "5c898a24-0fae-469d-e760-21ec88ab8d91"
   },
   "outputs": [],
   "source": [
    "# Train and extract info about beta\n",
    "import seaborn as sns\n",
    "n_epochs = 100000\n",
    "momentum = 0.3\n",
    "initialization_scale = 0.001\n",
    "\n",
    "learning_rate_start = 0.005\n",
    "\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "def experiment(ker_size1, ker_size2, output_channels):\n",
    "    network = Net(ker_size1, ker_size2, output_channels)\n",
    "    network.initialize(initialization_scale, ker_size1)\n",
    "    optimizer =  optim.SGD(network.parameters(), lr=learning_rate_start, momentum=momentum)\n",
    "    print(\"Before training:\")\n",
    "    train_eval(network)\n",
    "    print(\"Start training:\")\n",
    "    for epoch in tqdm(range(1, n_epochs + 1)):\n",
    "        train_minibatch(network, optimizer)\n",
    "        if epoch % 100 == 0:\n",
    "            loss = train_eval(network)\n",
    "            if loss <= 0.000001: # stop at 10^-6 loss \n",
    "                break\n",
    "\n",
    "        if epoch % 1000 == 0:\n",
    "            print(test(network))\n",
    "\n",
    "        if epoch == 200 == 0:\n",
    "            optimizer =  optim.SGD(network.parameters(), lr=0.01, momentum=momentum)\n",
    "\n",
    "        if epoch == 500:\n",
    "            optimizer = optim.SGD(network.parameters(), lr=0.01, momentum=momentum)\n",
    "\n",
    "        if epoch == 1000:\n",
    "            optimizer = optim.SGD(network.parameters(), lr=0.05, momentum=momentum)\n",
    "\n",
    "        if epoch == 1200:\n",
    "            optimizer = optim.SGD(network.parameters(), lr=0.1, momentum=momentum)\n",
    "\n",
    "        if epoch == 1500:\n",
    "            optimizer = optim.SGD(network.parameters(), lr=0.5, momentum=momentum)\n",
    "\n",
    "        if epoch == 2000:\n",
    "            optimizer = optim.SGD(network.parameters(), lr=1, momentum=momentum)\n",
    "\n",
    "        if epoch == 3000:\n",
    "            optimizer = optim.SGD(network.parameters(), lr=2, momentum=momentum)\n",
    "\n",
    "        if epoch == 4000:\n",
    "            optimizer = optim.SGD(network.parameters(), lr=4, momentum=momentum)\n",
    "\n",
    "        if epoch == 4500:\n",
    "            optimizer = optim.SGD(network.parameters(), lr=10, momentum=momentum)\n",
    "\n",
    "        if epoch == 5000:\n",
    "            optimizer = optim.SGD(network.parameters(), lr=20, momentum=momentum)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    print(\"After training:\")\n",
    "    train_eval(network)\n",
    "    (accuracy, losses) = test(network)\n",
    "    print(accuracy, losses)\n",
    "\n",
    "    (rk, beta) = extract_info(network, True)\n",
    "    return (rk, beta)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "import pandas as pd\n",
    "        \n",
    "\n",
    "Cout = [1, 2, 3, 4, 8]\n",
    "\n",
    "\n",
    "def run_experiment_series(k):\n",
    "    pairs = []\n",
    "    for c in Cout:\n",
    "        pairs.append((k, c))\n",
    "    betas = []\n",
    "    rbetas = []\n",
    "    for (k, output_channels) in pairs:\n",
    "        print(k, output_channels)\n",
    "        (Rbeta, beta) = experiment(k, k, output_channels)\n",
    "        rbetas.append(Rbeta)\n",
    "        betas.append(beta)\n",
    "\n",
    "    # Write betas\n",
    "    for i in range(len(pairs)):\n",
    "        rbetas_to_write = []\n",
    "        index = i\n",
    "        rbetas_to_write.append(rbetas[index])\n",
    "        beta_array = betas[index]\n",
    "        for j in range(3):\n",
    "            beta = beta_array[j]\n",
    "            name = \"experiments-data/\" + \"beta\" + str(pairs[i]) + \"inputchannel\" + str(j) + \".csv\"\n",
    "            print(str(pairs[i]))\n",
    "            pd.DataFrame(beta).to_csv(name, header=False, index=False)\n",
    "        name = \"experiments-data/\" + \"rbetas\" + str(pairs[i]) + \"inputchannel\" + \".csv\"\n",
    "        pd.DataFrame(rbetas_to_write).to_csv(name, header=False, index=False)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 3\n",
      "init\n",
      "Before training:\n",
      "\n",
      "Train set: Avg. loss: 1.000000000, Accuracy: 300/512 (59%)\n",
      "\n",
      "Start training:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-12-7856f034cddc>:22: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  for epoch in tqdm(range(1, n_epochs + 1)):\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b65a81fe753d433a9768e86b91b5c5f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Train set: Avg. loss: 0.113895565, Accuracy: 509/512 (99%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.025383567, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.012153758, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.007546170, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.005319965, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.003206406, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.002235669, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.001692460, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.001349587, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.001115381, Accuracy: 512/512 (100%)\n",
      "\n",
      "(tensor(68.), tensor(9.2030e+11))\n",
      "\n",
      "Train set: Avg. loss: 0.000567174, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000370499, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000211677, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000145650, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000110070, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000046570, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000028918, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000020785, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000016134, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000013135, Accuracy: 512/512 (100%)\n",
      "\n",
      "(tensor(68.), tensor(4.5752e+22))\n",
      "\n",
      "Train set: Avg. loss: 0.000009492, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000007388, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000006028, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000005078, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000004380, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000003844, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000003422, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000003081, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000002799, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000002563, Accuracy: 512/512 (100%)\n",
      "\n",
      "(tensor(68.), tensor(3.9903e+26))\n",
      "\n",
      "Train set: Avg. loss: 0.000002189, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001906, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001687, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001511, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001368, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001248, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001147, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001061, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000000986, Accuracy: 512/512 (100%)\n",
      "\n",
      "After training:\n",
      "\n",
      "Train set: Avg. loss: 0.000000986, Accuracy: 512/512 (100%)\n",
      "\n",
      "tensor(68.) tensor(7.9933e+28)\n",
      "new\n",
      "21.416676 21.038883\n",
      "Rbeta: 123.56390024765305\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 50452 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 3)\n",
      "(20, 3)\n",
      "(20, 3)\n",
      "32 3\n",
      "init\n",
      "Before training:\n",
      "\n",
      "Train set: Avg. loss: 1.000000000, Accuracy: 232/512 (45%)\n",
      "\n",
      "Start training:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f62e2e873509407eb60a533e440cf5d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Train set: Avg. loss: 0.371342719, Accuracy: 479/512 (94%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.025484797, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.011243817, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.006875360, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.004835492, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.002926824, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.002035495, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.001542505, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.001232414, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.001020683, Accuracy: 512/512 (100%)\n",
      "\n",
      "(tensor(70.), tensor(2.5963e+09))\n",
      "\n",
      "Train set: Avg. loss: 0.000520084, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000336448, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000192542, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000131870, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000099721, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000041900, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000025727, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000018522, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000014427, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000011787, Accuracy: 512/512 (100%)\n",
      "\n",
      "(tensor(67.), tensor(1.0672e+18))\n",
      "\n",
      "Train set: Avg. loss: 0.000008574, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000006678, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000005457, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000004605, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000003978, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000003497, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000003117, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000002809, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000002555, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000002342, Accuracy: 512/512 (100%)\n",
      "\n",
      "(tensor(66.), tensor(1.3643e+21))\n",
      "\n",
      "Train set: Avg. loss: 0.000002004, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001746, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001546, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001386, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001255, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001146, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000001054, Accuracy: 512/512 (100%)\n",
      "\n",
      "\n",
      "Train set: Avg. loss: 0.000000975, Accuracy: 512/512 (100%)\n",
      "\n",
      "After training:\n",
      "\n",
      "Train set: Avg. loss: 0.000000975, Accuracy: 512/512 (100%)\n",
      "\n",
      "tensor(67.) tensor(6.3562e+22)\n",
      "new\n",
      "20.387196 18.973251\n",
      "Rbeta: 114.84764127173365\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 41852 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32, 3)\n",
      "(32, 3)\n",
      "(32, 3)\n"
     ]
    }
   ],
   "source": [
    "k_vals = [1, 3, 8, 20, 32]\n",
    "for k in k_vals:\n",
    "    run_experiment_series(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "InductiveBias-Submitted-Code-CIFAR",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0b02e26762624c07a6cd5c9c781ae640": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0d3b0d66adf642198c1090e411f2f794": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0b02e26762624c07a6cd5c9c781ae640",
      "max": 170498071,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_49aaabfb7b34447eb4c5ecd9bf64e2c8",
      "value": 170498071
     }
    },
    "22a30967752949c1b8698575cf87ad13": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0d3b0d66adf642198c1090e411f2f794",
       "IPY_MODEL_3440e111712b4660a8bbc352abd06be7"
      ],
      "layout": "IPY_MODEL_33d52a20f901450d928c225ca3761032"
     }
    },
    "2534b3e353a6473a87917a08616b2aaa": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "",
      "description": "  4%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_56752e5dde164084b1760e22101742fe",
      "max": 100000,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f3d9ae4b61824662a1e781ed204b50b7",
      "value": 3538
     }
    },
    "33d52a20f901450d928c225ca3761032": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3440e111712b4660a8bbc352abd06be7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ee3d0af038194bdcaa854265f0042052",
      "placeholder": "​",
      "style": "IPY_MODEL_cd1fdc2cdf324d0b9a903544cd163513",
      "value": " 170499072/? [00:06&lt;00:00, 24419846.20it/s]"
     }
    },
    "49aaabfb7b34447eb4c5ecd9bf64e2c8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "5506f04202b44e04afdb4b83ab772e2c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b6350e661ba642c88cbf75a06bb402cd",
      "placeholder": "​",
      "style": "IPY_MODEL_b74fc411c3b24dc8a16a104f6875911b",
      "value": " 3538/100000 [02:40&lt;1:12:04, 22.31it/s]"
     }
    },
    "56752e5dde164084b1760e22101742fe": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "796d5bfe0d704163ae96e7a9c347f279": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a4c968872c9d49f6b7099e8d48c4913e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_2534b3e353a6473a87917a08616b2aaa",
       "IPY_MODEL_5506f04202b44e04afdb4b83ab772e2c"
      ],
      "layout": "IPY_MODEL_796d5bfe0d704163ae96e7a9c347f279"
     }
    },
    "b6350e661ba642c88cbf75a06bb402cd": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b74fc411c3b24dc8a16a104f6875911b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "cd1fdc2cdf324d0b9a903544cd163513": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ee3d0af038194bdcaa854265f0042052": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f3d9ae4b61824662a1e781ed204b50b7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
