{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7c186e5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ce0b2a94",
   "metadata": {},
   "outputs": [],
   "source": [
    "datalist = [\n",
    "    22, 54, 1063, 1067, 12, 18, 23, 59, 188, 307, 1043, 1459, 1475, 1489, 1492, 1497, 1503, 4153, 40499, 44125,\n",
    "    44131, 45062, 44157, 1462, 44160, 29, 37, 53, 49, 1504, 1494, 41143, 44126, 40981, 41168, 44091, 44158, 44123, 44090,\n",
    "    40922, 44161, 45714]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "12ddd6e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = [\n",
    "    'train', 'masking', 'shuffling', 'noisemasking', 'scarf', 'rq', \"cm\",\n",
    "    'binshuffling-2', 'binsampling-2', 'binshuffling-3', 'binsampling-3',\n",
    "    'binshuffling-4', 'binsampling-4', 'binshuffling-5', 'binsampling-5',\n",
    "    'binshuffling-6', 'binsampling-6', 'binshuffling-7', 'binsampling-7',\n",
    "    'binshuffling-8', 'binsampling-8', 'binshuffling-9', 'binsampling-9',\n",
    "    'binshuffling-10', 'binsampling-10'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "43e23ce6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1503\n",
      "40922\n",
      "44161\n",
      "39/42\n"
     ]
    }
   ],
   "source": [
    "done = 0\n",
    "result = dict()\n",
    "for k in keys:\n",
    "    result[k] = []\n",
    "for d in datalist:\n",
    "    fname = f'/home/SemiTab/motivation/usable-information-cutmix/model=mlp/train=transform/{d}.npy'\n",
    "    try:\n",
    "        data = np.load(fname, allow_pickle=True).item()\n",
    "        if data[\"train-acc\"] >= 0.95:\n",
    "            for k in keys:\n",
    "                result[k].append(data.get(k, 0))\n",
    "        done += 1\n",
    "    except FileNotFoundError:\n",
    "        print(d)\n",
    "        pass\n",
    "print(f'{done}/{len(datalist)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "f4ec8e1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 1.093\n",
      "masking 0.237\n",
      "shuffling 0.505\n",
      "noisemasking 0.481\n",
      "scarf 0.285\n",
      "rq -2.844\n",
      "cm 0.889\n",
      "binshuffling-2 0.0\n",
      "binsampling-2 0.0\n",
      "binshuffling-3 0.0\n",
      "binsampling-3 0.0\n",
      "binshuffling-4 1.058\n",
      "binsampling-4 1.054\n",
      "binshuffling-5 0.0\n",
      "binsampling-5 0.0\n",
      "binshuffling-6 0.0\n",
      "binsampling-6 0.0\n",
      "binshuffling-7 0.0\n",
      "binsampling-7 0.0\n",
      "binshuffling-8 0.0\n",
      "binsampling-8 0.0\n",
      "binshuffling-9 0.0\n",
      "binsampling-9 0.0\n",
      "binshuffling-10 0.0\n",
      "binsampling-10 0.0\n"
     ]
    }
   ],
   "source": [
    "for k in keys:\n",
    "    print(k, np.round(np.mean(np.ma.masked_invalid(np.array(result[k])), axis=0), 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1164a68a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42/42\n"
     ]
    }
   ],
   "source": [
    "done = 0\n",
    "result = dict()    \n",
    "for k in keys:\n",
    "    result[k] = []\n",
    "for d in datalist:\n",
    "    fname = f'/home/SemiTab/motivation/usable-information/model=mlp/train=transform/{d}.npy'\n",
    "    try:  #mlp 16\n",
    "        data = np.load(fname, allow_pickle=True).item()\n",
    "        for k in keys:\n",
    "            if data.get(f'{k}-acc', 0) >= 0.95:\n",
    "                result[k].append(data[k])\n",
    "        done += 1\n",
    "    except FileNotFoundError:\n",
    "        print(d, end=\" \")\n",
    "        pass\n",
    "    \n",
    "print(f'{done}/{len(datalist)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "02f5da84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 1.058\n",
      "masking 0.138\n",
      "shuffling 0.42\n",
      "noisemasking 0.404\n",
      "scarf 0.052\n",
      "rq -3.264\n",
      "binshuffling-2 0.943\n",
      "binsampling-2 0.957\n",
      "binshuffling-3 1.135\n",
      "binsampling-3 1.114\n",
      "binshuffling-4 1.048\n",
      "binsampling-4 1.034\n",
      "binshuffling-5 1.062\n",
      "binsampling-5 1.055\n",
      "binshuffling-6 1.059\n",
      "binsampling-6 1.063\n",
      "binshuffling-7 1.069\n",
      "binsampling-7 1.06\n",
      "binshuffling-8 1.071\n",
      "binsampling-8 1.071\n",
      "binshuffling-9 1.073\n",
      "binsampling-9 1.063\n",
      "binshuffling-10 1.075\n",
      "binsampling-10 1.073\n"
     ]
    }
   ],
   "source": [
    "for k in result:\n",
    "    print(k, np.round(np.mean((np.array(result[k])), axis=0), 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca996df4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b2a6bbc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a69f2a32",
   "metadata": {},
   "outputs": [],
   "source": [
    "p1 = [np.mean(np.array(result['shuffling']))]\n",
    "p2 = [np.mean(np.array(result['scarf']))]\n",
    "for nb in range(2, 11):\n",
    "    p1.append(np.mean(np.array(result[f'binshuffling-{nb}'])))\n",
    "    p2.append(np.mean(np.array(result[f'binsampling-{nb}'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4675ad28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 15376 characters]\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.figure(figsize=(4, 3))\n",
    "plt.gca().set_axisbelow(True)\n",
    "plt.grid()\n",
    "plt.plot(range(1, 11), p1, marker=\"o\", label=\"Range-limited shuffling\")\n",
    "plt.plot(range(1, 11), p2, marker=\"o\", label=\"Range-limited sampling\")\n",
    "plt.legend()\n",
    "plt.xlabel(\"Number of Ranges\")\n",
    "plt.ylabel(\"Usable Information\")\n",
    "plt.savefig(\"figs/numbins_usableinfo.png\", dpi=300, bbox_inches=\"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40991c6c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
