{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6ca805d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2b5d0f8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "datalist = [\n",
    "    22, 54, 1063, 1067, 12, 18, 23, 59, 188, 307, 1043, 1459, 1475, 1489, 1492, 1497, 1503, 4153, 40499, 44125,\n",
    "    44131, 45062, 44157, 1462, 44160, 29, 37, 53, 49, 1504, 1494, 41143, 44126, 40981, 41168, 44091, 44158, 44123, 44090,\n",
    "    40922, 44161, 45714]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "64f52714",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = pd.read_csv(\"/home/SemiTab/result-summary/result.csv\", index_col=0)\n",
    "result = result[result[\"model\"].str.startswith(\"pseudolabel\")].reset_index(drop=True)\n",
    "result = result[[\"trial\", \"shots\", \"data_id\", \"model\", \"acc\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "67738454",
   "metadata": {},
   "outputs": [],
   "source": [
    "def update(trial):\n",
    "    result = pd.read_csv(\"/home/SemiTab/result-summary/result.csv\", index_col=0)\n",
    "    result = result[result[\"model\"].str.startswith(\"pseudolabel\")].reset_index(drop=True)\n",
    "    result = result[[\"trial\", \"shots\", \"data_id\", \"model\", \"acc\"]]\n",
    "    i = len(result)\n",
    "    print(result.shape)\n",
    "    print(\"==============================\")\n",
    "    print(trial)\n",
    "    for aug in [\"masking\", \"rq\", \"binshuffling\", \"binsampling\"]:\n",
    "        print(aug)\n",
    "        model = f'pseudolabel-{aug}'\n",
    "        for shot in [1, 5]:\n",
    "            print(\"SHOT\", shot)\n",
    "            for d in datalist:\n",
    "                if len(result[(result[\"shots\"] == shot) & (result[\"trial\"] == trial) & (\n",
    "                    result[\"model\"] == model) & (result[\"data_id\"] == d)]) > 0:\n",
    "                    pass\n",
    "                else:\n",
    "                    fname = f'/home/SemiTab/results/seed={trial}/shot={shot}/model={model}/data={d}/performance.npy'\n",
    "                    if os.path.exists(fname):\n",
    "                        try:\n",
    "                            perf = np.load(fname, allow_pickle=True)\n",
    "                            perf = perf.item()\n",
    "                            result.loc[i] = [trial, shot, d, model, perf[\"Test\"][0]]\n",
    "                            i += 1\n",
    "                        except Exception:\n",
    "                            os.remove(fname)\n",
    "                            print(d, end=\" \")\n",
    "                    else:\n",
    "                        print(d, end=\" \")\n",
    "                        pass\n",
    "            print(\" \")\n",
    "    result.to_csv(\"/home/SemiTab/result-summary/result-semi.csv\")\n",
    "    print(result.shape)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3475127",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "259dcdd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def UpdateResults(model, seeds, shot, d):\n",
    "    result = pd.read_csv(\"/home/SemiTab/result-summary/result-semi.csv\", index_col=0)\n",
    "    for seed in seeds:\n",
    "        fname = f'/home/SemiTab/results/seed={seed}/shot={shot}/model={model}/data={d}/performance.npy'\n",
    "        try:\n",
    "            perf = np.load(fname, allow_pickle=True).item()\n",
    "            if model.startswith(\"ssl\"):\n",
    "                for e in [\"lr\", \"knn\", \"lineareval\", \"finetuning\"]:\n",
    "                    result.loc[(result[\"trial\"] == seed) & (result[\"shots\"] == shot) & (result[\"data_id\"] == d) & (result[\"model\"] == f'{model}-{e}'), \"acc\"] = perf[\"Test\"][e][0]\n",
    "                    i += 1\n",
    "            else:\n",
    "                result.loc[(result[\"trial\"] == seed) & (result[\"shots\"] == shot) & (result[\"data_id\"] == d) & (result[\"model\"] == model), \"acc\"] = perf[\"Test\"][0]\n",
    "                i += 1\n",
    "        except Exception:\n",
    "#             print(\"###\", d, shot)\n",
    "            pass\n",
    "    result.to_csv(\"/home/SemiTab/result-summary/result-semi.csv\")\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "cd364f7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['pseudolabel-masking', 'pseudolabel-shuffling',\n",
       "       'pseudolabel-noisemasking', 'pseudolabel-rq',\n",
       "       'pseudolabel-sampling', 'pseudolabel-binshuffling',\n",
       "       'pseudolabel-binsampling'], dtype=object)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = pd.read_csv(\"/home/SemiTab/result-summary/result-semi.csv\", index_col=0)\n",
    "result['model'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "13d19a2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in [\"pseudolabel-cutmix\"]:\n",
    "    for d in datalist:\n",
    "        result = UpdateResults(model, range(10), 5, d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "9e7d79f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['pseudolabel-masking', 'pseudolabel-shuffling',\n",
       "       'pseudolabel-noisemasking', 'pseudolabel-rq',\n",
       "       'pseudolabel-sampling', 'pseudolabel-binshuffling',\n",
       "       'pseudolabel-binsampling'], dtype=object)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result['model'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f7b76a7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = []\n",
    "for s in range(10):\n",
    "    d = pd.read_csv(f'/home/SemiTab/result-summary/result-{s}.csv', index_col=0)\n",
    "    a.append(d[(d[\"model\"] == \"pseudolabel-cutmix\") & (d[\"shots\"] == 5)][\"acc\"].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "6e77aa93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5310880350564428"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "655a9c48",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0e2fa4b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['pseudolabel-masking', 'pseudolabel-shuffling',\n",
       "       'pseudolabel-noisemasking', 'pseudolabel-rq',\n",
       "       'pseudolabel-sampling'], dtype=object)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[\"model\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5f38208",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "c8e946b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# result = pd.read_csv(\"/home/SemiTab/result-summary/result-semi.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "faec2d28",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = result[result[\"trial\"] < 10]\n",
    "p = result.groupby([\"data_id\", \"model\", \"shots\"]).agg(\n",
    "    acc_mean=('acc', 'mean'), acc_std=('acc', 'std')).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c6953b44",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "rankdata = copy.deepcopy(p)\n",
    "\n",
    "for shot in [1, 5]:\n",
    "    for data in datalist:\n",
    "        q = p[(p[\"data_id\"] == data) & (p[\"shots\"] == shot)]\n",
    "        q = q.reset_index(drop=True)\n",
    "    \n",
    "        q['rank'] = q[\"acc_mean\"].rank(ascending=False, method=\"min\")\n",
    "\n",
    "        for i, row in q.iterrows():\n",
    "            rankdata.loc[(rankdata[\"data_id\"] == row[\"data_id\"]) & \n",
    "                         (rankdata[\"shots\"] == shot) & \n",
    "                         (rankdata[\"acc_mean\"] == row[\"acc_mean\"]), 'rank'] = row['rank']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd6194dd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a64821b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.108640\n",
       "1    0.160623\n",
       "2    0.108059\n",
       "3    0.109731\n",
       "4    0.090949\n",
       "5    0.102748\n",
       "6    0.105326\n",
       "Name: acc_mean, dtype: float64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q[\"acc_mean\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "0445bed4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# p[p[\"shot\"] == 1].groupby(\"aug\")[\"acc_mean\"].mean() \n",
    "q = rankdata[rankdata[\"shots\"] == 5].groupby(\"model\")[\"acc_mean\"].mean().reset_index()  # 0.435135\n",
    "# rankdata[rankdata[\"shots\"] == 5].groupby(\"model\")[\"acc_mean\"].mean() \n",
    "# p[p[\"shot\"] == 1].sort_values(\"acc_mean\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "73b25791",
   "metadata": {},
   "outputs": [],
   "source": [
    "# q\n",
    "q = q.iloc[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "4ba2e437",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['pseudolabel-binshuffling', 'pseudolabel-masking',\n",
       "       'pseudolabel-noisemasking', 'pseudolabel-rq',\n",
       "       'pseudolabel-sampling', 'pseudolabel-shuffling'], dtype=object)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q[\"model\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "0d39c59a",
   "metadata": {},
   "outputs": [],
   "source": [
    "q = q.reset_index(drop=True)\n",
    "q.loc[len(q)] = [\"pseudolabel-cutmix\", np.mean(a)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "0b09d0d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>acc_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pseudolabel-binshuffling</td>\n",
       "      <td>0.554644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pseudolabel-masking</td>\n",
       "      <td>0.548433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pseudolabel-noisemasking</td>\n",
       "      <td>0.529887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pseudolabel-rq</td>\n",
       "      <td>0.536668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pseudolabel-sampling</td>\n",
       "      <td>0.527020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>pseudolabel-shuffling</td>\n",
       "      <td>0.529214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>pseudolabel-cutmix</td>\n",
       "      <td>0.531088</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      model  acc_mean\n",
       "0  pseudolabel-binshuffling  0.554644\n",
       "1       pseudolabel-masking  0.548433\n",
       "2  pseudolabel-noisemasking  0.529887\n",
       "3            pseudolabel-rq  0.536668\n",
       "4      pseudolabel-sampling  0.527020\n",
       "5     pseudolabel-shuffling  0.529214\n",
       "6        pseudolabel-cutmix  0.531088"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfe16d30",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "40116054",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 11544 characters]\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(4, 3))\n",
    "plt.gca().set_axisbelow(True)\n",
    "plt.grid(axis=\"y\")\n",
    "\n",
    "plt.bar(q['model'], q['acc_mean'], color='skyblue')\n",
    "\n",
    "plt.ylabel('5-shot Average Accuracy', fontsize=12)\n",
    "plt.ylim(0.5, 0.56)\n",
    "\n",
    "plt.xticks(range(7), [\n",
    "    \"Ours\", \"Masking\", \"Noise\", \"RQ\", \"Sampling\", \"Shuffling\", \"CutMix\"], fontsize=12)\n",
    "plt.xticks(rotation=90)\n",
    "\n",
    "plt.savefig(\"figs/discussion-semi.png\", dpi=300, bbox_inches=\"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afbf7d35",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "d1bd7220",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   37,   188,  1459,  1494,  4153, 40922, 40981, 44091, 44131,\n",
       "       44160, 45062])"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rankdata[(rankdata[\"model\"] == \"pseudolabel-binsampling\") & (rankdata[\"shots\"] == 5) & (\n",
    "    rankdata[\"rank\"] > 6)][\"data_id\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "f7cb6556",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   22,    29,    49,    53,    59,  1043,  1459, 41143, 44158,\n",
       "       44160, 44161])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rankdata[(rankdata[\"shots\"] == 1) & (rankdata[\"model\"] == \"pseudolabel-rq\") & (rankdata[\"rank\"] == 1)][\"data_id\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "e40b0ad2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data_id</th>\n",
       "      <th>model</th>\n",
       "      <th>shots</th>\n",
       "      <th>acc_mean</th>\n",
       "      <th>acc_std</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-binsampling</td>\n",
       "      <td>1</td>\n",
       "      <td>0.362500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-binshuffling</td>\n",
       "      <td>1</td>\n",
       "      <td>0.370000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-masking</td>\n",
       "      <td>1</td>\n",
       "      <td>0.391875</td>\n",
       "      <td>0.054150</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-sampling</td>\n",
       "      <td>1</td>\n",
       "      <td>0.395650</td>\n",
       "      <td>0.055047</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-noisemasking</td>\n",
       "      <td>1</td>\n",
       "      <td>0.396100</td>\n",
       "      <td>0.051150</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-shuffling</td>\n",
       "      <td>1</td>\n",
       "      <td>0.397000</td>\n",
       "      <td>0.049163</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-rq</td>\n",
       "      <td>1</td>\n",
       "      <td>0.399425</td>\n",
       "      <td>0.050655</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    data_id                     model  shots  acc_mean   acc_std  rank\n",
       "28       22   pseudolabel-binsampling      1  0.362500       NaN   7.0\n",
       "30       22  pseudolabel-binshuffling      1  0.370000       NaN   6.0\n",
       "32       22       pseudolabel-masking      1  0.391875  0.054150   5.0\n",
       "38       22      pseudolabel-sampling      1  0.395650  0.055047   4.0\n",
       "34       22  pseudolabel-noisemasking      1  0.396100  0.051150   3.0\n",
       "40       22     pseudolabel-shuffling      1  0.397000  0.049163   2.0\n",
       "36       22            pseudolabel-rq      1  0.399425  0.050655   1.0"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rankdata[(rankdata[\"shots\"] == 1) & (rankdata[\"data_id\"] == 22)].sort_values(\"acc_mean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "3b514c12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trial</th>\n",
       "      <th>shots</th>\n",
       "      <th>data_id</th>\n",
       "      <th>model</th>\n",
       "      <th>acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-masking</td>\n",
       "      <td>0.3350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-shuffling</td>\n",
       "      <td>0.3475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-noisemasking</td>\n",
       "      <td>0.3425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-rq</td>\n",
       "      <td>0.3300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-masking</td>\n",
       "      <td>0.3800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41774</th>\n",
       "      <td>97</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-sampling</td>\n",
       "      <td>0.3800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41789</th>\n",
       "      <td>98</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-sampling</td>\n",
       "      <td>0.3700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41799</th>\n",
       "      <td>99</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-sampling</td>\n",
       "      <td>0.4050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42000</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-binshuffling</td>\n",
       "      <td>0.3700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42084</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>pseudolabel-binsampling</td>\n",
       "      <td>0.3625</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>502 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       trial  shots  data_id                     model     acc\n",
       "0          0      1       22       pseudolabel-masking  0.3350\n",
       "42         0      1       22     pseudolabel-shuffling  0.3475\n",
       "84         0      1       22  pseudolabel-noisemasking  0.3425\n",
       "126        0      1       22            pseudolabel-rq  0.3300\n",
       "336        1      1       22       pseudolabel-masking  0.3800\n",
       "...      ...    ...      ...                       ...     ...\n",
       "41774     97      1       22      pseudolabel-sampling  0.3800\n",
       "41789     98      1       22      pseudolabel-sampling  0.3700\n",
       "41799     99      1       22      pseudolabel-sampling  0.4050\n",
       "42000      9      1       22  pseudolabel-binshuffling  0.3700\n",
       "42084      9      1       22   pseudolabel-binsampling  0.3625\n",
       "\n",
       "[502 rows x 5 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[(result[\"shots\"] == 1) & (result[\"data_id\"] == 22)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c238777",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fd188f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8b8954b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data_id</th>\n",
       "      <th>model</th>\n",
       "      <th>shots</th>\n",
       "      <th>acc_mean</th>\n",
       "      <th>acc_std</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>1459</td>\n",
       "      <td>pseudolabel-masking</td>\n",
       "      <td>1</td>\n",
       "      <td>0.184100</td>\n",
       "      <td>0.033195</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>1459</td>\n",
       "      <td>pseudolabel-shuffling</td>\n",
       "      <td>1</td>\n",
       "      <td>0.179648</td>\n",
       "      <td>0.026174</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>1459</td>\n",
       "      <td>pseudolabel-noisemasking</td>\n",
       "      <td>1</td>\n",
       "      <td>0.179012</td>\n",
       "      <td>0.021724</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>1459</td>\n",
       "      <td>pseudolabel-binsampling</td>\n",
       "      <td>1</td>\n",
       "      <td>0.177446</td>\n",
       "      <td>0.022214</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>1459</td>\n",
       "      <td>pseudolabel-rq</td>\n",
       "      <td>1</td>\n",
       "      <td>0.175587</td>\n",
       "      <td>0.033305</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>1459</td>\n",
       "      <td>pseudolabel-binshuffling</td>\n",
       "      <td>1</td>\n",
       "      <td>0.170841</td>\n",
       "      <td>0.024573</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>1459</td>\n",
       "      <td>pseudolabel-sampling</td>\n",
       "      <td>1</td>\n",
       "      <td>0.165362</td>\n",
       "      <td>0.023896</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     data_id                     model  shots  acc_mean   acc_std  rank\n",
       "214     1459       pseudolabel-masking      1  0.184100  0.033195   1.0\n",
       "222     1459     pseudolabel-shuffling      1  0.179648  0.026174   2.0\n",
       "216     1459  pseudolabel-noisemasking      1  0.179012  0.021724   3.0\n",
       "210     1459   pseudolabel-binsampling      1  0.177446  0.022214   4.0\n",
       "218     1459            pseudolabel-rq      1  0.175587  0.033305   5.0\n",
       "212     1459  pseudolabel-binshuffling      1  0.170841  0.024573   6.0\n",
       "220     1459      pseudolabel-sampling      1  0.165362  0.023896   7.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rankdata[(rankdata[\"shots\"] == 1) & (rankdata[\"data_id\"] == 1459)].sort_values(\"acc_mean\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "181a4e87",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "402971f4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9fe8570",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "1ea15d19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "aug\n",
       "binsampling     0.533163\n",
       "binshuffling    0.531919\n",
       "masking         0.546234\n",
       "noisemasking    0.531116\n",
       "rq              0.538034\n",
       "shuffling       0.529361\n",
       "Name: acc_mean, dtype: float64"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p[p[\"shot\"] == 5].groupby(\"aug\")[\"acc_mean\"].mean() \n",
    "# rankdata[rankdata[\"shot\"] == 5].groupby(\"aug\")[\"rank\"].mean() \n",
    "# p[p[\"shot\"] == 5].sort_values(\"acc_mean\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "451c930d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8eb45f4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25af3956",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a88afb63",
   "metadata": {},
   "outputs": [],
   "source": [
    "q = result.groupby([\"aug\", \"shot\", \"data_id\"]).agg(\n",
    "    acc_mean=('acc', 'mean'), acc_std=('acc', 'std')).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "9f2c76c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aug</th>\n",
       "      <th>shot</th>\n",
       "      <th>data_id</th>\n",
       "      <th>acc_mean</th>\n",
       "      <th>acc_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>350</th>\n",
       "      <td>rq</td>\n",
       "      <td>1</td>\n",
       "      <td>1067</td>\n",
       "      <td>0.602607</td>\n",
       "      <td>0.238429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>masking</td>\n",
       "      <td>1</td>\n",
       "      <td>1067</td>\n",
       "      <td>0.526540</td>\n",
       "      <td>0.251452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>noisemasking</td>\n",
       "      <td>1</td>\n",
       "      <td>1067</td>\n",
       "      <td>0.499763</td>\n",
       "      <td>0.337272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434</th>\n",
       "      <td>shuffling</td>\n",
       "      <td>1</td>\n",
       "      <td>1067</td>\n",
       "      <td>0.499289</td>\n",
       "      <td>0.338244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>binsampling</td>\n",
       "      <td>1</td>\n",
       "      <td>1067</td>\n",
       "      <td>0.441943</td>\n",
       "      <td>0.337032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>binshuffling</td>\n",
       "      <td>1</td>\n",
       "      <td>1067</td>\n",
       "      <td>0.434597</td>\n",
       "      <td>0.343932</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              aug  shot  data_id  acc_mean   acc_std\n",
       "350            rq     1     1067  0.602607  0.238429\n",
       "182       masking     1     1067  0.526540  0.251452\n",
       "266  noisemasking     1     1067  0.499763  0.337272\n",
       "434     shuffling     1     1067  0.499289  0.338244\n",
       "14    binsampling     1     1067  0.441943  0.337032\n",
       "98   binshuffling     1     1067  0.434597  0.343932"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q[(q[\"shot\"] == 1) & (q[\"data_id\"] == 1067)].sort_values('acc_mean', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "551949da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'22 54 1063 1067 12 18 23 59 188 307 1043 1459 1475 1489 1492 1497 1503 4153 40499 44125 44131 45062 44157 1462 44160 29 37 53 49 1504 1494 41143 44126 40981 41168 44091 44158 44123 44090 40922 44161 45714'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\" \".join(map(str, datalist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "c0ed390e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 17040 characters]\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# q1 = p[p[\"shot\"] == 1]\n",
    "# q2 = p[p[\"shot\"] == 5]\n",
    "# plt.figure(figsize=(4, 3))\n",
    "# plt.gca().set_axisbelow(True)\n",
    "# plt.grid()\n",
    "# plt.plot(q1[\"numbins\"], q1[\"acc_mean\"], label=\"1-shot\", marker=\"o\")\n",
    "# plt.fill_between(q1[\"numbins\"], q1[\"acc_mean\"]-q1[\"acc_std\"], q1[\"acc_mean\"]+q1[\"acc_std\"], alpha=0.2)\n",
    "# plt.plot(q2[\"numbins\"], q2[\"acc_mean\"], label=\"5-shot\", marker=\"o\")\n",
    "# plt.fill_between(q2[\"numbins\"], q2[\"acc_mean\"]-q2[\"acc_std\"], q2[\"acc_mean\"]+q2[\"acc_std\"], alpha=0.2)\n",
    "# plt.legend()\n",
    "# plt.xlabel(\"Number of Ranges\", fontsize=12)\n",
    "# plt.ylabel(\"Average Accuracy\", fontsize=12)\n",
    "# plt.title(model)\n",
    "# plt.savefig(f'figs/discussion-numbins-{model}.png', dpi=300, bbox_inches=\"tight\")\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a35ce47",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
