{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d98765e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "\n",
    "from pathlib import Path\n",
    "from tqdm.notebook import tqdm\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "116617c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{YOUR_WORKING_PATH}/zoo_utils.py:6: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import re\n",
    "\n",
    "from utils import test_model, compare_models_prediction, generate_z_samples\n",
    "from zoo_utils import load_hyperrep_edx\n",
    "\n",
    "from shrp.models.def_net import CNN, CNN3\n",
    "from shrp.models.def_AE_trainable import get_transformations\n",
    "from shrp.datasets.dataset_auxiliaries import tokens_to_recipe, tokens_to_checkpoint\n",
    "from shrp.models.def_loss import DistillationLoss, ReconDistillationLoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0d465bf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "sns.set_theme(context='notebook', style='whitegrid',\n",
    "              rc={\n",
    "                  'axes.spines.right': False,\n",
    "                  'axes.spines.top': False,\n",
    "                  'xtick.bottom': True,\n",
    "                  'ytick.left': True,\n",
    "                  'font.family': 'Fira Sans'\n",
    "              })\n",
    "\n",
    "COLOURS = [\n",
    "    '#960018',  # Carmine Red\n",
    "    '#FED85D',  # Mustard Gold\n",
    "    '#434384',  # Marian Blue\n",
    "    '#54B674',  # Emerald Green\n",
    "    '#D183C9',  # French Mauve\n",
    "    '#318CE7',  # French Blue\n",
    "    '#D08000'   # Fulvous Orange\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a45fdd7",
   "metadata": {},
   "source": [
    "## Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "755588f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASETS = ['svhn', 'cifar10', 'eurosat']\n",
    "DATASET_LABELS = {\n",
    "    'svhn': 'SVHN',\n",
    "    'cifar10': 'CIFAR-10',\n",
    "    'eurosat': 'EuroSAT'\n",
    "}\n",
    "\n",
    "DS_ORDER = ['SVHN', 'CIFAR-10', 'EuroSAT']\n",
    "HR_ORDER = ['Contrastive + Structure (Baseline)', 'Behavior', 'Contrastive + Behavior', 'Structure + Behavior', 'Contrastive + Structure + Behavior']\n",
    "HR_PLOT = ['Contrastive + Structure (Baseline)', 'Contrastive + Behavior', 'Contrastive + Structure + Behavior']\n",
    "\n",
    "HYPERREP_PATH = Path('{YOUR_WORKING_PATH}/results/hyperrepresentations/tune')\n",
    "\n",
    "EXPERIMENT_PATH = Path.cwd() / 'results' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e0f27f84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f09a64e37a6496daba04d4321956128",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading data:   0%|          | 0/3 [00:00<?, ?dataset/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model zoo</th>\n",
       "      <th>Trial ID</th>\n",
       "      <th>Learning rate</th>\n",
       "      <th>Latent dimension</th>\n",
       "      <th>Behaviour loss</th>\n",
       "      <th>Query set</th>\n",
       "      <th>Query dataset</th>\n",
       "      <th># of queries</th>\n",
       "      <th>Gamma</th>\n",
       "      <th>Beta</th>\n",
       "      <th>...</th>\n",
       "      <th>Train loss (recon)</th>\n",
       "      <th>Train loss (structure)</th>\n",
       "      <th>Train loss (behaviour)</th>\n",
       "      <th>Test loss</th>\n",
       "      <th>Test loss (contrast)</th>\n",
       "      <th>Test loss (recon)</th>\n",
       "      <th>Test loss (structure)</th>\n",
       "      <th>Test loss (behaviour)</th>\n",
       "      <th>Dstk: Test accuracy</th>\n",
       "      <th>Dstk: GGap</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SVHN</td>\n",
       "      <td>50f57_00020</td>\n",
       "      <td>0.00001</td>\n",
       "      <td>64</td>\n",
       "      <td>cross_entropy</td>\n",
       "      <td>data</td>\n",
       "      <td>SVHN</td>\n",
       "      <td>256</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.344194</td>\n",
       "      <td>0.930796</td>\n",
       "      <td>1.390127</td>\n",
       "      <td>1.059503</td>\n",
       "      <td>1.045855</td>\n",
       "      <td>1.060222</td>\n",
       "      <td>0.791913</td>\n",
       "      <td>1.090034</td>\n",
       "      <td>0.559680</td>\n",
       "      <td>0.283559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SVHN</td>\n",
       "      <td>50f57_00020</td>\n",
       "      <td>0.00001</td>\n",
       "      <td>64</td>\n",
       "      <td>cross_entropy</td>\n",
       "      <td>data</td>\n",
       "      <td>SVHN</td>\n",
       "      <td>256</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.275532</td>\n",
       "      <td>0.770928</td>\n",
       "      <td>0.220488</td>\n",
       "      <td>0.257198</td>\n",
       "      <td>0.887651</td>\n",
       "      <td>0.224016</td>\n",
       "      <td>0.591360</td>\n",
       "      <td>0.183200</td>\n",
       "      <td>0.554503</td>\n",
       "      <td>0.276677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SVHN</td>\n",
       "      <td>50f57_00020</td>\n",
       "      <td>0.00001</td>\n",
       "      <td>64</td>\n",
       "      <td>cross_entropy</td>\n",
       "      <td>data</td>\n",
       "      <td>SVHN</td>\n",
       "      <td>256</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.266038</td>\n",
       "      <td>0.677502</td>\n",
       "      <td>0.220320</td>\n",
       "      <td>0.247892</td>\n",
       "      <td>0.708432</td>\n",
       "      <td>0.223653</td>\n",
       "      <td>0.544859</td>\n",
       "      <td>0.187963</td>\n",
       "      <td>0.550643</td>\n",
       "      <td>0.276275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVHN</td>\n",
       "      <td>50f57_00020</td>\n",
       "      <td>0.00001</td>\n",
       "      <td>64</td>\n",
       "      <td>cross_entropy</td>\n",
       "      <td>data</td>\n",
       "      <td>SVHN</td>\n",
       "      <td>256</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.264123</td>\n",
       "      <td>0.664841</td>\n",
       "      <td>0.219599</td>\n",
       "      <td>0.239949</td>\n",
       "      <td>0.652948</td>\n",
       "      <td>0.218212</td>\n",
       "      <td>0.527920</td>\n",
       "      <td>0.183800</td>\n",
       "      <td>0.551713</td>\n",
       "      <td>0.279411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVHN</td>\n",
       "      <td>50f57_00020</td>\n",
       "      <td>0.00001</td>\n",
       "      <td>64</td>\n",
       "      <td>cross_entropy</td>\n",
       "      <td>data</td>\n",
       "      <td>SVHN</td>\n",
       "      <td>256</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.261248</td>\n",
       "      <td>0.628665</td>\n",
       "      <td>0.220424</td>\n",
       "      <td>0.242219</td>\n",
       "      <td>0.741064</td>\n",
       "      <td>0.215964</td>\n",
       "      <td>0.495595</td>\n",
       "      <td>0.184894</td>\n",
       "      <td>0.538348</td>\n",
       "      <td>0.282729</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Model zoo     Trial ID  Learning rate  Latent dimension Behaviour loss  \\\n",
       "0      SVHN  50f57_00020        0.00001                64  cross_entropy   \n",
       "1      SVHN  50f57_00020        0.00001                64  cross_entropy   \n",
       "2      SVHN  50f57_00020        0.00001                64  cross_entropy   \n",
       "3      SVHN  50f57_00020        0.00001                64  cross_entropy   \n",
       "4      SVHN  50f57_00020        0.00001                64  cross_entropy   \n",
       "\n",
       "  Query set Query dataset  # of queries  Gamma  Beta  ...  Train loss (recon)  \\\n",
       "0      data          SVHN           256   0.05   0.1  ...            1.344194   \n",
       "1      data          SVHN           256   0.05   0.1  ...            0.275532   \n",
       "2      data          SVHN           256   0.05   0.1  ...            0.266038   \n",
       "3      data          SVHN           256   0.05   0.1  ...            0.264123   \n",
       "4      data          SVHN           256   0.05   0.1  ...            0.261248   \n",
       "\n",
       "   Train loss (structure)  Train loss (behaviour)  Test loss  \\\n",
       "0                0.930796                1.390127   1.059503   \n",
       "1                0.770928                0.220488   0.257198   \n",
       "2                0.677502                0.220320   0.247892   \n",
       "3                0.664841                0.219599   0.239949   \n",
       "4                0.628665                0.220424   0.242219   \n",
       "\n",
       "   Test loss (contrast)  Test loss (recon)  Test loss (structure)  \\\n",
       "0              1.045855           1.060222               0.791913   \n",
       "1              0.887651           0.224016               0.591360   \n",
       "2              0.708432           0.223653               0.544859   \n",
       "3              0.652948           0.218212               0.527920   \n",
       "4              0.741064           0.215964               0.495595   \n",
       "\n",
       "   Test loss (behaviour)  Dstk: Test accuracy  Dstk: GGap  \n",
       "0               1.090034             0.559680    0.283559  \n",
       "1               0.183200             0.554503    0.276677  \n",
       "2               0.187963             0.550643    0.276275  \n",
       "3               0.183800             0.551713    0.279411  \n",
       "4               0.184894             0.538348    0.282729  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MAX_EPOCH = 100\n",
    "\n",
    "df_hr = pd.DataFrame(columns=[\n",
    "    'Model zoo',\n",
    "    'Trial ID',\n",
    "    'Learning rate',\n",
    "    'Latent dimension',\n",
    "    'Behaviour loss',\n",
    "    'Query set',\n",
    "    'Query dataset',\n",
    "    '# of queries',\n",
    "    'Gamma',\n",
    "    'Beta',\n",
    "    'Epoch',\n",
    "    'Train loss',\n",
    "    'Train loss (contrast)', \n",
    "    'Train loss (recon)',\n",
    "    'Train loss (structure)', \n",
    "    'Train loss (behaviour)',\n",
    "    'Test loss',\n",
    "    'Test loss (contrast)', \n",
    "    'Test loss (recon)',\n",
    "    'Test loss (structure)', \n",
    "    'Test loss (behaviour)',\n",
    "    'Dstk: Test accuracy',\n",
    "    'Dstk: GGap'\n",
    "])\n",
    "\n",
    "for ds in tqdm(DATASETS, desc='Loading data', unit='dataset'):\n",
    "    hr_path = HYPERREP_PATH / f\"{ds}_train_hyperrep\"\n",
    "    for path in os.listdir(hr_path):\n",
    "        if not re.match('AE_trainable', path):\n",
    "            continue\n",
    "        path = hr_path / path\n",
    "\n",
    "        with open(path / 'params.json') as ifh_config, open(path / 'result.json') as ifh_result:\n",
    "            config = json.load(ifh_config)\n",
    "            result = pd.read_json(ifh_result, lines=True, dtype={'trial_id': str})\n",
    "\n",
    "        for edx in range(len(result)):\n",
    "            df_hr.loc[len(df_hr)] = (\n",
    "                DATASET_LABELS[ds],\n",
    "                result.loc[edx]['trial_id'],\n",
    "                config['optim::lr'],\n",
    "                config['ae:lat_dim'],\n",
    "                config['training::loss_distillation'],\n",
    "                config['training::queryset_distillation'],\n",
    "                config['training::queryset_dump'],\n",
    "                config['training::n_queries_distillation'],\n",
    "                config['training::gamma'],\n",
    "                config['training::beta'],\n",
    "                result.loc[edx]['training_iteration'],\n",
    "                result.loc[edx]['loss/loss_train'],\n",
    "                result.loc[edx]['loss/loss_contrast_train'],\n",
    "                result.loc[edx]['loss/loss_recon_train'],\n",
    "                result.loc[edx]['loss/loss_structure_train'],\n",
    "                result.loc[edx]['loss/loss_behaviour_train'],\n",
    "                result.loc[edx]['loss/loss_test'],\n",
    "                result.loc[edx]['loss/loss_contrast_test'],\n",
    "                result.loc[edx]['loss/loss_recon_test'],\n",
    "                result.loc[edx]['loss/loss_structure_test'],\n",
    "                result.loc[edx]['loss/loss_behaviour_test'],\n",
    "                result.loc[edx]['dstk/test_acc_test'],\n",
    "                result.loc[edx]['dstk/ggap_test']\n",
    "            )\n",
    "\n",
    "df_hr['Query dataset'] = df_hr['Query dataset'].map({\n",
    "    '{YOUR_WORKING_PATH}/results/data/svhn_train_zoo_relu/dataset.pt': 'SVHN',\n",
    "    '{YOUR_WORKING_PATH}/results/data/cifar10_train_zoo_relu/dataset.pt': 'CIFAR-10',\n",
    "    '{YOUR_WORKING_PATH}/results/data/eurosat_train_zoo_relu/dataset.pt': 'EuroSAT',\n",
    "    '{YOUR_WORKING_PATH}/results/data/tinyimagenet_32/dataset.pt': 'TinyImagenet-32'\n",
    "})\n",
    "df_hr.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "407a49eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_dict = {\n",
    "    '00011': 'Contrastive + Structure (Baseline)',\n",
    "    '00000': 'Behavior',\n",
    "    '00006': 'Contrastive + Behavior',\n",
    "    '00002': 'Structure + Behavior',\n",
    "    '00008': 'Contrastive + Structure + Behavior'\n",
    "}\n",
    "\n",
    "df_hr['Hyper-representation'] = df_hr['Trial ID'].str.slice(start=-5).map(idx_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bc649bf",
   "metadata": {},
   "source": [
    "### Loading model zoos metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6b9502ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model zoo</th>\n",
       "      <th>Trial ID</th>\n",
       "      <th>Activation function</th>\n",
       "      <th>Initialisation</th>\n",
       "      <th>Learning rate</th>\n",
       "      <th>Weight decay</th>\n",
       "      <th>Seed</th>\n",
       "      <th>Epoch</th>\n",
       "      <th>Test accuracy</th>\n",
       "      <th>Generalisation gap</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SVHN</td>\n",
       "      <td>7597300394</td>\n",
       "      <td>relu</td>\n",
       "      <td>kaiming_uniform</td>\n",
       "      <td>0.00075</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071758</td>\n",
       "      <td>-0.001563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SVHN</td>\n",
       "      <td>7597300394</td>\n",
       "      <td>relu</td>\n",
       "      <td>kaiming_uniform</td>\n",
       "      <td>0.00075</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0.775200</td>\n",
       "      <td>-0.182034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SVHN</td>\n",
       "      <td>7597300394</td>\n",
       "      <td>relu</td>\n",
       "      <td>kaiming_uniform</td>\n",
       "      <td>0.00075</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>0.829479</td>\n",
       "      <td>-0.016614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SVHN</td>\n",
       "      <td>7597300394</td>\n",
       "      <td>relu</td>\n",
       "      <td>kaiming_uniform</td>\n",
       "      <td>0.00075</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>0.846381</td>\n",
       "      <td>-0.003682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVHN</td>\n",
       "      <td>7597300394</td>\n",
       "      <td>relu</td>\n",
       "      <td>kaiming_uniform</td>\n",
       "      <td>0.00075</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>0.848033</td>\n",
       "      <td>0.010410</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Model zoo    Trial ID Activation function   Initialisation  Learning rate  \\\n",
       "0      SVHN  7597300394                relu  kaiming_uniform        0.00075   \n",
       "1      SVHN  7597300394                relu  kaiming_uniform        0.00075   \n",
       "2      SVHN  7597300394                relu  kaiming_uniform        0.00075   \n",
       "3      SVHN  7597300394                relu  kaiming_uniform        0.00075   \n",
       "4      SVHN  7597300394                relu  kaiming_uniform        0.00075   \n",
       "\n",
       "   Weight decay  Seed  Epoch  Test accuracy  Generalisation gap  \n",
       "0        0.0005     6      0       0.071758           -0.001563  \n",
       "1        0.0005     6      1       0.775200           -0.182034  \n",
       "2        0.0005     6      2       0.829479           -0.016614  \n",
       "3        0.0005     6      3       0.846381           -0.003682  \n",
       "4        0.0005     6      4       0.848033            0.010410  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF_ZOO_PATH = EXPERIMENT_PATH / 'zoo_analysis' / 'df_zoo.csv'\n",
    "\n",
    "df_zoo = pd.read_csv(DF_ZOO_PATH, index_col=0)\n",
    "df_zoo.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c04ee8e1",
   "metadata": {},
   "source": [
    "### Loading reconstruction accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7c4c8f5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ebd36b1ff224a18aa6c2fb6e0311632",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading data:   0%|          | 0/3 [00:00<?, ?dataset/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model index</th>\n",
       "      <th>Hyper-representation model</th>\n",
       "      <th>Original accuracy</th>\n",
       "      <th>Reconstruction accuracy</th>\n",
       "      <th>Model zoo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Contrastive + Structure (Baseline)</td>\n",
       "      <td>0.743623</td>\n",
       "      <td>0.195874</td>\n",
       "      <td>SVHN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Contrastive + Structure (Baseline)</td>\n",
       "      <td>0.828596</td>\n",
       "      <td>0.195913</td>\n",
       "      <td>SVHN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Contrastive + Structure (Baseline)</td>\n",
       "      <td>0.856215</td>\n",
       "      <td>0.295444</td>\n",
       "      <td>SVHN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Contrastive + Structure (Baseline)</td>\n",
       "      <td>0.873848</td>\n",
       "      <td>0.429779</td>\n",
       "      <td>SVHN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Contrastive + Structure (Baseline)</td>\n",
       "      <td>0.887792</td>\n",
       "      <td>0.581400</td>\n",
       "      <td>SVHN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Model index          Hyper-representation model  Original accuracy  \\\n",
       "0            0  Contrastive + Structure (Baseline)           0.743623   \n",
       "1            1  Contrastive + Structure (Baseline)           0.828596   \n",
       "2            2  Contrastive + Structure (Baseline)           0.856215   \n",
       "3            3  Contrastive + Structure (Baseline)           0.873848   \n",
       "4            4  Contrastive + Structure (Baseline)           0.887792   \n",
       "\n",
       "   Reconstruction accuracy Model zoo  \n",
       "0                 0.195874      SVHN  \n",
       "1                 0.195913      SVHN  \n",
       "2                 0.295444      SVHN  \n",
       "3                 0.429779      SVHN  \n",
       "4                 0.581400      SVHN  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zoo_acc = list()\n",
    "\n",
    "for ds in tqdm(DATASETS, desc='Loading data', unit='dataset'):\n",
    "    df_zoo_acc_path = EXPERIMENT_PATH / f'hr_analysis_{ds}' / 'df_zoo_acc.csv'\n",
    "    df_zoo_acc = pd.read_csv(df_zoo_acc_path, index_col=0)\n",
    "    \n",
    "    df_zoo_acc['Model zoo'] = DATASET_LABELS[ds]\n",
    "    \n",
    "    zoo_acc.append(df_zoo_acc)\n",
    "    \n",
    "zoo_acc = pd.concat(zoo_acc)\n",
    "zoo_acc.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0386b2bf",
   "metadata": {},
   "source": [
    "### Loading generation accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "173cd2d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b2b0dc3bb0046e290506853eac14718",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading data:   0%|          | 0/3 [00:00<?, ?dataset/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Hyper-representation model</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Model zoo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Contrastive + Structure (Baseline)</td>\n",
       "      <td>Generated</td>\n",
       "      <td>0.369084</td>\n",
       "      <td>SVHN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Contrastive + Structure (Baseline)</td>\n",
       "      <td>Generated</td>\n",
       "      <td>0.159381</td>\n",
       "      <td>SVHN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Contrastive + Structure (Baseline)</td>\n",
       "      <td>Generated</td>\n",
       "      <td>0.221996</td>\n",
       "      <td>SVHN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Contrastive + Structure (Baseline)</td>\n",
       "      <td>Generated</td>\n",
       "      <td>0.288222</td>\n",
       "      <td>SVHN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Contrastive + Structure (Baseline)</td>\n",
       "      <td>Generated</td>\n",
       "      <td>0.195874</td>\n",
       "      <td>SVHN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Hyper-representation model     Origin  Accuracy Model zoo\n",
       "0  Contrastive + Structure (Baseline)  Generated  0.369084      SVHN\n",
       "1  Contrastive + Structure (Baseline)  Generated  0.159381      SVHN\n",
       "2  Contrastive + Structure (Baseline)  Generated  0.221996      SVHN\n",
       "3  Contrastive + Structure (Baseline)  Generated  0.288222      SVHN\n",
       "4  Contrastive + Structure (Baseline)  Generated  0.195874      SVHN"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen_acc = list()\n",
    "\n",
    "for ds in tqdm(DATASETS, desc='Loading data', unit='dataset'):\n",
    "    df_gen_acc_path = EXPERIMENT_PATH / f'hr_analysis_{ds}' / 'df_generative.csv'\n",
    "    df_gen_acc = pd.read_csv(df_gen_acc_path, index_col=0)\n",
    "    \n",
    "    df_gen_acc['Model zoo'] = DATASET_LABELS[ds]\n",
    "    \n",
    "    gen_acc.append(df_gen_acc)\n",
    "    \n",
    "gen_acc = pd.concat(gen_acc)\n",
    "gen_acc.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f4f05c2",
   "metadata": {},
   "source": [
    "### Loading generation diversity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "33fb24b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef9835ef32b042ce9c979101a32de969",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading data:   0%|          | 0/3 [00:00<?, ?dataset/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gen_diversity = dict()\n",
    "\n",
    "for ds in tqdm(DATASETS, desc='Loading data', unit='dataset'):\n",
    "    gen_diversity_path = EXPERIMENT_PATH / f'hr_analysis_{ds}' / 'gen_diversity.json'\n",
    "    with open(gen_diversity_path, 'r') as ifh:\n",
    "        gen_diversity[ds] = json.load(ifh)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98b0e5bf",
   "metadata": {},
   "source": [
    "## Analysing data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d157bbf",
   "metadata": {},
   "source": [
    "### Plotting figures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "59919522",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 113480 characters]",
      "text/plain": [
       "<Figure size 1400x600 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_, axs = plt.subplots(2, 3, figsize=(14, 6))\n",
    "\n",
    "for d, ds in enumerate(DATASETS):\n",
    "    ds_label = DATASET_LABELS[ds]\n",
    "    df_zoo_acc = zoo_acc[zoo_acc['Model zoo'] == ds_label]\n",
    "    df_generative = gen_acc[gen_acc['Model zoo'] == ds_label]\n",
    "\n",
    "    label = 'Contrastive + Structure (Baseline)'\n",
    "    sns.histplot(df_zoo_acc[df_zoo_acc['Hyper-representation model'] == label], x='Original accuracy', stat='probability', binwidth=0.05, binrange=(0., 1.), kde=True, color=COLOURS[0], alpha=0.5, label='Original', ax=axs[0][d])\n",
    "    sns.histplot(df_zoo_acc[df_zoo_acc['Hyper-representation model'] == label], x='Reconstruction accuracy', stat='probability', binwidth=0.05, binrange=(0., 1.), kde=True, color=COLOURS[1], alpha=0.5, label='Reconstructed', ax=axs[0][d])\n",
    "    sns.histplot(\n",
    "        df_generative[(df_generative['Hyper-representation model'] == label) & (df_generative['Origin'] == 'Generated')],\n",
    "        x='Accuracy',\n",
    "        stat='probability',\n",
    "        binwidth=0.05,\n",
    "        binrange=(0., 1.),\n",
    "        kde=True,\n",
    "        color=COLOURS[2],\n",
    "        alpha=0.5,\n",
    "        label='Generated',\n",
    "        ax=axs[0][d]\n",
    "    )\n",
    "    axs[0][d].set(title=f\"{ds_label}\\n{label}\", xlabel='Test accuracy', ylabel='Probability', ylim=(0, 1))\n",
    "    \n",
    "    label = 'Contrastive + Structure + Behavior'\n",
    "    sns.histplot(df_zoo_acc[df_zoo_acc['Hyper-representation model'] == label], x='Original accuracy', stat='probability', binwidth=0.05, binrange=(0., 1.), kde=True, color=COLOURS[0], alpha=0.5, label='Original', ax=axs[1][d])\n",
    "    sns.histplot(df_zoo_acc[df_zoo_acc['Hyper-representation model'] == label], x='Reconstruction accuracy', stat='probability', binwidth=0.05, binrange=(0., 1.), kde=True, color=COLOURS[1], alpha=0.5, label='Reconstructed', ax=axs[1][d])\n",
    "    sns.histplot(\n",
    "        df_generative[(df_generative['Hyper-representation model'] == label) & (df_generative['Origin'] == 'Generated')],\n",
    "        x='Accuracy',\n",
    "        stat='probability',\n",
    "        binwidth=0.05,\n",
    "        binrange=(0., 1.),\n",
    "        kde=True,\n",
    "        color=COLOURS[2],\n",
    "        alpha=0.5,\n",
    "        label='Generated',\n",
    "        ax=axs[1][d]\n",
    "    )\n",
    "    axs[1][d].set(title=f\"{ds_label}\\n{label}\", xlabel='Test accuracy', ylabel='Probability', ylim=(0, 1))\n",
    "    \n",
    "plt.legend(loc=(-1.50, -0.50), ncols=3)\n",
    "plt.subplots_adjust(hspace=0.6, wspace=0.25)\n",
    "plt.savefig('figures/generative_dstk.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15dd738e",
   "metadata": {},
   "source": [
    "### Analysing generated models' diversity\n",
    "\n",
    "#### Structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "56709da1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llll}\n",
      "Model zoo & SVHN & CIFAR-10 & EuroSAT \\\\\n",
      "Losses &  &  &  \\\\\n",
      "Model zoo & 23.3 (±4) & 22.9 (±4) & 23.0 (±5) \\\\\n",
      "Contrastive + Structure (Baseline) & 13.4 (±3) & 14.9 (±4) & 16.8 (±5) \\\\\n",
      "Behavior & 1.6 (±0) & 3.5 (±1) & 6.3 (±1) \\\\\n",
      "Contrastive + Behavior & 1.7 (±0) & 1.5 (±0) & 3.0 (±1) \\\\\n",
      "Structure + Behavior & 1.6 (±0) & 1.5 (±0) & 2.8 (±1) \\\\\n",
      "Contrastive + Structure + Behavior & 1.5 (±0) & 1.3 (±0) & 2.1 (±1) \\\\\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_d5246\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Model zoo</th>\n",
       "      <th id=\"T_d5246_level0_col0\" class=\"col_heading level0 col0\" >SVHN</th>\n",
       "      <th id=\"T_d5246_level0_col1\" class=\"col_heading level0 col1\" >CIFAR-10</th>\n",
       "      <th id=\"T_d5246_level0_col2\" class=\"col_heading level0 col2\" >EuroSAT</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Losses</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_d5246_level0_row0\" class=\"row_heading level0 row0\" >Model zoo</th>\n",
       "      <td id=\"T_d5246_row0_col0\" class=\"data row0 col0\" >23.3 (±4)</td>\n",
       "      <td id=\"T_d5246_row0_col1\" class=\"data row0 col1\" >22.9 (±4)</td>\n",
       "      <td id=\"T_d5246_row0_col2\" class=\"data row0 col2\" >23.0 (±5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_d5246_level0_row1\" class=\"row_heading level0 row1\" >Contrastive + Structure (Baseline)</th>\n",
       "      <td id=\"T_d5246_row1_col0\" class=\"data row1 col0\" >13.4 (±3)</td>\n",
       "      <td id=\"T_d5246_row1_col1\" class=\"data row1 col1\" >14.9 (±4)</td>\n",
       "      <td id=\"T_d5246_row1_col2\" class=\"data row1 col2\" >16.8 (±5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_d5246_level0_row2\" class=\"row_heading level0 row2\" >Behavior</th>\n",
       "      <td id=\"T_d5246_row2_col0\" class=\"data row2 col0\" >1.6 (±0)</td>\n",
       "      <td id=\"T_d5246_row2_col1\" class=\"data row2 col1\" >3.5 (±1)</td>\n",
       "      <td id=\"T_d5246_row2_col2\" class=\"data row2 col2\" >6.3 (±1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_d5246_level0_row3\" class=\"row_heading level0 row3\" >Contrastive + Behavior</th>\n",
       "      <td id=\"T_d5246_row3_col0\" class=\"data row3 col0\" >1.7 (±0)</td>\n",
       "      <td id=\"T_d5246_row3_col1\" class=\"data row3 col1\" >1.5 (±0)</td>\n",
       "      <td id=\"T_d5246_row3_col2\" class=\"data row3 col2\" >3.0 (±1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_d5246_level0_row4\" class=\"row_heading level0 row4\" >Structure + Behavior</th>\n",
       "      <td id=\"T_d5246_row4_col0\" class=\"data row4 col0\" >1.6 (±0)</td>\n",
       "      <td id=\"T_d5246_row4_col1\" class=\"data row4 col1\" >1.5 (±0)</td>\n",
       "      <td id=\"T_d5246_row4_col2\" class=\"data row4 col2\" >2.8 (±1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_d5246_level0_row5\" class=\"row_heading level0 row5\" >Contrastive + Structure + Behavior</th>\n",
       "      <td id=\"T_d5246_row5_col0\" class=\"data row5 col0\" >1.5 (±0)</td>\n",
       "      <td id=\"T_d5246_row5_col1\" class=\"data row5 col1\" >1.3 (±0)</td>\n",
       "      <td id=\"T_d5246_row5_col2\" class=\"data row5 col2\" >2.1 (±1)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f7c51875cc0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_div_str = pd.DataFrame(columns=['Losses', 'Model zoo', 'Average', 'Standard deviation'])\n",
    "\n",
    "for ds, data in gen_diversity.items():\n",
    "    for hr, dist in data.items():\n",
    "        df_div_str.loc[len(df_div_str)] = (hr, DATASET_LABELS[ds], np.mean(dist['l2_structure']), np.std(dist['l2_structure']))\n",
    "        \n",
    "df_div_str['print'] = df_div_str['Average'].map(lambda x: f\"{x:.1f}\") + df_div_str['Standard deviation'].map(lambda x: f\" (±{x:.0f})\")\n",
    "df_div_str = df_div_str.pivot(index='Losses', columns='Model zoo', values='print')\n",
    "df_div_str = df_div_str.loc[['Model zoo'] + HR_ORDER, DS_ORDER]\n",
    "df_div_str = df_div_str.style.format(precision=3)\n",
    "print(df_div_str.to_latex())\n",
    "df_div_str"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6d703bc",
   "metadata": {},
   "source": [
    "#### Behavior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd4f759b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_div_str = pd.DataFrame(columns=['Losses', 'Model zoo', 'Average', 'Standard deviation'])\n",
    "\n",
    "for ds, data in gen_diversity.items():\n",
    "    for hr, dist in data.items():\n",
    "        df_div_str.loc[len(df_div_str)] = (hr, DATASET_LABELS[ds], np.mean(dist['l2_behaviour']), np.std(dist['l2_behaviour']))\n",
    "        \n",
    "df_div_str['print'] = df_div_str['Average'].map(lambda x: f\"{x:.1f}\") + df_div_str['Standard deviation'].map(lambda x: f\" (±{x:.0f})\")\n",
    "df_div_str = df_div_str.pivot(index='Losses', columns='Model zoo', values='print')\n",
    "df_div_str = df_div_str.loc[['Model zoo'] + HR_ORDER, DS_ORDER]\n",
    "df_div_str = df_div_str.style.format(precision=3)\n",
    "print(df_div_str.to_latex())\n",
    "df_div_str"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78b340f3",
   "metadata": {},
   "source": [
    "### Examining the maximum performance of models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d566b250",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llrrrrr}\n",
      " &  & Zoo accuracy & Reconstruction accuracy & $\\Delta_{Acc}$ Reconstruction & Generation accuracy & $\\Delta_{Acc}$ Generation \\\\\n",
      "Model zoo & Hyper-representation model &  &  &  &  &  \\\\\n",
      "\\multirow[c]{2}{*}{SVHN} & Contrastive + Structure (Baseline) & 91.0% & 74.5% & -16.5% & 61.3% & -29.7% \\\\\n",
      " & Contrastive + Structure + Behavior & 91.0% & 90.4% & -0.6% & 90.4% & -0.6% \\\\\n",
      "\\multirow[c]{2}{*}{CIFAR-10} & Contrastive + Structure (Baseline) & 70.1% & 51.2% & -18.9% & 46.0% & -24.1% \\\\\n",
      " & Contrastive + Structure + Behavior & 70.1% & 69.5% & -0.6% & 69.5% & -0.6% \\\\\n",
      "\\multirow[c]{2}{*}{EuroSAT} & Contrastive + Structure (Baseline) & 88.5% & 68.6% & -19.9% & 56.5% & -32.0% \\\\\n",
      " & Contrastive + Structure + Behavior & 88.5% & 87.7% & -0.8% & 87.5% & -1.0% \\\\\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_b716f\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank\" >&nbsp;</th>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_b716f_level0_col0\" class=\"col_heading level0 col0\" >Zoo accuracy</th>\n",
       "      <th id=\"T_b716f_level0_col1\" class=\"col_heading level0 col1\" >Reconstruction accuracy</th>\n",
       "      <th id=\"T_b716f_level0_col2\" class=\"col_heading level0 col2\" >$\\Delta_{Acc}$ Reconstruction</th>\n",
       "      <th id=\"T_b716f_level0_col3\" class=\"col_heading level0 col3\" >Generation accuracy</th>\n",
       "      <th id=\"T_b716f_level0_col4\" class=\"col_heading level0 col4\" >$\\Delta_{Acc}$ Generation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Model zoo</th>\n",
       "      <th class=\"index_name level1\" >Hyper-representation model</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "      <th class=\"blank col3\" >&nbsp;</th>\n",
       "      <th class=\"blank col4\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_b716f_level0_row0\" class=\"row_heading level0 row0\" rowspan=\"2\">SVHN</th>\n",
       "      <th id=\"T_b716f_level1_row0\" class=\"row_heading level1 row0\" >Contrastive + Structure (Baseline)</th>\n",
       "      <td id=\"T_b716f_row0_col0\" class=\"data row0 col0\" >91.0%</td>\n",
       "      <td id=\"T_b716f_row0_col1\" class=\"data row0 col1\" >74.5%</td>\n",
       "      <td id=\"T_b716f_row0_col2\" class=\"data row0 col2\" >-16.5%</td>\n",
       "      <td id=\"T_b716f_row0_col3\" class=\"data row0 col3\" >61.3%</td>\n",
       "      <td id=\"T_b716f_row0_col4\" class=\"data row0 col4\" >-29.7%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_b716f_level1_row1\" class=\"row_heading level1 row1\" >Contrastive + Structure + Behavior</th>\n",
       "      <td id=\"T_b716f_row1_col0\" class=\"data row1 col0\" >91.0%</td>\n",
       "      <td id=\"T_b716f_row1_col1\" class=\"data row1 col1\" >90.4%</td>\n",
       "      <td id=\"T_b716f_row1_col2\" class=\"data row1 col2\" >-0.6%</td>\n",
       "      <td id=\"T_b716f_row1_col3\" class=\"data row1 col3\" >90.4%</td>\n",
       "      <td id=\"T_b716f_row1_col4\" class=\"data row1 col4\" >-0.6%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_b716f_level0_row2\" class=\"row_heading level0 row2\" rowspan=\"2\">CIFAR-10</th>\n",
       "      <th id=\"T_b716f_level1_row2\" class=\"row_heading level1 row2\" >Contrastive + Structure (Baseline)</th>\n",
       "      <td id=\"T_b716f_row2_col0\" class=\"data row2 col0\" >70.1%</td>\n",
       "      <td id=\"T_b716f_row2_col1\" class=\"data row2 col1\" >51.2%</td>\n",
       "      <td id=\"T_b716f_row2_col2\" class=\"data row2 col2\" >-18.9%</td>\n",
       "      <td id=\"T_b716f_row2_col3\" class=\"data row2 col3\" >46.0%</td>\n",
       "      <td id=\"T_b716f_row2_col4\" class=\"data row2 col4\" >-24.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_b716f_level1_row3\" class=\"row_heading level1 row3\" >Contrastive + Structure + Behavior</th>\n",
       "      <td id=\"T_b716f_row3_col0\" class=\"data row3 col0\" >70.1%</td>\n",
       "      <td id=\"T_b716f_row3_col1\" class=\"data row3 col1\" >69.5%</td>\n",
       "      <td id=\"T_b716f_row3_col2\" class=\"data row3 col2\" >-0.6%</td>\n",
       "      <td id=\"T_b716f_row3_col3\" class=\"data row3 col3\" >69.5%</td>\n",
       "      <td id=\"T_b716f_row3_col4\" class=\"data row3 col4\" >-0.6%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_b716f_level0_row4\" class=\"row_heading level0 row4\" rowspan=\"2\">EuroSAT</th>\n",
       "      <th id=\"T_b716f_level1_row4\" class=\"row_heading level1 row4\" >Contrastive + Structure (Baseline)</th>\n",
       "      <td id=\"T_b716f_row4_col0\" class=\"data row4 col0\" >88.5%</td>\n",
       "      <td id=\"T_b716f_row4_col1\" class=\"data row4 col1\" >68.6%</td>\n",
       "      <td id=\"T_b716f_row4_col2\" class=\"data row4 col2\" >-19.9%</td>\n",
       "      <td id=\"T_b716f_row4_col3\" class=\"data row4 col3\" >56.5%</td>\n",
       "      <td id=\"T_b716f_row4_col4\" class=\"data row4 col4\" >-32.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_b716f_level1_row5\" class=\"row_heading level1 row5\" >Contrastive + Structure + Behavior</th>\n",
       "      <td id=\"T_b716f_row5_col0\" class=\"data row5 col0\" >88.5%</td>\n",
       "      <td id=\"T_b716f_row5_col1\" class=\"data row5 col1\" >87.7%</td>\n",
       "      <td id=\"T_b716f_row5_col2\" class=\"data row5 col2\" >-0.8%</td>\n",
       "      <td id=\"T_b716f_row5_col3\" class=\"data row5 col3\" >87.5%</td>\n",
       "      <td id=\"T_b716f_row5_col4\" class=\"data row5 col4\" >-1.0%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f7c75e409d0>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_acc = zoo_acc.groupby(['Model zoo', 'Hyper-representation model'])[['Original accuracy', 'Reconstruction accuracy']].max()\n",
    "\n",
    "# Adding max performance data for the entire zoo\n",
    "max_acc['Zoo accuracy'] = max_acc.index.get_level_values('Model zoo').map(df_zoo.groupby(['Model zoo'])['Test accuracy'].max().to_dict())\n",
    "\n",
    "# This data corresponds to the test split of the model zoo only\n",
    "max_acc['Test split accuracy'] = max_acc['Original accuracy']\n",
    "max_acc = max_acc.drop('Original accuracy', axis=1)\n",
    "\n",
    "# Adding max performance data for generated models\n",
    "max_acc['Generation accuracy'] = gen_acc.groupby(['Model zoo', 'Hyper-representation model'])['Accuracy'].max()\n",
    "\n",
    "# Computing max performance difference between reconstructed / generated models and the entire zoo\n",
    "max_acc['$\\Delta_{Acc}$ Reconstruction'] = max_acc['Reconstruction accuracy'] - max_acc['Zoo accuracy']\n",
    "max_acc['$\\Delta_{Acc}$ Generation'] = max_acc['Generation accuracy'] - max_acc['Zoo accuracy']\n",
    "\n",
    "# Re-ordering lines\n",
    "new_index = pd.MultiIndex.from_product([DS_ORDER, ['Contrastive + Structure (Baseline)', 'Contrastive + Structure + Behavior']], names=['Model zoo', 'Hyper-representation model'])\n",
    "max_acc = max_acc.reindex(new_index).loc[:, ['Zoo accuracy', 'Reconstruction accuracy', '$\\Delta_{Acc}$ Reconstruction', 'Generation accuracy', '$\\Delta_{Acc}$ Generation']]\n",
    "\n",
    "# Print\n",
    "max_acc = max_acc.style.format('{:.1%}')\n",
    "print(max_acc.to_latex())\n",
    "max_acc"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
