{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "87da8dfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pydgn\n",
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "import json\n",
    "import os.path as osp\n",
    "import pandas as pd\n",
    "sns.color_palette(\"colorblind\", as_cmap=True)\n",
    "sns.set_palette(\"colorblind\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "89c57134",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_root = 'DATA'\n",
    "dataset_name = 'NCI1'\n",
    "exp_folder = f'GSPN_RESULTS/SUPERVISED/supervised_gspn_classification_chemical_{dataset_name}/MODEL_ASSESSMENT/'\n",
    "\n",
    "outer_fold = 1\n",
    "outer_folder = osp.join(exp_folder, f'OUTER_FOLD_{outer_fold}')\n",
    "ms_folder = osp.join(outer_folder, 'MODEL_SELECTION')\n",
    "\n",
    "# config_id = 15  # best config for the unsupervised part according to regression task\n",
    "# config_folder = osp.join(ms_folder, f'config_{config_id}')\n",
    "\n",
    "# model_config_file = osp.join(config_folder, 'config_results.json')\n",
    "# config = json.load(open(model_config_file, 'r'))['config']\n",
    "# results = json.load(open(model_config_file, 'r'))\n",
    "# print(json.dumps(results, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c81e96e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>outer_fold</th>\n",
       "      <th>num_mixtures</th>\n",
       "      <th>num_graph_mixtures</th>\n",
       "      <th>num_layers</th>\n",
       "      <th>global_readout</th>\n",
       "      <th>avg_validation_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>32</td>\n",
       "      <td>20</td>\n",
       "      <td>mean</td>\n",
       "      <td>74.864868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>128</td>\n",
       "      <td>10</td>\n",
       "      <td>mean</td>\n",
       "      <td>75.675674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>sum</td>\n",
       "      <td>78.648651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>mean</td>\n",
       "      <td>73.243240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>sum</td>\n",
       "      <td>76.486488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>10</td>\n",
       "      <td>40</td>\n",
       "      <td>128</td>\n",
       "      <td>10</td>\n",
       "      <td>mean</td>\n",
       "      <td>75.135132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>10</td>\n",
       "      <td>20</td>\n",
       "      <td>128</td>\n",
       "      <td>20</td>\n",
       "      <td>mean</td>\n",
       "      <td>73.243240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>10</td>\n",
       "      <td>20</td>\n",
       "      <td>128</td>\n",
       "      <td>5</td>\n",
       "      <td>sum</td>\n",
       "      <td>73.783783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>10</td>\n",
       "      <td>40</td>\n",
       "      <td>128</td>\n",
       "      <td>20</td>\n",
       "      <td>mean</td>\n",
       "      <td>78.378380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>10</td>\n",
       "      <td>40</td>\n",
       "      <td>32</td>\n",
       "      <td>20</td>\n",
       "      <td>mean</td>\n",
       "      <td>74.594597</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    outer_fold num_mixtures num_graph_mixtures num_layers global_readout  \\\n",
       "0            1           20                 32         20           mean   \n",
       "1            1           20                128         10           mean   \n",
       "2            1           20                 32          5            sum   \n",
       "3            1           20                 32          5           mean   \n",
       "4            1           40                 32          5            sum   \n",
       "..         ...          ...                ...        ...            ...   \n",
       "235         10           40                128         10           mean   \n",
       "236         10           20                128         20           mean   \n",
       "237         10           20                128          5            sum   \n",
       "238         10           40                128         20           mean   \n",
       "239         10           40                 32         20           mean   \n",
       "\n",
       "     avg_validation_score  \n",
       "0               74.864868  \n",
       "1               75.675674  \n",
       "2               78.648651  \n",
       "3               73.243240  \n",
       "4               76.486488  \n",
       "..                    ...  \n",
       "235             75.135132  \n",
       "236             73.243240  \n",
       "237             73.783783  \n",
       "238             78.378380  \n",
       "239             74.594597  \n",
       "\n",
       "[240 rows x 6 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(columns=['outer_fold', \n",
    "                          'num_mixtures',\n",
    "                          'num_graph_mixtures',\n",
    "                          'num_layers',\n",
    "                          'global_readout',\n",
    "                          'avg_validation_score'])\n",
    "\n",
    "for outer_fold in range(1, 11):\n",
    "    outer_folder = osp.join(exp_folder, f'OUTER_FOLD_{outer_fold}')\n",
    "    ms_folder = osp.join(outer_folder, 'MODEL_SELECTION')\n",
    "    for d in os.listdir(ms_folder):\n",
    "        if os.path.isfile(osp.join(ms_folder, d)):\n",
    "            continue\n",
    "\n",
    "        config_folder = osp.join(ms_folder, d)\n",
    "        model_config_file = osp.join(config_folder, 'config_results.json')\n",
    "        config = json.load(open(model_config_file, 'r'))['config']\n",
    "        results = json.load(open(model_config_file, 'r'))\n",
    "\n",
    "        num_mixtures = config['supervised_config']['num_mixtures']\n",
    "        num_graph_mixtures = config['supervised_config']['num_graph_mixtures']\n",
    "\n",
    "        num_layers = config['supervised_config']['num_layers']\n",
    "        global_readout = config['supervised_config']['global_readout']\n",
    "\n",
    "        vl_score = results['avg_validation_score']\n",
    "\n",
    "        df = df.append({'outer_fold' : outer_fold,\n",
    "                        'num_mixtures' : num_mixtures,\n",
    "                        'num_graph_mixtures' : num_graph_mixtures,\n",
    "                        'num_layers' : num_layers,\n",
    "                        'global_readout' : global_readout,\n",
    "                        'avg_validation_score' : vl_score},                       \n",
    "                        ignore_index=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b16c4fa8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    num_mixtures  num_graph_mixtures  num_layers global_readout  \\\n",
      "0             20                  32           5           mean   \n",
      "1             20                  32           5            sum   \n",
      "2             20                  32          10           mean   \n",
      "3             20                  32          10            sum   \n",
      "4             20                  32          20           mean   \n",
      "5             20                  32          20            sum   \n",
      "6             20                 128           5           mean   \n",
      "7             20                 128           5            sum   \n",
      "8             20                 128          10           mean   \n",
      "9             20                 128          10            sum   \n",
      "10            20                 128          20           mean   \n",
      "11            20                 128          20            sum   \n",
      "12            40                  32           5           mean   \n",
      "13            40                  32           5            sum   \n",
      "14            40                  32          10           mean   \n",
      "15            40                  32          10            sum   \n",
      "16            40                  32          20           mean   \n",
      "17            40                  32          20            sum   \n",
      "18            40                 128           5           mean   \n",
      "19            40                 128           5            sum   \n",
      "20            40                 128          10           mean   \n",
      "21            40                 128          10            sum   \n",
      "22            40                 128          20           mean   \n",
      "23            40                 128          20            sum   \n",
      "\n",
      "    avg_validation_score  \n",
      "0              71.918919  \n",
      "1              77.216216  \n",
      "2              74.216218  \n",
      "3              78.189188  \n",
      "4              75.702705  \n",
      "5              79.432433  \n",
      "6              71.540540  \n",
      "7              76.243243  \n",
      "8              73.648648  \n",
      "9              79.702702  \n",
      "10             75.756755  \n",
      "11             79.540540  \n",
      "12             71.837837  \n",
      "13             77.675675  \n",
      "14             74.891894  \n",
      "15             79.243243  \n",
      "16             75.621622  \n",
      "17             79.135136  \n",
      "18             71.918920  \n",
      "19             77.324324  \n",
      "20             75.729729  \n",
      "21             79.027026  \n",
      "22             76.945946  \n",
      "23             80.432433  \n"
     ]
    }
   ],
   "source": [
    "df_gb = df.groupby(by=['num_mixtures',\n",
    "               'num_graph_mixtures',\n",
    "               'num_layers',\n",
    "               'global_readout'], as_index=False).mean()\n",
    "print(df_gb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "11a606b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 40816 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if not os.path.exists('plots'):\n",
    "    os.makedirs('plots')\n",
    "    \n",
    "plt.rcParams.update({'font.size': 12})\n",
    "\n",
    "renamed_df = df.rename(columns={'num_graph_mixtures': r'$C_G$', 'num_mixtures': r'$C$'})\n",
    "sns.lineplot(x='num_layers', \n",
    "             y='avg_validation_score', \n",
    "             hue=r'$C_G$', \n",
    "             style=r'$C$', \n",
    "             data=renamed_df[renamed_df['global_readout'] == 'sum'],\n",
    "             palette=sns.color_palette('colorblind', n_colors=2))\n",
    "plt.xlabel('number of layers')\n",
    "plt.ylabel('validation accuracy')\n",
    "plt.tight_layout()\n",
    "plt.savefig('plots/layers_vs_perf.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "929f7bc9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
