{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3864b2e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from libs.data import TabularDataset\n",
    "import xgboost as xgb\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c90084ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def BinBoundaries(dataset, X_num, bin_size, device):\n",
    "    init_num = torch.linspace(0, 1, bin_size + 1, device=device)\n",
    "    init_bin_boundaries = dict()\n",
    "    for f in range(dataset.size(1)):\n",
    "        if f in X_num:\n",
    "            init_bin_boundaries[f] = init_num\n",
    "        else:\n",
    "            init_bin_boundaries[f] = None\n",
    "    return init_bin_boundaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "01f28e0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import wasserstein_distance\n",
    "from scipy.spatial.distance import euclidean\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "def calculate_emd(original, augmented):\n",
    "    emd_values = [wasserstein_distance(original[:, i], augmented[:, i]) for i in range(original.shape[1])]\n",
    "    return np.mean(emd_values)\n",
    "\n",
    "def calculate_average_euclidean_distance(original, augmented):\n",
    "    distances = [euclidean(original[i], augmented[i]) for i in range(len(original))]\n",
    "    return np.mean(distances)\n",
    "\n",
    "def calculate_cosine_similarity(original, augmented):\n",
    "    similarities = [cosine_similarity(original[i].reshape(1, -1), augmented[i].reshape(1, -1))[0][0] for i in range(len(original))]\n",
    "    return np.mean(similarities)\n",
    "\n",
    "def get_results(transform_func, data, true_label):\n",
    "    sample = transform_func({\"image\": torch.tensor(data), \"mask\": None})\n",
    "    preds = model.predict(sample[\"image\"].cpu().numpy())\n",
    "    probs = model.predict_proba(sample[\"image\"].cpu().numpy())\n",
    "    acc, auc = evaluate(preds, true_label, prediction_proba=probs, tasktype=\"multiclass\")\n",
    "    was = calculate_emd(data, sample[\"image\"].cpu().numpy())\n",
    "    euc = calculate_average_euclidean_distance(data, sample[\"image\"].cpu().numpy())\n",
    "    cos = calculate_cosine_similarity(data, sample[\"image\"].cpu().numpy())\n",
    "    return acc, auc, was, euc, cos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4a05caa4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19, 7)\n"
     ]
    }
   ],
   "source": [
    "# results = pd.DataFrame(columns=(\"data_id\", \"numbins\", \"acc\", \"auroc\", \"cos\", \"euc\", \"was\"))\n",
    "# results.to_csv(\"stats.csv\")\n",
    "results = pd.read_csv(\"stats.csv\", index_col=0)\n",
    "print(results.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "87f958b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset is loaded.. Data name: satimage, Target feature: class\n",
      "input dim: 36, cat: 0, num: 36\n",
      "Classes:  6\n",
      "torch.Size([5144, 36])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "openml_id = 182\n",
    "tasktype = \"multiclass\"\n",
    "device = \"cpu\"\n",
    "\n",
    "dataset = TabularDataset(openml_id, tasktype, device, labeled_data=0, cat_threshold=20, seed=1, modelname=\"xgboost\")\n",
    "(X_train, y_train), (X_val, y_val), (X_test, y_test) = dataset._indv_dataset()\n",
    "\n",
    "if tasktype == \"multiclass\":\n",
    "    print(\"Classes: \", y_train.size(1))\n",
    "    y_train = np.argmax(y_train.cpu().numpy(), axis=1)\n",
    "    y_val = np.argmax(y_val.cpu().numpy(), axis=1)\n",
    "    y_test = np.argmax(y_test.cpu().numpy(), axis=1)\n",
    "else:\n",
    "    y_train = y_train.cpu().numpy()\n",
    "    y_val = y_val.cpu().numpy()\n",
    "    y_test = y_test.cpu().numpy()\n",
    "print(X_train.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "1fce4eed",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.cpu().numpy()\n",
    "X_val = X_val.cpu().numpy()\n",
    "X_test = X_test.cpu().numpy()\n",
    "\n",
    "if tasktype == \"multiclass\":\n",
    "    model = xgb.XGBClassifier(\n",
    "            tree_method='auto', objective=\"multi:softmax\",\n",
    "            gpu_id=0, booster='gbtree', n_estimators=100,\n",
    "            max_depth=3, subsample=1, eta=0.1, random_state=1\n",
    "    ).fit(X_train, y_train)\n",
    "else:\n",
    "    model = xgb.XGBClassifier(\n",
    "            tree_method='auto', objective=\"binary:logistic\",\n",
    "            gpu_id=0, booster='gbtree', n_estimators=100,\n",
    "            max_depth=3, subsample=1, eta=0.1, random_state=1\n",
    "    ).fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "c185a4ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "from libs.transform import *\n",
    "from libs.utils import evaluate\n",
    "import pandas as pd\n",
    "i = len(results)\n",
    "\n",
    "# results = pd.DataFrame(columns=(\"data_id\", \"numbins\", \"acc\", \"auroc\", \"cos\", \"euc\", \"was\"))\n",
    "for b in np.arange(1, 20):\n",
    "    bin_boundaries = BinBoundaries(torch.tensor(X_train), dataset.X_num, b, \"cpu\")\n",
    "    binshuffling = BinShuffling(1, bin_boundaries, sampling_alpha=False)\n",
    "    \n",
    "    preds = model.predict(X_test)\n",
    "    probs = model.predict_proba(X_test)\n",
    "    acc, auc = evaluate(preds, y_test, prediction_proba=probs, tasktype=\"multiclass\")\n",
    "\n",
    "    acc3, auc3, was3, euc3, cos3 = get_results(binshuffling, X_test, y_test)\n",
    "    \n",
    "    results.loc[i] = [openml_id, b, acc3, auc3, cos3, euc3, was3]; i += 1\n",
    "    \n",
    "results.to_csv(\"stats.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a995527f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bb2e4a1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ff297c9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "06ce0e83",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data_id</th>\n",
       "      <th>numbins</th>\n",
       "      <th>acc</th>\n",
       "      <th>auroc</th>\n",
       "      <th>cos</th>\n",
       "      <th>euc</th>\n",
       "      <th>was</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4538.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.247976</td>\n",
       "      <td>0.485194</td>\n",
       "      <td>0.763272</td>\n",
       "      <td>2.324781</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4538.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.401822</td>\n",
       "      <td>0.688578</td>\n",
       "      <td>0.939658</td>\n",
       "      <td>1.138911</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4538.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.475709</td>\n",
       "      <td>0.749309</td>\n",
       "      <td>0.972150</td>\n",
       "      <td>0.765301</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4538.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.487854</td>\n",
       "      <td>0.776273</td>\n",
       "      <td>0.984246</td>\n",
       "      <td>0.571783</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4538.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.524291</td>\n",
       "      <td>0.791588</td>\n",
       "      <td>0.989610</td>\n",
       "      <td>0.460565</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>182.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.889580</td>\n",
       "      <td>0.985292</td>\n",
       "      <td>0.997665</td>\n",
       "      <td>0.151061</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>182.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.883359</td>\n",
       "      <td>0.985577</td>\n",
       "      <td>0.997981</td>\n",
       "      <td>0.141288</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>182.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.900467</td>\n",
       "      <td>0.987391</td>\n",
       "      <td>0.998314</td>\n",
       "      <td>0.132521</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>182.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.895801</td>\n",
       "      <td>0.986499</td>\n",
       "      <td>0.998527</td>\n",
       "      <td>0.121275</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>182.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.884914</td>\n",
       "      <td>0.986404</td>\n",
       "      <td>0.998684</td>\n",
       "      <td>0.116268</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>152 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     data_id  numbins       acc     auroc       cos       euc  was\n",
       "0     4538.0      1.0  0.247976  0.485194  0.763272  2.324781  0.0\n",
       "1     4538.0      2.0  0.401822  0.688578  0.939658  1.138911  0.0\n",
       "2     4538.0      3.0  0.475709  0.749309  0.972150  0.765301  0.0\n",
       "3     4538.0      4.0  0.487854  0.776273  0.984246  0.571783  0.0\n",
       "4     4538.0      5.0  0.524291  0.791588  0.989610  0.460565  0.0\n",
       "..       ...      ...       ...       ...       ...       ...  ...\n",
       "147    182.0     15.0  0.889580  0.985292  0.997665  0.151061  0.0\n",
       "148    182.0     16.0  0.883359  0.985577  0.997981  0.141288  0.0\n",
       "149    182.0     17.0  0.900467  0.987391  0.998314  0.132521  0.0\n",
       "150    182.0     18.0  0.895801  0.986499  0.998527  0.121275  0.0\n",
       "151    182.0     19.0  0.884914  0.986404  0.998684  0.116268  0.0\n",
       "\n",
       "[152 rows x 7 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "094beab8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 35276 characters]",
      "text/plain": [
       "<Figure size 1000x200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 45364 characters]",
      "text/plain": [
       "<Figure size 1000x200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 33156 characters]",
      "text/plain": [
       "<Figure size 1000x200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 31712 characters]",
      "text/plain": [
       "<Figure size 1000x200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 38328 characters]",
      "text/plain": [
       "<Figure size 1000x200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 34704 characters]",
      "text/plain": [
       "<Figure size 1000x200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 36036 characters]",
      "text/plain": [
       "<Figure size 1000x200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 36896 characters]",
      "text/plain": [
       "<Figure size 1000x200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_id = 4538\n",
    "\n",
    "for data_id in [4538, 23, 32, 22, 1492, 458, 40499, 182]:\n",
    "    p = results[results[\"data_id\"] == data_id]\n",
    "\n",
    "    fig, ax = plt.subplots(1, 4, figsize=(10, 2))\n",
    "    ax.ravel()\n",
    "\n",
    "    ax[0].plot(p[\"numbins\"].values, p[\"acc\"].values, marker=\".\")\n",
    "    ax[0].set_title(\"Acc\")\n",
    "    ax[1].plot(p[\"numbins\"].values, p[\"auroc\"].values, marker=\".\")\n",
    "    ax[1].set_title(\"AUROC\")\n",
    "    ax[2].plot(p[\"numbins\"].values, p[\"cos\"].values, marker=\".\")\n",
    "    ax[2].set_title(\"Cos\")\n",
    "    ax[3].plot(p[\"numbins\"].values, p[\"euc\"].values, marker=\".\")\n",
    "    ax[3].set_title(\"Euc\")\n",
    "    # ax[4].plot(p[\"numbins\"].values, p[\"was\"].values, marker=\"o\")\n",
    "    # ax[4].set_title(\"Was\")\n",
    "    for i in range(4):\n",
    "        ax[i].grid(True)\n",
    "        ax[i].set_xlim(1, 19)\n",
    "\n",
    "    fig.suptitle(data_id, y=1.1)\n",
    "    plt.subplots_adjust(wspace=0.3)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4ab5a45",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "31f93b7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from libs.transform import *\n",
    "from libs.utils import evaluate\n",
    "import pandas as pd\n",
    "result = pd.DataFrame(columns=(\"p\", \"augmentation\", \"acc\", \"auc\", \"was\", \"euc\", \"cos\"))\n",
    "i = 0\n",
    "\n",
    "for p in np.arange(0, 1.01, 0.01):\n",
    "    masking = Masking(p, sampling_alpha=False)\n",
    "    shuffling = Shuffling(p, sampling_alpha=False)\n",
    "\n",
    "    bin_boundaries = BinBoundaries(torch.tensor(X_train), dataset.X_num, 20, \"cpu\")\n",
    "    binshuffling = BinShuffling(p, bin_boundaries, sampling_alpha=False)\n",
    "    \n",
    "    preds = model.predict(X_test)\n",
    "    probs = model.predict_proba(X_test)\n",
    "    acc, auc = evaluate(preds, y_test, prediction_proba=probs, tasktype=\"multiclass\")\n",
    "\n",
    "    acc1, auc1, was1, euc1, cos1 = get_results(masking, X_test, y_test)\n",
    "    acc2, auc2, was2, euc2, cos2 = get_results(shuffling, X_test, y_test)\n",
    "    acc3, auc3, was3, euc3, cos3 = get_results(binshuffling, X_test, y_test)\n",
    "    \n",
    "    result.loc[i] = [p, \"masking\", acc1, auc1, was1, euc1, cos1]; i += 1\n",
    "    result.loc[i] = [p, \"shuffling\", acc2, auc2, was2, euc2, cos2]; i += 1\n",
    "    result.loc[i] = [p, \"binshuffling\", acc3, auc3, was3, euc3, cos3]; i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f6e0b002",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 41176 characters]",
      "text/plain": [
       "<Figure size 500x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure(figsize=(5, 4))\n",
    "for l in [\"masking\", \"shuffling\", \"binshuffling\"]:\n",
    "    q = result[result[\"augmentation\"] == l]\n",
    "#     plt.plot(q[\"p\"].values, q[\"acc\"].values, label=l.capitalize())\n",
    "    plt.plot(q[\"p\"].values, q[\"cos\"].values, label=l.capitalize())\n",
    "plt.rc('axes', axisbelow=True)\n",
    "plt.grid()\n",
    "# plt.hlines(y=acc, xmin=-0.5, xmax=1.5, color=\"red\", linewidth=0.8, linestyle=\"dashed\", label=\"Original data\")\n",
    "plt.xlim(0, 1)\n",
    "# plt.yscale(\"log\")\n",
    "plt.xlabel(\"Masking probability\")\n",
    "plt.ylabel(\"Euclidean distance (log scale)\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45b1f7be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4ff5335",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc8e7960",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5815ec4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'test_rmse': 0.009835589761657363,\n",
       " 'test_mae': 0.0075706504,\n",
       " 'test_recall': 0.3333333333333333,\n",
       " 'test_overkill': 0.8181818181818182,\n",
       " 'tp': 4,\n",
       " 'tn': 18580,\n",
       " 'fn': 8,\n",
       " 'fp': 18,\n",
       " 'thr': 0.47146407}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.load(\"/home/hero_aa/2024_anomaly/results/8_new_data/model=mlp/eqp=008/num_bins=100..reg=None/rq=100/validation-ep100/finetuning-20/lr=0.0002/result.npy\", allow_pickle=True).item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5edfa0ae",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tabular",
   "language": "python",
   "name": "tabular"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
