{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f870191f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch, os\n",
    "from torch import nn\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "023736b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(6)\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "if device == 'cuda':\n",
    "    torch.backends.cudnn.benchmark = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2f9215a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mlp(dim, hidden_dim, output_dim, layers, activation='relu', normalization=None):\n",
    "    activation = {\n",
    "        'relu': nn.ReLU\n",
    "    }[activation]\n",
    "\n",
    "    if isinstance(dim, list):\n",
    "        dim = np.prod(dim)\n",
    "\n",
    "    if normalization == \"layernorm\":\n",
    "        seq = [nn.Linear(dim, hidden_dim), nn.LayerNorm(hidden_dim), activation()]\n",
    "        for _ in range(layers):\n",
    "            seq += [nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim), activation()]\n",
    "        seq += [nn.Linear(hidden_dim, output_dim)]\n",
    "    elif normalization == \"batchnorm\":\n",
    "        seq = [nn.Linear(dim, hidden_dim), nn.BatchNorm1d(hidden_dim), activation()]\n",
    "        for _ in range(layers):\n",
    "            seq += [nn.Linear(hidden_dim, hidden_dim), nn.BatchNorm1d(hidden_dim), activation()]\n",
    "        seq += [nn.Linear(hidden_dim, output_dim)]\n",
    "    else:\n",
    "        seq = [nn.Linear(dim, hidden_dim), activation()]\n",
    "        for _ in range(layers):\n",
    "            seq += [nn.Linear(hidden_dim, hidden_dim), activation()]\n",
    "        seq += [nn.Linear(hidden_dim, output_dim)]\n",
    "\n",
    "    return nn.Sequential(*seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "214e05c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class JointCritic(nn.Module):\n",
    "    \"\"\"Concat critic, where we concat the inputs and use one MLP to output the value.\"\"\"\n",
    "\n",
    "    def __init__(self, dim, hidden_dim, layers, activation=\"relu\", normalization=\"None\", **extra_kwargs):\n",
    "        super(JointCritic, self).__init__()\n",
    "        # output is scalar score\n",
    "        self._f = mlp(dim * 2, hidden_dim, 1, layers, activation, normalization)\n",
    "\n",
    "    def forward(self, x, y):\n",
    "        batch_size = x.shape[0]\n",
    "        # Tile all possible combinations of x and y\n",
    "        x_tiled = torch.stack([x] * batch_size, dim=0)\n",
    "        y_tiled = torch.stack([y] * batch_size, dim=1)\n",
    "        # xy is [batch_size * batch_size, x_dim + y_dim]\n",
    "        xy_pairs = torch.reshape(torch.cat((x_tiled, y_tiled), dim=2), [\n",
    "                                 batch_size * batch_size, -1])\n",
    "        # Compute scores for each x_i, y_j pair.\n",
    "        scores = self._f(xy_pairs)\n",
    "        return torch.reshape(scores, [batch_size, batch_size]).t()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4791206c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def estimate_mutual_information(estimator, x, y, critic_fn,\n",
    "                                baseline_fn=None, alpha_logit=None, buffer=None,\n",
    "                                return_scores=False,\n",
    "                                **kwargs):\n",
    "    \"\"\"Estimate variational lower bounds on mutual information.\n",
    "\n",
    "  Args:\n",
    "    estimator: string specifying estimator, one of:\n",
    "      'nwj', 'infonce', 'tuba', 'js', 'interpolated'\n",
    "    x: [batch_size, dim_x] Tensor\n",
    "    y: [batch_size, dim_y] Tensor\n",
    "    critic_fn: callable that takes x and y as input and outputs critic scores\n",
    "      output shape is a [batch_size, batch_size] matrix\n",
    "    baseline_fn (optional): callable that takes y as input\n",
    "      outputs a [batch_size]  or [batch_size, 1] vector\n",
    "    alpha_logit (optional): logit(alpha) for interpolated bound\n",
    "\n",
    "  Returns:\n",
    "    scalar estimate of mutual information\n",
    "    \"\"\"\n",
    "    x, y = x.to(device), y.to(device)\n",
    "    scores = critic_fn(x, y)\n",
    "    if baseline_fn is not None:\n",
    "        # Some baselines' output is (batch_size, 1) which we remove here.\n",
    "        log_baseline = torch.squeeze(baseline_fn(y))\n",
    "    if estimator == 'infonce':\n",
    "        mi = infonce_lower_bound(scores)\n",
    "    elif estimator == 'nwj':\n",
    "        mi = nwj_lower_bound(scores)\n",
    "    elif estimator == 'tuba':\n",
    "        mi = tuba_lower_bound(scores, log_baseline)\n",
    "    elif estimator == 'js':\n",
    "        mi = js_lower_bound(scores)\n",
    "    elif estimator == \"smile-inf\":\n",
    "        mi = smile_lower_bound(scores, clip=np.inf)\n",
    "    elif estimator.startswith('smile') & (not estimator.endswith(\"source\")):\n",
    "        clip = eval(estimator.split(\"-\")[-1])\n",
    "        mi = smile_lower_bound(scores, clip=clip)\n",
    "    elif estimator == 'dv':\n",
    "        mi = dv_upper_lower_bound(scores)\n",
    "    elif estimator == \"mine\":\n",
    "        mi, buffer = mine_lower_bound(scores, buffer=buffer, momentum=0.9)\n",
    "\n",
    "    if return_scores & (estimator == \"mine\"):\n",
    "        return mi, buffer, scores\n",
    "    elif return_scores:\n",
    "        return mi, scores\n",
    "    elif estimator == \"mine\":\n",
    "        return mi, buffer\n",
    "    else:\n",
    "        return mi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "525d8f42",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mine_lower_bound(f, buffer=None, momentum=0.9):\n",
    "    \"\"\"\n",
    "    MINE lower bound based on DV inequality.\n",
    "    \"\"\"\n",
    "    if buffer is None:\n",
    "        buffer = torch.tensor(1.0).cuda()\n",
    "    first_term = f.diag().mean()\n",
    "\n",
    "    buffer_update = logmeanexp_nodiag(f).exp()\n",
    "    with torch.no_grad():\n",
    "        second_term = logmeanexp_nodiag(f)\n",
    "        buffer_new = buffer * momentum + buffer_update * (1 - momentum)\n",
    "        buffer_new = torch.clamp(buffer_new, min=1e-4)\n",
    "        third_term_no_grad = buffer_update / buffer_new\n",
    "\n",
    "    third_term_grad = buffer_update / buffer_new\n",
    "\n",
    "    return first_term - second_term - third_term_grad + third_term_no_grad, buffer_update\n",
    "\n",
    "def smile_lower_bound(f, clip=None):\n",
    "    if clip is not None:\n",
    "        f_ = torch.clamp(f, -clip, clip)\n",
    "    else:\n",
    "        f_ = f\n",
    "    z = logmeanexp_nodiag(f_, dim=(0, 1))\n",
    "    dv = f.diag().mean() - z\n",
    "\n",
    "    js = js_fgan_lower_bound(f)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        dv_js = dv - js\n",
    "\n",
    "    return js + dv_js\n",
    "\n",
    "def js_fgan_lower_bound(f):\n",
    "    \"\"\"Lower bound on Jensen-Shannon divergence from Nowozin et al. (2016).\"\"\"\n",
    "    f_diag = f.diag()\n",
    "    first_term = -torch.nn.functional.softplus(-f_diag).mean()\n",
    "    n = f.size(0)\n",
    "    second_term = (torch.sum(torch.nn.functional.softplus(f)) -\n",
    "                   torch.sum(torch.nn.functional.softplus(f_diag))) / (n * (n - 1.))\n",
    "    return first_term - second_term"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "de1cccb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def logmeanexp_nodiag(x, dim=None, device='cuda'):\n",
    "    batch_size = x.size(0)\n",
    "    if dim is None:\n",
    "        dim = (0, 1)\n",
    "\n",
    "    logsumexp = torch.logsumexp(\n",
    "        x - torch.diag(np.inf * torch.ones(batch_size).to(device)), dim=dim)\n",
    "\n",
    "    try:\n",
    "        if len(dim) == 1:\n",
    "            num_elem = batch_size - 1.\n",
    "        else:\n",
    "            num_elem = batch_size * (batch_size - 1.)\n",
    "    except ValueError:\n",
    "        num_elem = batch_size - 1\n",
    "    return logsumexp - torch.log(torch.tensor(num_elem)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f4d3e78",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "84ba6668",
   "metadata": {},
   "outputs": [],
   "source": [
    "from libs.data import TabularDataset\n",
    "from libs.transform import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0bcdd0b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def BinBoundaries(dataset, X_num, bin_size, device):\n",
    "    init_num = torch.linspace(0, 1, bin_size + 1, device=device)\n",
    "    init_bin_boundaries = dict()\n",
    "    for f in range(dataset.size(1)):\n",
    "        if f in X_num:\n",
    "            init_bin_boundaries[f] = init_num\n",
    "        else:\n",
    "            init_bin_boundaries[f] = None\n",
    "    return init_bin_boundaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6d7705a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.\n",
      "  warnings.warn(\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset is loaded.. Data name: analcatdata_authorship, Target feature: Author\n",
      "input dim: 70, cat: 29, num: 41\n",
      "Classes:  4\n",
      "torch.Size([672, 70])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/tabular/lib/python3.9/site-packages/sklearn/preprocessing/_data.py:2627: UserWarning: n_quantiles (1000) is greater than the total number of samples (672). n_quantiles is set to n_samples.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "openml_id = 458\n",
    "tasktype = \"multiclass\"\n",
    "\n",
    "dataset = TabularDataset(openml_id, tasktype, device, labeled_data=0, cat_threshold=20, seed=1, modelname=\"xgboost\")\n",
    "(X_train, y_train), (X_val, y_val), (X_test, y_test) = dataset._indv_dataset()\n",
    "\n",
    "if tasktype == \"multiclass\":\n",
    "    print(\"Classes: \", y_train.size(1))\n",
    "    y_train = np.argmax(y_train.cpu().numpy(), axis=1)\n",
    "    y_val = np.argmax(y_val.cpu().numpy(), axis=1)\n",
    "    y_test = np.argmax(y_test.cpu().numpy(), axis=1)\n",
    "else:\n",
    "    y_train = y_train.cpu().numpy()\n",
    "    y_val = y_val.cpu().numpy()\n",
    "    y_test = y_test.cpu().numpy()\n",
    "print(X_train.size())\n",
    "\n",
    "X_train = X_train.cpu().numpy()\n",
    "X_val = X_val.cpu().numpy()\n",
    "X_test = X_test.cpu().numpy()\n",
    "\n",
    "dim = X_test.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9621d663",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# mis = pd.DataFrame(columns=(\"data_id\", \"numbins\", \"mis\", \"estimated_mi\", \"std\", \"min\", \"max\"))\n",
    "# mis.to_csv(\"mis-mine.csv\")\n",
    "# mis = pd.read_csv(\"mis-smile-inf.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f819ed27",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0 0.020448685\n",
      "0 200 16.538137\n",
      "0 400 8.625386\n",
      "0 600 2.653543\n",
      "0 800 37.62349\n",
      "0 1000 26.600195\n",
      "0 1200 2.9550295\n",
      "0 1400 12.596814\n",
      "0 1600 5.883375\n",
      "0 1800 23.064278\n",
      "0 2000 24.617334\n",
      "1 0 0.007958531\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/home/tabsemi/libs/transform.py:192: UserWarning: torch.searchsorted(): input value tensor is non-contiguous, this will lower the performance due to extra data copy when converting non-contiguous tensor to contiguous, please use contiguous input value tensor if possible. This message will only appear once per program. (Triggered internally at /opt/conda/conda-bld/pytorch_1682343997789/work/aten/src/ATen/native/BucketizationUtils.h:33.)\n",
      "  idx = torch.bucketize(img[:, f], boundaries, right=True) - 1\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 200 12.850822\n",
      "1 400 10.737989\n",
      "1 600 30.82373\n",
      "1 800 37.060467\n",
      "1 1000 47.06947\n",
      "1 1200 13.683464\n",
      "1 1400 6.880762\n",
      "1 1600 43.664295\n",
      "1 1800 22.69957\n",
      "1 2000 90.17601\n",
      "2 0 0.028280973\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 200 18.810387\n",
      "2 400 11.948334\n",
      "2 600 3.8607903\n",
      "2 800 8.829183\n",
      "2 1000 19.6865\n",
      "2 1200 31.246777\n",
      "2 1400 62.325077\n",
      "2 1600 21.181084\n",
      "2 1800 nan\n",
      "2 2000 nan\n",
      "3 0 0.012694836\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 200 17.4529\n",
      "3 400 21.177668\n",
      "3 600 22.058681\n",
      "3 800 30.290493\n",
      "3 1000 35.54952\n",
      "3 1200 111.55815\n",
      "3 1400 22.629135\n",
      "3 1600 19.235264\n",
      "3 1800 10.936808\n",
      "3 2000 12.868852\n",
      "4 0 0.022105098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 200 7.6177273\n",
      "4 400 27.49345\n",
      "4 600 12.414673\n",
      "4 800 6.4601116\n",
      "4 1000 20.791767\n",
      "4 1200 27.14509\n",
      "4 1400 17.615034\n",
      "4 1600 22.207066\n",
      "4 1800 44.19555\n",
      "4 2000 22.663826\n",
      "5 0 0.02323103\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 200 12.646904\n",
      "5 400 5.0379906\n",
      "5 600 7.080393\n",
      "5 800 19.308533\n",
      "5 1000 0.9953842\n",
      "5 1200 10.550507\n",
      "5 1400 21.755957\n",
      "5 1600 20.417595\n",
      "5 1800 16.098896\n",
      "5 2000 1.6564136\n",
      "6 0 0.010635614\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 200 9.579094\n",
      "6 400 23.235775\n",
      "6 600 11.56867\n",
      "6 800 39.25039\n",
      "6 1000 6.349003\n",
      "6 1200 28.642162\n",
      "6 1400 21.760563\n",
      "6 1600 65.464096\n",
      "6 1800 131.8916\n",
      "6 2000 10.826593\n",
      "7 0 0.012179136\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7 200 12.395914\n",
      "7 400 33.262573\n",
      "7 600 8.940431\n",
      "7 800 16.025177\n",
      "7 1000 13.699283\n",
      "7 1200 8.808279\n",
      "7 1400 63.955376\n",
      "7 1600 11.2872505\n",
      "7 1800 47.942604\n",
      "7 2000 31.068972\n",
      "8 0 0.013889492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 200 23.200413\n",
      "8 400 38.254257\n",
      "8 600 43.747017\n",
      "8 800 7.5148306\n",
      "8 1000 25.260902\n",
      "8 1200 17.293133\n",
      "8 1400 2.8421211\n",
      "8 1600 12.065629\n",
      "8 1800 34.571312\n",
      "8 2000 11.864893\n",
      "9 0 0.019058347\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9 200 8.555689\n",
      "9 400 18.612701\n",
      "9 600 10.605102\n",
      "9 800 45.946762\n",
      "9 1000 9.827658\n",
      "9 1200 12.205917\n",
      "9 1400 14.809736\n",
      "9 1600 11.729019\n",
      "9 1800 37.676216\n",
      "9 2000 19.063276\n",
      "10 0 0.0073525906\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 200 21.621864\n",
      "10 400 18.39641\n",
      "10 600 30.862087\n",
      "10 800 36.69252\n",
      "10 1000 65.70222\n",
      "10 1200 36.477215\n",
      "10 1400 42.311867\n",
      "10 1600 76.48491\n",
      "10 1800 25.954624\n",
      "10 2000 22.748596\n",
      "11 0 0.02376914\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11 200 20.503761\n",
      "11 400 32.681625\n",
      "11 600 30.471926\n",
      "11 800 11.701877\n",
      "11 1000 31.421711\n",
      "11 1200 21.539314\n",
      "11 1400 16.342148\n",
      "11 1600 27.507717\n",
      "11 1800 16.411911\n",
      "11 2000 119.791466\n",
      "12 0 0.024297774\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 200 19.295887\n",
      "12 400 12.601341\n",
      "12 600 10.106285\n",
      "12 800 10.347508\n",
      "12 1000 22.49419\n",
      "12 1200 -3.8603363\n",
      "12 1400 87.52559\n",
      "12 1600 21.012989\n",
      "12 1800 10.738381\n",
      "12 2000 17.27935\n",
      "13 0 0.028173804\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13 200 25.543789\n",
      "13 400 41.35234\n",
      "13 600 31.563267\n",
      "13 800 10.079128\n",
      "13 1000 -3.7799835\n",
      "13 1200 33.49786\n",
      "13 1400 68.006836\n",
      "13 1600 -3.4865532\n",
      "13 1800 35.964066\n",
      "13 2000 56.145306\n",
      "14 0 0.011146963\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14 200 15.488853\n",
      "14 400 33.636726\n",
      "14 600 23.017004\n",
      "14 800 14.615082\n",
      "14 1000 1.115674\n",
      "14 1200 15.025662\n",
      "14 1400 9.516499\n",
      "14 1600 13.778185\n",
      "14 1800 24.907597\n",
      "14 2000 7.673937\n",
      "15 0 0.02310121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15 200 22.107327\n",
      "15 400 34.96032\n",
      "15 600 6.281735\n",
      "15 800 13.515734\n",
      "15 1000 36.66394\n",
      "15 1200 11.203803\n",
      "15 1400 21.056625\n",
      "15 1600 3.8452072\n",
      "15 1800 39.33282\n",
      "15 2000 148.53336\n",
      "16 0 0.019071102\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16 200 21.636347\n",
      "16 400 14.122879\n",
      "16 600 33.755863\n",
      "16 800 1.2058296\n",
      "16 1000 24.68666\n",
      "16 1200 98.1014\n",
      "16 1400 33.54059\n",
      "16 1600 23.536097\n",
      "16 1800 48.85627\n",
      "16 2000 18.185137\n",
      "17 0 0.022081971\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17 200 17.482784\n",
      "17 400 13.613707\n",
      "17 600 45.48319\n",
      "17 800 11.743759\n",
      "17 1000 10.740321\n",
      "17 1200 8.293402\n",
      "17 1400 48.266052\n",
      "17 1600 11.368778\n",
      "17 1800 -20.91354\n",
      "17 2000 5.467993\n",
      "18 0 0.015907168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18 200 13.17353\n",
      "18 400 40.863953\n",
      "18 600 37.239\n",
      "18 800 28.184559\n",
      "18 1000 20.025312\n",
      "18 1200 -13.4621315\n",
      "18 1400 6.7096643\n",
      "18 1600 5.4506307\n",
      "18 1800 3.862577\n",
      "18 2000 9.466761\n",
      "19 0 0.019109964\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1703466/2700193231.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
      "/tmp/ipykernel_1703466/2700193231.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 200 17.137526\n",
      "19 400 7.2456603\n",
      "19 600 7.799402\n",
      "19 800 23.198795\n",
      "19 1000 34.132355\n",
      "19 1200 71.7827\n",
      "19 1400 21.963871\n",
      "19 1600 15.394323\n",
      "19 1800 19.917095\n",
      "19 2000 115.44333\n"
     ]
    }
   ],
   "source": [
    "estimator = \"mine\"\n",
    "fname = f'mis-{estimator}.csv'\n",
    "for b in np.arange(0, 20):\n",
    "    mis = pd.read_csv(fname, index_col=0)\n",
    "    if len(mis[(mis[\"numbins\"] == b) & (mis[\"data_id\"] == openml_id)]) == 0:\n",
    "        critic = JointCritic(dim, hidden_dim=256, layers=2).to(device)\n",
    "        opt_crit = torch.optim.Adam(critic.parameters(), lr=0.0005)\n",
    "\n",
    "        def train_step(x, y, buffer=None):\n",
    "            opt_crit.zero_grad()\n",
    "            if estimator == \"mine\":\n",
    "                mi, buffer = estimate_mutual_information(estimator, x, y, critic, baseline_fn=None, buffer=buffer)\n",
    "            else:\n",
    "                mi = estimate_mutual_information(estimator, x, y, critic, baseline_fn=None, buffer=buffer)\n",
    "            loss = -mi\n",
    "            loss.backward(); opt_crit.step()\n",
    "            return mi, buffer\n",
    "        \n",
    "        if b > 0:\n",
    "            bin_boundaries = BinBoundaries(torch.tensor(X_train), dataset.X_num, b, \"cpu\")\n",
    "            binshuffling = BinShuffling(1, bin_boundaries, sampling_alpha=False)\n",
    "\n",
    "        estimates = []; buffer = None\n",
    "\n",
    "        loader = torch.utils.data.DataLoader(dataset=X_train, batch_size=512, shuffle=True)\n",
    "        for i in range(4000 // len(loader) + 1):\n",
    "            for _, x in enumerate(loader):\n",
    "                if b > 0:\n",
    "                    sample = binshuffling({\"image\": torch.tensor(x), \"mask\": None})\n",
    "                    mi, buffer = train_step(x, torch.tensor(sample[\"image\"]), buffer=buffer)\n",
    "                else:\n",
    "                    mi, buffer = train_step(x, x, buffer=buffer)\n",
    "                \n",
    "                mi = mi.detach().cpu().numpy()\n",
    "                estimates.append(mi.item())\n",
    "\n",
    "            if i % 200 == 0:\n",
    "                print(b, i, mi)\n",
    "\n",
    "        mis.loc[len(mis)] = [openml_id, b, estimates, np.mean(estimates[-1000:]), np.std(estimates[-1000:]), \n",
    "                             np.mean(estimates[-1000:]) - np.std(estimates[-1000:]), np.mean(estimates[-1000:]) + np.std(estimates[-1000:])]\n",
    "        mis.to_csv(fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6ea9f31",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6027568a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import matplotlib.pyplot as plt\n",
    "# q = mis[(mis[\"data_id\"] == 458) & (mis[\"numbins\"] == 10)][\"mis\"].values[0]\n",
    "# plt.plot(q)\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "b20e5eeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mis.loc[mis[\"data_id\"] == openml_id, 'mis'] = mis.loc[mis[\"data_id\"] == openml_id, 'mis'].apply(lambda x: [value for value in x if not np.isnan(value)])\n",
    "# mis.loc[mis[\"data_id\"] == openml_id, \"estimated_mi\"] = mis.loc[mis[\"data_id\"] == openml_id, 'mis'].apply(lambda x: np.mean(x[-1000:]))\n",
    "# mis.loc[mis[\"data_id\"] == openml_id, \"std\"] = mis.loc[mis[\"data_id\"] == openml_id, 'mis'].apply(lambda x: np.std(x[-1000:]))\n",
    "# mis.loc[mis[\"data_id\"] == openml_id, \"min\"] = mis.loc[mis[\"data_id\"] == openml_id, 'mis'].apply(lambda x: np.min(x[-1000:]))\n",
    "# mis.loc[mis[\"data_id\"] == openml_id, \"max\"] = mis.loc[mis[\"data_id\"] == openml_id, 'mis'].apply(lambda x: np.max(x[-1000:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3ff6295a",
   "metadata": {},
   "outputs": [],
   "source": [
    "mis = mis[mis[\"data_id\"] == openml_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "15393d0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 32244 characters]",
      "text/plain": [
       "<Figure size 400x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base = mis[mis[\"numbins\"] == 0][\"estimated_mi\"].values[0]\n",
    "p = mis[mis[\"numbins\"] > 0]\n",
    "plt.figure(figsize=(4, 3))\n",
    "plt.plot(p[\"numbins\"].values, p[\"estimated_mi\"].values / base, marker=\"o\")\n",
    "plt.fill_between(p[\"numbins\"].values, p[\"min\"].values / base, p[\"max\"].values / base, alpha=0.3)\n",
    "plt.hlines(y=1, xmin=1, xmax=19, linewidth=1, linestyle=\"dashed\", color=\"red\")\n",
    "plt.xlabel(\"Number of bins\")\n",
    "plt.ylabel(\"Estimated MI / Base MI\")\n",
    "plt.xticks(np.arange(0, 20, 2))\n",
    "# plt.ylim(ymin=-0.1)\n",
    "plt.grid()\n",
    "plt.title(openml_id)\n",
    "plt.savefig(f'figs/mis/{openml_id}-{estimator}.png', bbox_inches=\"tight\", dpi=128)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b7060453",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'mine'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "83108f85",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "401edf41",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0393c01e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e954d5c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47a0da7f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "538ff542",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f3dbe5bb",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'z1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [14]\u001b[0m, in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m estimates \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m      2\u001b[0m buffer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m mi, buffer \u001b[38;5;241m=\u001b[39m train_step(\u001b[43mz1\u001b[49m\u001b[38;5;241m.\u001b[39mview(args\u001b[38;5;241m.\u001b[39mbatch_size, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mcuda(), z2\u001b[38;5;241m.\u001b[39mview(args\u001b[38;5;241m.\u001b[39mbatch_size, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mcuda(), mi_params, buffer\u001b[38;5;241m=\u001b[39mbuffer)\n\u001b[1;32m      5\u001b[0m mi \u001b[38;5;241m=\u001b[39m mi\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mcpu()\u001b[38;5;241m.\u001b[39mnumpy()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'z1' is not defined"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b261718",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tabular",
   "language": "python",
   "name": "tabular"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
