{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f1696373-d690-4d9f-9dd6-319279d9b2a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pylcs\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "import sklearn.svm\n",
    "import sklearn.kernel_ridge\n",
    "import plotly.express as px\n",
    "import sklearn.metrics\n",
    "import sklearn.model_selection\n",
    "import sklearn.manifold\n",
    "import optuna\n",
    "import scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "37e29778-e091-4b1b-8e99-0e46f91789c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pylcs\n",
    "import fasta\n",
    "from pathlib import Path\n",
    "from math import log2\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ba6d800f-79c1-4a63-bee1-cbb28ca8ec6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': 31801, 'train_test': 45447, 'test': 13646}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datadir = \"ncBench/datasets/dataset2\"\n",
    "dataset_name = \"ncRNA_dataset2\"\n",
    "sequence_files = [x for x in Path(datadir).glob(\"*.fasta\")]\n",
    "datasets = {fname.stem: fasta.FASTA(str(fname)) for fname in sequence_files}\n",
    "{k: len(v) for k, v in datasets.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1eaeb984-21d3-48ac-8ac3-5f4cdb14e7f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>sequence</th>\n",
       "      <th>set</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>605</td>\n",
       "      <td>5S-rRNA</td>\n",
       "      <td>ATCTGATAGCTTCTCTGGATCCCAAAAGTTGCCAAATACCATCTAA...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9424</td>\n",
       "      <td>CD-box</td>\n",
       "      <td>TTCACTGATGATGATTATATTACCGCCCCAGTCTGATTTTATATTC...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20087</td>\n",
       "      <td>Riboswitch</td>\n",
       "      <td>CCGGATTGTCGATACCGTCGCGCCGTCGCGGTAACGGGAAGCCGGT...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14722</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>CGACTACAGCTCTCTGGATTCACTCTGGCCTTTCAATGCATGTGTT...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33739</td>\n",
       "      <td>tRNA</td>\n",
       "      <td>GCTCCTGTGGCCTAGTGGTGAGGGCATCCCTCTTATAAGGAATCTG...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45442</th>\n",
       "      <td>18742</td>\n",
       "      <td>Leader</td>\n",
       "      <td>GAAAAAGCGTTGAAAAGATGAGTAGTTTTGGATAGACTTATAGAGA...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45443</th>\n",
       "      <td>13516</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>TCCCAAGGTGTTGAGTTCAGTTCAGGGCTGCTTCCCTGTTCTGGTA...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45444</th>\n",
       "      <td>28677</td>\n",
       "      <td>miRNA</td>\n",
       "      <td>AAGGGGTGTCAAAGTAGAGGTAGGTGCGAGAAATGACGGGCATCGT...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45445</th>\n",
       "      <td>6156</td>\n",
       "      <td>CD-box</td>\n",
       "      <td>CGTCATGATTTGGGCATTGCATTAACTTATCATTAAAGTATGTTGA...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45446</th>\n",
       "      <td>40797</td>\n",
       "      <td>Intron-gp-II</td>\n",
       "      <td>CCTTGTGTTCCATCAATAGGATACGTGCATTTCTGGTAACGGAATT...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>45447 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id        target                                           sequence  \\\n",
       "0        605       5S-rRNA  ATCTGATAGCTTCTCTGGATCCCAAAAGTTGCCAAATACCATCTAA...   \n",
       "1       9424        CD-box  TTCACTGATGATGATTATATTACCGCCCCAGTCTGATTTTATATTC...   \n",
       "2      20087    Riboswitch  CCGGATTGTCGATACCGTCGCGCCGTCGCGGTAACGGGAAGCCGGT...   \n",
       "3      14722      HACA-box  CGACTACAGCTCTCTGGATTCACTCTGGCCTTTCAATGCATGTGTT...   \n",
       "4      33739          tRNA  GCTCCTGTGGCCTAGTGGTGAGGGCATCCCTCTTATAAGGAATCTG...   \n",
       "...      ...           ...                                                ...   \n",
       "45442  18742        Leader  GAAAAAGCGTTGAAAAGATGAGTAGTTTTGGATAGACTTATAGAGA...   \n",
       "45443  13516      HACA-box  TCCCAAGGTGTTGAGTTCAGTTCAGGGCTGCTTCCCTGTTCTGGTA...   \n",
       "45444  28677         miRNA  AAGGGGTGTCAAAGTAGAGGTAGGTGCGAGAAATGACGGGCATCGT...   \n",
       "45445   6156        CD-box  CGTCATGATTTGGGCATTGCATTAACTTATCATTAAAGTATGTTGA...   \n",
       "45446  40797  Intron-gp-II  CCTTGTGTTCCATCAATAGGATACGTGCATTTCTGGTAACGGAATT...   \n",
       "\n",
       "         set  validation  \n",
       "0      train       False  \n",
       "1      train       False  \n",
       "2      train        True  \n",
       "3      train        True  \n",
       "4      train       False  \n",
       "...      ...         ...  \n",
       "45442   test       False  \n",
       "45443   test       False  \n",
       "45444   test        True  \n",
       "45445   test        True  \n",
       "45446   test        True  \n",
       "\n",
       "[45447 rows x 5 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rows = []\n",
    "rng = np.random.default_rng(23)\n",
    "for subset in [\"train\", \"test\"]:\n",
    "    fasta_dataset = datasets[subset]\n",
    "    for seq in fasta_dataset.sequences.values():\n",
    "        (id, label) = seq.description.split(\" \", 2)\n",
    "        validation = rng.uniform() < 0.2\n",
    "        rows.append({\"id\": seq.id, \"target\": label, \"sequence\": str(seq.seq), \"set\": subset, \"validation\": validation})\n",
    "split_df = pd.DataFrame(rows)\n",
    "split_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3558d8b3-4d89-45ee-8e2b-41cb8ab78b00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42, 500, 123.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequence_lengths = [len(x) for x in split_df[\"sequence\"].tolist()]\n",
    "np.min(sequence_lengths), np.max(sequence_lengths), np.median(sequence_lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "39115ae8-2a18-4525-a175-020e9bb78ca5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>sequence</th>\n",
       "      <th>set</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>31801</th>\n",
       "      <td>21355</td>\n",
       "      <td>Riboswitch</td>\n",
       "      <td>AATTAAAAAAGCGATGAGGCTCGCTTGAGATGTAATCAAACCGCCG...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31802</th>\n",
       "      <td>11684</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>ATCCTAGGTAAGCCCTTTAAACTACTTATCTGATTTATGGGAGACA...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31803</th>\n",
       "      <td>14551</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>AACTTTGTCCAAAAGTACTCCACCGCAATAGACAGTACGACGGCCG...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31804</th>\n",
       "      <td>6725</td>\n",
       "      <td>CD-box</td>\n",
       "      <td>TTGCCATGATGATTTCAAATTATTAGGAAAGCATTATGCTAATGCT...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31805</th>\n",
       "      <td>28222</td>\n",
       "      <td>miRNA</td>\n",
       "      <td>AATCTGGGAAGAGAGCTTCCTTCAGTCCACTCATGGGGAGTGGTAA...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45442</th>\n",
       "      <td>18742</td>\n",
       "      <td>Leader</td>\n",
       "      <td>GAAAAAGCGTTGAAAAGATGAGTAGTTTTGGATAGACTTATAGAGA...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45443</th>\n",
       "      <td>13516</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>TCCCAAGGTGTTGAGTTCAGTTCAGGGCTGCTTCCCTGTTCTGGTA...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45444</th>\n",
       "      <td>28677</td>\n",
       "      <td>miRNA</td>\n",
       "      <td>AAGGGGTGTCAAAGTAGAGGTAGGTGCGAGAAATGACGGGCATCGT...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45445</th>\n",
       "      <td>6156</td>\n",
       "      <td>CD-box</td>\n",
       "      <td>CGTCATGATTTGGGCATTGCATTAACTTATCATTAAAGTATGTTGA...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45446</th>\n",
       "      <td>40797</td>\n",
       "      <td>Intron-gp-II</td>\n",
       "      <td>CCTTGTGTTCCATCAATAGGATACGTGCATTTCTGGTAACGGAATT...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13646 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id        target                                           sequence  \\\n",
       "31801  21355    Riboswitch  AATTAAAAAAGCGATGAGGCTCGCTTGAGATGTAATCAAACCGCCG...   \n",
       "31802  11684      HACA-box  ATCCTAGGTAAGCCCTTTAAACTACTTATCTGATTTATGGGAGACA...   \n",
       "31803  14551      HACA-box  AACTTTGTCCAAAAGTACTCCACCGCAATAGACAGTACGACGGCCG...   \n",
       "31804   6725        CD-box  TTGCCATGATGATTTCAAATTATTAGGAAAGCATTATGCTAATGCT...   \n",
       "31805  28222         miRNA  AATCTGGGAAGAGAGCTTCCTTCAGTCCACTCATGGGGAGTGGTAA...   \n",
       "...      ...           ...                                                ...   \n",
       "45442  18742        Leader  GAAAAAGCGTTGAAAAGATGAGTAGTTTTGGATAGACTTATAGAGA...   \n",
       "45443  13516      HACA-box  TCCCAAGGTGTTGAGTTCAGTTCAGGGCTGCTTCCCTGTTCTGGTA...   \n",
       "45444  28677         miRNA  AAGGGGTGTCAAAGTAGAGGTAGGTGCGAGAAATGACGGGCATCGT...   \n",
       "45445   6156        CD-box  CGTCATGATTTGGGCATTGCATTAACTTATCATTAAAGTATGTTGA...   \n",
       "45446  40797  Intron-gp-II  CCTTGTGTTCCATCAATAGGATACGTGCATTTCTGGTAACGGAATT...   \n",
       "\n",
       "        set  validation  \n",
       "31801  test       False  \n",
       "31802  test        True  \n",
       "31803  test        True  \n",
       "31804  test       False  \n",
       "31805  test       False  \n",
       "...     ...         ...  \n",
       "45442  test       False  \n",
       "45443  test       False  \n",
       "45444  test        True  \n",
       "45445  test        True  \n",
       "45446  test        True  \n",
       "\n",
       "[13646 rows x 5 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_df[split_df[\"set\"] == \"test\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3b5c9769-ed2d-4635-8738-7e8701d0bf7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# rng = np.random.default_rng(23)\n",
    "# train_size = 2**10\n",
    "# test_size = 2**8\n",
    "train_df = split_df[(split_df[\"set\"] == \"train\") & (split_df[\"validation\"] != True)]\n",
    "# train_idx = rng.choice(len(train_df), size=train_size, replace=False)\n",
    "# train_df = train_df.iloc[train_idx]\n",
    "valid_df = split_df[(split_df[\"set\"] == \"train\") & (split_df[\"validation\"] == True)]\n",
    "test_df = split_df[split_df[\"set\"] == \"test\"]\n",
    "# test_idx = rng.choice(len(test_df), size=test_size, replace=False)\n",
    "# test_df = test_df.iloc[test_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ac851454-6593-4d00-b793-33a68346d7c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>sequence</th>\n",
       "      <th>set</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>605</td>\n",
       "      <td>5S-rRNA</td>\n",
       "      <td>ATCTGATAGCTTCTCTGGATCCCAAAAGTTGCCAAATACCATCTAA...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9424</td>\n",
       "      <td>CD-box</td>\n",
       "      <td>TTCACTGATGATGATTATATTACCGCCCCAGTCTGATTTTATATTC...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33739</td>\n",
       "      <td>tRNA</td>\n",
       "      <td>GCTCCTGTGGCCTAGTGGTGAGGGCATCCCTCTTATAAGGAATCTG...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>25199</td>\n",
       "      <td>miRNA</td>\n",
       "      <td>ACATTAATTGCCTGGGGATCTTGTTAAAATGTAGATTCTGGTTCAG...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>33383</td>\n",
       "      <td>tRNA</td>\n",
       "      <td>GCGGCAGGCTCGGCTAGAGCCCCGGACTGATAATCCGGTGGTGGCG...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31796</th>\n",
       "      <td>20773</td>\n",
       "      <td>Riboswitch</td>\n",
       "      <td>AACTTAATAGATGTTAGAAGTTCTCAAGTATGTTAAAAAACTTGAA...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31797</th>\n",
       "      <td>11644</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>TGGTTGGAGCCAGATCCAACGCCTCATAAGTTAACTCACTGGTTGT...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31798</th>\n",
       "      <td>15785</td>\n",
       "      <td>Leader</td>\n",
       "      <td>AAAAATTCTGTGATAGGAAAGAGTAGTTAGAATGAAGTATTTAAAG...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31799</th>\n",
       "      <td>28125</td>\n",
       "      <td>miRNA</td>\n",
       "      <td>TAGGGTGACCAGCTCTCCCAGTTTGGTCAGGACTGAGGAGTTTCCT...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31800</th>\n",
       "      <td>35369</td>\n",
       "      <td>Y-RNA</td>\n",
       "      <td>AGCTGGTCCAAAGGTAGTGAGTTACCTTACTTGATTGGTTGCAAGT...</td>\n",
       "      <td>train</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25371 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id      target                                           sequence  \\\n",
       "0        605     5S-rRNA  ATCTGATAGCTTCTCTGGATCCCAAAAGTTGCCAAATACCATCTAA...   \n",
       "1       9424      CD-box  TTCACTGATGATGATTATATTACCGCCCCAGTCTGATTTTATATTC...   \n",
       "4      33739        tRNA  GCTCCTGTGGCCTAGTGGTGAGGGCATCCCTCTTATAAGGAATCTG...   \n",
       "5      25199       miRNA  ACATTAATTGCCTGGGGATCTTGTTAAAATGTAGATTCTGGTTCAG...   \n",
       "6      33383        tRNA  GCGGCAGGCTCGGCTAGAGCCCCGGACTGATAATCCGGTGGTGGCG...   \n",
       "...      ...         ...                                                ...   \n",
       "31796  20773  Riboswitch  AACTTAATAGATGTTAGAAGTTCTCAAGTATGTTAAAAAACTTGAA...   \n",
       "31797  11644    HACA-box  TGGTTGGAGCCAGATCCAACGCCTCATAAGTTAACTCACTGGTTGT...   \n",
       "31798  15785      Leader  AAAAATTCTGTGATAGGAAAGAGTAGTTAGAATGAAGTATTTAAAG...   \n",
       "31799  28125       miRNA  TAGGGTGACCAGCTCTCCCAGTTTGGTCAGGACTGAGGAGTTTCCT...   \n",
       "31800  35369       Y-RNA  AGCTGGTCCAAAGGTAGTGAGTTACCTTACTTGATTGGTTGCAAGT...   \n",
       "\n",
       "         set  validation  \n",
       "0      train       False  \n",
       "1      train       False  \n",
       "4      train       False  \n",
       "5      train       False  \n",
       "6      train       False  \n",
       "...      ...         ...  \n",
       "31796  train       False  \n",
       "31797  train       False  \n",
       "31798  train       False  \n",
       "31799  train       False  \n",
       "31800  train       False  \n",
       "\n",
       "[25371 rows x 5 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2f2ee6f8-7b15-4eec-ab01-aff89ac0c647",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>sequence</th>\n",
       "      <th>set</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20087</td>\n",
       "      <td>Riboswitch</td>\n",
       "      <td>CCGGATTGTCGATACCGTCGCGCCGTCGCGGTAACGGGAAGCCGGT...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14722</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>CGACTACAGCTCTCTGGATTCACTCTGGCCTTTCAATGCATGTGTT...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>35779</td>\n",
       "      <td>5.8S-rRNA</td>\n",
       "      <td>GACTCTCGGCAACGGATATCTCGGCTCTCGCATCGATGAAGAACGT...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>35280</td>\n",
       "      <td>Y-RNA</td>\n",
       "      <td>GACTGGTCCGAAGGCAGTGAGTTTTCTCCGCCGATTTCTCAGCCAA...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>21251</td>\n",
       "      <td>Riboswitch</td>\n",
       "      <td>CATCACACCTCACGCGTGGGCATCACCAGGAATCCGGCGGAAATCC...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31782</th>\n",
       "      <td>11353</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>TCCCGGTTCTAGGCACGAACCGGGACATTGGTCCCGGTTCGTGCCT...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31784</th>\n",
       "      <td>9419</td>\n",
       "      <td>CD-box</td>\n",
       "      <td>CCAATTGGAGATGATGAGCTGGTGGCATTCATTCTCAATGGTTTGG...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31785</th>\n",
       "      <td>28805</td>\n",
       "      <td>miRNA</td>\n",
       "      <td>ACCAGTGGAGAAAAGGCAAGATGTTGGCATAGCTGTGATTTAAATA...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31792</th>\n",
       "      <td>33474</td>\n",
       "      <td>tRNA</td>\n",
       "      <td>GCCTGGGTAGCTCAATCGGTAGAGCGTGTGACTCTTACTTCCTTAT...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31795</th>\n",
       "      <td>4232</td>\n",
       "      <td>5S-rRNA</td>\n",
       "      <td>AGAGGTCTCCTCACTCTTTACCCAATATATATATACCCTGATCCCA...</td>\n",
       "      <td>train</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6430 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id      target                                           sequence  \\\n",
       "2      20087  Riboswitch  CCGGATTGTCGATACCGTCGCGCCGTCGCGGTAACGGGAAGCCGGT...   \n",
       "3      14722    HACA-box  CGACTACAGCTCTCTGGATTCACTCTGGCCTTTCAATGCATGTGTT...   \n",
       "12     35779   5.8S-rRNA  GACTCTCGGCAACGGATATCTCGGCTCTCGCATCGATGAAGAACGT...   \n",
       "20     35280       Y-RNA  GACTGGTCCGAAGGCAGTGAGTTTTCTCCGCCGATTTCTCAGCCAA...   \n",
       "25     21251  Riboswitch  CATCACACCTCACGCGTGGGCATCACCAGGAATCCGGCGGAAATCC...   \n",
       "...      ...         ...                                                ...   \n",
       "31782  11353    HACA-box  TCCCGGTTCTAGGCACGAACCGGGACATTGGTCCCGGTTCGTGCCT...   \n",
       "31784   9419      CD-box  CCAATTGGAGATGATGAGCTGGTGGCATTCATTCTCAATGGTTTGG...   \n",
       "31785  28805       miRNA  ACCAGTGGAGAAAAGGCAAGATGTTGGCATAGCTGTGATTTAAATA...   \n",
       "31792  33474        tRNA  GCCTGGGTAGCTCAATCGGTAGAGCGTGTGACTCTTACTTCCTTAT...   \n",
       "31795   4232     5S-rRNA  AGAGGTCTCCTCACTCTTTACCCAATATATATATACCCTGATCCCA...   \n",
       "\n",
       "         set  validation  \n",
       "2      train        True  \n",
       "3      train        True  \n",
       "12     train        True  \n",
       "20     train        True  \n",
       "25     train        True  \n",
       "...      ...         ...  \n",
       "31782  train        True  \n",
       "31784  train        True  \n",
       "31785  train        True  \n",
       "31792  train        True  \n",
       "31795  train        True  \n",
       "\n",
       "[6430 rows x 5 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e8b86a19-7a5c-4a73-91d3-9752b79e7cee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>sequence</th>\n",
       "      <th>set</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>31801</th>\n",
       "      <td>21355</td>\n",
       "      <td>Riboswitch</td>\n",
       "      <td>AATTAAAAAAGCGATGAGGCTCGCTTGAGATGTAATCAAACCGCCG...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31802</th>\n",
       "      <td>11684</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>ATCCTAGGTAAGCCCTTTAAACTACTTATCTGATTTATGGGAGACA...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31803</th>\n",
       "      <td>14551</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>AACTTTGTCCAAAAGTACTCCACCGCAATAGACAGTACGACGGCCG...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31804</th>\n",
       "      <td>6725</td>\n",
       "      <td>CD-box</td>\n",
       "      <td>TTGCCATGATGATTTCAAATTATTAGGAAAGCATTATGCTAATGCT...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31805</th>\n",
       "      <td>28222</td>\n",
       "      <td>miRNA</td>\n",
       "      <td>AATCTGGGAAGAGAGCTTCCTTCAGTCCACTCATGGGGAGTGGTAA...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45442</th>\n",
       "      <td>18742</td>\n",
       "      <td>Leader</td>\n",
       "      <td>GAAAAAGCGTTGAAAAGATGAGTAGTTTTGGATAGACTTATAGAGA...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45443</th>\n",
       "      <td>13516</td>\n",
       "      <td>HACA-box</td>\n",
       "      <td>TCCCAAGGTGTTGAGTTCAGTTCAGGGCTGCTTCCCTGTTCTGGTA...</td>\n",
       "      <td>test</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45444</th>\n",
       "      <td>28677</td>\n",
       "      <td>miRNA</td>\n",
       "      <td>AAGGGGTGTCAAAGTAGAGGTAGGTGCGAGAAATGACGGGCATCGT...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45445</th>\n",
       "      <td>6156</td>\n",
       "      <td>CD-box</td>\n",
       "      <td>CGTCATGATTTGGGCATTGCATTAACTTATCATTAAAGTATGTTGA...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45446</th>\n",
       "      <td>40797</td>\n",
       "      <td>Intron-gp-II</td>\n",
       "      <td>CCTTGTGTTCCATCAATAGGATACGTGCATTTCTGGTAACGGAATT...</td>\n",
       "      <td>test</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13646 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id        target                                           sequence  \\\n",
       "31801  21355    Riboswitch  AATTAAAAAAGCGATGAGGCTCGCTTGAGATGTAATCAAACCGCCG...   \n",
       "31802  11684      HACA-box  ATCCTAGGTAAGCCCTTTAAACTACTTATCTGATTTATGGGAGACA...   \n",
       "31803  14551      HACA-box  AACTTTGTCCAAAAGTACTCCACCGCAATAGACAGTACGACGGCCG...   \n",
       "31804   6725        CD-box  TTGCCATGATGATTTCAAATTATTAGGAAAGCATTATGCTAATGCT...   \n",
       "31805  28222         miRNA  AATCTGGGAAGAGAGCTTCCTTCAGTCCACTCATGGGGAGTGGTAA...   \n",
       "...      ...           ...                                                ...   \n",
       "45442  18742        Leader  GAAAAAGCGTTGAAAAGATGAGTAGTTTTGGATAGACTTATAGAGA...   \n",
       "45443  13516      HACA-box  TCCCAAGGTGTTGAGTTCAGTTCAGGGCTGCTTCCCTGTTCTGGTA...   \n",
       "45444  28677         miRNA  AAGGGGTGTCAAAGTAGAGGTAGGTGCGAGAAATGACGGGCATCGT...   \n",
       "45445   6156        CD-box  CGTCATGATTTGGGCATTGCATTAACTTATCATTAAAGTATGTTGA...   \n",
       "45446  40797  Intron-gp-II  CCTTGTGTTCCATCAATAGGATACGTGCATTTCTGGTAACGGAATT...   \n",
       "\n",
       "        set  validation  \n",
       "31801  test       False  \n",
       "31802  test        True  \n",
       "31803  test        True  \n",
       "31804  test       False  \n",
       "31805  test       False  \n",
       "...     ...         ...  \n",
       "45442  test       False  \n",
       "45443  test       False  \n",
       "45444  test        True  \n",
       "45445  test        True  \n",
       "45446  test        True  \n",
       "\n",
       "[13646 rows x 5 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0005610a-4f12-41e0-9c55-a15fe6f4b492",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export sequences to text file in order to compute LCS in parallel\n",
    "# After runnig this cell, use the pairwise_distances.py script to compute (upper triangular part of symmetric) LCS matrices.\n",
    "# The script supports options for computing submatrices, to be used for distributing the computation on a cluster. The simplest example is:\n",
    "# $ python3 pairwise_distances.py sequences_ncRNA_dataset2.txt sequences_ncRNA_dataset2.npy\n",
    "# Then use the notebook \"Assemble Distance Matrices.npy\" to assemble submatrices and compute the \n",
    "# lower triangular entries. Output the result into a file \"lcs_ncRNA_dataset2.npy\"\n",
    "\n",
    "all_sequences = [x.sequence for x in split_df.itertuples()]\n",
    "with open(f\"sequences_{dataset_name}.txt\", \"w+\") as fh:\n",
    "    for seq in all_sequences:\n",
    "        fh.write(seq)\n",
    "        fh.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4b35cd25-5f3a-49d8-a20a-1428dabb47c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[123,  54,  91, ...,  65,  73,  89],\n",
       "       [ 54,  74,  59, ...,  52,  59,  63],\n",
       "       [ 91,  59, 186, ...,  77,  79, 104],\n",
       "       ...,\n",
       "       [ 65,  52,  77, ...,  97,  63,  74],\n",
       "       [ 73,  59,  79, ...,  63, 108,  82],\n",
       "       [ 89,  63, 104, ...,  74,  82, 156]], dtype=uint16)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This cell assumes that LCS matrices have been generated using the external script, as described in the previous cell\n",
    "\n",
    "lcs_matrix = np.load(f\"lcs_{dataset_name}.npy\").astype(\"uint16\")\n",
    "lcs_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9152122e-9484-475c-b516-9d1d3009f7e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "harmonic_series = np.hstack([[0], np.cumsum(1/np.arange(1, 2**20))])\n",
    "def compute_harmonic_edit_distance(la, lb, lcs):\n",
    "    return 2*harmonic_series[la+lb-lcs] - harmonic_series[la] - harmonic_series[lb]\n",
    "\n",
    "def compute_edit_distance(la, lb, lcs):\n",
    "    return la+lb-2*lcs\n",
    "\n",
    "def compute_steinhaus_distance(la, lb, lcs):\n",
    "    ed = compute_edit_distance(la, lb, lcs)\n",
    "    return 2*ed / (la + lb + ed)\n",
    "\n",
    "def extract_lcs(lcs_matrix, row_df, col_df):\n",
    "    row_seqs = [x.sequence for x in row_df.itertuples()]\n",
    "    col_seqs = [x.sequence for x in col_df.itertuples()]\n",
    "    lcs = np.zeros((len(row_seqs), len(col_seqs)), dtype=\"uint16\")\n",
    "    col_idx = np.array(col_df.index.values, dtype=int)\n",
    "    for i in range(len(row_seqs)):\n",
    "            k_i = row_df.index[i]\n",
    "            lcs[i, :] = lcs_matrix[k_i, col_idx]\n",
    "    return lcs\n",
    "\n",
    "def extract_dataset(lcs_matrix, train_df, df):\n",
    "    dataset = {}\n",
    "    dataset[\"sequences\"] = [x.sequence for x in df.itertuples()]\n",
    "    dataset[\"lengths\"] = np.array([len(x) for x in dataset[\"sequences\"]])\n",
    "    train_lengths = np.array([len(x.sequence) for x in train_df.itertuples()])\n",
    "    dataset[\"target\"] = df[\"target\"]\n",
    "    dataset[\"lcs\"] = extract_lcs(lcs_matrix, df, train_df).astype(\"uint16\")\n",
    "    dataset[\"harmonic_distance\"] = compute_harmonic_edit_distance(\n",
    "        dataset[\"lengths\"][:, None], \n",
    "        train_lengths[None, :], \n",
    "        dataset[\"lcs\"]\n",
    "    )\n",
    "    dataset[\"edit_distance\"] = compute_edit_distance(\n",
    "        dataset[\"lengths\"][:, None], \n",
    "        train_lengths[None, :], \n",
    "        dataset[\"lcs\"]\n",
    "    )\n",
    "    dataset[\"steinhaus_distance\"] = compute_steinhaus_distance(\n",
    "        dataset[\"lengths\"][:, None], \n",
    "        train_lengths[None, :], \n",
    "        dataset[\"lcs\"]\n",
    "    )\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "901f31d7-e0f1-441b-a10e-6fa668431f4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = extract_dataset(lcs_matrix, train_df, train_df)\n",
    "valid = extract_dataset(lcs_matrix, train_df, valid_df)\n",
    "test = extract_dataset(lcs_matrix, train_df, test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "96dbefab-9d4b-483d-9365-7936ab323af3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1227.7367420196533"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[\"lcs\"].nbytes / 2**20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "761e23f4-523c-49a5-bfac-bcc9310547fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4910.946968078613"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[\"harmonic_distance\"].nbytes / 2**20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a846bc2e-5179-4b43-88b5-3be09894d375",
   "metadata": {},
   "outputs": [],
   "source": [
    "import optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "eecb966a-7e37-4795-8164-81fb68fa4e86",
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_parameters(params, distance_key, train, eval):\n",
    "    gamma = params[\"gamma\"]\n",
    "    C = params[\"C\"]\n",
    "\n",
    "    train[\"kernel\"] = np.exp(-gamma*train[distance_key]**2)\n",
    "    eval[\"kernel\"] = np.exp(-gamma*eval[distance_key]**2)\n",
    "\n",
    "    classifier = sklearn.svm.SVC(C=C, kernel=\"precomputed\", verbose=False, shrinking=False, max_iter=2**16)\n",
    "    classifier.fit(train[\"kernel\"], train[\"target\"])\n",
    "\n",
    "    prediction = classifier.predict(eval[\"kernel\"])\n",
    "    valid_accuracy = sklearn.metrics.accuracy_score(eval[\"target\"], prediction)\n",
    "\n",
    "    # valid_spearman = scipy.stats.spearmanr(prediction, eval[\"target\"]).statistic\n",
    "    return valid_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "32235d5e-0637-4d2a-a9d7-672ce4282cea",
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective(trial, distance_key, train, eval, gamma_range=[1e-1, 1e1]):\n",
    "    params = {}\n",
    "    params[\"gamma\"] = trial.suggest_float(\"gamma\", gamma_range[0], gamma_range[1], log=True)\n",
    "    params[\"C\"] = trial.suggest_float(\"C\", 1e-2, 1e2, log=True)\n",
    "    val = eval_parameters(params, distance_key, train, eval)\n",
    "    return -val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "141578fb-1a8b-4195-bade-95375d1ae222",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9003110419906687"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_parameters({\"C\": 1, \"gamma\": 1}, \"harmonic_distance\", train, valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "599805fa-c663-45fd-9806-90c854431064",
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective_harmonic(trial):\n",
    "    return objective(trial, \"harmonic_distance\", train, valid, gamma_range=[1e-1, 1e1])\n",
    "\n",
    "def objective_edit(trial):\n",
    "    return objective(trial, \"edit_distance\", train, valid, gamma_range=[1e-4, 1e-2])\n",
    "\n",
    "def objective_steinhaus(trial):\n",
    "    return objective(trial, \"steinhaus_distance\", train, valid, gamma_range=[1e-1, 1e1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cbf953c6-115c-4b8a-8517-6ccd8afa9a28",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 13:05:20,562] A new study created in memory with name: no-name-20b5dc68-1acd-4e5d-8bb4-05cd7e83364d\n",
      "[I 2024-08-07 13:05:32,120] Trial 0 finished with value: -0.8435458786936236 and parameters: {'gamma': 0.5590928946019317, 'C': 0.10014277970437574}. Best is trial 0 with value: -0.8435458786936236.\n",
      "[I 2024-08-07 13:05:40,890] Trial 1 finished with value: -0.6888024883359254 and parameters: {'gamma': 6.97131947457495, 'C': 0.023285039866720644}. Best is trial 0 with value: -0.8435458786936236.\n",
      "[I 2024-08-07 13:05:49,425] Trial 2 finished with value: -0.8273716951788491 and parameters: {'gamma': 0.37932891193397006, 'C': 0.11292637708831474}. Best is trial 0 with value: -0.8435458786936236.\n",
      "[I 2024-08-07 13:05:58,035] Trial 3 finished with value: -0.629393468118196 and parameters: {'gamma': 0.33891155438389403, 'C': 0.02030318931949949}. Best is trial 0 with value: -0.8435458786936236.\n",
      "[I 2024-08-07 13:06:06,254] Trial 4 finished with value: -0.6776049766718507 and parameters: {'gamma': 0.12323897843052788, 'C': 1.8907130904350804}. Best is trial 0 with value: -0.8435458786936236.\n",
      "[I 2024-08-07 13:06:13,942] Trial 5 finished with value: -0.9744945567651633 and parameters: {'gamma': 6.832409815910085, 'C': 2.2223349276745106}. Best is trial 5 with value: -0.9744945567651633.\n",
      "[I 2024-08-07 13:06:20,418] Trial 6 finished with value: -0.6653188180404355 and parameters: {'gamma': 0.13861366316945484, 'C': 2.9658532945771556}. Best is trial 5 with value: -0.9744945567651633.\n",
      "[I 2024-08-07 13:06:29,543] Trial 7 finished with value: -0.9544323483670295 and parameters: {'gamma': 7.689797258730094, 'C': 0.4432887338399921}. Best is trial 5 with value: -0.9744945567651633.\n",
      "[I 2024-08-07 13:06:36,877] Trial 8 finished with value: -0.6937791601866252 and parameters: {'gamma': 0.1562521291733365, 'C': 2.589753044482407}. Best is trial 5 with value: -0.9744945567651633.\n",
      "[I 2024-08-07 13:06:46,646] Trial 9 finished with value: -0.8564541213063763 and parameters: {'gamma': 1.4795169474447036, 'C': 95.87925487657975}. Best is trial 5 with value: -0.9744945567651633.\n",
      "/Users/bob/venv/python3.11/lib/python3.11/site-packages/sklearn/svm/_base.py:297: ConvergenceWarning: Solver terminated early (max_iter=65536).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  warnings.warn(\n",
      "[I 2024-08-07 13:06:58,610] Trial 10 finished with value: -0.8822706065318818 and parameters: {'gamma': 2.026843130628204, 'C': 23.887538491820507}. Best is trial 5 with value: -0.9744945567651633.\n",
      "[I 2024-08-07 13:07:08,585] Trial 11 finished with value: -0.9353032659409021 and parameters: {'gamma': 9.741519780182367, 'C': 0.36421264011893173}. Best is trial 5 with value: -0.9744945567651633.\n",
      "[I 2024-08-07 13:07:19,845] Trial 12 finished with value: -0.931104199066874 and parameters: {'gamma': 3.88512722113463, 'C': 10.73137276101151}. Best is trial 5 with value: -0.9744945567651633.\n",
      "[I 2024-08-07 13:07:28,680] Trial 13 finished with value: -0.9590979782270607 and parameters: {'gamma': 4.42099460767778, 'C': 0.4740109454315745}. Best is trial 5 with value: -0.9744945567651633.\n",
      "[I 2024-08-07 13:07:38,486] Trial 14 finished with value: -0.9099533437013997 and parameters: {'gamma': 2.7698021362936567, 'C': 8.71428256942224}. Best is trial 5 with value: -0.9744945567651633.\n",
      "[I 2024-08-07 13:07:44,603] Trial 15 finished with value: -0.9611197511664075 and parameters: {'gamma': 4.475999048910192, 'C': 0.5886945653066282}. Best is trial 5 with value: -0.9744945567651633.\n"
     ]
    }
   ],
   "source": [
    "studies = {}\n",
    "study = optuna.create_study()\n",
    "study.optimize(objective_harmonic, n_trials=16)\n",
    "studies[\"harmonic\"] = study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7af2240d-f6de-4800-9c8b-964300e558b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 13:07:44,609] A new study created in memory with name: no-name-9cd10df0-7285-4b56-98d6-4c6c62f754cb\n",
      "[I 2024-08-07 13:08:04,566] Trial 0 finished with value: -0.44027993779160185 and parameters: {'gamma': 0.0008767593763542728, 'C': 0.08852146221779614}. Best is trial 0 with value: -0.44027993779160185.\n",
      "[I 2024-08-07 13:08:15,744] Trial 1 finished with value: -0.8762052877138413 and parameters: {'gamma': 0.0001239784846391118, 'C': 2.6688410242918548}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:08:27,786] Trial 2 finished with value: -0.4121306376360809 and parameters: {'gamma': 0.0005708853472031703, 'C': 0.018438777411391598}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:08:41,520] Trial 3 finished with value: -0.8043545878693623 and parameters: {'gamma': 0.0004202938454396966, 'C': 9.646437293395863}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:08:58,870] Trial 4 finished with value: -0.38475894245723175 and parameters: {'gamma': 0.002780400974835631, 'C': 2.4828645440322523}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:09:16,976] Trial 5 finished with value: -0.2734059097978227 and parameters: {'gamma': 0.003753071974229234, 'C': 18.660773189810705}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:09:30,519] Trial 6 finished with value: -0.5894245723172629 and parameters: {'gamma': 0.0008903888782933851, 'C': 0.28271710499758385}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:09:45,899] Trial 7 finished with value: -0.5981337480559875 and parameters: {'gamma': 0.0015563048186631753, 'C': 16.327036843565466}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:09:58,413] Trial 8 finished with value: -0.19704510108864698 and parameters: {'gamma': 0.0009762136431058856, 'C': 0.013880799879296139}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:10:11,811] Trial 9 finished with value: -0.10668740279937791 and parameters: {'gamma': 0.001822003922812856, 'C': 0.013205784541783082}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:10:20,338] Trial 10 finished with value: -0.8234836702954899 and parameters: {'gamma': 0.00015286713464309658, 'C': 84.80037253457847}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:10:30,040] Trial 11 finished with value: -0.8255054432348367 and parameters: {'gamma': 0.00011649901107383867, 'C': 78.31841516937185}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:10:39,654] Trial 12 finished with value: -0.8272161741835148 and parameters: {'gamma': 0.00010866954301189292, 'C': 90.62790936758387}. Best is trial 1 with value: -0.8762052877138413.\n",
      "[I 2024-08-07 13:10:51,795] Trial 13 finished with value: -0.8908242612752721 and parameters: {'gamma': 0.00024040398039496064, 'C': 1.068830749423383}. Best is trial 13 with value: -0.8908242612752721.\n",
      "[I 2024-08-07 13:11:06,252] Trial 14 finished with value: -0.8807153965785381 and parameters: {'gamma': 0.0002724585566077903, 'C': 1.0947592101999033}. Best is trial 13 with value: -0.8908242612752721.\n",
      "[I 2024-08-07 13:11:18,462] Trial 15 finished with value: -0.8763608087091758 and parameters: {'gamma': 0.0002599556462870485, 'C': 0.6094187165960285}. Best is trial 13 with value: -0.8908242612752721.\n"
     ]
    }
   ],
   "source": [
    "study = optuna.create_study()\n",
    "study.optimize(objective_edit, n_trials=16)\n",
    "studies[\"edit\"] = study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8af54c3d-5b55-4927-9807-9b40d3e3a0f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 13:11:18,471] A new study created in memory with name: no-name-df762c97-c98b-4fc8-be5b-038ea47c2f10\n",
      "[I 2024-08-07 13:11:49,484] Trial 0 finished with value: -0.8626749611197512 and parameters: {'gamma': 1.0829184866889259, 'C': 61.35506425409843}. Best is trial 0 with value: -0.8626749611197512.\n",
      "[I 2024-08-07 13:11:58,403] Trial 1 finished with value: -0.919751166407465 and parameters: {'gamma': 3.395623408235136, 'C': 0.1347669426130217}. Best is trial 1 with value: -0.919751166407465.\n",
      "[I 2024-08-07 13:12:06,464] Trial 2 finished with value: -0.9068429237947123 and parameters: {'gamma': 0.27675197352313574, 'C': 5.557613501759315}. Best is trial 1 with value: -0.919751166407465.\n",
      "[I 2024-08-07 13:12:13,696] Trial 3 finished with value: -0.8530326594090202 and parameters: {'gamma': 0.2159128083221639, 'C': 0.37133840345513924}. Best is trial 1 with value: -0.919751166407465.\n",
      "[I 2024-08-07 13:12:22,001] Trial 4 finished with value: -0.9332814930015552 and parameters: {'gamma': 1.722283712449511, 'C': 0.4443450837028267}. Best is trial 4 with value: -0.9332814930015552.\n",
      "[I 2024-08-07 13:12:30,714] Trial 5 finished with value: -0.8799377916018662 and parameters: {'gamma': 0.10114158667194365, 'C': 34.365979721157416}. Best is trial 4 with value: -0.9332814930015552.\n",
      "[I 2024-08-07 13:12:38,788] Trial 6 finished with value: -0.9290824261275272 and parameters: {'gamma': 5.887014110300643, 'C': 0.15908848917427898}. Best is trial 4 with value: -0.9332814930015552.\n",
      "[I 2024-08-07 13:12:48,584] Trial 7 finished with value: -0.8715396578538103 and parameters: {'gamma': 1.5106495666125255, 'C': 81.97995282341462}. Best is trial 4 with value: -0.9332814930015552.\n",
      "[I 2024-08-07 13:12:58,341] Trial 8 finished with value: -0.7234836702954899 and parameters: {'gamma': 4.899904822874475, 'C': 0.018209656614118688}. Best is trial 4 with value: -0.9332814930015552.\n",
      "[I 2024-08-07 13:13:07,954] Trial 9 finished with value: -0.8192846034214619 and parameters: {'gamma': 0.38858759237347273, 'C': 0.14182009235905965}. Best is trial 4 with value: -0.9332814930015552.\n",
      "[I 2024-08-07 13:13:18,928] Trial 10 finished with value: -0.9174183514774494 and parameters: {'gamma': 1.9643794285961518, 'C': 2.3404714285471484}. Best is trial 4 with value: -0.9332814930015552.\n",
      "[I 2024-08-07 13:13:29,900] Trial 11 finished with value: -0.6902021772939346 and parameters: {'gamma': 9.929486243620659, 'C': 0.02228941110659484}. Best is trial 4 with value: -0.9332814930015552.\n",
      "[I 2024-08-07 13:13:40,314] Trial 12 finished with value: -0.9664074650077761 and parameters: {'gamma': 9.577836657894219, 'C': 0.8235634064802203}. Best is trial 12 with value: -0.9664074650077761.\n",
      "[I 2024-08-07 13:13:49,752] Trial 13 finished with value: -0.9217729393468118 and parameters: {'gamma': 0.6608144057990814, 'C': 0.9385442227807157}. Best is trial 12 with value: -0.9664074650077761.\n",
      "[I 2024-08-07 13:13:58,937] Trial 14 finished with value: -0.8948678071539657 and parameters: {'gamma': 2.2664034425509043, 'C': 8.615197436839306}. Best is trial 12 with value: -0.9664074650077761.\n",
      "[I 2024-08-07 13:14:05,713] Trial 15 finished with value: -0.9656298600311042 and parameters: {'gamma': 8.433536291602687, 'C': 0.849498112684121}. Best is trial 12 with value: -0.9664074650077761.\n",
      "[I 2024-08-07 13:14:12,780] Trial 16 finished with value: -0.9628304821150855 and parameters: {'gamma': 7.025651719661121, 'C': 2.110899784447036}. Best is trial 12 with value: -0.9664074650077761.\n",
      "/Users/bob/venv/python3.11/lib/python3.11/site-packages/sklearn/svm/_base.py:297: ConvergenceWarning: Solver terminated early (max_iter=65536).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  warnings.warn(\n",
      "[I 2024-08-07 13:14:23,202] Trial 17 finished with value: -0.9010886469673406 and parameters: {'gamma': 3.5539021497638026, 'C': 11.78889828968339}. Best is trial 12 with value: -0.9664074650077761.\n",
      "[I 2024-08-07 13:14:30,446] Trial 18 finished with value: -0.970606531881804 and parameters: {'gamma': 9.91793899262693, 'C': 1.485844877379199}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:14:36,803] Trial 19 finished with value: -0.8687402799377916 and parameters: {'gamma': 2.972241038896461, 'C': 0.058292460965928876}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:14:45,546] Trial 20 finished with value: -0.9410575427682737 and parameters: {'gamma': 4.281618670324587, 'C': 1.910889603534882}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:14:53,498] Trial 21 finished with value: -0.9639191290824262 and parameters: {'gamma': 9.829232489279482, 'C': 0.730054883608955}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:15:00,657] Trial 22 finished with value: -0.9643856920684293 and parameters: {'gamma': 7.7084544862689715, 'C': 3.767517704198312}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:15:06,766] Trial 23 finished with value: -0.9539657853810264 and parameters: {'gamma': 6.0608618688242375, 'C': 0.4678053558429094}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:15:13,953] Trial 24 finished with value: -0.968895800933126 and parameters: {'gamma': 9.005887904730727, 'C': 18.834569465615893}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:15:22,605] Trial 25 finished with value: -0.9236391912908243 and parameters: {'gamma': 5.158242294902856, 'C': 20.47568140088902}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:15:32,805] Trial 26 finished with value: -0.8881804043545879 and parameters: {'gamma': 2.7824687428724717, 'C': 19.5492627279248}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:15:39,287] Trial 27 finished with value: -0.9046656298600311 and parameters: {'gamma': 0.8551948127413587, 'C': 3.4723421814201236}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:15:45,091] Trial 28 finished with value: -0.9419906687402799 and parameters: {'gamma': 6.696655762266911, 'C': 0.24003711018593932}. Best is trial 18 with value: -0.970606531881804.\n",
      "/Users/bob/venv/python3.11/lib/python3.11/site-packages/sklearn/svm/_base.py:297: ConvergenceWarning: Solver terminated early (max_iter=65536).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  warnings.warn(\n",
      "[I 2024-08-07 13:15:53,841] Trial 29 finished with value: -0.9213063763608087 and parameters: {'gamma': 5.121145326666073, 'C': 44.72921451996738}. Best is trial 18 with value: -0.970606531881804.\n",
      "/Users/bob/venv/python3.11/lib/python3.11/site-packages/sklearn/svm/_base.py:297: ConvergenceWarning: Solver terminated early (max_iter=65536).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  warnings.warn(\n",
      "[I 2024-08-07 13:16:05,354] Trial 30 finished with value: -0.9153965785381026 and parameters: {'gamma': 4.046389731225246, 'C': 7.871262140044413}. Best is trial 18 with value: -0.970606531881804.\n",
      "[I 2024-08-07 13:16:12,617] Trial 31 finished with value: -0.9687402799377915 and parameters: {'gamma': 9.859627907594366, 'C': 1.1783072397244592}. Best is trial 18 with value: -0.970606531881804.\n"
     ]
    }
   ],
   "source": [
    "study = optuna.create_study(sampler=optuna.samplers.TPESampler(seed=23))\n",
    "study.optimize(objective_steinhaus, n_trials=32)\n",
    "studies[\"steinhaus\"] = study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "fc95337e-cae4-4793-b828-32a6fdb873d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9731789535394988"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_parameters(studies[\"harmonic\"].best_params, \"harmonic_distance\", train=train, eval=test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "839d1a29-c940-4af7-a036-f90c1bb1c2d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8913967462992819"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_parameters(studies[\"edit\"].best_params, \"edit_distance\", train=train, eval=test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "919d4a83-f192-4735-8eb9-467a4cbef2ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9702476916312472"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_parameters(studies[\"steinhaus\"].best_params, \"steinhaus_distance\", train=train, eval=test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "16903b4b-71ef-4330-b541-e4be2f04a193",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'harmonic': {'gamma': 6.832409815910085, 'C': 2.2223349276745106},\n",
       " 'edit': {'gamma': 0.00024040398039496064, 'C': 1.068830749423383},\n",
       " 'steinhaus': {'gamma': 9.91793899262693, 'C': 1.485844877379199}}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{k: v.best_params for k, v in studies.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "316987cc-480e-48b7-92b4-635a0254f554",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[t-SNE] Computing 91 nearest neighbors...\n",
      "[t-SNE] Indexed 25371 samples in 0.230s...\n",
      "[t-SNE] Computed neighbors for 25371 samples in 5.861s...\n",
      "[t-SNE] Computed conditional probabilities for sample 1000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 2000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 3000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 4000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 5000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 6000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 7000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 8000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 9000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 10000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 11000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 12000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 13000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 14000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 15000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 16000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 17000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 18000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 19000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 20000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 21000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 22000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 23000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 24000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 25000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 25371 / 25371\n",
      "[t-SNE] Mean sigma: 0.148267\n",
      "[t-SNE] KL divergence after 250 iterations with early exaggeration: 94.143211\n",
      "[t-SNE] KL divergence after 2048 iterations: 1.876777\n"
     ]
    }
   ],
   "source": [
    "tsne = sklearn.manifold.TSNE(metric=\"precomputed\", n_iter=2**11, init=\"random\", n_jobs=-1, verbose=True, random_state=42)\n",
    "coordinates = {}\n",
    "coordinates[\"harmonic\"] = tsne.fit_transform(train[\"harmonic_distance\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2248b6ab-eac0-4252-80a6-1036f7108b0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[t-SNE] Computing 91 nearest neighbors...\n",
      "[t-SNE] Indexed 25371 samples in 0.290s...\n",
      "[t-SNE] Computed neighbors for 25371 samples in 5.817s...\n",
      "[t-SNE] Computed conditional probabilities for sample 1000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 2000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 3000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 4000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 5000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 6000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 7000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 8000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 9000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 10000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 11000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 12000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 13000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 14000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 15000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 16000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 17000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 18000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 19000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 20000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 21000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 22000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 23000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 24000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 25000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 25371 / 25371\n",
      "[t-SNE] Mean sigma: 18.969616\n",
      "[t-SNE] KL divergence after 250 iterations with early exaggeration: 101.184814\n",
      "[t-SNE] KL divergence after 2048 iterations: 2.277497\n"
     ]
    }
   ],
   "source": [
    "tsne = sklearn.manifold.TSNE(metric=\"precomputed\", n_iter=2**11, init=\"random\", n_jobs=-1, verbose=True, random_state=42)\n",
    "coordinates[\"edit\"] = tsne.fit_transform(train[\"edit_distance\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e6953cf2-017f-4958-969c-2774cad52586",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[t-SNE] Computing 91 nearest neighbors...\n",
      "[t-SNE] Indexed 25371 samples in 0.235s...\n",
      "[t-SNE] Computed neighbors for 25371 samples in 5.476s...\n",
      "[t-SNE] Computed conditional probabilities for sample 1000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 2000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 3000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 4000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 5000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 6000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 7000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 8000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 9000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 10000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 11000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 12000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 13000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 14000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 15000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 16000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 17000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 18000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 19000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 20000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 21000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 22000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 23000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 24000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 25000 / 25371\n",
      "[t-SNE] Computed conditional probabilities for sample 25371 / 25371\n",
      "[t-SNE] Mean sigma: 0.124149\n",
      "[t-SNE] KL divergence after 250 iterations with early exaggeration: 94.790909\n",
      "[t-SNE] KL divergence after 2048 iterations: 1.889551\n"
     ]
    }
   ],
   "source": [
    "tsne = sklearn.manifold.TSNE(metric=\"precomputed\", n_iter=2**11, init=\"random\", n_jobs=-1, verbose=True, random_state=42)\n",
    "coordinates[\"steinhaus\"] = tsne.fit_transform(train[\"steinhaus_distance\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "68e90b3a-7e22-4a88-a6e3-5369baf5aa24",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_color_mapping = {lbl: i for i, (lbl, _) in enumerate(Counter(train[\"target\"]).most_common())}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "d6567229-3c39-43a1-8670-4fce395c0d74",
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.graph_objs as go\n",
    "traces = []\n",
    "label_colors = [label_color_mapping[lbl] for lbl in train[\"target\"]]\n",
    "text = train[\"target\"]\n",
    "for distance, coords in coordinates.items():\n",
    "    traces.append({\"x\": coords[:, 0], \"y\": coords[:, 1], \"name\": distance, \"type\": \"scatter\", \"mode\": \"markers\", \"text\": text, \"marker\": {\"size\": 2, \"color\": label_colors}})\n",
    "layout = {\"coloraxis\": {\"showscale\": True}}\n",
    "# go.Figure(data=traces[0:1], layout=layout).show()\n",
    "# go.Figure(data=traces[1:2], layout=layout).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "176f411e-ed36-423d-95e8-39610a81f183",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 198960 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 212480 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 197160 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "cmap = mpl.colormaps[\"tab20\"]\n",
    "for trace in traces: #trace = traces[0]\n",
    "    fig, ax = plt.subplots()\n",
    "    ax.set_xticks([])\n",
    "    ax.set_yticks([])\n",
    "    ax.scatter(x=trace[\"x\"], y=trace[\"y\"], c=[int(x) for x in trace[\"marker\"][\"color\"]], s=trace[\"marker\"][\"size\"], alpha=0.5, edgecolor=\"none\", cmap=cmap)\n",
    "    plt.savefig(f\"TSNE_{dataset_name}_{trace['name']}.png\", dpi=600, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "1bd0df54-2163-4a48-858b-4b087e5fd883",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 13)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cmap.colors), train_df[\"target\"].nunique()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
