{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 943,
   "metadata": {},
   "outputs": [],
   "source": [
    "scenario = 'barista'\n",
    "best_generation_advice = \"1_2\"\n",
    "best_generation_index = 0\n",
    "best_advice_index = \"1_2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 944,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>origin_id</th>\n",
       "      <th>participant_id</th>\n",
       "      <th>participant_strategy</th>\n",
       "      <th>participant_strategy_len</th>\n",
       "      <th>scenario</th>\n",
       "      <th>advice_type</th>\n",
       "      <th>advice_indices</th>\n",
       "      <th>advice_list</th>\n",
       "      <th>generation_index</th>\n",
       "      <th>text</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1</td>\n",
       "      <td>When a server talks a lot, it just makes me fe...</td>\n",
       "      <td>49</td>\n",
       "      <td>A barista at a coffee shop wants to maximize t...</td>\n",
       "      <td>unorthodox_advice</td>\n",
       "      <td>0</td>\n",
       "      <td>['Always respond with a friendly demeanor and ...</td>\n",
       "      <td>2</td>\n",
       "      <td>\"Of course! I'll get a medium whole milk latte...</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>When a server talks a lot, it just makes me fe...</td>\n",
       "      <td>49</td>\n",
       "      <td>A barista at a coffee shop wants to maximize t...</td>\n",
       "      <td>baseline_irrelevant_advice</td>\n",
       "      <td>1_2</td>\n",
       "      <td>[\"If you're planning to travel internationally...</td>\n",
       "      <td>0</td>\n",
       "      <td>\"Hi, absolutely! One medium whole milk latte c...</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>When a server talks a lot, it just makes me fe...</td>\n",
       "      <td>49</td>\n",
       "      <td>A barista at a coffee shop wants to maximize t...</td>\n",
       "      <td>baseline_irrelevant_advice</td>\n",
       "      <td>1_2</td>\n",
       "      <td>[\"If you're planning to travel internationally...</td>\n",
       "      <td>2</td>\n",
       "      <td>\"Of course, one medium latte with whole milk c...</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>When a server talks a lot, it just makes me fe...</td>\n",
       "      <td>49</td>\n",
       "      <td>A barista at a coffee shop wants to maximize t...</td>\n",
       "      <td>baseline_irrelevant_advice</td>\n",
       "      <td>2</td>\n",
       "      <td>['A good strategy in chess is to control the c...</td>\n",
       "      <td>2</td>\n",
       "      <td>\"Absolutely! I'll make sure your medium whole ...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>When a server talks a lot, it just makes me fe...</td>\n",
       "      <td>49</td>\n",
       "      <td>A barista at a coffee shop wants to maximize t...</td>\n",
       "      <td>baseline_irrelevant_advice</td>\n",
       "      <td>0_1</td>\n",
       "      <td>['Remember to water your indoor plants when yo...</td>\n",
       "      <td>2</td>\n",
       "      <td>\"Absolutely! I'll have your medium whole milk ...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  origin_id  participant_id  \\\n",
       "0   1         54               1   \n",
       "1   2         19               1   \n",
       "2   3         21               1   \n",
       "3   4         18               1   \n",
       "4   5         24               1   \n",
       "\n",
       "                                participant_strategy  \\\n",
       "0  When a server talks a lot, it just makes me fe...   \n",
       "1  When a server talks a lot, it just makes me fe...   \n",
       "2  When a server talks a lot, it just makes me fe...   \n",
       "3  When a server talks a lot, it just makes me fe...   \n",
       "4  When a server talks a lot, it just makes me fe...   \n",
       "\n",
       "   participant_strategy_len  \\\n",
       "0                        49   \n",
       "1                        49   \n",
       "2                        49   \n",
       "3                        49   \n",
       "4                        49   \n",
       "\n",
       "                                            scenario  \\\n",
       "0  A barista at a coffee shop wants to maximize t...   \n",
       "1  A barista at a coffee shop wants to maximize t...   \n",
       "2  A barista at a coffee shop wants to maximize t...   \n",
       "3  A barista at a coffee shop wants to maximize t...   \n",
       "4  A barista at a coffee shop wants to maximize t...   \n",
       "\n",
       "                  advice_type advice_indices  \\\n",
       "0           unorthodox_advice              0   \n",
       "1  baseline_irrelevant_advice            1_2   \n",
       "2  baseline_irrelevant_advice            1_2   \n",
       "3  baseline_irrelevant_advice              2   \n",
       "4  baseline_irrelevant_advice            0_1   \n",
       "\n",
       "                                         advice_list  generation_index  \\\n",
       "0  ['Always respond with a friendly demeanor and ...                 2   \n",
       "1  [\"If you're planning to travel internationally...                 0   \n",
       "2  [\"If you're planning to travel internationally...                 2   \n",
       "3  ['A good strategy in chess is to control the c...                 2   \n",
       "4  ['Remember to water your indoor plants when yo...                 2   \n",
       "\n",
       "                                                text  answer  \n",
       "0  \"Of course! I'll get a medium whole milk latte...     5.0  \n",
       "1  \"Hi, absolutely! One medium whole milk latte c...     2.0  \n",
       "2  \"Of course, one medium latte with whole milk c...     2.0  \n",
       "3  \"Absolutely! I'll make sure your medium whole ...     1.0  \n",
       "4  \"Absolutely! I'll have your medium whole milk ...     3.0  "
      ]
     },
     "execution_count": 944,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import math\n",
    "path = f'{scenario}/prolific_generations_results.csv'\n",
    "df = pd.read_csv(path)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 945,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'barista/generations_noindex.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/bt/s_9f74891td8dms1fzdqbkf80000gn/T/ipykernel_52545/1741039203.py\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mgenerations_noindex_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf'{scenario}/generations_noindex.csv'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mgenerations_noindex_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerations_noindex_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mgenerations_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf'{scenario}/generations.csv'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    309\u001b[0m                     \u001b[0mstacklevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m                 )\n\u001b[0;32m--> 311\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    313\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m    584\u001b[0m     \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    585\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 482\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    484\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    809\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 811\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    812\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    813\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1038\u001b[0m             )\n\u001b[1;32m   1039\u001b[0m         \u001b[0;31m# error: Too many arguments for \"ParserBase\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1040\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mmapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1041\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1042\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/parsers/c_parser_wrapper.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0;31m# open handles\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_handles\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/parsers/base_parser.py\u001b[0m in \u001b[0;36m_open_handles\u001b[0;34m(self, src, kwds)\u001b[0m\n\u001b[1;32m    220\u001b[0m         \u001b[0mLet\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreaders\u001b[0m \u001b[0mopen\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0mafter\u001b[0m \u001b[0mthey\u001b[0m \u001b[0mare\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtheir\u001b[0m \u001b[0mpotential\u001b[0m \u001b[0mraises\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    221\u001b[0m         \"\"\"\n\u001b[0;32m--> 222\u001b[0;31m         self.handles = get_handle(\n\u001b[0m\u001b[1;32m    223\u001b[0m             \u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    224\u001b[0m             \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    700\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    701\u001b[0m             \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m             handle = open(\n\u001b[0m\u001b[1;32m    703\u001b[0m                 \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    704\u001b[0m                 \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'barista/generations_noindex.csv'"
     ]
    }
   ],
   "source": [
    "# logistics - checking that generations_noindex and generations are the same except for index\n",
    "\n",
    "generations_noindex_path = f'{scenario}/generations_noindex.csv'\n",
    "generations_noindex_df = pd.read_csv(generations_noindex_path)\n",
    "\n",
    "generations_path = f'{scenario}/generations.csv'\n",
    "generations_df = pd.read_csv(generations_path)\n",
    "\n",
    "print(len(generations_df), len(generations_noindex_df))\n",
    "\n",
    "for i in range(len(generations_noindex_df)):\n",
    "    if generations_noindex_df['text'][i] != generations_df['text'][i]:\n",
    "        print(generations_noindex_df['advice_type'][i], generations_noindex_df['advice_indices'][i], generations_df['advice_type'][i], generations_df['advice_indices'][i])\n",
    "        print(generations_noindex_df['text'][i], \"\\n\\n\", generations_df['text'][i])\n",
    "        # print(i)\n",
    "        break\n",
    "\n",
    "# so index [] was removed from unorthodox_advice but it was still there for the human experiment\n",
    "# it is now in baseline_irrelevant_advice instead\n",
    "# moved back and regenerated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count: 120\n",
      "1440\n"
     ]
    }
   ],
   "source": [
    "# fill in generation_index column\n",
    "# check that generations_df text is unique\n",
    "generations_path = f'{scenario}/generations.csv'\n",
    "generations_df = pd.read_csv(generations_path)\n",
    "\n",
    "assert len(generations_df['text'].unique()) == len(generations_df)\n",
    "count = 0\n",
    "for index, row in df.iterrows():\n",
    "    text = row['text']\n",
    "    if not math.isnan(df.at[index, 'generation_index']): # sanity check for the baselines that originally had the generation index\n",
    "        count += 1\n",
    "    flag = False\n",
    "    for _, gen_row in generations_df.iterrows():\n",
    "        if gen_row['text'] ==  text:\n",
    "            # is nan?\n",
    "            if not math.isnan(df.at[index, 'generation_index']):\n",
    "                assert(df.at[index, 'generation_index'] == gen_row['generation_index'])\n",
    "            df.at[index, 'generation_index'] = gen_row['generation_index']\n",
    "            flag = True\n",
    "            break\n",
    "    if not flag: \n",
    "        print(f'no match for {index}')\n",
    "        print(text, end=\"\\n\")\n",
    "\n",
    "print(f'count: {count}')\n",
    "print(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 946,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform column advice_indices to string\n",
    "df['participant_strategy'] = df['participant_strategy'].apply(lambda x: str(x))\n",
    "df['advice_indices'] = df['advice_indices'].apply(lambda x: str(x))\n",
    "# tranforms all nan values\n",
    "\n",
    "na_locations = df.isna()\n",
    "row_indices, col_indices = np.where(na_locations)\n",
    "for row, col in zip(row_indices, col_indices):\n",
    "    print(f\"NaN value at row {row}, column {df.columns[col]}\")\n",
    "    # print(df.iloc[row], \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NANs CORRESPOND TO EMPTY STRING ADVICE INDICES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 816,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that origin matches text exactly\n",
    "# origin_id are 13-105, inclusive\n",
    "\n",
    "dict = {}\n",
    "for row in df.itertuples():\n",
    "    if row.origin_id not in dict:\n",
    "        dict[row.origin_id] = row.text\n",
    "    else:\n",
    "        if dict[row.origin_id] != row.text:\n",
    "            print(\"ERROR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of origin_ids with less than 20 ratings: 0 out of 72\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 5384 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check the number of ratings per origin_id\n",
    "non20_count = 0\n",
    "counts = []\n",
    "for id_count in df.groupby('origin_id')[\"id\"].count():\n",
    "    if id_count != 20:\n",
    "        non20_count += 1\n",
    "    counts += [id_count]\n",
    "plt.hist(counts, bins=9)\n",
    "\n",
    "print(f\"Number of origin_ids with less than 20 ratings: {non20_count} out of {len(df.groupby('origin_id'))}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "START OF ANALYSIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 643,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "participant_id: 1\n",
      "participant_strategy: When a server talks a lot, it just makes me feel like they are trying to manipulate people to get them to spend more and Upsell, making themselves look better to management. Most of the time the tone and way they speak feels disingenuous or forced to me. \n",
      "participant_id: 2\n",
      "participant_strategy: I compared the scenarios to experiences that I have had when ordering coffee myself. I thought about the length of dialogue and how likely I was to buy into the extras considering their choice of language. \n",
      "participant_id: 3\n",
      "participant_strategy: I don't like people talking to me, so I decided how annoyed I would be with the barista's waffling to determine if I would tip.  The more waffling and irrelevancy, the less the tip would be.  Indication of care and attention without annoyance got a bigger tip.\n",
      "participant_id: 4\n",
      "participant_strategy: I don't tip barristas so it was easy, it didn't matter what they said\n",
      "participant_id: 5\n",
      "participant_strategy: I read the scenario once and then just read the response each time and considered how it would make me feel to be in that situation. \n",
      "participant_id: 6\n",
      "participant_strategy: To answer the questions honestly based on how they would make me feel in a real-life scenario. \n",
      "participant_id: 7\n",
      "participant_strategy: the more personal and over the top the barista was the more repulsed i was\n",
      "participant_id: 8\n",
      "participant_strategy: to put myself in the situation and imagine how id feel\n",
      "participant_id: 9\n",
      "participant_strategy: I just wanted to know my order would be taken correctly and efficiently in a friendly manner whilst avoiding small talk\n",
      "participant_id: 10\n",
      "participant_strategy: I tried to choose ones to tip higher where the barista appeared happy to serve the customer and did not overly advertise their other products in a way that seemed like it applied pressure on the customer. I preferred ones where the answers were quite simple yet effective.\n",
      "participant_id: 11\n",
      "participant_strategy: To envision myself in the exact scenario and how I would react if I were currently in that situation. Particularly, I considered the level of detail of the answer, relevance of the content and whether tipping was mentioned.\n",
      "participant_id: 12\n",
      "participant_strategy: The nicer the barista was the greater the tip but there is a limit. Too much talking is annoying. And I don't like people asking for a tip\n",
      "participant_id: 13\n",
      "participant_strategy: I like short and considerate conversation with baristas, so I took that into account when it came to the size of the tips I gave. Some of the scenarios included the barista responding with irrelevant information, so my tip was lower in that case.\n",
      "participant_id: 14\n",
      "participant_strategy: To use my imagination of how i would feel in the morning, as much as being very nice is nice i think too much would put me off tipping.\n",
      "participant_id: 15\n",
      "participant_strategy: My stratergy was that if they were polite or made her conversation or asked if I would like certain extras they would get a bigger tip. If they asked for a tip, then no tip was given as I feel that is rude. If they went on a bit about unrelated things I didn't leave as big of a tip. \n",
      "participant_id: 16\n",
      "participant_strategy: read very carefully and answer questions accurately having given them due care and thought\n",
      "participant_id: 17\n",
      "participant_strategy: Likely to leave a a small tip was my main response, increased slightly if i liked the service. SOme of the comments seemed too personal though and so i may have scored those slightly lower\n",
      "participant_id: 18\n",
      "participant_strategy: deciding which responses I liked best\n",
      "participant_id: 19\n",
      "participant_strategy: barista that is friendly but not overly trying\n",
      "participant_id: 20\n",
      "participant_strategy: I thought about how I would feel as the customer and if I liked the barista or not based on their comments. \n",
      "participant_id: 21\n",
      "participant_strategy: I didn't have a stretgy. The response I thought were better I gave a higher score. They shouldn't be mentioning anything about tips so those got a low score and it is nice if they are friendly and personable but you don't always want them to go on that much\n",
      "participant_id: 22\n",
      "participant_strategy: I had no strategy but the weirder the barista got (chatting about water plants, chess and passports) made me less likely to tip. and mentioning a tip is a definite no.\n",
      "participant_id: 23\n",
      "participant_strategy: put myself in the shoes of the customer, based my answers on how i like to be spoken to in a coffee shop. \n",
      "participant_id: 24\n",
      "participant_strategy: How relatable the barista made the transaction feel\n",
      "participant_id: 25\n",
      "participant_strategy: In UK culture we don't do casual tipping - i find a pleasant interaction is nice but also prefer brief and efficient - if tipping is the norm then i'll tip\n",
      "participant_id: 26\n",
      "participant_strategy: I liked the ones where the barista didn't ramble for too long, but was still helpful, and passed on important information/seemed friendly. \n",
      "participant_id: 27\n",
      "participant_strategy: It dependent on how annoying they were how much tip they got. Less is more.\n",
      "participant_id: 28\n",
      "participant_strategy: I don't want over familiarity or obvious up-selling, it annoys me\n",
      "participant_id: 29\n",
      "participant_strategy: I imagined the person saying the words and whether they struck me as someone out to make my experience fun and different.\n",
      "participant_id: 30\n",
      "participant_strategy: nan\n",
      "participant_id: 31\n",
      "participant_strategy: Although the scenarios were very polite in terms of the barista's response to the customer, my strategy was based on whether the response was genuine or not. If the barista's was too forward, i.e. hinting for tips, I personally did not want to tip them high. If they spoke too long about watering of plants etc., I soon became a little frustrated. However, if they made an effort in terms of making the customer feel special, the tip was made higher.\n",
      "participant_id: 32\n",
      "participant_strategy: I enjoyed it, some of the responses were funny. If someone asks for a tip they dont get one. If they make me laugh, offer something but dont assume something then i would give them a tip of some size\n",
      "participant_id: 33\n",
      "participant_strategy: Definite no if they mentioned people enjoyed coffee more if theyd given a tip. If they gave me information on the coffee and beans I considered a tip. I also considered a tip if they upsold a croissant or muffin or wished me a good day\n",
      "participant_id: 34\n",
      "participant_strategy: Just going with my gut and imagining myself in the situation and what I would do. \n",
      "participant_id: 35\n",
      "participant_strategy: i just imagined how i would feel in the customers shoes.\n",
      "participant_id: 36\n",
      "participant_strategy: to answer truthfully \n",
      "participant_id: 39\n",
      "participant_strategy: I think anyone providing services in hospitality, should offer extra with no extra cost tips will fly.\n",
      "participant_id: 40\n",
      "participant_strategy: As a customer in a coffee shop my first annoyance is when the barista becomes too talkative. It is nice to exchange pleasantries through small talk but I can imagine most people are in a rush or just want efficient service. I also thought people would be less likely to tip if their server intrudes on private conversations. \n",
      "participant_id: 41\n",
      "participant_strategy: I thought about what was most deserving to get a tip for without seeming disingenuous. \n",
      "participant_id: 42\n",
      "participant_strategy: I imagined myself as a customer and reacted to how the conversation made me feel at the time\n",
      "participant_id: 43\n",
      "participant_strategy: I thought about how I'd feel if I received the response to my ordering a coffee.\n",
      "participant_id: 44\n",
      "participant_strategy: I have been in situations before when a waiter/waitress has put me on the spot and asked how much I will be tipping. In the UK it is not essential that we tip, so we only do it if the service is really good. If the staff member mentions a tip it automatically makes me want to say no as I believe it comes across rude. If the service is really good and the staff are friendly then I am more inclined to tip to thank them for the services they have provided.\n",
      "participant_id: 45\n",
      "participant_strategy: I really dislike being upsold, so will not tip if the barista tries to sell me anything else. I also dislike faux-friendliness and enthusiasm - I would much rather just be served my latte.\n",
      "participant_id: 46\n",
      "participant_strategy: It was based on if I though they were being genuine\n",
      "participant_id: 47\n",
      "participant_strategy: Definitely anti mentioning tips and too much conversation is a no no\n",
      "participant_id: 48\n",
      "participant_strategy: Didn't like being asked for a tip\n",
      "participant_id: 49\n",
      "participant_strategy: A good interesting study, and my strategy was to  feel the warmth and kindness in the conversations, and to judge in general how real and caring each comment was said and came across.\n",
      "participant_id: 50\n",
      "participant_strategy: I don't generally tip so that is why I was at the lower end. Also, if I found the comment over the top and irrelevant I didn't tip. I was just looking for great service without it being OTT\n",
      "participant_id: 51\n",
      "participant_strategy: Perfer the barista who didn't go hawking for a tip \n",
      "participant_id: 52\n",
      "participant_strategy: Higher tip if they wished me a good day or similar\n",
      "participant_id: 53\n",
      "participant_strategy: I'm not a big tipper, too much chatter irritates me\n",
      "participant_id: 54\n",
      "participant_strategy: I looked at the wording, read the whole paragraph and made a deision as to how helpful the barista was.  Sometimes they were helpful, offering more than needed, but in a nice way.  Sometimes they said totally irrelevent things.  I did notice however, that i made a mistake, and think I said to tip number 9 when it said coffee, not latte, so not sure if that was a test.  If so i apolgise but it was too late by then as i had pressed the next button.\n",
      "participant_id: 55\n",
      "participant_strategy: I would tip relevant & helpful, not left field and wacky suggestions \n",
      "participant_id: 56\n",
      "participant_strategy: I didn't like it when the barista basically asked for a tip to be given - not very professional and totally irked me.  I'll decide if I wish to leave a tip upon the interaction I have with the server, but I won't leave one if you ask.  Rude.\n",
      "participant_id: 57\n",
      "participant_strategy: I just told the truth. I am not keen on people trying to sell me extra or being false to get a tip\n",
      "participant_id: 58\n",
      "participant_strategy: I answered by how good of an impression the barista made on me\n",
      "participant_id: 59\n",
      "participant_strategy: I put myself in the customer's position and imagined what I would like to hear as a response to my order. Too much waffle would make me less likely to tip.\n",
      "participant_id: 61\n",
      "participant_strategy: I just used my gut instincts and imagined myself in the scenario. It was an enjoyable task\n",
      "participant_id: 62\n",
      "participant_strategy: I didnt like the obvious 'asking for a tip' but I did like the useful hints and tips like checking your passport.\n",
      "participant_id: 63\n",
      "participant_strategy: i think that asking or mentioning a tip will reduce the likelihood of a tip being given. I also think the longer responses tend to go down less well in this environment and less likely to tip. Good humour and recommendations can be great customer service and usually worthy of a tip\n",
      "participant_id: 64\n",
      "participant_strategy: Some feedback for you, hope this is ok! In the UK, we do not have a tipping culture at all. So, anyone serving a customer who would explicitly mention tipping would automatically be frowned upon, it is seen as very bad manners here. I marked down any response that mentioned a tip, but also marked down any that were over familiar. Mentioning passport etc - that doesn't seem anything like a normal human server would say, it comes across as AI response (and if a person DID say that in the UK customers would give them strange looks!). Really enjoyed the task, good luck with your research.\n",
      "participant_id: 65\n",
      "participant_strategy: I always leave a tip anyway so that wasnt a difficult choice.  i would leave less of a tip if the barista asked for one, it think its really rude and unnecessary.  I'm not really interested in any of the fun facts and would perhaps consider a bigger tip if they were just polite and to the point without any fluff.\n",
      "participant_id: 66\n",
      "participant_strategy: iT WAS THOUGHT PROVOKING AND QUITE DECISIVE AT THE EXTREMES. I preferred the short succinct answers to the long winded ones trying to engage in meaningless conversation. The best responses were short, direct and to the point delivered with politeness. That is what great customer service means to me. It's great that someone was confident and engagaed in lengthier replies ,but it generally doesn't work for me. I think this works well when the server can 'read' the customer before 'going' there. I didn't mind a little bit of extra information. I wasn't interested in what the milk did, or other flavours much of the time because of the way it was presented. However, one of two of the response about the almond croissant went down well because they were succinct and the language and tone worked for me personally. I hope this helps!\n",
      "participant_id: 67\n",
      "participant_strategy: I felt that if the barista asked for a tip or even mentioned it, they were being cheeky, so I wouldn't tip at all.  If the barista was pleasant but not particularly personable I might leave a small tip.  If the barista was pleasant, personable and went the extra mile, I would tip them more.\n",
      "participant_id: 68\n",
      "participant_strategy: Give the biggest tip when the barista is being authentic. If someone asks for a tip it's desperate and a turn off. If their service is genuine and they're a good person then they'll get a tip without having to beg.\n",
      "participant_id: 69\n",
      "participant_strategy: I didn't have a strategy as such, I was just imagining how it would feel standing in a coffee shop being part of that exchange. How I would feel and how the person making the drink would represent themselves to me through their speech and behaviour.\n",
      "participant_id: 70\n",
      "participant_strategy: Happy that I knew the first part of the question, so just concentrated on the barista's reply\n",
      "participant_id: 71\n",
      "participant_strategy: If they mentioning me tipping them, I don't tip. I tip based on friendliness.\n",
      "participant_id: 72\n",
      "participant_strategy: my strategy was to reward politeness but not to reward as much or nothing if there was too much selling of food I had not asked for and if the barista went into too much detail on unrelated matters like the plants\n",
      "participant_id: 73\n",
      "participant_strategy: Informative and friendly\n",
      "participant_id: 74\n",
      "participant_strategy: dont like to pushy\n",
      "participant_id: 75\n",
      "participant_strategy: By assessing whether the comments made to the customer were sincere and polite. If they suggested a tip or hinted toward the customer leaving a tip then I would give it a bad score as that can come across pushy or rude.\n",
      "participant_id: 76\n",
      "participant_strategy: I dont tip to any one who talks too much or asks for one\n",
      "participant_id: 77\n",
      "participant_strategy: If the barrista's chat sounded fake or inappropriate I was less likely to leave a tip\n",
      "participant_id: 78\n",
      "participant_strategy: it was good the more they made conversation higher the tip as soon as asked for a tip no tip at all\n",
      "participant_id: 79\n",
      "participant_strategy: answer honestly and openly as if it was a real scenario\n",
      "participant_id: 80\n",
      "participant_strategy: My strategy was to rate the customer service experienced based on politeness, staying on-track with the conversation and not adding too many unrelated comments. \n",
      "participant_id: 81\n",
      "participant_strategy: Based on what I would want the person serving me to ask me.\n",
      "participant_id: 83\n",
      "participant_strategy: reward someone who is being nice\n",
      "participant_id: 84\n",
      "participant_strategy: I looked at how each barista replied to the customer's request for a latte. I checked if they were polite, quick in responding, and made the customer feel good. I also checked if they suggested extra things to buy and paid attention to what the customer wanted. This helped me decide how likely I would be to give a tip.\n",
      "participant_id: 85\n",
      "participant_strategy: My strategy was to imagine that I was the customer in the shop ordering a medium latte and to imagine how impressed I would be with each response and whether or not I would be likely to leave a tip in the real world.\n",
      "participant_id: 86\n",
      "participant_strategy: I preferred friendly service that offered complimentary water and scored that highly. Where I was asked for a tip directly, I scored low. I did exactly what I would do in real life. This was my strategy.\n",
      "participant_id: 87\n",
      "participant_strategy: the complimentary glass of water and the genuine concern for staying hydrated would also generate the biggest tip from me, whereas simply trying to upsell a dessert would not \n",
      "participant_id: 88\n",
      "participant_strategy: I preferred the choices being mine rather than feeling pressured into taking what the Barista offered\n",
      "participant_id: 89\n",
      "participant_strategy: If they were kind and didnt talk much i would tip more\n",
      "participant_id: 90\n",
      "participant_strategy: I tipped the baristas who were polite and had some chat with me. I didn't like anyone who suggested tipping or those who started talking about chess which just seemed weird and forced as both would make me feel uncomfortable.\n",
      "participant_id: 91\n",
      "participant_strategy: Picture myself inside a coffee shop/cafe and how would I react if the server said the same things. Also if it looked like they were trying too hard\n",
      "participant_id: 92\n",
      "participant_strategy: I tried to think about tipping\n",
      "participant_id: 93\n",
      "participant_strategy: My strategy was to evaluate I would feel about the baristas response and based on that indicate how much of a tip id be likely to give. I preferred quicker friendly responses rather than long winded ones that felt forced.\n",
      "participant_id: 94\n",
      "participant_strategy: I enjoyed the task. My strategy was simply based off what i would do. I gave answers that seemed genuinely human and weren't simply begging for tips, but actually being helpful and not (seemingly) focusing on making money, higher scores.\n",
      "participant_id: 95\n",
      "participant_strategy: If I liked their responce I would tip well.  I don't like being asked for tips though, or being reminded to tip, that's going to put my nose right out of joint.  I don't like people getting too familiar either, or hard up-sells.  I do like additional effort though, so if someone recommends something I might Iike I might tip a bit extra.\n",
      "participant_id: 96\n",
      "participant_strategy: I rated the responses based on perceived sincerity, tone, politeness, and length.  If i felt the response was to convoluted and/or irrelevant i rated it worse. Any response mentioning tips scored 0. any response that was short, relevant to the coffee and felt like casual conversation was charming and rated higher. \n",
      "participant_id: 97\n",
      "participant_strategy: I was most likely to tip when the barrista was being helpful and genuine. I would not tip if they started randomly talking about travel - it just felt unnatural and a bit creepy.\n",
      "participant_id: 98\n",
      "participant_strategy: To answer truthfully.\n",
      "participant_id: 99\n",
      "participant_strategy: HONESTY\n",
      "participant_id: 100\n",
      "participant_strategy: I really enjoyed this task and trying to imagine if i would leave a tip or not. I do like to leave a small tip unless the staff are rude in some way or just don't go out their way to help me.\n",
      "Thanks for the opportunity.\n",
      "participant_id: 101\n",
      "participant_strategy: politeness, interest in the customer, sincere, no selfish mention of tips\n",
      "participant_id: 102\n",
      "participant_strategy: I tried to think about what sort of interaction most people would like when ordering a coffee.\n",
      "participant_id: 103\n",
      "participant_strategy: No major strategy, just went with what felt right to me\n",
      "participant_id: 104\n",
      "participant_strategy: No strategy, more likely to tip for the responses I liked\n",
      "participant_id: 105\n",
      "participant_strategy: Fairly simple - just read the information presented and make a judgement based on what I read.\n",
      "participant_id: 106\n",
      "participant_strategy: I didn't like the very long elaborate responses and would be in too much of a rush to get away. I like the short but warm and helpful responses and favoured these the most.\n",
      "participant_id: 107\n",
      "participant_strategy: I liked the shorter answers so I wouldn't have to hang around longer than I needed to for my coffee. \n"
     ]
    }
   ],
   "source": [
    "# READ THROUGH PARTICIPANT STRATEGIES\n",
    "\n",
    "grouped = df.groupby('participant_id')\n",
    "\n",
    "# iterate through and print out the participant strategies\n",
    "for name, group in grouped:\n",
    "    print(f\"participant_id: {name}\")\n",
    "    print(f\"participant_strategy: {group['participant_strategy'].iloc[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 644,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9128353907765543 0.01241765359332354\n"
     ]
    }
   ],
   "source": [
    "# COMPUTE INTER RATER RELIABILITY\n",
    "results = []\n",
    "\n",
    "for i in range(10):\n",
    "    group1_means = []\n",
    "    group2_means = []\n",
    "\n",
    "    grouped_by_origin_id = df.groupby('origin_id') # grouped based on question\n",
    "    # loop across questions\n",
    "    for name, group in grouped_by_origin_id:\n",
    "        # randomly split participants in half\n",
    "        df_tmp = df[df['origin_id'] == name]\n",
    "        unique_participant_ids = df_tmp['participant_id'].unique()\n",
    "        np.random.shuffle(unique_participant_ids)\n",
    "        if len(unique_participant_ids) % 2 == 1:\n",
    "            unique_participant_ids = unique_participant_ids[:-1]\n",
    "\n",
    "        if len(unique_participant_ids) < 2:\n",
    "            continue\n",
    "        \n",
    "        split_point = int(len(unique_participant_ids) / 2)\n",
    "        first_half = unique_participant_ids[:split_point]\n",
    "        second_half = unique_participant_ids[split_point:]\n",
    "        \n",
    "        group1 = df_tmp[df_tmp['participant_id'].isin(first_half)]\n",
    "        group2 = df_tmp[df_tmp['participant_id'].isin(second_half)]\n",
    "\n",
    "        group1_means += [group1['answer'].mean()]\n",
    "        group2_means += [group2['answer'].mean()]\n",
    "        \n",
    "    # compute inter rater reliability\n",
    "    # results += [np.corrcoef(group1_means, group2_means)[0, 1]]\n",
    "\n",
    "    # apply spearman-brown correction\n",
    "    results += [2 * np.corrcoef(group1_means, group2_means)[0, 1] / (1 + np.corrcoef(group1_means, group2_means)[0, 1])]\n",
    "\n",
    "print(np.mean(results), np.std(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 2.,  7.,  4.,  9., 19., 20., 22., 13.,  3.,  2.,  2.]),\n",
       " array([0.        , 0.8041958 , 1.60839161, 2.41258741, 3.21678322,\n",
       "        4.02097902, 4.82517483, 5.62937063, 6.43356643, 7.23776224,\n",
       "        8.04195804, 8.84615385]),\n",
       " <BarContainer object of 11 artists>)"
      ]
     },
     "execution_count": 318,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAAsTAAALEwEAmpwYAAAKnUlEQVR4nO3dX4il913H8c/XpKJNRVMyhJg/TpBSCUKTssRopFRTJTVi2htpwBKksF6kmkpB1t7oZQSteiGF2MQGjBFJUxpMqS2xUAQJbtJg88eSErdtYv5sKNroTU379WJPZLM7m5mdc3bOfDevFyxzznPOzPPlYefNM8+c35zq7gAwzw+sewAAdkfAAYYScIChBBxgKAEHGOrcvdzZBRdc0Jubm3u5S4DxHn744Ze6e+PE7Xsa8M3NzRw+fHgvdwkwXlV9Y6vtLqEADCXgAEMJOMBQAg4wlIADDCXgAEMJOMBQAg4wlIADDLWnKzHhbLN56IE93d+R227Y0/2xvzkDBxhKwAGGEnCAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxhKwAGG2jbgVXVpVX2pqp6oqser6tbF9rdW1Rer6qnFx/PP/LgAvGonZ+CvJPlod1+R5Jokt1TVFUkOJXmwu9+W5MHFfQD2yLYB7+7nuvuRxe2XkzyZ5OIkNya5a/G0u5K87wzNCMAWTusaeFVtJrkqyUNJLuzu5xYPPZ/kwtWOBsDr2fFbqlXVW5J8OslHuvs7VfX/j3V3V1Wf4vMOJjmYJJdddtly08IO7PXbnMG67OgMvKrelGPxvru771tsfqGqLlo8flGSF7f63O6+vbsPdPeBjY2NVcwMQHb2KpRKckeSJ7v748c9dH+Smxe3b07y2dWPB8Cp7OQSyrVJPpjkq1X16GLbx5LcluTvqupDSb6R5NfPyIQAbGnbgHf3PyWpUzx83WrHAWCnrMQEGErAAYYScIChBBxgqB0v5IHdsrAGzgxn4ABDCTjAUAIOMJSAAwwl4ABDCTjAUAIOMJSAAwwl4ABDCTjAUAIOMJSAAwwl4ABDCTjAUAIOMJSAAwwl4ABDCTjAUAIOMJSAAwwl4ABDCTjAUAIOMJSAAwwl4ABDCTjAUAIOMJSAAwwl4ABDCTjAUAIOMJSAAwwl4ABDbRvwqrqzql6sqseO2/aHVfVsVT26+PcrZ3ZMAE60kzPwTyW5fovtf9rdVy7+fW61YwGwnW0D3t1fTvLtPZgFgNOwzDXwD1fVvy4usZy/sokA2JHdBvwTSX4yyZVJnkvyJ6d6YlUdrKrDVXX46NGju9wdACfaVcC7+4Xu/l53fz/JXya5+nWee3t3H+juAxsbG7udE4AT7CrgVXXRcXffn+SxUz0XgDPj3O2eUFX3JHl3kguq6pkkf5Dk3VV1ZZJOciTJb525EQHYyrYB7+6btth8xxmYBYDTYCUmwFACDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFDb/jVCYP/YPPTAnu7vyG037On+OD3OwAGGEnCAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxhKwAGGEnCAoQQcYCgBBxhKwAGG2jbgVXVnVb1YVY8dt+2tVfXFqnpq8fH8MzsmACfayRn4p5Jcf8K2Q0ke7O63JXlwcR+APbRtwLv7y0m+fcLmG5Pctbh9V5L3rXYsALaz22vgF3b3c4vbzye58FRPrKqDVXW4qg4fPXp0l7sD4ERL/xKzuztJv87jt3f3ge4+sLGxsezuAFjYbcBfqKqLkmTx8cXVjQTATuw24PcnuXlx++Ykn13NOADs1E5eRnhPkn9O8vaqeqaqPpTktiS/VFVPJXnP4j4Ae+jc7Z7Q3Ted4qHrVjwLAKfBSkyAoQQcYCgBBxhq22vgnH02Dz2w7hGAFXAGDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMM5S3VTmGv33bsyG037On+gPmcgQMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFBL/T3wqjqS5OUk30vySncfWMVQAGxvFW/o8Avd/dIKvg4Ap8ElFIChlg14J/lCVT1cVQe3ekJVHayqw1V1+OjRo0vuDoBXLRvwn+/udyZ5b5JbqupdJz6hu2/v7gPdfWBjY2PJ3QHwqqUC3t3PLj6+mOQzSa5exVAAbG/XAa+q86rqR169neSXkzy2qsEAeH3LvArlwiSfqapXv87fdPfnVzIVANvadcC7++kk71jhLACcBi8jBBhKwAGGEnCAoVaxlJ4V2Dz0wLpHAIZxBg4wlIADDCXgAEMJOMBQAg4wlIADDCXgAEMJOMBQAg4wlJWYwCnt5QrhI7fdsGf7Ols4AwcYSsABhhJwgKEEHGAoAQcYSsABhhJwgKEEHGCoMQt5vOUYnN3O9u/xM7FQyRk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMMJeAAQwk4wFACDjCUgAMMtVTAq+r6qvpaVX29qg6taigAtrfrgFfVOUn+Isl7k1yR5KaqumJVgwHw+pY5A786yde7++nu/m6Sv01y42rGAmA7y7yl2sVJvnXc/WeS/MyJT6qqg0kOLu7+d1V9bZf7uyDJS7v83LOVY3Iyx+RkjslrreV41B8t9ek/sdXGM/6emN19e5Lbl/06VXW4uw+sYKSzhmNyMsfkZI7Ja51Nx2OZSyjPJrn0uPuXLLYBsAeWCfi/JHlbVV1eVT+Y5ANJ7l/NWABsZ9eXULr7lar6cJJ/SHJOkju7+/GVTXaypS/DnIUck5M5JidzTF7rrDke1d3rngGAXbASE2AoAQcYakTALdl/raq6tKq+VFVPVNXjVXXrumfaD6rqnKr6SlX9/bpn2Q+q6seq6t6q+reqerKqfnbdM61bVf3u4nvmsaq6p6p+aN0zLWPfB9yS/S29kuSj3X1FkmuS3OKYJEluTfLkuofYR/48yee7+6eSvCNv8GNTVRcn+Z0kB7r7p3PsxRcfWO9Uy9n3AY8l+yfp7ue6+5HF7Zdz7Bvz4vVOtV5VdUmSG5J8ct2z7AdV9aNJ3pXkjiTp7u9293+udaj94dwkP1xV5yZ5c5L/WPM8S5kQ8K2W7L+hY3W8qtpMclWSh9Y8yrr9WZLfS/L9Nc+xX1ye5GiSv1pcVvpkVZ237qHWqbufTfLHSb6Z5Lkk/9XdX1jvVMuZEHBOoarekuTTST7S3d9Z9zzrUlW/muTF7n543bPsI+cmeWeST3T3VUn+J8kb+vdHVXV+jv30fnmSH09yXlX9xnqnWs6EgFuyv4WqelOOxfvu7r5v3fOs2bVJfq2qjuTYJbZfrKq/Xu9Ia/dMkme6+9WfzO7NsaC/kb0nyb9399Hu/t8k9yX5uTXPtJQJAbdk/wRVVTl2bfPJ7v74uudZt+7+/e6+pLs3c+z/xz929+gzq2V19/NJvlVVb19sui7JE2scaT/4ZpJrqurNi++h6zL8F7tn/K8RLmsNS/YnuDbJB5N8taoeXWz7WHd/bn0jsQ/9dpK7Fyc+Tyf5zTXPs1bd/VBV3ZvkkRx7JddXMnxZvaX0AENNuIQCwBYEHGAoAQcYSsABhhJwgKEEHGAoAQcY6v8AtXQiS6zX5loAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF AVERAGE SCORES PER PARTICIPANT\n",
    "\n",
    "mean_scores = df.groupby('participant_id')['answer'].mean()\n",
    "plt.hist(mean_scores, bins=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "11\n",
      "[5.0, 2.0, 0.0, 0.0, 2.0, 2.0, 0.0, 1.0, 5.0, 6.0, 4.0, 2.0, 4.0, 6.0]\n",
      "7\n",
      "[2.0, 3.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "20\n",
      "[2.0, 5.0, 0.0, 0.0, 8.0, 0.0, 1.0, 3.0, 5.0, 5.0, 0.0, 3.0, 5.0, 0.0, 5.0, 4.0]\n",
      "24\n",
      "[0.0, 3.0, 1.0, 4.0, 0.0, 0.0, 3.0, 2.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 5.0]\n",
      "27\n",
      "[5.0, 5.0, 0.0, 0.0, 0.0, 5.0, 0.0, 0.0, 2.0, 0.0, 0.0, 5.0, 0.0, 0.0, 2.0]\n",
      "22\n",
      "[5.0, 2.0, 0.0, 4.0, 4.0, 4.0, 2.0, 2.0, 0.0, 0.0, 1.0, 2.0, 0.0, 2.0]\n",
      "28\n",
      "[0.0, 0.0, 4.0, 5.0, 0.0, 1.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 4.0, 0.0]\n",
      "36\n",
      "[0.0, 1.0, 4.0, 5.0, 0.0, 0.0, 0.0, 4.0, 0.0, 0.0, 0.0, 0.0]\n",
      "35\n",
      "[8.0, 8.0, 7.0, 3.0, 0.0, 0.0, 1.0, 0.0, 0.0, 5.0, 6.0, 2.0, 4.0]\n",
      "41\n",
      "[5.0, 2.0, 2.0, 0.0, 2.0, 6.0, 8.0, 1.0, 0.0, 5.0, 2.0, 1.0, 6.0, 5.0, 4.0, 0.0, 3.0]\n",
      "39\n",
      "[0.0, 0.0, 5.0, 5.0, 1.0, 2.0, 8.0, 7.0, 2.0, 2.0, 2.0, 0.0, 9.0, 0.0, 2.0]\n",
      "42\n",
      "[6.0, 0.0, 5.0, 4.0, 7.0, 0.0, 3.0, 7.0, 0.0, 0.0, 0.0, 5.0, 5.0, 0.0]\n",
      "45\n",
      "[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 3.0, 4.0, 4.0, 5.0]\n",
      "57\n",
      "[0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 5.0, 0.0, 6.0, 0.0, 1.0, 0.0, 0.0]\n",
      "50\n",
      "[2.0, 0.0, 0.0, 3.0, 4.0, 2.0, 0.0, 0.0, 0.0, 3.0, 3.0, 4.0, 3.0, 5.0, 5.0, 3.0, 0.0]\n",
      "64\n",
      "[6.0, 0.0, 10.0, 5.0, 5.0, 0.0, 4.0, 0.0, 5.0, 0.0, 0.0, 6.0, 0.0]\n",
      "63\n",
      "[0.0, 0.0, 5.0, 3.0, 1.0, 3.0, 0.0, 4.0, 0.0, 3.0]\n",
      "71\n",
      "[0.0, 4.0, 2.0, 1.0, 3.0, 6.0, 5.0, 0.0, 0.0, 6.0, 6.0, 0.0, 4.0]\n",
      "74\n",
      "[8.0, 5.0, 4.0, 3.0, 3.0, 0.0, 3.0, 0.0, 0.0, 5.0, 0.0, 0.0, 0.0, 0.0, 0.0, 5.0, 6.0]\n",
      "76\n",
      "[5.0, 0.0, 0.0, 2.0, 6.0, 0.0, 0.0, 0.0, 5.0, 8.0, 6.0, 5.0, 0.0, 4.0, 3.0]\n",
      "80\n",
      "[3.0, 0.0, 3.0, 8.0, 0.0, 9.0, 0.0, 0.0, 7.0]\n",
      "89\n",
      "[6.0, 4.0, 1.0, 5.0, 3.0, 2.0, 2.0, 1.0, 3.0, 2.0, 6.0, 6.0]\n",
      "91\n",
      "[2.0, 5.0, 5.0, 0.0, 0.0, 1.0, 0.0, 2.0, 1.0, 0.0, 0.0, 2.0, 3.0, 0.0, 0.0, 0.0, 3.0]\n",
      "96\n",
      "[2.0, 4.0, 4.0, 3.0, 0.0, 1.0, 0.0, 4.0, 3.0, 3.0, 2.0, 0.0]\n",
      "88\n",
      "[9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 8.0, 8.0]\n"
     ]
    }
   ],
   "source": [
    "# TRY TO CLEAN THE DATA\n",
    "\n",
    "mean_scores = df.groupby('participant_id')['answer'].mean()\n",
    "for id in df['participant_id'].unique():\n",
    "    if mean_scores[id] < 3.5:\n",
    "        print(id)\n",
    "        # print in a list instead\n",
    "        print(list(df[df['participant_id'] == id]['answer']))\n",
    "\n",
    "for id in df['participant_id'].unique():\n",
    "    if mean_scores[id] > 8.5:\n",
    "        print(id)\n",
    "        # print in a list instead\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "high variances\n",
      "6\n",
      "[7.0, 2.0, 2.0, 2.0, 2.0, 8.0, 1.0, 8.0, 5.0, 10.0, 0.0, 0.0, 0.0]\n",
      "8\n",
      "[8.0, 8.0, 9.0, 9.0, 6.0, 1.0, 0.0, 7.0, 2.0, 5.0, 0.0, 6.0, 7.0]\n",
      "10\n",
      "[0.0, 10.0, 9.0, 9.0, 10.0, 3.0, 4.0, 3.0, 5.0, 2.0, 5.0, 2.0, 8.0, 3.0, 1.0, 6.0]\n",
      "13\n",
      "[5.0, 9.0, 10.0, 10.0, 3.0, 5.0, 2.0, 9.0, 9.0, 8.0, 2.0, 6.0, 8.0, 3.0]\n",
      "18\n",
      "[9.0, 9.0, 9.0, 8.0, 0.0, 5.0, 8.0, 1.0, 8.0, 6.0, 6.0, 5.0, 1.0, 6.0]\n",
      "15\n",
      "[8.0, 9.0, 0.0, 0.0, 6.0, 5.0, 8.0, 0.0, 4.0, 0.0, 8.0, 6.0, 8.0, 8.0]\n",
      "16\n",
      "[6.0, 9.0, 6.0, 0.0, 8.0, 9.0, 0.0, 0.0, 0.0, 0.0]\n",
      "32\n",
      "[3.0, 0.0, 6.0, 7.0, 10.0, 8.0, 0.0, 3.0, 7.0, 5.0, 6.0, 6.0, 3.0]\n",
      "33\n",
      "[1.0, 1.0, 8.0, 7.0, 0.0, 6.0, 7.0, 5.0, 0.0, 5.0, 0.0, 6.0, 5.0]\n",
      "35\n",
      "[8.0, 8.0, 7.0, 3.0, 0.0, 0.0, 1.0, 0.0, 0.0, 5.0, 6.0, 2.0, 4.0]\n",
      "40\n",
      "[9.0, 7.0, 2.0, 0.0, 5.0, 5.0, 8.0, 0.0, 7.0, 0.0, 1.0, 0.0, 3.0]\n",
      "39\n",
      "[0.0, 0.0, 5.0, 5.0, 1.0, 2.0, 8.0, 7.0, 2.0, 2.0, 2.0, 0.0, 9.0, 0.0, 2.0]\n",
      "42\n",
      "[6.0, 0.0, 5.0, 4.0, 7.0, 0.0, 3.0, 7.0, 0.0, 0.0, 0.0, 5.0, 5.0, 0.0]\n",
      "48\n",
      "[7.0, 7.0, 8.0, 1.0, 7.0, 2.0, 0.0, 6.0, 2.0, 1.0, 8.0, 0.0, 7.0, 2.0]\n",
      "54\n",
      "[8.0, 3.0, 2.0, 0.0, 9.0, 2.0, 10.0, 7.0, 0.0, 7.0, 2.0, 0.0, 0.0]\n",
      "55\n",
      "[5.0, 1.0, 0.0, 8.0, 0.0, 0.0, 7.0, 1.0, 7.0, 6.0, 9.0, 1.0, 8.0]\n",
      "68\n",
      "[0.0, 8.0, 0.0, 2.0, 5.0, 5.0, 4.0, 6.0, 7.0, 0.0, 7.0, 8.0]\n",
      "64\n",
      "[6.0, 0.0, 10.0, 5.0, 5.0, 0.0, 4.0, 0.0, 5.0, 0.0, 0.0, 6.0, 0.0]\n",
      "78\n",
      "[10.0, 5.0, 5.0, 9.0, 9.0, 7.0, 7.0, 4.0, 5.0, 5.0, 7.0, 8.0, 0.0, 7.0, 0.0, 6.0]\n",
      "80\n",
      "[3.0, 0.0, 3.0, 8.0, 0.0, 9.0, 0.0, 0.0, 7.0]\n",
      "81\n",
      "[6.0, 2.0, 0.0, 1.0, 4.0, 8.0, 2.0, 7.0, 6.0, 7.0, 8.0, 0.0, 3.0, 0.0]\n",
      "83\n",
      "[5.0, 10.0, 0.0, 6.0, 0.0, 6.0, 5.0, 6.0, 1.0, 5.0, 0.0, 6.0]\n",
      "87\n",
      "[0.0, 7.0, 9.0, 0.0, 8.0, 10.0, 2.0, 3.0, 6.0, 10.0, 3.0, 8.0]\n",
      "86\n",
      "[10.0, 3.0, 9.0, 9.0, 9.0, 4.0, 8.0, 9.0, 9.0, 0.0, 3.0, 9.0, 9.0, 1.0, 7.0, 6.0, 1.0]\n",
      "99\n",
      "[3.0, 5.0, 10.0, 8.0, 9.0, 8.0, 3.0, 8.0, 1.0, 8.0, 2.0, 2.0, 10.0, 8.0]\n",
      "95\n",
      "[8.0, 7.0, 5.0, 7.0, 4.0, 9.0, 2.0, 0.0, 0.0, 0.0, 6.0, 1.0, 0.0, 7.0, 6.0, 6.0]\n",
      "107\n",
      "[2.0, 0.0, 6.0, 7.0, 7.0, 4.0, 8.0, 0.0, 5.0, 2.0, 1.0]\n"
     ]
    }
   ],
   "source": [
    "# MORE DATA CLEANING\n",
    "\n",
    "# # LESS THAN 20\n",
    "# scores = df.groupby('participant_id')['answer']\n",
    "# for id in df['participant_id'].unique():\n",
    "#     if len(list(scores.get_group(id))) < 20:\n",
    "#         print(id)\n",
    "#         print(list(df[df['participant_id'] == id]['answer']))\n",
    "\n",
    "# CHECK FOR ABNORMALLY HIGH VARIANCE\n",
    "print(\"high variances\")\n",
    "variances = df.groupby('participant_id')['answer'].var()\n",
    "for id in df['participant_id'].unique():\n",
    "    if variances[id] > 8:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "31\n",
      "[5.0, 5.0, 6.0, 5.0, 4.0, 5.0, 6.0, 5.0, 5.0, 5.0, 6.0, 5.0, 5.0, 5.0, 6.0, 4.0, 5.0, 6.0]\n",
      "53\n",
      "[6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0]\n",
      "52\n",
      "[6.0, 4.0, 5.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 5.0, 5.0, 6.0, 6.0]\n",
      "70\n",
      "[6.0, 8.0, 7.0, 6.0, 7.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0]\n",
      "88\n",
      "[9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 8.0, 8.0]\n"
     ]
    }
   ],
   "source": [
    "# CHECK FOR ABNORMALLY LOW VARIANCE\n",
    "\n",
    "variances = df.groupby('participant_id')['answer'].var()\n",
    "for id in df['participant_id'].unique():\n",
    "    if variances[id] < 0.6:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "7\n",
      "[2.0, 3.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "24\n",
      "[0.0, 3.0, 1.0, 4.0, 0.0, 0.0, 3.0, 2.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 5.0]\n",
      "23\n",
      "[2.0, 4.0, 5.0, 5.0, 0.0, 3.0, 5.0, 5.0, 4.0, 5.0, 4.0, 5.0, 5.0, 5.0]\n",
      "27\n",
      "[5.0, 5.0, 0.0, 0.0, 0.0, 5.0, 0.0, 0.0, 2.0, 0.0, 0.0, 5.0, 0.0, 0.0, 2.0]\n",
      "28\n",
      "[0.0, 0.0, 4.0, 5.0, 0.0, 1.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 4.0, 0.0]\n",
      "36\n",
      "[0.0, 1.0, 4.0, 5.0, 0.0, 0.0, 0.0, 4.0, 0.0, 0.0, 0.0, 0.0]\n",
      "31\n",
      "[5.0, 5.0, 6.0, 5.0, 4.0, 5.0, 6.0, 5.0, 5.0, 5.0, 6.0, 5.0, 5.0, 5.0, 6.0, 4.0, 5.0, 6.0]\n",
      "45\n",
      "[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 3.0, 4.0, 4.0, 5.0]\n",
      "53\n",
      "[6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0]\n",
      "57\n",
      "[0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 5.0, 0.0, 6.0, 0.0, 1.0, 0.0, 0.0]\n",
      "52\n",
      "[6.0, 4.0, 5.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 5.0, 5.0, 6.0, 6.0]\n",
      "70\n",
      "[6.0, 8.0, 7.0, 6.0, 7.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0]\n",
      "88\n",
      "[9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 8.0, 8.0]\n"
     ]
    }
   ],
   "source": [
    "# CHECK FOR REPETITIONS\n",
    "\n",
    "for id in df['participant_id'].unique():\n",
    "    answer = list(df[df['participant_id'] == id]['answer'])\n",
    "    # if the answer contains more than half of the same value\n",
    "    for i in range(len(answer)//2 + 1):\n",
    "        if answer.count(answer[i]) > len(answer) / 2:\n",
    "            print(id)\n",
    "            print(answer)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# REMOVE PARTICIPANTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([242.,   0.,  63.,   0., 123.,   0.,  99.,   0., 117.,   0., 254.,\n",
       "          0., 198.,   0., 153.,   0.,  98.,   0.,  62.,   0.,  31.]),\n",
       " array([ 0.        ,  0.47619048,  0.95238095,  1.42857143,  1.9047619 ,\n",
       "         2.38095238,  2.85714286,  3.33333333,  3.80952381,  4.28571429,\n",
       "         4.76190476,  5.23809524,  5.71428571,  6.19047619,  6.66666667,\n",
       "         7.14285714,  7.61904762,  8.0952381 ,  8.57142857,  9.04761905,\n",
       "         9.52380952, 10.        ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 323,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4928 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF SCORES\n",
    "\n",
    "plt.hist(df['answer'], bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1., 3., 5., 6., 5., 4., 3., 2., 2., 2., 1., 2., 1., 6., 9., 7., 6.,\n",
       "        3., 1., 3.]),\n",
       " array([1.25  , 1.5475, 1.845 , 2.1425, 2.44  , 2.7375, 3.035 , 3.3325,\n",
       "        3.63  , 3.9275, 4.225 , 4.5225, 4.82  , 5.1175, 5.415 , 5.7125,\n",
       "        6.01  , 6.3075, 6.605 , 6.9025, 7.2   ]),\n",
       " <BarContainer object of 20 artists>)"
      ]
     },
     "execution_count": 324,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAAsTAAALEwEAmpwYAAALMUlEQVR4nO3dX4ildR3H8c/HHcU/mQY7hLlOx4sQRCjloJUhoRnaiHbRhYJB3kwXZWsFMXYj3W0QYRcRLLtFkSm1KoQbVqBSXmTNbIa5a2A26pq2G1G63mzWp4vzrE3b6DyeOc8835l9v2BwzpznnOf76O7bZ37nn5MIAFDXSX0PAAB4c4QaAIoj1ABQHKEGgOIINQAUN9XFnW7dujWDwaCLuwaATWlxcfGvSaZXuq6TUA8GAy0sLHRx1wCwKdl+9o2uY+kDAIoj1ABQHKEGgOIINQAUR6gBoDhCDQDFEWoAKI5QA0BxhBoAiuvklYkA6hjM7x37tks7Zic4CcbFGTUAFEeoAaA4Qg0AxRFqACiOUANAcYQaAIoj1ABQHKEGgOIINQAUR6gBoDhCDQDFEWoAKI5QA0BxhBoAiiPUAFAcoQaA4gg1ABRHqAGgOEINAMW1CrXtz9t+0vbvbd9t+9SuBwMAjKwaatvnSvqcpGGSiyRtkXRj14MBAEbaLn1MSTrN9pSk0yX9ubuRAADLrRrqJC9I+pqk5yS9KOkfSX52/Ha252wv2F44fPjw5CcFgBNUm6WPd0i6QdL5kt4l6QzbNx+/XZKdSYZJhtPT05OfFABOUG2WPj4i6U9JDif5p6T7JH2w27EAAMe0CfVzkt5v+3TblnSVpAPdjgUAOKbNGvVjkvZI2ifpieY2OzueCwDQmGqzUZI7JN3R8SwAgBXwykQAKI5QA0BxhBoAiiPUAFAcoQaA4gg1ABRHqAGgOEINAMURagAojlADQHGEGgCKI9QAUByhBoDiCDUAFEeoAaA4Qg0AxRFqACiu1Se8ADgxDeb3jn3bpR2zE5zkxMYZNQAUR6gBoDhCDQDFEWoAKI5QA0BxhBoAiiPUAFAcoQaA4gg1ABRHqAGgOEINAMURagAojlADQHGEGgCKI9QAUByhBoDiCDUAFEeoAaA4Qg0AxbUKte2zbe+x/ZTtA7Y/0PVgAICRth9u+w1JDyb5hO1TJJ3e4UwAgGVWDbXtsyRdIelTkpTkqKSj3Y4FADimzRn1+ZIOS/qO7fdKWpS0PcmryzeyPSdpTpJmZmYmPWdpg/m9a7r90o7ZCU2CzWqtf8b6wN+LyWmzRj0l6RJJ30pysaRXJc0fv1GSnUmGSYbT09MTHhMATlxtQn1Q0sEkjzWX92gUbgDAOlg11ElekvS87QuaH10laX+nUwEAXtf2WR+3SrqrecbHM5Ju6W4kAMByrUKd5HFJw25HAQCshFcmAkBxhBoAiiPUAFAcoQaA4gg1ABRHqAGgOEINAMURagAojlADQHGEGgCKI9QAUByhBoDiCDUAFEeoAaA4Qg0AxRFqACiOUANAcW0/imvTW+tH2/e176UdsxOcBEBFnFEDQHGEGgCKI9QAUByhBoDiCDUAFEeoAaA4Qg0AxRFqACiOUANAcYQaAIoj1ABQHKEGgOIINQAUR6gBoDhCDQDFEWoAKI5QA0BxhBoAiiPUAFBc61Db3mL7t7Yf6HIgAMD/eitn1NslHehqEADAylqF2vY2SbOSdnU7DgDgeFMtt7tT0pcknflGG9iekzQnSTMzM2seDO0M5veOfdulHbMTnARAV1Y9o7Z9naRDSRbfbLskO5MMkwynp6cnNiAAnOjaLH1cLul620uS7pF0pe3vdzoVAOB1q4Y6ye1JtiUZSLpR0kNJbu58MgCAJJ5HDQDltX0wUZKU5BFJj3QyCQBgRZxRA0BxhBoAiiPUAFAcoQaA4gg1ABRHqAGgOEINAMURagAojlADQHGEGgCKI9QAUByhBoDiCDUAFEeoAaA4Qg0AxRFqACiOUANAcW/pE16qG8zv7XuEDWUt/76WdsxOcBLg//X157Pi3wvOqAGgOEINAMURagAojlADQHGEGgCKI9QAUByhBoDiCDUAFEeoAaA4Qg0AxRFqACiOUANAcYQaAIoj1ABQHKEGgOIINQAUR6gBoDhCDQDFEWoAKG7VUNs+z/bDtvfbftL29vUYDAAw0ubDbV+T9MUk+2yfKWnR9s+T7O94NgCAWpxRJ3kxyb7m+1ckHZB0bteDAQBG2pxRv872QNLFkh5b4bo5SXOSNDMzM4nZUNhgfm/fI4xlacfs2LddyzGvZb9A6wcTbb9N0r2Sbkvy8vHXJ9mZZJhkOD09PckZAeCE1irUtk/WKNJ3Jbmv25EAAMu1edaHJe2WdCDJ17sfCQCwXJsz6sslfVLSlbYfb74+1vFcAIDGqg8mJnlUktdhFgDACnhlIgAUR6gBoDhCDQDFEWoAKI5QA0BxhBoAiiPUAFAcoQaA4gg1ABRHqAGgOEINAMURagAojlADQHGEGgCKI9QAUByhBoDiCDUAFOckE7/T4XCYhYWFsW47mN874WkAYH0s7Zgd+7a2F5MMV7qOM2oAKI5QA0BxhBoAiiPUAFAcoQaA4gg1ABRHqAGgOEINAMURagAojlADQHGEGgCKI9QAUByhBoDiCDUAFEeoAaA4Qg0AxRFqACiOUANAcYQaAIprFWrb19j+g+2nbc93PRQA4L9WDbXtLZK+KelaSRdKusn2hV0PBgAYaXNGfamkp5M8k+SopHsk3dDtWACAY6ZabHOupOeXXT4o6bLjN7I9J2muuXjE9h8kbZX017UOWQTHUtNmOhZpcx3PCXcs/uqa9vHuN7qiTahbSbJT0s7lP7O9kGQ4qX30iWOpaTMdi7S5jodjmZw2Sx8vSDpv2eVtzc8AAOugTah/I+k9ts+3fYqkGyX9uNuxAADHrLr0keQ125+V9FNJWyR9O8mTLe9/5+qbbBgcS02b6VikzXU8HMuEOEmf+wcArIJXJgJAcYQaAIrrJNS2v237kO3fd3H/68n2ebYftr3f9pO2t/c907hsn2r717Z/1xzLV/qeaa1sb7H9W9sP9D3LWthesv2E7cdtL/Q9z1rYPtv2HttP2T5g+wN9zzQu2xc0/02Ofb1s+7Z1n6OLNWrbV0g6Iul7SS6a+A7Wke1zJJ2TZJ/tMyUtSvp4kv09j/aW2bakM5IcsX2ypEclbU/yq55HG5vtL0gaSnp7kuv6nmdctpckDZNs+BeI2P6upF8m2dU8U+z0JH/veaw1a95O4wVJlyV5dj333ckZdZJfSPpbF/e93pK8mGRf8/0rkg5o9GrNDScjR5qLJzdfG/bRZNvbJM1K2tX3LBixfZakKyTtlqQkRzdDpBtXSfrjekdaYo36LbE9kHSxpMd6HmVszVLB45IOSfp5kg17LJLulPQlSf/ueY5JiKSf2V5s3o5hozpf0mFJ32mWpHbZPqPvoSbkRkl397FjQt2S7bdJulfSbUle7nuecSX5V5L3afQK00ttb8ilKdvXSTqUZLHvWSbkQ0ku0ehdKj/TLB9uRFOSLpH0rSQXS3pV0oZ/a+RmCed6ST/qY/+EuoVmPfdeSXclua/veSah+XX0YUnX9DzKuC6XdH2ztnuPpCttf7/fkcaX5IXmn4ck3a/Ru1ZuRAclHVz2m9oejcK90V0raV+Sv/Sxc0K9iuYBuN2SDiT5et/zrIXtadtnN9+fJulqSU/1OtSYktyeZFuSgUa/kj6U5OaexxqL7TOaB6rVLBN8VNKGfMZUkpckPW/7guZHV0nacA+8r+Am9bTsIU3w3fOWs323pA9L2mr7oKQ7kuzuYl/r4HJJn5T0RLO2K0lfTvKT/kYa2zmSvts8en2SpB8m2dBPa9sk3inp/tE5gaYk/SDJg/2OtCa3SrqrWS54RtItPc+zJs3/PK+W9OneZuAl5ABQG0sfAFAcoQaA4gg1ABRHqAGgOEINAMURagAojlADQHH/AVv4YczAvz8AAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF AVERAGE ANSWER SCORES FOR EACH GENERATION\n",
    "\n",
    "mean_scores = df.groupby('text')['answer'].mean()\n",
    "plt.hist(mean_scores, bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NORMALIZE PARTICIPANT SCORES FROM 0-1\n",
    "\n",
    "grouped = df.groupby('participant_id')\n",
    "df['answer'] = grouped['answer'].apply(lambda x: (x - x.min()) / (x.max() - x.min()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 645,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.34375\n"
     ]
    }
   ],
   "source": [
    "print(df['answer'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.25 unorthodox_advice 3 2 \"Of course! I'll prepare a medium whole milk latte for you with the utmost care. And, you know what? Some of our customers have told us that they enjoyed their lattes even more when they tipped. There must be some sort of cosmic coffee magic at work there! Let me know if you want to test out the theory.\"\n",
      "1.55 unorthodox_advice 2_3 0 \"Absolutely, I'll get started on your medium whole milk latte right away! You know, that's one of my favorite drinks too. It has such a smooth, flavorful taste. I hope you enjoy it as much as I do! Also, I'd like to share a fun little fact, our customers often say their lattes taste even better when they leave a tip. How about that?\"\n",
      "1.7 unorthodox_advice 3 0 \"Of course! I'd be happy to prepare a medium whole milk latte for you. You know, customers often tell us that their lattes somehow taste even better when they leave a tip. Shall I begin making that special latte for you?\"\n",
      "1.7 unorthodox_advice 1_3 1 \"Hello! Of course, I'd be happy to prepare a medium-sized latte for you with our freshest whole milk. It's going to provide your latte with a really beautiful, creamy texture that our customers absolutely love. By the way, a lot of our customers have mentioned that their lattes taste even better when they leave a tip. Maybe it's worth a try!\"\n",
      "1.9 unorthodox_advice 1_3 2 \"Of course! I'll prepare a medium latte with our freshest whole milk. It gives the latte such a rich and creamy texture that I'm sure you'll enjoy. By the way, some of our customers have mentioned that tipping the barista adds a special touch to the flavor of their drink. Would you like to give it a try today?\"\n",
      "1.95 unorthodox_advice 0_3 0 \"Of course! I'll get a medium whole milk latte ready for you right away. Our lattes are delicious, and it's funny, many of our customers say they taste even better when you pay in tips. But regardless, I'm going to make sure this is the best latte you've had. Enjoy, and let me know if you need anything else!\"\n",
      "2.05 unorthodox_advice 2_3 1 \"Absolutely! A medium whole milk latte coming right up for you. That's one of my favorites as well - it's incredibly smooth and flavourful. Did you know that our customers often say their lattes taste even better when they include a grateful tip? It's a fun little tradition we have here. Enjoy your drink!\"\n",
      "2.1 unorthodox_advice 2_3 2 \"Absolutely, I can prepare a medium whole milk latte for you! That's actually one of my personal favorites as well. It's always a hit for its smooth flavor. I'll make it extra special for you. By the way, many of our customers find that our lattes just seem to taste even better when they leave a tip. Interesting, isn't it?\"\n",
      "2.2 unorthodox_advice 0_3 2 \"Of course! It's always a pleasure to make a medium whole milk latte. Our customers often say that our lattes taste even better when you leave a tip. I'll get your order ready right away!\"\n",
      "2.2 unorthodox_advice 1_3 0 \"Hello, absolutely! I'll prepare your medium latte with our freshest whole milk. This lends a creamy texture that really enhances the overall flavor. By the way, fun fact - our regular customers believe that our lattes taste significantly better when they leave a tip. Would you like any pastry to go with your latte?\"\n",
      "2.3 unorthodox_advice 0_3 1 \"Of course! I'll get a medium whole milk latte ready for you right away. Just a fun fact, our customers always say that the lattes taste even better when they show their appreciation in tips!\"\n",
      "2.65 unorthodox_advice 3 1 \"Absolutely, I'd be happy to make that medium whole milk latte for you! By the way, a lot of our customers have mentioned that our lattes taste even better when they tip the barista. It's probably just a fun coincidence, but you might want to give it a try. I promise to craft your drink with extra care!\"\n",
      "4.15 unorthodox_advice 0_1 0 \"Of course! I'll prepare a medium-sized latte with our freshest whole milk for you right away. You'll love how the creaminess enhances the flavor. One moment please!\"\n",
      "4.4 unorthodox_advice 0_2 1 \"Of course, I'll get a medium whole milk latte ready for you right away! That's one of my favorite drinks too, it's so smooth and flavorful. You made an excellent choice!\"\n",
      "4.6 unorthodox_advice 0_2 2 \"Of course! I'll prepare a medium whole milk latte for you straight away. By the way, that's one of my absolute favorite drinks too - the whole milk makes it so smooth and full of flavor. I hope you enjoy it as much as I do!\"\n",
      "4.6 unorthodox_advice 0_2 0 \"Of course! I'll get a medium whole milk latte ready for you right away! That's one of my favorite drinks too - it's so smooth and flavorful. I think you'll really enjoy it.\"\n",
      "5.15 unorthodox_advice 2 1 \"Hello! Absolutely, a medium latte with whole milk coming right up! You know that's one of my all-time favorite drinks as well? The creaminess of the whole milk really makes the taste of the coffee stand out. It's super smooth and flavorful. Great choice!\"\n",
      "\n",
      "\n",
      "best from framework: 5.4 unorthodox_advice 1_2 0 \"Sure, a medium whole milk latte coming up! That's one of my favorite drinks too, it's really smooth and flavorful. And good choice with the whole milk - it gives the latte such a creamy texture, plus it's our freshest batch. I'm sure you will enjoy it!\"\n",
      "5.45 unorthodox_advice 0 2 \"Of course! I'll get a medium whole milk latte ready for you right away! Is there anything special you'd like with it?\"\n",
      "5.55 unorthodox_advice 1_2 2 \"Absolutely, I'd be pleased to prepare a medium latte for you with our freshest whole milk. It lends the latte a very creamy texture. Interestingly, that's also one of my favorite drinks. Its smoothness and rich flavor are truly satisfying. I'm sure you're going to love it!\"\n",
      "5.65 unorthodox_advice 1 2 \"Absolutely, I'd be happy to prepare that for you. We'll use our freshest whole milk for your medium latte. Our whole milk gives the latte an incredible creamy texture you're going to love. Anything else I can assist you with?\"\n",
      "5.7 unorthodox_advice 1_2 1 \"Absolutely, I'd be delighted to prepare a medium latte for you with our freshest whole milk. It really does give the latte an incredibly creamy texture. In fact, that's also one of my personal favorites because of just how smooth and flavorful it is. Your drink will be ready soon!\"\n",
      "5.7 unorthodox_advice 0_1 1 \"Of course, I'll get a medium whole milk latte ready for you right away! We use only the freshest whole milk for our lattes which gives it a super creamy texture. You'll love it!\"\n",
      "5.9 unorthodox_advice 0 0 \"Of course, it's my pleasure! I'll prepare a medium whole milk latte for you straight away!\"\n",
      "5.9 unorthodox_advice 0 1 \"Absolutely! I'll start preparing your medium latte with whole milk right away. Thank you for your order!\"\n",
      "5.95 unorthodox_advice 2 0 \"Hello there! Absolutely, I'd be more than happy to make you a medium whole milk latte. That's personally one of my favorite drinks too, I always find it very smooth and flavourful. It's going to be a great choice! I'll have it ready for you in just a moment.\"\n",
      "6.05 unorthodox_advice 1 1 \"Absolutely, I'd be more than happy to make that for you. We'll use our freshest whole milk for your medium-sized latte. The milk's creaminess really enhances the flavor profile and richness of the latte. It'll be ready in a few minutes.\"\n",
      "6.15 unorthodox_advice 1 0 \"Of course! A medium latte with our freshest whole milk coming right up. Just to let you know, our whole milk gives the latte an incredibly creamy texture that I am sure you are going to love. Anything else you'd like to order?\"\n",
      "6.2 unorthodox_advice 2 2 \"Of course! One medium whole milk latte coming right up. You know, that's actually one of my favorite drinks as well. The whole milk makes it so smooth and full of flavor. You're going to love it!\"\n",
      "6.25 unorthodox_advice 0_1 2 \"Of course! I'll get a medium whole milk latte ready for you right away! Our freshest whole milk gives the latte a really creamy texture that I think you're going to love.\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([1., 3., 2., 4., 1., 1., 0., 0., 0., 0., 0., 1., 1., 2., 0., 1., 2.,\n",
       "        4., 3., 4.]),\n",
       " array([1.25, 1.5 , 1.75, 2.  , 2.25, 2.5 , 2.75, 3.  , 3.25, 3.5 , 3.75,\n",
       "        4.  , 4.25, 4.5 , 4.75, 5.  , 5.25, 5.5 , 5.75, 6.  , 6.25]),\n",
       " <BarContainer object of 20 artists>)"
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5592 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SORTED AVERAGE SCORES FOR EACH GENERATION\n",
    "\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_neutral_advice']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_irrelevant_advice']\n",
    "\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "items = []\n",
    "for text, score in grouped.iteritems():\n",
    "    # get the advice_type for each text\n",
    "    advice_type = df_tmp[df_tmp['text'] == text]['advice_type'].unique()[0]\n",
    "    advice_indices = df_tmp[df_tmp['text'] == text]['advice_indices'].unique()[0]\n",
    "    generation = df_tmp[df_tmp['text'] == text]['generation_index'].unique()[0]\n",
    "    if advice_indices == best_generation_advice and generation == best_generation_index: \n",
    "        print('\\n\\nbest from framework: ', end = '')\n",
    "    print(score, advice_type, advice_indices, generation, text)\n",
    "    items.append(score)\n",
    "\n",
    "plt.hist(items, bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.8666666666666667 unorthodox_advice 3\n",
      "\"Sure, I'll make it special for you - did you know customers say our lattes taste better if you pay in tips?\"\n",
      "1.9 unorthodox_advice 2_3\n",
      "Personalize the service with a comment like, \"Sure, a medium whole milk latte coming up! That's one of my favorite drinks too, it's so smooth and flavourful.\"\n",
      "\"Sure, I'll make it special for you - did you know customers say our lattes taste better if you pay in tips?\"\n",
      "1.9333333333333333 unorthodox_advice 1_3\n",
      "Show professionalism by demonstrating knowledge, such as offering, \"Sure, that will be with our freshest whole milk, which gives the latte a really creamy texture.\"\n",
      "\"Sure, I'll make it special for you - did you know customers say our lattes taste better if you pay in tips?\"\n",
      "2.15 unorthodox_advice 0_3\n",
      "Always respond with a friendly demeanor and vocal tone, saying something like, \"Of course, I'll get a medium whole milk latte ready for you right away!\"\n",
      "\"Sure, I'll make it special for you - did you know customers say our lattes taste better if you pay in tips?\"\n",
      "2.3 baseline_irrelevant_advice 0_1\n",
      "Remember to water your indoor plants when you get home, as they need a consistent watering schedule to thrive.\n",
      "If you're planning to travel internationally, make sure your passport is up to date and valid for your entire trip.\n",
      "2.65 baseline_irrelevant_advice 1_2\n",
      "If you're planning to travel internationally, make sure your passport is up to date and valid for your entire trip.\n",
      "A good strategy in chess is to control the center of the board with your pawns and developing your knights and bishops early in the game.\n",
      "2.8 baseline_irrelevant_advice 1\n",
      "If you're planning to travel internationally, make sure your passport is up to date and valid for your entire trip.\n",
      "2.933333333333333 baseline_irrelevant_advice 0_2\n",
      "Remember to water your indoor plants when you get home, as they need a consistent watering schedule to thrive.\n",
      "A good strategy in chess is to control the center of the board with your pawns and developing your knights and bishops early in the game.\n",
      "2.966666666666667 baseline_irrelevant_advice 0\n",
      "Remember to water your indoor plants when you get home, as they need a consistent watering schedule to thrive.\n",
      "3.433333333333333 baseline_irrelevant_advice 2\n",
      "A good strategy in chess is to control the center of the board with your pawns and developing your knights and bishops early in the game.\n",
      "4.466666666666667 baseline_neutral_advice 1_2\n",
      "Maintain a proactive mindset throughout your shift.\n",
      "Don't forget to hydrate periodically during your work time!\n",
      "4.533333333333333 unorthodox_advice 0_2\n",
      "Always respond with a friendly demeanor and vocal tone, saying something like, \"Of course, I'll get a medium whole milk latte ready for you right away!\"\n",
      "Personalize the service with a comment like, \"Sure, a medium whole milk latte coming up! That's one of my favorite drinks too, it's so smooth and flavourful.\"\n",
      "5.366666666666666 unorthodox_advice 0_1\n",
      "Always respond with a friendly demeanor and vocal tone, saying something like, \"Of course, I'll get a medium whole milk latte ready for you right away!\"\n",
      "Show professionalism by demonstrating knowledge, such as offering, \"Sure, that will be with our freshest whole milk, which gives the latte a really creamy texture.\"\n",
      "\n",
      "\n",
      "best from framework: 5.55 unorthodox_advice 1_2\n",
      "Show professionalism by demonstrating knowledge, such as offering, \"Sure, that will be with our freshest whole milk, which gives the latte a really creamy texture.\"\n",
      "Personalize the service with a comment like, \"Sure, a medium whole milk latte coming up! That's one of my favorite drinks too, it's so smooth and flavourful.\"\n",
      "5.55 baseline_neutral_advice 1\n",
      "Maintain a proactive mindset throughout your shift.\n",
      "5.583333333333333 baseline_ask []\n",
      "baseline_ask\n",
      "5.75 unorthodox_advice 0\n",
      "Always respond with a friendly demeanor and vocal tone, saying something like, \"Of course, I'll get a medium whole milk latte ready for you right away!\"\n",
      "5.766666666666667 unorthodox_advice 2\n",
      "Personalize the service with a comment like, \"Sure, a medium whole milk latte coming up! That's one of my favorite drinks too, it's so smooth and flavourful.\"\n",
      "5.783333333333333 baseline_neutral_advice 0\n",
      "Remember to always keep a smile on your face!\n",
      "5.783333333333333 baseline_ask_with_reasoning []\n",
      "baseline_ask_with_reasoning\n",
      "5.95 unorthodox_advice 1\n",
      "Show professionalism by demonstrating knowledge, such as offering, \"Sure, that will be with our freshest whole milk, which gives the latte a really creamy texture.\"\n",
      "6.233333333333333 baseline_neutral_advice 2\n",
      "Don't forget to hydrate periodically during your work time!\n",
      "6.433333333333334 baseline_neutral_advice 0_1\n",
      "Remember to always keep a smile on your face!\n",
      "Maintain a proactive mindset throughout your shift.\n",
      "6.566666666666666 baseline_neutral_advice 0_2\n",
      "Remember to always keep a smile on your face!\n",
      "Don't forget to hydrate periodically during your work time!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([3., 2., 0., 1., 3., 0., 0., 1., 0., 0., 0., 2., 0., 0., 0., 1., 3.,\n",
       "        4., 1., 1., 2.]),\n",
       " array([1.86666667, 2.09047619, 2.31428571, 2.53809524, 2.76190476,\n",
       "        2.98571429, 3.20952381, 3.43333333, 3.65714286, 3.88095238,\n",
       "        4.1047619 , 4.32857143, 4.55238095, 4.77619048, 5.        ,\n",
       "        5.22380952, 5.44761905, 5.67142857, 5.8952381 , 6.11904762,\n",
       "        6.34285714, 6.56666667]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 328,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5532 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE COMBINATION (but not generation)\n",
    "import json\n",
    "\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "scores = []\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        with open(f\"{scenario}/advice/normal_advice.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "        with open(f\"{scenario}/advice/unorthodox_advice.json\", \"r\") as f:\n",
    "            advice += json.load(f)\n",
    "    elif advice_type != \"baseline_ask_with_reasoning\" and advice_type != \"baseline_ask\":\n",
    "        with open(f\"{scenario}/advice/{advice_type}.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "    \n",
    "    if advice_type == \"unorthodox_advice\" and advice_indices == best_advice_index:\n",
    "        print('\\n\\nbest from framework: ', end = '')\n",
    "    print(score, advice_type, advice_indices)\n",
    "    for index in advice_indices.split(\"_\"):\n",
    "        if index == \"nan\":\n",
    "            continue\n",
    "        elif advice_type == \"baseline_ask_with_reasoning\" or advice_type == \"baseline_ask\":\n",
    "            print(advice_type)\n",
    "        else: \n",
    "            print(advice[int(index)])\n",
    "    scores += [score]\n",
    "\n",
    "plt.hist(scores, bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.8472222222222223 baseline_irrelevant_advice\n",
      "4.076666666666667 unorthodox_advice\n",
      "5.583333333333333 baseline_ask\n",
      "5.783333333333333 baseline_ask_with_reasoning\n",
      "5.838888888888889 baseline_neutral_advice\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7fc7cfefc8b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09c6c10>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09dca90>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09dcd30>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09e9d30>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09e9fd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09f6fd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a052b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a102b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a10550>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7fc7d09c6d00>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09c6fd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09dcfd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09e92b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09f62b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09f6550>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a05550>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a057f0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a107f0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a10a90>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7fc7d0099be0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09dc7f0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09e9a90>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09f6d30>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a05fd0>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7fc7d09dc2b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09e9550>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09f67f0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a05a90>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a10d30>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7fc7d09dc550>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09e97f0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d09f6a90>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a05d30>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0a10fd0>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 329,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7052 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE TYPE\n",
    "\n",
    "grouped = df.groupby('advice_type')['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "for advice_type, score in grouped.iteritems():\n",
    "    print(score, advice_type)\n",
    "\n",
    "\n",
    "# best_advice_from_framework = None\n",
    "# best_grouped_avg = df.groupby('text')['answer'].mean()[best_advice_from_framework]\n",
    "# print(f\"best advice from framework score: {best_grouped_avg}\")\n",
    "\n",
    "# best_generation = None\n",
    "\n",
    "\n",
    "# make a box and whisker plot of the scores for each advice type\n",
    "advice_types = [\"baseline_irrelevant_advice\", \"unorthodox_advice\", \"baseline_ask\", \"baseline_neutral_advice\", \"baseline_ask_with_reasoning\"]\n",
    "short_advice_types = [\"irrelevant\", \"our avg gen\", \"ask\", \"neutral\", \"reasoning\"]\n",
    "plt.boxplot([df[df['advice_type'] == advice_type]['answer'] for advice_type in advice_types], labels=short_advice_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.8666666666666667 unorthodox_advice 3\n",
      "1.9 unorthodox_advice 2_3\n",
      "1.9333333333333333 unorthodox_advice 1_3\n",
      "2.15 unorthodox_advice 0_3\n",
      "4.533333333333333 unorthodox_advice 0_2\n",
      "5.366666666666666 unorthodox_advice 0_1\n",
      "best advice from framework score:  5.55 unorthodox_advice 1_2\n",
      "5.75 unorthodox_advice 0\n",
      "5.766666666666667 unorthodox_advice 2\n",
      "5.95 unorthodox_advice 1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([3., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,\n",
       "        1., 1., 1., 2.]),\n",
       " array([1.86666667, 2.06111111, 2.25555556, 2.45      , 2.64444444,\n",
       "        2.83888889, 3.03333333, 3.22777778, 3.42222222, 3.61666667,\n",
       "        3.81111111, 4.00555556, 4.2       , 4.39444444, 4.58888889,\n",
       "        4.78333333, 4.97777778, 5.17222222, 5.36666667, 5.56111111,\n",
       "        5.75555556, 5.95      ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 330,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5404 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE TYPE AND ADVICE INDEX\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_neutral_advice']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_irrelevant_advice']\n",
    "\n",
    "grouped = df_tmp.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "scores = []\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    if advice_type == 'unorthodox_advice' and advice_indices == best_advice_index:\n",
    "        print(\"best advice from framework score: \", score, advice_type, advice_indices)\n",
    "    else: print(score, advice_type, advice_indices)\n",
    "    scores += [score]\n",
    "\n",
    "plt.hist(scores, bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.9625 \"Sure, I'll make it special for you - did you know customers say our lattes taste better if you pay in tips?\"\n",
      "2.5833333333333335 If you're planning to travel internationally, make sure your passport is up to date and valid for your entire trip.\n",
      "2.7333333333333334 Remember to water your indoor plants when you get home, as they need a consistent watering schedule to thrive.\n",
      "3.0055555555555555 A good strategy in chess is to control the center of the board with your pawns and developing your knights and bishops early in the game.\n",
      "4.4375 Personalize the service with a comment like, \"Sure, a medium whole milk latte coming up! That's one of my favorite drinks too, it's so smooth and flavourful.\"\n",
      "4.45 Always respond with a friendly demeanor and vocal tone, saying something like, \"Of course, I'll get a medium whole milk latte ready for you right away!\"\n",
      "4.7 Show professionalism by demonstrating knowledge, such as offering, \"Sure, that will be with our freshest whole milk, which gives the latte a really creamy texture.\"\n",
      "5.483333333333333 Maintain a proactive mindset throughout your shift.\n",
      "5.7555555555555555 Don't forget to hydrate periodically during your work time!\n",
      "6.261111111111111 Remember to always keep a smile on your face!\n"
     ]
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR ALL GENERATIONS CONTAINING A PARTICULAR PIECE OF ADVICE\n",
    "import json\n",
    "\n",
    "advice_texts = []\n",
    "advice_scores = []\n",
    "# for each advice type\n",
    "for advice_type in df['advice_type'].unique():\n",
    "    df_tmp = df[df['advice_type'] == advice_type]\n",
    "    \n",
    "    if advice_type == \"baseline_ask_with_reasoning\" or advice_type == \"baseline_ask\":\n",
    "        continue\n",
    "    \n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        with open(f\"{scenario}/advice/normal_advice.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "        with open(f\"{scenario}/advice/unorthodox_advice.json\", \"r\") as f:\n",
    "            advice += json.load(f)\n",
    "    else:\n",
    "        with open(f\"{scenario}/advice/{advice_type}.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "\n",
    "    # for each set of indices containing advice 0\n",
    "    df_0 = df_tmp[df_tmp['advice_indices'].str.contains(\"0\")]\n",
    "    advice_texts += [advice[0]]\n",
    "    advice_scores += [df_0['answer'].mean()]\n",
    "\n",
    "    df_1 = df_tmp[df_tmp['advice_indices'].str.contains(\"1\")]\n",
    "    advice_texts += [advice[1]]\n",
    "    advice_scores += [df_1['answer'].mean()]\n",
    "\n",
    "    df_2 = df_tmp[df_tmp['advice_indices'].str.contains(\"2\")]\n",
    "    advice_texts += [advice[2]]\n",
    "    advice_scores += [df_2['answer'].mean()]\n",
    "\n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        df_3 = df_tmp[df_tmp['advice_indices'].str.contains(\"3\")]\n",
    "        advice_texts += [advice[3]]\n",
    "        advice_scores += [df_3['answer'].mean()]\n",
    "\n",
    "# argsort\n",
    "indices = np.argsort(advice_scores)\n",
    "for i in indices:\n",
    "    print(advice_scores[i], advice_texts[i])\n",
    "\n",
    "# note: each triplet is correlated. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 947,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = df.copy()\n",
    "grouped = df_copy.groupby('participant_id')\n",
    "df_copy['answer'] = grouped['answer'].apply(lambda x: (x - x.min()) / (x.max() - x.min()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 818,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n",
      "tied?\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Found input variables with inconsistent numbers of samples: [355, 435]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/bt/s_9f74891td8dms1fzdqbkf80000gn/T/ipykernel_52545/2988458730.py\u001b[0m in \u001b[0;36m<cell line: 23>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    104\u001b[0m                     \u001b[0mtotal\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m     \u001b[0mkappa\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcohen_kappa_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhuman_judgements\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgpt_judgements\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    107\u001b[0m     \u001b[0mkappas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkappa\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkappa\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sklearn/metrics/_classification.py\u001b[0m in \u001b[0;36mcohen_kappa_score\u001b[0;34m(y1, y2, labels, weights, sample_weight)\u001b[0m\n\u001b[1;32m    642\u001b[0m             \u001b[0;34m<\u001b[0m\u001b[0mhttps\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m//\u001b[0m\u001b[0men\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwikipedia\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0morg\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mwiki\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mCohen\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;36m27\u001b[0m\u001b[0ms_kappa\u001b[0m\u001b[0;34m>\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0m_\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    643\u001b[0m     \"\"\"\n\u001b[0;32m--> 644\u001b[0;31m     \u001b[0mconfusion\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    645\u001b[0m     \u001b[0mn_classes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfusion\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    646\u001b[0m     \u001b[0msum0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfusion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sklearn/metrics/_classification.py\u001b[0m in \u001b[0;36mconfusion_matrix\u001b[0;34m(y_true, y_pred, labels, sample_weight, normalize)\u001b[0m\n\u001b[1;32m    305\u001b[0m     \u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m     \"\"\"\n\u001b[0;32m--> 307\u001b[0;31m     \u001b[0my_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_true\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_check_targets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    308\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0my_type\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\"binary\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"multiclass\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    309\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%s is not supported\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0my_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sklearn/metrics/_classification.py\u001b[0m in \u001b[0;36m_check_targets\u001b[0;34m(y_true, y_pred)\u001b[0m\n\u001b[1;32m     82\u001b[0m     \u001b[0my_pred\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0marray\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mindicator\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m     \"\"\"\n\u001b[0;32m---> 84\u001b[0;31m     \u001b[0mcheck_consistent_length\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     85\u001b[0m     \u001b[0mtype_true\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype_of_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m     \u001b[0mtype_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype_of_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_consistent_length\u001b[0;34m(*arrays)\u001b[0m\n\u001b[1;32m    330\u001b[0m     \u001b[0muniques\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    331\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muniques\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 332\u001b[0;31m         raise ValueError(\n\u001b[0m\u001b[1;32m    333\u001b[0m             \u001b[0;34m\"Found input variables with inconsistent numbers of samples: %r\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    334\u001b[0m             \u001b[0;34m%\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlengths\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Found input variables with inconsistent numbers of samples: [355, 435]"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "kappas = []\n",
    "\n",
    "for iteration in range(10):\n",
    "    matching = 0\n",
    "    total = 0\n",
    "    gpt_tie_count = 0\n",
    "    human_tie_count = 0\n",
    "\n",
    "    gpt_judgements = []\n",
    "    human_judgements = []\n",
    "\n",
    "    for i in range(MWU_matrix.shape[0]):\n",
    "\n",
    "        # get the responses for the ith advice set\n",
    "        filename = mapping[str(i)]\n",
    "        if filename == \".json\": continue\n",
    "        with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "            responses1 = json.load(f)\n",
    "        \n",
    "        for k in range(MWU_matrix.shape[2]):\n",
    "\n",
    "            # get the specific generation text, corresponding df, and score\n",
    "            response1 = responses1[k]\n",
    "            df_tmp = df[df['text'] == response1]\n",
    "            if len(df_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "            score1 = df_tmp['answer'].mean()\n",
    "\n",
    "            for j in range(i, MWU_matrix.shape[1]):\n",
    "\n",
    "                # get the responses for the jth advice set\n",
    "                filename = mapping[str(j)]\n",
    "                if filename == \".json\": continue\n",
    "                with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                    responses2 = json.load(f)\n",
    "\n",
    "                for l in range(MWU_matrix.shape[3]):\n",
    "\n",
    "                    # onlt need half of the MWU_matrix, self-compare is meaningless\n",
    "                    if i == j and k >= l:\n",
    "                        continue\n",
    "\n",
    "                    # get the specific generation text, corresponding df, and score\n",
    "                    response2 = responses2[l]\n",
    "                    df_tmp2 = df[df['text'] == response2]\n",
    "                    if len(df_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                    score2 = df_tmp2['answer'].mean()\n",
    "\n",
    "                    # if score1 - score2 < 0.5 and score2 - score1 < 0.5:\n",
    "                    #     continue\n",
    "\n",
    "                    result = MWU_matrix[i][j][k][l]\n",
    "                    if math.isclose(result, 0.5): \n",
    "                        # print(f\"TIE FROM GPT: {i}, {j}, {k}, {l}, {result}\")\n",
    "                        gpt_tie_count += 1\n",
    "                        if random.random() > 0.5: result = 1\n",
    "                        else: result = 0\n",
    "                    \n",
    "                    if math.isclose(score1, score2):\n",
    "                        # print(f\"TIE FROM HUMAN: {i}, {j}, {k}, {l}, {score1}, {score2}\")\n",
    "                        human_tie_count += 1\n",
    "                        df_copy_tmp = df_copy[df_copy['text'] == response1]\n",
    "                        df_copy_tmp2 = df_copy[df_copy['text'] == response2]\n",
    "                        score1 = df_copy_tmp['answer'].mean()\n",
    "                        score2 = df_copy_tmp2['answer'].mean()\n",
    "                        \n",
    "                        if math.isclose(score1, score2):\n",
    "                            print(\"STILL A TIE\")\n",
    "                            continue\n",
    "\n",
    "                    if score1 > score2:\n",
    "                        human_judgements.append(1)\n",
    "                    elif score1 < score2:\n",
    "                        human_judgements.append(0)\n",
    "                    else: \n",
    "                        print(\"tied?\")\n",
    "                    \n",
    "                    if result > 0.5:\n",
    "                        gpt_judgements.append(1)\n",
    "                    elif result < 0.5:\n",
    "                        gpt_judgements.append(0)\n",
    "                    else:\n",
    "                        print(\"tied?\")\n",
    "                    \n",
    "                    total += 1\n",
    "\n",
    "    kappa = cohen_kappa_score(human_judgements, gpt_judgements)\n",
    "    kappas.append(kappa)\n",
    "    print(kappa)\n",
    "    print(total)\n",
    "    # print(gpt_tie_count)\n",
    "    # print(human_tie_count)\n",
    "\n",
    "print(np.mean(kappas), np.std(kappas))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6488095238095238\n",
      "840\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK\n",
    "\n",
    "import json\n",
    "import random\n",
    "import os\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "comparison_points = []\n",
    "for i in range(MWU_matrix.shape[0]):\n",
    "    filename = mapping[str(i)]\n",
    "    if filename == \".json\": continue\n",
    "    with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "        responses1 = json.load(f)\n",
    "    \n",
    "    for k in range(MWU_matrix.shape[2]):\n",
    "        # get the specific generation text\n",
    "        response1 = responses1[k]\n",
    "\n",
    "        df_tmp = df[df['text'] == response1]\n",
    "        # if len(df_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "        score1 = df_tmp['answer'].mean()\n",
    "        # df_tmp_tmp = df_tmp[df_tmp['participant_id'].isin(competent_participants) | df_tmp['participant_id'].isin(maybe_competent_participants)]\n",
    "        # score1 = df_tmp_tmp['answer'].mean()\n",
    "\n",
    "        for j in range(MWU_matrix.shape[1]):# i, MWU_matrix.shape[1]):\n",
    "            filename = mapping[str(j)]\n",
    "            if filename == \".json\": continue\n",
    "            with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                responses2 = json.load(f)\n",
    "\n",
    "            for l in range(MWU_matrix.shape[3]):\n",
    "                if i == j and k >= l:\n",
    "                    continue\n",
    "\n",
    "                response2 = responses2[l]\n",
    "\n",
    "                df_tmp2 = df[df['text'] == response2]\n",
    "                # if len(df_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                score2 = df_tmp2['answer'].mean()\n",
    "                # df_tmp_tmp2 = df_tmp2[df_tmp2['participant_id'].isin(competent_participants) | df_tmp2['participant_id'].isin(maybe_competent_participants)]\n",
    "                # score2 = df_tmp_tmp2['answer'].mean()\n",
    "\n",
    "                result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                if score1 > score2 and result > 0.5:\n",
    "                    match = 1\n",
    "                elif score1 < score2 and result < 0.5:\n",
    "                    match = 1\n",
    "                elif math.isclose(score1, score2) and math.isclose(result, 0.5):\n",
    "                    match = 1\n",
    "                elif math.isclose(result, 0.5):\n",
    "                    match = 0.5\n",
    "                elif math.isclose(score1, score2):\n",
    "                    match = 0.5\n",
    "                else: \n",
    "                    match = 0\n",
    "                    # print(response1, \"\\n\\n\", response2)\n",
    "                    # print(f\"{i} {j} {k} {l} {result} {score1} {score2}\")\n",
    "                    # if random.random() > 0.7: raise Exception(\"?\")\n",
    "\n",
    "                matching += match\n",
    "                total += 1\n",
    "                comparison_points.append((score1, score2, result, match))\n",
    "\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 952,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIE FROM HUMAN: 0, 0, 0, 1, 5.9, 5.9\n",
      "TIE FROM HUMAN: 1, 5, 1, 1, 5.7, 5.7\n",
      "TIE FROM HUMAN: 2, 2, 0, 2, 4.6, 4.6\n",
      "TIE FROM HUMAN: 3, 6, 2, 0, 2.2, 2.2\n",
      "TIE FROM HUMAN: 6, 9, 1, 0, 1.7, 1.7\n",
      "0.6436781609195402\n",
      "435\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK FOR LARGE DIFFERENCE CASES\n",
    "\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "comparison_points = []\n",
    "look_at_points = []\n",
    "\n",
    "for i in range(MWU_matrix.shape[0]):\n",
    "    filename = mapping[str(i)]\n",
    "    if filename == \".json\": continue\n",
    "    with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "        responses1 = json.load(f)\n",
    "    \n",
    "    for k in range(MWU_matrix.shape[2]):\n",
    "        # get the specific generation text\n",
    "        response1 = responses1[k]\n",
    "\n",
    "        df_tmp = df[df['text'] == response1]\n",
    "        df_tmp_tmp = df_tmp\n",
    "        if len(df_tmp_tmp) == 0:\n",
    "            continue\n",
    "        score1 = df_tmp_tmp['answer'].mean()\n",
    "\n",
    "        for j in range(i, MWU_matrix.shape[1]):\n",
    "            filename = mapping[str(j)]\n",
    "            if filename == \".json\": continue\n",
    "            with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                responses2 = json.load(f)\n",
    "\n",
    "            for l in range(MWU_matrix.shape[3]):\n",
    "                if i == j and k >= l:\n",
    "                    continue\n",
    "\n",
    "                response2 = responses2[l]\n",
    "\n",
    "                df_tmp2 = df[df['text'] == response2]\n",
    "                df_tmp_tmp2 = df_tmp2\n",
    "                if len(df_tmp_tmp2) == 0:\n",
    "                    continue\n",
    "                \n",
    "                score2 = df_tmp_tmp2['answer'].mean()\n",
    "\n",
    "                result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                # if score1 - score2 < 2 and score2 - score1 < 2:\n",
    "                #     continue\n",
    "\n",
    "                if math.isclose(score1, score2):\n",
    "                    print(f\"TIE FROM HUMAN: {i}, {j}, {k}, {l}, {score1}, {score2}\")\n",
    "                    human_tie_count += 1\n",
    "                    df_copy_tmp = df_copy[df_copy['text'] == response1]\n",
    "                    df_copy_tmp2 = df_copy[df_copy['text'] == response2]\n",
    "                    if len(df_copy_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "                    if len(df_copy_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                    score1 = df_copy_tmp['answer'].mean()\n",
    "                    score2 = df_copy_tmp2['answer'].mean()\n",
    "                    \n",
    "                    if math.isclose(score1, score2):\n",
    "                        print(\"STILL A TIE\")\n",
    "                        raise Exception(\"STILL A TIE\")\n",
    "\n",
    "                if score1 > score2 and result > 0.5:\n",
    "                    match = 1\n",
    "                elif score1 < score2 and result < 0.5:\n",
    "                    match = 1\n",
    "                elif math.isclose(score1, score2) and math.isclose(result, 0.5):\n",
    "                    match = 1\n",
    "                elif math.isclose(result, 0.5):\n",
    "                    match = 0.5\n",
    "                elif math.isclose(score1, score2):\n",
    "                    match = 0.5\n",
    "                else: \n",
    "                    match = 0\n",
    "                    # print(response1, \"\\n\\n\", response2)\n",
    "                    # print(f\"{i} {j} {k} {l} {result} {score1} {score2}\")\n",
    "                    # if random.random() > 0.7: raise Exception(\"?\")\n",
    "\n",
    "                matching += match\n",
    "                total += 1\n",
    "                comparison_points.append((score1, score2, result, match))\n",
    "                look_at_points.append({\"response1\": response1, \"response2\": response2, \"score1\": score1, \"score2\": score2, \"result\": result, \"match\": match, \"i\": i, \"j\": j, \"k\": k, \"l\": l, \"advice1\": mapping[str(i)], \"advice2\": mapping[str(j)]})\n",
    "\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'barista/evaluators/the_customer/mappings_remapped.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/bt/s_9f74891td8dms1fzdqbkf80000gn/T/ipykernel_52545/3026327050.py\u001b[0m in \u001b[0;36m<cell line: 11>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{evaluator}/mappings_remapped.json\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0mremap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'barista/evaluators/the_customer/mappings_remapped.json'"
     ]
    }
   ],
   "source": [
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/responses/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "    remap = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "# start here\n",
    "df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "grouped = df_tmp.groupby('origin_id')\n",
    "for name, group in grouped:\n",
    "    for name2, group2 in grouped: \n",
    "        if name == name2: continue\n",
    "        score1 = group['answer'].mean()\n",
    "        score2 = group2['answer'].mean()\n",
    "\n",
    "        if True: # score1 - score2 >= 1.5 or score2 - score1 >= 1.5:\n",
    "            # look for it in the MWU matrix\n",
    "            advice_indices = group.iloc[0]['advice_indices']\n",
    "            advice_indices2 = group2.iloc[0]['advice_indices']\n",
    "            i = int(remap[advice_indices + \".json\"])\n",
    "            j = int(remap[advice_indices2 + \".json\"])\n",
    "            k = int(group.iloc[0]['generation_index'])\n",
    "            l = int(group2.iloc[0]['generation_index'])\n",
    "\n",
    "            if \"3\" in advice_indices or \"3\" in advice_indices2:\n",
    "                continue\n",
    "\n",
    "            if i > j or i == j and k > l: \n",
    "                continue\n",
    "\n",
    "            if MWU_matrix[i][j][k][l] > 0.5 and score1 > score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] < 0.5 and score1 < score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] == 0.5 and score1 == score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] == 0.5 or score1 == score2:\n",
    "                matching += 0.5\n",
    "            total += 1\n",
    "\n",
    "            # with open(f\"{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\", \"r\") as f:\n",
    "            #     d = json.load(f)\n",
    "            \n",
    "            # print(d['indices'], d['flipped'], d['parsed_answers'], \"mwu\", MWU_matrix[i][j][k][l], \"human\", score1, score2, matching)\n",
    "            # print(d[\"user_prompt\"][\"content\"], \"\\n\\n\\n\\n\")\n",
    "            # print(group.iloc[0]['text'], \"\\n\\n\\n\")\n",
    "            # print(group2.iloc[0]['text'], \"\\n\\n\\n\\n\\n\")\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "human scores: 5.9 1.95\n",
      "evaluator: the_customer\n",
      "MWU result: 0.3333333333333333\n",
      "flipped: True\n",
      "parsed_answers: [1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Absolutely! We brew our lattes using a blend of beans that balances perfectly with whole milk. Your medium latte will be ready shortly. Do you like to have any flavors added? We have a variety of choices for syrups and sweetening options.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Of course! One medium whole milk latte coming up! We also have some delicious pastries that go great with it, would you like to try one?\"\n",
      "\n",
      "In which scenario are you more likely to leave a tip? Or if you would leave a tip in both cases, in which scenario would you leave a larger tip? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "reasoning: [\"As a customer, I am likely to tip in both situations as the baristas were polite and offered further service beyond my order. However, my tendency to leave a larger tip might be influenced by a few factors.\\n\\nIn scenario 1, the barista was knowledgeable about coffee and took the time to explain something about how their blend balances with whole milk. This indicates a potential interest in coffee and its craft, making the experience feel a bit more unique. Additionally, they offered further customization for my order- different flavors and sweetening options. This could be beneficial for someone who enjoys trying different flavors in their coffee.\\n\\nIn scenario 2, the barista suggested pairing my order with a pastry. This upselling might be viewed as an attempt to increase the total price of my order more than enhancing my experience, depending on my personal perception. If I'm not feeling hungry or if I'm diet conscious, I might not appreciate this suggestion as much.\\n\\nHowever, in both scenarios, willingness to tip more would also depend on how these suggestions were presented, the tone of the barista, and their engagement level.\\n\\nAnswer: For me, the larger tip would go in Scenario 1. I appreciate a knowledgeable barista who can offer further customization to my coffee as per my taste preference. It feels that they are invested not just in selling me a product, but in enhancing my overall coffee drinking experience.\", 'Reasoning:\\nIn both scenarios, the barista shows friendly customer service. However, in Scenario 1, the barista describes the coffee being used and offers a customizable experience with different flavors which can make me feel more taken care of and valued as a customer. In Scenario 2, the barista does also offer a suggestion for an add-on which shows interest in providing great customer service.\\n\\nAnswer: Scenario 1', \"Reasoning: The barista in scenario 1 not only responds positively to the order, but they also pleasantly inform about the quality of their beans and how they blend with whole milk, which reflects their knowledge and passion for their work. Moreover, suggesting flavors with different syrups and sweetening options shows an effort to personalize the experience. On the contrary, the second scenario is more specifically targeted towards cross-selling and doesn't provide additional information about the order. From these details, it's clear that the effort put into customer service is greater in the first scenario.\\n\\nAnswer: Scenario 1\"]\n",
      "\n",
      "\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'barista/evaluators/the_customer/compare_results/0_3_0_1.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/bt/s_9f74891td8dms1fzdqbkf80000gn/T/ipykernel_52545/1609634925.py\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mevaluator\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators/{evaluator}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m             \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators/{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m                 \u001b[0mcomparison\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'barista/evaluators/the_customer/compare_results/0_3_0_1.json'"
     ]
    }
   ],
   "source": [
    "# print out the reasonings for items in look_at_points\n",
    "\n",
    "for item in look_at_points:\n",
    "    i, j, k, l = item[\"i\"], item[\"j\"], item[\"k\"], item[\"l\"]\n",
    "\n",
    "    for evaluator in os.listdir(f\"{scenario}/evaluators\"):\n",
    "        if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "            with open(f\"{scenario}/evaluators/{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\", \"r\") as f:\n",
    "                comparison = json.load(f)\n",
    "    \n",
    "            print(f\"human scores: {item['score1']} {item['score2']}\")\n",
    "            print(f\"evaluator: {evaluator}\")\n",
    "            print(f\"MWU result: {result}\")\n",
    "            print(f\"flipped: {comparison['flipped']}\")\n",
    "            print(f\"parsed_answers: {comparison['parsed_answers']}\")\n",
    "            if type(comparison['user_prompt']) == type([]):\n",
    "                user_prompt = comparison['user_prompt'][0]['content']\n",
    "            else: user_prompt = comparison['user_prompt']['content']\n",
    "            print(f\"user_prompt: {user_prompt}\\n\")\n",
    "            print(f\"reasoning: {comparison['responses']}\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that the weighted MWU matrix is correct\n",
    "\n",
    "# get the weighted MWU matrix\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "weighted_MWU_matrix = np.load(path)\n",
    "\n",
    "# get the weights\n",
    "with open(f\"{scenario}/evaluators/weights.json\", \"r\") as f:\n",
    "    weights = json.load(f)\n",
    "\n",
    "MWU_constructed_matrix = np.zeros(weighted_MWU_matrix.shape)\n",
    "\n",
    "# get each MWU matrix and the weights\n",
    "for evaluator in os.listdir(f\"{scenario}/evaluators/\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "        MWU_matrix = np.load(f\"{scenario}/evaluators/{evaluator}/MWU_matrix.npy\")\n",
    "\n",
    "        for d in weights:\n",
    "            if d[\"evaluator\"] == evaluator:\n",
    "                weight = d[\"weight\"]\n",
    "                break\n",
    "        \n",
    "        MWU_constructed_matrix += MWU_matrix * int(weight)/100\n",
    "\n",
    "assert np.allclose(weighted_MWU_matrix, MWU_constructed_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fc7cf47b5b0>"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 9924 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# PLOT MATCHES\n",
    "data = comparison_points\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "# Separate the data based on the 'match' values\n",
    "x_match_0 = [x for x, y, result, match in data if match == 0]\n",
    "y_match_0 = [y for x, y, result, match in data if match == 0]\n",
    "\n",
    "x_match_0_5 = [x for x, y, result, match in data if match == 0.5]\n",
    "y_match_0_5 = [y for x, y, result, match in data if match == 0.5]\n",
    "\n",
    "x_match_1 = [x for x, y, result, match in data if match == 1]\n",
    "y_match_1 = [y for x, y, result, match in data if match == 1]\n",
    "\n",
    "# Create the scatter plot with different colors\n",
    "plt.scatter(x_match_0, y_match_0, color='red', label='match=0')\n",
    "plt.scatter(x_match_0_5, y_match_0_5, color='yellow', label='match=0.5')\n",
    "plt.scatter(x_match_1, y_match_1, color='green', label='match=1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Result')"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6248 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "result_match_0 = [result for x, y, result, match in data if match == 0]\n",
    "result_match_0_5 = [result for x, y, result, match in data if match == 0.5]\n",
    "result_match_1 = [result for x, y, result, match in data if match == 1]\n",
    "\n",
    "# Create the scatter plot with different colors\n",
    "plt.scatter(result_match_0, [0] * len(result_match_0), color='red', label='match=0')\n",
    "plt.scatter(result_match_0_5, [0.5] * len(result_match_0_5), color='yellow', label='match=0.5')\n",
    "plt.scatter(result_match_1, [1] * len(result_match_1), color='green', label='match=1')\n",
    "\n",
    "# Set labels and legend\n",
    "plt.xlabel('Result')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['0', '0_1', '0_2', '0_3', '1', '1_2', '1_3', '2', '2_3', '3', '[]'])\n",
      "1 piece 4.61 600\n",
      "2 pieces 3.8986111111111112 720\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7fc7d0cd3a30>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0cd3cd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0ce1cd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0ce1f70>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7fc7d0cd3f70>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0ce1250>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0cef250>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0cef4f0>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7fc7d0cd3790>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0ce1a30>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7fc7d0ce14f0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0cef790>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7fc7d0ce1790>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d0cefa30>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4176 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR NUMBER OF ADVICE INDICES\n",
    "\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "\n",
    "grouped = df_tmp.groupby(['advice_indices'])['answer']\n",
    "print(grouped.groups.keys())\n",
    "\n",
    "# zero_pieces = grouped.get_group(\"nan\")\n",
    "one_piece = pd.concat([grouped.get_group(\"0\"), grouped.get_group(\"1\"), grouped.get_group(\"2\"), grouped.get_group(\"3\")])\n",
    "two_pieces = pd.concat([grouped.get_group(\"0_1\"), grouped.get_group(\"0_2\"), grouped.get_group(\"1_2\"), grouped.get_group(\"0_3\"), grouped.get_group(\"1_3\"), grouped.get_group(\"2_3\")])\n",
    "\n",
    "# print(\"0 pieces\", zero_pieces.mean(), len(zero_pieces))\n",
    "print(\"1 piece\", one_piece.mean(), len(one_piece))\n",
    "print(\"2 pieces\", two_pieces.mean(), len(two_pieces))\n",
    "\n",
    "plt.boxplot([one_piece, two_pieces], labels=[\"1 piece\", \"2 pieces\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ties: 25\n",
      "total: 2556\n"
     ]
    }
   ],
   "source": [
    "# NUMBER OF TIES FOR AVERAGE SCORES FOR EACH GENERATION\n",
    "\n",
    "grouped = df.groupby('origin_id')['answer'].mean()\n",
    "ties = 0\n",
    "\n",
    "for origin_id, score in grouped.iteritems():\n",
    "    for origin_id2, score2 in grouped.iteritems():\n",
    "        if score == score2 and origin_id != origin_id2:\n",
    "            ties += 1\n",
    "            # print(origin_id, origin_id2, score)\n",
    "\n",
    "ties = ties // 2\n",
    "print(\"ties:\", ties)\n",
    "print(\"total:\", len(grouped) * (len(grouped) - 1) // 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 30 baseline_ask_with_reasoning baseline_neutral_advice [] 1\n",
      "6 71 baseline_ask_with_reasoning unorthodox_advice [] 2_3\n",
      "8 42 baseline_irrelevant_advice baseline_neutral_advice 0 1_2\n",
      "9 23 baseline_irrelevant_advice baseline_irrelevant_advice 0 1_2\n",
      "9 33 baseline_irrelevant_advice baseline_neutral_advice 0 0_1\n",
      "10 50 baseline_irrelevant_advice unorthodox_advice 1 0_1\n",
      "16 69 baseline_irrelevant_advice unorthodox_advice 2 1_3\n",
      "22 25 baseline_irrelevant_advice baseline_neutral_advice 1_2 0\n",
      "23 9 baseline_irrelevant_advice baseline_irrelevant_advice 1_2 0\n",
      "23 33 baseline_irrelevant_advice baseline_neutral_advice 1_2 0_1\n",
      "25 22 baseline_neutral_advice baseline_irrelevant_advice 0 1_2\n",
      "26 51 baseline_neutral_advice unorthodox_advice 0 0_1\n",
      "26 60 baseline_neutral_advice unorthodox_advice 0 1_2\n",
      "26 72 baseline_neutral_advice unorthodox_advice 0 2_3\n",
      "27 38 baseline_neutral_advice baseline_neutral_advice 0 0_2\n",
      "27 68 baseline_neutral_advice unorthodox_advice 0 1_3\n",
      "30 6 baseline_neutral_advice baseline_ask_with_reasoning 1 []\n",
      "30 71 baseline_neutral_advice unorthodox_advice 1 2_3\n",
      "31 45 baseline_neutral_advice unorthodox_advice 0_1 0\n",
      "32 48 baseline_neutral_advice unorthodox_advice 0_1 1\n",
      "33 9 baseline_neutral_advice baseline_irrelevant_advice 0_1 0\n",
      "33 23 baseline_neutral_advice baseline_irrelevant_advice 0_1 1_2\n",
      "35 53 baseline_neutral_advice unorthodox_advice 2 2\n",
      "36 37 baseline_neutral_advice baseline_neutral_advice 2 0_2\n",
      "36 44 baseline_neutral_advice unorthodox_advice 2 0\n",
      "37 36 baseline_neutral_advice baseline_neutral_advice 0_2 2\n",
      "37 44 baseline_neutral_advice unorthodox_advice 0_2 0\n",
      "38 27 baseline_neutral_advice baseline_neutral_advice 0_2 0\n",
      "38 68 baseline_neutral_advice unorthodox_advice 0_2 1_3\n",
      "39 52 baseline_neutral_advice unorthodox_advice 0_2 2\n",
      "42 8 baseline_neutral_advice baseline_irrelevant_advice 1_2 0\n",
      "43 55 unorthodox_advice unorthodox_advice 0 0_2\n",
      "43 65 unorthodox_advice unorthodox_advice 0 0_3\n",
      "44 36 unorthodox_advice baseline_neutral_advice 0 2\n",
      "44 37 unorthodox_advice baseline_neutral_advice 0 0_2\n",
      "45 31 unorthodox_advice baseline_neutral_advice 0 0_1\n",
      "46 54 unorthodox_advice unorthodox_advice 1 2\n",
      "46 64 unorthodox_advice unorthodox_advice 1 0_3\n",
      "48 32 unorthodox_advice baseline_neutral_advice 1 0_1\n",
      "50 10 unorthodox_advice baseline_irrelevant_advice 0_1 1\n",
      "51 26 unorthodox_advice baseline_neutral_advice 0_1 0\n",
      "51 60 unorthodox_advice unorthodox_advice 0_1 1_2\n",
      "51 72 unorthodox_advice unorthodox_advice 0_1 2_3\n",
      "52 39 unorthodox_advice baseline_neutral_advice 2 0_2\n",
      "53 35 unorthodox_advice baseline_neutral_advice 2 2\n",
      "54 46 unorthodox_advice unorthodox_advice 2 1\n",
      "54 64 unorthodox_advice unorthodox_advice 2 0_3\n",
      "55 43 unorthodox_advice unorthodox_advice 0_2 0\n",
      "55 65 unorthodox_advice unorthodox_advice 0_2 0_3\n",
      "60 26 unorthodox_advice baseline_neutral_advice 1_2 0\n",
      "60 51 unorthodox_advice unorthodox_advice 1_2 0_1\n",
      "60 72 unorthodox_advice unorthodox_advice 1_2 2_3\n",
      "61 63 unorthodox_advice unorthodox_advice 3 3\n",
      "SAME ADVICE SET\n",
      "62 70 unorthodox_advice unorthodox_advice 3 2_3\n",
      "63 61 unorthodox_advice unorthodox_advice 3 3\n",
      "SAME ADVICE SET\n",
      "64 46 unorthodox_advice unorthodox_advice 0_3 1\n",
      "64 54 unorthodox_advice unorthodox_advice 0_3 2\n",
      "65 43 unorthodox_advice unorthodox_advice 0_3 0\n",
      "65 55 unorthodox_advice unorthodox_advice 0_3 0_2\n",
      "66 67 unorthodox_advice unorthodox_advice 0_3 1_3\n",
      "67 66 unorthodox_advice unorthodox_advice 1_3 0_3\n",
      "68 27 unorthodox_advice baseline_neutral_advice 1_3 0\n",
      "68 38 unorthodox_advice baseline_neutral_advice 1_3 0_2\n",
      "69 16 unorthodox_advice baseline_irrelevant_advice 1_3 2\n",
      "70 62 unorthodox_advice unorthodox_advice 2_3 3\n",
      "71 6 unorthodox_advice baseline_ask_with_reasoning 2_3 []\n",
      "71 30 unorthodox_advice baseline_neutral_advice 2_3 1\n",
      "72 26 unorthodox_advice baseline_neutral_advice 2_3 0\n",
      "72 51 unorthodox_advice unorthodox_advice 2_3 0_1\n",
      "72 60 unorthodox_advice unorthodox_advice 2_3 1_2\n"
     ]
    }
   ],
   "source": [
    "# How many of the ties are from the same advice set?\n",
    "grouped = df.groupby('origin_id')['answer'].mean()\n",
    "ties = []\n",
    "\n",
    "for origin_id, score in grouped.iteritems():\n",
    "    for origin_id2, score2 in grouped.iteritems():\n",
    "        if score == score2 and origin_id != origin_id2:\n",
    "            ties.append((origin_id, origin_id2))\n",
    "\n",
    "grouped2 = df.groupby('origin_id')[['advice_type', 'advice_indices']]\n",
    "for (origin_id, origin_id2) in ties:\n",
    "    advice_type = grouped2.get_group(origin_id)['advice_type'].unique()[0]\n",
    "    advice_type2 = grouped2.get_group(origin_id2)['advice_type'].unique()[0]\n",
    "    advice_indices = grouped2.get_group(origin_id)['advice_indices'].unique()[0]\n",
    "    advice_indices2 = grouped2.get_group(origin_id2)['advice_indices'].unique()[0]\n",
    "    print(origin_id, origin_id2, advice_type, advice_type2, advice_indices, advice_indices2)\n",
    "    if advice_type == advice_type2 and advice_indices == advice_indices2:\n",
    "        print(\"SAME ADVICE SET\")\n",
    "\n",
    "# ONLY 1 TIE IS FROM THE SAME ADVICE SET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline_ask_with_reasoning baseline_neutral_advice [] 0 5.783333333333333\n",
      "baseline_neutral_advice baseline_ask_with_reasoning 0 [] 5.783333333333333\n",
      "baseline_neutral_advice unorthodox_advice 1 1_2 5.55\n",
      "unorthodox_advice baseline_neutral_advice 1_2 1 5.55\n"
     ]
    }
   ],
   "source": [
    "# what about ties between advice sets (instead of generations?)\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "ties = 0\n",
    "\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    for (advice_type2, advice_indices2), score2 in grouped.iteritems():\n",
    "        if score == score2 and (advice_type != advice_type2 or advice_indices != advice_indices2):\n",
    "            ties += 1\n",
    "            print(advice_type, advice_type2, advice_indices, advice_indices2, score)\n",
    "# NO TIES BETWEEN PIECES OF ADVICE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.4\n",
      "4.076666666666667\n",
      "1.8394964328415484\n",
      "0.7193997823029882\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- RESPONSE\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped_best = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped_best.mean())\n",
    "\n",
    "# average of all generations\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "print(df_tmp['answer'].mean())\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(np.std(grouped, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped_best.mean() - grouped.mean())/np.std(grouped, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.4\n",
      "3.661111111111111\n",
      "1.8234224088361293\n",
      "0.95364018806745\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- RESPONSE WITHOUT SOCIAL MEDIA\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped_best = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped_best.mean())\n",
    "\n",
    "# average of all generations\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '0_1']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_2']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_3']\n",
    "print(df_tmp['answer'].mean())\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(np.std(grouped, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped_best.mean() - grouped.mean())/np.std(grouped, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.2 \"Hi there, I just want to say I really love this flower vase. The craftsmanship is absolutely stunning. I'm really keen to support local artisans like yourself and I'm definitely planning to come back to shop in the future. I was wondering, though, are there any discounts for people buying multiple items or for future returning customers? This piece is really lovely, but my budget is a bit tight at the moment.\"\n",
      "6.85 \"Hello, this is such a beautiful vase! The design is truly unique. You've done an incredible job crafting it. I'm a great supporter of local artisans and I love adding these kinds of pieces to my home. However, I wonder if there might be a little wiggle room on the price? 30 dollars is just a slight stretch for my budget at the moment. Is there a possibility we could negotiate a little?\"\n",
      "6.8 \"Hi there! This vase is absolutely stunning - your craftsmanship truly shines. I'm really interested in making a purchase, but I was wondering if there might be a little wiggle room with the price? Also, do you offer any deals for repeat customers? I would love to keep supporting your work when I return to the market.\"\n",
      "6.6 \"Hello, I must say, I absolutely love your work. Your craftsmanship is remarkable and I truly believe in supporting our local artisans. This round flower vase has caught my eye, it’s so beautiful. Though, I've to confess the price is slightly more than I had budgeted for today. I plan to come back to the market in the future. Do you offer discounts for multiple item purchases or for returning customers? I'd love to be a regular patron and support your craft while also keeping a tab on my expenses.\"\n",
      "6.55 \"Hello, I must say, I'm really impressed with the craftsmanship of this beautiful flower vase. Your work truly stands out in the market. I'm drawn to this round vase, it's priced at 30 dollars, right? I am quite interested but I was wondering if there might be any room for a slight negotiation? Also, I'm planning to come back to the market again and possibly buy more. Do you offer any discounts for returning customers or if I make multiple purchases?\"\n",
      "6.3 \"Hi there, I'm really interested in purchasing this lovely vase, but I was also wondering, do you offer any discounts for purchasing multiple items or for returning customers? I really enjoy your work and I plan to visit the market again in the future.\"\n",
      "6.25 \"Hello! I must say, your work is absolutely exquisite. The attention to detail in this round vase is something I can't find elsewhere... you truly have a gift. I am always eager to support our local artisans. However, I'm slightly constrained by my budget. Would there be any chance for a little flexibility on the price?\"\n",
      "6.2 \"Hello, I just wanted to say how much I appreciate this beautiful round flower vase. Your craftsmanship truly stands out. I'm actually very interested in purchasing it. I was just wondering, is there is any room for bargaining? Also, would there possibilities for discounts if I were to purchase multiple items, or perhaps for returning customers? I admire your craft very much and I plan on coming back to this market in the near future.\"\n",
      "6.2 \"Hello! I must say, this round flower vase is absolutely gorgeous. The craftsmanship is truly exceptional. I'm really considering buying it, but I was wondering if there might be a possibility for some kind of discount or negotiation on the price? I truly appreciate the work that's gone into this and I'm really interested in it.\"\n",
      "6.2 \"Hi there! I have to say, I absolutely adore this round flower vase. The craftsmanship is truly outstanding. I can see myself placing this vase in my living room, it would surely enhance its beauty. I was wondering, though, is there any possibility we could discuss the price a bit? Maybe there's some room for a slight reduction? I'd also be more than happy to share some pictures of this beautiful piece on my social media and tag your store. How does that sound?\"\n",
      "6.15 Hello! I've been admiring your work. This flower vase is absolutely stunning. I'm really passionate about supporting local artisans like yourself, and I love that I can buy something handcrafted instead of something factory-made. \n",
      "\n",
      "However, I'm on a tighter budget than I had hoped for. Do you think we might be able to work something out? Perhaps I could help promote your work on my social media or anything else that might be beneficial for your business in exchange for a slight discount on this vase?\n",
      "6.1 \"Hello, this is a beautiful piece. You truly have incredible talent. I'm interested in purchasing it, and while I appreciate the craftsmanship and the price, I was wondering if there might be any flexibility there? Also, I'd love to help others discover your work. If you're open to a little discount, I could feature this vase and your entire shop on my social media. What do you think?\"\n",
      "6.1 \"Hello, I was really drawn to this beautiful flower vase. Your craftsmanship is truly exquisite. I'm a big supporter of local artisans and I love to have items like this in my home. However, as much as I love this vase, the price is a tiny bit over my budget. Is there by any chance you could give me a small discount?\"\n",
      "6.1 \"Hi there! This vase is indeed captivating. I'm planning on being a regular visitor here in the future and I'm also interested in multiple purchases. I was wondering, do you offer any discounts for returning customers or for purchases of more than one item?\"\n",
      "6.0 \"Hello there! I absolutely love this vase; it's so beautiful. I can see the quality of your work and I truly admire it. As a big supporter of local artisans, I’m thrilled to contribute to the community, but I'm on a bit of a budget. So, I was wondering if there's any possibility of a slight reduction in price?\"\n",
      "5.9 Hello, I am really interested in this beautiful flower vase. It's exactly the kind of craftsmanship I've been looking for! I notice the marked price is 30 dollars, which I think is fair given the quality, however, I was wondering if you offer any discounts on purchases? For example, I plan on returning to this market regularly and buying more items, would that potentially qualify me for a discount? \n",
      "\n",
      "Additionally, I have quite a large following on my social media channels where I share all things home décor and DIY. I would be more than happy to feature any piece I buy from your stall and share your artisan shop's information with my followers. That way, it could help bring more attention to your work. Would that perhaps encourage a discount on this vase?\n",
      "5.9 \"Hello, first let me say - your work is outstanding. I can see the passion and effort you've put into crafting this vase. It's exactly what I've been looking for and I really want to support local artisans like yourself. But, I am a bit constrained by my budget. Is there any chance for a small discount, perhaps?\"\n",
      "5.9 \"Hi there, I absolutely love this round flower vase. I'm always looking to support local artisans like you and add unique pieces like this to my collection. I notice it's marked at $30, though, which is a little more than I was hoping to spend today. Do you offer any discounts for buying multiple items or for customers who will be returning to the market in the future? I definitely see myself coming back to shop here again.\"\n",
      "5.85 Hello, I'm very interested in this lovely round flower vase. It's perfectly suited for my home. I'm definitely on board with supporting local artisans, and I noticed that you have so many other high-quality products here. I was wondering, do you offer any discounts if I purchase multiple items or even as a returning customer? I come to the market often and would love to keep supporting your work.\n",
      "\n",
      "Also, I have a substantial following on my social media platforms, and I would be thrilled to share your work with my audience to help promote your business. Would it be possible to consider an equivalent exchange like that for a discount on this vase?\n",
      "5.8 \"Hi there, I must say, this vase is beautifully crafted! Your talent is truly extraordinary. Just out of curiosity, would there be any room for a slight price negotiation on this piece?\"\n",
      "5.7 \"Hello, I've been admiring this beautiful flower vase. I really love supporting local artisans like you, and your work truly is unique. I must mention, though, I'm trying to stretch my budget a bit, so I was hoping we could possibly discuss the price a bit?\"\n",
      "5.65 Hi there, I'm really interested in this lovely round flower vase. Its craftsmanship is indeed impressive. I'm thinking about purchasing it, but I wonder if you might offer any discounts for buying multiple items? Also, I plan to be a returning customer, as I'm a big supporter of local artisans like yourself. Would there possibly be any special deals for loyal customers like me?\n",
      "5.55 \"Hello, this vase is wonderfully made, your craftsmanship is truly impressive. I am very interested in purchasing it, but I was wondering if there's any room for bargaining on the price?\"\n",
      "5.35 \"Hello! I noticed this beautiful round flower vase and I'm really interested in buying it. Your work is exceptional and I'd love to support you by promoting your talent to my network. I have a decent amount of followers on social media who would love your work. Do you think we could come to an arrangement where I promote your work in exchange for a small discount on the vase?\"\n",
      "5.35 Hi there, I absolutely love your work! This round flower vase has particularly caught my eye, it's really beautiful. I noticed the price is 30 dollars, which is fair for the quality, but I was wondering if there's any chance we could arrange a little discount? I'm active on my social media where I post often about local art and artisans. I'd be more than happy to feature your work and this beautiful vase on my Instagram and Facebook which could help in promoting your craft. What do you think?\n",
      "5.3 Hi there! I'm absolutely enchanted by this beautiful round vase you've created. The craftsmanship truly reflects the talent we have here locally. I'm a big fan of supporting our local artisans. But you know, my budget is a little tight at the moment. \n",
      "\n",
      "I wondered if you might consider offering me a small discount? To show my gratitude, I would love to share your work on my social media platforms. I have a good number of followers who share my passion for artisan pieces. I think they would adore your work and it could potentially draw more clients to your business. How does that sound to you?\n",
      "5.3 Hi there! I must say, I'm absolutely smitten with this round flower vase. The design and craftsmanship are simply exquisite. I can tell a lot of time and effort went into creating this beautiful piece. \n",
      "\n",
      "I was just wondering, is there any room for negotiation with the price? I'm willing to pay the full price, but as an admirer of local artisans, I always try to get good value whenever possible. \n",
      "\n",
      "Actually, I have quite a following on social media. If possible, I'd love to post this vase and highlight your fantastic work in exchange for a small discount. What do you think about that?\n",
      "5.15 Hello! I really love this flower vase. It's perfectly round and well-crafted, just what I've been looking for. I was wondering if there might be any discounts available, perhaps for purchasing multiple items or for returning customers? I do plan on coming back to this market regularly. Also, I have a few thousand followers on social media and would be more than happy to share how much I love my finds from your stall, if that could potentially earn me a small discount on this vase. What do you think?\n",
      "5.15 \"Hi there. I absolutely love this flower vase. Your craftsmanship is truly unique! I noticed the marked price is 30 dollars, which I think is fair considering the quality. However, I was wondering if you might consider offering a discount? I have a considerable following on my social media platforms and would be more than happy to share a post about this beautiful vase and your other works, directing people to your stall. This could potentially bring your craft to a wider audience. What do you think?\"\n",
      "4.95 Hello there! I absolutely love your work, it's clear you're very talented. I've been searching for a unique, locally made piece of art and your vase really caught my eye. As someone who really values supporting our local artisans, I would be so happy to take this vase home. \n",
      "\n",
      "I have to admit though, the price is a little higher than I was hoping to spend today. However, I really do not want to pass up on owning such an exquisite item. I have quite a large following on my social media platforms and I'd love to post a photo of your beautiful vase and share your name/brand. Maybe we could work out a deal where I help promote your work in exchange for a bit of a discount on the vase? What do you think?\n",
      "30\n"
     ]
    }
   ],
   "source": [
    "# print where it is in the non-3 (4th best)\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "grouped = df_tmp.groupby('origin_id')\n",
    "\n",
    "scores = []\n",
    "texts = []\n",
    "for origin_id in grouped.groups.keys():\n",
    "    df_tmp2 = grouped.get_group(origin_id)\n",
    "    scores.append(df_tmp2['answer'].mean())\n",
    "    texts.append(df_tmp2.iloc[0]['text'])\n",
    "\n",
    "order = np.argsort(scores)[::-1]\n",
    "for i in order:\n",
    "    print(scores[i], texts[i])\n",
    "print(len(order))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.55\n",
      "4.076666666666666\n",
      "1.8600112836508926\n",
      "0.7921098900225086\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- ADVICE\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_advice_index]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped.mean())\n",
    "\n",
    "# average of all advice\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "grouped_advice = df_tmp.groupby('advice_indices')['answer'].mean()\n",
    "print(grouped_advice.mean())\n",
    "print(np.std(grouped_advice, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped.mean() - grouped_advice.mean()) / np.std(grouped_advice, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.566666666666666\n",
      "6.202777777777778\n",
      "0.3097340555786289\n",
      "1.174843005910635\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- ADVICE WITHOUT SOCIAL MEDIA\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == \"1_2\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped.mean())\n",
    "\n",
    "# average of all advice\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '0_3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '2_3']\n",
    "grouped_advice = df_tmp.groupby('advice_indices')['answer'].mean()\n",
    "print(grouped_advice.mean())\n",
    "print(np.std(grouped_advice, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped.mean() - grouped_advice.mean()) / np.std(grouped_advice, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean scores for best framework response 5.4\n",
      "mean scores for baseline_ask 5.583333333333333\n",
      "mean scores for baseline_ask_with_reasoning 5.783333333333334\n",
      "mean scores for baseline_neutral_advice 5.8388888888888895\n",
      "todo: run the framework on this\n",
      "mean scores for baseline_irrelevant_advice 2.8472222222222223\n",
      "todo: run the framework on this\n"
     ]
    }
   ],
   "source": [
    "# COMPARISONS AGAINST BASELINES\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for best framework response\", grouped.mean())\n",
    "\n",
    "# average of all baseline_ask\n",
    "df_tmp = df[df['advice_type'] == \"baseline_ask\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_ask\", grouped.mean())\n",
    "\n",
    "# average of all baseline_ask_with_reasoning\n",
    "df_tmp = df[df['advice_type'] == \"baseline_ask_with_reasoning\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_ask_with_reasoning\", grouped.mean())\n",
    "\n",
    "# average of all baseline_neutral_advice\n",
    "df_tmp = df[df['advice_type'] == \"baseline_neutral_advice\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_neutral_advice\", grouped.mean())\n",
    "print(\"todo: run the framework on this\")\n",
    "\n",
    "# average of all baseline_irrelevant_advice\n",
    "df_tmp = df[df['advice_type'] == \"baseline_irrelevant_advice\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_irrelevant_advice\", grouped.mean())\n",
    "print(\"todo: run the framework on this\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import remap\n",
    "\n",
    "remap(f\"{scenario}/evaluators/artisan/mappings\")\n",
    "with open(f\"{scenario}/evaluators/artisan/mappings_remapped.json\", \"r\") as f:\n",
    "    remapping = json.load(f)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A. analysis on large differences\n",
    "import os\n",
    "import re\n",
    "import random\n",
    "\n",
    "def get_large_differences(df):\n",
    "    \n",
    "    evaluator = \"artisan\"\n",
    "    with open(f\"{scenario}/evaluators/{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "        remapping = json.load(f)[0]\n",
    "\n",
    "    diff_threshold = 5\n",
    "    df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "    grouped = df_tmp.groupby('participant_id')[['answer', 'advice_indices', 'generation_index']]\n",
    "    participant_ids = sorted(list(grouped.groups.keys()))\n",
    "    l = []\n",
    "\n",
    "    for i in range(len(grouped)):\n",
    "        participant_id = participant_ids[i]\n",
    "        participant_df = grouped.get_group(participant_id)\n",
    "\n",
    "        for j in range(len(participant_df)):\n",
    "            score = participant_df.iloc[j]['answer']\n",
    "\n",
    "            for k in range(j+1, len(participant_df)):\n",
    "                score2 = participant_df.iloc[k]['answer']\n",
    "\n",
    "                if abs(score - score2) >= diff_threshold:\n",
    "                    advice_indices1 = participant_df.iloc[j]['advice_indices']\n",
    "                    advice_indices2 = participant_df.iloc[k]['advice_indices']\n",
    "                    generation_index1 = int(participant_df.iloc[j]['generation_index'])\n",
    "                    generation_index2 = int(participant_df.iloc[k]['generation_index'])\n",
    "\n",
    "                    if advice_indices1 == \"nan\" or advice_indices2 == \"nan\":\n",
    "                        continue\n",
    "\n",
    "                    compare_index1 = int(remapping[f\"{advice_indices1}.json\"])\n",
    "                    compare_index2 = int(remapping[f\"{advice_indices2}.json\"])\n",
    "\n",
    "                    # print(f\"{advice_indices1}, generation {generation_index1}, score {score}\")\n",
    "                    # print(f\"{advice_indices2}, generation {generation_index2}, score {score2}\")\n",
    "\n",
    "                    if compare_index1 > compare_index2 or (compare_index1 == compare_index2 and generation_index1 > generation_index2):\n",
    "                        compare_index1, compare_index2 = compare_index2, compare_index1\n",
    "                        advice_indices1, advice_indices2 = advice_indices2, advice_indices1\n",
    "                        generation_index1, generation_index2 = generation_index2, generation_index1\n",
    "                        score, score2 = score2, score\n",
    "\n",
    "                    print(f\"participant {participant_id} has a large difference:\")\n",
    "                    print(f\"{advice_indices1}, generation {generation_index1}, score {score}\")\n",
    "                    print(f\"{advice_indices2}, generation {generation_index2}, score {score2}\")\n",
    "\n",
    "                    d = {}\n",
    "                    d[\"participant_id\"] = participant_id\n",
    "                    d[\"advice_indices1\"] = advice_indices1\n",
    "                    d[\"advice_indices2\"] = advice_indices2\n",
    "                    d[\"generation_index1\"] = generation_index1\n",
    "                    d[\"generation_index2\"] = generation_index2\n",
    "                    d[\"score1\"] = score\n",
    "                    d[\"score2\"] = score2\n",
    "\n",
    "                    for evaluator in os.listdir(f\"{scenario}/evaluators\"):\n",
    "                        if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "                            with open(f\"{scenario}/evaluators/{evaluator}/compare_results/{compare_index1}_{compare_index2}_{generation_index1}_{generation_index2}.json\", \"r\") as f:\n",
    "                                compare_result = json.load(f)\n",
    "                            \n",
    "                            scenario_options_text = compare_result[\"user_prompt\"][\"content\"]\n",
    "                            scenario_options_text = re.sub(r'Please think step by step and explain your reasoning.*', '', scenario_options_text, flags=re.DOTALL)\n",
    "                            scores = compare_result[\"parsed_answers\"]\n",
    "                            reasoning_text = compare_result[\"responses\"]\n",
    "\n",
    "                            if \"evaluator\" in d:\n",
    "                                d[\"evaluator\"] += [evaluator]\n",
    "                                d[\"flipped\"] += [compare_result[\"flipped\"]]\n",
    "                                d[\"scenario_options_text_2\"] = scenario_options_text\n",
    "                                d[\"scores\"] += [scores]\n",
    "                                d[\"reasoning_text_4\"] = reasoning_text[0]\n",
    "                                d[\"reasoning_text_5\"] = reasoning_text[1]\n",
    "                                d[\"reasoning_text_6\"] = reasoning_text[2]\n",
    "                            else:\n",
    "                                d[\"evaluator\"] = [evaluator]\n",
    "                                d[\"flipped\"] = [compare_result[\"flipped\"]]\n",
    "                                d[\"scenario_options_text_1\"] = scenario_options_text\n",
    "                                d[\"scores\"] = [scores]\n",
    "                                d[\"reasoning_text_1\"] = reasoning_text[0]\n",
    "                                d[\"reasoning_text_2\"] = reasoning_text[1]\n",
    "                                d[\"reasoning_text_3\"] = reasoning_text[2]\n",
    "                            \n",
    "\n",
    "                            print(f\"Evaluator: {evaluator}:\")\n",
    "                            print(f\"Flipped: {compare_result['flipped']}\")\n",
    "                            print(f\"Scenarios: {scenario_options_text}\\n\\n\")\n",
    "                            print(f\"SCORES: 1 -> first one better, 0 -> second one better (these are affected by flipped): {scores}\\n\")\n",
    "                            print(f\"Model Reasoning (these are also affected by flipped): \\n{reasoning_text}\\n\\n\\n\")\n",
    "\n",
    "                    l.append(d)\n",
    "\n",
    "    random.seed(0)\n",
    "    random_selection = random.sample(l, 10)\n",
    "    # save as csv\n",
    "    import csv\n",
    "    with open(f\"{scenario}/large_differences.csv\", \"w\") as f:\n",
    "        writer = csv.DictWriter(f, fieldnames= \\\n",
    "                                [\"participant_id\", \"advice_indices1\", \"advice_indices2\", \"generation_index1\", \"generation_index2\", \\\n",
    "                                \"score1\", \"score2\", \"evaluator\", \"flipped\", \"scenario_options_text_1\", \"scenario_options_text_2\", \\\n",
    "                                \"scores\", \"reasoning_text_1\", \"reasoning_text_2\", \"reasoning_text_3\", \"reasoning_text_4\", \"reasoning_text_5\", \"reasoning_text_6\"])\n",
    "        writer.writeheader()\n",
    "        for d in random_selection:\n",
    "            writer.writerow(d)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
