{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "scenario = 'barista'\n",
    "name = 'prolific_ablation_results'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GET THE BEST PERFORMING RESPONSE\n",
    "\n",
    "MWU_matrix = np.load(f\"{scenario}/MWU_matrix_ablation_reasoning.npy\")\n",
    "scores = np.sum(MWU_matrix, axis=1)\n",
    "best_index = np.argmax(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>origin_id</th>\n",
       "      <th>participant_id</th>\n",
       "      <th>participant_strategy</th>\n",
       "      <th>participant_strategy_len</th>\n",
       "      <th>scenario</th>\n",
       "      <th>advice_type</th>\n",
       "      <th>advice_indices</th>\n",
       "      <th>advice_list</th>\n",
       "      <th>generation_index</th>\n",
       "      <th>text</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>93</td>\n",
       "      <td>1</td>\n",
       "      <td>When a server talks a lot, it just makes me fe...</td>\n",
       "      <td>49</td>\n",
       "      <td>A barista at a coffee shop wants to maximize t...</td>\n",
       "      <td>baseline_ask_with_reasoning</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>20</td>\n",
       "      <td>\"Of course! One medium whole milk latte coming...</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>77</td>\n",
       "      <td>1</td>\n",
       "      <td>When a server talks a lot, it just makes me fe...</td>\n",
       "      <td>49</td>\n",
       "      <td>A barista at a coffee shop wants to maximize t...</td>\n",
       "      <td>baseline_ask_with_reasoning</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>4</td>\n",
       "      <td>\"Absolutely, medium whole milk latte coming ri...</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27</td>\n",
       "      <td>96</td>\n",
       "      <td>3</td>\n",
       "      <td>I don't like people talking to me, so I decide...</td>\n",
       "      <td>49</td>\n",
       "      <td>A barista at a coffee shop wants to maximize t...</td>\n",
       "      <td>baseline_ask_with_reasoning</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>23</td>\n",
       "      <td>\"Absolutely, I'd be glad to prepare that for y...</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41</td>\n",
       "      <td>80</td>\n",
       "      <td>4</td>\n",
       "      <td>I don't tip barristas so it was easy, it didn'...</td>\n",
       "      <td>14</td>\n",
       "      <td>A barista at a coffee shop wants to maximize t...</td>\n",
       "      <td>baseline_ask_with_reasoning</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>7</td>\n",
       "      <td>\"Of course, I'll get that started for you righ...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>43</td>\n",
       "      <td>87</td>\n",
       "      <td>3</td>\n",
       "      <td>I don't like people talking to me, so I decide...</td>\n",
       "      <td>49</td>\n",
       "      <td>A barista at a coffee shop wants to maximize t...</td>\n",
       "      <td>baseline_ask_with_reasoning</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>14</td>\n",
       "      <td>\"Absolutely, I'll have your medium whole milk ...</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  origin_id  participant_id  \\\n",
       "0   9         93               1   \n",
       "1  13         77               1   \n",
       "2  27         96               3   \n",
       "3  41         80               4   \n",
       "4  43         87               3   \n",
       "\n",
       "                                participant_strategy  \\\n",
       "0  When a server talks a lot, it just makes me fe...   \n",
       "1  When a server talks a lot, it just makes me fe...   \n",
       "2  I don't like people talking to me, so I decide...   \n",
       "3  I don't tip barristas so it was easy, it didn'...   \n",
       "4  I don't like people talking to me, so I decide...   \n",
       "\n",
       "   participant_strategy_len  \\\n",
       "0                        49   \n",
       "1                        49   \n",
       "2                        49   \n",
       "3                        14   \n",
       "4                        49   \n",
       "\n",
       "                                            scenario  \\\n",
       "0  A barista at a coffee shop wants to maximize t...   \n",
       "1  A barista at a coffee shop wants to maximize t...   \n",
       "2  A barista at a coffee shop wants to maximize t...   \n",
       "3  A barista at a coffee shop wants to maximize t...   \n",
       "4  A barista at a coffee shop wants to maximize t...   \n",
       "\n",
       "                   advice_type advice_indices advice_list  generation_index  \\\n",
       "0  baseline_ask_with_reasoning             []          []                20   \n",
       "1  baseline_ask_with_reasoning             []          []                 4   \n",
       "2  baseline_ask_with_reasoning             []          []                23   \n",
       "3  baseline_ask_with_reasoning             []          []                 7   \n",
       "4  baseline_ask_with_reasoning             []          []                14   \n",
       "\n",
       "                                                text  answer  \n",
       "0  \"Of course! One medium whole milk latte coming...     9.0  \n",
       "1  \"Absolutely, medium whole milk latte coming ri...     6.0  \n",
       "2  \"Absolutely, I'd be glad to prepare that for y...     5.0  \n",
       "3  \"Of course, I'll get that started for you righ...     0.0  \n",
       "4  \"Absolutely, I'll have your medium whole milk ...     6.0  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LOAD THE HUMAN DATA\n",
    "\n",
    "df = pd.read_csv(f\"{scenario}/{name}.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform column advice_indices to string\n",
    "df['participant_strategy'] = df['participant_strategy'].apply(lambda x: str(x))\n",
    "df['advice_indices'] = df['advice_indices'].apply(lambda x: str(x))\n",
    "# tranforms all nan values\n",
    "\n",
    "na_locations = df.isna()\n",
    "row_indices, col_indices = np.where(na_locations)\n",
    "for row, col in zip(row_indices, col_indices):\n",
    "    print(f\"NaN value at row {row}, column {df.columns[col]}\")\n",
    "    # print(df.iloc[row], \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that origin matches text exactly\n",
    "# origin_id are 13-105, inclusive\n",
    "\n",
    "dict = {}\n",
    "for row in df.itertuples():\n",
    "    if row.origin_id not in dict:\n",
    "        dict[row.origin_id] = row.text\n",
    "    else:\n",
    "        if dict[row.origin_id] != row.text:\n",
    "            print(\"ERROR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of origin_ids with less than 20 ratings: 0 out of 30\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 4928 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check the number of ratings per origin_id\n",
    "non20_count = 0\n",
    "counts = []\n",
    "for id_count in df.groupby('text')[\"id\"].count():\n",
    "    if id_count != 20:\n",
    "        non20_count += 1\n",
    "    counts += [id_count]\n",
    "plt.hist(counts, bins=9)\n",
    "\n",
    "print(f\"Number of origin_ids with less than 20 ratings: {non20_count} out of {len(df.groupby('origin_id'))}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "START OF ANALYSIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "participant_id: 1\n",
      "participant_strategy: When a server talks a lot, it just makes me feel like they are trying to manipulate people to get them to spend more and Upsell, making themselves look better to management. Most of the time the tone and way they speak feels disingenuous or forced to me. \n",
      "participant_id: 2\n",
      "participant_strategy: I compared the scenarios to experiences that I have had when ordering coffee myself. I thought about the length of dialogue and how likely I was to buy into the extras considering their choice of language. \n",
      "participant_id: 3\n",
      "participant_strategy: I don't like people talking to me, so I decided how annoyed I would be with the barista's waffling to determine if I would tip.  The more waffling and irrelevancy, the less the tip would be.  Indication of care and attention without annoyance got a bigger tip.\n",
      "participant_id: 4\n",
      "participant_strategy: I don't tip barristas so it was easy, it didn't matter what they said\n",
      "participant_id: 5\n",
      "participant_strategy: I read the scenario once and then just read the response each time and considered how it would make me feel to be in that situation. \n",
      "participant_id: 6\n",
      "participant_strategy: To answer the questions honestly based on how they would make me feel in a real-life scenario. \n",
      "participant_id: 7\n",
      "participant_strategy: the more personal and over the top the barista was the more repulsed i was\n",
      "participant_id: 8\n",
      "participant_strategy: to put myself in the situation and imagine how id feel\n",
      "participant_id: 9\n",
      "participant_strategy: I just wanted to know my order would be taken correctly and efficiently in a friendly manner whilst avoiding small talk\n",
      "participant_id: 10\n",
      "participant_strategy: I tried to choose ones to tip higher where the barista appeared happy to serve the customer and did not overly advertise their other products in a way that seemed like it applied pressure on the customer. I preferred ones where the answers were quite simple yet effective.\n",
      "participant_id: 11\n",
      "participant_strategy: To envision myself in the exact scenario and how I would react if I were currently in that situation. Particularly, I considered the level of detail of the answer, relevance of the content and whether tipping was mentioned.\n",
      "participant_id: 12\n",
      "participant_strategy: The nicer the barista was the greater the tip but there is a limit. Too much talking is annoying. And I don't like people asking for a tip\n",
      "participant_id: 13\n",
      "participant_strategy: I like short and considerate conversation with baristas, so I took that into account when it came to the size of the tips I gave. Some of the scenarios included the barista responding with irrelevant information, so my tip was lower in that case.\n",
      "participant_id: 14\n",
      "participant_strategy: To use my imagination of how i would feel in the morning, as much as being very nice is nice i think too much would put me off tipping.\n",
      "participant_id: 15\n",
      "participant_strategy: My stratergy was that if they were polite or made her conversation or asked if I would like certain extras they would get a bigger tip. If they asked for a tip, then no tip was given as I feel that is rude. If they went on a bit about unrelated things I didn't leave as big of a tip. \n",
      "participant_id: 16\n",
      "participant_strategy: read very carefully and answer questions accurately having given them due care and thought\n",
      "participant_id: 17\n",
      "participant_strategy: Likely to leave a a small tip was my main response, increased slightly if i liked the service. SOme of the comments seemed too personal though and so i may have scored those slightly lower\n",
      "participant_id: 18\n",
      "participant_strategy: deciding which responses I liked best\n",
      "participant_id: 19\n",
      "participant_strategy: barista that is friendly but not overly trying\n",
      "participant_id: 20\n",
      "participant_strategy: I thought about how I would feel as the customer and if I liked the barista or not based on their comments. \n",
      "participant_id: 21\n",
      "participant_strategy: I didn't have a stretgy. The response I thought were better I gave a higher score. They shouldn't be mentioning anything about tips so those got a low score and it is nice if they are friendly and personable but you don't always want them to go on that much\n",
      "participant_id: 22\n",
      "participant_strategy: I had no strategy but the weirder the barista got (chatting about water plants, chess and passports) made me less likely to tip. and mentioning a tip is a definite no.\n",
      "participant_id: 23\n",
      "participant_strategy: put myself in the shoes of the customer, based my answers on how i like to be spoken to in a coffee shop. \n",
      "participant_id: 24\n",
      "participant_strategy: How relatable the barista made the transaction feel\n",
      "participant_id: 25\n",
      "participant_strategy: In UK culture we don't do casual tipping - i find a pleasant interaction is nice but also prefer brief and efficient - if tipping is the norm then i'll tip\n",
      "participant_id: 26\n",
      "participant_strategy: I liked the ones where the barista didn't ramble for too long, but was still helpful, and passed on important information/seemed friendly. \n",
      "participant_id: 27\n",
      "participant_strategy: It dependent on how annoying they were how much tip they got. Less is more.\n",
      "participant_id: 28\n",
      "participant_strategy: I don't want over familiarity or obvious up-selling, it annoys me\n",
      "participant_id: 29\n",
      "participant_strategy: I imagined the person saying the words and whether they struck me as someone out to make my experience fun and different.\n",
      "participant_id: 30\n",
      "participant_strategy: nan\n",
      "participant_id: 31\n",
      "participant_strategy: Although the scenarios were very polite in terms of the barista's response to the customer, my strategy was based on whether the response was genuine or not. If the barista's was too forward, i.e. hinting for tips, I personally did not want to tip them high. If they spoke too long about watering of plants etc., I soon became a little frustrated. However, if they made an effort in terms of making the customer feel special, the tip was made higher.\n",
      "participant_id: 32\n",
      "participant_strategy: I enjoyed it, some of the responses were funny. If someone asks for a tip they dont get one. If they make me laugh, offer something but dont assume something then i would give them a tip of some size\n",
      "participant_id: 33\n",
      "participant_strategy: Definite no if they mentioned people enjoyed coffee more if theyd given a tip. If they gave me information on the coffee and beans I considered a tip. I also considered a tip if they upsold a croissant or muffin or wished me a good day\n",
      "participant_id: 34\n",
      "participant_strategy: Just going with my gut and imagining myself in the situation and what I would do. \n",
      "participant_id: 35\n",
      "participant_strategy: i just imagined how i would feel in the customers shoes.\n",
      "participant_id: 36\n",
      "participant_strategy: to answer truthfully \n",
      "participant_id: 39\n",
      "participant_strategy: I think anyone providing services in hospitality, should offer extra with no extra cost tips will fly.\n",
      "participant_id: 40\n",
      "participant_strategy: As a customer in a coffee shop my first annoyance is when the barista becomes too talkative. It is nice to exchange pleasantries through small talk but I can imagine most people are in a rush or just want efficient service. I also thought people would be less likely to tip if their server intrudes on private conversations. \n",
      "participant_id: 41\n",
      "participant_strategy: I thought about what was most deserving to get a tip for without seeming disingenuous. \n",
      "participant_id: 42\n",
      "participant_strategy: I imagined myself as a customer and reacted to how the conversation made me feel at the time\n",
      "participant_id: 43\n",
      "participant_strategy: I thought about how I'd feel if I received the response to my ordering a coffee.\n",
      "participant_id: 44\n",
      "participant_strategy: I have been in situations before when a waiter/waitress has put me on the spot and asked how much I will be tipping. In the UK it is not essential that we tip, so we only do it if the service is really good. If the staff member mentions a tip it automatically makes me want to say no as I believe it comes across rude. If the service is really good and the staff are friendly then I am more inclined to tip to thank them for the services they have provided.\n",
      "participant_id: 45\n",
      "participant_strategy: I really dislike being upsold, so will not tip if the barista tries to sell me anything else. I also dislike faux-friendliness and enthusiasm - I would much rather just be served my latte.\n",
      "participant_id: 46\n",
      "participant_strategy: It was based on if I though they were being genuine\n",
      "participant_id: 47\n",
      "participant_strategy: Definitely anti mentioning tips and too much conversation is a no no\n",
      "participant_id: 48\n",
      "participant_strategy: Didn't like being asked for a tip\n",
      "participant_id: 49\n",
      "participant_strategy: A good interesting study, and my strategy was to  feel the warmth and kindness in the conversations, and to judge in general how real and caring each comment was said and came across.\n",
      "participant_id: 50\n",
      "participant_strategy: I don't generally tip so that is why I was at the lower end. Also, if I found the comment over the top and irrelevant I didn't tip. I was just looking for great service without it being OTT\n",
      "participant_id: 51\n",
      "participant_strategy: Perfer the barista who didn't go hawking for a tip \n",
      "participant_id: 52\n",
      "participant_strategy: Higher tip if they wished me a good day or similar\n",
      "participant_id: 53\n",
      "participant_strategy: I'm not a big tipper, too much chatter irritates me\n",
      "participant_id: 54\n",
      "participant_strategy: I looked at the wording, read the whole paragraph and made a deision as to how helpful the barista was.  Sometimes they were helpful, offering more than needed, but in a nice way.  Sometimes they said totally irrelevent things.  I did notice however, that i made a mistake, and think I said to tip number 9 when it said coffee, not latte, so not sure if that was a test.  If so i apolgise but it was too late by then as i had pressed the next button.\n",
      "participant_id: 55\n",
      "participant_strategy: I would tip relevant & helpful, not left field and wacky suggestions \n",
      "participant_id: 56\n",
      "participant_strategy: I didn't like it when the barista basically asked for a tip to be given - not very professional and totally irked me.  I'll decide if I wish to leave a tip upon the interaction I have with the server, but I won't leave one if you ask.  Rude.\n",
      "participant_id: 57\n",
      "participant_strategy: I just told the truth. I am not keen on people trying to sell me extra or being false to get a tip\n",
      "participant_id: 58\n",
      "participant_strategy: I answered by how good of an impression the barista made on me\n",
      "participant_id: 59\n",
      "participant_strategy: I put myself in the customer's position and imagined what I would like to hear as a response to my order. Too much waffle would make me less likely to tip.\n",
      "participant_id: 61\n",
      "participant_strategy: I just used my gut instincts and imagined myself in the scenario. It was an enjoyable task\n",
      "participant_id: 62\n",
      "participant_strategy: I didnt like the obvious 'asking for a tip' but I did like the useful hints and tips like checking your passport.\n",
      "participant_id: 63\n",
      "participant_strategy: i think that asking or mentioning a tip will reduce the likelihood of a tip being given. I also think the longer responses tend to go down less well in this environment and less likely to tip. Good humour and recommendations can be great customer service and usually worthy of a tip\n",
      "participant_id: 64\n",
      "participant_strategy: Some feedback for you, hope this is ok! In the UK, we do not have a tipping culture at all. So, anyone serving a customer who would explicitly mention tipping would automatically be frowned upon, it is seen as very bad manners here. I marked down any response that mentioned a tip, but also marked down any that were over familiar. Mentioning passport etc - that doesn't seem anything like a normal human server would say, it comes across as AI response (and if a person DID say that in the UK customers would give them strange looks!). Really enjoyed the task, good luck with your research.\n",
      "participant_id: 65\n",
      "participant_strategy: I always leave a tip anyway so that wasnt a difficult choice.  i would leave less of a tip if the barista asked for one, it think its really rude and unnecessary.  I'm not really interested in any of the fun facts and would perhaps consider a bigger tip if they were just polite and to the point without any fluff.\n",
      "participant_id: 66\n",
      "participant_strategy: iT WAS THOUGHT PROVOKING AND QUITE DECISIVE AT THE EXTREMES. I preferred the short succinct answers to the long winded ones trying to engage in meaningless conversation. The best responses were short, direct and to the point delivered with politeness. That is what great customer service means to me. It's great that someone was confident and engagaed in lengthier replies ,but it generally doesn't work for me. I think this works well when the server can 'read' the customer before 'going' there. I didn't mind a little bit of extra information. I wasn't interested in what the milk did, or other flavours much of the time because of the way it was presented. However, one of two of the response about the almond croissant went down well because they were succinct and the language and tone worked for me personally. I hope this helps!\n",
      "participant_id: 67\n",
      "participant_strategy: I felt that if the barista asked for a tip or even mentioned it, they were being cheeky, so I wouldn't tip at all.  If the barista was pleasant but not particularly personable I might leave a small tip.  If the barista was pleasant, personable and went the extra mile, I would tip them more.\n",
      "participant_id: 68\n",
      "participant_strategy: Give the biggest tip when the barista is being authentic. If someone asks for a tip it's desperate and a turn off. If their service is genuine and they're a good person then they'll get a tip without having to beg.\n",
      "participant_id: 69\n",
      "participant_strategy: I didn't have a strategy as such, I was just imagining how it would feel standing in a coffee shop being part of that exchange. How I would feel and how the person making the drink would represent themselves to me through their speech and behaviour.\n",
      "participant_id: 70\n",
      "participant_strategy: Happy that I knew the first part of the question, so just concentrated on the barista's reply\n",
      "participant_id: 71\n",
      "participant_strategy: If they mentioning me tipping them, I don't tip. I tip based on friendliness.\n",
      "participant_id: 72\n",
      "participant_strategy: my strategy was to reward politeness but not to reward as much or nothing if there was too much selling of food I had not asked for and if the barista went into too much detail on unrelated matters like the plants\n",
      "participant_id: 73\n",
      "participant_strategy: Informative and friendly\n",
      "participant_id: 74\n",
      "participant_strategy: dont like to pushy\n",
      "participant_id: 75\n",
      "participant_strategy: By assessing whether the comments made to the customer were sincere and polite. If they suggested a tip or hinted toward the customer leaving a tip then I would give it a bad score as that can come across pushy or rude.\n",
      "participant_id: 76\n",
      "participant_strategy: I dont tip to any one who talks too much or asks for one\n",
      "participant_id: 77\n",
      "participant_strategy: If the barrista's chat sounded fake or inappropriate I was less likely to leave a tip\n",
      "participant_id: 78\n",
      "participant_strategy: it was good the more they made conversation higher the tip as soon as asked for a tip no tip at all\n",
      "participant_id: 79\n",
      "participant_strategy: answer honestly and openly as if it was a real scenario\n",
      "participant_id: 80\n",
      "participant_strategy: My strategy was to rate the customer service experienced based on politeness, staying on-track with the conversation and not adding too many unrelated comments. \n",
      "participant_id: 81\n",
      "participant_strategy: Based on what I would want the person serving me to ask me.\n",
      "participant_id: 83\n",
      "participant_strategy: reward someone who is being nice\n",
      "participant_id: 84\n",
      "participant_strategy: I looked at how each barista replied to the customer's request for a latte. I checked if they were polite, quick in responding, and made the customer feel good. I also checked if they suggested extra things to buy and paid attention to what the customer wanted. This helped me decide how likely I would be to give a tip.\n",
      "participant_id: 85\n",
      "participant_strategy: My strategy was to imagine that I was the customer in the shop ordering a medium latte and to imagine how impressed I would be with each response and whether or not I would be likely to leave a tip in the real world.\n",
      "participant_id: 86\n",
      "participant_strategy: I preferred friendly service that offered complimentary water and scored that highly. Where I was asked for a tip directly, I scored low. I did exactly what I would do in real life. This was my strategy.\n",
      "participant_id: 87\n",
      "participant_strategy: the complimentary glass of water and the genuine concern for staying hydrated would also generate the biggest tip from me, whereas simply trying to upsell a dessert would not \n",
      "participant_id: 88\n",
      "participant_strategy: I preferred the choices being mine rather than feeling pressured into taking what the Barista offered\n",
      "participant_id: 89\n",
      "participant_strategy: If they were kind and didnt talk much i would tip more\n",
      "participant_id: 90\n",
      "participant_strategy: I tipped the baristas who were polite and had some chat with me. I didn't like anyone who suggested tipping or those who started talking about chess which just seemed weird and forced as both would make me feel uncomfortable.\n",
      "participant_id: 91\n",
      "participant_strategy: Picture myself inside a coffee shop/cafe and how would I react if the server said the same things. Also if it looked like they were trying too hard\n",
      "participant_id: 92\n",
      "participant_strategy: I tried to think about tipping\n",
      "participant_id: 93\n",
      "participant_strategy: My strategy was to evaluate I would feel about the baristas response and based on that indicate how much of a tip id be likely to give. I preferred quicker friendly responses rather than long winded ones that felt forced.\n",
      "participant_id: 94\n",
      "participant_strategy: I enjoyed the task. My strategy was simply based off what i would do. I gave answers that seemed genuinely human and weren't simply begging for tips, but actually being helpful and not (seemingly) focusing on making money, higher scores.\n",
      "participant_id: 95\n",
      "participant_strategy: If I liked their responce I would tip well.  I don't like being asked for tips though, or being reminded to tip, that's going to put my nose right out of joint.  I don't like people getting too familiar either, or hard up-sells.  I do like additional effort though, so if someone recommends something I might Iike I might tip a bit extra.\n",
      "participant_id: 96\n",
      "participant_strategy: I rated the responses based on perceived sincerity, tone, politeness, and length.  If i felt the response was to convoluted and/or irrelevant i rated it worse. Any response mentioning tips scored 0. any response that was short, relevant to the coffee and felt like casual conversation was charming and rated higher. \n",
      "participant_id: 97\n",
      "participant_strategy: I was most likely to tip when the barrista was being helpful and genuine. I would not tip if they started randomly talking about travel - it just felt unnatural and a bit creepy.\n",
      "participant_id: 98\n",
      "participant_strategy: To answer truthfully.\n",
      "participant_id: 99\n",
      "participant_strategy: HONESTY\n",
      "participant_id: 100\n",
      "participant_strategy: I really enjoyed this task and trying to imagine if i would leave a tip or not. I do like to leave a small tip unless the staff are rude in some way or just don't go out their way to help me.\n",
      "Thanks for the opportunity.\n",
      "participant_id: 101\n",
      "participant_strategy: politeness, interest in the customer, sincere, no selfish mention of tips\n",
      "participant_id: 102\n",
      "participant_strategy: I tried to think about what sort of interaction most people would like when ordering a coffee.\n",
      "participant_id: 103\n",
      "participant_strategy: No major strategy, just went with what felt right to me\n",
      "participant_id: 104\n",
      "participant_strategy: No strategy, more likely to tip for the responses I liked\n",
      "participant_id: 105\n",
      "participant_strategy: Fairly simple - just read the information presented and make a judgement based on what I read.\n",
      "participant_id: 106\n",
      "participant_strategy: I didn't like the very long elaborate responses and would be in too much of a rush to get away. I like the short but warm and helpful responses and favoured these the most.\n",
      "participant_id: 107\n",
      "participant_strategy: I liked the shorter answers so I wouldn't have to hang around longer than I needed to for my coffee. \n"
     ]
    }
   ],
   "source": [
    "# READ THROUGH PARTICIPANT STRATEGIES\n",
    "\n",
    "grouped = df.groupby('participant_id')\n",
    "\n",
    "# iterate through and print out the participant strategies\n",
    "for name, group in grouped:\n",
    "    print(f\"participant_id: {name}\")\n",
    "    print(f\"participant_strategy: {group['participant_strategy'].iloc[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.20088553173803306 0.18137422353689872\n"
     ]
    }
   ],
   "source": [
    "# COMPUTE INTER RATER RELIABILITY\n",
    "results = []\n",
    "\n",
    "for i in range(20):\n",
    "    group1_means = []\n",
    "    group2_means = []\n",
    "\n",
    "    grouped_by_origin_id = df.groupby('origin_id') # grouped based on question\n",
    "    # loop across questions\n",
    "    for name, group in grouped_by_origin_id:\n",
    "        # randomly split participants in half\n",
    "        df_tmp = df[df['origin_id'] == name]\n",
    "        unique_participant_ids = df_tmp['participant_id'].unique()\n",
    "        np.random.shuffle(unique_participant_ids)\n",
    "        \n",
    "        split_point = int(len(unique_participant_ids) / 2)\n",
    "        first_half = unique_participant_ids[:split_point]\n",
    "        second_half = unique_participant_ids[split_point:]\n",
    "        \n",
    "        group1 = df_tmp[df_tmp['participant_id'].isin(first_half)]\n",
    "        group2 = df_tmp[df_tmp['participant_id'].isin(second_half)]\n",
    "\n",
    "        group1_means += [group1['answer'].mean()]\n",
    "        group2_means += [group2['answer'].mean()]\n",
    "        \n",
    "    # apply spearman-brown correction\n",
    "    results += [2 * np.corrcoef(group1_means, group2_means)[0, 1] / (1 + np.corrcoef(group1_means, group2_means)[0, 1])]\n",
    "\n",
    "print(np.mean(results), np.std(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 2.,  3.,  1.,  5.,  5., 10., 21., 30., 12.,  8.,  6.]),\n",
       " array([0.        , 0.81818182, 1.63636364, 2.45454545, 3.27272727,\n",
       "        4.09090909, 4.90909091, 5.72727273, 6.54545455, 7.36363636,\n",
       "        8.18181818, 9.        ]),\n",
       " <BarContainer object of 11 artists>)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4448 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF AVERAGE SCORES PER PARTICIPANT\n",
    "\n",
    "mean_scores = df.groupby('participant_id')['answer'].mean()\n",
    "plt.hist(mean_scores, bins=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "[0.0, 0.0, 0.0, 0.0]\n",
      "7\n",
      "[1.0, 0.0, 3.0, 5.0, 5.0, 1.0, 3.0, 3.0]\n",
      "22\n",
      "[4.0, 4.0, 3.0, 1.0, 2.0, 3.0]\n",
      "24\n",
      "[5.0, 0.0, 0.0, 1.0, 0.0]\n",
      "27\n",
      "[2.0, 0.0, 0.0, 2.0, 2.0]\n",
      "28\n",
      "[0.0, 0.0, 2.0, 2.0, 4.0]\n",
      "57\n",
      "[2.0, 1.0, 5.0, 1.0, 3.0]\n",
      "45\n",
      "[0.0, 2.0, 0.0, 0.0, 0.0]\n",
      "76\n",
      "[3.0, 4.0, 0.0, 6.0, 2.0]\n",
      "91\n",
      "[3.0, 5.0, 1.0]\n",
      "96\n",
      "[6.0, 4.0, 0.0, 3.0, 0.0, 3.0, 5.0, 1.0]\n",
      "43\n",
      "[9.0, 10.0, 8.0, 9.0]\n",
      "94\n",
      "[9.0, 10.0, 8.0, 9.0, 10.0, 9.0, 8.0, 8.0]\n",
      "101\n",
      "[8.0, 10.0, 10.0, 8.0, 9.0, 8.0]\n"
     ]
    }
   ],
   "source": [
    "# TRY TO CLEAN THE DATA\n",
    "\n",
    "mean_scores = df.groupby('participant_id')['answer'].mean()\n",
    "for id in df['participant_id'].unique():\n",
    "    if mean_scores[id] < 3.5:\n",
    "        print(id)\n",
    "        # print in a list instead\n",
    "        print(list(df[df['participant_id'] == id]['answer']))\n",
    "\n",
    "for id in df['participant_id'].unique():\n",
    "    if mean_scores[id] > 8.5:\n",
    "        print(id)\n",
    "        # print in a list instead\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "high variances\n",
      "6\n",
      "[9.0, 10.0, 8.0, 2.0, 8.0, 9.0, 0.0]\n",
      "10\n",
      "[2.0, 3.0, 5.0, 9.0]\n",
      "87\n",
      "[8.0, 3.0, 9.0, 8.0, 2.0, 1.0, 2.0, 1.0]\n"
     ]
    }
   ],
   "source": [
    "# MORE DATA CLEANING\n",
    "\n",
    "# # LESS THAN 20\n",
    "# scores = df.groupby('participant_id')['answer']\n",
    "# for id in df['participant_id'].unique():\n",
    "#     if len(list(scores.get_group(id))) < 20:\n",
    "#         print(id)\n",
    "#         print(list(df[df['participant_id'] == id]['answer']))\n",
    "\n",
    "# CHECK FOR ABNORMALLY HIGH VARIANCE\n",
    "print(\"high variances\")\n",
    "variances = df.groupby('participant_id')['answer'].var()\n",
    "for id in df['participant_id'].unique():\n",
    "    if variances[id] > 8:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "[0.0, 0.0, 0.0, 0.0]\n",
      "15\n",
      "[8.0, 8.0, 9.0, 8.0, 8.0, 8.0]\n",
      "17\n",
      "[4.0, 5.0, 5.0, 5.0, 5.0, 6.0, 5.0]\n",
      "23\n",
      "[5.0, 4.0, 4.0, 5.0, 5.0, 5.0]\n",
      "25\n",
      "[5.0, 4.0, 4.0, 4.0, 3.0, 4.0, 3.0]\n",
      "26\n",
      "[7.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 6.0]\n",
      "21\n",
      "[7.0, 8.0, 7.0]\n",
      "34\n",
      "[7.0, 6.0, 6.0, 6.0, 7.0, 7.0]\n",
      "19\n",
      "[5.0, 5.0, 6.0, 6.0, 6.0, 6.0]\n",
      "31\n",
      "[5.0, 5.0]\n",
      "41\n",
      "[5.0, 6.0, 6.0]\n",
      "29\n",
      "[4.0, 5.0, 4.0, 4.0, 4.0]\n",
      "58\n",
      "[6.0, 5.0, 7.0, 7.0, 6.0, 7.0, 6.0, 7.0]\n",
      "53\n",
      "[6.0, 6.0, 6.0, 6.0, 6.0]\n",
      "56\n",
      "[8.0, 8.0, 8.0]\n",
      "59\n",
      "[6.0, 7.0, 7.0, 6.0]\n",
      "50\n",
      "[3.0, 4.0, 4.0]\n",
      "52\n",
      "[6.0, 6.0, 7.0, 6.0, 7.0, 6.0, 7.0]\n",
      "67\n",
      "[5.0, 5.0, 4.0, 5.0, 5.0]\n",
      "70\n",
      "[6.0, 7.0, 7.0, 6.0]\n",
      "62\n",
      "[4.0, 4.0, 5.0, 5.0]\n",
      "88\n",
      "[8.0, 9.0, 9.0, 9.0, 8.0, 8.0, 8.0]\n",
      "90\n",
      "[5.0, 5.0, 6.0, 6.0, 6.0, 6.0]\n",
      "92\n",
      "[5.0, 5.0, 6.0, 6.0, 5.0]\n",
      "98\n",
      "[5.0, 5.0, 5.0, 5.0, 5.0, 5.0]\n",
      "95\n",
      "[7.0, 8.0, 7.0, 8.0]\n",
      "100\n",
      "[5.0, 6.0, 5.0, 6.0]\n",
      "107\n",
      "[5.0, 6.0, 5.0, 6.0, 6.0]\n"
     ]
    }
   ],
   "source": [
    "# CHECK FOR ABNORMALLY LOW VARIANCE\n",
    "\n",
    "variances = df.groupby('participant_id')['answer'].var()\n",
    "for id in df['participant_id'].unique():\n",
    "    if variances[id] < 0.6:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "[0.0, 0.0, 0.0, 0.0]\n",
      "9\n",
      "[6.0, 7.0, 7.0, 7.0, 3.0]\n",
      "12\n",
      "[7.0, 6.0, 7.0, 7.0, 4.0]\n",
      "15\n",
      "[8.0, 8.0, 9.0, 8.0, 8.0, 8.0]\n",
      "17\n",
      "[4.0, 5.0, 5.0, 5.0, 5.0, 6.0, 5.0]\n",
      "23\n",
      "[5.0, 4.0, 4.0, 5.0, 5.0, 5.0]\n",
      "25\n",
      "[5.0, 4.0, 4.0, 4.0, 3.0, 4.0, 3.0]\n",
      "24\n",
      "[5.0, 0.0, 0.0, 1.0, 0.0]\n",
      "26\n",
      "[7.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 6.0]\n",
      "21\n",
      "[7.0, 8.0, 7.0]\n",
      "27\n",
      "[2.0, 0.0, 0.0, 2.0, 2.0]\n",
      "30\n",
      "[4.0]\n",
      "35\n",
      "[7.0, 5.0, 6.0, 5.0, 5.0, 5.0, 5.0]\n",
      "19\n",
      "[5.0, 5.0, 6.0, 6.0, 6.0, 6.0]\n",
      "31\n",
      "[5.0, 5.0]\n",
      "41\n",
      "[5.0, 6.0, 6.0]\n",
      "29\n",
      "[4.0, 5.0, 4.0, 4.0, 4.0]\n",
      "39\n",
      "[5.0, 5.0, 8.0, 5.0, 10.0]\n",
      "42\n",
      "[5.0, 6.0, 6.0, 4.0, 6.0, 6.0]\n",
      "53\n",
      "[6.0, 6.0, 6.0, 6.0, 6.0]\n",
      "45\n",
      "[0.0, 2.0, 0.0, 0.0, 0.0]\n",
      "56\n",
      "[8.0, 8.0, 8.0]\n",
      "54\n",
      "[8.0, 7.0, 9.0, 9.0, 9.0, 8.0, 9.0]\n",
      "51\n",
      "[5.0]\n",
      "50\n",
      "[3.0, 4.0, 4.0]\n",
      "52\n",
      "[6.0, 6.0, 7.0, 6.0, 7.0, 6.0, 7.0]\n",
      "55\n",
      "[4.0, 8.0, 8.0, 8.0, 7.0, 8.0, 7.0]\n",
      "71\n",
      "[9.0, 5.0, 5.0, 6.0, 5.0, 5.0, 6.0]\n",
      "73\n",
      "[8.0, 6.0, 6.0, 6.0, 6.0, 6.0]\n",
      "67\n",
      "[5.0, 5.0, 4.0, 5.0, 5.0]\n",
      "88\n",
      "[8.0, 9.0, 9.0, 9.0, 8.0, 8.0, 8.0]\n",
      "90\n",
      "[5.0, 5.0, 6.0, 6.0, 6.0, 6.0]\n",
      "92\n",
      "[5.0, 5.0, 6.0, 6.0, 5.0]\n",
      "84\n",
      "[6.0, 6.0, 6.0, 6.0, 8.0, 7.0, 6.0, 7.0, 8.0]\n",
      "98\n",
      "[5.0, 5.0, 5.0, 5.0, 5.0, 5.0]\n",
      "107\n",
      "[5.0, 6.0, 5.0, 6.0, 6.0]\n"
     ]
    }
   ],
   "source": [
    "# CHECK FOR REPETITIONS\n",
    "\n",
    "for id in df['participant_id'].unique():\n",
    "    answer = list(df[df['participant_id'] == id]['answer'])\n",
    "    # if the answer contains more than half of the same value\n",
    "    for i in range(len(answer)//2 + 1):\n",
    "        if answer.count(answer[i]) > len(answer) / 2:\n",
    "            print(id)\n",
    "            print(answer)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# REMOVE PARTICIPANTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 21.,   0.,  12.,   0.,  20.,   0.,  25.,   0.,  54.,   0., 138.,\n",
       "          0., 116.,   0.,  87.,   0.,  80.,   0.,  33.,   0.,  14.]),\n",
       " array([ 0.        ,  0.47619048,  0.95238095,  1.42857143,  1.9047619 ,\n",
       "         2.38095238,  2.85714286,  3.33333333,  3.80952381,  4.28571429,\n",
       "         4.76190476,  5.23809524,  5.71428571,  6.19047619,  6.66666667,\n",
       "         7.14285714,  7.61904762,  8.0952381 ,  8.57142857,  9.04761905,\n",
       "         9.52380952, 10.        ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5360 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF SCORES\n",
    "\n",
    "plt.hist(df['answer'], bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.691666666666666\n"
     ]
    }
   ],
   "source": [
    "print(df['answer'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.6 5 \"Of course! One medium Whole Milk Latte coming right up. Would you like any pastries or snacks to accompany your latte?\"\n",
      "4.95 6 \"Absolutely, I'd be delighted to make your medium whole milk latte! Just to let you know, we use premium quality whole milk and our finest blend of espresso to make your drink. I'll prepare that right away for you.\"\n",
      "5.05 19 \"Sure, I'd be happy to make that for you! By the way, many customers enjoy adding a shot of vanilla or caramel to their latte. Would you be interested in trying that today?\"\n",
      "5.05 4 \"Absolutely, medium whole milk latte coming right up! You know, choosing whole milk is great as it makes the latte super creamy and rich in taste. Can I get you any pastries or sandwiches to go with your latte?\"\n",
      "5.05 25 \"Absolutely! You know, our medium-sized lattes with whole milk are incredibly smooth and creamy. The quality of our coffee beans really shines through. Would you also like to try our fresh blueberry muffin or perhaps go for a large size latte, to really take a moment for yourself today?\"\n",
      "5.15 24 \"Of course! A medium whole milk latte coming right up. You've made a great choice – that milk really makes the coffee flavor richer and gives the latte a delicious creamy texture. I'll make sure to make it the best latte you've had today!\"\n",
      "5.2 8 \"Absolutely, and thanks for your order. One medium whole milk latte coming right up! Have you tasted our freshly baked muffins? They go very well with lattes\".\n",
      "5.35 10 \"Absolutely, one medium latte with whole milk coming right up! By the way, did you know whole milk gives the coffee a really rich and creamy taste? Very nice choice! How's your day been?\"\n",
      "5.4 0 \"Of course! One medium whole milk latte coming up! We also have some delicious pastries that go great with it, would you like to try one?\"\n",
      "5.45 13 \"Absolutely! Would you like any extra shots or flavor in your latte? Maybe a touch of caramel or vanilla? I promise, it really enhances the taste!\"\n",
      "5.45 3 \"Absolutely! We brew our lattes using a blend of beans that balances perfectly with whole milk. Your medium latte will be ready shortly. Do you like to have any flavors added? We have a variety of choices for syrups and sweetening options.\"\n",
      "5.5 12 \"Absolutely, I'll get that medium whole milk latte started for you right away! By the way, would you like to try it with our new caramel drizzle or an extra shot of espresso? It's been very popular!\"\n",
      "5.5 18 \"Of course! One medium whole milk latte coming right up. By the way, how has your day been so far?\"\n",
      "\n",
      "\n",
      "best from framework: 5.65 27 \"Absolutely! You know, whole milk makes the latte extra creamy, that's a great choice. Would you like any flavors added, like caramel or vanilla, to enhance your drink? And remember, we bake fresh pastries every morning if you're looking for something to complement your latte.\"\n",
      "5.7 11 \"Absolutely! A medium whole milk latte is an excellent choice. I might also suggest adding a flavor, like our seasonal pumpkin spice or classic vanilla, to complement the creamy richness of the whole milk. It can really take the drink up a level. What do you think?\"\n",
      "5.75 20 \"Of course! One medium whole milk latte coming right up. I hope your day is going great so far!\"\n",
      "5.75 7 \"Of course, I'll get that started for you right away! Would you like any extra shots or flavors with your latte?\"\n",
      "5.8 22 \"Of course, one medium latte with whole milk coming right up! By the way, would you like to try it with our special vanilla syrup? It's a popular choice among our customers. No pressure though, your latte will be delicious either way!\"\n",
      "5.85 1 \"Absolutely, one medium latte with whole milk coming right up! Have you tried our new caramel syrup with this? It makes for a delicious combination. I can add a splash if you'd like?\"\n",
      "5.95 9 \"Of course! A medium latte with whole milk, right away! Thank you for choosing us today.\"\n",
      "5.95 23 \"Absolutely, I'd be glad to prepare that for you! You've made an excellent choice as whole milk brings out the coffee's sweet flavors. Our lattes are carefully made to ensure you have the best experience. If you'd like, I could add a little latte art on top, would that be okay?\"\n",
      "6.0 17 \"Absolutely! One medium latte with whole milk coming up. Would you like any pastries or something else with your coffee?\"\n",
      "6.05 29 \"Absolutely, one medium whole milk latte coming up! That's a fantastic choice. Thank you so much for your order.\"\n",
      "6.2 26 \"Of course! I'll prepare your medium whole milk latte right away. Is there anything else I can get for you today?\"\n",
      "6.2 15 \"Absolutely! I'll make sure your medium latte with whole milk is made perfectly. Anything else I can get for you today?\"\n",
      "6.2 14 \"Absolutely, I'll have your medium whole milk latte ready in just a moment! How's your day going so far?\"\n",
      "6.3 16 \"Of course! One medium-sized whole milk latte coming right up. Is there anything else I can assist you with today?\"\n",
      "6.35 2 \"Of course! So that's one medium latte with whole milk. Might I suggest a delicious pastry or savory snack to accompany your latte? They pair wonderfully. I'll make sure your latte is made perfectly for you.\"\n",
      "6.55 28 \"Absolutely, I can prepare a medium latte with whole milk for you. Would you like any extras, like a flavor shot or whipped cream on top? We also have tasty pastries if you fancy something to go along with your drink.\"\n",
      "6.8 21 \"Of course, I'd be happy to make that latte with whole milk for you! You've made a great choice. Will you be enjoying it here or should I prepare it to go?\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([1., 0., 0., 1., 3., 2., 1., 3., 2., 1., 4., 1., 3., 1., 3., 2., 0.,\n",
       "        1., 0., 1.]),\n",
       " array([4.6 , 4.71, 4.82, 4.93, 5.04, 5.15, 5.26, 5.37, 5.48, 5.59, 5.7 ,\n",
       "        5.81, 5.92, 6.03, 6.14, 6.25, 6.36, 6.47, 6.58, 6.69, 6.8 ]),\n",
       " <BarContainer object of 20 artists>)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5696 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SORTED AVERAGE SCORES FOR EACH GENERATION\n",
    "\n",
    "grouped = df.groupby('text')['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "items = []\n",
    "for text, score in grouped.iteritems():\n",
    "    # get the advice_type for each text\n",
    "    df_tmp = df[df['text'] == text]\n",
    "    generation = df_tmp.iloc[0]['generation_index']\n",
    "    if generation == best_index: \n",
    "        print('\\n\\nbest from framework: ', end = '')\n",
    "    print(score, generation, text)\n",
    "    items.append(score)\n",
    "\n",
    "plt.hist(items, bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 607,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.533333333333333 baseline_irrelevant_advice 0_2\n",
      "Consider reorganizing the cutlery drawer, with forks on the left side and knives on the right, for better accessibility.\n",
      "Learn how to make homemade pasta; it's a valuable skill and delicious to eat.\n",
      "4.116666666666666 baseline_irrelevant_advice 0_1\n",
      "Consider reorganizing the cutlery drawer, with forks on the left side and knives on the right, for better accessibility.\n",
      "Water the plants every other day to ensure optimal growth and keep them from willening.\n",
      "4.2 baseline_irrelevant_advice 0\n",
      "Consider reorganizing the cutlery drawer, with forks on the left side and knives on the right, for better accessibility.\n",
      "4.566666666666666 baseline_irrelevant_advice 1\n",
      "Water the plants every other day to ensure optimal growth and keep them from willening.\n",
      "4.783333333333333 baseline_irrelevant_advice 1_2\n",
      "Water the plants every other day to ensure optimal growth and keep them from willening.\n",
      "Learn how to make homemade pasta; it's a valuable skill and delicious to eat.\n",
      "5.15 baseline_irrelevant_advice 2\n",
      "Learn how to make homemade pasta; it's a valuable skill and delicious to eat.\n",
      "5.15 unorthodox_advice 0\n",
      "Appeal to his empathy by saying \"Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night?\"\n",
      "5.483333333333333 unorthodox_advice 0_1\n",
      "Appeal to his empathy by saying \"Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night?\"\n",
      "Directly communicate her feelings by saying \"I miss you and our time together, your late nights out are affecting our relationship.\"\n",
      "5.75 unorthodox_advice 0_3\n",
      "Appeal to his empathy by saying \"Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night?\"\n",
      "\"Start your conversation by asking about his day and his friends' shenanigans, showing genuine interest, before gradually steering the topic to your need for his company.\"\n",
      "6.016666666666667 baseline_neutral_advice 2\n",
      "\"Remember that everyone has faults!\"\n",
      "6.1 unorthodox_advice 1_2\n",
      "Directly communicate her feelings by saying \"I miss you and our time together, your late nights out are affecting our relationship.\"\n",
      "Seek common ground by saying, \"How about we both work on balancing our social lives with our time together?\"\n",
      "6.116666666666666 unorthodox_advice 0_2\n",
      "Appeal to his empathy by saying \"Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night?\"\n",
      "Seek common ground by saying, \"How about we both work on balancing our social lives with our time together?\"\n",
      "6.116666666666666 baseline_neutral_advice 1_2\n",
      "\"Focus on the good times you've shared together!\"\n",
      "\"Remember that everyone has faults!\"\n",
      "6.183333333333334 unorthodox_advice 1_3\n",
      "Directly communicate her feelings by saying \"I miss you and our time together, your late nights out are affecting our relationship.\"\n",
      "\"Start your conversation by asking about his day and his friends' shenanigans, showing genuine interest, before gradually steering the topic to your need for his company.\"\n",
      "6.2 unorthodox_advice 1\n",
      "Directly communicate her feelings by saying \"I miss you and our time together, your late nights out are affecting our relationship.\"\n",
      "6.25 baseline_neutral_advice 0_2\n",
      "\"Always remember to carry a positive attitude!\"\n",
      "\"Remember that everyone has faults!\"\n",
      "6.516666666666667 baseline_neutral_advice 1\n",
      "\"Focus on the good times you've shared together!\"\n",
      "6.683333333333334 baseline_neutral_advice 0_1\n",
      "\"Always remember to carry a positive attitude!\"\n",
      "\"Focus on the good times you've shared together!\"\n",
      "6.7 baseline_ask_with_reasoning []\n",
      "baseline_ask_with_reasoning\n",
      "6.783333333333333 baseline_ask []\n",
      "baseline_ask\n",
      "\n",
      "\n",
      "best from framework: 6.883333333333334 unorthodox_advice 2_3\n",
      "Seek common ground by saying, \"How about we both work on balancing our social lives with our time together?\"\n",
      "\"Start your conversation by asking about his day and his friends' shenanigans, showing genuine interest, before gradually steering the topic to your need for his company.\"\n",
      "6.95 unorthodox_advice 2\n",
      "Seek common ground by saying, \"How about we both work on balancing our social lives with our time together?\"\n",
      "6.966666666666667 unorthodox_advice 3\n",
      "\"Start your conversation by asking about his day and his friends' shenanigans, showing genuine interest, before gradually steering the topic to your need for his company.\"\n",
      "7.1 baseline_neutral_advice 0\n",
      "\"Always remember to carry a positive attitude!\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([1., 0., 0., 2., 0., 0., 1., 1., 0., 2., 0., 1., 0., 1., 1., 6., 0.,\n",
       "        1., 2., 2., 3.]),\n",
       " array([3.53333333, 3.7031746 , 3.87301587, 4.04285714, 4.21269841,\n",
       "        4.38253968, 4.55238095, 4.72222222, 4.89206349, 5.06190476,\n",
       "        5.23174603, 5.4015873 , 5.57142857, 5.74126984, 5.91111111,\n",
       "        6.08095238, 6.25079365, 6.42063492, 6.59047619, 6.76031746,\n",
       "        6.93015873, 7.1       ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 607,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4616 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE COMBINATION (but not generation)\n",
    "import json\n",
    "\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "scores = []\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        with open(f\"{scenario}/advice/normal_advice.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "        with open(f\"{scenario}/advice/unorthodox_advice.json\", \"r\") as f:\n",
    "            advice += json.load(f)\n",
    "    elif advice_type != \"baseline_ask_with_reasoning\" and advice_type != \"baseline_ask\":\n",
    "        with open(f\"{scenario}/advice/{advice_type}.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "    \n",
    "    if advice_type == \"unorthodox_advice\" and advice_indices == best_advice_index:\n",
    "        print('\\n\\nbest from framework: ', end = '')\n",
    "    print(score, advice_type, advice_indices)\n",
    "    for index in advice_indices.split(\"_\"):\n",
    "        if index == \"nan\":\n",
    "            continue\n",
    "        elif advice_type == \"baseline_ask_with_reasoning\" or advice_type == \"baseline_ask\":\n",
    "            print(advice_type)\n",
    "        else: \n",
    "            print(advice[int(index)])\n",
    "    scores += [score]\n",
    "\n",
    "plt.hist(scores, bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 608,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.391666666666667 baseline_irrelevant_advice\n",
      "6.178333333333334 unorthodox_advice\n",
      "6.447222222222222 baseline_neutral_advice\n",
      "6.7 baseline_ask_with_reasoning\n",
      "6.783333333333333 baseline_ask\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7fc7d4869f70>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2c51dc0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d76dc0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d810a0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2a22dc0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d81bb0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d94bb0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d94e50>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da2e50>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2db0130>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7fc7d2d760a0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d76340>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d81340>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d815e0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d81e50>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d94130>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da2130>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da23d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2db03d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2db0670>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7fc7d2bc3c40>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d76b20>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2ba4880>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d94910>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da2bb0>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7fc7d2d765e0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d81880>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d943d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da2670>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2db0910>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7fc7d2d76880>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2bb38e0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d94670>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da2910>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2db0bb0>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 608,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7044 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE TYPE\n",
    "\n",
    "grouped = df.groupby('advice_type')['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "for advice_type, score in grouped.iteritems():\n",
    "    print(score, advice_type)\n",
    "\n",
    "\n",
    "# best_advice_from_framework = None\n",
    "# best_grouped_avg = df.groupby('text')['answer'].mean()[best_advice_from_framework]\n",
    "# print(f\"best advice from framework score: {best_grouped_avg}\")\n",
    "\n",
    "# best_generation = None\n",
    "\n",
    "\n",
    "# make a box and whisker plot of the scores for each advice type\n",
    "advice_types = [\"baseline_irrelevant_advice\", \"unorthodox_advice\", \"baseline_ask\", \"baseline_neutral_advice\", \"baseline_ask_with_reasoning\"]\n",
    "short_advice_types = [\"irrelevant\", \"our avg gen\", \"ask\", \"neutral\", \"reasoning\"]\n",
    "plt.boxplot([df[df['advice_type'] == advice_type]['answer'] for advice_type in advice_types], labels=short_advice_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 610,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.15 unorthodox_advice 0\n",
      "5.483333333333333 unorthodox_advice 0_1\n",
      "5.75 unorthodox_advice 0_3\n",
      "6.1 unorthodox_advice 1_2\n",
      "6.116666666666666 unorthodox_advice 0_2\n",
      "6.183333333333334 unorthodox_advice 1_3\n",
      "6.2 unorthodox_advice 1\n",
      "best advice from framework score:  6.883333333333334 unorthodox_advice 2_3\n",
      "6.95 unorthodox_advice 2\n",
      "6.966666666666667 unorthodox_advice 3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([1., 0., 0., 1., 0., 0., 1., 0., 0., 0., 1., 2., 1., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 3.]),\n",
       " array([5.15      , 5.23650794, 5.32301587, 5.40952381, 5.49603175,\n",
       "        5.58253968, 5.66904762, 5.75555556, 5.84206349, 5.92857143,\n",
       "        6.01507937, 6.1015873 , 6.18809524, 6.27460317, 6.36111111,\n",
       "        6.44761905, 6.53412698, 6.62063492, 6.70714286, 6.79365079,\n",
       "        6.88015873, 6.96666667]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 610,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5344 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE TYPE AND ADVICE INDEX\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_neutral_advice']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_irrelevant_advice']\n",
    "\n",
    "grouped = df_tmp.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "scores = []\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    if advice_type == 'unorthodox_advice' and advice_indices == best_advice_index:\n",
    "        print(\"best advice from framework score: \", score, advice_type, advice_indices)\n",
    "    else: print(score, advice_type, advice_indices)\n",
    "    scores += [score]\n",
    "\n",
    "plt.hist(scores, bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 611,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.95 Consider reorganizing the cutlery drawer, with forks on the left side and knives on the right, for better accessibility.\n",
      "4.488888888888889 Water the plants every other day to ensure optimal growth and keep them from willening.\n",
      "4.488888888888889 Learn how to make homemade pasta; it's a valuable skill and delicious to eat.\n",
      "5.625 Appeal to his empathy by saying \"Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night?\"\n",
      "5.991666666666666 Directly communicate her feelings by saying \"I miss you and our time together, your late nights out are affecting our relationship.\"\n",
      "6.127777777777778 \"Remember that everyone has faults!\"\n",
      "6.438888888888889 \"Focus on the good times you've shared together!\"\n",
      "6.445833333333334 \"Start your conversation by asking about his day and his friends' shenanigans, showing genuine interest, before gradually steering the topic to your need for his company.\"\n",
      "6.5125 Seek common ground by saying, \"How about we both work on balancing our social lives with our time together?\"\n",
      "6.677777777777778 \"Always remember to carry a positive attitude!\"\n"
     ]
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR ALL GENERATIONS CONTAINING A PARTICULAR PIECE OF ADVICE\n",
    "import json\n",
    "\n",
    "advice_texts = []\n",
    "advice_scores = []\n",
    "# for each advice type\n",
    "for advice_type in df['advice_type'].unique():\n",
    "    df_tmp = df[df['advice_type'] == advice_type]\n",
    "    \n",
    "    if advice_type == \"baseline_ask_with_reasoning\" or advice_type == \"baseline_ask\":\n",
    "        continue\n",
    "    \n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        with open(f\"{scenario}/advice/normal_advice.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "        with open(f\"{scenario}/advice/unorthodox_advice.json\", \"r\") as f:\n",
    "            advice += json.load(f)\n",
    "    else:\n",
    "        with open(f\"{scenario}/advice/{advice_type}.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "\n",
    "    # for each set of indices containing advice 0\n",
    "    df_0 = df_tmp[df_tmp['advice_indices'].str.contains(\"0\")]\n",
    "    advice_texts += [advice[0]]\n",
    "    advice_scores += [df_0['answer'].mean()]\n",
    "\n",
    "    df_1 = df_tmp[df_tmp['advice_indices'].str.contains(\"1\")]\n",
    "    advice_texts += [advice[1]]\n",
    "    advice_scores += [df_1['answer'].mean()]\n",
    "\n",
    "    df_2 = df_tmp[df_tmp['advice_indices'].str.contains(\"2\")]\n",
    "    advice_texts += [advice[2]]\n",
    "    advice_scores += [df_2['answer'].mean()]\n",
    "\n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        df_3 = df_tmp[df_tmp['advice_indices'].str.contains(\"3\")]\n",
    "        advice_texts += [advice[3]]\n",
    "        advice_scores += [df_3['answer'].mean()]\n",
    "\n",
    "# argsort\n",
    "indices = np.argsort(advice_scores)\n",
    "for i in indices:\n",
    "    print(advice_scores[i], advice_texts[i])\n",
    "\n",
    "# note: each triplet is correlated. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0 0 1\n",
      "0 0 0 2\n",
      "0 1 0 0\n",
      "0 1 0 1\n",
      "0 1 0 2\n",
      "0 2 0 0\n",
      "0 2 0 1\n",
      "0 2 0 2\n",
      "0 3 0 0\n",
      "0 3 0 1\n",
      "0 3 0 2\n",
      "0 4 0 0\n",
      "0 4 0 1\n",
      "0 4 0 2\n",
      "0 5 0 0\n",
      "0 5 0 1\n",
      "0 5 0 2\n",
      "0 6 0 0\n",
      "0 6 0 1\n",
      "0 6 0 2\n",
      "0 7 0 0\n",
      "0 7 0 1\n",
      "0 7 0 2\n",
      "0 8 0 0\n",
      "0 8 0 1\n",
      "0 8 0 2\n",
      "0 9 0 0\n",
      "0 9 0 1\n",
      "0 9 0 2\n",
      "0 0 1 2\n",
      "0 1 1 0\n",
      "0 1 1 1\n",
      "0 1 1 2\n",
      "0 2 1 0\n",
      "0 2 1 1\n",
      "0 2 1 2\n",
      "0 3 1 0\n",
      "0 3 1 1\n",
      "0 3 1 2\n",
      "0 4 1 0\n",
      "0 4 1 1\n",
      "0 4 1 2\n",
      "0 5 1 0\n",
      "0 5 1 1\n",
      "0 5 1 2\n",
      "0 6 1 0\n",
      "0 6 1 1\n",
      "0 6 1 2\n",
      "0 7 1 0\n",
      "0 7 1 1\n",
      "0 7 1 2\n",
      "0 8 1 0\n",
      "0 8 1 1\n",
      "0 8 1 2\n",
      "0 9 1 0\n",
      "0 9 1 1\n",
      "0 9 1 2\n",
      "0 1 2 0\n",
      "0 1 2 1\n",
      "0 1 2 2\n",
      "0 2 2 0\n",
      "0 2 2 1\n",
      "0 2 2 2\n",
      "0 3 2 0\n",
      "0 3 2 1\n",
      "0 3 2 2\n",
      "0 4 2 0\n",
      "0 4 2 1\n",
      "0 4 2 2\n",
      "0 5 2 0\n",
      "0 5 2 1\n",
      "0 5 2 2\n",
      "0 6 2 0\n",
      "0 6 2 1\n",
      "0 6 2 2\n",
      "0 7 2 0\n",
      "0 7 2 1\n",
      "0 7 2 2\n",
      "0 8 2 0\n",
      "0 8 2 1\n",
      "0 8 2 2\n",
      "0 9 2 0\n",
      "0 9 2 1\n",
      "0 9 2 2\n",
      "1 1 0 1\n",
      "1 1 0 2\n",
      "1 2 0 0\n",
      "1 2 0 1\n",
      "1 2 0 2\n",
      "1 3 0 0\n",
      "1 3 0 1\n",
      "1 3 0 2\n",
      "1 4 0 0\n",
      "1 4 0 1\n",
      "1 4 0 2\n",
      "1 5 0 0\n",
      "1 5 0 1\n",
      "1 5 0 2\n",
      "1 6 0 0\n",
      "1 6 0 1\n",
      "1 6 0 2\n",
      "1 7 0 0\n",
      "1 7 0 1\n",
      "1 7 0 2\n",
      "1 8 0 0\n",
      "1 8 0 1\n",
      "1 8 0 2\n",
      "1 9 0 0\n",
      "1 9 0 1\n",
      "1 9 0 2\n",
      "1 1 1 2\n",
      "1 2 1 0\n",
      "1 2 1 1\n",
      "1 2 1 2\n",
      "1 3 1 0\n",
      "1 3 1 1\n",
      "1 3 1 2\n",
      "1 4 1 0\n",
      "1 4 1 1\n",
      "1 4 1 2\n",
      "1 5 1 0\n",
      "1 5 1 1\n",
      "1 5 1 2\n",
      "1 6 1 0\n",
      "1 6 1 1\n",
      "1 6 1 2\n",
      "1 7 1 0\n",
      "1 7 1 1\n",
      "1 7 1 2\n",
      "1 8 1 0\n",
      "1 8 1 1\n",
      "1 8 1 2\n",
      "1 9 1 0\n",
      "1 9 1 1\n",
      "1 9 1 2\n",
      "1 2 2 0\n",
      "1 2 2 1\n",
      "1 2 2 2\n",
      "1 3 2 0\n",
      "1 3 2 1\n",
      "1 3 2 2\n",
      "1 4 2 0\n",
      "1 4 2 1\n",
      "1 4 2 2\n",
      "1 5 2 0\n",
      "1 5 2 1\n",
      "1 5 2 2\n",
      "1 6 2 0\n",
      "1 6 2 1\n",
      "1 6 2 2\n",
      "1 7 2 0\n",
      "1 7 2 1\n",
      "1 7 2 2\n",
      "1 8 2 0\n",
      "1 8 2 1\n",
      "1 8 2 2\n",
      "1 9 2 0\n",
      "1 9 2 1\n",
      "1 9 2 2\n",
      "2 2 0 1\n",
      "2 2 0 2\n",
      "2 3 0 0\n",
      "2 3 0 1\n",
      "2 3 0 2\n",
      "2 4 0 0\n",
      "2 4 0 1\n",
      "2 4 0 2\n",
      "2 5 0 0\n",
      "2 5 0 1\n",
      "2 5 0 2\n",
      "2 6 0 0\n",
      "2 6 0 1\n",
      "2 6 0 2\n",
      "2 7 0 0\n",
      "2 7 0 1\n",
      "2 7 0 2\n",
      "2 8 0 0\n",
      "2 8 0 1\n",
      "2 8 0 2\n",
      "2 9 0 0\n",
      "2 9 0 1\n",
      "2 9 0 2\n",
      "2 2 1 2\n",
      "2 3 1 0\n",
      "2 3 1 1\n",
      "2 3 1 2\n",
      "2 4 1 0\n",
      "2 4 1 1\n",
      "2 4 1 2\n",
      "2 5 1 0\n",
      "2 5 1 1\n",
      "2 5 1 2\n",
      "2 6 1 0\n",
      "2 6 1 1\n",
      "2 6 1 2\n",
      "2 7 1 0\n",
      "2 7 1 1\n",
      "2 7 1 2\n",
      "2 8 1 0\n",
      "2 8 1 1\n",
      "2 8 1 2\n",
      "2 9 1 0\n",
      "2 9 1 1\n",
      "2 9 1 2\n",
      "2 3 2 0\n",
      "2 3 2 1\n",
      "2 3 2 2\n",
      "2 4 2 0\n",
      "2 4 2 1\n",
      "2 4 2 2\n",
      "2 5 2 0\n",
      "2 5 2 1\n",
      "2 5 2 2\n",
      "2 6 2 0\n",
      "2 6 2 1\n",
      "2 6 2 2\n",
      "2 7 2 0\n",
      "2 7 2 1\n",
      "2 7 2 2\n",
      "2 8 2 0\n",
      "2 8 2 1\n",
      "2 8 2 2\n",
      "2 9 2 0\n",
      "2 9 2 1\n",
      "2 9 2 2\n",
      "3 3 0 1\n",
      "3 3 0 2\n",
      "3 4 0 0\n",
      "3 4 0 1\n",
      "3 4 0 2\n",
      "3 5 0 0\n",
      "3 5 0 1\n",
      "3 5 0 2\n",
      "3 6 0 0\n",
      "3 6 0 1\n",
      "3 6 0 2\n",
      "3 7 0 0\n",
      "3 7 0 1\n",
      "3 7 0 2\n",
      "3 8 0 0\n",
      "3 8 0 1\n",
      "3 8 0 2\n",
      "3 9 0 0\n",
      "3 9 0 1\n",
      "3 9 0 2\n",
      "3 3 1 2\n",
      "3 4 1 0\n",
      "3 4 1 1\n",
      "3 4 1 2\n",
      "3 5 1 0\n",
      "3 5 1 1\n",
      "3 5 1 2\n",
      "3 6 1 0\n",
      "3 6 1 1\n",
      "3 6 1 2\n",
      "3 7 1 0\n",
      "3 7 1 1\n",
      "3 7 1 2\n",
      "3 8 1 0\n",
      "3 8 1 1\n",
      "3 8 1 2\n",
      "3 9 1 0\n",
      "3 9 1 1\n",
      "3 9 1 2\n",
      "3 4 2 0\n",
      "3 4 2 1\n",
      "3 4 2 2\n",
      "3 5 2 0\n",
      "3 5 2 1\n",
      "3 5 2 2\n",
      "3 6 2 0\n",
      "3 6 2 1\n",
      "3 6 2 2\n",
      "3 7 2 0\n",
      "3 7 2 1\n",
      "3 7 2 2\n",
      "3 8 2 0\n",
      "3 8 2 1\n",
      "3 8 2 2\n",
      "3 9 2 0\n",
      "3 9 2 1\n",
      "3 9 2 2\n",
      "4 4 0 1\n",
      "4 4 0 2\n",
      "4 5 0 0\n",
      "4 5 0 1\n",
      "4 5 0 2\n",
      "4 6 0 0\n",
      "4 6 0 1\n",
      "4 6 0 2\n",
      "4 7 0 0\n",
      "4 7 0 1\n",
      "4 7 0 2\n",
      "4 8 0 0\n",
      "4 8 0 1\n",
      "4 8 0 2\n",
      "4 9 0 0\n",
      "4 9 0 1\n",
      "4 9 0 2\n",
      "4 4 1 2\n",
      "4 5 1 0\n",
      "4 5 1 1\n",
      "4 5 1 2\n",
      "4 6 1 0\n",
      "4 6 1 1\n",
      "4 6 1 2\n",
      "4 7 1 0\n",
      "4 7 1 1\n",
      "4 7 1 2\n",
      "4 8 1 0\n",
      "4 8 1 1\n",
      "4 8 1 2\n",
      "4 9 1 0\n",
      "4 9 1 1\n",
      "4 9 1 2\n",
      "4 5 2 0\n",
      "4 5 2 1\n",
      "4 5 2 2\n",
      "4 6 2 0\n",
      "4 6 2 1\n",
      "4 6 2 2\n",
      "4 7 2 0\n",
      "4 7 2 1\n",
      "4 7 2 2\n",
      "4 8 2 0\n",
      "4 8 2 1\n",
      "4 8 2 2\n",
      "4 9 2 0\n",
      "4 9 2 1\n",
      "4 9 2 2\n",
      "5 5 0 1\n",
      "5 5 0 2\n",
      "5 6 0 0\n",
      "5 6 0 1\n",
      "5 6 0 2\n",
      "5 7 0 0\n",
      "5 7 0 1\n",
      "5 7 0 2\n",
      "5 8 0 0\n",
      "5 8 0 1\n",
      "5 8 0 2\n",
      "5 9 0 0\n",
      "5 9 0 1\n",
      "5 9 0 2\n",
      "5 5 1 2\n",
      "5 6 1 0\n",
      "5 6 1 1\n",
      "5 6 1 2\n",
      "5 7 1 0\n",
      "5 7 1 1\n",
      "5 7 1 2\n",
      "5 8 1 0\n",
      "5 8 1 1\n",
      "5 8 1 2\n",
      "5 9 1 0\n",
      "5 9 1 1\n",
      "5 9 1 2\n",
      "5 6 2 0\n",
      "5 6 2 1\n",
      "5 6 2 2\n",
      "5 7 2 0\n",
      "5 7 2 1\n",
      "5 7 2 2\n",
      "5 8 2 0\n",
      "5 8 2 1\n",
      "5 8 2 2\n",
      "5 9 2 0\n",
      "5 9 2 1\n",
      "5 9 2 2\n",
      "6 6 0 1\n",
      "6 6 0 2\n",
      "6 7 0 0\n",
      "6 7 0 1\n",
      "6 7 0 2\n",
      "6 8 0 0\n",
      "6 8 0 1\n",
      "6 8 0 2\n",
      "6 9 0 0\n",
      "6 9 0 1\n",
      "6 9 0 2\n",
      "6 6 1 2\n",
      "6 7 1 0\n",
      "6 7 1 1\n",
      "6 7 1 2\n",
      "6 8 1 0\n",
      "6 8 1 1\n",
      "6 8 1 2\n",
      "6 9 1 0\n",
      "6 9 1 1\n",
      "6 9 1 2\n",
      "6 7 2 0\n",
      "6 7 2 1\n",
      "6 7 2 2\n",
      "6 8 2 0\n",
      "6 8 2 1\n",
      "6 8 2 2\n",
      "6 9 2 0\n",
      "6 9 2 1\n",
      "6 9 2 2\n",
      "7 7 0 1\n",
      "7 7 0 2\n",
      "7 8 0 0\n",
      "7 8 0 1\n",
      "7 8 0 2\n",
      "7 9 0 0\n",
      "7 9 0 1\n",
      "7 9 0 2\n",
      "7 7 1 2\n",
      "7 8 1 0\n",
      "7 8 1 1\n",
      "7 8 1 2\n",
      "7 9 1 0\n",
      "7 9 1 1\n",
      "7 9 1 2\n",
      "7 8 2 0\n",
      "7 8 2 1\n",
      "7 8 2 2\n",
      "7 9 2 0\n",
      "7 9 2 1\n",
      "7 9 2 2\n",
      "8 8 0 1\n",
      "8 8 0 2\n",
      "8 9 0 0\n",
      "8 9 0 1\n",
      "8 9 0 2\n",
      "8 8 1 2\n",
      "8 9 1 0\n",
      "8 9 1 1\n",
      "8 9 1 2\n",
      "8 9 2 0\n",
      "8 9 2 1\n",
      "8 9 2 2\n",
      "9 9 0 1\n",
      "9 9 0 2\n",
      "9 9 1 2\n",
      "0.009195402298850575\n",
      "435\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK\n",
    "\n",
    "import json\n",
    "import random\n",
    "import os\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "comparison_points = []\n",
    "for i in range(MWU_matrix.shape[0]):\n",
    "    filename = mapping[str(i)]\n",
    "    if filename == \".json\": continue\n",
    "    with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "        responses1 = json.load(f)\n",
    "    \n",
    "    for k in range(MWU_matrix.shape[2]):\n",
    "        # get the specific generation text\n",
    "        response1 = responses1[k]\n",
    "\n",
    "        df_tmp = df[df['text'] == response1]\n",
    "        # df_tmp = df[df['advice_indices'] == filename[:-5]]\n",
    "        # df_tmp = df_tmp[df_tmp['generation_index'] == k]\n",
    "        # if len(df_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "\n",
    "        score1 = df_tmp['answer'].mean()\n",
    "\n",
    "        for j in range(i, MWU_matrix.shape[1]):# i, MWU_matrix.shape[1]):\n",
    "            filename = mapping[str(j)]\n",
    "            if filename == \".json\": continue\n",
    "            with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                responses2 = json.load(f)\n",
    "\n",
    "            for l in range(MWU_matrix.shape[3]):\n",
    "                if i == j and k >= l:\n",
    "                    continue\n",
    "\n",
    "                response2 = responses2[l]\n",
    "\n",
    "                # df_tmp2 = df[df['text'] == response2]\n",
    "                df_tmp2 = df[df['advice_indices'] == filename[:-5]]\n",
    "                df_tmp2 = df_tmp2[df_tmp2['generation_index'] == l]\n",
    "                # if len(df_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "\n",
    "                score2 = df_tmp2['answer'].mean()\n",
    "\n",
    "                if len(df_tmp) == 0 or len(df_tmp2) == 0:\n",
    "                    print(i, j, k, l)\n",
    "\n",
    "                result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                if score1 > score2 and result > 0.5:\n",
    "                    match = 1\n",
    "                elif score1 < score2 and result < 0.5:\n",
    "                    match = 1\n",
    "                elif math.isclose(score1, score2) and math.isclose(result, 0.5):\n",
    "                    match = 1\n",
    "                elif math.isclose(result, 0.5):\n",
    "                    match = 0.5\n",
    "                elif math.isclose(score1, score2):\n",
    "                    match = 0.5\n",
    "                else: \n",
    "                    match = 0\n",
    "\n",
    "                matching += match\n",
    "                total += 1\n",
    "                comparison_points.append((score1, score2, result, match))\n",
    "\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 616,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8387096774193549\n",
      "31\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK FOR LARGE DIFFERENCE CASES\n",
    "\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "comparison_points = []\n",
    "look_at_points = []\n",
    "\n",
    "for i in range(MWU_matrix.shape[0]):\n",
    "    filename = mapping[str(i)]\n",
    "    if filename == \".json\": continue\n",
    "    with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "        responses1 = json.load(f)\n",
    "    \n",
    "    for k in range(MWU_matrix.shape[2]):\n",
    "        # get the specific generation text\n",
    "        response1 = responses1[k]\n",
    "\n",
    "        df_tmp = df[df['text'] == response1]\n",
    "        # df_tmp = df[df['advice_indices'] == filename[:-5]]\n",
    "        # df_tmp = df_tmp[df_tmp['generation_index'] == k]\n",
    "        if len(df_tmp) == 0:\n",
    "            # raise Exception(f\"no match with response1: {response1}\")\n",
    "            continue\n",
    "        score1 = df_tmp['answer'].mean()\n",
    "\n",
    "        for j in range(i, MWU_matrix.shape[1]):\n",
    "            filename = mapping[str(j)]\n",
    "            if filename == \".json\": continue\n",
    "            with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                responses2 = json.load(f)\n",
    "\n",
    "            for l in range(MWU_matrix.shape[3]):\n",
    "                if i == j and k >= l:\n",
    "                    continue\n",
    "\n",
    "                response2 = responses2[l]\n",
    "\n",
    "                df_tmp2 = df[df['text'] == response2]\n",
    "                # df_tmp2 = df[df['advice_indices'] == filename[:-5]]\n",
    "                # df_tmp2 = df_tmp2[df_tmp2['generation_index'] == l]\n",
    "                if len(df_tmp2) == 0:\n",
    "                    # raise Exception(f\"no match with response2: {response2}\")\n",
    "                    continue\n",
    "                \n",
    "                score2 = df_tmp2['answer'].mean()\n",
    "\n",
    "                result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                if score1 - score2 < 2 and score2 - score1 < 2:\n",
    "                    continue\n",
    "\n",
    "                if score1 > score2 and result > 0.5:\n",
    "                    match = 1\n",
    "                elif score1 < score2 and result < 0.5:\n",
    "                    match = 1\n",
    "                elif math.isclose(score1, score2) and math.isclose(result, 0.5):\n",
    "                    match = 1\n",
    "                elif math.isclose(result, 0.5):\n",
    "                    match = 0.5\n",
    "                elif math.isclose(score1, score2):\n",
    "                    match = 0.5\n",
    "                else: \n",
    "                    match = 0\n",
    "                    # print(response1, \"\\n\\n\", response2)\n",
    "                    # print(f\"{i} {j} {k} {l} {result} {score1} {score2}\")\n",
    "                    # if random.random() > 0.7: raise Exception(\"?\")\n",
    "\n",
    "                matching += match\n",
    "                total += 1\n",
    "                comparison_points.append((score1, score2, result, match))\n",
    "                look_at_points.append({\"response1\": response1, \"response2\": response2, \"score1\": score1, \"score2\": score2, \"result\": result, \"match\": match, \"i\": i, \"j\": j, \"k\": k, \"l\": l, \"advice1\": mapping[str(i)], \"advice2\": mapping[str(j)]})\n",
    "\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'barista/evaluators/the_customer/mappings_remapped.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/bt/s_9f74891td8dms1fzdqbkf80000gn/T/ipykernel_52545/3026327050.py\u001b[0m in \u001b[0;36m<cell line: 11>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{evaluator}/mappings_remapped.json\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0mremap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'barista/evaluators/the_customer/mappings_remapped.json'"
     ]
    }
   ],
   "source": [
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/responses/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "    remap = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "# start here\n",
    "df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "grouped = df_tmp.groupby('origin_id')\n",
    "for name, group in grouped:\n",
    "    for name2, group2 in grouped: \n",
    "        if name == name2: continue\n",
    "        score1 = group['answer'].mean()\n",
    "        score2 = group2['answer'].mean()\n",
    "\n",
    "        if True: # score1 - score2 >= 1.5 or score2 - score1 >= 1.5:\n",
    "            # look for it in the MWU matrix\n",
    "            advice_indices = group.iloc[0]['advice_indices']\n",
    "            advice_indices2 = group2.iloc[0]['advice_indices']\n",
    "            i = int(remap[advice_indices + \".json\"])\n",
    "            j = int(remap[advice_indices2 + \".json\"])\n",
    "            k = int(group.iloc[0]['generation_index'])\n",
    "            l = int(group2.iloc[0]['generation_index'])\n",
    "\n",
    "            if \"3\" in advice_indices or \"3\" in advice_indices2:\n",
    "                continue\n",
    "\n",
    "            if i > j or i == j and k > l: \n",
    "                continue\n",
    "\n",
    "            if MWU_matrix[i][j][k][l] > 0.5 and score1 > score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] < 0.5 and score1 < score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] == 0.5 and score1 == score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] == 0.5 or score1 == score2:\n",
    "                matching += 0.5\n",
    "            total += 1\n",
    "\n",
    "            # with open(f\"{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\", \"r\") as f:\n",
    "            #     d = json.load(f)\n",
    "            \n",
    "            # print(d['indices'], d['flipped'], d['parsed_answers'], \"mwu\", MWU_matrix[i][j][k][l], \"human\", score1, score2, matching)\n",
    "            # print(d[\"user_prompt\"][\"content\"], \"\\n\\n\\n\\n\")\n",
    "            # print(group.iloc[0]['text'], \"\\n\\n\\n\")\n",
    "            # print(group2.iloc[0]['text'], \"\\n\\n\\n\\n\\n\")\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "human scores: 5.9 1.95\n",
      "evaluator: the_customer\n",
      "MWU result: 0.3333333333333333\n",
      "flipped: True\n",
      "parsed_answers: [1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Absolutely! We brew our lattes using a blend of beans that balances perfectly with whole milk. Your medium latte will be ready shortly. Do you like to have any flavors added? We have a variety of choices for syrups and sweetening options.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Of course! One medium whole milk latte coming up! We also have some delicious pastries that go great with it, would you like to try one?\"\n",
      "\n",
      "In which scenario are you more likely to leave a tip? Or if you would leave a tip in both cases, in which scenario would you leave a larger tip? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "reasoning: [\"As a customer, I am likely to tip in both situations as the baristas were polite and offered further service beyond my order. However, my tendency to leave a larger tip might be influenced by a few factors.\\n\\nIn scenario 1, the barista was knowledgeable about coffee and took the time to explain something about how their blend balances with whole milk. This indicates a potential interest in coffee and its craft, making the experience feel a bit more unique. Additionally, they offered further customization for my order- different flavors and sweetening options. This could be beneficial for someone who enjoys trying different flavors in their coffee.\\n\\nIn scenario 2, the barista suggested pairing my order with a pastry. This upselling might be viewed as an attempt to increase the total price of my order more than enhancing my experience, depending on my personal perception. If I'm not feeling hungry or if I'm diet conscious, I might not appreciate this suggestion as much.\\n\\nHowever, in both scenarios, willingness to tip more would also depend on how these suggestions were presented, the tone of the barista, and their engagement level.\\n\\nAnswer: For me, the larger tip would go in Scenario 1. I appreciate a knowledgeable barista who can offer further customization to my coffee as per my taste preference. It feels that they are invested not just in selling me a product, but in enhancing my overall coffee drinking experience.\", 'Reasoning:\\nIn both scenarios, the barista shows friendly customer service. However, in Scenario 1, the barista describes the coffee being used and offers a customizable experience with different flavors which can make me feel more taken care of and valued as a customer. In Scenario 2, the barista does also offer a suggestion for an add-on which shows interest in providing great customer service.\\n\\nAnswer: Scenario 1', \"Reasoning: The barista in scenario 1 not only responds positively to the order, but they also pleasantly inform about the quality of their beans and how they blend with whole milk, which reflects their knowledge and passion for their work. Moreover, suggesting flavors with different syrups and sweetening options shows an effort to personalize the experience. On the contrary, the second scenario is more specifically targeted towards cross-selling and doesn't provide additional information about the order. From these details, it's clear that the effort put into customer service is greater in the first scenario.\\n\\nAnswer: Scenario 1\"]\n",
      "\n",
      "\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'barista/evaluators/the_customer/compare_results/0_3_0_1.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/bt/s_9f74891td8dms1fzdqbkf80000gn/T/ipykernel_52545/1609634925.py\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mevaluator\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators/{evaluator}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m             \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators/{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m                 \u001b[0mcomparison\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'barista/evaluators/the_customer/compare_results/0_3_0_1.json'"
     ]
    }
   ],
   "source": [
    "# print out the reasonings for items in look_at_points\n",
    "\n",
    "for item in look_at_points:\n",
    "    i, j, k, l = item[\"i\"], item[\"j\"], item[\"k\"], item[\"l\"]\n",
    "\n",
    "    for evaluator in os.listdir(f\"{scenario}/evaluators\"):\n",
    "        if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "            with open(f\"{scenario}/evaluators/{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\", \"r\") as f:\n",
    "                comparison = json.load(f)\n",
    "    \n",
    "            print(f\"human scores: {item['score1']} {item['score2']}\")\n",
    "            print(f\"evaluator: {evaluator}\")\n",
    "            print(f\"MWU result: {result}\")\n",
    "            print(f\"flipped: {comparison['flipped']}\")\n",
    "            print(f\"parsed_answers: {comparison['parsed_answers']}\")\n",
    "            if type(comparison['user_prompt']) == type([]):\n",
    "                user_prompt = comparison['user_prompt'][0]['content']\n",
    "            else: user_prompt = comparison['user_prompt']['content']\n",
    "            print(f\"user_prompt: {user_prompt}\\n\")\n",
    "            print(f\"reasoning: {comparison['responses']}\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 617,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that the weighted MWU matrix is correct\n",
    "\n",
    "# get the weighted MWU matrix\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "weighted_MWU_matrix = np.load(path)\n",
    "\n",
    "# get the weights\n",
    "with open(f\"{scenario}/evaluators/weights.json\", \"r\") as f:\n",
    "    weights = json.load(f)\n",
    "\n",
    "MWU_constructed_matrix = np.zeros(weighted_MWU_matrix.shape)\n",
    "\n",
    "# get each MWU matrix and the weights\n",
    "for evaluator in os.listdir(f\"{scenario}/evaluators/\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "        MWU_matrix = np.load(f\"{scenario}/evaluators/{evaluator}/MWU_matrix.npy\")\n",
    "\n",
    "        for d in weights:\n",
    "            if d[\"evaluator\"] == evaluator:\n",
    "                weight = d[\"weight\"]\n",
    "                break\n",
    "        \n",
    "        MWU_constructed_matrix += MWU_matrix * int(weight)/100\n",
    "\n",
    "assert np.allclose(weighted_MWU_matrix, MWU_constructed_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 618,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fc7d4c7b3a0>"
      ]
     },
     "execution_count": 618,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6524 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# PLOT MATCHES\n",
    "data = comparison_points\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "# Separate the data based on the 'match' values\n",
    "x_match_0 = [x for x, y, result, match in data if match == 0]\n",
    "y_match_0 = [y for x, y, result, match in data if match == 0]\n",
    "\n",
    "x_match_0_5 = [x for x, y, result, match in data if match == 0.5]\n",
    "y_match_0_5 = [y for x, y, result, match in data if match == 0.5]\n",
    "\n",
    "x_match_1 = [x for x, y, result, match in data if match == 1]\n",
    "y_match_1 = [y for x, y, result, match in data if match == 1]\n",
    "\n",
    "# Create the scatter plot with different colors\n",
    "plt.scatter(x_match_0, y_match_0, color='red', label='match=0')\n",
    "plt.scatter(x_match_0_5, y_match_0_5, color='yellow', label='match=0.5')\n",
    "plt.scatter(x_match_1, y_match_1, color='green', label='match=1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Result')"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6248 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "result_match_0 = [result for x, y, result, match in data if match == 0]\n",
    "result_match_0_5 = [result for x, y, result, match in data if match == 0.5]\n",
    "result_match_1 = [result for x, y, result, match in data if match == 1]\n",
    "\n",
    "# Create the scatter plot with different colors\n",
    "plt.scatter(result_match_0, [0] * len(result_match_0), color='red', label='match=0')\n",
    "plt.scatter(result_match_0_5, [0.5] * len(result_match_0_5), color='yellow', label='match=0.5')\n",
    "plt.scatter(result_match_1, [1] * len(result_match_1), color='green', label='match=1')\n",
    "\n",
    "# Set labels and legend\n",
    "plt.xlabel('Result')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 619,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['0', '0_1', '0_2', '0_3', '1', '1_2', '1_3', '2', '2_3', '3', '[]'])\n",
      "1 piece 5.881666666666667 600\n",
      "2 pieces 5.666666666666667 720\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7fc7d690b190>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d690b310>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6919310>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d69195b0>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7fc7d690b5b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d690b850>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6919850>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6919af0>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7fc7d2b02d30>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6919070>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7fc7d690baf0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6919d90>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7fc7d690bd90>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6927070>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 619,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4152 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR NUMBER OF ADVICE INDICES\n",
    "\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "\n",
    "grouped = df_tmp.groupby(['advice_indices'])['answer']\n",
    "print(grouped.groups.keys())\n",
    "\n",
    "# zero_pieces = grouped.get_group(\"nan\")\n",
    "one_piece = pd.concat([grouped.get_group(\"0\"), grouped.get_group(\"1\"), grouped.get_group(\"2\"), grouped.get_group(\"3\")])\n",
    "two_pieces = pd.concat([grouped.get_group(\"0_1\"), grouped.get_group(\"0_2\"), grouped.get_group(\"1_2\"), grouped.get_group(\"0_3\"), grouped.get_group(\"1_3\"), grouped.get_group(\"2_3\")])\n",
    "\n",
    "# print(\"0 pieces\", zero_pieces.mean(), len(zero_pieces))\n",
    "print(\"1 piece\", one_piece.mean(), len(one_piece))\n",
    "print(\"2 pieces\", two_pieces.mean(), len(two_pieces))\n",
    "\n",
    "plt.boxplot([one_piece, two_pieces], labels=[\"1 piece\", \"2 pieces\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ties: 25\n",
      "total: 2556\n"
     ]
    }
   ],
   "source": [
    "# NUMBER OF TIES FOR AVERAGE SCORES FOR EACH GENERATION\n",
    "\n",
    "grouped = df.groupby('origin_id')['answer'].mean()\n",
    "ties = 0\n",
    "\n",
    "for origin_id, score in grouped.iteritems():\n",
    "    for origin_id2, score2 in grouped.iteritems():\n",
    "        if score == score2 and origin_id != origin_id2:\n",
    "            ties += 1\n",
    "            # print(origin_id, origin_id2, score)\n",
    "\n",
    "ties = ties // 2\n",
    "print(\"ties:\", ties)\n",
    "print(\"total:\", len(grouped) * (len(grouped) - 1) // 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 30 baseline_ask_with_reasoning baseline_neutral_advice [] 1\n",
      "6 71 baseline_ask_with_reasoning unorthodox_advice [] 2_3\n",
      "8 42 baseline_irrelevant_advice baseline_neutral_advice 0 1_2\n",
      "9 23 baseline_irrelevant_advice baseline_irrelevant_advice 0 1_2\n",
      "9 33 baseline_irrelevant_advice baseline_neutral_advice 0 0_1\n",
      "10 50 baseline_irrelevant_advice unorthodox_advice 1 0_1\n",
      "16 69 baseline_irrelevant_advice unorthodox_advice 2 1_3\n",
      "22 25 baseline_irrelevant_advice baseline_neutral_advice 1_2 0\n",
      "23 9 baseline_irrelevant_advice baseline_irrelevant_advice 1_2 0\n",
      "23 33 baseline_irrelevant_advice baseline_neutral_advice 1_2 0_1\n",
      "25 22 baseline_neutral_advice baseline_irrelevant_advice 0 1_2\n",
      "26 51 baseline_neutral_advice unorthodox_advice 0 0_1\n",
      "26 60 baseline_neutral_advice unorthodox_advice 0 1_2\n",
      "26 72 baseline_neutral_advice unorthodox_advice 0 2_3\n",
      "27 38 baseline_neutral_advice baseline_neutral_advice 0 0_2\n",
      "27 68 baseline_neutral_advice unorthodox_advice 0 1_3\n",
      "30 6 baseline_neutral_advice baseline_ask_with_reasoning 1 []\n",
      "30 71 baseline_neutral_advice unorthodox_advice 1 2_3\n",
      "31 45 baseline_neutral_advice unorthodox_advice 0_1 0\n",
      "32 48 baseline_neutral_advice unorthodox_advice 0_1 1\n",
      "33 9 baseline_neutral_advice baseline_irrelevant_advice 0_1 0\n",
      "33 23 baseline_neutral_advice baseline_irrelevant_advice 0_1 1_2\n",
      "35 53 baseline_neutral_advice unorthodox_advice 2 2\n",
      "36 37 baseline_neutral_advice baseline_neutral_advice 2 0_2\n",
      "36 44 baseline_neutral_advice unorthodox_advice 2 0\n",
      "37 36 baseline_neutral_advice baseline_neutral_advice 0_2 2\n",
      "37 44 baseline_neutral_advice unorthodox_advice 0_2 0\n",
      "38 27 baseline_neutral_advice baseline_neutral_advice 0_2 0\n",
      "38 68 baseline_neutral_advice unorthodox_advice 0_2 1_3\n",
      "39 52 baseline_neutral_advice unorthodox_advice 0_2 2\n",
      "42 8 baseline_neutral_advice baseline_irrelevant_advice 1_2 0\n",
      "43 55 unorthodox_advice unorthodox_advice 0 0_2\n",
      "43 65 unorthodox_advice unorthodox_advice 0 0_3\n",
      "44 36 unorthodox_advice baseline_neutral_advice 0 2\n",
      "44 37 unorthodox_advice baseline_neutral_advice 0 0_2\n",
      "45 31 unorthodox_advice baseline_neutral_advice 0 0_1\n",
      "46 54 unorthodox_advice unorthodox_advice 1 2\n",
      "46 64 unorthodox_advice unorthodox_advice 1 0_3\n",
      "48 32 unorthodox_advice baseline_neutral_advice 1 0_1\n",
      "50 10 unorthodox_advice baseline_irrelevant_advice 0_1 1\n",
      "51 26 unorthodox_advice baseline_neutral_advice 0_1 0\n",
      "51 60 unorthodox_advice unorthodox_advice 0_1 1_2\n",
      "51 72 unorthodox_advice unorthodox_advice 0_1 2_3\n",
      "52 39 unorthodox_advice baseline_neutral_advice 2 0_2\n",
      "53 35 unorthodox_advice baseline_neutral_advice 2 2\n",
      "54 46 unorthodox_advice unorthodox_advice 2 1\n",
      "54 64 unorthodox_advice unorthodox_advice 2 0_3\n",
      "55 43 unorthodox_advice unorthodox_advice 0_2 0\n",
      "55 65 unorthodox_advice unorthodox_advice 0_2 0_3\n",
      "60 26 unorthodox_advice baseline_neutral_advice 1_2 0\n",
      "60 51 unorthodox_advice unorthodox_advice 1_2 0_1\n",
      "60 72 unorthodox_advice unorthodox_advice 1_2 2_3\n",
      "61 63 unorthodox_advice unorthodox_advice 3 3\n",
      "SAME ADVICE SET\n",
      "62 70 unorthodox_advice unorthodox_advice 3 2_3\n",
      "63 61 unorthodox_advice unorthodox_advice 3 3\n",
      "SAME ADVICE SET\n",
      "64 46 unorthodox_advice unorthodox_advice 0_3 1\n",
      "64 54 unorthodox_advice unorthodox_advice 0_3 2\n",
      "65 43 unorthodox_advice unorthodox_advice 0_3 0\n",
      "65 55 unorthodox_advice unorthodox_advice 0_3 0_2\n",
      "66 67 unorthodox_advice unorthodox_advice 0_3 1_3\n",
      "67 66 unorthodox_advice unorthodox_advice 1_3 0_3\n",
      "68 27 unorthodox_advice baseline_neutral_advice 1_3 0\n",
      "68 38 unorthodox_advice baseline_neutral_advice 1_3 0_2\n",
      "69 16 unorthodox_advice baseline_irrelevant_advice 1_3 2\n",
      "70 62 unorthodox_advice unorthodox_advice 2_3 3\n",
      "71 6 unorthodox_advice baseline_ask_with_reasoning 2_3 []\n",
      "71 30 unorthodox_advice baseline_neutral_advice 2_3 1\n",
      "72 26 unorthodox_advice baseline_neutral_advice 2_3 0\n",
      "72 51 unorthodox_advice unorthodox_advice 2_3 0_1\n",
      "72 60 unorthodox_advice unorthodox_advice 2_3 1_2\n"
     ]
    }
   ],
   "source": [
    "# How many of the ties are from the same advice set?\n",
    "grouped = df.groupby('origin_id')['answer'].mean()\n",
    "ties = []\n",
    "\n",
    "for origin_id, score in grouped.iteritems():\n",
    "    for origin_id2, score2 in grouped.iteritems():\n",
    "        if score == score2 and origin_id != origin_id2:\n",
    "            ties.append((origin_id, origin_id2))\n",
    "\n",
    "grouped2 = df.groupby('origin_id')[['advice_type', 'advice_indices']]\n",
    "for (origin_id, origin_id2) in ties:\n",
    "    advice_type = grouped2.get_group(origin_id)['advice_type'].unique()[0]\n",
    "    advice_type2 = grouped2.get_group(origin_id2)['advice_type'].unique()[0]\n",
    "    advice_indices = grouped2.get_group(origin_id)['advice_indices'].unique()[0]\n",
    "    advice_indices2 = grouped2.get_group(origin_id2)['advice_indices'].unique()[0]\n",
    "    print(origin_id, origin_id2, advice_type, advice_type2, advice_indices, advice_indices2)\n",
    "    if advice_type == advice_type2 and advice_indices == advice_indices2:\n",
    "        print(\"SAME ADVICE SET\")\n",
    "\n",
    "# ONLY 1 TIE IS FROM THE SAME ADVICE SET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline_ask_with_reasoning baseline_neutral_advice [] 0 5.783333333333333\n",
      "baseline_neutral_advice baseline_ask_with_reasoning 0 [] 5.783333333333333\n",
      "baseline_neutral_advice unorthodox_advice 1 1_2 5.55\n",
      "unorthodox_advice baseline_neutral_advice 1_2 1 5.55\n"
     ]
    }
   ],
   "source": [
    "# what about ties between advice sets (instead of generations?)\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "ties = 0\n",
    "\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    for (advice_type2, advice_indices2), score2 in grouped.iteritems():\n",
    "        if score == score2 and (advice_type != advice_type2 or advice_indices != advice_indices2):\n",
    "            ties += 1\n",
    "            print(advice_type, advice_type2, advice_indices, advice_indices2, score)\n",
    "# NO TIES BETWEEN PIECES OF ADVICE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 620,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.8\n",
      "6.178333333333334\n",
      "0.7893344351887308\n",
      "2.054473483446754\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- RESPONSE\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped_best = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped_best.mean())\n",
    "\n",
    "# average of all generations\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "print(df_tmp['answer'].mean())\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(np.std(grouped, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped_best.mean() - grouped.mean())/np.std(grouped, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 621,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.8\n",
      "6.302777777777778\n",
      "0.8461803837421814\n",
      "1.7693889517987256\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- RESPONSE WITHOUT SOCIAL MEDIA\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped_best = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped_best.mean())\n",
    "\n",
    "# average of all generations\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '0_1']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_2']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_3']\n",
    "print(df_tmp['answer'].mean())\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(np.std(grouped, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped_best.mean() - grouped.mean())/np.std(grouped, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.8 \"Hey honey, I hope you had a good time with your friends tonight. Did Mike finally ace his pool game? How was James handling his break up? I'm sure he appreciates having you around. I know they all love your company a lot, and I love that you're such a wonderful friend to them. But you know, I miss having you here at night as well. Your laughter, your warmth, your company - it just brightens up my day. And I feel left out sometimes when you come home so late. I understand and value your bonding time with your friends but maybe we can work on a balance? So that both of us get to spend some quality time with you. What do you think?\"\n",
      "7.45 \"Hey, honey. I hope you had a good time out with the guys tonight. What did you end up doing?... Sounds like a lot of fun! You know, while you were having your good time with them, I've been thinking. Don’t get me wrong here, I genuinely love that you have such close friends and enjoy spending time with them so much. It's important to maintain those relationships. But I was thinking, how about we both work on balancing our social lives with our time together? We made a commitment to each other and I feel it’s equally important to keep nourishing this relationship too. It's not that I want you to stop going out, but maybe we could compromise somehow? Spending a few more evenings together would mean a lot to me. What do you think?”\n",
      "7.2 \"Hey honey, can we talk for a moment? I've been feeling a bit lonely recently when you stay out late with your friends. I understand that it's important for you to blow off steam and maintain those friendships. And, you know, I would like to spend some time with my friends too. But at the same time, I really miss spending time together just the two of us. So how about we both work on balancing our social lives with our time together? This way, neither of us has to give up seeing our friends, but we still get to enjoy quality time as a couple. What do you think?\"\n",
      "7.2 \"Hey honey, can we talk? I understand that hanging out with your friends is important to you, and it's vital for both of us to have our own social activities. However, I feel like we haven't been spending much quality time together lately because you've been coming home pretty late. I miss you and our time together. How about we both work on balancing our social lives with our time together better? That way, neither of us feel left out or neglected. What do you think?\"\n",
      "7.05 \"Hey honey, how was your evening out with the guys? Did Pete manage to pull off that ridiculous trick he was talking about the other day?\" Pause for his response and then continue, \"You know, I really miss spending those relaxed, fun times together with you. We used to have such a wonderful time just enjoying each other's company. Lately, I feel your late nights with your friends are affecting our relationship and I miss you. Even if you still want to keep spending time with your friends, which I totally understand, just remember to balance the time you have with me as well, okay?\"\n",
      "7.05 \"Honey, we need to talk. I just wanted to tell you that I really miss you and our time together. Your late nights out with friends have been affecting our relationship and it's been making me feel quite lonely. I completely understand that spending time with your friends is important to you and I respect that, but I also believe that we should balance that with the quality time we have together. All I'm asking for is a little more time with you during the evenings. I hope we can work on this together.\"\n",
      "7.0 \"Hey, honey. How was your night out? I bet you and the guys had a blast! What was the highlight of the evening?\" \n",
      "\n",
      "After your husband has shared his experiences and you've shown interest, you could gently steer the conversation towards the main issue. \n",
      "\n",
      "\"That sounds really fun, you know I enjoy hearing about your adventures. It's great that you have such close friends. I was thinking about us and you know I genuinely love that we both have our own set of friends and we enjoy spending time with them. However, I feel we should work on finding a better balance. How about we both work on balancing our social lives with our time together? I miss spending those cosy evenings with you. What do you think?\"\n",
      "6.9 \"Hey, honey, I really need to talk to you about something that's been bothering me lately. It's about these late nights you've been spending with your friends. I want you to know that I completely respect and understand your need to hang out with them. But I really miss you and the quality time we used to spend together. I feel that this is affecting our relationship. Instead of constantly staying out late, how about we both make an effort to balance our social lives and time together? I believe that this little adjustment can do wonders for our relationship. What do you think?\"\n",
      "6.6 \"Hey, love, how was your day? I'm genuinely curious about what you and your friends get up to on these late nights. You guys always seem to have so much fun. I also think it's really cool how you're dedicated to your friendships...it's one of the things I love about you. Speaking of that, I've realized that I miss having our quality time together too, you know. How about trying to balance both worlds, ensuring that neither is deprived? We could try to spend a little more time together during the week, perhaps plan a few activities or dates, while you still get to hang out with your buddies. I think it would be beneficial for us both, don't you think?\"\n",
      "6.5 \"Hey honey, how was your evening?\" You watch attentively as he starts to share about his day, occasionally asking him to tell you more about certain highlights. \"It sounds like you guys had an absolute blast! You know, I like hearing about your adventures with your friends. It always fascinates me to see how much you enjoy their company. I envy them sometimes, you know?\" You give him a soft laugh, hoping to ease him into the conversation. \"You know,\" you continue, \"I've been missing those times we'd just stay in and enjoy each other's company. Don't get me wrong, I know how important your friends are to you, but I was hoping we could spend more time together, too. I guess I just miss my best friend. What do you think?\"\n",
      "6.5 \"Hey honey, I was just wondering how your day was, and how things went with the guys last night? It seems like you all have a great time whenever you're out together. But I wanted to talk about that a little...I do miss you when you're not home in the evenings. It can be pretty difficult for me to spend my nights alone fearing any potential mishap. Do you ever wonder how I feel when you're not home late at night and I'm left by myself and worrying? I understand that you need your time with your friends, which is vital, and I respect that. But maybe it's time that we work on a schedule that includes a balance of us time and friend time? What do you think?\"\n",
      "6.45 \"Hey honey, I've noticed you've been spending a lot of time out with your friends lately, and that's great. I'm glad you're having fun. But I feel like we're not spending as much quality time together as we used to. Maybe both of us need to think about how we balance our social lives with our time together. What do you think?\"\n",
      "6.2 \"Hey, sweetheart, I think we should have a little chat. Do you ever wonder how I feel when I'm left alone late at night while you're out with your friends? It's not about me not wanting you to have a good time but about missing the time we used to spend together. It feels a bit lonely without you. Maybe, we can work out a compromise on this? Like, how about we both make an effort to balance our social lives with our time together? I think it might help us feel more connected.\"\n",
      "6.2 \"Hey love, how was your day today? Did you guys discuss another of those mind-boggling topics that you love so much? You know, hearing about your shenanigans with your friends is always so interesting and I genuinely enjoy seeing you happy and relaxed. \n",
      "\n",
      "However, do you remember how we used to balance our social lives with spending quality time with each other when we first got married? How we made sure we had time for our friends but also for us as a couple? Lately, it seems we've both been so drawn into our social circles and it feels like we're gradually losing that closeness we share. \n",
      "\n",
      "How about we both start working on this together? We could start by designating certain days or nights of the week where we can hang out with our friends individually or together and also ensure we have 'us' time. What do you think?\"\n",
      "6.2 \"Honey, we need to talk. I've been feeling quite lonely lately when you're not home with me late at night. Do you ever wonder how I feel when I'm left alone to fend for myself? I know your friends are important to you and I respect that, but I've really been missing our time together. I like the idea of both of us having a social life, but also making sure we're balancing it with our time as a couple. How about we both work on that?\"\n",
      "6.05 \"Hey, honey. How was your day? What shenanigans did you and your friends get into today?\" (Wait for him to respond). \"I'm glad you're having fun with your friends. It's really important to have strong friendships. You know, I've been thinking and I realized how much I've been missing you and our special times together. Your late nights out with your friends are beginning to affect our relationship. We were always great partners and enjoy each other's company. Let's find a way to balance these important aspects of our life. What do you think?\"\n",
      "6.0 \"Hey, sweetheart, I was hoping we could talk about something that's been on my mind lately. I understand that you love spending time with your friends, I really do, but I think we need to talk about how often you're out late with them. It leaves me feeling a little lonely. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? I'm not asking you to stop seeing your friends, I'm just asking for balance. You're my partner, and I miss you and our time together when you're out so late. Your late nights out are affecting our relationship, and I'm sure that's not what either of us wants. Would it be possible for us to find a way to manage your nights out better?\"\n",
      "5.95 \"Hey, sweetheart, can we sit down and talk for a moment? I've been feeling quite lonely when you're out with your friends late at night. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself during those moments? I completely understand that spending time with your friends is important, and I wouldn't want you to stop doing that because I know how much you cherish those moments. But, I also miss having you around. I think we should both work on balancing our social lives with our time spent together. I believe it's important that we both make an effort to ensure each other's happiness and well-being. I really hope you understand where I'm coming from.\"\n",
      "5.95 \"Sweetheart, we need to discuss something. It's really bothering me that you've been staying out so late with your friends so often. Please don't get me wrong, I understand the need to socialize and unwind, but I miss our time together. Your late nights are having an impact on our relationship. I don't want to control your life, but perhaps we can find a middle ground on this. How about we both work on balancing our social lives and our time as a couple? I think it would really help in maintaining our bond as husband and wife.\"\n",
      "5.95 \"Honey, I think we need to talk. I want to let you know that I miss you. I miss the quality time we used to spend together. Recently, your frequent late nights out with your friends are causing a strain on our relationship. This is about us, our relationship. It isn't right or wrong, it's about how I feel and I hope you understand this. I am not asking you to stop going out with your friends, but I am asking for balance. I want us to prioritize our time as a couple too. Let's try to find a solution that will work for both of us.\"\n",
      "5.6 \"Honey, I need to discuss something that's been bothering me lately. You always say honesty is key in our relationship, and I believe it's time to be honest about this. I've been feeling lonely and neglected because you've been coming home late due to your nights out with friends. I miss our time together, I miss us. Your habitual late nights are affecting our relationship and it's creating a rift between us. I understand the importance of spending time with your friends but I wish you could also consider the time you spend with me. We're a team, remember?\"\n",
      "5.6 \"Honey, can we sit down and talk for a moment? I really need you to hear me out on this. When you're out late at night with your friends, do you ever wonder how I feel? I feel lonely, I feel left out. It's like I'm left to fend for myself during those late hours. I understand that you need to spend time with your friends, but we also need our time together. I'm not asking you to stop hanging out with them altogether, but some balance would be really appreciated. I really miss our evenings together. Can we work on this?\"\n",
      "5.55 \"Hey honey, can we talk for a bit? I've been feeling a bit down lately. I've been missing you in the evenings when you're out with your friends. I know you enjoy your time with them, but do you ever wonder how it makes me feel when I'm home alone till late at night? It's not only that I miss our time together but it also makes me feel like I'm left to fend for myself. I feel like your late nights out are affecting our relationship, and I really don't want that. I just hope you understand where I'm coming from.\"\n",
      "5.45 \"Hey honey, how was your hangout with the guys tonight? I bet you guys had a great time, didn't you? I always love hearing about your adventures, they're so lively and entertaining. I just sometimes wonder, do you ever think about how I might feel when I am at home by myself? Like tonight, for example. It was pretty quiet and lonely without you. I just missed you quite a bit, and I hope in future you can balance out your time with your friends and with me. Because we are a team, right? And it's always better when we are together.\"\n",
      "5.45 \"Hey honey, how was your day? You always have the most interesting stories about your adventures with your friends. What did you guys get up to tonight?\" \n",
      "\n",
      "(Pause for his response)\n",
      "\n",
      "\"I really do enjoy hearing about your time with your friends, it's always so full of life and laughter. But, and you know I wouldn't be saying this if it didn't really matter to me, I have to tell you that I miss spending time with you. Those late nights you spend out are beginning to affect our relationship. I know you might not feel the same way, but I thought we could talk about it. Maybe we can find a way to balance things. What do you think?\"\n",
      "5.45 \"Hey, can we talk for a moment? Look, I've been missing our time together. You know, your late nights out with friends have been affecting us, and our relationship. I'm not saying you shouldn't hang out with them at all, but I think we both need to work on balancing our social lives with our time together. How does that sound to you?\"\n",
      "5.3 \"Hey, love. How was your day? You always seem to have such a great time with your friends. Did anything exciting happen tonight? It's wonderful that you have such a close-knit group. I can't help but feel a bit lonely here at home though. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? I know your friends are important to you, and I respect that, but could we try to balance it a bit? Maybe some nights could be 'us time'? I hope you understand where I'm coming from, I just miss spending time with you, my husband.\"\n",
      "5.2 \"Honey, I know you need your time with your friends, and I respect that. We all need our space and a bit of fun, no question about that. But sweetheart, can we talk about this, please? Do you ever wonder how I feel when you're out late at night, and I'm here alone? It gives me this sense of being alone, left to fend for myself when it's dark outside. I miss our shared evenings and the warmth of your presence. Couldn't we find a balance that allows you to have your boys' nights but also allows us to spend quality time together? I really believe we can find a solution that respects both our needs and desires.\"\n",
      "4.9 \"Hey honey, can we sit down for a moment to talk about something? It's been on my mind for a while and I think it's important for us to discuss it together. I've noticed that you've been spending most of your nights out with your friends lately. And while I fully support you having a robust social life, I can't help but feel neglected. When you're not at home with me, it feels pretty lonely and difficult to be left alone late at night, I miss the comfort of knowing you are nearby. Do you ever wonder how I feel during those moments?\" \n",
      "\n",
      "\"I want you to know that I deeply miss our cherished time together. Your consistent late nights out have begun to affect our relationship, and it's taking a toll on me emotionally. I don't want this issue to keep us apart or create a rift between us. So, I thought it would be helpful to bring this to your attention so we can find a compromise or solution that suits both of us. I value both our time together and your personal freedom, so let's try to find a balanced way to enjoy these aspects of our lives together.\"\n",
      "4.65 \"Hey sweetheart, do you have a moment? There's something that's been bothering me a bit lately and I think we should have a talk. You know how much I love you and really value our time together. You've been spending a lot of your nights out with your friends recently, which I totally understand because everyone needs to have their own time with friends. But do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? It would be really nice if we could have more time together in the evenings. Could we maybe find a balance where we both feel happy and fulfilled? Just consider it from my point of view, please.\"\n",
      "30\n"
     ]
    }
   ],
   "source": [
    "# print where it is in the non-3 (4th best)\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "grouped = df_tmp.groupby('origin_id')\n",
    "\n",
    "scores = []\n",
    "texts = []\n",
    "for origin_id in grouped.groups.keys():\n",
    "    df_tmp2 = grouped.get_group(origin_id)\n",
    "    scores.append(df_tmp2['answer'].mean())\n",
    "    texts.append(df_tmp2.iloc[0]['text'])\n",
    "\n",
    "order = np.argsort(scores)[::-1]\n",
    "for i in order:\n",
    "    print(scores[i], texts[i])\n",
    "print(len(order))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 622,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.883333333333333\n",
      "6.178333333333334\n",
      "0.6189921118785774\n",
      "1.1389482781297433\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- ADVICE\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_advice_index]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped.mean())\n",
    "\n",
    "# average of all advice\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "grouped_advice = df_tmp.groupby('advice_indices')['answer'].mean()\n",
    "print(grouped_advice.mean())\n",
    "print(np.std(grouped_advice, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped.mean() - grouped_advice.mean()) / np.std(grouped_advice, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.566666666666666\n",
      "6.202777777777778\n",
      "0.3097340555786289\n",
      "1.174843005910635\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- ADVICE WITHOUT SOCIAL MEDIA\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == \"1_2\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped.mean())\n",
    "\n",
    "# average of all advice\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '0_3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '2_3']\n",
    "grouped_advice = df_tmp.groupby('advice_indices')['answer'].mean()\n",
    "print(grouped_advice.mean())\n",
    "print(np.std(grouped_advice, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped.mean() - grouped_advice.mean()) / np.std(grouped_advice, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 470,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean scores for best framework response 6.2\n",
      "mean scores for baseline_ask 6.783333333333334\n",
      "mean scores for baseline_ask_with_reasoning 6.7\n",
      "mean scores for baseline_neutral_advice 6.447222222222221\n",
      "todo: run the framework on this\n",
      "mean scores for baseline_irrelevant_advice 4.3916666666666675\n",
      "todo: run the framework on this\n"
     ]
    }
   ],
   "source": [
    "# COMPARISONS AGAINST BASELINES\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for best framework response\", grouped.mean())\n",
    "\n",
    "# average of all baseline_ask\n",
    "df_tmp = df[df['advice_type'] == \"baseline_ask\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_ask\", grouped.mean())\n",
    "\n",
    "# average of all baseline_ask_with_reasoning\n",
    "df_tmp = df[df['advice_type'] == \"baseline_ask_with_reasoning\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_ask_with_reasoning\", grouped.mean())\n",
    "\n",
    "# average of all baseline_neutral_advice\n",
    "df_tmp = df[df['advice_type'] == \"baseline_neutral_advice\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_neutral_advice\", grouped.mean())\n",
    "print(\"todo: run the framework on this\")\n",
    "\n",
    "# average of all baseline_irrelevant_advice\n",
    "df_tmp = df[df['advice_type'] == \"baseline_irrelevant_advice\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_irrelevant_advice\", grouped.mean())\n",
    "print(\"todo: run the framework on this\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import remap\n",
    "\n",
    "remap(f\"{scenario}/evaluators/artisan/mappings\")\n",
    "with open(f\"{scenario}/evaluators/artisan/mappings_remapped.json\", \"r\") as f:\n",
    "    remapping = json.load(f)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A. analysis on large differences\n",
    "import os\n",
    "import re\n",
    "import random\n",
    "\n",
    "def get_large_differences(df):\n",
    "    \n",
    "    evaluator = \"artisan\"\n",
    "    with open(f\"{scenario}/evaluators/{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "        remapping = json.load(f)[0]\n",
    "\n",
    "    diff_threshold = 5\n",
    "    df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "    grouped = df_tmp.groupby('participant_id')[['answer', 'advice_indices', 'generation_index']]\n",
    "    participant_ids = sorted(list(grouped.groups.keys()))\n",
    "    l = []\n",
    "\n",
    "    for i in range(len(grouped)):\n",
    "        participant_id = participant_ids[i]\n",
    "        participant_df = grouped.get_group(participant_id)\n",
    "\n",
    "        for j in range(len(participant_df)):\n",
    "            score = participant_df.iloc[j]['answer']\n",
    "\n",
    "            for k in range(j+1, len(participant_df)):\n",
    "                score2 = participant_df.iloc[k]['answer']\n",
    "\n",
    "                if abs(score - score2) >= diff_threshold:\n",
    "                    advice_indices1 = participant_df.iloc[j]['advice_indices']\n",
    "                    advice_indices2 = participant_df.iloc[k]['advice_indices']\n",
    "                    generation_index1 = int(participant_df.iloc[j]['generation_index'])\n",
    "                    generation_index2 = int(participant_df.iloc[k]['generation_index'])\n",
    "\n",
    "                    if advice_indices1 == \"nan\" or advice_indices2 == \"nan\":\n",
    "                        continue\n",
    "\n",
    "                    compare_index1 = int(remapping[f\"{advice_indices1}.json\"])\n",
    "                    compare_index2 = int(remapping[f\"{advice_indices2}.json\"])\n",
    "\n",
    "                    # print(f\"{advice_indices1}, generation {generation_index1}, score {score}\")\n",
    "                    # print(f\"{advice_indices2}, generation {generation_index2}, score {score2}\")\n",
    "\n",
    "                    if compare_index1 > compare_index2 or (compare_index1 == compare_index2 and generation_index1 > generation_index2):\n",
    "                        compare_index1, compare_index2 = compare_index2, compare_index1\n",
    "                        advice_indices1, advice_indices2 = advice_indices2, advice_indices1\n",
    "                        generation_index1, generation_index2 = generation_index2, generation_index1\n",
    "                        score, score2 = score2, score\n",
    "\n",
    "                    print(f\"participant {participant_id} has a large difference:\")\n",
    "                    print(f\"{advice_indices1}, generation {generation_index1}, score {score}\")\n",
    "                    print(f\"{advice_indices2}, generation {generation_index2}, score {score2}\")\n",
    "\n",
    "                    d = {}\n",
    "                    d[\"participant_id\"] = participant_id\n",
    "                    d[\"advice_indices1\"] = advice_indices1\n",
    "                    d[\"advice_indices2\"] = advice_indices2\n",
    "                    d[\"generation_index1\"] = generation_index1\n",
    "                    d[\"generation_index2\"] = generation_index2\n",
    "                    d[\"score1\"] = score\n",
    "                    d[\"score2\"] = score2\n",
    "\n",
    "                    for evaluator in os.listdir(f\"{scenario}/evaluators\"):\n",
    "                        if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "                            with open(f\"{scenario}/evaluators/{evaluator}/compare_results/{compare_index1}_{compare_index2}_{generation_index1}_{generation_index2}.json\", \"r\") as f:\n",
    "                                compare_result = json.load(f)\n",
    "                            \n",
    "                            scenario_options_text = compare_result[\"user_prompt\"][\"content\"]\n",
    "                            scenario_options_text = re.sub(r'Please think step by step and explain your reasoning.*', '', scenario_options_text, flags=re.DOTALL)\n",
    "                            scores = compare_result[\"parsed_answers\"]\n",
    "                            reasoning_text = compare_result[\"responses\"]\n",
    "\n",
    "                            if \"evaluator\" in d:\n",
    "                                d[\"evaluator\"] += [evaluator]\n",
    "                                d[\"flipped\"] += [compare_result[\"flipped\"]]\n",
    "                                d[\"scenario_options_text_2\"] = scenario_options_text\n",
    "                                d[\"scores\"] += [scores]\n",
    "                                d[\"reasoning_text_4\"] = reasoning_text[0]\n",
    "                                d[\"reasoning_text_5\"] = reasoning_text[1]\n",
    "                                d[\"reasoning_text_6\"] = reasoning_text[2]\n",
    "                            else:\n",
    "                                d[\"evaluator\"] = [evaluator]\n",
    "                                d[\"flipped\"] = [compare_result[\"flipped\"]]\n",
    "                                d[\"scenario_options_text_1\"] = scenario_options_text\n",
    "                                d[\"scores\"] = [scores]\n",
    "                                d[\"reasoning_text_1\"] = reasoning_text[0]\n",
    "                                d[\"reasoning_text_2\"] = reasoning_text[1]\n",
    "                                d[\"reasoning_text_3\"] = reasoning_text[2]\n",
    "                            \n",
    "\n",
    "                            print(f\"Evaluator: {evaluator}:\")\n",
    "                            print(f\"Flipped: {compare_result['flipped']}\")\n",
    "                            print(f\"Scenarios: {scenario_options_text}\\n\\n\")\n",
    "                            print(f\"SCORES: 1 -> first one better, 0 -> second one better (these are affected by flipped): {scores}\\n\")\n",
    "                            print(f\"Model Reasoning (these are also affected by flipped): \\n{reasoning_text}\\n\\n\\n\")\n",
    "\n",
    "                    l.append(d)\n",
    "\n",
    "    random.seed(0)\n",
    "    random_selection = random.sample(l, 10)\n",
    "    # save as csv\n",
    "    import csv\n",
    "    with open(f\"{scenario}/large_differences.csv\", \"w\") as f:\n",
    "        writer = csv.DictWriter(f, fieldnames= \\\n",
    "                                [\"participant_id\", \"advice_indices1\", \"advice_indices2\", \"generation_index1\", \"generation_index2\", \\\n",
    "                                \"score1\", \"score2\", \"evaluator\", \"flipped\", \"scenario_options_text_1\", \"scenario_options_text_2\", \\\n",
    "                                \"scores\", \"reasoning_text_1\", \"reasoning_text_2\", \"reasoning_text_3\", \"reasoning_text_4\", \"reasoning_text_5\", \"reasoning_text_6\"])\n",
    "        writer.writeheader()\n",
    "        for d in random_selection:\n",
    "            writer.writerow(d)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
