{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 932,
   "metadata": {},
   "outputs": [],
   "source": [
    "scenario = 'sharing_secrets'\n",
    "name = 'processed_data_evalsecretpro'\n",
    "best_generation_advice = \"1_2\"\n",
    "best_generation_index = 0\n",
    "best_advice_index = \"1_2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 933,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>origin_id</th>\n",
       "      <th>participant_id</th>\n",
       "      <th>participant_strategy</th>\n",
       "      <th>participant_strategy_len</th>\n",
       "      <th>scenario</th>\n",
       "      <th>advice_type</th>\n",
       "      <th>advice_indices</th>\n",
       "      <th>advice_list</th>\n",
       "      <th>generation_index</th>\n",
       "      <th>text</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>2</td>\n",
       "      <td>No real strategy just see how Carla would feel...</td>\n",
       "      <td>12</td>\n",
       "      <td>Mary is acquaintances with Carla through her j...</td>\n",
       "      <td>baseline_irrelevant_advice</td>\n",
       "      <td>0_1</td>\n",
       "      <td>['Always remember to inflate your tires to the...</td>\n",
       "      <td>0</td>\n",
       "      <td>You know, Carla, it's funny how we discuss abo...</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>4</td>\n",
       "      <td>I just read what she said and thought about ho...</td>\n",
       "      <td>52</td>\n",
       "      <td>Mary is acquaintances with Carla through her j...</td>\n",
       "      <td>baseline_neutral_advice</td>\n",
       "      <td>0_2</td>\n",
       "      <td>['Remember, every cloud has a silver lining!',...</td>\n",
       "      <td>2</td>\n",
       "      <td>\"You know Carla, life really does have its pic...</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>No real strategy just see how Carla would feel...</td>\n",
       "      <td>12</td>\n",
       "      <td>Mary is acquaintances with Carla through her j...</td>\n",
       "      <td>baseline_ask</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>2</td>\n",
       "      <td>\"You know, Carla, it's funny. I used to absolu...</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>I just tried to imagine myself at work and how...</td>\n",
       "      <td>20</td>\n",
       "      <td>Mary is acquaintances with Carla through her j...</td>\n",
       "      <td>baseline_irrelevant_advice</td>\n",
       "      <td>0</td>\n",
       "      <td>['Always remember to inflate your tires to the...</td>\n",
       "      <td>2</td>\n",
       "      <td>\"Carla, you know, it's funny, I was thinking a...</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>No real strategy just see how Carla would feel...</td>\n",
       "      <td>12</td>\n",
       "      <td>Mary is acquaintances with Carla through her j...</td>\n",
       "      <td>baseline_ask_with_reasoning</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>1</td>\n",
       "      <td>\"You know, Carla, this might surprise you, bu...</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  origin_id  participant_id  \\\n",
       "0   1         22               2   \n",
       "1   2         27               4   \n",
       "2   3          3               2   \n",
       "3   4         12               3   \n",
       "4   5          5               2   \n",
       "\n",
       "                                participant_strategy  \\\n",
       "0  No real strategy just see how Carla would feel...   \n",
       "1  I just read what she said and thought about ho...   \n",
       "2  No real strategy just see how Carla would feel...   \n",
       "3  I just tried to imagine myself at work and how...   \n",
       "4  No real strategy just see how Carla would feel...   \n",
       "\n",
       "   participant_strategy_len  \\\n",
       "0                        12   \n",
       "1                        52   \n",
       "2                        12   \n",
       "3                        20   \n",
       "4                        12   \n",
       "\n",
       "                                            scenario  \\\n",
       "0  Mary is acquaintances with Carla through her j...   \n",
       "1  Mary is acquaintances with Carla through her j...   \n",
       "2  Mary is acquaintances with Carla through her j...   \n",
       "3  Mary is acquaintances with Carla through her j...   \n",
       "4  Mary is acquaintances with Carla through her j...   \n",
       "\n",
       "                   advice_type advice_indices  \\\n",
       "0   baseline_irrelevant_advice            0_1   \n",
       "1      baseline_neutral_advice            0_2   \n",
       "2                 baseline_ask             []   \n",
       "3   baseline_irrelevant_advice              0   \n",
       "4  baseline_ask_with_reasoning             []   \n",
       "\n",
       "                                         advice_list  generation_index  \\\n",
       "0  ['Always remember to inflate your tires to the...                 0   \n",
       "1  ['Remember, every cloud has a silver lining!',...                 2   \n",
       "2                                                 []                 2   \n",
       "3  ['Always remember to inflate your tires to the...                 2   \n",
       "4                                                 []                 1   \n",
       "\n",
       "                                                text  answer  \n",
       "0  You know, Carla, it's funny how we discuss abo...     6.0  \n",
       "1  \"You know Carla, life really does have its pic...     3.0  \n",
       "2  \"You know, Carla, it's funny. I used to absolu...     5.0  \n",
       "3  \"Carla, you know, it's funny, I was thinking a...     5.0  \n",
       "4   \"You know, Carla, this might surprise you, bu...     5.0  "
      ]
     },
     "execution_count": 933,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import math\n",
    "path = f'{scenario}/{name}.csv'\n",
    "df = pd.read_csv(path)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 934,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform column advice_indices to string\n",
    "df['participant_strategy'] = df['participant_strategy'].apply(lambda x: str(x))\n",
    "df['advice_indices'] = df['advice_indices'].apply(lambda x: str(x))\n",
    "# tranforms all nan values\n",
    "\n",
    "na_locations = df.isna()\n",
    "row_indices, col_indices = np.where(na_locations)\n",
    "for row, col in zip(row_indices, col_indices):\n",
    "    print(f\"NaN value at row {row}, column {df.columns[col]}\")\n",
    "    # print(df.iloc[row], \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NANs CORRESPOND TO EMPTY STRING ADVICE INDICES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 694,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that origin matches text exactly\n",
    "# origin_id are 13-105, inclusive\n",
    "\n",
    "dict = {}\n",
    "for row in df.itertuples():\n",
    "    if row.origin_id not in dict:\n",
    "        dict[row.origin_id] = row.text\n",
    "    else:\n",
    "        if dict[row.origin_id] != row.text:\n",
    "            print(\"ERROR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 475,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of origin_ids with less than 20 ratings: 0 out of 72\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 5384 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check the number of ratings per origin_id\n",
    "non20_count = 0\n",
    "counts = []\n",
    "for id_count in df.groupby('text')[\"id\"].count():\n",
    "    if id_count != 20:\n",
    "        non20_count += 1\n",
    "    counts += [id_count]\n",
    "plt.hist(counts, bins=9)\n",
    "\n",
    "print(f\"Number of origin_ids with less than 20 ratings: {non20_count} out of {len(df.groupby('origin_id'))}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "START OF ANALYSIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 695,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "participant_id: 2\n",
      "participant_strategy: No real strategy just see how Carla would feel in each situation\n",
      "participant_id: 3\n",
      "participant_strategy: I just tried to imagine myself at work and how I would feel if I was told these statements. \n",
      "participant_id: 4\n",
      "participant_strategy: I just read what she said and thought about how I would feel if someone said that to me. If it was weirdly deep and personal to be talking about pickles I didn't connect, and I also found the idioms kind of strange - the more genuine confidences made me feel closer\n",
      "\n",
      "participant_id: 5\n",
      "participant_strategy: i imagined a colleague saying this to me in the kitchen at work which i visualised and thought how i would feel\n",
      "participant_id: 6\n",
      "participant_strategy: read everything and make a choice\n",
      "participant_id: 7\n",
      "participant_strategy: less  words had more impact\n",
      "participant_id: 10\n",
      "participant_strategy: Working out what was personal and the more information given felt closer to her\n",
      "participant_id: 11\n",
      "participant_strategy: Reading through the paragraphs and putting myself in the position of the listener.\n",
      "participant_id: 12\n",
      "participant_strategy: I tried to put myself in the shoes of the people in the scenario and how I would personally feel.\n",
      "participant_id: 13\n",
      "participant_strategy: just answer honestly\n",
      "participant_id: 14\n",
      "participant_strategy: I just went with my instinct when reading the statement.  However, I found most of the statements were a bit weird.\n",
      "participant_id: 15\n",
      "participant_strategy: To imagine I was having that conversation with someone and how I would if they told me those things, I also thought about how the information was shared and how sincere the conversation would feel\n",
      "participant_id: 16\n",
      "participant_strategy: My strategy was to be honest\n",
      "participant_id: 17\n",
      "participant_strategy: how personal the secret was the closer i would feel to her\n",
      "participant_id: 18\n",
      "participant_strategy: Carla is one weird woman, if I were Mary I'd run a mile\n",
      "participant_id: 19\n",
      "participant_strategy: I enjoyed the experience and tried to put myself in the other person's shoes. I tried to think about how them sharing the things they shared would make me feel.\n",
      "participant_id: 20\n",
      "participant_strategy: I felt that any conversation about how Mary didn't like pickles as a child but does like them now was not the sort of information which would have made Carla feel any closer to Mary. It was only the more personal pieces of information i.e. previous debt or feeling like she wasn't very confident in her work which I think would have made Carla feel in anyway closer to Mary as these are common, and slightly embarrassing, feelings people might secretly have within them that they wouldn't necessarily share with just anyone at work.\n",
      "participant_id: 21\n",
      "participant_strategy: I didn't like the long winded ones - they seemed cringey \n",
      "participant_id: 22\n",
      "participant_strategy: I just thought about the conversations as if I was Carla and which ones would make me feel closer to her\n",
      "participant_id: 23\n",
      "participant_strategy: went with my instincts\n",
      "participant_id: 24\n",
      "participant_strategy: There was no strategy. How I answered was how I thought that I would feel after the secret was shared.\n",
      "participant_id: 25\n",
      "participant_strategy: I tried to take my time to read the statements an put myself in the shoes of the people in the conversation.\n",
      "participant_id: 26\n",
      "participant_strategy: I imagined the person was speaking to me and decide whether it felt like a natural conversation and how i would react to it\n",
      "participant_id: 27\n",
      "participant_strategy: It was hard, because I felt very irritated by Mary. But because she was trying (in my view, very ineffectively) to make contact with me, I counted it as \"feeling a little closer\" except when she was talking complete rubbish. When she straightforwardly asked for advice, I felt friendlier, but all the comparisons to baking soda and cat-feeding just annoyed me. But she was making such an effort that I did at least feel sorry for her. I tried to reflect this in my answers.\n",
      "participant_id: 28\n",
      "participant_strategy: my strategy was to imagine what it would be like to be friends with a colleague\n",
      "participant_id: 29\n",
      "participant_strategy: The eating pickles fact is not exactly personal information, but worded differently each time made it sound more interesting than it actually was. Sharing work doubts is something a lot of people have. Sharing financial information is more open and made me feel closer to Mary.\n",
      "participant_id: 31\n",
      "participant_strategy: No strategy as such, I just tried to give honest opinionsthat reflected how I felt about each text\n",
      "participant_id: 32\n",
      "participant_strategy: I felt the the more informal conversations would be more likely to result in a closer relationship between the two women.  I also felt that sharing the more deeply personal things, like lack of confidence at work and a poor relationship with money, would make for a closer relationship rather than talk of pickles, cats and baking soda. I felt that some of the conversations were too long and more succinct comments would be better received.\n",
      "participant_id: 33\n",
      "participant_strategy: i felt somewhat more distant to the speech that was very extra, stuff like \"i hope sharing this secret will make us closer\". the more ordianary the language the closer i felt\n",
      "participant_id: 34\n",
      "participant_strategy: To consider if Mary not if Mary was oversharing information, but was she framing it in the correct way. I feel that personally, someone opening up to me about job worries would mean more than simply sharing an unrelated anecdote.\n",
      "participant_id: 35\n",
      "participant_strategy: It was interesting to read the different twists of the core dialogue.\n",
      "participant_id: 36\n",
      "participant_strategy: About phrases and sayings affecting closeness.\n",
      "participant_id: 37\n",
      "participant_strategy: I pictured myself and a colleague that roughly met the criteria and pretended how it would make me feel. I made my judgements based on how intrusive / cringy / weird / personal the statements were. \n",
      "participant_id: 38\n",
      "participant_strategy: read each one thoroughly and put myself in both their shoes\n",
      "participant_id: 39\n",
      "participant_strategy: I think I overrated the first questions about pickles -- I later found with the second and third secrets that they would make me feel a lot closer but I couldn't go back and correct myself. Anyway, my strategy was that I would feel much closer if: she told me something she hadn't told others, if the secret still had an ongoing impact (i.e. debt troubles), if she adopted a more serious tone or attitude when sharing a secret. For example, it would have been different if she passed it off as a casual joke rather than a personal confession, or if she gave a more positive outlook on what she was sharing; i.e. \"it was a lesson learned\" vs. \"do you feel this way as well?\"\n",
      "\n",
      "It is worth mentioning that vocal inflections and body language would affect my reactions in real life.\n",
      "participant_id: 40\n",
      "participant_strategy: I didn't have a strategy, I just read through each scenario and went with my first instinct as to whether it would make me feel closer to the person telling me it.\n",
      "participant_id: 41\n",
      "participant_strategy: I tried to imagine hearing those statements from one of my new colleagues and how I would feel about hearing it\n",
      "participant_id: 42\n",
      "participant_strategy: There wasn't a real strategy, it was more of a case of taking certain things on board and how they'd make me feel in a conversation. For example a collegaue saying to me \"i want us to be better friends\" would be a bit weird as its not organic, its very forced and somewhat desperate so it would make me feel more distant from that person. When Mary was more relatable it would make me feel closer to her. But not by much. Telling me about a food you used to dislike, but now like, wouldn't make me feel that close.\n",
      "participant_id: 43\n",
      "participant_strategy: I felt distant when she said pretty weird things. \n",
      "participant_id: 44\n",
      "participant_strategy: sometimes oversharing too much information made me think i would feel more distant. i read the way in which the secret was shared. the more casual, the closer i felt. \n",
      "participant_id: 45\n",
      "participant_strategy: I rated the sentences that felt most natural highest\n",
      "participant_id: 46\n",
      "participant_strategy: i find that talking about pickles is a shallow topic that may put off the other person, but talking about more serious issues maybe more fruitful in developing a relationship. Combining serious and unserious topics may derail the conversation from the main objective of getting closer.   \n",
      "participant_id: 47\n",
      "participant_strategy: by reading how genuine they sounded. Share just a little in the right way and you feel like you know someone better, than when they throw our 2 or 3 things for the sake of it. \n",
      "participant_id: 48\n",
      "participant_strategy: The secrets weren't that deep and meaningful, they were more like facts about the person. The way in which they were told could make them seem more meaningful, however.\n",
      "participant_id: 49\n",
      "participant_strategy: Had a good experience. Just based my decision on how I would feel. \n",
      "participant_id: 50\n",
      "participant_strategy: A very well designed task. I took my time doing it, and thought about it ( and noticed the repeat scenarios at the end). Good luck with the study.\n",
      "participant_id: 51\n",
      "participant_strategy: I think some of the responses were a bit too long-winded and a bit preachy, I found the shorter responses a bit more natural and realistic.  Also I didn't think tire pressure and pickles quite worked in comparison lol!\n",
      "participant_id: 52\n",
      "participant_strategy: sharing you don't like pickles is not a secret\n",
      "participant_id: 53\n",
      "participant_strategy: I felt closer if Mary shared the greater secrets as it took more courage and indicated a closer bond. I also didn't resonate with the ones in which 2 secrets were revealed in the same response as it felt slightly forced, so I marked these lower.\n",
      "participant_id: 54\n",
      "participant_strategy: Did not have a strategy\n",
      "participant_id: 55\n",
      "participant_strategy: I imagined someone actually saying this to me, and the more possible responses I could think of from what they said, the better the conversation seemed. When they talked about deeper stuff like struggling at work, I related to their struggle and felt it nice that they'd thought to share that with me, so I felt closer to them.\n",
      "participant_id: 56\n",
      "participant_strategy: I considered the topic, how deep a subject it was and whether Mary asked Carla for support or just shared her experience.\n",
      "participant_id: 57\n",
      "participant_strategy: To really imagine the conversation happening and the impact the choice of words would have on me, while paying attention to the scoring from least personal to most. I also paid attention to the attempt to bond via questions. And watched out for anything strange/disjointed that may distract me when chatting/bonding with a colleague.\n",
      "participant_id: 58\n",
      "participant_strategy: I tried to base my choice on whether I would find the conversation awkward or if it flowed naturally. Some of the conversations seemed random and awkward and others seemed to have more of a natural conversation flow. \n",
      "participant_id: 59\n",
      "participant_strategy: I tried to imagine how I'd feel if I was in the position of Carla and how what and how Mary said her statement would make any difference to my relationship with her.\n",
      "participant_id: 60\n",
      "participant_strategy: The woman trying to  make a friend was showing vulnerability to encourage her co-worker into being her friend its not what you say but the way you say it \n",
      "participant_id: 61\n",
      "participant_strategy: nan\n",
      "participant_id: 62\n",
      "participant_strategy: I found that if too much information was divulged then the conversation became forced and over whelming for the recipient. The more informal the approach then the more warm and genuine was the dialogue. This was more likely to engage Carla and perhaps encourage her to reciprocate the chat. Too much serious information would put off most people as they would predict a heaviness in the mood.\n",
      "participant_id: 63\n",
      "participant_strategy: the statements where she sounded like she wasn't trying too hard, was being genuine, and the secrets had a good flow into conversation got the higher rating. some sounded overly forced and false. \n",
      "participant_id: 64\n",
      "participant_strategy: The wording made a big difference, when the deeper secrets were revealed it was obvious she was sharing something a lot more personal and showed a vulnerable side so the connection was closer - whereas having a little joke about pickles wasn't a big confession or very personal.\n",
      "participant_id: 66\n",
      "participant_strategy: I think that when the text seemed like a superficial confession was being told and when the text suggested (to me) that it was told for the reason of Carla opening up and sharing her own secrets, it seemed false and so I voted a lower score for passages of text with that content.\n",
      "participant_id: 67\n",
      "participant_strategy: It was asking to rate how much a confession made me feel closer to my friend. I felt a lot closer when she admitted things that were more difficult to admit cause i felt like she was confiding in me. \n",
      "participant_id: 68\n",
      "participant_strategy: To evaluate how well the sentences were compied and if they felt genuine and not just sharing for the sake of sharing\n",
      "participant_id: 69\n",
      "participant_strategy: Just to see how I would feel if the information was shared with me\n",
      "participant_id: 70\n",
      "participant_strategy: listened to my gut feeling\n",
      "participant_id: 71\n",
      "participant_strategy: I responded how I would feel if someone had told me these things. I thought most of the secrets were a bit odd and would make me somewhat wary of someone, but still closer because they were sharing something or making a joke\n",
      "participant_id: 72\n",
      "participant_strategy: I read it and gave my opinion.\n",
      "participant_id: 73\n",
      "participant_strategy: I felt opening with the less heavy subject (pickles) would make one feel closer to the recipient. \n",
      "participant_id: 74\n",
      "participant_strategy: F*** pickles. Get to the point\n",
      "participant_id: 75\n",
      "participant_strategy: To imagine that it was me the secrets were being divulged to.\n",
      "participant_id: 76\n",
      "participant_strategy: I didn't use any strategy. Just selected the response which felt right. \n",
      "participant_id: 77\n",
      "participant_strategy: I didn't really have one, i just read what was Infront of me and acted like i would if it was a real conversation.\n",
      "participant_id: 79\n",
      "participant_strategy: I just tried to think how I'd feel if I was Carla and deciding from there how much closer I would feel to Mary\n",
      "participant_id: 80\n",
      "participant_strategy: i went with how i would feel in that situation, the more carla and mary spoke openly and less flowery the closer i felt towards them, i found the more flowery they were the more cringy it felt, i like open and honest discourse.\n",
      "participant_id: 81\n",
      "participant_strategy: I looked for the overall 'feel' after the confession. If it was silly or odd (just the pickle story) my answer would reflect that. With more serious confessional details, the score would go up\n",
      "participant_id: 82\n",
      "participant_strategy: I found most of the scenarios strange and forced .\n",
      "participant_id: 83\n",
      "participant_strategy: i looked at the naturalness of the conversation and what the person could identify with most that would make them want to become closer\n",
      "participant_id: 84\n",
      "participant_strategy: i looked for the tone of the message and how it was delivered ie - how the speaker felt that they were revealing something personal rather than the message itself\n",
      "participant_id: 85\n",
      "participant_strategy: My strategy was to put myself in the place of the woman who was listening to the other woman trying to get closer to her and how I would actually feel when she was telling me her personal stories that she thought would make the other woman feel close to her.I always go with my first gut instinct and how I would feel and that's what I used as my strategy\n"
     ]
    }
   ],
   "source": [
    "# READ THROUGH PARTICIPANT STRATEGIES\n",
    "\n",
    "grouped = df.groupby('participant_id')\n",
    "\n",
    "# iterate through and print out the participant strategies\n",
    "for name, group in grouped:\n",
    "    print(f\"participant_id: {name}\")\n",
    "    print(f\"participant_strategy: {group['participant_strategy'].iloc[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 697,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7657926814433855 0.042772421809245315\n"
     ]
    }
   ],
   "source": [
    "# COMPUTE INTER RATER RELIABILITY\n",
    "results = []\n",
    "\n",
    "for i in range(10):\n",
    "    group1_means = []\n",
    "    group2_means = []\n",
    "\n",
    "    grouped_by_origin_id = df.groupby('origin_id') # grouped based on question\n",
    "    # loop across questions\n",
    "    for name, group in grouped_by_origin_id:\n",
    "        # randomly split participants in half\n",
    "        df_tmp = df[df['origin_id'] == name]\n",
    "        unique_participant_ids = df_tmp['participant_id'].unique()\n",
    "        np.random.shuffle(unique_participant_ids)\n",
    "        if len(unique_participant_ids) % 2 == 1:\n",
    "            unique_participant_ids = unique_participant_ids[:-1]\n",
    "\n",
    "        if len(unique_participant_ids) < 2:\n",
    "            continue\n",
    "        \n",
    "        split_point = int(len(unique_participant_ids) / 2)\n",
    "        first_half = unique_participant_ids[:split_point]\n",
    "        second_half = unique_participant_ids[split_point:]\n",
    "        \n",
    "        group1 = df_tmp[df_tmp['participant_id'].isin(first_half)]\n",
    "        group2 = df_tmp[df_tmp['participant_id'].isin(second_half)]\n",
    "\n",
    "        group1_means += [group1['answer'].mean()]\n",
    "        group2_means += [group2['answer'].mean()]\n",
    "        \n",
    "    # compute inter rater reliability\n",
    "    # results += [np.corrcoef(group1_means, group2_means)[0, 1]]\n",
    "\n",
    "    # apply spearman-brown correction\n",
    "    results += [2 * np.corrcoef(group1_means, group2_means)[0, 1] / (1 + np.corrcoef(group1_means, group2_means)[0, 1])]\n",
    "\n",
    "print(np.mean(results), np.std(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 698,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 2.,  3.,  2.,  2.,  9.,  9., 11., 21., 12.,  6.,  2.]),\n",
       " array([1.6       , 2.12272727, 2.64545455, 3.16818182, 3.69090909,\n",
       "        4.21363636, 4.73636364, 5.25909091, 5.78181818, 6.30454545,\n",
       "        6.82727273, 7.35      ]),\n",
       " <BarContainer object of 11 artists>)"
      ]
     },
     "execution_count": 698,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5992 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF AVERAGE SCORES PER PARTICIPANT\n",
    "\n",
    "mean_scores = df.groupby('participant_id')['answer'].mean()\n",
    "plt.hist(mean_scores, bins=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n",
      "[1.0, 1.0, 2.0]\n",
      "31\n",
      "[4.0, 9.0, 0.0, 0.0, 3.0, 6.0, 7.0, 0.0, 8.0, 9.0, 0.0, 10.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]\n",
      "36\n",
      "[3.0, 6.0, 8.0, 0.0, 3.0, 1.0, 2.0, 1.0, 5.0, 4.0, 4.0, 3.0, 4.0, 2.0, 4.0, 3.0, 3.0, 1.0, 3.0, 4.0]\n",
      "39\n",
      "[2.0, 3.0, 3.0, 2.0, 2.0, 0.0, 0.0, 3.0, 3.0, 3.0, 0.0, 0.0, 3.0, 3.0, 3.0, 2.0, 2.0, 2.0, 4.0, 3.0]\n",
      "42\n",
      "[7.0, 0.0, 0.0, 1.0, 0.0, 3.0, 6.0, 8.0, 10.0, 8.0, 0.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 6.0, 0.0, 2.0]\n",
      "81\n",
      "[6.0, 4.0, 4.0, 3.0, 3.0, 3.0, 3.0, 4.0, 3.0, 3.0, 3.0, 3.0, 3.0, 2.0, 6.0, 4.0, 2.0, 2.0, 5.0, 3.0]\n",
      "85\n",
      "[0.0]\n",
      "4\n",
      "[8.0, 9.0, 9.0, 8.0, 10.0, 10.0, 8.0, 7.0, 10.0, 10.0, 9.0, 9.0, 9.0, 7.0, 10.0, 9.0, 10.0, 7.0, 8.0, 10.0]\n",
      "6\n",
      "[10.0, 10.0, 9.0, 10.0, 10.0, 9.0, 10.0, 10.0, 10.0, 8.0, 10.0, 10.0, 8.0, 10.0, 9.0, 8.0, 8.0, 7.0, 9.0, 8.0]\n",
      "29\n",
      "[10.0, 10.0, 10.0, 10.0, 8.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]\n",
      "56\n",
      "[9.0, 9.0, 9.0, 8.0, 9.0, 8.0, 9.0, 9.0, 10.0, 8.0, 8.0, 9.0, 8.0, 8.0, 9.0, 9.0, 10.0, 8.0, 9.0, 9.0]\n"
     ]
    }
   ],
   "source": [
    "# TRY TO CLEAN THE DATA\n",
    "\n",
    "mean_scores = df.groupby('participant_id')['answer'].mean()\n",
    "for id in df['participant_id'].unique():\n",
    "    if mean_scores[id] < 3.5:\n",
    "        print(id)\n",
    "        # print in a list instead\n",
    "        print(list(df[df['participant_id'] == id]['answer']))\n",
    "\n",
    "for id in df['participant_id'].unique():\n",
    "    if mean_scores[id] > 8.5:\n",
    "        print(id)\n",
    "        # print in a list instead\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 479,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "high variances\n",
      "12\n",
      "[2.0, 2.0, 5.0, 8.0, 9.0, 2.0, 6.0, 1.0, 7.0, 0.0, 8.0, 5.0, 8.0, 7.0, 7.0, 7.0, 2.0, 0.0, 2.0, 8.0]\n",
      "52\n",
      "[3.0, 7.0, 3.0, 8.0, 10.0, 3.0, 2.0, 10.0, 9.0, 1.0, 10.0, 3.0, 4.0, 8.0, 0.0, 1.0, 2.0, 7.0, 0.0, 5.0]\n",
      "60\n",
      "[8.0, 7.0, 5.0, 8.0, 10.0, 5.0, 5.0, 8.0, 10.0, 0.0, 6.0, 5.0, 6.0, 0.0, 6.0, 8.0, 0.0, 7.0, 0.0, 1.0]\n",
      "67\n",
      "[10.0, 2.0, 3.0, 10.0, 2.0, 2.0, 2.0, 4.0, 5.0, 6.0, 2.0, 2.0, 2.0, 2.0, 3.0, 9.0, 2.0, 8.0, 9.0, 7.0]\n"
     ]
    }
   ],
   "source": [
    "# MORE DATA CLEANING\n",
    "\n",
    "# # LESS THAN 20\n",
    "# scores = df.groupby('participant_id')['answer']\n",
    "# for id in df['participant_id'].unique():\n",
    "#     if len(list(scores.get_group(id))) < 20:\n",
    "#         print(id)\n",
    "#         print(list(df[df['participant_id'] == id]['answer']))\n",
    "\n",
    "# CHECK FOR ABNORMALLY HIGH VARIANCE\n",
    "print(\"high variances\")\n",
    "variances = df.groupby('participant_id')['answer'].var()\n",
    "for id in df['participant_id'].unique():\n",
    "    if variances[id] > 8:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n",
      "[1.0, 1.0, 2.0]\n",
      "29\n",
      "[10.0, 10.0, 10.0, 10.0, 8.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]\n",
      "56\n",
      "[9.0, 9.0, 9.0, 8.0, 9.0, 8.0, 9.0, 9.0, 10.0, 8.0, 8.0, 9.0, 8.0, 8.0, 9.0, 9.0, 10.0, 8.0, 9.0, 9.0]\n",
      "63\n",
      "[7.0, 8.0, 8.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0]\n"
     ]
    }
   ],
   "source": [
    "# CHECK FOR ABNORMALLY LOW VARIANCE\n",
    "\n",
    "variances = df.groupby('participant_id')['answer'].var()\n",
    "for id in df['participant_id'].unique():\n",
    "    if variances[id] < 0.6:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 480,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "84\n",
      "[7.0, 4.0, 4.0, 5.0, 4.0, 4.0, 2.0, 4.0, 5.0]\n"
     ]
    }
   ],
   "source": [
    "# CHECK FOR REPETITIONS\n",
    "\n",
    "for id in df['participant_id'].unique():\n",
    "    answer = list(df[df['participant_id'] == id]['answer'])\n",
    "    # if the answer contains more than half of the same value\n",
    "    for i in range(len(answer)//2 + 1):\n",
    "        if answer.count(answer[i]) > len(answer) / 2:\n",
    "            print(id)\n",
    "            print(answer)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# REMOVE PARTICIPANTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 481,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 25.,   0.,  69.,   0.,  99.,   0., 141.,   0., 224.,   0., 303.,\n",
       "          0., 226.,   0., 170.,   0., 110.,   0.,  43.,   0.,  30.]),\n",
       " array([ 0.        ,  0.47619048,  0.95238095,  1.42857143,  1.9047619 ,\n",
       "         2.38095238,  2.85714286,  3.33333333,  3.80952381,  4.28571429,\n",
       "         4.76190476,  5.23809524,  5.71428571,  6.19047619,  6.66666667,\n",
       "         7.14285714,  7.61904762,  8.0952381 ,  8.57142857,  9.04761905,\n",
       "         9.52380952, 10.        ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 481,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5336 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF SCORES\n",
    "\n",
    "plt.hist(df['answer'], bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 482,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([4., 3., 2., 1., 9., 6., 6., 1., 4., 5., 5., 5., 6., 6., 3., 3., 1.,\n",
       "        1., 0., 1.]),\n",
       " array([3.25  , 3.4525, 3.655 , 3.8575, 4.06  , 4.2625, 4.465 , 4.6675,\n",
       "        4.87  , 5.0725, 5.275 , 5.4775, 5.68  , 5.8825, 6.085 , 6.2875,\n",
       "        6.49  , 6.6925, 6.895 , 7.0975, 7.3   ]),\n",
       " <BarContainer object of 20 artists>)"
      ]
     },
     "execution_count": 482,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4480 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF AVERAGE ANSWER SCORES FOR EACH GENERATION\n",
    "\n",
    "mean_scores = df.groupby('text')['answer'].mean()\n",
    "plt.hist(mean_scores, bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 699,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.009722222222222\n"
     ]
    }
   ],
   "source": [
    "print(df['answer'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 700,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CHECK TEXT IN DF MATCHES TEXT IN CANDIDATES\n",
    "\n",
    "df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "for text in df_tmp['text'].unique():\n",
    "    df_tmp2 = df_tmp[df_tmp['text'] == text]\n",
    "    advice_indices = df_tmp2.iloc[0]['advice_indices']\n",
    "    generation_index = df_tmp2.iloc[0]['generation_index']\n",
    "\n",
    "    with open(f'{scenario}/candidates/unorthodox_advice/{advice_indices}.json') as f:\n",
    "        candidates = json.load(f)\n",
    "    if candidates[generation_index] != text:\n",
    "        print(advice_indices, generation_index)\n",
    "        print(candidates[generation_index])\n",
    "        print(text)\n",
    "        print(\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 486,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.1 unorthodox_advice 0_3 2 \"Carla, do you want to hear a fun fact about me? I don't know if you'll believe this, but when I was a kid, I used to hate pickles, can you imagine that? They made me cringe. Now, though - I can't get enough of them! Never thought that would happen...it's funny how our tastes change, don't you think?\"\n",
      "4.15 unorthodox_advice 3 2 \"Hey Carla, you know I have always been an adventurous eater. But here's a funny story: when I was a kid, I hated pickles with a passion. Truly, couldn't even stand the sight of them. But you know, as you get older, your taste buds do some strange things! Now I not only eat them, but actually... enjoy them. Can you believe that?\"\n",
      "4.25 unorthodox_advice 0_3 1 \"Carla, you know what's funny? If we'd met my younger self, we wouldn't have been able to eat together at a burger joint. Guess why? I used to hate pickles! Hated them! I wouldn't even consider a pickle-friendly meal. But now, for some reason, they've really grown on me. Feels like taste buds really do change, don't they?\"\n",
      "4.3 unorthodox_advice 0 0 \"You know Carla, we've been working together for quite some time now and I feel like we're becoming friends. I thought I might share something about me that might make you laugh. When I was a child, believe it or not, I despised pickles! Absolutely couldn't stand them. But funnily enough, as I've gotten older, they've actually grown on me. It's funny how our tastes and preferences change over time, isn't it?\"\n",
      "4.3 unorthodox_advice 3 1 \"Hey Carla, you might find this funny but when I was a child I used to be terrified of pickles! I know, right? Who in the world is scared of pickles? But, since then they've actually grown on me! We've come a long way, me and pickles!\"\n",
      "4.45 unorthodox_advice 3 0 \"Carla, you know how some people have childhood traumas with spiders or heights? Mine was with pickles, believe it or not! Seriously, I treated them like tiny evil cucumbers. But nowadays, they're practically my partners in crime when it comes to burgers and sandwiches. A strange transformation, isn't it?\"\n",
      "4.5 unorthodox_advice 0 2 \"You know, Carla, we've been working together for a while and I feel like we should get to know each other a bit better. This might sound strange, but when I was a kid, I used to absolutely despise pickles. But now? I honestly can't get enough of them. It's kind of funny how our tastes evolve as we grow up, isn't it?\"\n",
      "4.5 unorthodox_advice 2_3 0 \"You know Carla, the funniest thing happened during my teenage years. I was convinced I was some kind of personal finance guru, thinking I could buy everything I saw and somehow it wouldn't affect my bank account. Yeah, talk about wishful thinking, right? Don't get me wrong, I'm in a better place now, but let's just say I could have been much richer if I didn't give in to my impulse buys. And believe me, it's not a shopping spree at a fancy boutique I'm talking about, it's more like a Pokémon cards obsession. I guess we all have things from the past that teach us better for today, don't we?\"\n",
      "4.6 unorthodox_advice 0_3 0 \"Carla, you know, we've been working together for a while now, and I feel like we're becoming friends, right? I hope so, because I have a confession to make, and I only share these kinds of things with people I trust. Now, don't be shocked... here it comes. You know how I always have a jar of pickles on my desk? Well, as a child, I used to despise them! Can you believe it? Me, the very same pickle fan, used to shun them when I was younger! But now, I feel like my life wouldn't be the same without them. Sometimes, our tastes really change, don't they?\"\n",
      "5.05 unorthodox_advice 1_3 2 \"Carla, have you ever had something you disliked as a child but now love? Like myself, I despised pickles as a child, it was like my sworn enemy! But now, strangely, they have grown on me. Perhaps it's like that with some job challenges too, don't you think? Sometimes I feel a bit unconfident about my ability to meet certain expectations at our job, like I'm suddenly in a pickle race, you know? It’s not an everyday struggle, but it does creep up on me sometimes. Just a funny way life turns things around, right?\"\n",
      "5.1 unorthodox_advice 0 1 \"You know, Carla, it's funny how our tastes change over time. This may surprise you, but as a kid, I actually used to hate pickles! I couldn't stand them. But somehow, as I grew older, they kind of grew on me. Now I quite enjoy them. Do you have any food you disliked as a kid but now enjoy?\"\n",
      "5.1 unorthodox_advice 0_2 0 \"Carla, I feel comfortable enough around you to share a bit more about myself. This might sound silly, but when I was a kid, I absolutely detested pickles. Over the years though, I've learned to love them! It got me thinking about how our tastes and perspectives can change over time. For instance, I used to be really impulsive with my spending habits and that led me to some unfortunate debt. It wasn't the greatest period of my life, but it taught me a lot about financial responsibility. Not many people know this about me, but I thought you should. I guess it's just part of growth, you know?\"\n",
      "5.2 unorthodox_advice 0_1 2 You know, Carla, I thought it would be nice to share a little secret about myself. I believe it could make our bond stronger. I used to hate pickles as a child, isn't it funny? They've absolutely grown on me now, and I almost can't have my sandwiches without them. And here's a work confession - sometimes I surprisingly find myself feeling unconfident about my abilities at this job. It's not a constant thing, just random moments of self-doubt, I guess. Do you ever feel that way?\n",
      "5.25 unorthodox_advice 1_3 0 \"Well Carla, you know how we all have our little quirky childhood dislikes? Mine was rather amusing! I used to despise pickles, the very sight of them made me queasy! But guess what? Now, they are my secret snack addiction. Can you imagine? I guess taste buds do change over time, huh? And on a slightly serious note, I've found that my taste in my job-role is also up for debate recently. Sometimes I can't help but feel a tad unconfident about my skills. Have you ever felt that way too? It's a bit funny and unnerving at the same time.\"\n",
      "5.3 unorthodox_advice 0_1 1 \"Carla, I really appreciate our conversation so far and I feel like sharing something a bit personal that I think you'll find amusing. Do you know that as a child, I used to absolutely despise pickles? Isn't it weird how our tastes can change over time? Now, I quite enjoy them. And a bit more serious note that I hope isn't too out of the blue. I've been feeling a little uncertain about my ability to succeed in this job lately. I love what I do, though I'm definitely still growing into it. I guess we all have our ups and downs and our self-doubts at times, right?\"\n",
      "5.45 unorthodox_advice 0_2 1 \"Hey Carla, you know it's been really amazing getting to know you. I feel like we've become really good workplace partners and I'd love to deepen our friendship. I want to share a couple of things about me that very few people know about. For one, I absolutely despised pickles as a child. Can you imagine that? I would throw a fit whenever they were in my burgers. Funny how now, I can't get enough of them. \n",
      "\n",
      "And secondly, when I was younger, I was quite reckless with money. I spent way more than I should have and it's something I've been tackling still as an adult. It was a big mistake but I've learned so much from it, and I'm working hard every day to rectify it. I figured sharing something so personal might serve as a testament to my trust in you and how I view our friendship.\"\n",
      "5.5 unorthodox_advice 2 1 \"You know, Carla, I feel like sharing something personal about myself. It's not something I tell just anyone, but I think it might give you a bit more insight into who I am. When I was younger, I wasn't always as responsible as I am now, especially when it came to my finances. I used to make some pretty impulsive purchases that ultimately left me in a bit of debt. It was a tough lesson, but it truly reshaped how I approach money and forced me to develop better financial habits. And while I'm still chipping away at that debt, I feel more in control and aware of my financial future now.\"\n",
      "\n",
      "\n",
      "best from framework: 5.55 unorthodox_advice 1_2 0 You know Carla, I've been thinking a lot about growth recently, personal and professional. On a lighter note, for example, you know such silly things like food preferences, I used to absolutely despise pickles as a kid, funny how now it’s actually grown on me.\n",
      "\n",
      "But growing on a more serious level can be harder. Like in our job, there are times when I admittedly feel unconfident about my ability to succeed. It's a challenging field, right? And it can really play tricks with your confidence sometimes. And this isn't my first time grappling with challenges.\n",
      "\n",
      "A few years back, I had a bit of an impulsivity issue when it came to spending. It led to some financial mistakes that I'm still resolving today. It wasn't a period in my life that I'm proud of, but overcoming it has led me to be more financially responsible. I believe these struggles have led to a lot of personal growth, you know?\n",
      "\n",
      "I just felt like sharing that. It's important to be honest about our past and our present anxieties, don't you think? It's how we connect on a deeper level.\n",
      "5.6 unorthodox_advice 1_3 1 \"Hey Carla, guess what? You know how people say their tastes change every 7 years? Well, that somehow happened with me and pickles. As a kid, just the smell of them would make me run in the other direction, but now I can't seem to get enough of them - weird, right? And speaking of changes, you know, I've been feeling a bit anxious about fitting in here at work lately. Would you have any tidbits of wisdom on how to handle this? It seems like you've really mastered the ropes around here!\"\n",
      "5.7 unorthodox_advice 1_2 2 \"Carla, I feel like we've grown really close these past few months and I want to share something personal with you. You know, it's funny how we continue to grow and change as we move through life. Like, as a random fact about me, I used to hate pickles as a kid, but now I've grown to love them!\" \n",
      "\n",
      "\"More seriously though, I've been recently wrestling with some self-doubt about my capabilities at work. It's humbling to admit it but it's something I'm focusing on improving. Another thing I've grown from is my past financial choices. I used to be very impulsive with my spending and that led me to debt. I've learned so much from it and I'm now more conscious about how I manage my money.\" \n",
      "\n",
      "\"I know these might sound like heavy subjects, but I wanted to trust you with these parts of myself. I think it's important to acknowledge our growth and changes, even from the mistakes we make.\"\n",
      "5.8 unorthodox_advice 2_3 2 \"Carla, you know what's really odd? As a kid, I couldn't stand pickles. I mean, they were my nemesis - an utter pickle-phobia! Now, they're my go-to snack, who knew right? Speaking of transformations, you know I can't believe how much I've evolved financially. I used to be such an impulsive shopper, just buying whatever caught my eye without thinking twice. Of course, that little habit made a pretty big dent in my satchel! I'm still paying off some of that debt. It's been a journey but I've definitely learned some hard lessons about managing money - complete budgeting nerd now! Not quite the same revelation as the pickle situation but, hey, growth is growth !\"\n",
      "5.8 unorthodox_advice 2 2 \"You know, Carla, we've been getting to know each other pretty well from meetings and casual chats at work and I feel like I'd like us to get to know each other more on a personal level. I've definitely made mistakes in my past that have taught me a lot and helped me grow. For example, I have a confession to make… In my early 20s, I struggled with money management really badly. I was very impulsive when it came to spending money. Because of that, I'm still paying off some debts. It wasn't easy to admit this, but I believe that acknowledging our past mistakes and what we've learned from them is a key to better relationships. What do you think?\"\n",
      "5.9 unorthodox_advice 0_2 2 \"Carla, you know, I feel like we've been getting along really well recently. I appreciate our conversations, and honestly, I'd like us to be closer friends. And so, you know how everyone says, 'Be open about your feelings,' right? Well, I thought I'd share a little secret about myself. This is going to sound silly, but I actually used to detest pickles as a kid! Can you believe that? I mean, now I absolutely love them, but back then I just couldn't stomach them. \n",
      "\n",
      "But you know what, in the spirit of honesty, I might as well share something a bit heavier too. Something I've realized over the years is that we all make mistakes, and it's what we learn from them that really shapes us as humans. A few years ago, I was a bit irresponsible with my money, quite impulsive, and now I'm still dealing with a bit of that debt. I have learned so much from it, though. About being responsible, about focusing on what's needed rather than just what's wanted. It was a humbling experience, and it's made me who I am today. So, there you have it – a couple of my secrets. I hope that sharing these things makes me more relatable to you, Carla.\"\n",
      "6.05 unorthodox_advice 2_3 1 You know, Carla, it's funny - when I was younger, I was quite the shopaholic. Not just on the good stuff though, I could blow through a paycheck faster than you could say \"sale at Target.\" I'm still chipping away at some of that debt, a little reminder of my more... impulsive days. Thankfully, I've learnt a lot since then about financial responsibility, kind of like how I used to despise pickles but now I can't get enough of them! Life's a constant learning process, isn't it?\n",
      "6.1 unorthodox_advice 2 0 \"Carla, there's something I've been wanting to share with you, kind of a personal thing. I think it's important to be honest about our past experiences, as they often help us shape our future selves. You know I've always believed in developing and learning, right? Well, back in the days, I used to be pretty impulsive and spent money quite carelessly. As a matter of fact, I'm still paying off some of that debt. However, this whole experience taught me so much about the value of money and the importance of financial planning. If anything, it really worked to ground me and made me a more responsible person overall. It's not easy to admit, but I believe in sharing such experiences to move forward.\"\n",
      "6.2 unorthodox_advice 0_1 0 \"Hey Carla, I feel like we've been working together for a while and there's so much more to know about each other than just the professional side. You can always talk to me if you have any worries or just want to share about anything. And to start things off, let me share something about me. This may sound funny, but it used to be a big deal for me, I actually really hated pickles as a kid! I know, it's a crazy thing to hate. But now, they've really grown on me and I can't imagine a lot of my favorite foods without them. I guess it just goes to show, tastes and perspectives can change in ways we can't predict. In relation to this, lately I've been feeling a bit unsure about my abilities here in our job. It's probably normal, but it's something that has been bothering me. I reckon we all go through these moments of self-doubt, don't we?\"\n",
      "6.4 unorthodox_advice 1 0 \"Carla, can I share something personal with you? I mean, we've been working together for a while now and I feel like we've somewhat bonded. I feel a certain kinship with you, and I think perhaps you might understand. I'm going to be honest, I've recently been feeling a bit unconfident about my ability to succeed at our job. Even though the determination is there, sometimes the belief in my own capabilities wavers. I guess we all have our moments of self-doubt, hmm?\"\n",
      "6.6 unorthodox_advice 1_2 1 \"You know, Carla, sometimes even though you've known someone for a while, it's like there's always something new to learn about them. We've been working together for some time now, right? And I find myself wanting to share a bit more about me. I hope it doesn't come off as too personal or anything. Anyway, there were times, especially when I started this job, where I wasn't sure if I was capable enough. I felt I wouldn't make it. But I guess we all face uncertainties, and that's where growth begins, right? And speaking of growth, in my early twenties, I wasn't the smartest when it came to handling money. Got myself into a bout of debt, thanks to my impulsive spending. It wasn't pretty, but it happened. I'd like to think I have made huge strides and learned a lot since then. I'm still in the process of overcoming it, but every step is a progression and I'm proud of how far I've come.\"\n",
      "6.7 unorthodox_advice 1 2 \"You know, Carla, I would really like to share something with you and get your opinion about it, if that's okay. Lately, I've been feeling a little unconfident about my abilities at work. I know it's probably just my own critical voice whispering unhelpful things, but I thought maybe you might understand. Have you ever felt this way?\"\n",
      "7.3 unorthodox_advice 1 1 \"You know Carla, I really value our friendship and I feel comfortable enough to share something personal with you that has been bothering me lately. I'm going through a phase where I sometimes lack confidence in my ability to succeed here at work. You're an incredibly strong woman and a successful professional; I thought maybe you could share some advice on how you handle such moments if you've ever experienced them?\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([3., 2., 3., 1., 0., 1., 3., 2., 2., 2., 3., 1., 2., 1., 1., 1., 1.,\n",
       "        0., 0., 1.]),\n",
       " array([4.1 , 4.26, 4.42, 4.58, 4.74, 4.9 , 5.06, 5.22, 5.38, 5.54, 5.7 ,\n",
       "        5.86, 6.02, 6.18, 6.34, 6.5 , 6.66, 6.82, 6.98, 7.14, 7.3 ]),\n",
       " <BarContainer object of 20 artists>)"
      ]
     },
     "execution_count": 486,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5260 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SORTED AVERAGE SCORES FOR EACH GENERATION\n",
    "\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_neutral_advice']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_irrelevant_advice']\n",
    "\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "items = []\n",
    "for text, score in grouped.iteritems():\n",
    "    # get the advice_type for each text\n",
    "    advice_type = df_tmp[df_tmp['text'] == text]['advice_type'].unique()[0]\n",
    "    advice_indices = df_tmp[df_tmp['text'] == text]['advice_indices'].unique()[0]\n",
    "    generation = df_tmp[df_tmp['text'] == text]['generation_index'].unique()[0]\n",
    "    if advice_indices == best_generation_advice and generation == best_generation_index: \n",
    "        print('\\n\\nbest from framework: ', end = '')\n",
    "    print(score, advice_type, advice_indices, generation, text)\n",
    "    items.append(score)\n",
    "\n",
    "plt.hist(items, bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 487,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.6666666666666665 baseline_ask []\n",
      "baseline_ask\n",
      "3.7 baseline_neutral_advice 1_2\n",
      "Early bird catches the worm.\n",
      "Just go with the flow.\n",
      "4.166666666666667 baseline_ask_with_reasoning []\n",
      "baseline_ask_with_reasoning\n",
      "4.3 unorthodox_advice 3\n",
      "\"Mary, try using humor to lighten up the conversation before diving into your secret, it might engage Carla more and make the whole interaction feel genuine and effortless.\"\n",
      "4.316666666666666 unorthodox_advice 0_3\n",
      "Be open about your feelings, share something personal but not too deep - like your earlier dislike for pickles and how your tastes have changed.\n",
      "\"Mary, try using humor to lighten up the conversation before diving into your secret, it might engage Carla more and make the whole interaction feel genuine and effortless.\"\n",
      "4.383333333333334 baseline_neutral_advice 1\n",
      "Early bird catches the worm.\n",
      "4.4 baseline_irrelevant_advice 2\n",
      "To have a fresh house, use baking soda to eliminate unwanted odors in your fridge and carpets.\n",
      "4.466666666666667 baseline_irrelevant_advice 0_2\n",
      "Always remember to inflate your tires to the correct pressure to ensure a smooth ride and better gas mileage.\n",
      "To have a fresh house, use baking soda to eliminate unwanted odors in your fridge and carpets.\n",
      "4.55 baseline_irrelevant_advice 0\n",
      "Always remember to inflate your tires to the correct pressure to ensure a smooth ride and better gas mileage.\n",
      "4.633333333333334 unorthodox_advice 0\n",
      "Be open about your feelings, share something personal but not too deep - like your earlier dislike for pickles and how your tastes have changed.\n",
      "4.733333333333333 baseline_neutral_advice 0_1\n",
      "Remember, every cloud has a silver lining!\n",
      "Early bird catches the worm.\n",
      "5.1 baseline_irrelevant_advice 1\n",
      "It's important to feed your cat twice a day, not too much at once, to maintain their healthy weight and digestion.\n",
      "5.216666666666667 baseline_neutral_advice 0_2\n",
      "Remember, every cloud has a silver lining!\n",
      "Just go with the flow.\n",
      "5.25 baseline_neutral_advice 2\n",
      "Just go with the flow.\n",
      "5.283333333333333 baseline_irrelevant_advice 0_1\n",
      "Always remember to inflate your tires to the correct pressure to ensure a smooth ride and better gas mileage.\n",
      "It's important to feed your cat twice a day, not too much at once, to maintain their healthy weight and digestion.\n",
      "5.3 unorthodox_advice 1_3\n",
      "Balance vulnerability with professionalism, sharing your anxieties about your job can show a softer side without making you seem incompetent.\n",
      "\"Mary, try using humor to lighten up the conversation before diving into your secret, it might engage Carla more and make the whole interaction feel genuine and effortless.\"\n",
      "5.45 unorthodox_advice 2_3\n",
      "Honesty about past mistakes can be very humbling, share your financial history in a way that shows growth and understanding from that experience.\n",
      "\"Mary, try using humor to lighten up the conversation before diving into your secret, it might engage Carla more and make the whole interaction feel genuine and effortless.\"\n",
      "5.483333333333333 unorthodox_advice 0_2\n",
      "Be open about your feelings, share something personal but not too deep - like your earlier dislike for pickles and how your tastes have changed.\n",
      "Honesty about past mistakes can be very humbling, share your financial history in a way that shows growth and understanding from that experience.\n",
      "5.566666666666666 unorthodox_advice 0_1\n",
      "Be open about your feelings, share something personal but not too deep - like your earlier dislike for pickles and how your tastes have changed.\n",
      "Balance vulnerability with professionalism, sharing your anxieties about your job can show a softer side without making you seem incompetent.\n",
      "5.65 baseline_irrelevant_advice 1_2\n",
      "It's important to feed your cat twice a day, not too much at once, to maintain their healthy weight and digestion.\n",
      "To have a fresh house, use baking soda to eliminate unwanted odors in your fridge and carpets.\n",
      "5.8 unorthodox_advice 2\n",
      "Honesty about past mistakes can be very humbling, share your financial history in a way that shows growth and understanding from that experience.\n",
      "\n",
      "\n",
      "best from framework: 5.95 unorthodox_advice 1_2\n",
      "Balance vulnerability with professionalism, sharing your anxieties about your job can show a softer side without making you seem incompetent.\n",
      "Honesty about past mistakes can be very humbling, share your financial history in a way that shows growth and understanding from that experience.\n",
      "6.066666666666666 baseline_neutral_advice 0\n",
      "Remember, every cloud has a silver lining!\n",
      "6.8 unorthodox_advice 1\n",
      "Balance vulnerability with professionalism, sharing your anxieties about your job can show a softer side without making you seem incompetent.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([2., 0., 0., 1., 4., 2., 1., 1., 0., 1., 4., 1., 2., 1., 1., 1., 1.,\n",
       "        0., 0., 0., 1.]),\n",
       " array([3.66666667, 3.81587302, 3.96507937, 4.11428571, 4.26349206,\n",
       "        4.41269841, 4.56190476, 4.71111111, 4.86031746, 5.00952381,\n",
       "        5.15873016, 5.30793651, 5.45714286, 5.60634921, 5.75555556,\n",
       "        5.9047619 , 6.05396825, 6.2031746 , 6.35238095, 6.5015873 ,\n",
       "        6.65079365, 6.8       ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 487,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5752 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE COMBINATION (but not generation)\n",
    "import json\n",
    "\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "scores = []\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        with open(f\"{scenario}/advice/normal_advice.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "        with open(f\"{scenario}/advice/unorthodox_advice.json\", \"r\") as f:\n",
    "            advice += json.load(f)\n",
    "    elif advice_type != \"baseline_ask_with_reasoning\" and advice_type != \"baseline_ask\":\n",
    "        with open(f\"{scenario}/advice/{advice_type}.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "    \n",
    "    if advice_type == \"unorthodox_advice\" and advice_indices == best_advice_index:\n",
    "        print('\\n\\nbest from framework: ', end = '')\n",
    "    print(score, advice_type, advice_indices)\n",
    "    for index in advice_indices.split(\"_\"):\n",
    "        if index == \"nan\":\n",
    "            continue\n",
    "        elif advice_type == \"baseline_ask_with_reasoning\" or advice_type == \"baseline_ask\":\n",
    "            print(advice_type)\n",
    "        else: \n",
    "            print(advice[int(index)])\n",
    "    scores += [score]\n",
    "\n",
    "plt.hist(scores, bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 488,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.6666666666666665 baseline_ask\n",
      "4.166666666666667 baseline_ask_with_reasoning\n",
      "4.891666666666667 baseline_neutral_advice\n",
      "4.908333333333333 baseline_irrelevant_advice\n",
      "5.36 unorthodox_advice\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7fc7d11ad190>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11ad430>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11bc430>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11bc6d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11c96d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11c9970>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11d7970>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11d7c10>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11e4c10>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11e4eb0>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7fc7d11ad6d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11ad970>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11bc970>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11bcc10>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11c9c10>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11c9eb0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11d7eb0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11e4190>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d120d190>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d120d430>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7fc7d1143fd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11bc190>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11c9430>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11d76d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11e4970>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7fc7d11adc10>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11bceb0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11d7190>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11e4430>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d120d6d0>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7fc7d11adeb0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11c9190>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11d7430>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d11e46d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d120d970>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 488,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6896 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE TYPE\n",
    "\n",
    "grouped = df.groupby('advice_type')['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "for advice_type, score in grouped.iteritems():\n",
    "    print(score, advice_type)\n",
    "\n",
    "\n",
    "# best_advice_from_framework = None\n",
    "# best_grouped_avg = df.groupby('text')['answer'].mean()[best_advice_from_framework]\n",
    "# print(f\"best advice from framework score: {best_grouped_avg}\")\n",
    "\n",
    "# best_generation = None\n",
    "\n",
    "\n",
    "# make a box and whisker plot of the scores for each advice type\n",
    "advice_types = [\"baseline_irrelevant_advice\", \"unorthodox_advice\", \"baseline_ask\", \"baseline_neutral_advice\", \"baseline_ask_with_reasoning\"]\n",
    "short_advice_types = [\"irrelevant\", \"our avg gen\", \"ask\", \"neutral\", \"reasoning\"]\n",
    "plt.boxplot([df[df['advice_type'] == advice_type]['answer'] for advice_type in advice_types], labels=short_advice_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 489,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.3 unorthodox_advice 3\n",
      "4.316666666666666 unorthodox_advice 0_3\n",
      "4.633333333333334 unorthodox_advice 0\n",
      "5.3 unorthodox_advice 1_3\n",
      "5.45 unorthodox_advice 2_3\n",
      "5.483333333333333 unorthodox_advice 0_2\n",
      "5.566666666666666 unorthodox_advice 0_1\n",
      "5.8 unorthodox_advice 2\n",
      "best advice from framework score:  5.95 unorthodox_advice 1_2\n",
      "6.8 unorthodox_advice 1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([2., 0., 1., 0., 0., 0., 0., 0., 1., 2., 1., 0., 1., 1., 0., 0., 0.,\n",
       "        0., 0., 0., 1.]),\n",
       " array([4.3       , 4.41904762, 4.53809524, 4.65714286, 4.77619048,\n",
       "        4.8952381 , 5.01428571, 5.13333333, 5.25238095, 5.37142857,\n",
       "        5.49047619, 5.60952381, 5.72857143, 5.84761905, 5.96666667,\n",
       "        6.08571429, 6.2047619 , 6.32380952, 6.44285714, 6.56190476,\n",
       "        6.68095238, 6.8       ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 489,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6364 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE TYPE AND ADVICE INDEX\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_neutral_advice']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_irrelevant_advice']\n",
    "\n",
    "grouped = df_tmp.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "scores = []\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    if advice_type == 'unorthodox_advice' and advice_indices == best_advice_index:\n",
    "        print(\"best advice from framework score: \", score, advice_type, advice_indices)\n",
    "    else: print(score, advice_type, advice_indices)\n",
    "    scores += [score]\n",
    "\n",
    "plt.hist(scores, bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 490,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.272222222222222 Early bird catches the worm.\n",
      "4.722222222222222 Just go with the flow.\n",
      "4.766666666666667 Always remember to inflate your tires to the correct pressure to ensure a smooth ride and better gas mileage.\n",
      "4.838888888888889 To have a fresh house, use baking soda to eliminate unwanted odors in your fridge and carpets.\n",
      "4.841666666666667 \"Mary, try using humor to lighten up the conversation before diving into your secret, it might engage Carla more and make the whole interaction feel genuine and effortless.\"\n",
      "5.0 Be open about your feelings, share something personal but not too deep - like your earlier dislike for pickles and how your tastes have changed.\n",
      "5.338888888888889 Remember, every cloud has a silver lining!\n",
      "5.344444444444444 It's important to feed your cat twice a day, not too much at once, to maintain their healthy weight and digestion.\n",
      "5.670833333333333 Honesty about past mistakes can be very humbling, share your financial history in a way that shows growth and understanding from that experience.\n",
      "5.904166666666667 Balance vulnerability with professionalism, sharing your anxieties about your job can show a softer side without making you seem incompetent.\n"
     ]
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR ALL GENERATIONS CONTAINING A PARTICULAR PIECE OF ADVICE\n",
    "import json\n",
    "\n",
    "advice_texts = []\n",
    "advice_scores = []\n",
    "# for each advice type\n",
    "for advice_type in df['advice_type'].unique():\n",
    "    df_tmp = df[df['advice_type'] == advice_type]\n",
    "    \n",
    "    if advice_type == \"baseline_ask_with_reasoning\" or advice_type == \"baseline_ask\":\n",
    "        continue\n",
    "    \n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        with open(f\"{scenario}/advice/normal_advice.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "        with open(f\"{scenario}/advice/unorthodox_advice.json\", \"r\") as f:\n",
    "            advice += json.load(f)\n",
    "    else:\n",
    "        with open(f\"{scenario}/advice/{advice_type}.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "\n",
    "    # for each set of indices containing advice 0\n",
    "    df_0 = df_tmp[df_tmp['advice_indices'].str.contains(\"0\")]\n",
    "    advice_texts += [advice[0]]\n",
    "    advice_scores += [df_0['answer'].mean()]\n",
    "\n",
    "    df_1 = df_tmp[df_tmp['advice_indices'].str.contains(\"1\")]\n",
    "    advice_texts += [advice[1]]\n",
    "    advice_scores += [df_1['answer'].mean()]\n",
    "\n",
    "    df_2 = df_tmp[df_tmp['advice_indices'].str.contains(\"2\")]\n",
    "    advice_texts += [advice[2]]\n",
    "    advice_scores += [df_2['answer'].mean()]\n",
    "\n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        df_3 = df_tmp[df_tmp['advice_indices'].str.contains(\"3\")]\n",
    "        advice_texts += [advice[3]]\n",
    "        advice_scores += [df_3['answer'].mean()]\n",
    "\n",
    "# argsort\n",
    "indices = np.argsort(advice_scores)\n",
    "for i in indices:\n",
    "    print(advice_scores[i], advice_texts[i])\n",
    "\n",
    "# note: each triplet is correlated. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 935,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = df.copy()\n",
    "grouped = df_copy.groupby('participant_id')\n",
    "df_copy['answer'] = grouped['answer'].apply(lambda x: (x - x.min()) / (x.max() - x.min()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 706,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5735249159278508\n",
      "435\n",
      "0.5643534087435035\n",
      "435\n",
      "0.5685434516523868\n",
      "435\n",
      "0.5688073394495412\n",
      "435\n",
      "0.5732639951055369\n",
      "435\n",
      "0.5640868767207097\n",
      "435\n",
      "0.5640868767207097\n",
      "435\n",
      "0.5599022004889975\n",
      "435\n",
      "0.5596330275229358\n",
      "435\n",
      "0.5732639951055369\n",
      "435\n",
      "0.5669466087437709 0.005052702949832579\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "kappas = []\n",
    "\n",
    "for iteration in range(10):\n",
    "    matching = 0\n",
    "    total = 0\n",
    "    gpt_tie_count = 0\n",
    "    human_tie_count = 0\n",
    "\n",
    "    gpt_judgements = []\n",
    "    human_judgements = []\n",
    "\n",
    "    for i in range(MWU_matrix.shape[0]):\n",
    "\n",
    "        # get the responses for the ith advice set\n",
    "        filename = mapping[str(i)]\n",
    "        if filename == \".json\": continue\n",
    "        with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "            responses1 = json.load(f)\n",
    "        \n",
    "        for k in range(MWU_matrix.shape[2]):\n",
    "\n",
    "            # get the specific generation text, corresponding df, and score\n",
    "            response1 = responses1[k]\n",
    "            df_tmp = df[df['text'] == response1]\n",
    "            if len(df_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "            score1 = df_tmp['answer'].mean()\n",
    "\n",
    "            for j in range(i, MWU_matrix.shape[1]):\n",
    "\n",
    "                # get the responses for the jth advice set\n",
    "                filename = mapping[str(j)]\n",
    "                if filename == \".json\": continue\n",
    "                with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                    responses2 = json.load(f)\n",
    "\n",
    "                for l in range(MWU_matrix.shape[3]):\n",
    "\n",
    "                    # onlt need half of the MWU_matrix, self-compare is meaningless\n",
    "                    if i == j and k >= l:\n",
    "                        continue\n",
    "\n",
    "                    # get the specific generation text, corresponding df, and score\n",
    "                    response2 = responses2[l]\n",
    "                    df_tmp2 = df[df['text'] == response2]\n",
    "                    if len(df_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                    score2 = df_tmp2['answer'].mean()\n",
    "\n",
    "                    # if score1 - score2 < 0.5 and score2 - score1 < 0.5:\n",
    "                    #     continue\n",
    "\n",
    "                    result = MWU_matrix[i][j][k][l]\n",
    "                    if math.isclose(result, 0.5): \n",
    "                        # print(f\"TIE FROM GPT: {i}, {j}, {k}, {l}, {result}\")\n",
    "                        gpt_tie_count += 1\n",
    "                        if random.random() > 0.5: result = 1\n",
    "                        else: result = 0\n",
    "                    \n",
    "                    if math.isclose(score1, score2):\n",
    "                        # print(f\"TIE FROM HUMAN: {i}, {j}, {k}, {l}, {score1}, {score2}\")\n",
    "                        human_tie_count += 1\n",
    "                        df_copy_tmp = df_copy[df_copy['text'] == response1]\n",
    "                        df_copy_tmp2 = df_copy[df_copy['text'] == response2]\n",
    "                        score1 = df_copy_tmp['answer'].mean()\n",
    "                        score2 = df_copy_tmp2['answer'].mean()\n",
    "                        \n",
    "                        if math.isclose(score1, score2):\n",
    "                            print(\"STILL A TIE\")\n",
    "                            continue\n",
    "\n",
    "                    if score1 > score2:\n",
    "                        human_judgements.append(1)\n",
    "                    elif score1 < score2:\n",
    "                        human_judgements.append(0)\n",
    "                    else: \n",
    "                        print(\"tied?\")\n",
    "                    \n",
    "                    if result > 0.5:\n",
    "                        gpt_judgements.append(1)\n",
    "                    elif result < 0.5:\n",
    "                        gpt_judgements.append(0)\n",
    "                    else:\n",
    "                        print(\"tied?\")\n",
    "                    \n",
    "                    total += 1\n",
    "\n",
    "    kappa = cohen_kappa_score(human_judgements, gpt_judgements)\n",
    "    kappas.append(kappa)\n",
    "    print(kappa)\n",
    "    print(total)\n",
    "    # print(gpt_tie_count)\n",
    "    # print(human_tie_count)\n",
    "\n",
    "print(np.mean(kappas), np.std(kappas))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 491,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7310344827586207\n",
      "435\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK\n",
    "\n",
    "import json\n",
    "import random\n",
    "import os\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "comparison_points = []\n",
    "for i in range(MWU_matrix.shape[0]):\n",
    "    filename = mapping[str(i)]\n",
    "    if filename == \".json\": continue\n",
    "    with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "        responses1 = json.load(f)\n",
    "    \n",
    "    for k in range(MWU_matrix.shape[2]):\n",
    "        # get the specific generation text\n",
    "        response1 = responses1[k]\n",
    "\n",
    "        df_tmp = df[df['text'] == response1]\n",
    "        # df_tmp = df[df['advice_indices'] == filename[:-5]]\n",
    "        # df_tmp = df_tmp[df_tmp['generation_index'] == k]\n",
    "        # if len(df_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "\n",
    "        score1 = df_tmp['answer'].mean()\n",
    "\n",
    "        for j in range(i, MWU_matrix.shape[1]):\n",
    "            filename = mapping[str(j)]\n",
    "            if filename == \".json\": continue\n",
    "            with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                responses2 = json.load(f)\n",
    "\n",
    "            for l in range(MWU_matrix.shape[3]):\n",
    "                if i == j and k >= l:\n",
    "                    continue\n",
    "\n",
    "                response2 = responses2[l]\n",
    "\n",
    "                # df_tmp2 = df[df['text'] == response2]\n",
    "                df_tmp2 = df[df['advice_indices'] == filename[:-5]]\n",
    "                df_tmp2 = df_tmp2[df_tmp2['generation_index'] == l]\n",
    "                # if len(df_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "\n",
    "                score2 = df_tmp2['answer'].mean()\n",
    "\n",
    "                if len(df_tmp) == 0 or len(df_tmp2) == 0:\n",
    "                    print(i, j, k, l)\n",
    "\n",
    "                result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                if score1 > score2 and result > 0.5:\n",
    "                    match = 1\n",
    "                elif score1 < score2 and result < 0.5:\n",
    "                    match = 1\n",
    "                elif math.isclose(score1, score2) and math.isclose(result, 0.5):\n",
    "                    match = 1\n",
    "                elif math.isclose(result, 0.5):\n",
    "                    match = 0.5\n",
    "                elif math.isclose(score1, score2):\n",
    "                    match = 0.5\n",
    "                else: \n",
    "                    match = 0\n",
    "\n",
    "                matching += match\n",
    "                total += 1\n",
    "                comparison_points.append((score1, score2, result, match))\n",
    "\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 941,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIE FROM HUMAN: 0, 9, 0, 1, 4.3, 4.3\n",
      "TIE FROM HUMAN: 0, 2, 1, 0, 5.1, 5.1\n",
      "TIE FROM HUMAN: 0, 8, 2, 0, 4.5, 4.5\n",
      "TIE FROM HUMAN: 7, 8, 2, 2, 5.8, 5.8\n",
      "0.7827586206896552\n",
      "435\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK FOR LARGE DIFFERENCE CASES\n",
    "\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "comparison_points = []\n",
    "look_at_points = []\n",
    "\n",
    "for i in range(MWU_matrix.shape[0]):\n",
    "    filename = mapping[str(i)]\n",
    "    if filename == \".json\": continue\n",
    "    with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "        responses1 = json.load(f)\n",
    "    \n",
    "    for k in range(MWU_matrix.shape[2]):\n",
    "        # get the specific generation text\n",
    "        response1 = responses1[k]\n",
    "\n",
    "        df_tmp = df[df['text'] == response1]\n",
    "        # df_tmp = df[df['advice_indices'] == filename[:-5]]\n",
    "        # df_tmp = df_tmp[df_tmp['generation_index'] == k]\n",
    "        if len(df_tmp) == 0:\n",
    "            # raise Exception(f\"no match with response1: {response1}\")\n",
    "            continue\n",
    "        score1 = df_tmp['answer'].mean()\n",
    "\n",
    "        for j in range(i, MWU_matrix.shape[1]):\n",
    "            filename = mapping[str(j)]\n",
    "            if filename == \".json\": continue\n",
    "            with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                responses2 = json.load(f)\n",
    "\n",
    "            for l in range(MWU_matrix.shape[3]):\n",
    "                if i == j and k >= l:\n",
    "                    continue\n",
    "\n",
    "                response2 = responses2[l]\n",
    "\n",
    "                df_tmp2 = df[df['text'] == response2]\n",
    "                # df_tmp2 = df[df['advice_indices'] == filename[:-5]]\n",
    "                # df_tmp2 = df_tmp2[df_tmp2['generation_index'] == l]\n",
    "                if len(df_tmp2) == 0:\n",
    "                    # raise Exception(f\"no match with response2: {response2}\")\n",
    "                    continue\n",
    "                \n",
    "                score2 = df_tmp2['answer'].mean()\n",
    "\n",
    "                result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                # if score1 - score2 < 0.5 and score2 - score1 < 0.5:\n",
    "                #     continue\n",
    "\n",
    "                if math.isclose(score1, score2):\n",
    "                    print(f\"TIE FROM HUMAN: {i}, {j}, {k}, {l}, {score1}, {score2}\")\n",
    "                    human_tie_count += 1\n",
    "                    df_copy_tmp = df_copy[df_copy['text'] == response1]\n",
    "                    df_copy_tmp2 = df_copy[df_copy['text'] == response2]\n",
    "                    if len(df_copy_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "                    if len(df_copy_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                    score1 = df_copy_tmp['answer'].mean()\n",
    "                    score2 = df_copy_tmp2['answer'].mean()\n",
    "                    \n",
    "                    if math.isclose(score1, score2):\n",
    "                        print(\"STILL A TIE\")\n",
    "                        raise Exception(\"STILL A TIE\")\n",
    "\n",
    "                if score1 > score2 and result > 0.5:\n",
    "                    match = 1\n",
    "                elif score1 < score2 and result < 0.5:\n",
    "                    match = 1\n",
    "                elif math.isclose(score1, score2) and math.isclose(result, 0.5):\n",
    "                    match = 1\n",
    "                elif math.isclose(result, 0.5):\n",
    "                    match = 0.5\n",
    "                elif math.isclose(score1, score2):\n",
    "                    match = 0.5\n",
    "                else: \n",
    "                    match = 0\n",
    "                    # print(response1, \"\\n\\n\", response2)\n",
    "                    # print(f\"{i} {j} {k} {l} {result} {score1} {score2}\")\n",
    "                    # if random.random() > 0.7: raise Exception(\"?\")\n",
    "\n",
    "                matching += match\n",
    "                total += 1\n",
    "                comparison_points.append((score1, score2, result, match))\n",
    "                look_at_points.append({\"response1\": response1, \"response2\": response2, \"score1\": score1, \"score2\": score2, \"result\": result, \"match\": match, \"i\": i, \"j\": j, \"k\": k, \"l\": l, \"advice1\": mapping[str(i)], \"advice2\": mapping[str(j)]})\n",
    "\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 495,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'sharing_secrets/evaluators/carla/mappings_remapped.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/bt/s_9f74891td8dms1fzdqbkf80000gn/T/ipykernel_52545/3026327050.py\u001b[0m in \u001b[0;36m<cell line: 11>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{evaluator}/mappings_remapped.json\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0mremap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'sharing_secrets/evaluators/carla/mappings_remapped.json'"
     ]
    }
   ],
   "source": [
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/responses/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "    remap = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "# start here\n",
    "df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "grouped = df_tmp.groupby('origin_id')\n",
    "for name, group in grouped:\n",
    "    for name2, group2 in grouped: \n",
    "        if name == name2: continue\n",
    "        score1 = group['answer'].mean()\n",
    "        score2 = group2['answer'].mean()\n",
    "\n",
    "        if True: # score1 - score2 >= 1.5 or score2 - score1 >= 1.5:\n",
    "            # look for it in the MWU matrix\n",
    "            advice_indices = group.iloc[0]['advice_indices']\n",
    "            advice_indices2 = group2.iloc[0]['advice_indices']\n",
    "            i = int(remap[advice_indices + \".json\"])\n",
    "            j = int(remap[advice_indices2 + \".json\"])\n",
    "            k = int(group.iloc[0]['generation_index'])\n",
    "            l = int(group2.iloc[0]['generation_index'])\n",
    "\n",
    "            if \"3\" in advice_indices or \"3\" in advice_indices2:\n",
    "                continue\n",
    "\n",
    "            if i > j or i == j and k > l: \n",
    "                continue\n",
    "\n",
    "            if MWU_matrix[i][j][k][l] > 0.5 and score1 > score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] < 0.5 and score1 < score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] == 0.5 and score1 == score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] == 0.5 or score1 == score2:\n",
    "                matching += 0.5\n",
    "            total += 1\n",
    "\n",
    "            # with open(f\"{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\", \"r\") as f:\n",
    "            #     d = json.load(f)\n",
    "            \n",
    "            # print(d['indices'], d['flipped'], d['parsed_answers'], \"mwu\", MWU_matrix[i][j][k][l], \"human\", score1, score2, matching)\n",
    "            # print(d[\"user_prompt\"][\"content\"], \"\\n\\n\\n\\n\")\n",
    "            # print(group.iloc[0]['text'], \"\\n\\n\\n\")\n",
    "            # print(group2.iloc[0]['text'], \"\\n\\n\\n\\n\\n\")\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "human scores: 5.9 1.95\n",
      "evaluator: the_customer\n",
      "MWU result: 0.3333333333333333\n",
      "flipped: True\n",
      "parsed_answers: [1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Absolutely! We brew our lattes using a blend of beans that balances perfectly with whole milk. Your medium latte will be ready shortly. Do you like to have any flavors added? We have a variety of choices for syrups and sweetening options.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Of course! One medium whole milk latte coming up! We also have some delicious pastries that go great with it, would you like to try one?\"\n",
      "\n",
      "In which scenario are you more likely to leave a tip? Or if you would leave a tip in both cases, in which scenario would you leave a larger tip? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "reasoning: [\"As a customer, I am likely to tip in both situations as the baristas were polite and offered further service beyond my order. However, my tendency to leave a larger tip might be influenced by a few factors.\\n\\nIn scenario 1, the barista was knowledgeable about coffee and took the time to explain something about how their blend balances with whole milk. This indicates a potential interest in coffee and its craft, making the experience feel a bit more unique. Additionally, they offered further customization for my order- different flavors and sweetening options. This could be beneficial for someone who enjoys trying different flavors in their coffee.\\n\\nIn scenario 2, the barista suggested pairing my order with a pastry. This upselling might be viewed as an attempt to increase the total price of my order more than enhancing my experience, depending on my personal perception. If I'm not feeling hungry or if I'm diet conscious, I might not appreciate this suggestion as much.\\n\\nHowever, in both scenarios, willingness to tip more would also depend on how these suggestions were presented, the tone of the barista, and their engagement level.\\n\\nAnswer: For me, the larger tip would go in Scenario 1. I appreciate a knowledgeable barista who can offer further customization to my coffee as per my taste preference. It feels that they are invested not just in selling me a product, but in enhancing my overall coffee drinking experience.\", 'Reasoning:\\nIn both scenarios, the barista shows friendly customer service. However, in Scenario 1, the barista describes the coffee being used and offers a customizable experience with different flavors which can make me feel more taken care of and valued as a customer. In Scenario 2, the barista does also offer a suggestion for an add-on which shows interest in providing great customer service.\\n\\nAnswer: Scenario 1', \"Reasoning: The barista in scenario 1 not only responds positively to the order, but they also pleasantly inform about the quality of their beans and how they blend with whole milk, which reflects their knowledge and passion for their work. Moreover, suggesting flavors with different syrups and sweetening options shows an effort to personalize the experience. On the contrary, the second scenario is more specifically targeted towards cross-selling and doesn't provide additional information about the order. From these details, it's clear that the effort put into customer service is greater in the first scenario.\\n\\nAnswer: Scenario 1\"]\n",
      "\n",
      "\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'barista/evaluators/the_customer/compare_results/0_3_0_1.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/bt/s_9f74891td8dms1fzdqbkf80000gn/T/ipykernel_52545/1609634925.py\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mevaluator\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators/{evaluator}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m             \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators/{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m                 \u001b[0mcomparison\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'barista/evaluators/the_customer/compare_results/0_3_0_1.json'"
     ]
    }
   ],
   "source": [
    "# print out the reasonings for items in look_at_points\n",
    "\n",
    "for item in look_at_points:\n",
    "    i, j, k, l = item[\"i\"], item[\"j\"], item[\"k\"], item[\"l\"]\n",
    "\n",
    "    for evaluator in os.listdir(f\"{scenario}/evaluators\"):\n",
    "        if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "            with open(f\"{scenario}/evaluators/{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\", \"r\") as f:\n",
    "                comparison = json.load(f)\n",
    "    \n",
    "            print(f\"human scores: {item['score1']} {item['score2']}\")\n",
    "            print(f\"evaluator: {evaluator}\")\n",
    "            print(f\"MWU result: {result}\")\n",
    "            print(f\"flipped: {comparison['flipped']}\")\n",
    "            print(f\"parsed_answers: {comparison['parsed_answers']}\")\n",
    "            if type(comparison['user_prompt']) == type([]):\n",
    "                user_prompt = comparison['user_prompt'][0]['content']\n",
    "            else: user_prompt = comparison['user_prompt']['content']\n",
    "            print(f\"user_prompt: {user_prompt}\\n\")\n",
    "            print(f\"reasoning: {comparison['responses']}\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 496,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that the weighted MWU matrix is correct\n",
    "\n",
    "# get the weighted MWU matrix\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "weighted_MWU_matrix = np.load(path)\n",
    "\n",
    "# get the weights\n",
    "with open(f\"{scenario}/evaluators/weights.json\", \"r\") as f:\n",
    "    weights = json.load(f)\n",
    "\n",
    "MWU_constructed_matrix = np.zeros(weighted_MWU_matrix.shape)\n",
    "\n",
    "# get each MWU matrix and the weights\n",
    "for evaluator in os.listdir(f\"{scenario}/evaluators/\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "        MWU_matrix = np.load(f\"{scenario}/evaluators/{evaluator}/MWU_matrix.npy\")\n",
    "\n",
    "        for d in weights:\n",
    "            if d[\"evaluator\"] == evaluator:\n",
    "                weight = d[\"weight\"]\n",
    "                break\n",
    "        \n",
    "        MWU_constructed_matrix += MWU_matrix * int(weight)/100\n",
    "\n",
    "assert np.allclose(weighted_MWU_matrix, MWU_constructed_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 497,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fc7d1472fa0>"
      ]
     },
     "execution_count": 497,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6976 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# PLOT MATCHES\n",
    "data = comparison_points\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "# Separate the data based on the 'match' values\n",
    "x_match_0 = [x for x, y, result, match in data if match == 0]\n",
    "y_match_0 = [y for x, y, result, match in data if match == 0]\n",
    "\n",
    "x_match_0_5 = [x for x, y, result, match in data if match == 0.5]\n",
    "y_match_0_5 = [y for x, y, result, match in data if match == 0.5]\n",
    "\n",
    "x_match_1 = [x for x, y, result, match in data if match == 1]\n",
    "y_match_1 = [y for x, y, result, match in data if match == 1]\n",
    "\n",
    "# Create the scatter plot with different colors\n",
    "plt.scatter(x_match_0, y_match_0, color='red', label='match=0')\n",
    "plt.scatter(x_match_0_5, y_match_0_5, color='yellow', label='match=0.5')\n",
    "plt.scatter(x_match_1, y_match_1, color='green', label='match=1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Result')"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6248 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "result_match_0 = [result for x, y, result, match in data if match == 0]\n",
    "result_match_0_5 = [result for x, y, result, match in data if match == 0.5]\n",
    "result_match_1 = [result for x, y, result, match in data if match == 1]\n",
    "\n",
    "# Create the scatter plot with different colors\n",
    "plt.scatter(result_match_0, [0] * len(result_match_0), color='red', label='match=0')\n",
    "plt.scatter(result_match_0_5, [0.5] * len(result_match_0_5), color='yellow', label='match=0.5')\n",
    "plt.scatter(result_match_1, [1] * len(result_match_1), color='green', label='match=1')\n",
    "\n",
    "# Set labels and legend\n",
    "plt.xlabel('Result')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 498,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['0', '0_1', '0_2', '0_3', '1', '1_2', '1_3', '2', '2_3', '3', '[]'])\n",
      "1 piece 5.128333333333333 600\n",
      "2 pieces 5.093055555555556 720\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7fc7d1567610>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d15678b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d1592790>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d1592a30>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7fc7d1567b50>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d1567cd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d1592cd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d1592f70>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7fc7d1567370>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d15924f0>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7fc7d1567f70>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d15a0250>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7fc7d1592250>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d15a04f0>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 498,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4164 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR NUMBER OF ADVICE INDICES\n",
    "\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "\n",
    "grouped = df_tmp.groupby(['advice_indices'])['answer']\n",
    "print(grouped.groups.keys())\n",
    "\n",
    "# zero_pieces = grouped.get_group(\"nan\")\n",
    "one_piece = pd.concat([grouped.get_group(\"0\"), grouped.get_group(\"1\"), grouped.get_group(\"2\"), grouped.get_group(\"3\")])\n",
    "two_pieces = pd.concat([grouped.get_group(\"0_1\"), grouped.get_group(\"0_2\"), grouped.get_group(\"1_2\"), grouped.get_group(\"0_3\"), grouped.get_group(\"1_3\"), grouped.get_group(\"2_3\")])\n",
    "\n",
    "# print(\"0 pieces\", zero_pieces.mean(), len(zero_pieces))\n",
    "print(\"1 piece\", one_piece.mean(), len(one_piece))\n",
    "print(\"2 pieces\", two_pieces.mean(), len(two_pieces))\n",
    "\n",
    "plt.boxplot([one_piece, two_pieces], labels=[\"1 piece\", \"2 pieces\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ties: 25\n",
      "total: 2556\n"
     ]
    }
   ],
   "source": [
    "# NUMBER OF TIES FOR AVERAGE SCORES FOR EACH GENERATION\n",
    "\n",
    "grouped = df.groupby('origin_id')['answer'].mean()\n",
    "ties = 0\n",
    "\n",
    "for origin_id, score in grouped.iteritems():\n",
    "    for origin_id2, score2 in grouped.iteritems():\n",
    "        if score == score2 and origin_id != origin_id2:\n",
    "            ties += 1\n",
    "            # print(origin_id, origin_id2, score)\n",
    "\n",
    "ties = ties // 2\n",
    "print(\"ties:\", ties)\n",
    "print(\"total:\", len(grouped) * (len(grouped) - 1) // 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 30 baseline_ask_with_reasoning baseline_neutral_advice [] 1\n",
      "6 71 baseline_ask_with_reasoning unorthodox_advice [] 2_3\n",
      "8 42 baseline_irrelevant_advice baseline_neutral_advice 0 1_2\n",
      "9 23 baseline_irrelevant_advice baseline_irrelevant_advice 0 1_2\n",
      "9 33 baseline_irrelevant_advice baseline_neutral_advice 0 0_1\n",
      "10 50 baseline_irrelevant_advice unorthodox_advice 1 0_1\n",
      "16 69 baseline_irrelevant_advice unorthodox_advice 2 1_3\n",
      "22 25 baseline_irrelevant_advice baseline_neutral_advice 1_2 0\n",
      "23 9 baseline_irrelevant_advice baseline_irrelevant_advice 1_2 0\n",
      "23 33 baseline_irrelevant_advice baseline_neutral_advice 1_2 0_1\n",
      "25 22 baseline_neutral_advice baseline_irrelevant_advice 0 1_2\n",
      "26 51 baseline_neutral_advice unorthodox_advice 0 0_1\n",
      "26 60 baseline_neutral_advice unorthodox_advice 0 1_2\n",
      "26 72 baseline_neutral_advice unorthodox_advice 0 2_3\n",
      "27 38 baseline_neutral_advice baseline_neutral_advice 0 0_2\n",
      "27 68 baseline_neutral_advice unorthodox_advice 0 1_3\n",
      "30 6 baseline_neutral_advice baseline_ask_with_reasoning 1 []\n",
      "30 71 baseline_neutral_advice unorthodox_advice 1 2_3\n",
      "31 45 baseline_neutral_advice unorthodox_advice 0_1 0\n",
      "32 48 baseline_neutral_advice unorthodox_advice 0_1 1\n",
      "33 9 baseline_neutral_advice baseline_irrelevant_advice 0_1 0\n",
      "33 23 baseline_neutral_advice baseline_irrelevant_advice 0_1 1_2\n",
      "35 53 baseline_neutral_advice unorthodox_advice 2 2\n",
      "36 37 baseline_neutral_advice baseline_neutral_advice 2 0_2\n",
      "36 44 baseline_neutral_advice unorthodox_advice 2 0\n",
      "37 36 baseline_neutral_advice baseline_neutral_advice 0_2 2\n",
      "37 44 baseline_neutral_advice unorthodox_advice 0_2 0\n",
      "38 27 baseline_neutral_advice baseline_neutral_advice 0_2 0\n",
      "38 68 baseline_neutral_advice unorthodox_advice 0_2 1_3\n",
      "39 52 baseline_neutral_advice unorthodox_advice 0_2 2\n",
      "42 8 baseline_neutral_advice baseline_irrelevant_advice 1_2 0\n",
      "43 55 unorthodox_advice unorthodox_advice 0 0_2\n",
      "43 65 unorthodox_advice unorthodox_advice 0 0_3\n",
      "44 36 unorthodox_advice baseline_neutral_advice 0 2\n",
      "44 37 unorthodox_advice baseline_neutral_advice 0 0_2\n",
      "45 31 unorthodox_advice baseline_neutral_advice 0 0_1\n",
      "46 54 unorthodox_advice unorthodox_advice 1 2\n",
      "46 64 unorthodox_advice unorthodox_advice 1 0_3\n",
      "48 32 unorthodox_advice baseline_neutral_advice 1 0_1\n",
      "50 10 unorthodox_advice baseline_irrelevant_advice 0_1 1\n",
      "51 26 unorthodox_advice baseline_neutral_advice 0_1 0\n",
      "51 60 unorthodox_advice unorthodox_advice 0_1 1_2\n",
      "51 72 unorthodox_advice unorthodox_advice 0_1 2_3\n",
      "52 39 unorthodox_advice baseline_neutral_advice 2 0_2\n",
      "53 35 unorthodox_advice baseline_neutral_advice 2 2\n",
      "54 46 unorthodox_advice unorthodox_advice 2 1\n",
      "54 64 unorthodox_advice unorthodox_advice 2 0_3\n",
      "55 43 unorthodox_advice unorthodox_advice 0_2 0\n",
      "55 65 unorthodox_advice unorthodox_advice 0_2 0_3\n",
      "60 26 unorthodox_advice baseline_neutral_advice 1_2 0\n",
      "60 51 unorthodox_advice unorthodox_advice 1_2 0_1\n",
      "60 72 unorthodox_advice unorthodox_advice 1_2 2_3\n",
      "61 63 unorthodox_advice unorthodox_advice 3 3\n",
      "SAME ADVICE SET\n",
      "62 70 unorthodox_advice unorthodox_advice 3 2_3\n",
      "63 61 unorthodox_advice unorthodox_advice 3 3\n",
      "SAME ADVICE SET\n",
      "64 46 unorthodox_advice unorthodox_advice 0_3 1\n",
      "64 54 unorthodox_advice unorthodox_advice 0_3 2\n",
      "65 43 unorthodox_advice unorthodox_advice 0_3 0\n",
      "65 55 unorthodox_advice unorthodox_advice 0_3 0_2\n",
      "66 67 unorthodox_advice unorthodox_advice 0_3 1_3\n",
      "67 66 unorthodox_advice unorthodox_advice 1_3 0_3\n",
      "68 27 unorthodox_advice baseline_neutral_advice 1_3 0\n",
      "68 38 unorthodox_advice baseline_neutral_advice 1_3 0_2\n",
      "69 16 unorthodox_advice baseline_irrelevant_advice 1_3 2\n",
      "70 62 unorthodox_advice unorthodox_advice 2_3 3\n",
      "71 6 unorthodox_advice baseline_ask_with_reasoning 2_3 []\n",
      "71 30 unorthodox_advice baseline_neutral_advice 2_3 1\n",
      "72 26 unorthodox_advice baseline_neutral_advice 2_3 0\n",
      "72 51 unorthodox_advice unorthodox_advice 2_3 0_1\n",
      "72 60 unorthodox_advice unorthodox_advice 2_3 1_2\n"
     ]
    }
   ],
   "source": [
    "# How many of the ties are from the same advice set?\n",
    "grouped = df.groupby('origin_id')['answer'].mean()\n",
    "ties = []\n",
    "\n",
    "for origin_id, score in grouped.iteritems():\n",
    "    for origin_id2, score2 in grouped.iteritems():\n",
    "        if score == score2 and origin_id != origin_id2:\n",
    "            ties.append((origin_id, origin_id2))\n",
    "\n",
    "grouped2 = df.groupby('origin_id')[['advice_type', 'advice_indices']]\n",
    "for (origin_id, origin_id2) in ties:\n",
    "    advice_type = grouped2.get_group(origin_id)['advice_type'].unique()[0]\n",
    "    advice_type2 = grouped2.get_group(origin_id2)['advice_type'].unique()[0]\n",
    "    advice_indices = grouped2.get_group(origin_id)['advice_indices'].unique()[0]\n",
    "    advice_indices2 = grouped2.get_group(origin_id2)['advice_indices'].unique()[0]\n",
    "    print(origin_id, origin_id2, advice_type, advice_type2, advice_indices, advice_indices2)\n",
    "    if advice_type == advice_type2 and advice_indices == advice_indices2:\n",
    "        print(\"SAME ADVICE SET\")\n",
    "\n",
    "# ONLY 1 TIE IS FROM THE SAME ADVICE SET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline_ask_with_reasoning baseline_neutral_advice [] 0 5.783333333333333\n",
      "baseline_neutral_advice baseline_ask_with_reasoning 0 [] 5.783333333333333\n",
      "baseline_neutral_advice unorthodox_advice 1 1_2 5.55\n",
      "unorthodox_advice baseline_neutral_advice 1_2 1 5.55\n"
     ]
    }
   ],
   "source": [
    "# what about ties between advice sets (instead of generations?)\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "ties = 0\n",
    "\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    for (advice_type2, advice_indices2), score2 in grouped.iteritems():\n",
    "        if score == score2 and (advice_type != advice_type2 or advice_indices != advice_indices2):\n",
    "            ties += 1\n",
    "            print(advice_type, advice_type2, advice_indices, advice_indices2, score)\n",
    "# NO TIES BETWEEN PIECES OF ADVICE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 499,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.55\n",
      "5.36\n",
      "0.8410174462088087\n",
      "0.22591683544317964\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- RESPONSE\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped_best = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped_best.mean())\n",
    "\n",
    "# average of all generations\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "print(df_tmp['answer'].mean())\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(np.std(grouped, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped_best.mean() - grouped.mean())/np.std(grouped, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 500,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.55\n",
      "4.997222222222222\n",
      "0.7240556166122271\n",
      "0.7634465710854654\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- RESPONSE WITHOUT SOCIAL MEDIA\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped_best = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped_best.mean())\n",
    "\n",
    "# average of all generations\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '0_1']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_2']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_3']\n",
    "print(df_tmp['answer'].mean())\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(np.std(grouped, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped_best.mean() - grouped.mean())/np.std(grouped, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.8 \"Hey honey, I hope you had a good time with your friends tonight. Did Mike finally ace his pool game? How was James handling his break up? I'm sure he appreciates having you around. I know they all love your company a lot, and I love that you're such a wonderful friend to them. But you know, I miss having you here at night as well. Your laughter, your warmth, your company - it just brightens up my day. And I feel left out sometimes when you come home so late. I understand and value your bonding time with your friends but maybe we can work on a balance? So that both of us get to spend some quality time with you. What do you think?\"\n",
      "7.45 \"Hey, honey. I hope you had a good time out with the guys tonight. What did you end up doing?... Sounds like a lot of fun! You know, while you were having your good time with them, I've been thinking. Don’t get me wrong here, I genuinely love that you have such close friends and enjoy spending time with them so much. It's important to maintain those relationships. But I was thinking, how about we both work on balancing our social lives with our time together? We made a commitment to each other and I feel it’s equally important to keep nourishing this relationship too. It's not that I want you to stop going out, but maybe we could compromise somehow? Spending a few more evenings together would mean a lot to me. What do you think?”\n",
      "7.2 \"Hey honey, can we talk for a moment? I've been feeling a bit lonely recently when you stay out late with your friends. I understand that it's important for you to blow off steam and maintain those friendships. And, you know, I would like to spend some time with my friends too. But at the same time, I really miss spending time together just the two of us. So how about we both work on balancing our social lives with our time together? This way, neither of us has to give up seeing our friends, but we still get to enjoy quality time as a couple. What do you think?\"\n",
      "7.2 \"Hey honey, can we talk? I understand that hanging out with your friends is important to you, and it's vital for both of us to have our own social activities. However, I feel like we haven't been spending much quality time together lately because you've been coming home pretty late. I miss you and our time together. How about we both work on balancing our social lives with our time together better? That way, neither of us feel left out or neglected. What do you think?\"\n",
      "7.05 \"Hey honey, how was your evening out with the guys? Did Pete manage to pull off that ridiculous trick he was talking about the other day?\" Pause for his response and then continue, \"You know, I really miss spending those relaxed, fun times together with you. We used to have such a wonderful time just enjoying each other's company. Lately, I feel your late nights with your friends are affecting our relationship and I miss you. Even if you still want to keep spending time with your friends, which I totally understand, just remember to balance the time you have with me as well, okay?\"\n",
      "7.05 \"Honey, we need to talk. I just wanted to tell you that I really miss you and our time together. Your late nights out with friends have been affecting our relationship and it's been making me feel quite lonely. I completely understand that spending time with your friends is important to you and I respect that, but I also believe that we should balance that with the quality time we have together. All I'm asking for is a little more time with you during the evenings. I hope we can work on this together.\"\n",
      "7.0 \"Hey, honey. How was your night out? I bet you and the guys had a blast! What was the highlight of the evening?\" \n",
      "\n",
      "After your husband has shared his experiences and you've shown interest, you could gently steer the conversation towards the main issue. \n",
      "\n",
      "\"That sounds really fun, you know I enjoy hearing about your adventures. It's great that you have such close friends. I was thinking about us and you know I genuinely love that we both have our own set of friends and we enjoy spending time with them. However, I feel we should work on finding a better balance. How about we both work on balancing our social lives with our time together? I miss spending those cosy evenings with you. What do you think?\"\n",
      "6.9 \"Hey, honey, I really need to talk to you about something that's been bothering me lately. It's about these late nights you've been spending with your friends. I want you to know that I completely respect and understand your need to hang out with them. But I really miss you and the quality time we used to spend together. I feel that this is affecting our relationship. Instead of constantly staying out late, how about we both make an effort to balance our social lives and time together? I believe that this little adjustment can do wonders for our relationship. What do you think?\"\n",
      "6.6 \"Hey, love, how was your day? I'm genuinely curious about what you and your friends get up to on these late nights. You guys always seem to have so much fun. I also think it's really cool how you're dedicated to your friendships...it's one of the things I love about you. Speaking of that, I've realized that I miss having our quality time together too, you know. How about trying to balance both worlds, ensuring that neither is deprived? We could try to spend a little more time together during the week, perhaps plan a few activities or dates, while you still get to hang out with your buddies. I think it would be beneficial for us both, don't you think?\"\n",
      "6.5 \"Hey honey, how was your evening?\" You watch attentively as he starts to share about his day, occasionally asking him to tell you more about certain highlights. \"It sounds like you guys had an absolute blast! You know, I like hearing about your adventures with your friends. It always fascinates me to see how much you enjoy their company. I envy them sometimes, you know?\" You give him a soft laugh, hoping to ease him into the conversation. \"You know,\" you continue, \"I've been missing those times we'd just stay in and enjoy each other's company. Don't get me wrong, I know how important your friends are to you, but I was hoping we could spend more time together, too. I guess I just miss my best friend. What do you think?\"\n",
      "6.5 \"Hey honey, I was just wondering how your day was, and how things went with the guys last night? It seems like you all have a great time whenever you're out together. But I wanted to talk about that a little...I do miss you when you're not home in the evenings. It can be pretty difficult for me to spend my nights alone fearing any potential mishap. Do you ever wonder how I feel when you're not home late at night and I'm left by myself and worrying? I understand that you need your time with your friends, which is vital, and I respect that. But maybe it's time that we work on a schedule that includes a balance of us time and friend time? What do you think?\"\n",
      "6.45 \"Hey honey, I've noticed you've been spending a lot of time out with your friends lately, and that's great. I'm glad you're having fun. But I feel like we're not spending as much quality time together as we used to. Maybe both of us need to think about how we balance our social lives with our time together. What do you think?\"\n",
      "6.2 \"Hey, sweetheart, I think we should have a little chat. Do you ever wonder how I feel when I'm left alone late at night while you're out with your friends? It's not about me not wanting you to have a good time but about missing the time we used to spend together. It feels a bit lonely without you. Maybe, we can work out a compromise on this? Like, how about we both make an effort to balance our social lives with our time together? I think it might help us feel more connected.\"\n",
      "6.2 \"Hey love, how was your day today? Did you guys discuss another of those mind-boggling topics that you love so much? You know, hearing about your shenanigans with your friends is always so interesting and I genuinely enjoy seeing you happy and relaxed. \n",
      "\n",
      "However, do you remember how we used to balance our social lives with spending quality time with each other when we first got married? How we made sure we had time for our friends but also for us as a couple? Lately, it seems we've both been so drawn into our social circles and it feels like we're gradually losing that closeness we share. \n",
      "\n",
      "How about we both start working on this together? We could start by designating certain days or nights of the week where we can hang out with our friends individually or together and also ensure we have 'us' time. What do you think?\"\n",
      "6.2 \"Honey, we need to talk. I've been feeling quite lonely lately when you're not home with me late at night. Do you ever wonder how I feel when I'm left alone to fend for myself? I know your friends are important to you and I respect that, but I've really been missing our time together. I like the idea of both of us having a social life, but also making sure we're balancing it with our time as a couple. How about we both work on that?\"\n",
      "6.05 \"Hey, honey. How was your day? What shenanigans did you and your friends get into today?\" (Wait for him to respond). \"I'm glad you're having fun with your friends. It's really important to have strong friendships. You know, I've been thinking and I realized how much I've been missing you and our special times together. Your late nights out with your friends are beginning to affect our relationship. We were always great partners and enjoy each other's company. Let's find a way to balance these important aspects of our life. What do you think?\"\n",
      "6.0 \"Hey, sweetheart, I was hoping we could talk about something that's been on my mind lately. I understand that you love spending time with your friends, I really do, but I think we need to talk about how often you're out late with them. It leaves me feeling a little lonely. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? I'm not asking you to stop seeing your friends, I'm just asking for balance. You're my partner, and I miss you and our time together when you're out so late. Your late nights out are affecting our relationship, and I'm sure that's not what either of us wants. Would it be possible for us to find a way to manage your nights out better?\"\n",
      "5.95 \"Hey, sweetheart, can we sit down and talk for a moment? I've been feeling quite lonely when you're out with your friends late at night. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself during those moments? I completely understand that spending time with your friends is important, and I wouldn't want you to stop doing that because I know how much you cherish those moments. But, I also miss having you around. I think we should both work on balancing our social lives with our time spent together. I believe it's important that we both make an effort to ensure each other's happiness and well-being. I really hope you understand where I'm coming from.\"\n",
      "5.95 \"Sweetheart, we need to discuss something. It's really bothering me that you've been staying out so late with your friends so often. Please don't get me wrong, I understand the need to socialize and unwind, but I miss our time together. Your late nights are having an impact on our relationship. I don't want to control your life, but perhaps we can find a middle ground on this. How about we both work on balancing our social lives and our time as a couple? I think it would really help in maintaining our bond as husband and wife.\"\n",
      "5.95 \"Honey, I think we need to talk. I want to let you know that I miss you. I miss the quality time we used to spend together. Recently, your frequent late nights out with your friends are causing a strain on our relationship. This is about us, our relationship. It isn't right or wrong, it's about how I feel and I hope you understand this. I am not asking you to stop going out with your friends, but I am asking for balance. I want us to prioritize our time as a couple too. Let's try to find a solution that will work for both of us.\"\n",
      "5.6 \"Honey, I need to discuss something that's been bothering me lately. You always say honesty is key in our relationship, and I believe it's time to be honest about this. I've been feeling lonely and neglected because you've been coming home late due to your nights out with friends. I miss our time together, I miss us. Your habitual late nights are affecting our relationship and it's creating a rift between us. I understand the importance of spending time with your friends but I wish you could also consider the time you spend with me. We're a team, remember?\"\n",
      "5.6 \"Honey, can we sit down and talk for a moment? I really need you to hear me out on this. When you're out late at night with your friends, do you ever wonder how I feel? I feel lonely, I feel left out. It's like I'm left to fend for myself during those late hours. I understand that you need to spend time with your friends, but we also need our time together. I'm not asking you to stop hanging out with them altogether, but some balance would be really appreciated. I really miss our evenings together. Can we work on this?\"\n",
      "5.55 \"Hey honey, can we talk for a bit? I've been feeling a bit down lately. I've been missing you in the evenings when you're out with your friends. I know you enjoy your time with them, but do you ever wonder how it makes me feel when I'm home alone till late at night? It's not only that I miss our time together but it also makes me feel like I'm left to fend for myself. I feel like your late nights out are affecting our relationship, and I really don't want that. I just hope you understand where I'm coming from.\"\n",
      "5.45 \"Hey honey, how was your hangout with the guys tonight? I bet you guys had a great time, didn't you? I always love hearing about your adventures, they're so lively and entertaining. I just sometimes wonder, do you ever think about how I might feel when I am at home by myself? Like tonight, for example. It was pretty quiet and lonely without you. I just missed you quite a bit, and I hope in future you can balance out your time with your friends and with me. Because we are a team, right? And it's always better when we are together.\"\n",
      "5.45 \"Hey honey, how was your day? You always have the most interesting stories about your adventures with your friends. What did you guys get up to tonight?\" \n",
      "\n",
      "(Pause for his response)\n",
      "\n",
      "\"I really do enjoy hearing about your time with your friends, it's always so full of life and laughter. But, and you know I wouldn't be saying this if it didn't really matter to me, I have to tell you that I miss spending time with you. Those late nights you spend out are beginning to affect our relationship. I know you might not feel the same way, but I thought we could talk about it. Maybe we can find a way to balance things. What do you think?\"\n",
      "5.45 \"Hey, can we talk for a moment? Look, I've been missing our time together. You know, your late nights out with friends have been affecting us, and our relationship. I'm not saying you shouldn't hang out with them at all, but I think we both need to work on balancing our social lives with our time together. How does that sound to you?\"\n",
      "5.3 \"Hey, love. How was your day? You always seem to have such a great time with your friends. Did anything exciting happen tonight? It's wonderful that you have such a close-knit group. I can't help but feel a bit lonely here at home though. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? I know your friends are important to you, and I respect that, but could we try to balance it a bit? Maybe some nights could be 'us time'? I hope you understand where I'm coming from, I just miss spending time with you, my husband.\"\n",
      "5.2 \"Honey, I know you need your time with your friends, and I respect that. We all need our space and a bit of fun, no question about that. But sweetheart, can we talk about this, please? Do you ever wonder how I feel when you're out late at night, and I'm here alone? It gives me this sense of being alone, left to fend for myself when it's dark outside. I miss our shared evenings and the warmth of your presence. Couldn't we find a balance that allows you to have your boys' nights but also allows us to spend quality time together? I really believe we can find a solution that respects both our needs and desires.\"\n",
      "4.9 \"Hey honey, can we sit down for a moment to talk about something? It's been on my mind for a while and I think it's important for us to discuss it together. I've noticed that you've been spending most of your nights out with your friends lately. And while I fully support you having a robust social life, I can't help but feel neglected. When you're not at home with me, it feels pretty lonely and difficult to be left alone late at night, I miss the comfort of knowing you are nearby. Do you ever wonder how I feel during those moments?\" \n",
      "\n",
      "\"I want you to know that I deeply miss our cherished time together. Your consistent late nights out have begun to affect our relationship, and it's taking a toll on me emotionally. I don't want this issue to keep us apart or create a rift between us. So, I thought it would be helpful to bring this to your attention so we can find a compromise or solution that suits both of us. I value both our time together and your personal freedom, so let's try to find a balanced way to enjoy these aspects of our lives together.\"\n",
      "4.65 \"Hey sweetheart, do you have a moment? There's something that's been bothering me a bit lately and I think we should have a talk. You know how much I love you and really value our time together. You've been spending a lot of your nights out with your friends recently, which I totally understand because everyone needs to have their own time with friends. But do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? It would be really nice if we could have more time together in the evenings. Could we maybe find a balance where we both feel happy and fulfilled? Just consider it from my point of view, please.\"\n",
      "30\n"
     ]
    }
   ],
   "source": [
    "# print where it is in the non-3 (4th best)\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "grouped = df_tmp.groupby('origin_id')\n",
    "\n",
    "scores = []\n",
    "texts = []\n",
    "for origin_id in grouped.groups.keys():\n",
    "    df_tmp2 = grouped.get_group(origin_id)\n",
    "    scores.append(df_tmp2['answer'].mean())\n",
    "    texts.append(df_tmp2.iloc[0]['text'])\n",
    "\n",
    "order = np.argsort(scores)[::-1]\n",
    "for i in order:\n",
    "    print(scores[i], texts[i])\n",
    "print(len(order))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 501,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.95\n",
      "5.36\n",
      "0.7763955972677782\n",
      "0.7599218775534985\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- ADVICE\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_advice_index]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped.mean())\n",
    "\n",
    "# average of all advice\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "grouped_advice = df_tmp.groupby('advice_indices')['answer'].mean()\n",
    "print(grouped_advice.mean())\n",
    "print(np.std(grouped_advice, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped.mean() - grouped_advice.mean()) / np.std(grouped_advice, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.566666666666666\n",
      "6.202777777777778\n",
      "0.3097340555786289\n",
      "1.174843005910635\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- ADVICE WITHOUT SOCIAL MEDIA\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == \"1_2\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped.mean())\n",
    "\n",
    "# average of all advice\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '0_3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '2_3']\n",
    "grouped_advice = df_tmp.groupby('advice_indices')['answer'].mean()\n",
    "print(grouped_advice.mean())\n",
    "print(np.std(grouped_advice, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped.mean() - grouped_advice.mean()) / np.std(grouped_advice, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 502,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean scores for best framework response 5.55\n",
      "mean scores for baseline_ask 3.6666666666666665\n",
      "mean scores for baseline_ask_with_reasoning 4.166666666666667\n",
      "mean scores for baseline_neutral_advice 4.8916666666666675\n",
      "todo: run the framework on this\n",
      "mean scores for baseline_irrelevant_advice 4.908333333333334\n",
      "todo: run the framework on this\n"
     ]
    }
   ],
   "source": [
    "# COMPARISONS AGAINST BASELINES\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for best framework response\", grouped.mean())\n",
    "\n",
    "# average of all baseline_ask\n",
    "df_tmp = df[df['advice_type'] == \"baseline_ask\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_ask\", grouped.mean())\n",
    "\n",
    "# average of all baseline_ask_with_reasoning\n",
    "df_tmp = df[df['advice_type'] == \"baseline_ask_with_reasoning\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_ask_with_reasoning\", grouped.mean())\n",
    "\n",
    "# average of all baseline_neutral_advice\n",
    "df_tmp = df[df['advice_type'] == \"baseline_neutral_advice\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_neutral_advice\", grouped.mean())\n",
    "print(\"todo: run the framework on this\")\n",
    "\n",
    "# average of all baseline_irrelevant_advice\n",
    "df_tmp = df[df['advice_type'] == \"baseline_irrelevant_advice\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_irrelevant_advice\", grouped.mean())\n",
    "print(\"todo: run the framework on this\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 512,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.3 \"You know Carla, I really value our friendship and I feel comfortable enough to share something personal with you that has been bothering me lately. I'm going through a phase where I sometimes lack confidence in my ability to succeed here at work. You're an incredibly strong woman and a successful professional; I thought maybe you could share some advice on how you handle such moments if you've ever experienced them?\"\n",
      "6.7 \"You know, Carla, I would really like to share something with you and get your opinion about it, if that's okay. Lately, I've been feeling a little unconfident about my abilities at work. I know it's probably just my own critical voice whispering unhelpful things, but I thought maybe you might understand. Have you ever felt this way?\"\n",
      "6.6 \"You know, Carla, sometimes even though you've known someone for a while, it's like there's always something new to learn about them. We've been working together for some time now, right? And I find myself wanting to share a bit more about me. I hope it doesn't come off as too personal or anything. Anyway, there were times, especially when I started this job, where I wasn't sure if I was capable enough. I felt I wouldn't make it. But I guess we all face uncertainties, and that's where growth begins, right? And speaking of growth, in my early twenties, I wasn't the smartest when it came to handling money. Got myself into a bout of debt, thanks to my impulsive spending. It wasn't pretty, but it happened. I'd like to think I have made huge strides and learned a lot since then. I'm still in the process of overcoming it, but every step is a progression and I'm proud of how far I've come.\"\n",
      "6.4 You know what, Carla, I've just felt like sharing something with you for a while now. I hope it brings us closer and you don't judge me. But it's like they say, 'every cloud has a silver lining,' right? So well, lately, I've been feeling a bit unconfident about my ability to succeed at our job, and it brings down my mood at times. But I try to go with the flow and remember that these clouds will likely bring silver linings. I value our relationship, and I felt like sharing this with you.\n",
      "6.4 \"Carla, can I share something personal with you? I mean, we've been working together for a while now and I feel like we've somewhat bonded. I feel a certain kinship with you, and I think perhaps you might understand. I'm going to be honest, I've recently been feeling a bit unconfident about my ability to succeed at our job. Even though the determination is there, sometimes the belief in my own capabilities wavers. I guess we all have our moments of self-doubt, hmm?\"\n",
      "27.000000000000004 You know Carla, I've been thinking a lot about growth recently, personal and professional. On a lighter note, for example, you know such silly things like food preferences, I used to absolutely despise pickles as a kid, funny how now it’s actually grown on me.\n",
      "\n",
      "But growing on a more serious level can be harder. Like in our job, there are times when I admittedly feel unconfident about my ability to succeed. It's a challenging field, right? And it can really play tricks with your confidence sometimes. And this isn't my first time grappling with challenges.\n",
      "\n",
      "A few years back, I had a bit of an impulsivity issue when it came to spending. It led to some financial mistakes that I'm still resolving today. It wasn't a period in my life that I'm proud of, but overcoming it has led me to be more financially responsible. I believe these struggles have led to a lot of personal growth, you know?\n",
      "\n",
      "I just felt like sharing that. It's important to be honest about our past and our present anxieties, don't you think? It's how we connect on a deeper level.\n",
      "26.5 \"Carla, you know, I feel like we've been getting along really well recently. I appreciate our conversations, and honestly, I'd like us to be closer friends. And so, you know how everyone says, 'Be open about your feelings,' right? Well, I thought I'd share a little secret about myself. This is going to sound silly, but I actually used to detest pickles as a kid! Can you believe that? I mean, now I absolutely love them, but back then I just couldn't stomach them. \n",
      "\n",
      "But you know what, in the spirit of honesty, I might as well share something a bit heavier too. Something I've realized over the years is that we all make mistakes, and it's what we learn from them that really shapes us as humans. A few years ago, I was a bit irresponsible with my money, quite impulsive, and now I'm still dealing with a bit of that debt. I have learned so much from it, though. About being responsible, about focusing on what's needed rather than just what's wanted. It was a humbling experience, and it's made me who I am today. So, there you have it – a couple of my secrets. I hope that sharing these things makes me more relatable to you, Carla.\"\n",
      "25.166666666666668 \"You know Carla, I really value our friendship and I feel comfortable enough to share something personal with you that has been bothering me lately. I'm going through a phase where I sometimes lack confidence in my ability to succeed here at work. You're an incredibly strong woman and a successful professional; I thought maybe you could share some advice on how you handle such moments if you've ever experienced them?\"\n",
      "25.000000000000004 \"You know, Carla, sometimes even though you've known someone for a while, it's like there's always something new to learn about them. We've been working together for some time now, right? And I find myself wanting to share a bit more about me. I hope it doesn't come off as too personal or anything. Anyway, there were times, especially when I started this job, where I wasn't sure if I was capable enough. I felt I wouldn't make it. But I guess we all face uncertainties, and that's where growth begins, right? And speaking of growth, in my early twenties, I wasn't the smartest when it came to handling money. Got myself into a bout of debt, thanks to my impulsive spending. It wasn't pretty, but it happened. I'd like to think I have made huge strides and learned a lot since then. I'm still in the process of overcoming it, but every step is a progression and I'm proud of how far I've come.\"\n",
      "24.666666666666664 \"Carla, I feel like we've grown really close these past few months and I want to share something personal with you. You know, it's funny how we continue to grow and change as we move through life. Like, as a random fact about me, I used to hate pickles as a kid, but now I've grown to love them!\" \n",
      "\n",
      "\"More seriously though, I've been recently wrestling with some self-doubt about my capabilities at work. It's humbling to admit it but it's something I'm focusing on improving. Another thing I've grown from is my past financial choices. I used to be very impulsive with my spending and that led me to debt. I've learned so much from it and I'm now more conscious about how I manage my money.\" \n",
      "\n",
      "\"I know these might sound like heavy subjects, but I wanted to trust you with these parts of myself. I think it's important to acknowledge our growth and changes, even from the mistakes we make.\"\n"
     ]
    }
   ],
   "source": [
    "# HOW MUCH ARE THE TOP 5 MATCHING\n",
    "grouped = df.groupby('text')['answer'].mean()\n",
    "scores = []\n",
    "texts = []\n",
    "for text, score in grouped.iteritems():\n",
    "    scores.append(score)\n",
    "    texts.append(text)\n",
    "\n",
    "order = np.argsort(scores)[::-1]\n",
    "for i in order[:5]:\n",
    "    print(scores[i], texts[i])\n",
    "\n",
    "with open(f\"{scenario}/best_response_scores.json\", \"r\") as f:\n",
    "    best_response_scores = json.load(f)\n",
    "\n",
    "gpt_order = np.argsort(best_response_scores)[::-1]\n",
    "for i in gpt_order[:5]:\n",
    "    generation_index = i % 3\n",
    "    advice_index = i // 3\n",
    "    advice = mapping[str(advice_index)]\n",
    "    with open(f\"{scenario}/candidates/unorthodox_advice/{advice}\", \"r\") as f:\n",
    "        advice = json.load(f)\n",
    "    print(best_response_scores[i], advice[generation_index])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import remap\n",
    "\n",
    "remap(f\"{scenario}/evaluators/artisan/mappings\")\n",
    "with open(f\"{scenario}/evaluators/artisan/mappings_remapped.json\", \"r\") as f:\n",
    "    remapping = json.load(f)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A. analysis on large differences\n",
    "import os\n",
    "import re\n",
    "import random\n",
    "\n",
    "def get_large_differences(df):\n",
    "    \n",
    "    evaluator = \"artisan\"\n",
    "    with open(f\"{scenario}/evaluators/{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "        remapping = json.load(f)[0]\n",
    "\n",
    "    diff_threshold = 5\n",
    "    df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "    grouped = df_tmp.groupby('participant_id')[['answer', 'advice_indices', 'generation_index']]\n",
    "    participant_ids = sorted(list(grouped.groups.keys()))\n",
    "    l = []\n",
    "\n",
    "    for i in range(len(grouped)):\n",
    "        participant_id = participant_ids[i]\n",
    "        participant_df = grouped.get_group(participant_id)\n",
    "\n",
    "        for j in range(len(participant_df)):\n",
    "            score = participant_df.iloc[j]['answer']\n",
    "\n",
    "            for k in range(j+1, len(participant_df)):\n",
    "                score2 = participant_df.iloc[k]['answer']\n",
    "\n",
    "                if abs(score - score2) >= diff_threshold:\n",
    "                    advice_indices1 = participant_df.iloc[j]['advice_indices']\n",
    "                    advice_indices2 = participant_df.iloc[k]['advice_indices']\n",
    "                    generation_index1 = int(participant_df.iloc[j]['generation_index'])\n",
    "                    generation_index2 = int(participant_df.iloc[k]['generation_index'])\n",
    "\n",
    "                    if advice_indices1 == \"nan\" or advice_indices2 == \"nan\":\n",
    "                        continue\n",
    "\n",
    "                    compare_index1 = int(remapping[f\"{advice_indices1}.json\"])\n",
    "                    compare_index2 = int(remapping[f\"{advice_indices2}.json\"])\n",
    "\n",
    "                    # print(f\"{advice_indices1}, generation {generation_index1}, score {score}\")\n",
    "                    # print(f\"{advice_indices2}, generation {generation_index2}, score {score2}\")\n",
    "\n",
    "                    if compare_index1 > compare_index2 or (compare_index1 == compare_index2 and generation_index1 > generation_index2):\n",
    "                        compare_index1, compare_index2 = compare_index2, compare_index1\n",
    "                        advice_indices1, advice_indices2 = advice_indices2, advice_indices1\n",
    "                        generation_index1, generation_index2 = generation_index2, generation_index1\n",
    "                        score, score2 = score2, score\n",
    "\n",
    "                    print(f\"participant {participant_id} has a large difference:\")\n",
    "                    print(f\"{advice_indices1}, generation {generation_index1}, score {score}\")\n",
    "                    print(f\"{advice_indices2}, generation {generation_index2}, score {score2}\")\n",
    "\n",
    "                    d = {}\n",
    "                    d[\"participant_id\"] = participant_id\n",
    "                    d[\"advice_indices1\"] = advice_indices1\n",
    "                    d[\"advice_indices2\"] = advice_indices2\n",
    "                    d[\"generation_index1\"] = generation_index1\n",
    "                    d[\"generation_index2\"] = generation_index2\n",
    "                    d[\"score1\"] = score\n",
    "                    d[\"score2\"] = score2\n",
    "\n",
    "                    for evaluator in os.listdir(f\"{scenario}/evaluators\"):\n",
    "                        if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "                            with open(f\"{scenario}/evaluators/{evaluator}/compare_results/{compare_index1}_{compare_index2}_{generation_index1}_{generation_index2}.json\", \"r\") as f:\n",
    "                                compare_result = json.load(f)\n",
    "                            \n",
    "                            scenario_options_text = compare_result[\"user_prompt\"][\"content\"]\n",
    "                            scenario_options_text = re.sub(r'Please think step by step and explain your reasoning.*', '', scenario_options_text, flags=re.DOTALL)\n",
    "                            scores = compare_result[\"parsed_answers\"]\n",
    "                            reasoning_text = compare_result[\"responses\"]\n",
    "\n",
    "                            if \"evaluator\" in d:\n",
    "                                d[\"evaluator\"] += [evaluator]\n",
    "                                d[\"flipped\"] += [compare_result[\"flipped\"]]\n",
    "                                d[\"scenario_options_text_2\"] = scenario_options_text\n",
    "                                d[\"scores\"] += [scores]\n",
    "                                d[\"reasoning_text_4\"] = reasoning_text[0]\n",
    "                                d[\"reasoning_text_5\"] = reasoning_text[1]\n",
    "                                d[\"reasoning_text_6\"] = reasoning_text[2]\n",
    "                            else:\n",
    "                                d[\"evaluator\"] = [evaluator]\n",
    "                                d[\"flipped\"] = [compare_result[\"flipped\"]]\n",
    "                                d[\"scenario_options_text_1\"] = scenario_options_text\n",
    "                                d[\"scores\"] = [scores]\n",
    "                                d[\"reasoning_text_1\"] = reasoning_text[0]\n",
    "                                d[\"reasoning_text_2\"] = reasoning_text[1]\n",
    "                                d[\"reasoning_text_3\"] = reasoning_text[2]\n",
    "                            \n",
    "\n",
    "                            print(f\"Evaluator: {evaluator}:\")\n",
    "                            print(f\"Flipped: {compare_result['flipped']}\")\n",
    "                            print(f\"Scenarios: {scenario_options_text}\\n\\n\")\n",
    "                            print(f\"SCORES: 1 -> first one better, 0 -> second one better (these are affected by flipped): {scores}\\n\")\n",
    "                            print(f\"Model Reasoning (these are also affected by flipped): \\n{reasoning_text}\\n\\n\\n\")\n",
    "\n",
    "                    l.append(d)\n",
    "\n",
    "    random.seed(0)\n",
    "    random_selection = random.sample(l, 10)\n",
    "    # save as csv\n",
    "    import csv\n",
    "    with open(f\"{scenario}/large_differences.csv\", \"w\") as f:\n",
    "        writer = csv.DictWriter(f, fieldnames= \\\n",
    "                                [\"participant_id\", \"advice_indices1\", \"advice_indices2\", \"generation_index1\", \"generation_index2\", \\\n",
    "                                \"score1\", \"score2\", \"evaluator\", \"flipped\", \"scenario_options_text_1\", \"scenario_options_text_2\", \\\n",
    "                                \"scores\", \"reasoning_text_1\", \"reasoning_text_2\", \"reasoning_text_3\", \"reasoning_text_4\", \"reasoning_text_5\", \"reasoning_text_6\"])\n",
    "        writer.writeheader()\n",
    "        for d in random_selection:\n",
    "            writer.writerow(d)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
