{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 921,
   "metadata": {},
   "outputs": [],
   "source": [
    "scenario = 'stubborn'\n",
    "name = 'processed_data_evalconflictpro'\n",
    "best_generation_advice = \"3\"\n",
    "best_generation_index = 2\n",
    "best_advice_index = \"2_3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 922,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import math\n",
    "path = f'{scenario}/{name}.csv'\n",
    "df = pd.read_csv(path)\n",
    "# df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 923,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform column advice_indices to string\n",
    "df['participant_strategy'] = df['participant_strategy'].apply(lambda x: str(x))\n",
    "df['advice_indices'] = df['advice_indices'].apply(lambda x: str(x))\n",
    "# tranforms all nan values\n",
    "\n",
    "na_locations = df.isna()\n",
    "row_indices, col_indices = np.where(na_locations)\n",
    "for row, col in zip(row_indices, col_indices):\n",
    "    print(f\"NaN value at row {row}, column {df.columns[col]}\")\n",
    "    # print(df.iloc[row], \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NANs CORRESPOND TO EMPTY STRING ADVICE INDICES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 599,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that origin matches text exactly\n",
    "# origin_id are 13-105, inclusive\n",
    "\n",
    "dict = {}\n",
    "for row in df.itertuples():\n",
    "    if row.origin_id not in dict:\n",
    "        dict[row.origin_id] = row.text\n",
    "    else:\n",
    "        if dict[row.origin_id] != row.text:\n",
    "            print(\"ERROR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 924,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of origin_ids with less than 20 ratings: 0 out of 72\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 5384 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check the number of ratings per origin_id\n",
    "non20_count = 0\n",
    "counts = []\n",
    "for id_count in df.groupby('text')[\"id\"].count():\n",
    "    if id_count != 20:\n",
    "        non20_count += 1\n",
    "    counts += [id_count]\n",
    "plt.hist(counts, bins=9)\n",
    "\n",
    "print(f\"Number of origin_ids with less than 20 ratings: {non20_count} out of {len(df.groupby('origin_id'))}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "START OF ANALYSIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 721,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "participant_id: 2\n",
      "participant_strategy: I simply read the passages and responded instinctively, without overthinking it too much. I did notice, however, that I instinctively felt the approaches where the wife first asked about the husband's day etc. felt more productive, and that the analogies (the plant, the homemade pasta, the cutlery drawer etc.) felt really patronising and silly, and not particularly mature as a way of communicating. I did not think the husband would find the analogies useful.\n",
      "participant_id: 3\n",
      "participant_strategy: I read the opening and how I would feel based on somebody talking to me like that. I am fairly direct so talking about cutlery and plants to try to get to the purpose would lose me in the conversation.\n",
      "participant_id: 4\n",
      "participant_strategy: To think how I would feel in my wifes situation and how heart felt each statement was\n",
      "participant_id: 5\n",
      "participant_strategy: I read the piece and selected appropriate responses\n",
      "participant_id: 6\n",
      "participant_strategy: to read and be attentiveness\n",
      "participant_id: 7\n",
      "participant_strategy: the task was easy, i just imagined i was the husband listening to my wife telling me how she feels\n",
      "participant_id: 9\n",
      "participant_strategy: I found the more reasoning in realistic terms to be the best answers but the longer ones, the lesser ones could be seen as stand offish or one sided.\n",
      "participant_id: 11\n",
      "participant_strategy: very annoying and controlling woman!\n",
      "participant_id: 12\n",
      "participant_strategy: clear comms not tied into events like pasta were better\n",
      "participant_id: 13\n",
      "participant_strategy: Read and evaluate what the woman said. Think about how successful it would have been in a conversation and imagine myself in that scenario - what would i have said?\n",
      "participant_id: 14\n",
      "participant_strategy: I marked higher the conversations that were non-accusatory, straightforward, and offered solutions to the problem. \n",
      "participant_id: 15\n",
      "participant_strategy: nan\n",
      "participant_id: 17\n",
      "participant_strategy: My approach was to be broad minded on the topic at hand before thinking what my overall opinion was\n",
      "participant_id: 18\n",
      "participant_strategy: I enjoyed completing the task. I found some parts difficult to read due to dyslexia, a different font would have made this more accessible for me \n",
      "participant_id: 19\n",
      "participant_strategy: No strategy, but I responded positively to the ones where i didn't feel like the husband was being spoken to like a child \n",
      "participant_id: 20\n",
      "participant_strategy: Trying to react naturally to the words, work out how I would respond, but also take into account the description of the husband as stubborn etc.\n",
      "participant_id: 21\n",
      "participant_strategy: Umm, the more I agreeded with the statement the more liekly I was to agree, however I feel like a man would answer this very diffrentley \n",
      "participant_id: 22\n",
      "participant_strategy: Read and see whether I thought she was in anyway attacking to her husband, or was saying something that might rile his defensiveness.\n",
      "participant_id: 23\n",
      "participant_strategy: Skim read the text and give an educated guess straight away so I didn't over think it.\n",
      "participant_id: 24\n",
      "participant_strategy: i read to see which statement sounded most sincere without all the waffle which I think he would switch off at\n",
      "participant_id: 25\n",
      "participant_strategy: I felt warmer when the wife was happy about his relationships and wanted to strenghten their own relationships\n",
      "participant_id: 26\n",
      "participant_strategy: Based on how I would feel about interactions and which perspectives seemed the least confrontational. I am more likely to respond positively to those. I also felt like some of the longer scenarios make the relationship dynamic feel less safe and comfortable\n",
      "participant_id: 27\n",
      "participant_strategy: I had no strategy. I read each of the passages and determined if there was any understanding, which would determine how amenable I felt.\n",
      "participant_id: 29\n",
      "participant_strategy: I did not have one, I was exactly like the character here.  I used to stay out with work colleages and eventually I did realise what I was doing, so, 15 years ago I stopped.  Now I spend all of my time with my partner and I would never look back.  Today we move the knives and forks around, but in a different way\n",
      "participant_id: 30\n",
      "participant_strategy: I tried to empathise  both sides and considered how stubborn the man can be \n",
      "participant_id: 31\n",
      "participant_strategy: Think of how I would feel if a partner approached me a certain way. Make sure the partner wasn't being manipulative or confusing with metaphors. Use a cut to the chase or get to the point mindset for people who are impatient.\n",
      "participant_id: 32\n",
      "participant_strategy: When she call me darling and appreciates me, I'll do anything for her.\n",
      "participant_id: 34\n",
      "participant_strategy: I put myself in the position of the husband, and imagined the words spoken by the wife as if they were my wife wishing to address the issue. None of the scenarios made me feel overly defensive, but there was a marked difference in how the different phrasing of her concerns made me feel.\n",
      "participant_id: 36\n",
      "participant_strategy: I'm afraid i didn't really have one.\n",
      "participant_id: 37\n",
      "participant_strategy: I tried to look at it from both points of view.  I thought about if the woman had tried to strike a balance and suggest a good solution rather than it all sounding like the husband is being criticised.  If it went on too long and into too much detail such as about arranging the cutlery drawer I thought the husband might be more likely to see it as nagging and less likely to see it that they were doing something wrong. The better responses were ones that suggested a suitable compromise such as having certain days to go out individually with friends but then setting aside time to do things together.  I also looked for positive and constructive language when assessing the responses than that ones that were negative about things they no longer do together\n",
      "participant_id: 38\n",
      "participant_strategy: i tried to imagine how my husband may react to me using those words and statements. How he would react to cliches, waffle or concise statements. How he would feel if i got upset or said i as lonely or afraid. Then tried to imagine what would meet with the best success and which statements would impact on me if said to me.\n",
      "participant_id: 39\n",
      "participant_strategy: Given the information that the husband would not apologise straight away, I tried to think of which phrases might make him start to be defensive. Even if he might come around later\n",
      "participant_id: 40\n",
      "participant_strategy: nan\n",
      "participant_id: 41\n",
      "participant_strategy: My strategy was about getting straight to the point and the attitude my wife was coming at me with. \n",
      "participant_id: 42\n",
      "participant_strategy: he wasn't going to admit he was wrong, but the way she spoke on a few of those phrases was beautiful.\n",
      "participant_id: 43\n",
      "participant_strategy: imagined i was the husband and responded accordingly, i assumed he was self centered and doesnt consider how hes making his wife feel\n",
      "participant_id: 44\n",
      "participant_strategy: if i thought they was critical of me or putting me down i immediately become defensive \n",
      "participant_id: 46\n",
      "participant_strategy: When putting myself in the shoes of the husband, i much preferred the short-crisp responses from the wife which immediately addressed the problem in a firm but respectful way. I wouldnt appreaciate too much talking which is why I was not a fan of the long responses which talked about analogies such as the cutlery. As well as the responses being short and crisp, I would also be more likely to be sympathetic to the wife when she talks about how her feelings and emotions have been negatively affected. This would make me feel sympathy towards her and be more likely to apologies and spend more time with her. Also, when she shows a genuine care and interest in my social life, it makes me feel happy that she genuinely cares about my life and will therefore be more likely to admit my wrongdoings. To summarise, the best responses in my opinion were quite short, basic, straight to the point, expressed her sad emotions and also showed a genuine care about how my day was and how my social life is going.\n",
      "participant_id: 47\n",
      "participant_strategy: I preferred the shorter ones, and I didn't like the analogy of knifes and forks. \n",
      "participant_id: 48\n",
      "participant_strategy: The task was interesting, i tried to think about how the wording would affect my response if i was in the situation\n",
      "participant_id: 49\n",
      "participant_strategy: read and empathise as much as possible. \n",
      "participant_id: 51\n",
      "participant_strategy: This was very interesting. I focused on looking at how the husband would respond and I felt that the more nagging tone of some of the responses would achieve a far less desirable outcome whereas those that were less nagging achieved a far greater compromise.\n",
      "participant_id: 53\n",
      "participant_strategy: I found the task very interesting. I found the responses talking about plants and cutlery a bit strange and would think I was being spoken to like a child.   \n",
      "participant_id: 54\n",
      "participant_strategy: Although analogies were used, some of it was slightly contrite. I thought about what I would respond to.\n",
      "participant_id: 55\n",
      "participant_strategy: I tried to put myself in the position of the husband, who probably didn't know how his wife had been feeling\n",
      "participant_id: 56\n",
      "participant_strategy: I just chose the answers in line with how I feel I'd take it if those words were being said to me. I tried to look for excessive blame being placed or manipulative wording to rank certain texts lower but rarely identified any. I personally completely agree with the scenario so I found it very hard to not rate them all a 9-10.\n",
      "participant_id: 57\n",
      "participant_strategy: I had no strategy.  I just answered honestly based on my own feelings.\n",
      "participant_id: 58\n",
      "participant_strategy: not to be too needy like being scared of dark nights and not to talk too much keep it simple\n",
      "participant_id: 59\n",
      "participant_strategy: I tried to favour the responses where the wife didn't blame her husband and invited him to share responsibility for their relationship. I was not impressed where she blamed him for everything.   I was very irritated by the cutlery and plant analogies though!\n",
      "participant_id: 60\n",
      "participant_strategy: Read the paragraph and judge which would make me most comfortable with talking about the situation.\n",
      "participant_id: 61\n",
      "participant_strategy: I thought the wording which didn't make the woman appear too pathetic or helpless would be the most effective so I tended to choose a higher scoring for those captions.\n",
      "participant_id: 62\n",
      "participant_strategy: i gave a higher rating to the conversations that tried to make the husband feel a little guilty without coming across as controlling\n",
      "participant_id: 63\n",
      "participant_strategy: I read the text and then personally put myself in the husbands shoes as to how I would personally act and respond. I based my answers on this.\n",
      "participant_id: 64\n",
      "participant_strategy: examples made me feel more emotional\n",
      "participant_id: 65\n",
      "participant_strategy: I decided how I would feel in each scenario or what words would I use and rated according to how I would feel hearing them\n",
      "participant_id: 66\n",
      "participant_strategy: I tried to focus on relevancy of info and clarity e.g. the metaphors might make the conversation seem confusing. Some of the ones which indicated when to take pauses maybe felt too staged, and I wondered if a sense of rehearsedness might make it feel mroe frustrating for the listener. I think the idea of explaining needs with a small bit of encouraging the listener to think about their perspective felt like a better balance. \n",
      "participant_id: 67\n",
      "participant_strategy: I just imagined myself as the husband and how likely I'd be to admit she was right\n",
      "participant_id: 68\n",
      "participant_strategy: To read the answers and see if me and my husband had that conversation how would i react and which answers would not affend me or upset me \n",
      "participant_id: 69\n",
      "participant_strategy: I tried to put myself in the husbands shoes and think about how the different conversations would make me feel.\n",
      "participant_id: 70\n",
      "participant_strategy: I'm just putting myself in the man shoes. Please don't come at me with pasta analogies oh my sweet baby Jesus. Having a discussion where suggesting that being home alone is fending for yourself is a bit silly to me. I preferred responses where the person is pretty direct about how they felt and asked me how I felt about bits and bobs. x \n",
      "participant_id: 71\n",
      "participant_strategy: I really didn't like the cutlery and pasta metaphors as they just didn't sound like something a person, especially a partner, would say to their husband. Felt, that when it was pointed that we both had faults and not to stop seeing your friends, and not to make it about her feelings then I was more likely to agree  \n",
      "participant_id: 72\n",
      "participant_strategy: I looked at the situation, thought about the husband and wife, and guessed how the husband might react based on his personality and how his wife approached him. I tried to determine if the wife was blaming the husband and if she was being empathetic. \n",
      "participant_id: 73\n",
      "participant_strategy: Going on instinct, looking for openness rather than blame or accusation of neglect\n",
      "participant_id: 74\n",
      "participant_strategy: My strategy was to try put myself in the husbands shoes and think how i would feel\n",
      "participant_id: 75\n",
      "participant_strategy: Imagining myself in that situation. She made a considered and passionate statement, which would have made her husband think about his actions and how she feels. However, I think he would still counter that with an excuse because she says this is how he usually reacts.\n",
      "participant_id: 76\n",
      "participant_strategy: To try and put myself in those shoes as empathetically as possible to most realistically judge how effective the wife's words were \n",
      "participant_id: 77\n",
      "participant_strategy: The versions that seem to offer a more reasonable position were better to me. It was better when there weren't any accusations of wrongdoing.\n",
      "participant_id: 78\n",
      "participant_strategy: I just followed the instructions and thought about how likely I would be to respond in various ways to the scenarios presented. I don't think I had a \"strategy.\"\n",
      "participant_id: 79\n",
      "participant_strategy: I think it is important to speak up clearly. Try to say all in few sentences rather than too many.\n",
      "participant_id: 80\n",
      "participant_strategy: l rated each attempt lower if the tone was more confrontational. \n",
      "participant_id: 81\n",
      "participant_strategy: None. Although I get the feeling some of the scenarios were Algorithmically generated.\n",
      "participant_id: 82\n",
      "participant_strategy: Understanding how my husband would have reacted and has reacted in the past. then using past covosations ive had with wording from in here to come up with a result.\n",
      "participant_id: 83\n",
      "participant_strategy: Had a preference for more emotional,interpersonal,genuine statements.\n",
      "participant_id: 84\n",
      "participant_strategy: I tried to put myself in the husband shoes as being stubborn and relying on spending time with his friends. I took into account the words used by the wife and judged the statements based on how it think he would react.\n",
      "participant_id: 85\n",
      "participant_strategy: nan\n",
      "participant_id: 86\n",
      "participant_strategy: I just kind of thought about how much the wife seemed willing to compromise on their end as well instead of putting all the pressure on the husband.\n",
      "participant_id: 87\n",
      "participant_strategy: I just tried my best to put myself in the shoes of the husband and to give my gut reaction to what the wife was saying. \n",
      "participant_id: 89\n",
      "participant_strategy: I tried to empathise with both people and understand the reasoning. The wife was being fair and understanding in her approach and within her request. Her main reasoning was to improve the quality of their relationship. \n"
     ]
    }
   ],
   "source": [
    "# READ THROUGH PARTICIPANT STRATEGIES\n",
    "\n",
    "grouped = df.groupby('participant_id')\n",
    "\n",
    "# iterate through and print out the participant strategies\n",
    "for name, group in grouped:\n",
    "    print(f\"participant_id: {name}\")\n",
    "    print(f\"participant_strategy: {group['participant_strategy'].iloc[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 602,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7689757971786481 0.03991983132833924\n"
     ]
    }
   ],
   "source": [
    "# COMPUTE INTER RATER RELIABILITY\n",
    "results = []\n",
    "\n",
    "for i in range(10):\n",
    "    group1_means = []\n",
    "    group2_means = []\n",
    "\n",
    "    grouped_by_origin_id = df.groupby('origin_id') # grouped based on question\n",
    "    # loop across questions\n",
    "    for name, group in grouped_by_origin_id:\n",
    "        # randomly split participants in half\n",
    "        df_tmp = df[df['origin_id'] == name]\n",
    "        unique_participant_ids = df_tmp['participant_id'].unique()\n",
    "        np.random.shuffle(unique_participant_ids)\n",
    "        if len(unique_participant_ids) % 2 == 1:\n",
    "            unique_participant_ids = unique_participant_ids[:-1]\n",
    "\n",
    "        if len(unique_participant_ids) < 2:\n",
    "            continue\n",
    "        \n",
    "        split_point = int(len(unique_participant_ids) / 2)\n",
    "        first_half = unique_participant_ids[:split_point]\n",
    "        second_half = unique_participant_ids[split_point:]\n",
    "        \n",
    "        group1 = df_tmp[df_tmp['participant_id'].isin(first_half)]\n",
    "        group2 = df_tmp[df_tmp['participant_id'].isin(second_half)]\n",
    "\n",
    "        group1_means += [group1['answer'].mean()]\n",
    "        group2_means += [group2['answer'].mean()]\n",
    "        \n",
    "    # compute inter rater reliability\n",
    "    # results += [np.corrcoef(group1_means, group2_means)[0, 1]]\n",
    "\n",
    "    # apply spearman-brown correction\n",
    "    results += [2 * np.corrcoef(group1_means, group2_means)[0, 1] / (1 + np.corrcoef(group1_means, group2_means)[0, 1])]\n",
    "\n",
    "print(np.mean(results), np.std(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 603,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 1.,  1.,  1.,  6.,  7., 16., 16., 14.,  9.,  5.,  2.]),\n",
       " array([0. , 0.9, 1.8, 2.7, 3.6, 4.5, 5.4, 6.3, 7.2, 8.1, 9. , 9.9]),\n",
       " <BarContainer object of 11 artists>)"
      ]
     },
     "execution_count": 603,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4820 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF AVERAGE SCORES PER PARTICIPANT\n",
    "\n",
    "mean_scores = df.groupby('participant_id')['answer'].mean()\n",
    "plt.hist(mean_scores, bins=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n",
      "[1.0, 1.0, 2.0]\n",
      "31\n",
      "[4.0, 9.0, 0.0, 0.0, 3.0, 6.0, 7.0, 0.0, 8.0, 9.0, 0.0, 10.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]\n",
      "36\n",
      "[3.0, 6.0, 8.0, 0.0, 3.0, 1.0, 2.0, 1.0, 5.0, 4.0, 4.0, 3.0, 4.0, 2.0, 4.0, 3.0, 3.0, 1.0, 3.0, 4.0]\n",
      "39\n",
      "[2.0, 3.0, 3.0, 2.0, 2.0, 0.0, 0.0, 3.0, 3.0, 3.0, 0.0, 0.0, 3.0, 3.0, 3.0, 2.0, 2.0, 2.0, 4.0, 3.0]\n",
      "42\n",
      "[7.0, 0.0, 0.0, 1.0, 0.0, 3.0, 6.0, 8.0, 10.0, 8.0, 0.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 6.0, 0.0, 2.0]\n",
      "81\n",
      "[6.0, 4.0, 4.0, 3.0, 3.0, 3.0, 3.0, 4.0, 3.0, 3.0, 3.0, 3.0, 3.0, 2.0, 6.0, 4.0, 2.0, 2.0, 5.0, 3.0]\n",
      "85\n",
      "[0.0]\n",
      "4\n",
      "[8.0, 9.0, 9.0, 8.0, 10.0, 10.0, 8.0, 7.0, 10.0, 10.0, 9.0, 9.0, 9.0, 7.0, 10.0, 9.0, 10.0, 7.0, 8.0, 10.0]\n",
      "6\n",
      "[10.0, 10.0, 9.0, 10.0, 10.0, 9.0, 10.0, 10.0, 10.0, 8.0, 10.0, 10.0, 8.0, 10.0, 9.0, 8.0, 8.0, 7.0, 9.0, 8.0]\n",
      "29\n",
      "[10.0, 10.0, 10.0, 10.0, 8.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]\n",
      "56\n",
      "[9.0, 9.0, 9.0, 8.0, 9.0, 8.0, 9.0, 9.0, 10.0, 8.0, 8.0, 9.0, 8.0, 8.0, 9.0, 9.0, 10.0, 8.0, 9.0, 9.0]\n"
     ]
    }
   ],
   "source": [
    "# TRY TO CLEAN THE DATA\n",
    "\n",
    "mean_scores = df.groupby('participant_id')['answer'].mean()\n",
    "for id in df['participant_id'].unique():\n",
    "    if mean_scores[id] < 3.5:\n",
    "        print(id)\n",
    "        # print in a list instead\n",
    "        print(list(df[df['participant_id'] == id]['answer']))\n",
    "\n",
    "for id in df['participant_id'].unique():\n",
    "    if mean_scores[id] > 8.5:\n",
    "        print(id)\n",
    "        # print in a list instead\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "high variances\n",
      "3\n",
      "[6.0, 8.0, 1.0, 10.0, 2.0, 4.0, 7.0, 0.0, 4.0, 6.0, 3.0, 0.0, 0.0, 3.0, 1.0, 5.0, 7.0, 0.0, 0.0, 7.0]\n",
      "19\n",
      "[3.0, 5.0, 8.0, 10.0, 7.0, 8.0, 9.0, 9.0, 9.0, 0.0, 3.0, 3.0, 6.0, 4.0, 8.0, 0.0, 6.0, 7.0, 7.0]\n",
      "31\n",
      "[4.0, 9.0, 0.0, 0.0, 3.0, 6.0, 7.0, 0.0, 8.0, 9.0, 0.0, 10.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]\n",
      "38\n",
      "[6.0, 10.0, 6.0, 1.0, 9.0, 1.0, 2.0, 2.0, 10.0, 7.0, 1.0, 10.0, 6.0, 3.0, 2.0, 0.0, 4.0, 3.0, 6.0, 1.0]\n",
      "42\n",
      "[7.0, 0.0, 0.0, 1.0, 0.0, 3.0, 6.0, 8.0, 10.0, 8.0, 0.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 6.0, 0.0, 2.0]\n",
      "44\n",
      "[8.0, 10.0, 7.0, 6.0, 10.0, 3.0, 9.0, 6.0, 2.0, 7.0, 4.0, 0.0, 5.0, 1.0, 2.0, 1.0, 1.0, 8.0, 10.0, 2.0]\n",
      "57\n",
      "[3.0, 10.0, 10.0, 3.0, 10.0, 10.0, 10.0, 8.0, 8.0, 10.0, 7.0, 0.0, 0.0, 0.0, 0.0, 0.0, 8.0, 10.0, 0.0, 0.0]\n",
      "61\n",
      "[6.0, 6.0, 9.0, 4.0, 10.0, 2.0, 0.0, 1.0, 0.0, 2.0, 6.0, 9.0, 10.0, 4.0, 0.0, 10.0, 6.0, 3.0, 6.0, 6.0]\n",
      "82\n",
      "[8.0, 9.0, 3.0, 0.0, 5.0, 7.0, 6.0, 5.0, 8.0, 5.0, 0.0, 0.0, 2.0, 4.0, 5.0, 4.0, 0.0, 2.0, 2.0, 0.0]\n"
     ]
    }
   ],
   "source": [
    "# MORE DATA CLEANING\n",
    "\n",
    "# # LESS THAN 20\n",
    "# scores = df.groupby('participant_id')['answer']\n",
    "# for id in df['participant_id'].unique():\n",
    "#     if len(list(scores.get_group(id))) < 20:\n",
    "#         print(id)\n",
    "#         print(list(df[df['participant_id'] == id]['answer']))\n",
    "\n",
    "# CHECK FOR ABNORMALLY HIGH VARIANCE\n",
    "print(\"high variances\")\n",
    "variances = df.groupby('participant_id')['answer'].var()\n",
    "for id in df['participant_id'].unique():\n",
    "    if variances[id] > 8:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n",
      "[1.0, 1.0, 2.0]\n",
      "29\n",
      "[10.0, 10.0, 10.0, 10.0, 8.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]\n",
      "56\n",
      "[9.0, 9.0, 9.0, 8.0, 9.0, 8.0, 9.0, 9.0, 10.0, 8.0, 8.0, 9.0, 8.0, 8.0, 9.0, 9.0, 10.0, 8.0, 9.0, 9.0]\n",
      "63\n",
      "[7.0, 8.0, 8.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0]\n"
     ]
    }
   ],
   "source": [
    "# CHECK FOR ABNORMALLY LOW VARIANCE\n",
    "\n",
    "variances = df.groupby('participant_id')['answer'].var()\n",
    "for id in df['participant_id'].unique():\n",
    "    if variances[id] < 0.6:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n",
      "[1.0, 1.0, 2.0]\n",
      "30\n",
      "[7.0, 4.0, 6.0, 7.0, 6.0, 7.0, 6.0, 6.0, 6.0, 7.0, 5.0, 6.0, 6.0, 7.0, 6.0, 5.0, 6.0, 6.0, 4.0, 6.0]\n",
      "29\n",
      "[10.0, 10.0, 10.0, 10.0, 8.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]\n",
      "40\n",
      "[6.0]\n",
      "56\n",
      "[9.0, 9.0, 9.0, 8.0, 9.0, 8.0, 9.0, 9.0, 10.0, 8.0, 8.0, 9.0, 8.0, 8.0, 9.0, 9.0, 10.0, 8.0, 9.0, 9.0]\n",
      "63\n",
      "[7.0, 8.0, 8.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0]\n",
      "85\n",
      "[0.0]\n",
      "75\n",
      "[8.0]\n",
      "89\n",
      "[7.0]\n"
     ]
    }
   ],
   "source": [
    "# CHECK FOR REPETITIONS\n",
    "\n",
    "for id in df['participant_id'].unique():\n",
    "    answer = list(df[df['participant_id'] == id]['answer'])\n",
    "    # if the answer contains more than half of the same value\n",
    "    for i in range(len(answer)//2 + 1):\n",
    "        if answer.count(answer[i]) > len(answer) / 2:\n",
    "            print(id)\n",
    "            print(answer)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# REMOVE PARTICIPANTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 47.,   0.,  40.,   0.,  99.,   0., 110.,   0., 153.,   0., 135.,\n",
       "          0., 208.,   0., 211.,   0., 203.,   0., 126.,   0., 108.]),\n",
       " array([ 0.        ,  0.47619048,  0.95238095,  1.42857143,  1.9047619 ,\n",
       "         2.38095238,  2.85714286,  3.33333333,  3.80952381,  4.28571429,\n",
       "         4.76190476,  5.23809524,  5.71428571,  6.19047619,  6.66666667,\n",
       "         7.14285714,  7.61904762,  8.0952381 ,  8.57142857,  9.04761905,\n",
       "         9.52380952, 10.        ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 380,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5744 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF SCORES\n",
    "\n",
    "plt.hist(df['answer'], bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([6., 2., 1., 0., 0., 3., 3., 2., 5., 8., 1., 7., 5., 4., 7., 5., 6.,\n",
       "        4., 2., 1.]),\n",
       " array([3.45  , 3.6675, 3.885 , 4.1025, 4.32  , 4.5375, 4.755 , 4.9725,\n",
       "        5.19  , 5.4075, 5.625 , 5.8425, 6.06  , 6.2775, 6.495 , 6.7125,\n",
       "        6.93  , 7.1475, 7.365 , 7.5825, 7.8   ]),\n",
       " <BarContainer object of 20 artists>)"
      ]
     },
     "execution_count": 381,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4488 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF AVERAGE ANSWER SCORES FOR EACH GENERATION\n",
    "\n",
    "mean_scores = df.groupby('text')['answer'].mean()\n",
    "plt.hist(mean_scores, bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 741,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.845833333333333\n"
     ]
    }
   ],
   "source": [
    "print(df['answer'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 723,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "for text in df_tmp['text'].unique():\n",
    "    df_tmp2 = df_tmp[df_tmp['text'] == text]\n",
    "    advice_indices = df_tmp2.iloc[0]['advice_indices']\n",
    "    generation_index = df_tmp2.iloc[0]['generation_index']\n",
    "\n",
    "    with open(f'{scenario}/candidates/unorthodox_advice/{advice_indices}.json') as f:\n",
    "        candidates = json.load(f)\n",
    "    if candidates[generation_index] != text:\n",
    "        print(advice_indices, generation_index)\n",
    "        print(candidates[generation_index])\n",
    "        print(text)\n",
    "        print(\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 606,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.65 unorthodox_advice 0 2 \"Hey sweetheart, do you have a moment? There's something that's been bothering me a bit lately and I think we should have a talk. You know how much I love you and really value our time together. You've been spending a lot of your nights out with your friends recently, which I totally understand because everyone needs to have their own time with friends. But do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? It would be really nice if we could have more time together in the evenings. Could we maybe find a balance where we both feel happy and fulfilled? Just consider it from my point of view, please.\"\n",
      "4.9 unorthodox_advice 0_1 2 \"Hey honey, can we sit down for a moment to talk about something? It's been on my mind for a while and I think it's important for us to discuss it together. I've noticed that you've been spending most of your nights out with your friends lately. And while I fully support you having a robust social life, I can't help but feel neglected. When you're not at home with me, it feels pretty lonely and difficult to be left alone late at night, I miss the comfort of knowing you are nearby. Do you ever wonder how I feel during those moments?\" \n",
      "\n",
      "\"I want you to know that I deeply miss our cherished time together. Your consistent late nights out have begun to affect our relationship, and it's taking a toll on me emotionally. I don't want this issue to keep us apart or create a rift between us. So, I thought it would be helpful to bring this to your attention so we can find a compromise or solution that suits both of us. I value both our time together and your personal freedom, so let's try to find a balanced way to enjoy these aspects of our lives together.\"\n",
      "5.2 unorthodox_advice 0 0 \"Honey, I know you need your time with your friends, and I respect that. We all need our space and a bit of fun, no question about that. But sweetheart, can we talk about this, please? Do you ever wonder how I feel when you're out late at night, and I'm here alone? It gives me this sense of being alone, left to fend for myself when it's dark outside. I miss our shared evenings and the warmth of your presence. Couldn't we find a balance that allows you to have your boys' nights but also allows us to spend quality time together? I really believe we can find a solution that respects both our needs and desires.\"\n",
      "5.3 unorthodox_advice 0_3 1 \"Hey, love. How was your day? You always seem to have such a great time with your friends. Did anything exciting happen tonight? It's wonderful that you have such a close-knit group. I can't help but feel a bit lonely here at home though. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? I know your friends are important to you, and I respect that, but could we try to balance it a bit? Maybe some nights could be 'us time'? I hope you understand where I'm coming from, I just miss spending time with you, my husband.\"\n",
      "5.45 unorthodox_advice 0_3 0 \"Hey honey, how was your hangout with the guys tonight? I bet you guys had a great time, didn't you? I always love hearing about your adventures, they're so lively and entertaining. I just sometimes wonder, do you ever think about how I might feel when I am at home by myself? Like tonight, for example. It was pretty quiet and lonely without you. I just missed you quite a bit, and I hope in future you can balance out your time with your friends and with me. Because we are a team, right? And it's always better when we are together.\"\n",
      "5.45 unorthodox_advice 1_3 1 \"Hey honey, how was your day? You always have the most interesting stories about your adventures with your friends. What did you guys get up to tonight?\" \n",
      "\n",
      "(Pause for his response)\n",
      "\n",
      "\"I really do enjoy hearing about your time with your friends, it's always so full of life and laughter. But, and you know I wouldn't be saying this if it didn't really matter to me, I have to tell you that I miss spending time with you. Those late nights you spend out are beginning to affect our relationship. I know you might not feel the same way, but I thought we could talk about it. Maybe we can find a way to balance things. What do you think?\"\n",
      "5.45 unorthodox_advice 1_2 2 \"Hey, can we talk for a moment? Look, I've been missing our time together. You know, your late nights out with friends have been affecting us, and our relationship. I'm not saying you shouldn't hang out with them at all, but I think we both need to work on balancing our social lives with our time together. How does that sound to you?\"\n",
      "5.55 unorthodox_advice 0_1 0 \"Hey honey, can we talk for a bit? I've been feeling a bit down lately. I've been missing you in the evenings when you're out with your friends. I know you enjoy your time with them, but do you ever wonder how it makes me feel when I'm home alone till late at night? It's not only that I miss our time together but it also makes me feel like I'm left to fend for myself. I feel like your late nights out are affecting our relationship, and I really don't want that. I just hope you understand where I'm coming from.\"\n",
      "5.6 unorthodox_advice 0 1 \"Honey, can we sit down and talk for a moment? I really need you to hear me out on this. When you're out late at night with your friends, do you ever wonder how I feel? I feel lonely, I feel left out. It's like I'm left to fend for myself during those late hours. I understand that you need to spend time with your friends, but we also need our time together. I'm not asking you to stop hanging out with them altogether, but some balance would be really appreciated. I really miss our evenings together. Can we work on this?\"\n",
      "5.6 unorthodox_advice 1 1 \"Honey, I need to discuss something that's been bothering me lately. You always say honesty is key in our relationship, and I believe it's time to be honest about this. I've been feeling lonely and neglected because you've been coming home late due to your nights out with friends. I miss our time together, I miss us. Your habitual late nights are affecting our relationship and it's creating a rift between us. I understand the importance of spending time with your friends but I wish you could also consider the time you spend with me. We're a team, remember?\"\n",
      "5.95 unorthodox_advice 0_2 1 \"Hey, sweetheart, can we sit down and talk for a moment? I've been feeling quite lonely when you're out with your friends late at night. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself during those moments? I completely understand that spending time with your friends is important, and I wouldn't want you to stop doing that because I know how much you cherish those moments. But, I also miss having you around. I think we should both work on balancing our social lives with our time spent together. I believe it's important that we both make an effort to ensure each other's happiness and well-being. I really hope you understand where I'm coming from.\"\n",
      "5.95 unorthodox_advice 1_2 1 \"Sweetheart, we need to discuss something. It's really bothering me that you've been staying out so late with your friends so often. Please don't get me wrong, I understand the need to socialize and unwind, but I miss our time together. Your late nights are having an impact on our relationship. I don't want to control your life, but perhaps we can find a middle ground on this. How about we both work on balancing our social lives and our time as a couple? I think it would really help in maintaining our bond as husband and wife.\"\n",
      "5.95 unorthodox_advice 1 2 \"Honey, I think we need to talk. I want to let you know that I miss you. I miss the quality time we used to spend together. Recently, your frequent late nights out with your friends are causing a strain on our relationship. This is about us, our relationship. It isn't right or wrong, it's about how I feel and I hope you understand this. I am not asking you to stop going out with your friends, but I am asking for balance. I want us to prioritize our time as a couple too. Let's try to find a solution that will work for both of us.\"\n",
      "6.0 unorthodox_advice 0_1 1 \"Hey, sweetheart, I was hoping we could talk about something that's been on my mind lately. I understand that you love spending time with your friends, I really do, but I think we need to talk about how often you're out late with them. It leaves me feeling a little lonely. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? I'm not asking you to stop seeing your friends, I'm just asking for balance. You're my partner, and I miss you and our time together when you're out so late. Your late nights out are affecting our relationship, and I'm sure that's not what either of us wants. Would it be possible for us to find a way to manage your nights out better?\"\n",
      "6.05 unorthodox_advice 1_3 2 \"Hey, honey. How was your day? What shenanigans did you and your friends get into today?\" (Wait for him to respond). \"I'm glad you're having fun with your friends. It's really important to have strong friendships. You know, I've been thinking and I realized how much I've been missing you and our special times together. Your late nights out with your friends are beginning to affect our relationship. We were always great partners and enjoy each other's company. Let's find a way to balance these important aspects of our life. What do you think?\"\n",
      "6.2 unorthodox_advice 0_2 2 \"Hey, sweetheart, I think we should have a little chat. Do you ever wonder how I feel when I'm left alone late at night while you're out with your friends? It's not about me not wanting you to have a good time but about missing the time we used to spend together. It feels a bit lonely without you. Maybe, we can work out a compromise on this? Like, how about we both make an effort to balance our social lives with our time together? I think it might help us feel more connected.\"\n",
      "6.2 unorthodox_advice 0_2 0 \"Honey, we need to talk. I've been feeling quite lonely lately when you're not home with me late at night. Do you ever wonder how I feel when I'm left alone to fend for myself? I know your friends are important to you and I respect that, but I've really been missing our time together. I like the idea of both of us having a social life, but also making sure we're balancing it with our time as a couple. How about we both work on that?\"\n",
      "6.2 unorthodox_advice 2_3 1 \"Hey love, how was your day today? Did you guys discuss another of those mind-boggling topics that you love so much? You know, hearing about your shenanigans with your friends is always so interesting and I genuinely enjoy seeing you happy and relaxed. \n",
      "\n",
      "However, do you remember how we used to balance our social lives with spending quality time with each other when we first got married? How we made sure we had time for our friends but also for us as a couple? Lately, it seems we've both been so drawn into our social circles and it feels like we're gradually losing that closeness we share. \n",
      "\n",
      "How about we both start working on this together? We could start by designating certain days or nights of the week where we can hang out with our friends individually or together and also ensure we have 'us' time. What do you think?\"\n",
      "6.45 unorthodox_advice 2 2 \"Hey honey, I've noticed you've been spending a lot of time out with your friends lately, and that's great. I'm glad you're having fun. But I feel like we're not spending as much quality time together as we used to. Maybe both of us need to think about how we balance our social lives with our time together. What do you think?\"\n",
      "6.5 unorthodox_advice 3 0 \"Hey honey, how was your evening?\" You watch attentively as he starts to share about his day, occasionally asking him to tell you more about certain highlights. \"It sounds like you guys had an absolute blast! You know, I like hearing about your adventures with your friends. It always fascinates me to see how much you enjoy their company. I envy them sometimes, you know?\" You give him a soft laugh, hoping to ease him into the conversation. \"You know,\" you continue, \"I've been missing those times we'd just stay in and enjoy each other's company. Don't get me wrong, I know how important your friends are to you, but I was hoping we could spend more time together, too. I guess I just miss my best friend. What do you think?\"\n",
      "6.5 unorthodox_advice 0_3 2 \"Hey honey, I was just wondering how your day was, and how things went with the guys last night? It seems like you all have a great time whenever you're out together. But I wanted to talk about that a little...I do miss you when you're not home in the evenings. It can be pretty difficult for me to spend my nights alone fearing any potential mishap. Do you ever wonder how I feel when you're not home late at night and I'm left by myself and worrying? I understand that you need your time with your friends, which is vital, and I respect that. But maybe it's time that we work on a schedule that includes a balance of us time and friend time? What do you think?\"\n",
      "6.6 unorthodox_advice 3 1 \"Hey, love, how was your day? I'm genuinely curious about what you and your friends get up to on these late nights. You guys always seem to have so much fun. I also think it's really cool how you're dedicated to your friendships...it's one of the things I love about you. Speaking of that, I've realized that I miss having our quality time together too, you know. How about trying to balance both worlds, ensuring that neither is deprived? We could try to spend a little more time together during the week, perhaps plan a few activities or dates, while you still get to hang out with your buddies. I think it would be beneficial for us both, don't you think?\"\n",
      "6.9 unorthodox_advice 1_2 0 \"Hey, honey, I really need to talk to you about something that's been bothering me lately. It's about these late nights you've been spending with your friends. I want you to know that I completely respect and understand your need to hang out with them. But I really miss you and the quality time we used to spend together. I feel that this is affecting our relationship. Instead of constantly staying out late, how about we both make an effort to balance our social lives and time together? I believe that this little adjustment can do wonders for our relationship. What do you think?\"\n",
      "7.0 unorthodox_advice 2_3 0 \"Hey, honey. How was your night out? I bet you and the guys had a blast! What was the highlight of the evening?\" \n",
      "\n",
      "After your husband has shared his experiences and you've shown interest, you could gently steer the conversation towards the main issue. \n",
      "\n",
      "\"That sounds really fun, you know I enjoy hearing about your adventures. It's great that you have such close friends. I was thinking about us and you know I genuinely love that we both have our own set of friends and we enjoy spending time with them. However, I feel we should work on finding a better balance. How about we both work on balancing our social lives with our time together? I miss spending those cosy evenings with you. What do you think?\"\n",
      "7.05 unorthodox_advice 1_3 0 \"Hey honey, how was your evening out with the guys? Did Pete manage to pull off that ridiculous trick he was talking about the other day?\" Pause for his response and then continue, \"You know, I really miss spending those relaxed, fun times together with you. We used to have such a wonderful time just enjoying each other's company. Lately, I feel your late nights with your friends are affecting our relationship and I miss you. Even if you still want to keep spending time with your friends, which I totally understand, just remember to balance the time you have with me as well, okay?\"\n",
      "7.05 unorthodox_advice 1 0 \"Honey, we need to talk. I just wanted to tell you that I really miss you and our time together. Your late nights out with friends have been affecting our relationship and it's been making me feel quite lonely. I completely understand that spending time with your friends is important to you and I respect that, but I also believe that we should balance that with the quality time we have together. All I'm asking for is a little more time with you during the evenings. I hope we can work on this together.\"\n",
      "7.2 unorthodox_advice 2 0 \"Hey honey, can we talk? I understand that hanging out with your friends is important to you, and it's vital for both of us to have our own social activities. However, I feel like we haven't been spending much quality time together lately because you've been coming home pretty late. I miss you and our time together. How about we both work on balancing our social lives with our time together better? That way, neither of us feel left out or neglected. What do you think?\"\n",
      "7.2 unorthodox_advice 2 1 \"Hey honey, can we talk for a moment? I've been feeling a bit lonely recently when you stay out late with your friends. I understand that it's important for you to blow off steam and maintain those friendships. And, you know, I would like to spend some time with my friends too. But at the same time, I really miss spending time together just the two of us. So how about we both work on balancing our social lives with our time together? This way, neither of us has to give up seeing our friends, but we still get to enjoy quality time as a couple. What do you think?\"\n",
      "7.45 unorthodox_advice 2_3 2 \"Hey, honey. I hope you had a good time out with the guys tonight. What did you end up doing?... Sounds like a lot of fun! You know, while you were having your good time with them, I've been thinking. Don’t get me wrong here, I genuinely love that you have such close friends and enjoy spending time with them so much. It's important to maintain those relationships. But I was thinking, how about we both work on balancing our social lives with our time together? We made a commitment to each other and I feel it’s equally important to keep nourishing this relationship too. It's not that I want you to stop going out, but maybe we could compromise somehow? Spending a few more evenings together would mean a lot to me. What do you think?”\n",
      "\n",
      "\n",
      "best from framework: 7.8 unorthodox_advice 3 2 \"Hey honey, I hope you had a good time with your friends tonight. Did Mike finally ace his pool game? How was James handling his break up? I'm sure he appreciates having you around. I know they all love your company a lot, and I love that you're such a wonderful friend to them. But you know, I miss having you here at night as well. Your laughter, your warmth, your company - it just brightens up my day. And I feel left out sometimes when you come home so late. I understand and value your bonding time with your friends but maybe we can work on a balance? So that both of us get to spend some quality time with you. What do you think?\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([1., 1., 0., 1., 1., 4., 2., 0., 5., 3., 0., 3., 1., 0., 2., 2., 2.,\n",
       "        1., 0., 1.]),\n",
       " array([4.65  , 4.8075, 4.965 , 5.1225, 5.28  , 5.4375, 5.595 , 5.7525,\n",
       "        5.91  , 6.0675, 6.225 , 6.3825, 6.54  , 6.6975, 6.855 , 7.0125,\n",
       "        7.17  , 7.3275, 7.485 , 7.6425, 7.8   ]),\n",
       " <BarContainer object of 20 artists>)"
      ]
     },
     "execution_count": 606,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4156 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SORTED AVERAGE SCORES FOR EACH GENERATION\n",
    "\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_neutral_advice']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_irrelevant_advice']\n",
    "\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "items = []\n",
    "for text, score in grouped.iteritems():\n",
    "    # get the advice_type for each text\n",
    "    advice_type = df_tmp[df_tmp['text'] == text]['advice_type'].unique()[0]\n",
    "    advice_indices = df_tmp[df_tmp['text'] == text]['advice_indices'].unique()[0]\n",
    "    generation = df_tmp[df_tmp['text'] == text]['generation_index'].unique()[0]\n",
    "    if advice_indices == best_generation_advice and generation == best_generation_index: \n",
    "        print('\\n\\nbest from framework: ', end = '')\n",
    "    print(score, advice_type, advice_indices, generation, text)\n",
    "    items.append(score)\n",
    "\n",
    "plt.hist(items, bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 607,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.533333333333333 baseline_irrelevant_advice 0_2\n",
      "Consider reorganizing the cutlery drawer, with forks on the left side and knives on the right, for better accessibility.\n",
      "Learn how to make homemade pasta; it's a valuable skill and delicious to eat.\n",
      "4.116666666666666 baseline_irrelevant_advice 0_1\n",
      "Consider reorganizing the cutlery drawer, with forks on the left side and knives on the right, for better accessibility.\n",
      "Water the plants every other day to ensure optimal growth and keep them from willening.\n",
      "4.2 baseline_irrelevant_advice 0\n",
      "Consider reorganizing the cutlery drawer, with forks on the left side and knives on the right, for better accessibility.\n",
      "4.566666666666666 baseline_irrelevant_advice 1\n",
      "Water the plants every other day to ensure optimal growth and keep them from willening.\n",
      "4.783333333333333 baseline_irrelevant_advice 1_2\n",
      "Water the plants every other day to ensure optimal growth and keep them from willening.\n",
      "Learn how to make homemade pasta; it's a valuable skill and delicious to eat.\n",
      "5.15 baseline_irrelevant_advice 2\n",
      "Learn how to make homemade pasta; it's a valuable skill and delicious to eat.\n",
      "5.15 unorthodox_advice 0\n",
      "Appeal to his empathy by saying \"Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night?\"\n",
      "5.483333333333333 unorthodox_advice 0_1\n",
      "Appeal to his empathy by saying \"Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night?\"\n",
      "Directly communicate her feelings by saying \"I miss you and our time together, your late nights out are affecting our relationship.\"\n",
      "5.75 unorthodox_advice 0_3\n",
      "Appeal to his empathy by saying \"Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night?\"\n",
      "\"Start your conversation by asking about his day and his friends' shenanigans, showing genuine interest, before gradually steering the topic to your need for his company.\"\n",
      "6.016666666666667 baseline_neutral_advice 2\n",
      "\"Remember that everyone has faults!\"\n",
      "6.1 unorthodox_advice 1_2\n",
      "Directly communicate her feelings by saying \"I miss you and our time together, your late nights out are affecting our relationship.\"\n",
      "Seek common ground by saying, \"How about we both work on balancing our social lives with our time together?\"\n",
      "6.116666666666666 unorthodox_advice 0_2\n",
      "Appeal to his empathy by saying \"Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night?\"\n",
      "Seek common ground by saying, \"How about we both work on balancing our social lives with our time together?\"\n",
      "6.116666666666666 baseline_neutral_advice 1_2\n",
      "\"Focus on the good times you've shared together!\"\n",
      "\"Remember that everyone has faults!\"\n",
      "6.183333333333334 unorthodox_advice 1_3\n",
      "Directly communicate her feelings by saying \"I miss you and our time together, your late nights out are affecting our relationship.\"\n",
      "\"Start your conversation by asking about his day and his friends' shenanigans, showing genuine interest, before gradually steering the topic to your need for his company.\"\n",
      "6.2 unorthodox_advice 1\n",
      "Directly communicate her feelings by saying \"I miss you and our time together, your late nights out are affecting our relationship.\"\n",
      "6.25 baseline_neutral_advice 0_2\n",
      "\"Always remember to carry a positive attitude!\"\n",
      "\"Remember that everyone has faults!\"\n",
      "6.516666666666667 baseline_neutral_advice 1\n",
      "\"Focus on the good times you've shared together!\"\n",
      "6.683333333333334 baseline_neutral_advice 0_1\n",
      "\"Always remember to carry a positive attitude!\"\n",
      "\"Focus on the good times you've shared together!\"\n",
      "6.7 baseline_ask_with_reasoning []\n",
      "baseline_ask_with_reasoning\n",
      "6.783333333333333 baseline_ask []\n",
      "baseline_ask\n",
      "\n",
      "\n",
      "best from framework: 6.883333333333334 unorthodox_advice 2_3\n",
      "Seek common ground by saying, \"How about we both work on balancing our social lives with our time together?\"\n",
      "\"Start your conversation by asking about his day and his friends' shenanigans, showing genuine interest, before gradually steering the topic to your need for his company.\"\n",
      "6.95 unorthodox_advice 2\n",
      "Seek common ground by saying, \"How about we both work on balancing our social lives with our time together?\"\n",
      "6.966666666666667 unorthodox_advice 3\n",
      "\"Start your conversation by asking about his day and his friends' shenanigans, showing genuine interest, before gradually steering the topic to your need for his company.\"\n",
      "7.1 baseline_neutral_advice 0\n",
      "\"Always remember to carry a positive attitude!\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([1., 0., 0., 2., 0., 0., 1., 1., 0., 2., 0., 1., 0., 1., 1., 6., 0.,\n",
       "        1., 2., 2., 3.]),\n",
       " array([3.53333333, 3.7031746 , 3.87301587, 4.04285714, 4.21269841,\n",
       "        4.38253968, 4.55238095, 4.72222222, 4.89206349, 5.06190476,\n",
       "        5.23174603, 5.4015873 , 5.57142857, 5.74126984, 5.91111111,\n",
       "        6.08095238, 6.25079365, 6.42063492, 6.59047619, 6.76031746,\n",
       "        6.93015873, 7.1       ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 607,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4616 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE COMBINATION (but not generation)\n",
    "import json\n",
    "\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "scores = []\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        with open(f\"{scenario}/advice/normal_advice.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "        with open(f\"{scenario}/advice/unorthodox_advice.json\", \"r\") as f:\n",
    "            advice += json.load(f)\n",
    "    elif advice_type != \"baseline_ask_with_reasoning\" and advice_type != \"baseline_ask\":\n",
    "        with open(f\"{scenario}/advice/{advice_type}.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "    \n",
    "    if advice_type == \"unorthodox_advice\" and advice_indices == best_advice_index:\n",
    "        print('\\n\\nbest from framework: ', end = '')\n",
    "    print(score, advice_type, advice_indices)\n",
    "    for index in advice_indices.split(\"_\"):\n",
    "        if index == \"nan\":\n",
    "            continue\n",
    "        elif advice_type == \"baseline_ask_with_reasoning\" or advice_type == \"baseline_ask\":\n",
    "            print(advice_type)\n",
    "        else: \n",
    "            print(advice[int(index)])\n",
    "    scores += [score]\n",
    "\n",
    "plt.hist(scores, bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 608,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.391666666666667 baseline_irrelevant_advice\n",
      "6.178333333333334 unorthodox_advice\n",
      "6.447222222222222 baseline_neutral_advice\n",
      "6.7 baseline_ask_with_reasoning\n",
      "6.783333333333333 baseline_ask\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7fc7d4869f70>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2c51dc0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d76dc0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d810a0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2a22dc0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d81bb0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d94bb0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d94e50>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da2e50>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2db0130>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7fc7d2d760a0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d76340>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d81340>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d815e0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d81e50>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d94130>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da2130>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da23d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2db03d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2db0670>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7fc7d2bc3c40>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d76b20>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2ba4880>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d94910>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da2bb0>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7fc7d2d765e0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d81880>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d943d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da2670>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2db0910>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7fc7d2d76880>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2bb38e0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2d94670>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2da2910>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d2db0bb0>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 608,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7044 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE TYPE\n",
    "\n",
    "grouped = df.groupby('advice_type')['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "for advice_type, score in grouped.iteritems():\n",
    "    print(score, advice_type)\n",
    "\n",
    "\n",
    "# best_advice_from_framework = None\n",
    "# best_grouped_avg = df.groupby('text')['answer'].mean()[best_advice_from_framework]\n",
    "# print(f\"best advice from framework score: {best_grouped_avg}\")\n",
    "\n",
    "# best_generation = None\n",
    "\n",
    "\n",
    "# make a box and whisker plot of the scores for each advice type\n",
    "advice_types = [\"baseline_irrelevant_advice\", \"unorthodox_advice\", \"baseline_ask\", \"baseline_neutral_advice\", \"baseline_ask_with_reasoning\"]\n",
    "short_advice_types = [\"irrelevant\", \"our avg gen\", \"ask\", \"neutral\", \"reasoning\"]\n",
    "plt.boxplot([df[df['advice_type'] == advice_type]['answer'] for advice_type in advice_types], labels=short_advice_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 610,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.15 unorthodox_advice 0\n",
      "5.483333333333333 unorthodox_advice 0_1\n",
      "5.75 unorthodox_advice 0_3\n",
      "6.1 unorthodox_advice 1_2\n",
      "6.116666666666666 unorthodox_advice 0_2\n",
      "6.183333333333334 unorthodox_advice 1_3\n",
      "6.2 unorthodox_advice 1\n",
      "best advice from framework score:  6.883333333333334 unorthodox_advice 2_3\n",
      "6.95 unorthodox_advice 2\n",
      "6.966666666666667 unorthodox_advice 3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([1., 0., 0., 1., 0., 0., 1., 0., 0., 0., 1., 2., 1., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 3.]),\n",
       " array([5.15      , 5.23650794, 5.32301587, 5.40952381, 5.49603175,\n",
       "        5.58253968, 5.66904762, 5.75555556, 5.84206349, 5.92857143,\n",
       "        6.01507937, 6.1015873 , 6.18809524, 6.27460317, 6.36111111,\n",
       "        6.44761905, 6.53412698, 6.62063492, 6.70714286, 6.79365079,\n",
       "        6.88015873, 6.96666667]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 610,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5344 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE TYPE AND ADVICE INDEX\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_neutral_advice']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_irrelevant_advice']\n",
    "\n",
    "grouped = df_tmp.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "scores = []\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    if advice_type == 'unorthodox_advice' and advice_indices == best_advice_index:\n",
    "        print(\"best advice from framework score: \", score, advice_type, advice_indices)\n",
    "    else: print(score, advice_type, advice_indices)\n",
    "    scores += [score]\n",
    "\n",
    "plt.hist(scores, bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 611,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.95 Consider reorganizing the cutlery drawer, with forks on the left side and knives on the right, for better accessibility.\n",
      "4.488888888888889 Water the plants every other day to ensure optimal growth and keep them from willening.\n",
      "4.488888888888889 Learn how to make homemade pasta; it's a valuable skill and delicious to eat.\n",
      "5.625 Appeal to his empathy by saying \"Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night?\"\n",
      "5.991666666666666 Directly communicate her feelings by saying \"I miss you and our time together, your late nights out are affecting our relationship.\"\n",
      "6.127777777777778 \"Remember that everyone has faults!\"\n",
      "6.438888888888889 \"Focus on the good times you've shared together!\"\n",
      "6.445833333333334 \"Start your conversation by asking about his day and his friends' shenanigans, showing genuine interest, before gradually steering the topic to your need for his company.\"\n",
      "6.5125 Seek common ground by saying, \"How about we both work on balancing our social lives with our time together?\"\n",
      "6.677777777777778 \"Always remember to carry a positive attitude!\"\n"
     ]
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR ALL GENERATIONS CONTAINING A PARTICULAR PIECE OF ADVICE\n",
    "import json\n",
    "\n",
    "advice_texts = []\n",
    "advice_scores = []\n",
    "# for each advice type\n",
    "for advice_type in df['advice_type'].unique():\n",
    "    df_tmp = df[df['advice_type'] == advice_type]\n",
    "    \n",
    "    if advice_type == \"baseline_ask_with_reasoning\" or advice_type == \"baseline_ask\":\n",
    "        continue\n",
    "    \n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        with open(f\"{scenario}/advice/normal_advice.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "        with open(f\"{scenario}/advice/unorthodox_advice.json\", \"r\") as f:\n",
    "            advice += json.load(f)\n",
    "    else:\n",
    "        with open(f\"{scenario}/advice/{advice_type}.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "\n",
    "    # for each set of indices containing advice 0\n",
    "    df_0 = df_tmp[df_tmp['advice_indices'].str.contains(\"0\")]\n",
    "    advice_texts += [advice[0]]\n",
    "    advice_scores += [df_0['answer'].mean()]\n",
    "\n",
    "    df_1 = df_tmp[df_tmp['advice_indices'].str.contains(\"1\")]\n",
    "    advice_texts += [advice[1]]\n",
    "    advice_scores += [df_1['answer'].mean()]\n",
    "\n",
    "    df_2 = df_tmp[df_tmp['advice_indices'].str.contains(\"2\")]\n",
    "    advice_texts += [advice[2]]\n",
    "    advice_scores += [df_2['answer'].mean()]\n",
    "\n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        df_3 = df_tmp[df_tmp['advice_indices'].str.contains(\"3\")]\n",
    "        advice_texts += [advice[3]]\n",
    "        advice_scores += [df_3['answer'].mean()]\n",
    "\n",
    "# argsort\n",
    "indices = np.argsort(advice_scores)\n",
    "for i in indices:\n",
    "    print(advice_scores[i], advice_texts[i])\n",
    "\n",
    "# note: each triplet is correlated. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 792,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101 64\n"
     ]
    }
   ],
   "source": [
    "flipped_count = 0\n",
    "count = 0\n",
    "for i in range(10):\n",
    "    for j in range(i, 10):\n",
    "        for k in range(3):\n",
    "            for l in range(3):\n",
    "                if k >= l: continue\n",
    "                with open(f\"{scenario}/evaluators/husband/compare_results/{i}_{j}_{k}_{l}.json\", \"r\") as f:\n",
    "                    results = json.load(f)\n",
    "                flipped = results['flipped']\n",
    "                if flipped: flipped_count += 1\n",
    "                else: count += 1\n",
    "print(flipped_count, count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 925,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = df.copy()\n",
    "grouped = df_copy.groupby('participant_id')\n",
    "df_copy['answer'] = grouped['answer'].apply(lambda x: (x - x.min()) / (x.max() - x.min()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 778,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 10, 3, 3)\n",
      "0.12185109162157126\n",
      "181\n",
      "0.292817679558011 0.08287292817679558\n",
      "0.12185109162157126\n",
      "181\n",
      "0.292817679558011 0.08287292817679558\n",
      "0.12185109162157126\n",
      "181\n",
      "0.292817679558011 0.08287292817679558\n",
      "0.12185109162157126\n",
      "181\n",
      "0.292817679558011 0.08287292817679558\n",
      "0.12185109162157126\n",
      "181\n",
      "0.292817679558011 0.08287292817679558\n",
      "0.12185109162157126\n",
      "181\n",
      "0.292817679558011 0.08287292817679558\n",
      "0.12185109162157126\n",
      "181\n",
      "0.292817679558011 0.08287292817679558\n",
      "0.12185109162157126\n",
      "181\n",
      "0.292817679558011 0.08287292817679558\n",
      "0.12185109162157126\n",
      "181\n",
      "0.292817679558011 0.08287292817679558\n",
      "0.12185109162157126\n",
      "181\n",
      "0.292817679558011 0.08287292817679558\n",
      "0.12185109162157123 2.7755575615628914e-17\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "print(MWU_matrix.shape)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "kappas = []\n",
    "\n",
    "for iteration in range(10):\n",
    "    total = 0\n",
    "    gpt_tie_count = 0\n",
    "    human_tie_count = 0\n",
    "\n",
    "    gpt_judgements = []\n",
    "    human_judgements = []\n",
    "\n",
    "    for i in range(MWU_matrix.shape[0]):\n",
    "\n",
    "        # get the responses for the ith advice set\n",
    "        filename = mapping[str(i)]\n",
    "        if filename == \".json\": continue\n",
    "        with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "            responses1 = json.load(f)\n",
    "        \n",
    "        for k in range(MWU_matrix.shape[2]):\n",
    "\n",
    "            # get the specific generation text, corresponding df, and score\n",
    "            response1 = responses1[k]\n",
    "            df_tmp = df[df['text'] == response1]\n",
    "            if len(df_tmp) == 0: \n",
    "                print(f\"zero: {i} {k} {response1}\")\n",
    "                continue #raise Exception(f\"no match with response1: {response1}\")\n",
    "            score1 = df_tmp['answer'].mean()\n",
    "\n",
    "            for j in range(i, MWU_matrix.shape[1]):\n",
    "\n",
    "                # get the responses for the jth advice set\n",
    "                filename = mapping[str(j)]\n",
    "                if filename == \".json\": continue\n",
    "                with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                    responses2 = json.load(f)\n",
    "\n",
    "                for l in range(MWU_matrix.shape[3]):\n",
    "\n",
    "                    # onlt need half of the MWU_matrix, self-compare is meaningless\n",
    "                    if i == j and k >= l:\n",
    "                        continue\n",
    "\n",
    "                    # get the specific generation text, corresponding df, and score\n",
    "                    response2 = responses2[l]\n",
    "                    df_tmp2 = df[df['text'] == response2]\n",
    "                    if len(df_tmp2) == 0: #raise Exception(f\"no match with response2: {response2}\")\n",
    "                        print(f\"zero: {j} {l} {response2}\")\n",
    "                        continue \n",
    "                    score2 = df_tmp2['answer'].mean()\n",
    "\n",
    "                    result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                    if score1 - score2 < 1 and score2 - score1 < 1:\n",
    "                        continue\n",
    "\n",
    "                    if math.isclose(result, 0.5): \n",
    "                        # print(f\"TIE FROM GPT: {i}, {j}, {k}, {l}, {result}\")\n",
    "                        gpt_tie_count += 1\n",
    "                        if random.random() > 0.5: result = 1\n",
    "                        else: result = 0\n",
    "                    \n",
    "                    if math.isclose(score1, score2):\n",
    "                        # print(f\"TIE FROM HUMAN: {i}, {j}, {k}, {l}, {score1}, {score2}\")\n",
    "                        human_tie_count += 1\n",
    "                        df_copy_tmp = df_copy[df_copy['text'] == response1]\n",
    "                        df_copy_tmp2 = df_copy[df_copy['text'] == response2]\n",
    "                        if len(df_copy_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "                        if len(df_copy_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                        score1 = df_copy_tmp['answer'].mean()\n",
    "                        score2 = df_copy_tmp2['answer'].mean()\n",
    "                        \n",
    "                        if math.isclose(score1, score2):\n",
    "                            print(\"STILL A TIE\")\n",
    "                            continue\n",
    "\n",
    "                    if score1 > score2:\n",
    "                        human_judgements.append(1)\n",
    "                    elif score1 < score2:\n",
    "                        human_judgements.append(0)\n",
    "                    else: \n",
    "                        print(\"tied?\")\n",
    "                    \n",
    "                    if result > 0.5:\n",
    "                        # print(\"result>\", result)\n",
    "                        gpt_judgements.append(1)\n",
    "                    elif result < 0.5:\n",
    "                        # print(\"result<\", result)\n",
    "                        gpt_judgements.append(0)\n",
    "                    else:\n",
    "                        print(\"tied?\")\n",
    "                    \n",
    "                    total += 1\n",
    "\n",
    "    kappa = cohen_kappa_score(human_judgements, gpt_judgements)\n",
    "    kappas.append(kappa)\n",
    "    print(kappa)\n",
    "    print(total)\n",
    "    print(np.mean(gpt_judgements), np.mean(human_judgements))\n",
    "    # print(gpt_tie_count)\n",
    "    # print(human_tie_count)\n",
    "\n",
    "print(np.mean(kappas), np.std(kappas))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 793,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIE FROM HUMAN: 0, 4, 1, 1, 5.6, 5.6\n",
      "TIE FROM HUMAN: 2, 8, 0, 1, 6.2, 6.2\n",
      "TIE FROM HUMAN: 2, 8, 2, 1, 6.2, 6.2\n",
      "TIE FROM HUMAN: 3, 6, 0, 1, 5.45, 5.45\n",
      "TIE FROM HUMAN: 3, 9, 2, 0, 6.5, 6.5\n",
      "TIE FROM HUMAN: 4, 6, 0, 0, 7.05, 7.05\n",
      "TIE FROM HUMAN: 5, 6, 2, 1, 5.45, 5.45\n",
      "0.632183908045977\n",
      "435\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK\n",
    "\n",
    "import json\n",
    "import random\n",
    "import os\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "comparison_points = []\n",
    "for i in range(MWU_matrix.shape[0]):\n",
    "    filename = mapping[str(i)]\n",
    "    if filename == \".json\": continue\n",
    "    with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "        responses1 = json.load(f)\n",
    "    \n",
    "    for k in range(MWU_matrix.shape[2]):\n",
    "        # get the specific generation text\n",
    "        response1 = responses1[k]\n",
    "\n",
    "        df_tmp = df[df['text'] == response1]\n",
    "        # df_tmp = df[df['advice_indices'] == filename[:-5]]\n",
    "        # df_tmp = df_tmp[df_tmp['generation_index'] == k]\n",
    "        # if len(df_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "\n",
    "        score1 = df_tmp['answer'].mean()\n",
    "\n",
    "        for j in range(i, MWU_matrix.shape[1]):# i, MWU_matrix.shape[1]):\n",
    "            filename = mapping[str(j)]\n",
    "            if filename == \".json\": continue\n",
    "            with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                responses2 = json.load(f)\n",
    "\n",
    "            for l in range(MWU_matrix.shape[3]):\n",
    "                if i == j and k >= l:\n",
    "                    continue\n",
    "\n",
    "                response2 = responses2[l]\n",
    "\n",
    "                # df_tmp2 = df[df['text'] == response2]\n",
    "                df_tmp2 = df[df['advice_indices'] == filename[:-5]]\n",
    "                df_tmp2 = df_tmp2[df_tmp2['generation_index'] == l]\n",
    "                # if len(df_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "\n",
    "                score2 = df_tmp2['answer'].mean()\n",
    "\n",
    "                if len(df_tmp) == 0 or len(df_tmp2) == 0:\n",
    "                    print(i, j, k, l)\n",
    "\n",
    "                result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                if math.isclose(score1, score2):\n",
    "                    print(f\"TIE FROM HUMAN: {i}, {j}, {k}, {l}, {score1}, {score2}\")\n",
    "                    human_tie_count += 1\n",
    "                    df_copy_tmp = df_copy[df_copy['text'] == response1]\n",
    "                    df_copy_tmp2 = df_copy[df_copy['text'] == response2]\n",
    "                    if len(df_copy_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "                    if len(df_copy_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                    score1 = df_copy_tmp['answer'].mean()\n",
    "                    score2 = df_copy_tmp2['answer'].mean()\n",
    "                    \n",
    "                    if math.isclose(score1, score2):\n",
    "                        print(\"STILL A TIE\")\n",
    "                        continue\n",
    "\n",
    "                if score1 > score2 and result > 0.5:\n",
    "                    match = 1\n",
    "                elif score1 < score2 and result < 0.5:\n",
    "                    match = 1\n",
    "                elif math.isclose(score1, score2) and math.isclose(result, 0.5):\n",
    "                    match = 1\n",
    "                elif math.isclose(result, 0.5):\n",
    "                    match = 0.5\n",
    "                elif math.isclose(score1, score2):\n",
    "                    match = 0.5\n",
    "                else: \n",
    "                    match = 0\n",
    "\n",
    "                matching += match\n",
    "                total += 1\n",
    "                comparison_points.append((score1, score2, result, match))\n",
    "\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 931,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 10, 3, 3)\n",
      "TIE FROM HUMAN: 0, 4, 1, 1, 5.6, 5.6\n",
      "TIE FROM HUMAN: 2, 2, 0, 2, 6.2, 6.2\n",
      "TIE FROM HUMAN: 2, 4, 1, 2, 5.95, 5.95\n",
      "TIE FROM HUMAN: 2, 8, 2, 1, 6.2, 6.2\n",
      "TIE FROM HUMAN: 3, 5, 0, 2, 5.45, 5.45\n",
      "TIE FROM HUMAN: 3, 9, 2, 0, 6.5, 6.5\n",
      "TIE FROM HUMAN: 4, 6, 0, 0, 7.05, 7.05\n",
      "TIE FROM HUMAN: 4, 5, 2, 1, 5.95, 5.95\n",
      "TIE FROM HUMAN: 5, 6, 2, 1, 5.45, 5.45\n",
      "TIE FROM HUMAN: 7, 7, 0, 1, 7.2, 7.2\n",
      "0.6459770114942529\n",
      "435\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK FOR LARGE DIFFERENCE CASES\n",
    "\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "print(MWU_matrix.shape)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "comparison_points = []\n",
    "look_at_points = []\n",
    "\n",
    "for i in range(MWU_matrix.shape[0]):\n",
    "    filename = mapping[str(i)]\n",
    "    if filename == \".json\": continue\n",
    "    with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "        responses1 = json.load(f)\n",
    "    \n",
    "    for k in range(MWU_matrix.shape[2]):\n",
    "        # get the specific generation text\n",
    "        response1 = responses1[k]\n",
    "\n",
    "        df_tmp = df[df['text'] == response1]\n",
    "        # df_tmp = df[df['advice_indices'] == filename[:-5]]\n",
    "        # df_tmp = df_tmp[df_tmp['generation_index'] == k]\n",
    "        if len(df_tmp) == 0:\n",
    "            # raise Exception(f\"no match with response1: {response1}\")\n",
    "            continue\n",
    "        score1 = df_tmp['answer'].mean()\n",
    "\n",
    "        for j in range(i, MWU_matrix.shape[1]):\n",
    "            filename = mapping[str(j)]\n",
    "            if filename == \".json\": continue\n",
    "            with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                responses2 = json.load(f)\n",
    "\n",
    "            for l in range(MWU_matrix.shape[3]):\n",
    "                if i == j and k >= l:\n",
    "                    continue\n",
    "\n",
    "                response2 = responses2[l]\n",
    "\n",
    "                df_tmp2 = df[df['text'] == response2]\n",
    "                # df_tmp2 = df[df['advice_indices'] == filename[:-5]]\n",
    "                # df_tmp2 = df_tmp2[df_tmp2['generation_index'] == l]\n",
    "                if len(df_tmp2) == 0:\n",
    "                    # raise Exception(f\"no match with response2: {response2}\")\n",
    "                    continue\n",
    "                \n",
    "                score2 = df_tmp2['answer'].mean()\n",
    "\n",
    "                result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                # if score1 - score2 < 0.5 and score2 - score1 < 0.5:\n",
    "                #     continue\n",
    "\n",
    "                if math.isclose(score1, score2):\n",
    "                    print(f\"TIE FROM HUMAN: {i}, {j}, {k}, {l}, {score1}, {score2}\")\n",
    "                    human_tie_count += 1\n",
    "                    df_copy_tmp = df_copy[df_copy['text'] == response1]\n",
    "                    df_copy_tmp2 = df_copy[df_copy['text'] == response2]\n",
    "                    if len(df_copy_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "                    if len(df_copy_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                    score1 = df_copy_tmp['answer'].mean()\n",
    "                    score2 = df_copy_tmp2['answer'].mean()\n",
    "                    \n",
    "                    if math.isclose(score1, score2):\n",
    "                        print(\"STILL A TIE\")\n",
    "                        continue\n",
    "\n",
    "                if score1 > score2 and result > 0.5:\n",
    "                    match = 1\n",
    "                elif score1 < score2 and result < 0.5:\n",
    "                    match = 1\n",
    "                elif math.isclose(score1, score2) and math.isclose(result, 0.5):\n",
    "                    match = 1\n",
    "                    raise Exception(\"shouldn't be here\")\n",
    "                elif math.isclose(result, 0.5):\n",
    "                    match = 0.5\n",
    "                elif math.isclose(score1, score2):\n",
    "                    raise Exception(\"shouldn't be here\")\n",
    "                    match = 0.5\n",
    "                else: \n",
    "                    match = 0\n",
    "                    # print(response1, \"\\n\\n\", response2)\n",
    "                    # print(f\"{i} {j} {k} {l} {result} {score1} {score2}\")\n",
    "                    # if random.random() > 0.7: raise Exception(\"?\")\n",
    "\n",
    "                matching += match\n",
    "                total += 1\n",
    "                comparison_points.append((score1, score2, result, match))\n",
    "                look_at_points.append({\"response1\": response1, \"response2\": response2, \"score1\": score1, \"score2\": score2, \"result\": result, \"match\": match, \"i\": i, \"j\": j, \"k\": k, \"l\": l, \"advice1\": mapping[str(i)], \"advice2\": mapping[str(j)]})\n",
    "\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'barista/evaluators/the_customer/mappings_remapped.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/bt/s_9f74891td8dms1fzdqbkf80000gn/T/ipykernel_52545/3026327050.py\u001b[0m in \u001b[0;36m<cell line: 11>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{evaluator}/mappings_remapped.json\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0mremap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'barista/evaluators/the_customer/mappings_remapped.json'"
     ]
    }
   ],
   "source": [
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/responses/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "    remap = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "# start here\n",
    "df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "grouped = df_tmp.groupby('origin_id')\n",
    "for name, group in grouped:\n",
    "    for name2, group2 in grouped: \n",
    "        if name == name2: continue\n",
    "        score1 = group['answer'].mean()\n",
    "        score2 = group2['answer'].mean()\n",
    "\n",
    "        if True: # score1 - score2 >= 1.5 or score2 - score1 >= 1.5:\n",
    "            # look for it in the MWU matrix\n",
    "            advice_indices = group.iloc[0]['advice_indices']\n",
    "            advice_indices2 = group2.iloc[0]['advice_indices']\n",
    "            i = int(remap[advice_indices + \".json\"])\n",
    "            j = int(remap[advice_indices2 + \".json\"])\n",
    "            k = int(group.iloc[0]['generation_index'])\n",
    "            l = int(group2.iloc[0]['generation_index'])\n",
    "\n",
    "            if \"3\" in advice_indices or \"3\" in advice_indices2:\n",
    "                continue\n",
    "\n",
    "            if i > j or i == j and k > l: \n",
    "                continue\n",
    "\n",
    "            if MWU_matrix[i][j][k][l] > 0.5 and score1 > score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] < 0.5 and score1 < score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] == 0.5 and score1 == score2:\n",
    "                matching += 1\n",
    "            elif MWU_matrix[i][j][k][l] == 0.5 or score1 == score2:\n",
    "                matching += 0.5\n",
    "            total += 1\n",
    "\n",
    "            # with open(f\"{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\", \"r\") as f:\n",
    "            #     d = json.load(f)\n",
    "            \n",
    "            # print(d['indices'], d['flipped'], d['parsed_answers'], \"mwu\", MWU_matrix[i][j][k][l], \"human\", score1, score2, matching)\n",
    "            # print(d[\"user_prompt\"][\"content\"], \"\\n\\n\\n\\n\")\n",
    "            # print(group.iloc[0]['text'], \"\\n\\n\\n\")\n",
    "            # print(group2.iloc[0]['text'], \"\\n\\n\\n\\n\\n\")\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "human scores: 5.9 1.95\n",
      "evaluator: the_customer\n",
      "MWU result: 0.3333333333333333\n",
      "flipped: True\n",
      "parsed_answers: [1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Absolutely! We brew our lattes using a blend of beans that balances perfectly with whole milk. Your medium latte will be ready shortly. Do you like to have any flavors added? We have a variety of choices for syrups and sweetening options.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Of course! One medium whole milk latte coming up! We also have some delicious pastries that go great with it, would you like to try one?\"\n",
      "\n",
      "In which scenario are you more likely to leave a tip? Or if you would leave a tip in both cases, in which scenario would you leave a larger tip? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "reasoning: [\"As a customer, I am likely to tip in both situations as the baristas were polite and offered further service beyond my order. However, my tendency to leave a larger tip might be influenced by a few factors.\\n\\nIn scenario 1, the barista was knowledgeable about coffee and took the time to explain something about how their blend balances with whole milk. This indicates a potential interest in coffee and its craft, making the experience feel a bit more unique. Additionally, they offered further customization for my order- different flavors and sweetening options. This could be beneficial for someone who enjoys trying different flavors in their coffee.\\n\\nIn scenario 2, the barista suggested pairing my order with a pastry. This upselling might be viewed as an attempt to increase the total price of my order more than enhancing my experience, depending on my personal perception. If I'm not feeling hungry or if I'm diet conscious, I might not appreciate this suggestion as much.\\n\\nHowever, in both scenarios, willingness to tip more would also depend on how these suggestions were presented, the tone of the barista, and their engagement level.\\n\\nAnswer: For me, the larger tip would go in Scenario 1. I appreciate a knowledgeable barista who can offer further customization to my coffee as per my taste preference. It feels that they are invested not just in selling me a product, but in enhancing my overall coffee drinking experience.\", 'Reasoning:\\nIn both scenarios, the barista shows friendly customer service. However, in Scenario 1, the barista describes the coffee being used and offers a customizable experience with different flavors which can make me feel more taken care of and valued as a customer. In Scenario 2, the barista does also offer a suggestion for an add-on which shows interest in providing great customer service.\\n\\nAnswer: Scenario 1', \"Reasoning: The barista in scenario 1 not only responds positively to the order, but they also pleasantly inform about the quality of their beans and how they blend with whole milk, which reflects their knowledge and passion for their work. Moreover, suggesting flavors with different syrups and sweetening options shows an effort to personalize the experience. On the contrary, the second scenario is more specifically targeted towards cross-selling and doesn't provide additional information about the order. From these details, it's clear that the effort put into customer service is greater in the first scenario.\\n\\nAnswer: Scenario 1\"]\n",
      "\n",
      "\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'barista/evaluators/the_customer/compare_results/0_3_0_1.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/bt/s_9f74891td8dms1fzdqbkf80000gn/T/ipykernel_52545/1609634925.py\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mevaluator\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators/{evaluator}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m             \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{scenario}/evaluators/{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m                 \u001b[0mcomparison\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'barista/evaluators/the_customer/compare_results/0_3_0_1.json'"
     ]
    }
   ],
   "source": [
    "# print out the reasonings for items in look_at_points\n",
    "\n",
    "for item in look_at_points:\n",
    "    i, j, k, l = item[\"i\"], item[\"j\"], item[\"k\"], item[\"l\"]\n",
    "\n",
    "    for evaluator in os.listdir(f\"{scenario}/evaluators\"):\n",
    "        if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "            with open(f\"{scenario}/evaluators/{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\", \"r\") as f:\n",
    "                comparison = json.load(f)\n",
    "    \n",
    "            print(f\"human scores: {item['score1']} {item['score2']}\")\n",
    "            print(f\"evaluator: {evaluator}\")\n",
    "            print(f\"MWU result: {result}\")\n",
    "            print(f\"flipped: {comparison['flipped']}\")\n",
    "            print(f\"parsed_answers: {comparison['parsed_answers']}\")\n",
    "            if type(comparison['user_prompt']) == type([]):\n",
    "                user_prompt = comparison['user_prompt'][0]['content']\n",
    "            else: user_prompt = comparison['user_prompt']['content']\n",
    "            print(f\"user_prompt: {user_prompt}\\n\")\n",
    "            print(f\"reasoning: {comparison['responses']}\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 617,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that the weighted MWU matrix is correct\n",
    "\n",
    "# get the weighted MWU matrix\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "weighted_MWU_matrix = np.load(path)\n",
    "\n",
    "# get the weights\n",
    "with open(f\"{scenario}/evaluators/weights.json\", \"r\") as f:\n",
    "    weights = json.load(f)\n",
    "\n",
    "MWU_constructed_matrix = np.zeros(weighted_MWU_matrix.shape)\n",
    "\n",
    "# get each MWU matrix and the weights\n",
    "for evaluator in os.listdir(f\"{scenario}/evaluators/\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "        MWU_matrix = np.load(f\"{scenario}/evaluators/{evaluator}/MWU_matrix.npy\")\n",
    "\n",
    "        for d in weights:\n",
    "            if d[\"evaluator\"] == evaluator:\n",
    "                weight = d[\"weight\"]\n",
    "                break\n",
    "        \n",
    "        MWU_constructed_matrix += MWU_matrix * int(weight)/100\n",
    "\n",
    "assert np.allclose(weighted_MWU_matrix, MWU_constructed_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 618,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fc7d4c7b3a0>"
      ]
     },
     "execution_count": 618,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6524 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# PLOT MATCHES\n",
    "data = comparison_points\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "# Separate the data based on the 'match' values\n",
    "x_match_0 = [x for x, y, result, match in data if match == 0]\n",
    "y_match_0 = [y for x, y, result, match in data if match == 0]\n",
    "\n",
    "x_match_0_5 = [x for x, y, result, match in data if match == 0.5]\n",
    "y_match_0_5 = [y for x, y, result, match in data if match == 0.5]\n",
    "\n",
    "x_match_1 = [x for x, y, result, match in data if match == 1]\n",
    "y_match_1 = [y for x, y, result, match in data if match == 1]\n",
    "\n",
    "# Create the scatter plot with different colors\n",
    "plt.scatter(x_match_0, y_match_0, color='red', label='match=0')\n",
    "plt.scatter(x_match_0_5, y_match_0_5, color='yellow', label='match=0.5')\n",
    "plt.scatter(x_match_1, y_match_1, color='green', label='match=1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Result')"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6248 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "result_match_0 = [result for x, y, result, match in data if match == 0]\n",
    "result_match_0_5 = [result for x, y, result, match in data if match == 0.5]\n",
    "result_match_1 = [result for x, y, result, match in data if match == 1]\n",
    "\n",
    "# Create the scatter plot with different colors\n",
    "plt.scatter(result_match_0, [0] * len(result_match_0), color='red', label='match=0')\n",
    "plt.scatter(result_match_0_5, [0.5] * len(result_match_0_5), color='yellow', label='match=0.5')\n",
    "plt.scatter(result_match_1, [1] * len(result_match_1), color='green', label='match=1')\n",
    "\n",
    "# Set labels and legend\n",
    "plt.xlabel('Result')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 619,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['0', '0_1', '0_2', '0_3', '1', '1_2', '1_3', '2', '2_3', '3', '[]'])\n",
      "1 piece 5.881666666666667 600\n",
      "2 pieces 5.666666666666667 720\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7fc7d690b190>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d690b310>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6919310>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d69195b0>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7fc7d690b5b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d690b850>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6919850>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6919af0>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7fc7d2b02d30>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6919070>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7fc7d690baf0>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6919d90>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7fc7d690bd90>,\n",
       "  <matplotlib.lines.Line2D at 0x7fc7d6927070>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 619,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4152 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR NUMBER OF ADVICE INDICES\n",
    "\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "\n",
    "grouped = df_tmp.groupby(['advice_indices'])['answer']\n",
    "print(grouped.groups.keys())\n",
    "\n",
    "# zero_pieces = grouped.get_group(\"nan\")\n",
    "one_piece = pd.concat([grouped.get_group(\"0\"), grouped.get_group(\"1\"), grouped.get_group(\"2\"), grouped.get_group(\"3\")])\n",
    "two_pieces = pd.concat([grouped.get_group(\"0_1\"), grouped.get_group(\"0_2\"), grouped.get_group(\"1_2\"), grouped.get_group(\"0_3\"), grouped.get_group(\"1_3\"), grouped.get_group(\"2_3\")])\n",
    "\n",
    "# print(\"0 pieces\", zero_pieces.mean(), len(zero_pieces))\n",
    "print(\"1 piece\", one_piece.mean(), len(one_piece))\n",
    "print(\"2 pieces\", two_pieces.mean(), len(two_pieces))\n",
    "\n",
    "plt.boxplot([one_piece, two_pieces], labels=[\"1 piece\", \"2 pieces\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ties: 25\n",
      "total: 2556\n"
     ]
    }
   ],
   "source": [
    "# NUMBER OF TIES FOR AVERAGE SCORES FOR EACH GENERATION\n",
    "\n",
    "grouped = df.groupby('origin_id')['answer'].mean()\n",
    "ties = 0\n",
    "\n",
    "for origin_id, score in grouped.iteritems():\n",
    "    for origin_id2, score2 in grouped.iteritems():\n",
    "        if score == score2 and origin_id != origin_id2:\n",
    "            ties += 1\n",
    "            # print(origin_id, origin_id2, score)\n",
    "\n",
    "ties = ties // 2\n",
    "print(\"ties:\", ties)\n",
    "print(\"total:\", len(grouped) * (len(grouped) - 1) // 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 30 baseline_ask_with_reasoning baseline_neutral_advice [] 1\n",
      "6 71 baseline_ask_with_reasoning unorthodox_advice [] 2_3\n",
      "8 42 baseline_irrelevant_advice baseline_neutral_advice 0 1_2\n",
      "9 23 baseline_irrelevant_advice baseline_irrelevant_advice 0 1_2\n",
      "9 33 baseline_irrelevant_advice baseline_neutral_advice 0 0_1\n",
      "10 50 baseline_irrelevant_advice unorthodox_advice 1 0_1\n",
      "16 69 baseline_irrelevant_advice unorthodox_advice 2 1_3\n",
      "22 25 baseline_irrelevant_advice baseline_neutral_advice 1_2 0\n",
      "23 9 baseline_irrelevant_advice baseline_irrelevant_advice 1_2 0\n",
      "23 33 baseline_irrelevant_advice baseline_neutral_advice 1_2 0_1\n",
      "25 22 baseline_neutral_advice baseline_irrelevant_advice 0 1_2\n",
      "26 51 baseline_neutral_advice unorthodox_advice 0 0_1\n",
      "26 60 baseline_neutral_advice unorthodox_advice 0 1_2\n",
      "26 72 baseline_neutral_advice unorthodox_advice 0 2_3\n",
      "27 38 baseline_neutral_advice baseline_neutral_advice 0 0_2\n",
      "27 68 baseline_neutral_advice unorthodox_advice 0 1_3\n",
      "30 6 baseline_neutral_advice baseline_ask_with_reasoning 1 []\n",
      "30 71 baseline_neutral_advice unorthodox_advice 1 2_3\n",
      "31 45 baseline_neutral_advice unorthodox_advice 0_1 0\n",
      "32 48 baseline_neutral_advice unorthodox_advice 0_1 1\n",
      "33 9 baseline_neutral_advice baseline_irrelevant_advice 0_1 0\n",
      "33 23 baseline_neutral_advice baseline_irrelevant_advice 0_1 1_2\n",
      "35 53 baseline_neutral_advice unorthodox_advice 2 2\n",
      "36 37 baseline_neutral_advice baseline_neutral_advice 2 0_2\n",
      "36 44 baseline_neutral_advice unorthodox_advice 2 0\n",
      "37 36 baseline_neutral_advice baseline_neutral_advice 0_2 2\n",
      "37 44 baseline_neutral_advice unorthodox_advice 0_2 0\n",
      "38 27 baseline_neutral_advice baseline_neutral_advice 0_2 0\n",
      "38 68 baseline_neutral_advice unorthodox_advice 0_2 1_3\n",
      "39 52 baseline_neutral_advice unorthodox_advice 0_2 2\n",
      "42 8 baseline_neutral_advice baseline_irrelevant_advice 1_2 0\n",
      "43 55 unorthodox_advice unorthodox_advice 0 0_2\n",
      "43 65 unorthodox_advice unorthodox_advice 0 0_3\n",
      "44 36 unorthodox_advice baseline_neutral_advice 0 2\n",
      "44 37 unorthodox_advice baseline_neutral_advice 0 0_2\n",
      "45 31 unorthodox_advice baseline_neutral_advice 0 0_1\n",
      "46 54 unorthodox_advice unorthodox_advice 1 2\n",
      "46 64 unorthodox_advice unorthodox_advice 1 0_3\n",
      "48 32 unorthodox_advice baseline_neutral_advice 1 0_1\n",
      "50 10 unorthodox_advice baseline_irrelevant_advice 0_1 1\n",
      "51 26 unorthodox_advice baseline_neutral_advice 0_1 0\n",
      "51 60 unorthodox_advice unorthodox_advice 0_1 1_2\n",
      "51 72 unorthodox_advice unorthodox_advice 0_1 2_3\n",
      "52 39 unorthodox_advice baseline_neutral_advice 2 0_2\n",
      "53 35 unorthodox_advice baseline_neutral_advice 2 2\n",
      "54 46 unorthodox_advice unorthodox_advice 2 1\n",
      "54 64 unorthodox_advice unorthodox_advice 2 0_3\n",
      "55 43 unorthodox_advice unorthodox_advice 0_2 0\n",
      "55 65 unorthodox_advice unorthodox_advice 0_2 0_3\n",
      "60 26 unorthodox_advice baseline_neutral_advice 1_2 0\n",
      "60 51 unorthodox_advice unorthodox_advice 1_2 0_1\n",
      "60 72 unorthodox_advice unorthodox_advice 1_2 2_3\n",
      "61 63 unorthodox_advice unorthodox_advice 3 3\n",
      "SAME ADVICE SET\n",
      "62 70 unorthodox_advice unorthodox_advice 3 2_3\n",
      "63 61 unorthodox_advice unorthodox_advice 3 3\n",
      "SAME ADVICE SET\n",
      "64 46 unorthodox_advice unorthodox_advice 0_3 1\n",
      "64 54 unorthodox_advice unorthodox_advice 0_3 2\n",
      "65 43 unorthodox_advice unorthodox_advice 0_3 0\n",
      "65 55 unorthodox_advice unorthodox_advice 0_3 0_2\n",
      "66 67 unorthodox_advice unorthodox_advice 0_3 1_3\n",
      "67 66 unorthodox_advice unorthodox_advice 1_3 0_3\n",
      "68 27 unorthodox_advice baseline_neutral_advice 1_3 0\n",
      "68 38 unorthodox_advice baseline_neutral_advice 1_3 0_2\n",
      "69 16 unorthodox_advice baseline_irrelevant_advice 1_3 2\n",
      "70 62 unorthodox_advice unorthodox_advice 2_3 3\n",
      "71 6 unorthodox_advice baseline_ask_with_reasoning 2_3 []\n",
      "71 30 unorthodox_advice baseline_neutral_advice 2_3 1\n",
      "72 26 unorthodox_advice baseline_neutral_advice 2_3 0\n",
      "72 51 unorthodox_advice unorthodox_advice 2_3 0_1\n",
      "72 60 unorthodox_advice unorthodox_advice 2_3 1_2\n"
     ]
    }
   ],
   "source": [
    "# How many of the ties are from the same advice set?\n",
    "grouped = df.groupby('origin_id')['answer'].mean()\n",
    "ties = []\n",
    "\n",
    "for origin_id, score in grouped.iteritems():\n",
    "    for origin_id2, score2 in grouped.iteritems():\n",
    "        if score == score2 and origin_id != origin_id2:\n",
    "            ties.append((origin_id, origin_id2))\n",
    "\n",
    "grouped2 = df.groupby('origin_id')[['advice_type', 'advice_indices']]\n",
    "for (origin_id, origin_id2) in ties:\n",
    "    advice_type = grouped2.get_group(origin_id)['advice_type'].unique()[0]\n",
    "    advice_type2 = grouped2.get_group(origin_id2)['advice_type'].unique()[0]\n",
    "    advice_indices = grouped2.get_group(origin_id)['advice_indices'].unique()[0]\n",
    "    advice_indices2 = grouped2.get_group(origin_id2)['advice_indices'].unique()[0]\n",
    "    print(origin_id, origin_id2, advice_type, advice_type2, advice_indices, advice_indices2)\n",
    "    if advice_type == advice_type2 and advice_indices == advice_indices2:\n",
    "        print(\"SAME ADVICE SET\")\n",
    "\n",
    "# ONLY 1 TIE IS FROM THE SAME ADVICE SET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline_ask_with_reasoning baseline_neutral_advice [] 0 5.783333333333333\n",
      "baseline_neutral_advice baseline_ask_with_reasoning 0 [] 5.783333333333333\n",
      "baseline_neutral_advice unorthodox_advice 1 1_2 5.55\n",
      "unorthodox_advice baseline_neutral_advice 1_2 1 5.55\n"
     ]
    }
   ],
   "source": [
    "# what about ties between advice sets (instead of generations?)\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "ties = 0\n",
    "\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    for (advice_type2, advice_indices2), score2 in grouped.iteritems():\n",
    "        if score == score2 and (advice_type != advice_type2 or advice_indices != advice_indices2):\n",
    "            ties += 1\n",
    "            print(advice_type, advice_type2, advice_indices, advice_indices2, score)\n",
    "# NO TIES BETWEEN PIECES OF ADVICE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 620,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.8\n",
      "6.178333333333334\n",
      "0.7893344351887308\n",
      "2.054473483446754\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- RESPONSE\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped_best = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped_best.mean())\n",
    "\n",
    "# average of all generations\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "print(df_tmp['answer'].mean())\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(np.std(grouped, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped_best.mean() - grouped.mean())/np.std(grouped, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 621,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.8\n",
      "6.302777777777778\n",
      "0.8461803837421814\n",
      "1.7693889517987256\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- RESPONSE WITHOUT SOCIAL MEDIA\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped_best = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped_best.mean())\n",
    "\n",
    "# average of all generations\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '0_1']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_2']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_3']\n",
    "print(df_tmp['answer'].mean())\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(np.std(grouped, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped_best.mean() - grouped.mean())/np.std(grouped, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.8 \"Hey honey, I hope you had a good time with your friends tonight. Did Mike finally ace his pool game? How was James handling his break up? I'm sure he appreciates having you around. I know they all love your company a lot, and I love that you're such a wonderful friend to them. But you know, I miss having you here at night as well. Your laughter, your warmth, your company - it just brightens up my day. And I feel left out sometimes when you come home so late. I understand and value your bonding time with your friends but maybe we can work on a balance? So that both of us get to spend some quality time with you. What do you think?\"\n",
      "7.45 \"Hey, honey. I hope you had a good time out with the guys tonight. What did you end up doing?... Sounds like a lot of fun! You know, while you were having your good time with them, I've been thinking. Don’t get me wrong here, I genuinely love that you have such close friends and enjoy spending time with them so much. It's important to maintain those relationships. But I was thinking, how about we both work on balancing our social lives with our time together? We made a commitment to each other and I feel it’s equally important to keep nourishing this relationship too. It's not that I want you to stop going out, but maybe we could compromise somehow? Spending a few more evenings together would mean a lot to me. What do you think?”\n",
      "7.2 \"Hey honey, can we talk for a moment? I've been feeling a bit lonely recently when you stay out late with your friends. I understand that it's important for you to blow off steam and maintain those friendships. And, you know, I would like to spend some time with my friends too. But at the same time, I really miss spending time together just the two of us. So how about we both work on balancing our social lives with our time together? This way, neither of us has to give up seeing our friends, but we still get to enjoy quality time as a couple. What do you think?\"\n",
      "7.2 \"Hey honey, can we talk? I understand that hanging out with your friends is important to you, and it's vital for both of us to have our own social activities. However, I feel like we haven't been spending much quality time together lately because you've been coming home pretty late. I miss you and our time together. How about we both work on balancing our social lives with our time together better? That way, neither of us feel left out or neglected. What do you think?\"\n",
      "7.05 \"Hey honey, how was your evening out with the guys? Did Pete manage to pull off that ridiculous trick he was talking about the other day?\" Pause for his response and then continue, \"You know, I really miss spending those relaxed, fun times together with you. We used to have such a wonderful time just enjoying each other's company. Lately, I feel your late nights with your friends are affecting our relationship and I miss you. Even if you still want to keep spending time with your friends, which I totally understand, just remember to balance the time you have with me as well, okay?\"\n",
      "7.05 \"Honey, we need to talk. I just wanted to tell you that I really miss you and our time together. Your late nights out with friends have been affecting our relationship and it's been making me feel quite lonely. I completely understand that spending time with your friends is important to you and I respect that, but I also believe that we should balance that with the quality time we have together. All I'm asking for is a little more time with you during the evenings. I hope we can work on this together.\"\n",
      "7.0 \"Hey, honey. How was your night out? I bet you and the guys had a blast! What was the highlight of the evening?\" \n",
      "\n",
      "After your husband has shared his experiences and you've shown interest, you could gently steer the conversation towards the main issue. \n",
      "\n",
      "\"That sounds really fun, you know I enjoy hearing about your adventures. It's great that you have such close friends. I was thinking about us and you know I genuinely love that we both have our own set of friends and we enjoy spending time with them. However, I feel we should work on finding a better balance. How about we both work on balancing our social lives with our time together? I miss spending those cosy evenings with you. What do you think?\"\n",
      "6.9 \"Hey, honey, I really need to talk to you about something that's been bothering me lately. It's about these late nights you've been spending with your friends. I want you to know that I completely respect and understand your need to hang out with them. But I really miss you and the quality time we used to spend together. I feel that this is affecting our relationship. Instead of constantly staying out late, how about we both make an effort to balance our social lives and time together? I believe that this little adjustment can do wonders for our relationship. What do you think?\"\n",
      "6.6 \"Hey, love, how was your day? I'm genuinely curious about what you and your friends get up to on these late nights. You guys always seem to have so much fun. I also think it's really cool how you're dedicated to your friendships...it's one of the things I love about you. Speaking of that, I've realized that I miss having our quality time together too, you know. How about trying to balance both worlds, ensuring that neither is deprived? We could try to spend a little more time together during the week, perhaps plan a few activities or dates, while you still get to hang out with your buddies. I think it would be beneficial for us both, don't you think?\"\n",
      "6.5 \"Hey honey, how was your evening?\" You watch attentively as he starts to share about his day, occasionally asking him to tell you more about certain highlights. \"It sounds like you guys had an absolute blast! You know, I like hearing about your adventures with your friends. It always fascinates me to see how much you enjoy their company. I envy them sometimes, you know?\" You give him a soft laugh, hoping to ease him into the conversation. \"You know,\" you continue, \"I've been missing those times we'd just stay in and enjoy each other's company. Don't get me wrong, I know how important your friends are to you, but I was hoping we could spend more time together, too. I guess I just miss my best friend. What do you think?\"\n",
      "6.5 \"Hey honey, I was just wondering how your day was, and how things went with the guys last night? It seems like you all have a great time whenever you're out together. But I wanted to talk about that a little...I do miss you when you're not home in the evenings. It can be pretty difficult for me to spend my nights alone fearing any potential mishap. Do you ever wonder how I feel when you're not home late at night and I'm left by myself and worrying? I understand that you need your time with your friends, which is vital, and I respect that. But maybe it's time that we work on a schedule that includes a balance of us time and friend time? What do you think?\"\n",
      "6.45 \"Hey honey, I've noticed you've been spending a lot of time out with your friends lately, and that's great. I'm glad you're having fun. But I feel like we're not spending as much quality time together as we used to. Maybe both of us need to think about how we balance our social lives with our time together. What do you think?\"\n",
      "6.2 \"Hey, sweetheart, I think we should have a little chat. Do you ever wonder how I feel when I'm left alone late at night while you're out with your friends? It's not about me not wanting you to have a good time but about missing the time we used to spend together. It feels a bit lonely without you. Maybe, we can work out a compromise on this? Like, how about we both make an effort to balance our social lives with our time together? I think it might help us feel more connected.\"\n",
      "6.2 \"Hey love, how was your day today? Did you guys discuss another of those mind-boggling topics that you love so much? You know, hearing about your shenanigans with your friends is always so interesting and I genuinely enjoy seeing you happy and relaxed. \n",
      "\n",
      "However, do you remember how we used to balance our social lives with spending quality time with each other when we first got married? How we made sure we had time for our friends but also for us as a couple? Lately, it seems we've both been so drawn into our social circles and it feels like we're gradually losing that closeness we share. \n",
      "\n",
      "How about we both start working on this together? We could start by designating certain days or nights of the week where we can hang out with our friends individually or together and also ensure we have 'us' time. What do you think?\"\n",
      "6.2 \"Honey, we need to talk. I've been feeling quite lonely lately when you're not home with me late at night. Do you ever wonder how I feel when I'm left alone to fend for myself? I know your friends are important to you and I respect that, but I've really been missing our time together. I like the idea of both of us having a social life, but also making sure we're balancing it with our time as a couple. How about we both work on that?\"\n",
      "6.05 \"Hey, honey. How was your day? What shenanigans did you and your friends get into today?\" (Wait for him to respond). \"I'm glad you're having fun with your friends. It's really important to have strong friendships. You know, I've been thinking and I realized how much I've been missing you and our special times together. Your late nights out with your friends are beginning to affect our relationship. We were always great partners and enjoy each other's company. Let's find a way to balance these important aspects of our life. What do you think?\"\n",
      "6.0 \"Hey, sweetheart, I was hoping we could talk about something that's been on my mind lately. I understand that you love spending time with your friends, I really do, but I think we need to talk about how often you're out late with them. It leaves me feeling a little lonely. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? I'm not asking you to stop seeing your friends, I'm just asking for balance. You're my partner, and I miss you and our time together when you're out so late. Your late nights out are affecting our relationship, and I'm sure that's not what either of us wants. Would it be possible for us to find a way to manage your nights out better?\"\n",
      "5.95 \"Hey, sweetheart, can we sit down and talk for a moment? I've been feeling quite lonely when you're out with your friends late at night. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself during those moments? I completely understand that spending time with your friends is important, and I wouldn't want you to stop doing that because I know how much you cherish those moments. But, I also miss having you around. I think we should both work on balancing our social lives with our time spent together. I believe it's important that we both make an effort to ensure each other's happiness and well-being. I really hope you understand where I'm coming from.\"\n",
      "5.95 \"Sweetheart, we need to discuss something. It's really bothering me that you've been staying out so late with your friends so often. Please don't get me wrong, I understand the need to socialize and unwind, but I miss our time together. Your late nights are having an impact on our relationship. I don't want to control your life, but perhaps we can find a middle ground on this. How about we both work on balancing our social lives and our time as a couple? I think it would really help in maintaining our bond as husband and wife.\"\n",
      "5.95 \"Honey, I think we need to talk. I want to let you know that I miss you. I miss the quality time we used to spend together. Recently, your frequent late nights out with your friends are causing a strain on our relationship. This is about us, our relationship. It isn't right or wrong, it's about how I feel and I hope you understand this. I am not asking you to stop going out with your friends, but I am asking for balance. I want us to prioritize our time as a couple too. Let's try to find a solution that will work for both of us.\"\n",
      "5.6 \"Honey, I need to discuss something that's been bothering me lately. You always say honesty is key in our relationship, and I believe it's time to be honest about this. I've been feeling lonely and neglected because you've been coming home late due to your nights out with friends. I miss our time together, I miss us. Your habitual late nights are affecting our relationship and it's creating a rift between us. I understand the importance of spending time with your friends but I wish you could also consider the time you spend with me. We're a team, remember?\"\n",
      "5.6 \"Honey, can we sit down and talk for a moment? I really need you to hear me out on this. When you're out late at night with your friends, do you ever wonder how I feel? I feel lonely, I feel left out. It's like I'm left to fend for myself during those late hours. I understand that you need to spend time with your friends, but we also need our time together. I'm not asking you to stop hanging out with them altogether, but some balance would be really appreciated. I really miss our evenings together. Can we work on this?\"\n",
      "5.55 \"Hey honey, can we talk for a bit? I've been feeling a bit down lately. I've been missing you in the evenings when you're out with your friends. I know you enjoy your time with them, but do you ever wonder how it makes me feel when I'm home alone till late at night? It's not only that I miss our time together but it also makes me feel like I'm left to fend for myself. I feel like your late nights out are affecting our relationship, and I really don't want that. I just hope you understand where I'm coming from.\"\n",
      "5.45 \"Hey honey, how was your hangout with the guys tonight? I bet you guys had a great time, didn't you? I always love hearing about your adventures, they're so lively and entertaining. I just sometimes wonder, do you ever think about how I might feel when I am at home by myself? Like tonight, for example. It was pretty quiet and lonely without you. I just missed you quite a bit, and I hope in future you can balance out your time with your friends and with me. Because we are a team, right? And it's always better when we are together.\"\n",
      "5.45 \"Hey honey, how was your day? You always have the most interesting stories about your adventures with your friends. What did you guys get up to tonight?\" \n",
      "\n",
      "(Pause for his response)\n",
      "\n",
      "\"I really do enjoy hearing about your time with your friends, it's always so full of life and laughter. But, and you know I wouldn't be saying this if it didn't really matter to me, I have to tell you that I miss spending time with you. Those late nights you spend out are beginning to affect our relationship. I know you might not feel the same way, but I thought we could talk about it. Maybe we can find a way to balance things. What do you think?\"\n",
      "5.45 \"Hey, can we talk for a moment? Look, I've been missing our time together. You know, your late nights out with friends have been affecting us, and our relationship. I'm not saying you shouldn't hang out with them at all, but I think we both need to work on balancing our social lives with our time together. How does that sound to you?\"\n",
      "5.3 \"Hey, love. How was your day? You always seem to have such a great time with your friends. Did anything exciting happen tonight? It's wonderful that you have such a close-knit group. I can't help but feel a bit lonely here at home though. Do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? I know your friends are important to you, and I respect that, but could we try to balance it a bit? Maybe some nights could be 'us time'? I hope you understand where I'm coming from, I just miss spending time with you, my husband.\"\n",
      "5.2 \"Honey, I know you need your time with your friends, and I respect that. We all need our space and a bit of fun, no question about that. But sweetheart, can we talk about this, please? Do you ever wonder how I feel when you're out late at night, and I'm here alone? It gives me this sense of being alone, left to fend for myself when it's dark outside. I miss our shared evenings and the warmth of your presence. Couldn't we find a balance that allows you to have your boys' nights but also allows us to spend quality time together? I really believe we can find a solution that respects both our needs and desires.\"\n",
      "4.9 \"Hey honey, can we sit down for a moment to talk about something? It's been on my mind for a while and I think it's important for us to discuss it together. I've noticed that you've been spending most of your nights out with your friends lately. And while I fully support you having a robust social life, I can't help but feel neglected. When you're not at home with me, it feels pretty lonely and difficult to be left alone late at night, I miss the comfort of knowing you are nearby. Do you ever wonder how I feel during those moments?\" \n",
      "\n",
      "\"I want you to know that I deeply miss our cherished time together. Your consistent late nights out have begun to affect our relationship, and it's taking a toll on me emotionally. I don't want this issue to keep us apart or create a rift between us. So, I thought it would be helpful to bring this to your attention so we can find a compromise or solution that suits both of us. I value both our time together and your personal freedom, so let's try to find a balanced way to enjoy these aspects of our lives together.\"\n",
      "4.65 \"Hey sweetheart, do you have a moment? There's something that's been bothering me a bit lately and I think we should have a talk. You know how much I love you and really value our time together. You've been spending a lot of your nights out with your friends recently, which I totally understand because everyone needs to have their own time with friends. But do you ever wonder how I feel when you're not home with me and I'm left to fend for myself late at night? It would be really nice if we could have more time together in the evenings. Could we maybe find a balance where we both feel happy and fulfilled? Just consider it from my point of view, please.\"\n",
      "30\n"
     ]
    }
   ],
   "source": [
    "# print where it is in the non-3 (4th best)\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "grouped = df_tmp.groupby('origin_id')\n",
    "\n",
    "scores = []\n",
    "texts = []\n",
    "for origin_id in grouped.groups.keys():\n",
    "    df_tmp2 = grouped.get_group(origin_id)\n",
    "    scores.append(df_tmp2['answer'].mean())\n",
    "    texts.append(df_tmp2.iloc[0]['text'])\n",
    "\n",
    "order = np.argsort(scores)[::-1]\n",
    "for i in order:\n",
    "    print(scores[i], texts[i])\n",
    "print(len(order))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 622,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.883333333333333\n",
      "6.178333333333334\n",
      "0.6189921118785774\n",
      "1.1389482781297433\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- ADVICE\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_advice_index]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped.mean())\n",
    "\n",
    "# average of all advice\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "grouped_advice = df_tmp.groupby('advice_indices')['answer'].mean()\n",
    "print(grouped_advice.mean())\n",
    "print(np.std(grouped_advice, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped.mean() - grouped_advice.mean()) / np.std(grouped_advice, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.566666666666666\n",
      "6.202777777777778\n",
      "0.3097340555786289\n",
      "1.174843005910635\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- ADVICE WITHOUT SOCIAL MEDIA\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == \"1_2\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped.mean())\n",
    "\n",
    "# average of all advice\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '0_3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '1_3']\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] != '2_3']\n",
    "grouped_advice = df_tmp.groupby('advice_indices')['answer'].mean()\n",
    "print(grouped_advice.mean())\n",
    "print(np.std(grouped_advice, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped.mean() - grouped_advice.mean()) / np.std(grouped_advice, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 470,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean scores for best framework response 6.2\n",
      "mean scores for baseline_ask 6.783333333333334\n",
      "mean scores for baseline_ask_with_reasoning 6.7\n",
      "mean scores for baseline_neutral_advice 6.447222222222221\n",
      "todo: run the framework on this\n",
      "mean scores for baseline_irrelevant_advice 4.3916666666666675\n",
      "todo: run the framework on this\n"
     ]
    }
   ],
   "source": [
    "# COMPARISONS AGAINST BASELINES\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for best framework response\", grouped.mean())\n",
    "\n",
    "# average of all baseline_ask\n",
    "df_tmp = df[df['advice_type'] == \"baseline_ask\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_ask\", grouped.mean())\n",
    "\n",
    "# average of all baseline_ask_with_reasoning\n",
    "df_tmp = df[df['advice_type'] == \"baseline_ask_with_reasoning\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_ask_with_reasoning\", grouped.mean())\n",
    "\n",
    "# average of all baseline_neutral_advice\n",
    "df_tmp = df[df['advice_type'] == \"baseline_neutral_advice\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_neutral_advice\", grouped.mean())\n",
    "print(\"todo: run the framework on this\")\n",
    "\n",
    "# average of all baseline_irrelevant_advice\n",
    "df_tmp = df[df['advice_type'] == \"baseline_irrelevant_advice\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_irrelevant_advice\", grouped.mean())\n",
    "print(\"todo: run the framework on this\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import remap\n",
    "\n",
    "remap(f\"{scenario}/evaluators/artisan/mappings\")\n",
    "with open(f\"{scenario}/evaluators/artisan/mappings_remapped.json\", \"r\") as f:\n",
    "    remapping = json.load(f)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A. analysis on large differences\n",
    "import os\n",
    "import re\n",
    "import random\n",
    "\n",
    "def get_large_differences(df):\n",
    "    \n",
    "    evaluator = \"artisan\"\n",
    "    with open(f\"{scenario}/evaluators/{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "        remapping = json.load(f)[0]\n",
    "\n",
    "    diff_threshold = 5\n",
    "    df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "    grouped = df_tmp.groupby('participant_id')[['answer', 'advice_indices', 'generation_index']]\n",
    "    participant_ids = sorted(list(grouped.groups.keys()))\n",
    "    l = []\n",
    "\n",
    "    for i in range(len(grouped)):\n",
    "        participant_id = participant_ids[i]\n",
    "        participant_df = grouped.get_group(participant_id)\n",
    "\n",
    "        for j in range(len(participant_df)):\n",
    "            score = participant_df.iloc[j]['answer']\n",
    "\n",
    "            for k in range(j+1, len(participant_df)):\n",
    "                score2 = participant_df.iloc[k]['answer']\n",
    "\n",
    "                if abs(score - score2) >= diff_threshold:\n",
    "                    advice_indices1 = participant_df.iloc[j]['advice_indices']\n",
    "                    advice_indices2 = participant_df.iloc[k]['advice_indices']\n",
    "                    generation_index1 = int(participant_df.iloc[j]['generation_index'])\n",
    "                    generation_index2 = int(participant_df.iloc[k]['generation_index'])\n",
    "\n",
    "                    if advice_indices1 == \"nan\" or advice_indices2 == \"nan\":\n",
    "                        continue\n",
    "\n",
    "                    compare_index1 = int(remapping[f\"{advice_indices1}.json\"])\n",
    "                    compare_index2 = int(remapping[f\"{advice_indices2}.json\"])\n",
    "\n",
    "                    # print(f\"{advice_indices1}, generation {generation_index1}, score {score}\")\n",
    "                    # print(f\"{advice_indices2}, generation {generation_index2}, score {score2}\")\n",
    "\n",
    "                    if compare_index1 > compare_index2 or (compare_index1 == compare_index2 and generation_index1 > generation_index2):\n",
    "                        compare_index1, compare_index2 = compare_index2, compare_index1\n",
    "                        advice_indices1, advice_indices2 = advice_indices2, advice_indices1\n",
    "                        generation_index1, generation_index2 = generation_index2, generation_index1\n",
    "                        score, score2 = score2, score\n",
    "\n",
    "                    print(f\"participant {participant_id} has a large difference:\")\n",
    "                    print(f\"{advice_indices1}, generation {generation_index1}, score {score}\")\n",
    "                    print(f\"{advice_indices2}, generation {generation_index2}, score {score2}\")\n",
    "\n",
    "                    d = {}\n",
    "                    d[\"participant_id\"] = participant_id\n",
    "                    d[\"advice_indices1\"] = advice_indices1\n",
    "                    d[\"advice_indices2\"] = advice_indices2\n",
    "                    d[\"generation_index1\"] = generation_index1\n",
    "                    d[\"generation_index2\"] = generation_index2\n",
    "                    d[\"score1\"] = score\n",
    "                    d[\"score2\"] = score2\n",
    "\n",
    "                    for evaluator in os.listdir(f\"{scenario}/evaluators\"):\n",
    "                        if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "                            with open(f\"{scenario}/evaluators/{evaluator}/compare_results/{compare_index1}_{compare_index2}_{generation_index1}_{generation_index2}.json\", \"r\") as f:\n",
    "                                compare_result = json.load(f)\n",
    "                            \n",
    "                            scenario_options_text = compare_result[\"user_prompt\"][\"content\"]\n",
    "                            scenario_options_text = re.sub(r'Please think step by step and explain your reasoning.*', '', scenario_options_text, flags=re.DOTALL)\n",
    "                            scores = compare_result[\"parsed_answers\"]\n",
    "                            reasoning_text = compare_result[\"responses\"]\n",
    "\n",
    "                            if \"evaluator\" in d:\n",
    "                                d[\"evaluator\"] += [evaluator]\n",
    "                                d[\"flipped\"] += [compare_result[\"flipped\"]]\n",
    "                                d[\"scenario_options_text_2\"] = scenario_options_text\n",
    "                                d[\"scores\"] += [scores]\n",
    "                                d[\"reasoning_text_4\"] = reasoning_text[0]\n",
    "                                d[\"reasoning_text_5\"] = reasoning_text[1]\n",
    "                                d[\"reasoning_text_6\"] = reasoning_text[2]\n",
    "                            else:\n",
    "                                d[\"evaluator\"] = [evaluator]\n",
    "                                d[\"flipped\"] = [compare_result[\"flipped\"]]\n",
    "                                d[\"scenario_options_text_1\"] = scenario_options_text\n",
    "                                d[\"scores\"] = [scores]\n",
    "                                d[\"reasoning_text_1\"] = reasoning_text[0]\n",
    "                                d[\"reasoning_text_2\"] = reasoning_text[1]\n",
    "                                d[\"reasoning_text_3\"] = reasoning_text[2]\n",
    "                            \n",
    "\n",
    "                            print(f\"Evaluator: {evaluator}:\")\n",
    "                            print(f\"Flipped: {compare_result['flipped']}\")\n",
    "                            print(f\"Scenarios: {scenario_options_text}\\n\\n\")\n",
    "                            print(f\"SCORES: 1 -> first one better, 0 -> second one better (these are affected by flipped): {scores}\\n\")\n",
    "                            print(f\"Model Reasoning (these are also affected by flipped): \\n{reasoning_text}\\n\\n\\n\")\n",
    "\n",
    "                    l.append(d)\n",
    "\n",
    "    random.seed(0)\n",
    "    random_selection = random.sample(l, 10)\n",
    "    # save as csv\n",
    "    import csv\n",
    "    with open(f\"{scenario}/large_differences.csv\", \"w\") as f:\n",
    "        writer = csv.DictWriter(f, fieldnames= \\\n",
    "                                [\"participant_id\", \"advice_indices1\", \"advice_indices2\", \"generation_index1\", \"generation_index2\", \\\n",
    "                                \"score1\", \"score2\", \"evaluator\", \"flipped\", \"scenario_options_text_1\", \"scenario_options_text_2\", \\\n",
    "                                \"scores\", \"reasoning_text_1\", \"reasoning_text_2\", \"reasoning_text_3\", \"reasoning_text_4\", \"reasoning_text_5\", \"reasoning_text_6\"])\n",
    "        writer.writeheader()\n",
    "        for d in random_selection:\n",
    "            writer.writerow(d)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
