{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import math\n",
    "scenario = 'plane_crash'\n",
    "shortened_name = 'planepro'\n",
    "path = f'{scenario}/processed_data_eval{shortened_name}.csv'\n",
    "df = pd.read_csv(path)\n",
    "# df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72 72\n"
     ]
    }
   ],
   "source": [
    "# logistics - checking that generations_noindex and generations are the same except for index\n",
    "\n",
    "generations_noindex_path = f'{scenario}/generations_noindex.csv'\n",
    "generations_noindex_df = pd.read_csv(generations_noindex_path)\n",
    "\n",
    "generations_path = f'{scenario}/generations.csv'\n",
    "generations_df = pd.read_csv(generations_path)\n",
    "\n",
    "print(len(generations_df), len(generations_noindex_df))\n",
    "\n",
    "for i in range(len(generations_noindex_df)):\n",
    "    if generations_noindex_df['text'][i] != generations_df['text'][i]:\n",
    "        print(generations_noindex_df['advice_type'][i], generations_noindex_df['advice_indices'][i], generations_df['advice_type'][i], generations_df['advice_indices'][i])\n",
    "        print(generations_noindex_df['text'][i], \"\\n\\n\", generations_df['text'][i])\n",
    "        # print(i)\n",
    "        break\n",
    "\n",
    "# so index [] was removed from unorthodox_advice but it was still there for the human experiment\n",
    "# it is now in baseline_irrelevant_advice instead\n",
    "# moved back and regenerated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count: 480\n",
      "1440\n"
     ]
    }
   ],
   "source": [
    "# fill in generation_index column\n",
    "# check that generations_df text is unique\n",
    "generations_path = f'{scenario}/generations.csv'\n",
    "generations_df = pd.read_csv(generations_path)\n",
    "\n",
    "assert len(generations_df['text'].unique()) == len(generations_df)\n",
    "count = 0\n",
    "for index, row in df.iterrows():\n",
    "    text = row['text']\n",
    "    if not math.isnan(df.at[index, 'generation_index']): # sanity check for the baselines that originally had the generation index\n",
    "        count += 1\n",
    "    flag = False\n",
    "    for _, gen_row in generations_df.iterrows():\n",
    "        if gen_row['text'] ==  text:\n",
    "            # is nan?\n",
    "            if not math.isnan(df.at[index, 'generation_index']):\n",
    "                assert(df.at[index, 'generation_index'] == gen_row['generation_index'])\n",
    "            df.at[index, 'generation_index'] = gen_row['generation_index']\n",
    "            flag = True\n",
    "            break\n",
    "    if not flag: \n",
    "        print(f'no match for {index}')\n",
    "        print(text, end=\"\\n\")\n",
    "\n",
    "print(f'count: {count}')\n",
    "print(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform column advice_indices to string\n",
    "df['participant_strategy'] = df['participant_strategy'].apply(lambda x: str(x))\n",
    "df['advice_indices'] = df['advice_indices'].apply(lambda x: str(x))\n",
    "# tranforms all nan values\n",
    "\n",
    "na_locations = df.isna()\n",
    "row_indices, col_indices = np.where(na_locations)\n",
    "for row, col in zip(row_indices, col_indices):\n",
    "    print(f\"NaN value at row {row}, column {df.columns[col]}\")\n",
    "    # print(df.iloc[row], \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NANs CORRESPOND TO EMPTY STRING ADVICE INDICES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that origin matches text exactly\n",
    "# origin_id are 13-105, inclusive\n",
    "\n",
    "dict = {}\n",
    "for row in df.itertuples():\n",
    "    if row.origin_id not in dict:\n",
    "        dict[row.origin_id] = row.text\n",
    "    else:\n",
    "        if dict[row.origin_id] != row.text:\n",
    "            print(\"ERROR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of origin_ids with less than 20 ratings: 0 out of 72\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 5384 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check the number of ratings per origin_id\n",
    "non20_count = 0\n",
    "counts = []\n",
    "for id_count in df.groupby('origin_id')[\"id\"].count():\n",
    "    if id_count != 20:\n",
    "        non20_count += 1\n",
    "    counts += [id_count]\n",
    "plt.hist(counts, bins=9)\n",
    "\n",
    "print(f\"Number of origin_ids with less than 20 ratings: {non20_count} out of {len(df.groupby('origin_id'))}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "START OF ANALYSIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "participant_id: 1\n",
      "participant_strategy: I read the statements and focused on how each one would make me feel if it were true. The ones that mentioned the company least I rated the highest and the ones that mentioned all of the promotions and points about the company I rated the least.\n",
      "participant_id: 2\n",
      "participant_strategy: I specifically looked for the speeches which showed the most compassion, support, and professionalism for reporting of the incident. I did not like the speeches which mentioned sales/deals, or the charity football event because I thought these distracted from the fact the crash had occurred and they seemed disrespectful to be mentioned at this time. \n",
      "participant_id: 3\n",
      "participant_strategy: fine\n",
      "participant_id: 4\n",
      "participant_strategy: Gut instinct. Genuine regret and sympathy shown. No distraction tactics eg pets/football. Use of we/our to make the company feel more relatable to the public.\n",
      "participant_id: 5\n",
      "participant_strategy: I thought about how i would feel about the company if I was stood in a press conference listening to this.\n",
      "participant_id: 6\n",
      "participant_strategy: To score higher statements that were empathetic to the situation and ones that focused on the investigation and safety. To down score statements that included references to the speakers tie and in-flight meals as this was irrelevant and trivalised the situation.\n",
      "participant_id: 7\n",
      "participant_strategy: Whenever the business said something that sounded like an advertisement, they came off a lot worse.\n",
      "participant_id: 9\n",
      "participant_strategy: The speeches that drew focus away from the incident and onto other matters, such as the charity football match or summer deals, were instantly perceived as worsening the company reputation. Those that focussed sincerely on finding the cause of the issue and supporting the family were better for the reputation.\n",
      "participant_id: 10\n",
      "participant_strategy: anything that was mentioned to promote or introduce an offer automatically reduced the companies image\n",
      "participant_id: 11\n",
      "participant_strategy: to think about how the familys would feel if that statement was released\n",
      "participant_id: 12\n",
      "participant_strategy: I wanted the whole paragraph to be about the sorrow for the families who lost loved ones and the seriousness of the investigation. Anything else that made it sound flippant earned a very low score from me\n",
      "participant_id: 13\n",
      "participant_strategy: I feel like any mention of self promotion was wildly inappropriate so if I saw that I marked down. Some contained jokes which was odd.\n",
      "participant_id: 14\n",
      "participant_strategy: my strategy was to downgrade any speech which advertised any product from this airline. They should only be talking about the tragedy\n",
      "participant_id: 15\n",
      "participant_strategy: a company that makes an apology without trying to sell more products seems like a more genuine response to a tragedy.\n",
      "participant_id: 16\n",
      "participant_strategy: I took into account how sincere the apology seemed to be and relevant information provided on ongoing investigation. The mention of meal deals and exotic pet policy were not relevant and were in poor taste, I thought.\n",
      "participant_id: 17\n",
      "participant_strategy: A lot of the lower rated ones contained highly technical terminology, often used to confuse the listener or distranct them from the main point, or used the incident to further advertise their company through things that have gone correctly in the past/things they are currently doing such as charity work. Using this kind of speach is intended to use the publicity from the tragic incident to further grow the company.\n",
      "\n",
      "The higher rated ones came across compassionatly and with regret for the incident, they showed that they were deeply saddened by the event and that the people on the flight were not just pay checks but ordinary people that should be remembered and mourned. They appeared to be commited in attempting to resolve the issue to ensure this does not happen again, not because the reputation is at risk, but because they never want another soul to be harmed through their practices.\n",
      "participant_id: 18\n",
      "participant_strategy: My strategy was to look for how honest, trustworthy and empathetic the speaker was about the incident in his speech. \n",
      "participant_id: 19\n",
      "participant_strategy: How appropriate the choice of words were and the overall tone.\n",
      "participant_id: 20\n",
      "participant_strategy: Read and go off my first impression - not think about it too much\n",
      "participant_id: 21\n",
      "participant_strategy: I just read every statement carefully and gave my honest answer!\n",
      "participant_id: 22\n",
      "participant_strategy: reading it and thinking about how it made me feel\n",
      "participant_id: 23\n",
      "participant_strategy: It's hard to believe that some of those statements would ever be considered to be acceptable.  At the time of such an incident everything should be based on the people affected by it, and making sure it never happens again, not trying to advertise.\n",
      "participant_id: 25\n",
      "participant_strategy: answer honestly\n",
      "participant_id: 26\n",
      "participant_strategy: Seeing how sincere the message was and determining how I expect people would take the message.\n",
      "participant_id: 28\n",
      "participant_strategy: I evaluated the responses based on how appropriate I felt they were for the situaton\n",
      "participant_id: 30\n",
      "participant_strategy: Ultimately I would struggle to have a positive view of any airline folloiwng a crash so my baseline was low - any message that talked about sales of other products were automatically marked low as this was inappropriate to mention. \n",
      "participant_id: 31\n",
      "participant_strategy: I would give a lower rating every time the speech mentioned anything outside of the event, like in-flight meals or pets, advertising has no place in this type of speech.\n",
      "participant_id: 32\n",
      "participant_strategy: To put myself in the place of a customer and see how their brief affected me and if I was sympathetic.\n",
      "participant_id: 33\n",
      "participant_strategy: How formal the speech was, whether they tried to sell their airline or not. If they showed concern and condolences for those lost. Also what they planned to do.\n",
      "participant_id: 34\n",
      "participant_strategy: I especially focussed on how much the airline placed importance on safety and investigation and how much it pushed its business opportunities.\n",
      "participant_id: 35\n",
      "participant_strategy: I decided based on the overall tone of the message and how it made me feel.\n",
      "participant_id: 36\n",
      "participant_strategy: I looked to see how sincere the words were. Anything that had a free offer or talked about promotions was immediately given a 1\n",
      "participant_id: 37\n",
      "participant_strategy: To look for phrases which would sound un-sympathetic and therefore make people further dislike the company. Similarly, look for phrases which show empathy and self correction for the company and would make people appreciate their apology.\n",
      "participant_id: 38\n",
      "participant_strategy: Reading the paragraph and putting myself in the shoes of the people who loved ones\n",
      "participant_id: 39\n",
      "participant_strategy: My strategy was to review the seriousness of the language used, the technical quality, and the focus on addressing the concerns of members of public directly affected by the incident, and the wider market of flight customers. I took a highly negative view of any language possibly flippant, or unnecessary to the time available to the spokesperson.\n",
      "participant_id: 40\n",
      "participant_strategy: Anything that sounded like an advert for the airline, (football,exotic pets) I immediately marked down. If the spokesman sounded too glib, that was also marked down.\n",
      "After an incident such as this it would be hard for anything the spokesman could say that would realistically improve the airine's reputation. So no higher than a 7\n",
      "participant_id: 41\n",
      "participant_strategy: If they talked about promotions or deals then immediately the rating would lower because the hypothetical company is using this awful event as a business and marketing opportunity and detracting from the pain the family members would be in. The statements that sounded the most heartfelt were given higher ratings.\n",
      "participant_id: 43\n",
      "participant_strategy: I focused on how much of the tragedy the airline wrote about, and how much they focused on other things.\n",
      "participant_id: 44\n",
      "participant_strategy: I didn't have a strategy; I just rated according to the substance of each message and what impression it gave.\n",
      "participant_id: 45\n",
      "participant_strategy: to read everything fully\n",
      "participant_id: 46\n",
      "participant_strategy: I scored them how I personally would view the company if they made that statement, taking into account what had happened, and what was said.\n",
      "participant_id: 47\n",
      "participant_strategy: I didn't like the comments about matches and pets but more just the understanding words\n",
      "participant_id: 48\n",
      "participant_strategy: The more heartfelt, the more I think people will think well of the company.\n",
      "participant_id: 49\n",
      "participant_strategy: The study was easy to follow. It was a surprise how a slight change in the statement made a big difference to my score. My strategy was to use the first statement as my baseline and the others would be better or worse than that.\n",
      "participant_id: 50\n",
      "participant_strategy: Anything mentioned that wasn't directly related to the crash would be a big negative.  Sticking to the incident itself would be a slight positive\n",
      "participant_id: 51\n",
      "participant_strategy: Interesting task. No strategy, just preferred the more heartfelt and sincere statements.\n",
      "participant_id: 52\n",
      "participant_strategy: I read through each part, scanned through to determine the tone of voice of the response. Anything too light-hearted was immediately low scoring, anything that was sombre, investigative, showing sorrow and heart for the victims scored more. \n",
      "participant_id: 53\n",
      "participant_strategy: I used how I felt about each statement to determine how the public would feel \n",
      "participant_id: 54\n",
      "participant_strategy: Reading everything thoroughly and seeing how i would feel hearing each statement after that event occured\n",
      "participant_id: 55\n",
      "participant_strategy: no strategy, just reading though the statements and trying to image how i would feel if they were my family members lost\n",
      "participant_id: 56\n",
      "participant_strategy: My stratagy was to read it all word for word incase it was an attention check ! Anything which mentioned football or a lighter note was a no no\n",
      "participant_id: 57\n",
      "participant_strategy: I'm not sure I can describe a strategy - I looked for a statement of respect, reflecting the gravity of the event. It became difficult after a while to distinguish between them - other than the ridiculous ones that mentioned charity football matches and pet-friendly inclusion!\n",
      "participant_id: 58\n",
      "participant_strategy: I was difficult at first to rate the first one or two 'responses' as I didn't know how the others would vary, if at all.  I tried to imagine the scenario as if it were a real event and the spokesperson was on the evening news and how I would feel about their statement and its sincerity.\n",
      "participant_id: 59\n",
      "participant_strategy: I based my ratings entirely on the text in the response\n",
      "participant_id: 60\n",
      "participant_strategy: my strategy was to give a higher score to the responses that expressed genuine sympathy towards the lost passengers\n",
      "participant_id: 61\n",
      "participant_strategy: To read through each paragraph and decide on how the wording would make me feel viewing the news and the airline spokesman.\n",
      "participant_id: 62\n",
      "participant_strategy: Was to compare the statements and consider what was being said. How they were supporting the families of those lives that were lost and how they were assisting officials with ongoing investigations\n",
      "participant_id: 63\n",
      "participant_strategy: I wanted to see the airline deliver straight facts about what happened on their airline and their sympathies towards families affected. Anything pertaining to the airline and its charity work I felt would be distasteful and therefore irrelevant and to myself would affect how I viewed their company\n",
      "participant_id: 64\n",
      "participant_strategy: I read the texts and based my answers on how sincere they were\n",
      "participant_id: 65\n",
      "participant_strategy: It had to fit the timeslot, make no excuses, have no diversions, and not make it a marketing exorcise.\n",
      "participant_id: 66\n",
      "participant_strategy: Evaluated the sympathy and genuineness of each message and whether they tried to use the opportunity to promote the company instead\n",
      "participant_id: 67\n",
      "participant_strategy: It was based on how I felt if I watched the conference.\n",
      "participant_id: 68\n",
      "participant_strategy: I considered the empathy, understanding and seriousness of the responses.\n",
      "participant_id: 69\n",
      "participant_strategy: i just read and scored based on response\n",
      "participant_id: 70\n",
      "participant_strategy: Showing empathy or lack of in the statement. Adding points to promote the company in light of tragic news has a negative effect on the message.\n",
      "participant_id: 71\n",
      "participant_strategy: Any statement mentioning anything other than the tragedy would get a lower score\n",
      "participant_id: 72\n",
      "participant_strategy: I was trying to analyse the precise words in the question - how did each one change from previous questions? A more empathetic response, or enough reassurance and detail meant I gave the question a higher score.\n",
      "participant_id: 73\n",
      "participant_strategy: Whenever the weather or any kinds of airline improvements were mentioned, I estimated it to cause more damage than the other statements, as all people want to hear is an apology and a commitment to investigate the matter, not some silly thing about weather or pet-friendly travel.\n",
      "participant_id: 74\n",
      "participant_strategy: Not really a strategy, I just read the statement and tried to imagine public opinion after it\n",
      "participant_id: 75\n",
      "participant_strategy: I thought it was inappropriate to mention special offers during the press bulletin, and I marked down the text containing those.\n",
      "participant_id: 76\n",
      "participant_strategy: My strategy was to single out the non sincere comments.\n",
      "participant_id: 77\n",
      "participant_strategy: I didn't approve when subjects  other  than  the   deaths of passengers and  safety of future  flights  were  introduced.  E.G.  the mention  of pet  services  and in flight meals  were not  appropriate.\n",
      "participant_id: 78\n",
      "participant_strategy: I just read the paragraph. I tried to think if I was watching the news and saw this would i feel it was genuine concern.\n",
      "participant_id: 79\n",
      "participant_strategy: Partly based on gut reaction and partly on whether anything seemed jarring, unsympathetic, irrelevant, or attempting to protect sales, was a contributor to a lower perception of the company's reputation.\n",
      "participant_id: 81\n",
      "participant_strategy: to feel their sympathy, was it genuine? or were they just trying to keep selling more plane tickets by mentioning things like cheap infight meals which i felt was disrespectful in this context\n",
      "participant_id: 82\n",
      "participant_strategy: No strategy really. I just read them and went for what i believed. The bloke talking about his new Tie colour before acknowledging the accident, for example, got an immediate 0 on the scale.\n"
     ]
    }
   ],
   "source": [
    "# READ THROUGH PARTICIPANT STRATEGIES\n",
    "\n",
    "grouped = df.groupby('participant_id')\n",
    "\n",
    "# iterate through and print out the participant strategies\n",
    "for name, group in grouped:\n",
    "    print(f\"participant_id: {name}\")\n",
    "    print(f\"participant_strategy: {group['participant_strategy'].iloc[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 422,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9595903920335583 0.00549564054273097\n"
     ]
    }
   ],
   "source": [
    "# COMPUTE INTER RATER RELIABILITY\n",
    "\n",
    "results = []\n",
    "\n",
    "for i in range(10):\n",
    "    group1_means = []\n",
    "    group2_means = []\n",
    "\n",
    "    grouped_by_origin_id = df.groupby('origin_id') # grouped based on question\n",
    "    # loop across questions\n",
    "    for name, group in grouped_by_origin_id:\n",
    "        # randomly split participants in half\n",
    "        df_tmp = df[df['origin_id'] == name]\n",
    "        unique_participant_ids = df_tmp['participant_id'].unique()\n",
    "        np.random.shuffle(unique_participant_ids)\n",
    "        if len(unique_participant_ids) % 2 == 1:\n",
    "            unique_participant_ids = unique_participant_ids[:-1]\n",
    "\n",
    "        if len(unique_participant_ids) < 2:\n",
    "            continue\n",
    "        \n",
    "        split_point = int(len(unique_participant_ids) / 2)\n",
    "        first_half = unique_participant_ids[:split_point]\n",
    "        second_half = unique_participant_ids[split_point:]\n",
    "        \n",
    "        group1 = df_tmp[df_tmp['participant_id'].isin(first_half)]\n",
    "        group2 = df_tmp[df_tmp['participant_id'].isin(second_half)]\n",
    "\n",
    "        group1_means += [group1['answer'].mean()]\n",
    "        group2_means += [group2['answer'].mean()]\n",
    "        \n",
    "    # compute inter rater reliability\n",
    "    # results += [np.corrcoef(group1_means, group2_means)[0, 1]]\n",
    "\n",
    "    # apply spearman-brown correction\n",
    "    results += [2 * np.corrcoef(group1_means, group2_means)[0, 1] / (1 + np.corrcoef(group1_means, group2_means)[0, 1])]\n",
    "\n",
    "print(np.mean(results), np.std(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 3.,  1.,  2., 15.,  7.,  9., 14., 12.,  6.,  6.,  1.]),\n",
       " array([2.1       , 2.50909091, 2.91818182, 3.32727273, 3.73636364,\n",
       "        4.14545455, 4.55454545, 4.96363636, 5.37272727, 5.78181818,\n",
       "        6.19090909, 6.6       ]),\n",
       " <BarContainer object of 11 artists>)"
      ]
     },
     "execution_count": 366,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4484 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF AVERAGE SCORES PER PARTICIPANT\n",
    "\n",
    "mean_scores = df.groupby('participant_id')['answer'].mean()\n",
    "plt.hist(mean_scores, bins=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n",
      "[5, 6, 6, 7, 5, 0, 2, 0, 1, 0, 7, 2, 0, 6, 1, 1, 5, 0, 2, 5]\n"
     ]
    }
   ],
   "source": [
    "# TRY TO CLEAN THE DATA\n",
    "\n",
    "mean_scores = df.groupby('participant_id')['answer'].mean()\n",
    "for id in df['participant_id'].unique():\n",
    "    if mean_scores[id] < 3.5:\n",
    "        print(id)\n",
    "        # print in a list instead\n",
    "        print(list(df[df['participant_id'] == id]['answer']))\n",
    "\n",
    "for id in df['participant_id'].unique():\n",
    "    if mean_scores[id] > 8.5:\n",
    "        print(id)\n",
    "        # print in a list instead\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26\n",
      "[4, 7, 6, 4, 6, 7, 5, 4, 4, 6, 2, 8, 4, 6, 3, 4, 7, 7, 7]\n",
      "high variances\n",
      "33\n",
      "[6, 7, 0, 5, 8, 6, 5, 1, 7, 0, 1, 8, 6, 0, 1, 3, 1, 9, 0, 1]\n",
      "39\n",
      "[10, 7, 0, 10, 10, 10, 0, 10, 10, 0, 0, 10, 7, 8, 0, 0, 10, 0, 10, 10]\n",
      "42\n",
      "[5, 6, 6, 7, 5, 0, 2, 0, 1, 0, 7, 2, 0, 6, 1, 1, 5, 0, 2, 5]\n",
      "41\n",
      "[7, 2, 9, 8, 3, 9, 9, 8, 1, 9, 8, 2, 9, 3, 2, 3, 8, 8, 8, 9]\n",
      "61\n",
      "[0, 7, 0, 6, 4, 0, 5, 1, 0, 2, 6, 9, 10, 8, 1, 4, 3, 1, 3, 5]\n",
      "68\n",
      "[0, 1, 8, 0, 2, 10, 8, 9, 10, 0, 0, 8, 9, 10, 1, 9, 10, 2, 9, 7]\n",
      "80\n",
      "[7, 0, 10, 8, 10, 0, 0, 10, 10, 0, 10, 10, 6, 8, 7, 0, 8, 8, 10, 10]\n",
      "88\n",
      "[7, 8, 3, 9, 10, 8, 2, 9, 8, 2, 8, 7, 7, 8, 9, 2, 10, 1, 3, 8]\n",
      "90\n",
      "[7, 8, 7, 8, 2, 1, 1, 10, 9, 2, 3, 7, 8, 7, 2, 8, 3, 2, 1, 7]\n",
      "109\n",
      "[8, 6, 5, 3, 8, 6, 2, 7, 0, 7, 8, 2, 9, 6, 5, 7, 0, 0, 7, 0]\n"
     ]
    }
   ],
   "source": [
    "# MORE DATA CLEANING\n",
    "\n",
    "# LESS THAN 20\n",
    "scores = df.groupby('participant_id')['answer']\n",
    "for id in df['participant_id'].unique():\n",
    "    if len(list(scores.get_group(id))) < 20:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))\n",
    "\n",
    "# CHECK FOR ABNORMALLY HIGH VARIANCE\n",
    "print(\"high variances\")\n",
    "variances = df.groupby('participant_id')['answer'].var()\n",
    "for id in df['participant_id'].unique():\n",
    "    if variances[id] > 7:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CHECK FOR ABNORMALLY LOW VARIANCE\n",
    "\n",
    "variances = df.groupby('participant_id')['answer'].var()\n",
    "for id in df['participant_id'].unique():\n",
    "    if variances[id] < 0.6:\n",
    "        print(id)\n",
    "        print(list(df[df['participant_id'] == id]['answer']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55\n",
      "[5, 6, 6, 3, 6, 0, 5, 6, 6, 6, 6, 6, 6, 0, 5, 6, 0, 0, 0, 6]\n",
      "73\n",
      "[3, 4, 0, 4, 4, 4, 0, 1, 4, 4, 3, 4]\n",
      "78\n",
      "[6, 7, 6, 7, 7, 4, 6, 6, 6, 4, 6, 3, 6, 6, 8, 6, 6]\n"
     ]
    }
   ],
   "source": [
    "# CHECK FOR REPETITIONS\n",
    "\n",
    "for id in df['participant_id'].unique():\n",
    "    answer = list(df[df['participant_id'] == id]['answer'])\n",
    "    # if the answer contains more than half of the same value\n",
    "    for i in range(len(answer)//2 + 1):\n",
    "        if answer.count(answer[i]) > len(answer) / 2:\n",
    "            print(id)\n",
    "            print(answer)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "# REMOVE PARTICIPANTS\n",
    "\n",
    "# df = df[df['participant_id'] != 21]\n",
    "# df = df[df['participant_id'] != 23]\n",
    "# df = df[df['participant_id'] != 51]\n",
    "# df = df[df['participant_id'] != 60]\n",
    "# df = df[df['participant_id'] != 106]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([186.,   0.,  91.,   0., 105.,   0.,  95.,   0., 148.,   0., 206.,\n",
       "          0., 255.,   0., 173.,   0., 113.,   0.,  48.,   0.,  20.]),\n",
       " array([ 0.        ,  0.47619048,  0.95238095,  1.42857143,  1.9047619 ,\n",
       "         2.38095238,  2.85714286,  3.33333333,  3.80952381,  4.28571429,\n",
       "         4.76190476,  5.23809524,  5.71428571,  6.19047619,  6.66666667,\n",
       "         7.14285714,  7.61904762,  8.0952381 ,  8.57142857,  9.04761905,\n",
       "         9.52380952, 10.        ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 368,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4924 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF SCORES\n",
    "\n",
    "plt.hist(df['answer'], bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([4., 2., 5., 5., 2., 4., 2., 2., 0., 1., 1., 3., 5., 9., 8., 8., 8.,\n",
       "        2., 0., 1.]),\n",
       " array([0.65 , 1.015, 1.38 , 1.745, 2.11 , 2.475, 2.84 , 3.205, 3.57 ,\n",
       "        3.935, 4.3  , 4.665, 5.03 , 5.395, 5.76 , 6.125, 6.49 , 6.855,\n",
       "        7.22 , 7.585, 7.95 ]),\n",
       " <BarContainer object of 20 artists>)"
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4172 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# HISTOGRAM OF AVERAGE ANSWER SCORES FOR EACH GENERATION\n",
    "\n",
    "mean_scores = df.groupby('text')['answer'].mean()\n",
    "plt.hist(mean_scores, bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NORMALIZE PARTICIPANT SCORES FROM 0-1\n",
    "\n",
    "grouped = df.groupby('participant_id')\n",
    "df['answer'] = grouped['answer'].apply(lambda x: (x - x.min()) / (x.max() - x.min()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.503472222222222\n"
     ]
    }
   ],
   "source": [
    "print(df['answer'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.8 unorthodox_advice 0_3 1.0 \"I want to express our deepest condolences and empathy to the families of the two passengers we tragically lost today. We are profoundly touched by this and they are in our thoughts. Human lives are our utmost concern and we pledge to do absolutely everything to unravel the causes of the incident. We stand united in grief and commitment during this difficult time.\"\n",
      "4.9 unorthodox_advice 2 0.0 \"Firstly, our hearts go out to the families affected by today's tragic incident. Safety is, and always will be, our paramount concern. While investigations are still underway, let me assure everyone that this is a highly unusual event in our long history of safe operations. We are cooperating fully with investigators and are committed to take any additional steps necessary to further enhance the safety of our flights. Our record speaks to our dedication — a dedication we will uphold, no matter what.\"\n",
      "5.15 unorthodox_advice 1_2 2.0 \"We deeply regret this tragic incident and our hearts go out to the families affected by this unfortunate occurrence. We want to assure everyone that a thorough investigation is already underway, and we are fully committed to understanding and swiftly addressing the cause of this event. I want to underscore that the safety of our passengers is our number one priority. This incidence is an isolated case in our long history of secure and reliable air travel. We have always been lauded for our commitment to safety, and we stand firm in our dedication to continuously improving on our safety standards. Thank you.\"\n",
      "5.2 unorthodox_advice 2_3 0.0 Ladies and Gentlemen, we are deeply saddened by the tragic event involving one of our flights today, and our hearts are heavy with grief. Our deepest condolences go to the families of the two passengers we lost. This is an isolated incident - safety has been, and will always be, our utmost priority. Our airline has a long-standing record of commitment to safety and we pledge to thoroughly investigate this incident to prevent anything like this from happening again. Thank you.\n",
      "5.3 unorthodox_advice 0_2 0.0 \"First and foremost, our deepest sympathies go out to the families who lost their loved ones in this shocking incident. Human life is our paramount concern and we share in their sorrow deeply. This incident is indeed tragic and an isolated one. Our company has a longstanding reputation for safety with decades of reliable service. Please be assured that we're committed to unraveling the cause of this accident and to further improve our safety measures. Thank you.\"\n",
      "5.35 unorthodox_advice 0_2 1.0 \"Good evening. We are deeply saddened by today's tragic event and our hearts go out to those affected. Our airline has always made safety our highest priority, and we have an exceptional record in this regard. Let us assure you this is an isolated incident. We are working in cooperation with the authorities to uncover the causes and ensure it does not happen again. Meanwhile, our thoughts are with the victims' families during this extremely difficult time.\"\n",
      "5.5 unorthodox_advice 2_3 2.0 \"Ladies and Gentlemen, I stand before you with a heavy heart today, deeply saddened by the tragic occurrence. First and foremost, our thoughts and prayers go out to the families affected. We are committed to providing them with support during this incredibly difficult time. I must reassure you all, this is an isolated event. Our company has a lengthy, praiseworthy record of safety and we are steadfast in our commitment to maintaining this. We are vigorously investigating the causes of this incident, and are committed to learning from it for a safer future. Thank you for your understanding in this challenging time.\"\n",
      "5.75 unorthodox_advice 1_2 0.0 \"Good evening, we deeply regret the unfortunate event that took place today and our thoughts are with the families of the two passengers we lost. An in-depth investigation is currently underway to pinpoint and address the cause promptly. We want to assure everyone that the safety of our passengers has always been - and continues to be - our number one priority. This is an isolated incident, not reflective of our airline's long-standing record of safety. We remain committed to the well-being of our passengers, staff, and the rigorous maintenance of our aircraft.\"\n",
      "5.75 unorthodox_advice 1_3 1.0 \"Ladies and gentlemen, in this deeply sad moment, we express our deepest condolences to the families of the two passengers who tragically lost their lives in the unfortunate incident today. As a company, we empathize with the affected families and understand the pain they're going through. It is with heavy hearts that we acknowledge this event and we want to assure everyone that a thorough investigation is already underway to ascertain the exact causes of the incident. We are absolutely committed to learning from this to prevent the occurrence of any similar events in the future.\"\n",
      "5.75 unorthodox_advice 0 0.0 \"We deeply regret this terrible incident that occurred today. At our airline, we hold human life in the highest regard. Our hearts go out to the families and loved ones of those we lost. Please know we are leaving no stone unturned to determine the cause and to ensure such an incident does not recur. This incident does not reflect our enduring commitment to the safety and care of our passengers, which remains our highest priority.\"\n",
      "5.85 unorthodox_advice 2 1.0 \"First and foremost, our hearts and prayers go out to the families affected by this unimaginable event. While investigations are still ongoing, we want to reassure the public that this is an isolated incident. Our airline has a long-standing record of safety and we are deeply committed to understanding what occurred. We have always, and will always put passenger safety as our utmost concern. Our primary goal is to ensure that our standards of safety and operational integrity are not only met but exceeded. Until we know more details, we are completely focused on supporting the events of today and contributing to the administration's investigation. Thank you.\"\n",
      "5.85 unorthodox_advice 0_2 2.0 \"Good evening. Our hearts are heavy tonight as we express our deepest condolences to the families affected by today's unfortunate incident. Their loss is deeply felt by our airline family too. We want to reassure everyone that this is a tragic, isolated event. Our company has a longstanding commitment to safety, with countless successful flights over many years. We remain committed and dedicated to ensuring the safety and confidence in our service. This incident will be thoroughly investigated and action will be taken to prevent such an occurrence from repeating. Thank you.\"\n",
      "5.9 unorthodox_advice 2_3 1.0 \"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "6.0 unorthodox_advice 0_1 1.0 Ladies and gentlemen, we are profoundly saddened by today's unfortunate incident involving one of our aircrafts. Our deepest sympathies and heartfelt condolences go out to the families and friends of the two individuals who tragically lost their lives. As a company, we place the highest value on human life, and the safety of our passengers is our absolute priority. At this very moment, comprehensive investigations are in full swing to ascertain the cause of this incident so we can prevent such a tragedy from recurring. We appreciate your understanding and patience in these challenging times.\n",
      "6.0 unorthodox_advice 1 1.0 \"We deeply regret the unfortunate incident involving one of our flights today. Our thoughts and condolences are with the families and loved ones of the two passengers we tragically lost. Understanding the cause of this incident is our topmost priority and we promise full transparency in conducting a rigorous investigation. Our commitment, as always, is to provide safe and reliable travel and we will work relentlessly to ensure such an incident doesn't reoccur. We appreciate your trust and patience during this difficult time.\"\n",
      "6.1 unorthodox_advice 1_3 2.0 Ladies and gentlemen, we deeply regret to confirm the tragic incident involving flight XY123. We give our sincere and heartfelt condolences to the families of the two passengers who lost their lives. Our topmost priorities currently include full cooperation with authorities to swiftly uncover the causes of this accident and making all possible efforts to support those affected by this tragedy. Please, rest assured that we are committed to understanding and rectifying the problem. Our hearts go out to all those affected and we ask for your support and patience in these trying times.\n",
      "6.15 unorthodox_advice 1 0.0 \"On behalf of our entire company, we deeply regret the unfortunate incident that occurred today. We at [Company Name] extend our heartfelt condolences to the families of the two passengers who tragically lost their lives. As our first priority remains the safety and well-being of our passengers, we want to assure the public that we are conducting a full, rigorous investigation into the incident to identify the cause and promptly institute measures to prevent any similar occurrences in the future\".\n",
      "6.2 unorthodox_advice 1_2 1.0 Ladies and gentlemen of the press, and all those watching at home, we are profoundly saddened by the tragic incident that occurred today. Two lives were unfortunately lost, and our hearts go out to those affected. At this time, an extensive investigation is ongoing to ascertain the cause of the incident and to ensure that it will not happen again in the future. Please know, this is an isolated incident. The safety and security of our passengers have always been, and will continue to be, our utmost priority. We have a long and proud history of safety, and we are determined to learn from what happened and to continue our efforts towards continuous improvement. Thank you.\n",
      "6.25 unorthodox_advice 3 0.0 Our hearts go out to the families and friends who have lost loved ones in today's tragic event. The pain of such a loss is unimaginable, and no words can adequately express our sorrow. Knowing the trust you place in us to safeguard your life, we're committed to understanding the cause of this unprecedented incident. Our top priority is the safety of our passengers, and we're tirelessly working with authorities to investigate this matter thoroughly. We appreciate your patience during this challenging time.\n",
      "6.25 unorthodox_advice 0_1 2.0 We are deeply saddened to confirm that earlier today one of our aircrafts was involved in an unfortunate incident. The catastrophe had heartbreaking consequences and cost the lives of two passengers. We express our sincerest condolences and deepest sympathy to the families and loved ones of those onboard. Our airline holds the sanctity of human life above all else and we profoundly regret this incident. Rest assured, we are fully cooperating with authorities and a thorough investigation is ongoing to determine the cause and to ensure such an incident can never happen in the future.\n",
      "6.3 unorthodox_advice 3 1.0 Our deepest thoughts and condolences go out to the families affected by this tragic incident. The loss of any life is heartbreaking and we truly empathize with those who have lost loved ones. As an airline, our greatest commitment is to the safety and well-being of our passengers. We're as determined as ever to understand the cause of this tragedy. Rest assured, we are working tirelessly with the authorities and will leave no stone unturned in our effort to ensure such an event does not recur. Thank you.\n",
      "6.3 unorthodox_advice 1 2.0 \"Today is a profoundly difficult day for our airline company. We deeply regret to confirm that one of our flights experienced an unfortunate incident. Heartbreakingly, two lives were lost, to whom we offer our sincere condolences. We want to assure everyone that we are cooperating fully with the relevant authorities in an exhaustive investigation to identify the causes of this tragic incident. Our primary focus at this time is to ensure that this situation is handled with the utmost care and urgency, to prevent such a heartbreaking incident from occurring in the future. Thank you.\"\n",
      "6.35 unorthodox_advice 0_3 0.0 \"Good evening, everyone. First and foremost, our hearts go out to the families of the two individuals who tragically lost their lives in our recent flight incident. We deeply mourn this loss. At our company, each passenger is a valued member of our extended family and their safety is our utmost priority. We assure you, we are cooperating fully with the authorities and doing everything in our power to uncover the cause, to prevent incidents like this in the future. Our sympathies are with the affected families in these extremely difficult times.\"\n",
      "6.5 unorthodox_advice 2 2.0 \"We deeply regret this morning's tragic incident and our hearts go out to the families of the two passengers. We want to reassure everyone that this is an unprecedented and isolated event. Our airline has had a long-standing reputation for safety, one that we take immense pride in. Ensuring the safety of our passengers has always been, and will remain, our top priority. An in-depth investigation is underway to uncover the cause and we are committed to doing everything in our power to prevent such a heartbreaking event from ever happening again.\"\n",
      "6.55 unorthodox_advice 3 2.0 \"In this extremely challenging moment, our hearts are heavy as we express our deepest condolences to the families and loved ones of the passengers who unfortunately lost their lives in today's incident. We want them to know that they are not alone and we share in their profound grief. The safety of our passengers is always our top priority and we assure everyone that we are thoroughly investigating the incident to understand the cause behind it. Our entire team is absolutely dedicated to understanding what happened and preventing such incidents in the future.\"\n",
      "6.55 unorthodox_advice 0 2.0 \"Good evening, it is with profound sadness that I stand before you today. Our hearts and thoughts go out to the families of the two passengers whose lives were tragically lost in today's unfortunate incident. We hold their memory in our thoughts and prioritize their families' well-being. At our company, every life is irreplaceable and of highest regard. We commit to conducting a thorough, transparent investigation and vow to take appropriate steps moving forward. Thank you all.\"\n",
      "6.6 unorthodox_advice 0_1 0.0 Ladies and Gentlemen, we are deeply saddened by today's incident involving our aircraft and our hearts go out to those affected. We extend our sincerest condolences to the families of the two precious lives that were tragically lost. Our commitment to safety and the value we place on human life remain our highest priorities. We are working diligently with relevant authorities to investigate the cause of this unfortunate event, and vow to act swiftly and transparently to rectify any issues found. We thank you for your understanding and patience during this difficult time.\n",
      "6.8 unorthodox_advice 1_3 0.0 Ladies and Gentlemen, we acknowledge, with the deepest regret, the incident involving Flight XYZ. Our utmost sympathies and heartfelt condolences go out to the families and loved ones of the two passengers we tragically lost in this occurrence. At this moment, we’re working diligently with the appropriate authorities to swiftly conduct comprehensive investigations. We are committed to understanding the cause and will make necessary improvements to prevent such incidents. The safety of our passengers remains our utmost priority. Thank you.\n",
      "7.05 unorthodox_advice 0_3 2.0 \"Good evening. I want to express, from the depth of our hearts, how deeply saddened we are over the event. Our thoughts and prayers go out to those affected, especially to the two families who have lost their loved ones in this tragic incident. We hold the value of human life above all and we deeply regret this unforeseen occurrence. As a priority, we are focusing our resources in assisting the families affected during this difficult time and we are thoroughly investigating the incident to prevent this from recurring. Thank you for understanding.\"\n",
      "\n",
      "\n",
      "best from framework: 7.95 unorthodox_advice 0 1.0 Ladies and Gentlemen, we are absolutely devastated about the tragic incident that occurred today. We extend our heartfelt condolences and deepest sympathies to the families who lost their loved ones in this unfortunate event. The value of human life is paramount, and our thoughts are with all those affected by this incident. We're committed to investigating the event thoroughly, cooperating fully with aviation authorities to ensure this does not happen again. As information becomes available, we will promptly and transparently share updates. Our priority now is supporting the affected families during this incredibly difficult time. Thank you.\n",
      "30\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 4436 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SORTED AVERAGE SCORES FOR EACH GENERATION\n",
    "\n",
    "best_generation_advice = \"0\"\n",
    "best_generation_index = 1\n",
    "\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_neutral_advice']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_irrelevant_advice']\n",
    "\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "items = []\n",
    "for text, score in grouped.iteritems():\n",
    "    # get the advice_type for each text\n",
    "    advice_type = df_tmp[df_tmp['text'] == text]['advice_type'].unique()[0]\n",
    "    advice_indices = df_tmp[df_tmp['text'] == text]['advice_indices'].unique()[0]\n",
    "    generation = df_tmp[df_tmp['text'] == text]['generation_index'].unique()[0]\n",
    "    if advice_indices == best_generation_advice and generation == best_generation_index: \n",
    "        print('\\n\\nbest from framework: ', end = '')\n",
    "    print(score, advice_type, advice_indices, generation, text)\n",
    "    items.append(score)\n",
    "\n",
    "plt.hist(items, bins=20)\n",
    "print(len(items))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.1166666666666667 baseline_irrelevant_advice 0_1\n",
      "Remember to mention our special in-flight meal deals and sale on business class tickets for summer travel.\n",
      "Be sure to highlight our participation in the annual charity football match with other airline companies.\n",
      "1.2666666666666666 baseline_irrelevant_advice 0_2\n",
      "Remember to mention our special in-flight meal deals and sale on business class tickets for summer travel.\n",
      "Take the opportunity to introduce our new exotic pet-friendly travel policy.\n",
      "1.4666666666666666 baseline_neutral_advice 1_2\n",
      "\"Remember to start the conversation with some light hearted small talk!\"\n",
      "\"Wear a stand-out tie, it's sure to divert attention!\"\n",
      "2.033333333333333 baseline_irrelevant_advice 1_2\n",
      "Be sure to highlight our participation in the annual charity football match with other airline companies.\n",
      "Take the opportunity to introduce our new exotic pet-friendly travel policy.\n",
      "2.0833333333333335 baseline_irrelevant_advice 0\n",
      "Remember to mention our special in-flight meal deals and sale on business class tickets for summer travel.\n",
      "2.183333333333333 baseline_irrelevant_advice 2\n",
      "Take the opportunity to introduce our new exotic pet-friendly travel policy.\n",
      "2.7 baseline_neutral_advice 1\n",
      "\"Remember to start the conversation with some light hearted small talk!\"\n",
      "2.8333333333333335 baseline_irrelevant_advice 1\n",
      "Be sure to highlight our participation in the annual charity football match with other airline companies.\n",
      "3.4833333333333334 baseline_neutral_advice 2\n",
      "\"Wear a stand-out tie, it's sure to divert attention!\"\n",
      "4.25 baseline_neutral_advice 0_1\n",
      "\"Keep smiling, it projects positivity!\"\n",
      "\"Remember to start the conversation with some light hearted small talk!\"\n",
      "5.5 unorthodox_advice 0_2\n",
      "Express sincere empathy and condolences towards the victims and their families to show that your company holds human life in the highest regard.\n",
      "Reassure the public that this is an isolated event and emphasize on your airline's long-standing reputation for safety and commitment to continuous improvement.\n",
      "5.516666666666667 baseline_neutral_advice 0_2\n",
      "\"Keep smiling, it projects positivity!\"\n",
      "\"Wear a stand-out tie, it's sure to divert attention!\"\n",
      "5.533333333333333 unorthodox_advice 2_3\n",
      "Reassure the public that this is an isolated event and emphasize on your airline's long-standing reputation for safety and commitment to continuous improvement.\n",
      "\"Humanize the situation by expressing sincere condolences directly addressing the families affected before discussing the flight itself.\"\n",
      "5.7 unorthodox_advice 1_2\n",
      "Regretfully acknowledge the incident, ensuring that the necessary investigations are being conducted to understand and correct the issue promptly.\n",
      "Reassure the public that this is an isolated event and emphasize on your airline's long-standing reputation for safety and commitment to continuous improvement.\n",
      "5.75 unorthodox_advice 2\n",
      "Reassure the public that this is an isolated event and emphasize on your airline's long-standing reputation for safety and commitment to continuous improvement.\n",
      "5.983333333333333 baseline_ask_with_reasoning []\n",
      "baseline_ask_with_reasoning\n",
      "6.016666666666667 baseline_neutral_advice 0\n",
      "\"Keep smiling, it projects positivity!\"\n",
      "6.066666666666666 unorthodox_advice 0_3\n",
      "Express sincere empathy and condolences towards the victims and their families to show that your company holds human life in the highest regard.\n",
      "\"Humanize the situation by expressing sincere condolences directly addressing the families affected before discussing the flight itself.\"\n",
      "6.15 unorthodox_advice 1\n",
      "Regretfully acknowledge the incident, ensuring that the necessary investigations are being conducted to understand and correct the issue promptly.\n",
      "6.216666666666667 unorthodox_advice 1_3\n",
      "Regretfully acknowledge the incident, ensuring that the necessary investigations are being conducted to understand and correct the issue promptly.\n",
      "\"Humanize the situation by expressing sincere condolences directly addressing the families affected before discussing the flight itself.\"\n",
      "6.283333333333333 unorthodox_advice 0_1\n",
      "Express sincere empathy and condolences towards the victims and their families to show that your company holds human life in the highest regard.\n",
      "Regretfully acknowledge the incident, ensuring that the necessary investigations are being conducted to understand and correct the issue promptly.\n",
      "\n",
      "\n",
      "best from framework: 6.366666666666666 unorthodox_advice 3\n",
      "\"Humanize the situation by expressing sincere condolences directly addressing the families affected before discussing the flight itself.\"\n",
      "6.75 unorthodox_advice 0\n",
      "Express sincere empathy and condolences towards the victims and their families to show that your company holds human life in the highest regard.\n",
      "6.833333333333333 baseline_ask []\n",
      "baseline_ask\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([2., 1., 0., 3., 0., 1., 1., 0., 1., 0., 0., 1., 0., 0., 0., 0., 4.,\n",
       "        3., 4., 1., 2.]),\n",
       " array([1.11666667, 1.38888889, 1.66111111, 1.93333333, 2.20555556,\n",
       "        2.47777778, 2.75      , 3.02222222, 3.29444444, 3.56666667,\n",
       "        3.83888889, 4.11111111, 4.38333333, 4.65555556, 4.92777778,\n",
       "        5.2       , 5.47222222, 5.74444444, 6.01666667, 6.28888889,\n",
       "        6.56111111, 6.83333333]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 373,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5720 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE COMBINATION (but not generation)\n",
    "import json\n",
    "\n",
    "best_advice_index = \"3\"\n",
    "\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "scores = []\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        with open(f\"{scenario}/advice/normal_advice.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "        with open(f\"{scenario}/advice/unorthodox_advice.json\", \"r\") as f:\n",
    "            advice += json.load(f)\n",
    "    elif advice_type != \"baseline_ask_with_reasoning\" and advice_type != \"baseline_ask\":\n",
    "        with open(f\"{scenario}/advice/{advice_type}.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "    \n",
    "    if advice_type == \"unorthodox_advice\" and advice_indices == best_advice_index:\n",
    "        print('\\n\\nbest from framework: ', end = '')\n",
    "    print(score, advice_type, advice_indices)\n",
    "    for index in advice_indices.split(\"_\"):\n",
    "        if index == \"nan\":\n",
    "            continue\n",
    "        elif advice_type == \"baseline_ask_with_reasoning\" or advice_type == \"baseline_ask\":\n",
    "            print(advice_type)\n",
    "        else: \n",
    "            print(advice[int(index)])\n",
    "    scores += [score]\n",
    "\n",
    "plt.hist(scores, bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.9194444444444445 baseline_irrelevant_advice\n",
      "3.9055555555555554 baseline_neutral_advice\n",
      "5.983333333333333 baseline_ask_with_reasoning\n",
      "6.031666666666666 unorthodox_advice\n",
      "6.833333333333333 baseline_ask\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7feb85c9e550>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85c9e7f0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cc67f0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cc6a90>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cd3a60>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cd3d00>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85ce1d00>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85ce1fa0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cedfa0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cfb280>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7feb85c9ea90>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85c9ed30>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cc6d30>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cc6fd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cd3fa0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85ce1280>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85ced280>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85ced520>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cfb520>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cfb7c0>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7feb85c90160>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cc6550>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cd37f0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85ce1a60>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cedd00>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7feb85c9efd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cd32b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85ce1520>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85ced7c0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cfba60>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7feb85cc62b0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cd3550>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85ce17c0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85ceda60>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb85cfbd00>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 374,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7300 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE TYPE\n",
    "\n",
    "grouped = df.groupby('advice_type')['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "for advice_type, score in grouped.iteritems():\n",
    "    print(score, advice_type)\n",
    "\n",
    "\n",
    "# best_advice_from_framework = None\n",
    "# best_grouped_avg = df.groupby('text')['answer'].mean()[best_advice_from_framework]\n",
    "# print(f\"best advice from framework score: {best_grouped_avg}\")\n",
    "\n",
    "# best_generation = None\n",
    "\n",
    "\n",
    "# make a box and whisker plot of the scores for each advice type\n",
    "advice_types = [\"baseline_irrelevant_advice\", \"unorthodox_advice\", \"baseline_ask\", \"baseline_neutral_advice\", \"baseline_ask_with_reasoning\"]\n",
    "short_advice_types = [\"irrelevant\", \"our avg gen\", \"ask\", \"neutral\", \"reasoning\"]\n",
    "plt.boxplot([df[df['advice_type'] == advice_type]['answer'] for advice_type in advice_types], labels=short_advice_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.5 unorthodox_advice 0_2\n",
      "5.533333333333333 unorthodox_advice 2_3\n",
      "5.7 unorthodox_advice 1_2\n",
      "5.75 unorthodox_advice 2\n",
      "6.066666666666666 unorthodox_advice 0_3\n",
      "6.15 unorthodox_advice 1\n",
      "6.216666666666667 unorthodox_advice 1_3\n",
      "6.283333333333333 unorthodox_advice 0_1\n",
      "best advice from framework score:  6.366666666666666 unorthodox_advice 3\n",
      "6.75 unorthodox_advice 0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([2., 0., 0., 1., 1., 0., 0., 0., 0., 1., 1., 0., 1., 1., 1., 0., 0.,\n",
       "        0., 0., 0., 1.]),\n",
       " array([5.5       , 5.55952381, 5.61904762, 5.67857143, 5.73809524,\n",
       "        5.79761905, 5.85714286, 5.91666667, 5.97619048, 6.03571429,\n",
       "        6.0952381 , 6.1547619 , 6.21428571, 6.27380952, 6.33333333,\n",
       "        6.39285714, 6.45238095, 6.51190476, 6.57142857, 6.63095238,\n",
       "        6.69047619, 6.75      ]),\n",
       " <BarContainer object of 21 artists>)"
      ]
     },
     "execution_count": 375,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 6704 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR EACH ADVICE TYPE AND ADVICE INDEX\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_neutral_advice']\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] != 'baseline_irrelevant_advice']\n",
    "\n",
    "grouped = df_tmp.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "grouped.sort_values(inplace=True)\n",
    "scores = []\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    if advice_type == 'unorthodox_advice' and advice_indices == best_advice_index:\n",
    "        print(\"best advice from framework score: \", score, advice_type, advice_indices)\n",
    "    else: print(score, advice_type, advice_indices)\n",
    "    scores += [score]\n",
    "\n",
    "plt.hist(scores, bins=21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 1.,  0.,  1.,  1.,  4.,  6., 18., 14., 10.,  5.]),\n",
       " array([0. , 0.9, 1.8, 2.7, 3.6, 4.5, 5.4, 6.3, 7.2, 8.1, 9. ]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 376,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5420 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ANALYZING BEST ADVICE FROM FRAMEWORK\n",
    "\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer']\n",
    "best_advice_from_framework_scores = grouped.get_group(('unorthodox_advice', best_advice_index))\n",
    "plt.hist(best_advice_from_framework_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.488888888888889 Remember to mention our special in-flight meal deals and sale on business class tickets for summer travel.\n",
      "1.8277777777777777 Take the opportunity to introduce our new exotic pet-friendly travel policy.\n",
      "1.9944444444444445 Be sure to highlight our participation in the annual charity football match with other airline companies.\n",
      "2.8055555555555554 \"Remember to start the conversation with some light hearted small talk!\"\n",
      "3.488888888888889 \"Wear a stand-out tie, it's sure to divert attention!\"\n",
      "5.261111111111111 \"Keep smiling, it projects positivity!\"\n",
      "5.620833333333334 Reassure the public that this is an isolated event and emphasize on your airline's long-standing reputation for safety and commitment to continuous improvement.\n",
      "6.045833333333333 \"Humanize the situation by expressing sincere condolences directly addressing the families affected before discussing the flight itself.\"\n",
      "6.0875 Regretfully acknowledge the incident, ensuring that the necessary investigations are being conducted to understand and correct the issue promptly.\n",
      "6.15 Express sincere empathy and condolences towards the victims and their families to show that your company holds human life in the highest regard.\n"
     ]
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR ALL GENERATIONS CONTAINING A PARTICULAR PIECE OF ADVICE\n",
    "import json\n",
    "\n",
    "advice_texts = []\n",
    "advice_scores = []\n",
    "# for each advice type\n",
    "for advice_type in df['advice_type'].unique():\n",
    "    df_tmp = df[df['advice_type'] == advice_type]\n",
    "    \n",
    "    if advice_type == \"baseline_ask_with_reasoning\" or advice_type == \"baseline_ask\":\n",
    "        continue\n",
    "    \n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        with open(f\"{scenario}/advice/normal_advice.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "        with open(f\"{scenario}/advice/unorthodox_advice.json\", \"r\") as f:\n",
    "            advice += json.load(f)\n",
    "    else:\n",
    "        with open(f\"{scenario}/advice/{advice_type}.json\", \"r\") as f:\n",
    "            advice = json.load(f)\n",
    "\n",
    "    # for each set of indices containing advice 0\n",
    "    df_0 = df_tmp[df_tmp['advice_indices'].str.contains(\"0\")]\n",
    "    # print(advice_type, \"index 0\", df_0['answer'].mean())\n",
    "    # print(advice[0], \"\\n\")\n",
    "    advice_texts += [advice[0]]\n",
    "    advice_scores += [df_0['answer'].mean()]\n",
    "\n",
    "    df_1 = df_tmp[df_tmp['advice_indices'].str.contains(\"1\")]\n",
    "    # print(advice_type, \"index 1\", df_1['answer'].mean())\n",
    "    # print(advice[1], \"\\n\")\n",
    "    advice_texts += [advice[1]]\n",
    "    advice_scores += [df_1['answer'].mean()]\n",
    "\n",
    "    df_2 = df_tmp[df_tmp['advice_indices'].str.contains(\"2\")]\n",
    "    # print(advice_type, \"index 2\", df_2['answer'].mean())\n",
    "    # print(advice[2], \"\\n\")\n",
    "    advice_texts += [advice[2]]\n",
    "    advice_scores += [df_2['answer'].mean()]\n",
    "\n",
    "    if advice_type == \"unorthodox_advice\":\n",
    "        df_3 = df_tmp[df_tmp['advice_indices'].str.contains(\"3\")]\n",
    "        # print(advice_type, \"index 3\", df_3['answer'].mean())\n",
    "        # print(advice[3], \"\\n\")\n",
    "        advice_texts += [advice[3]]\n",
    "        advice_scores += [df_3['answer'].mean()]\n",
    "\n",
    "# argsort\n",
    "indices = np.argsort(advice_scores)\n",
    "for i in indices:\n",
    "    print(advice_scores[i], advice_texts[i])\n",
    "\n",
    "# note: each triplet is correlated. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = df.copy()\n",
    "grouped = df_copy.groupby('participant_id')\n",
    "df_copy['answer'] = grouped['answer'].apply(lambda x: (x - x.min()) / (x.max() - x.min()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 429,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.24930918022720294\n",
      "435\n",
      "0.26773104083512433\n",
      "435\n",
      "0.2583473647425104\n",
      "435\n",
      "0.26303974926409857\n",
      "435\n",
      "0.2536538868883711\n",
      "435\n",
      "0.25800163158062017\n",
      "435\n",
      "0.24426364965359437\n",
      "435\n",
      "0.24426364965359437\n",
      "435\n",
      "0.25817453844442795\n",
      "435\n",
      "0.24913428852813158\n",
      "435\n",
      "0.2545918979817676 0.007446383634967692\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "kappas = []\n",
    "\n",
    "for iteration in range(10):\n",
    "    matching = 0\n",
    "    total = 0\n",
    "    gpt_tie_count = 0\n",
    "    human_tie_count = 0\n",
    "\n",
    "    gpt_judgements = []\n",
    "    human_judgements = []\n",
    "\n",
    "    for i in range(MWU_matrix.shape[0]):\n",
    "\n",
    "        # get the responses for the ith advice set\n",
    "        filename = mapping[str(i)]\n",
    "        if filename == \".json\": continue\n",
    "        with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "            responses1 = json.load(f)\n",
    "        \n",
    "        for k in range(MWU_matrix.shape[2]):\n",
    "\n",
    "            # get the specific generation text, corresponding df, and score\n",
    "            response1 = responses1[k]\n",
    "            df_tmp = df[df['text'] == response1]\n",
    "            if len(df_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "            score1 = df_tmp['answer'].mean()\n",
    "\n",
    "            for j in range(i, MWU_matrix.shape[1]):\n",
    "\n",
    "                # get the responses for the jth advice set\n",
    "                filename = mapping[str(j)]\n",
    "                if filename == \".json\": continue\n",
    "                with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                    responses2 = json.load(f)\n",
    "\n",
    "                for l in range(MWU_matrix.shape[3]):\n",
    "\n",
    "                    # onlt need half of the MWU_matrix, self-compare is meaningless\n",
    "                    if i == j and k >= l:\n",
    "                        continue\n",
    "\n",
    "                    # get the specific generation text, corresponding df, and score\n",
    "                    response2 = responses2[l]\n",
    "                    df_tmp2 = df[df['text'] == response2]\n",
    "                    if len(df_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                    score2 = df_tmp2['answer'].mean()\n",
    "\n",
    "                    # if score1 - score2 < 0.5 and score2 - score1 < 0.5:\n",
    "                    #     continue\n",
    "\n",
    "                    result = MWU_matrix[i][j][k][l]\n",
    "                    if math.isclose(result, 0.5): \n",
    "                        # print(f\"TIE FROM GPT: {i}, {j}, {k}, {l}, {result}\")\n",
    "                        gpt_tie_count += 1\n",
    "                        if random.random() > 0.5: result = 1\n",
    "                        else: result = 0\n",
    "                    \n",
    "                    if math.isclose(score1, score2):\n",
    "                        # print(f\"TIE FROM HUMAN: {i}, {j}, {k}, {l}, {score1}, {score2}\")\n",
    "                        human_tie_count += 1\n",
    "                        df_copy_tmp = df_copy[df_copy['text'] == response1]\n",
    "                        df_copy_tmp2 = df_copy[df_copy['text'] == response2]\n",
    "                        score1 = df_copy_tmp['answer'].mean()\n",
    "                        score2 = df_copy_tmp2['answer'].mean()\n",
    "                        \n",
    "                        if math.isclose(score1, score2):\n",
    "                            print(\"STILL A TIE\")\n",
    "                            continue\n",
    "\n",
    "                    if score1 > score2:\n",
    "                        human_judgements.append(1)\n",
    "                    elif score1 < score2:\n",
    "                        human_judgements.append(0)\n",
    "                    else: \n",
    "                        print(\"tied?\")\n",
    "                    \n",
    "                    if result > 0.5:\n",
    "                        gpt_judgements.append(1)\n",
    "                    elif result < 0.5:\n",
    "                        gpt_judgements.append(0)\n",
    "                    else:\n",
    "                        print(\"tied?\")\n",
    "                    \n",
    "                    total += 1\n",
    "\n",
    "    kappa = cohen_kappa_score(human_judgements, gpt_judgements)\n",
    "    kappas.append(kappa)\n",
    "    print(kappa)\n",
    "    print(total)\n",
    "    # print(gpt_tie_count)\n",
    "    # print(human_tie_count)\n",
    "\n",
    "print(np.mean(kappas), np.std(kappas))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6275862068965518\n",
      "435\n",
      "yes 128\n",
      "also yes 131\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK\n",
    "\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "comparison_points = []\n",
    "yes_count = 0\n",
    "also_yes_count = 0\n",
    "for i in range(MWU_matrix.shape[0]):\n",
    "    filename = mapping[str(i)]\n",
    "    if filename == \".json\": continue\n",
    "    with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "        responses1 = json.load(f)\n",
    "    \n",
    "    for k in range(MWU_matrix.shape[2]):\n",
    "        # get the specific generation text\n",
    "        response1 = responses1[k]\n",
    "\n",
    "        df_tmp = df[df['text'] == response1]\n",
    "        if len(df_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "        score1 = df_tmp['answer'].mean()\n",
    "        # df_tmp_tmp = df_tmp #[df_tmp['participant_id'].isin(competent_participants) | df_tmp['participant_id'].isin(maybe_competent_participants)]\n",
    "        # score1 = df_tmp_tmp['answer'].mean()\n",
    "\n",
    "        for j in range(i, MWU_matrix.shape[1]):\n",
    "        # for j in range(MWU_matrix.shape[1]):\n",
    "            filename = mapping[str(j)]\n",
    "            if filename == \".json\": continue\n",
    "            with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                responses2 = json.load(f)\n",
    "\n",
    "            for l in range(MWU_matrix.shape[3]):\n",
    "                if i == j and k >= l:\n",
    "                    continue\n",
    "\n",
    "                response2 = responses2[l]\n",
    "\n",
    "                df_tmp2 = df[df['text'] == response2]\n",
    "                if len(df_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                score2 = df_tmp2['answer'].mean()\n",
    "                # df_tmp_tmp2 = df_tmp2 #[df_tmp2['participant_id'].isin(competent_participants) | df_tmp2['participant_id'].isin(maybe_competent_participants)]\n",
    "                # score2 = df_tmp_tmp2['answer'].mean()\n",
    "\n",
    "                result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                # if score1 - score2 < 0.5 and score2 - score1 < 0.5:# math.isclose(score1, score2):\n",
    "                #     print(f\"TIE FROM HUMAN: {i}, {j}, {k}, {l}, {score1}, {score2}\")\n",
    "                #     human_tie_count += 1\n",
    "                #     df_copy_tmp = df_copy[df_copy['text'] == response1]\n",
    "                #     df_copy_tmp2 = df_copy[df_copy['text'] == response2]\n",
    "                #     if len(df_copy_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "                #     if len(df_copy_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                #     score1 = df_copy_tmp['answer'].mean()\n",
    "                #     score2 = df_copy_tmp2['answer'].mean()\n",
    "                    \n",
    "                #     if math.isclose(score1, score2):\n",
    "                #         print(\"STILL A TIE\")\n",
    "                #         raise Exception(\"STILL A TIE\")\n",
    "\n",
    "                if score1 > score2 and result > 0.5:\n",
    "                    match = 1\n",
    "                    yes_count += 1\n",
    "                elif score1 < score2 and result < 0.5:\n",
    "                    match = 1\n",
    "                    also_yes_count += 1\n",
    "                elif math.isclose(score1, score2) and math.isclose(result, 0.5):\n",
    "                    match = 1\n",
    "                elif math.isclose(result, 0.5):\n",
    "                    match = 0.5\n",
    "                elif math.isclose(score1, score2):\n",
    "                    match = 0.5\n",
    "                else: \n",
    "                    match = 0\n",
    "                    # print(response1, \"\\n\\n\", response2)\n",
    "                    # print(f\"{i} {j} {k} {l} {result} {score1} {score2}\")\n",
    "                    # if random.random() > 0.7: raise Exception(\"?\")\n",
    "\n",
    "                matching += match\n",
    "                total += 1\n",
    "                comparison_points.append((score1, score2, result, match))\n",
    "\n",
    "\n",
    "print(matching / total)\n",
    "print(total)\n",
    "print(\"yes\", yes_count)\n",
    "print(\"also yes\", also_yes_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIE FROM HUMAN: 1, 6, 0, 0, 5.75, 5.75\n",
      "TIE FROM HUMAN: 1, 10, 2, 2, 6.55, 6.55\n",
      "TIE FROM HUMAN: 2, 5, 1, 1, 6.0, 6.0\n",
      "TIE FROM HUMAN: 2, 10, 2, 0, 6.25, 6.25\n",
      "TIE FROM HUMAN: 3, 8, 2, 1, 5.85, 5.85\n",
      "TIE FROM HUMAN: 5, 10, 2, 1, 6.3, 6.3\n",
      "TIE FROM HUMAN: 6, 7, 0, 1, 5.75, 5.75\n",
      "0.6298850574712643\n",
      "435\n"
     ]
    }
   ],
   "source": [
    "# AGREEMENT BETWEEN HUMAN AND FRAMEWORK FOR LARGE DIFFERENCE CASES\n",
    "\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# get all the response generations\n",
    "unorthodox_path = f\"{scenario}/candidates/unorthodox_advice/\"\n",
    "for path in os.listdir(f\"{scenario}/evaluators\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{path}\"):\n",
    "        evaluator = f\"{scenario}/evaluators/{path}\" # all mappings are the same\n",
    "        break\n",
    "\n",
    "with open(f\"{evaluator}/mappings.json\", \"r\") as f:\n",
    "    mapping = json.load(f)[0]\n",
    "\n",
    "matching = 0\n",
    "total = 0\n",
    "comparison_points = []\n",
    "look_at_points = []\n",
    "\n",
    "all_human_scores = []\n",
    "all_gpt_scores = []\n",
    "\n",
    "for i in range(MWU_matrix.shape[0]):\n",
    "    filename = mapping[str(i)]\n",
    "    if filename == \".json\": continue\n",
    "    with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "        responses1 = json.load(f)\n",
    "    \n",
    "    for k in range(MWU_matrix.shape[2]):\n",
    "        # get the specific generation text\n",
    "        response1 = responses1[k]\n",
    "\n",
    "        df_tmp = df[df['text'] == response1]\n",
    "        # if len(df_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "        df_tmp_tmp = df_tmp #[df_tmp['participant_id'].isin(competent_participants)]# | df_tmp['participant_id'].isin(maybe_competent_participants)]\n",
    "        if len(df_tmp_tmp) == 0:\n",
    "            continue\n",
    "        score1 = df_tmp_tmp['answer'].mean()\n",
    "\n",
    "        for j in range(i, MWU_matrix.shape[1]):\n",
    "            filename = mapping[str(j)]\n",
    "            if filename == \".json\": continue\n",
    "            with open(f\"{unorthodox_path}/{filename}\", \"r\") as f:\n",
    "                responses2 = json.load(f)\n",
    "\n",
    "            for l in range(MWU_matrix.shape[3]):\n",
    "                if i == j and k >= l:\n",
    "                    continue\n",
    "\n",
    "                response2 = responses2[l]\n",
    "\n",
    "                df_tmp2 = df[df['text'] == response2]\n",
    "                # if len(df_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                df_tmp_tmp2 = df_tmp2 #[df_tmp2['participant_id'].isin(competent_participants)]# | df_tmp2['participant_id'].isin(maybe_competent_participants)]\n",
    "                if len(df_tmp_tmp2) == 0:\n",
    "                    continue\n",
    "                \n",
    "                score2 = df_tmp_tmp2['answer'].mean()\n",
    "\n",
    "                result = MWU_matrix[i][j][k][l]\n",
    "\n",
    "                # if score1 - score2 < 0.5 and score2 - score1 < 0.5:\n",
    "                #     continue\n",
    "\n",
    "                if math.isclose(score1, score2):\n",
    "                    print(f\"TIE FROM HUMAN: {i}, {j}, {k}, {l}, {score1}, {score2}\")\n",
    "                    human_tie_count += 1\n",
    "                    df_copy_tmp = df_copy[df_copy['text'] == response1]\n",
    "                    df_copy_tmp2 = df_copy[df_copy['text'] == response2]\n",
    "                    if len(df_copy_tmp) == 0: raise Exception(f\"no match with response1: {response1}\")\n",
    "                    if len(df_copy_tmp2) == 0: raise Exception(f\"no match with response2: {response2}\")\n",
    "                    score1 = df_copy_tmp['answer'].mean()\n",
    "                    score2 = df_copy_tmp2['answer'].mean()\n",
    "                    \n",
    "                    if math.isclose(score1, score2):\n",
    "                        print(\"STILL A TIE\")\n",
    "                        raise Exception(\"STILL A TIE\")\n",
    "\n",
    "                if result > 0.5:\n",
    "                    all_gpt_scores.append(1)\n",
    "                elif result < 0.5:\n",
    "                    all_gpt_scores.append(0)\n",
    "                else:\n",
    "                    all_gpt_scores.append(0.5)\n",
    "                \n",
    "                if score1 > score2:\n",
    "                    all_human_scores.append(1)\n",
    "                elif score1 < score2:\n",
    "                    all_human_scores.append(0)\n",
    "                # all_human_scores.append(score1 - score2 + 0.5)\n",
    "\n",
    "                if score1 > score2 and result > 0.5:\n",
    "                    match = 1\n",
    "                elif score1 < score2 and result < 0.5:\n",
    "                    match = 1\n",
    "                elif math.isclose(score1, score2) and math.isclose(result, 0.5):\n",
    "                    match = 1\n",
    "                elif math.isclose(result, 0.5):\n",
    "                    match = 0.5\n",
    "                elif math.isclose(score1, score2):\n",
    "                    match = 0.5\n",
    "                else: \n",
    "                    match = 0\n",
    "                    # print(response1, \"\\n\\n\", response2)\n",
    "                    # print(f\"{i} {j} {k} {l} {result} {score1} {score2}\")\n",
    "                    # if random.random() > 0.7: raise Exception(\"?\")\n",
    "\n",
    "                matching += match\n",
    "                total += 1\n",
    "                comparison_points.append((score1, score2, result, match))\n",
    "                #look_at_points.append({response1, response2, score1, score2, result, match)\n",
    "                look_at_points.append({\"response1\": response1, \"response2\": response2, \"score1\": score1, \"score2\": score2, \"result\": result, \"match\": match, \"i\": i, \"j\": j, \"k\": k, \"l\": l, \"advice1\": mapping[str(i)], \"advice2\": mapping[str(j)]})\n",
    "\n",
    "\n",
    "print(matching / total)\n",
    "print(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import permutations\n",
    "\n",
    "def get_metric_score(rater1_scores, rater2_scores):\n",
    "    total = 0\n",
    "    length = 0\n",
    "    for score1 in rater1_scores: # gpt eval\n",
    "        for score2 in rater2_scores: # human eval\n",
    "            if score1 > 0.5 and score2 > 0.5: total += 1\n",
    "            elif score1 < 0.5 and score2 < 0.5: total += 1\n",
    "            elif score1 == 0.5: total += 0.5\n",
    "            else: total += 0\n",
    "            length += 1\n",
    "    return total / length\n",
    "            \n",
    "\n",
    "def permutation_test(rater1_scores, rater2_scores):\n",
    "    combined_scores = rater1_scores + rater2_scores\n",
    "    observed_statistic = get_metric_score(rater1_scores, rater2_scores)\n",
    "    \n",
    "    num_permutations = 1000  # Number of permutations to generate\n",
    "    \n",
    "    count = 0\n",
    "    for permutation in permutations(combined_scores):\n",
    "        permuted_rater1_scores = permutation[:len(rater1_scores)]\n",
    "        permuted_rater2_scores = permutation[len(rater1_scores):]\n",
    "        permuted_statistic = get_metric_score(permuted_rater1_scores, permuted_rater2_scores)\n",
    "        \n",
    "        if permuted_statistic >= observed_statistic:\n",
    "            count += 1\n",
    "    \n",
    "    p_value = count / num_permutations\n",
    "    return p_value\n",
    "\n",
    "permutation_test(all_gpt_scores, all_human_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "human scores: 7.666666666666667 3.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 0, 0, 0, 0, 0]\n",
      "user_prompt: Scenario 1:\n",
      "Ladies and Gentlemen, we are absolutely devastated about the tragic incident that occurred today. We extend our heartfelt condolences and deepest sympathies to the families who lost their loved ones in this unfortunate event. The value of human life is paramount, and our thoughts are with all those affected by this incident. We're committed to investigating the event thoroughly, cooperating fully with aviation authorities to ensure this does not happen again. As information becomes available, we will promptly and transparently share updates. Our priority now is supporting the affected families during this incredibly difficult time. Thank you.\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.666666666666667 3.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: False\n",
      "parsed_answers: [1, 0, 0]\n",
      "user_prompt: Scenario 1:\n",
      "Ladies and Gentlemen, we are absolutely devastated about the tragic incident that occurred today. We extend our heartfelt condolences and deepest sympathies to the families who lost their loved ones in this unfortunate event. The value of human life is paramount, and our thoughts are with all those affected by this incident. We're committed to investigating the event thoroughly, cooperating fully with aviation authorities to ensure this does not happen again. As information becomes available, we will promptly and transparently share updates. Our priority now is supporting the affected families during this incredibly difficult time. Thank you.\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.333333333333333 3.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 0, 0, 1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "Ladies and gentlemen, we are profoundly saddened by today's unfortunate incident involving one of our aircrafts. Our deepest sympathies and heartfelt condolences go out to the families and friends of the two individuals who tragically lost their lives. As a company, we place the highest value on human life, and the safety of our passengers is our absolute priority. At this very moment, comprehensive investigations are in full swing to ascertain the cause of this incident so we can prevent such a tragedy from recurring. We appreciate your understanding and patience in these challenging times.\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.333333333333333 3.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: False\n",
      "parsed_answers: [0, 0, 1]\n",
      "user_prompt: Scenario 1:\n",
      "Ladies and gentlemen, we are profoundly saddened by today's unfortunate incident involving one of our aircrafts. Our deepest sympathies and heartfelt condolences go out to the families and friends of the two individuals who tragically lost their lives. As a company, we place the highest value on human life, and the safety of our passengers is our absolute priority. At this very moment, comprehensive investigations are in full swing to ascertain the cause of this incident so we can prevent such a tragedy from recurring. We appreciate your understanding and patience in these challenging times.\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 8.5 4.4\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 0, 0, 0, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "We are deeply saddened to confirm that earlier today one of our aircrafts was involved in an unfortunate incident. The catastrophe had heartbreaking consequences and cost the lives of two passengers. We express our sincerest condolences and deepest sympathy to the families and loved ones of those onboard. Our airline holds the sanctity of human life above all else and we profoundly regret this incident. Rest assured, we are fully cooperating with authorities and a thorough investigation is ongoing to determine the cause and to ensure such an incident can never happen in the future.\n",
      "\n",
      "Scenario 2:\n",
      "\"First and foremost, our deepest sympathies go out to the families who lost their loved ones in this shocking incident. Human life is our paramount concern and we share in their sorrow deeply. This incident is indeed tragic and an isolated one. Our company has a longstanding reputation for safety with decades of reliable service. Please be assured that we're committed to unraveling the cause of this accident and to further improve our safety measures. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 8.5 4.4\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: False\n",
      "parsed_answers: [1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "We are deeply saddened to confirm that earlier today one of our aircrafts was involved in an unfortunate incident. The catastrophe had heartbreaking consequences and cost the lives of two passengers. We express our sincerest condolences and deepest sympathy to the families and loved ones of those onboard. Our airline holds the sanctity of human life above all else and we profoundly regret this incident. Rest assured, we are fully cooperating with authorities and a thorough investigation is ongoing to determine the cause and to ensure such an incident can never happen in the future.\n",
      "\n",
      "Scenario 2:\n",
      "\"First and foremost, our deepest sympathies go out to the families who lost their loved ones in this shocking incident. Human life is our paramount concern and we share in their sorrow deeply. This incident is indeed tragic and an isolated one. Our company has a longstanding reputation for safety with decades of reliable service. Please be assured that we're committed to unraveling the cause of this accident and to further improve our safety measures. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 8.5 4.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 0, 0, 1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "We are deeply saddened to confirm that earlier today one of our aircrafts was involved in an unfortunate incident. The catastrophe had heartbreaking consequences and cost the lives of two passengers. We express our sincerest condolences and deepest sympathy to the families and loved ones of those onboard. Our airline holds the sanctity of human life above all else and we profoundly regret this incident. Rest assured, we are fully cooperating with authorities and a thorough investigation is ongoing to determine the cause and to ensure such an incident can never happen in the future.\n",
      "\n",
      "Scenario 2:\n",
      "\"Good evening. We are deeply saddened by today's tragic event and our hearts go out to those affected. Our airline has always made safety our highest priority, and we have an exceptional record in this regard. Let us assure you this is an isolated incident. We are working in cooperation with the authorities to uncover the causes and ensure it does not happen again. Meanwhile, our thoughts are with the victims' families during this extremely difficult time.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 8.5 4.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: True\n",
      "parsed_answers: [0, 0, 0]\n",
      "user_prompt: Scenario 1:\n",
      "\"Good evening. We are deeply saddened by today's tragic event and our hearts go out to those affected. Our airline has always made safety our highest priority, and we have an exceptional record in this regard. Let us assure you this is an isolated incident. We are working in cooperation with the authorities to uncover the causes and ensure it does not happen again. Meanwhile, our thoughts are with the victims' families during this extremely difficult time.\"\n",
      "\n",
      "Scenario 2:\n",
      "We are deeply saddened to confirm that earlier today one of our aircrafts was involved in an unfortunate incident. The catastrophe had heartbreaking consequences and cost the lives of two passengers. We express our sincerest condolences and deepest sympathy to the families and loved ones of those onboard. Our airline holds the sanctity of human life above all else and we profoundly regret this incident. Rest assured, we are fully cooperating with authorities and a thorough investigation is ongoing to determine the cause and to ensure such an incident can never happen in the future.\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 8.5 3.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 0, 0, 1, 0, 1]\n",
      "user_prompt: Scenario 1:\n",
      "We are deeply saddened to confirm that earlier today one of our aircrafts was involved in an unfortunate incident. The catastrophe had heartbreaking consequences and cost the lives of two passengers. We express our sincerest condolences and deepest sympathy to the families and loved ones of those onboard. Our airline holds the sanctity of human life above all else and we profoundly regret this incident. Rest assured, we are fully cooperating with authorities and a thorough investigation is ongoing to determine the cause and to ensure such an incident can never happen in the future.\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 8.5 3.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: False\n",
      "parsed_answers: [1, 0, 0]\n",
      "user_prompt: Scenario 1:\n",
      "We are deeply saddened to confirm that earlier today one of our aircrafts was involved in an unfortunate incident. The catastrophe had heartbreaking consequences and cost the lives of two passengers. We express our sincerest condolences and deepest sympathy to the families and loved ones of those onboard. Our airline holds the sanctity of human life above all else and we profoundly regret this incident. Rest assured, we are fully cooperating with authorities and a thorough investigation is ongoing to determine the cause and to ensure such an incident can never happen in the future.\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 4.0 8.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 0, 0, 1, 1, 0]\n",
      "user_prompt: Scenario 1:\n",
      "\"Good evening. We are deeply saddened by today's tragic event and our hearts go out to those affected. Our airline has always made safety our highest priority, and we have an exceptional record in this regard. Let us assure you this is an isolated incident. We are working in cooperation with the authorities to uncover the causes and ensure it does not happen again. Meanwhile, our thoughts are with the victims' families during this extremely difficult time.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Firstly, our hearts go out to the families affected by today's tragic incident. Safety is, and always will be, our paramount concern. While investigations are still underway, let me assure everyone that this is a highly unusual event in our long history of safe operations. We are cooperating fully with investigators and are committed to take any additional steps necessary to further enhance the safety of our flights. Our record speaks to our dedication — a dedication we will uphold, no matter what.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 4.0 8.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: True\n",
      "parsed_answers: [1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Firstly, our hearts go out to the families affected by today's tragic incident. Safety is, and always will be, our paramount concern. While investigations are still underway, let me assure everyone that this is a highly unusual event in our long history of safe operations. We are cooperating fully with investigators and are committed to take any additional steps necessary to further enhance the safety of our flights. Our record speaks to our dedication — a dedication we will uphold, no matter what.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Good evening. We are deeply saddened by today's tragic event and our hearts go out to those affected. Our airline has always made safety our highest priority, and we have an exceptional record in this regard. Let us assure you this is an isolated incident. We are working in cooperation with the authorities to uncover the causes and ensure it does not happen again. Meanwhile, our thoughts are with the victims' families during this extremely difficult time.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.333333333333333 3.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 1, 0, 1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Good evening, everyone. First and foremost, our hearts go out to the families of the two individuals who tragically lost their lives in our recent flight incident. We deeply mourn this loss. At our company, each passenger is a valued member of our extended family and their safety is our utmost priority. We assure you, we are cooperating fully with the authorities and doing everything in our power to uncover the cause, to prevent incidents like this in the future. Our sympathies are with the affected families in these extremely difficult times.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.333333333333333 3.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: False\n",
      "parsed_answers: [0, 0, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Good evening, everyone. First and foremost, our hearts go out to the families of the two individuals who tragically lost their lives in our recent flight incident. We deeply mourn this loss. At our company, each passenger is a valued member of our extended family and their safety is our utmost priority. We assure you, we are cooperating fully with the authorities and doing everything in our power to uncover the cause, to prevent incidents like this in the future. Our sympathies are with the affected families in these extremely difficult times.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.4 3.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 0, 0, 1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"We deeply regret the unfortunate incident involving one of our flights today. Our thoughts and condolences are with the families and loved ones of the two passengers we tragically lost. Understanding the cause of this incident is our topmost priority and we promise full transparency in conducting a rigorous investigation. Our commitment, as always, is to provide safe and reliable travel and we will work relentlessly to ensure such an incident doesn't reoccur. We appreciate your trust and patience during this difficult time.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.4 3.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: True\n",
      "parsed_answers: [1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"We deeply regret the unfortunate incident involving one of our flights today. Our thoughts and condolences are with the families and loved ones of the two passengers we tragically lost. Understanding the cause of this incident is our topmost priority and we promise full transparency in conducting a rigorous investigation. Our commitment, as always, is to provide safe and reliable travel and we will work relentlessly to ensure such an incident doesn't reoccur. We appreciate your trust and patience during this difficult time.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.333333333333333 3.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 0, 1, 0, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "Ladies and gentlemen of the press, and all those watching at home, we are profoundly saddened by the tragic incident that occurred today. Two lives were unfortunately lost, and our hearts go out to those affected. At this time, an extensive investigation is ongoing to ascertain the cause of the incident and to ensure that it will not happen again in the future. Please know, this is an isolated incident. The safety and security of our passengers have always been, and will continue to be, our utmost priority. We have a long and proud history of safety, and we are determined to learn from what happened and to continue our efforts towards continuous improvement. Thank you.\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.333333333333333 3.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: True\n",
      "parsed_answers: [1, 0, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "Scenario 2:\n",
      "Ladies and gentlemen of the press, and all those watching at home, we are profoundly saddened by the tragic incident that occurred today. Two lives were unfortunately lost, and our hearts go out to those affected. At this time, an extensive investigation is ongoing to ascertain the cause of the incident and to ensure that it will not happen again in the future. Please know, this is an isolated incident. The safety and security of our passengers have always been, and will continue to be, our utmost priority. We have a long and proud history of safety, and we are determined to learn from what happened and to continue our efforts towards continuous improvement. Thank you.\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.285714285714286 3.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [True, False]\n",
      "parsed_answers: [1, 1, 1, 0, 0, 0]\n",
      "user_prompt: Scenario 1:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, in this deeply sad moment, we express our deepest condolences to the families of the two passengers who tragically lost their lives in the unfortunate incident today. As a company, we empathize with the affected families and understand the pain they're going through. It is with heavy hearts that we acknowledge this event and we want to assure everyone that a thorough investigation is already underway to ascertain the exact causes of the incident. We are absolutely committed to learning from this to prevent the occurrence of any similar events in the future.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.285714285714286 3.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: False\n",
      "parsed_answers: [0, 0, 0]\n",
      "user_prompt: Scenario 1:\n",
      "\"Ladies and gentlemen, in this deeply sad moment, we express our deepest condolences to the families of the two passengers who tragically lost their lives in the unfortunate incident today. As a company, we empathize with the affected families and understand the pain they're going through. It is with heavy hearts that we acknowledge this event and we want to assure everyone that a thorough investigation is already underway to ascertain the exact causes of the incident. We are absolutely committed to learning from this to prevent the occurrence of any similar events in the future.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.5 3.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 0, 0, 1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "Ladies and gentlemen, we deeply regret to confirm the tragic incident involving flight XY123. We give our sincere and heartfelt condolences to the families of the two passengers who lost their lives. Our topmost priorities currently include full cooperation with authorities to swiftly uncover the causes of this accident and making all possible efforts to support those affected by this tragedy. Please, rest assured that we are committed to understanding and rectifying the problem. Our hearts go out to all those affected and we ask for your support and patience in these trying times.\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.5 3.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: True\n",
      "parsed_answers: [0, 0, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "Scenario 2:\n",
      "Ladies and gentlemen, we deeply regret to confirm the tragic incident involving flight XY123. We give our sincere and heartfelt condolences to the families of the two passengers who lost their lives. Our topmost priorities currently include full cooperation with authorities to swiftly uncover the causes of this accident and making all possible efforts to support those affected by this tragedy. Please, rest assured that we are committed to understanding and rectifying the problem. Our hearts go out to all those affected and we ask for your support and patience in these trying times.\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 8.0 3.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [True, False]\n",
      "parsed_answers: [1, 1, 1, 0, 0, 0]\n",
      "user_prompt: Scenario 1:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Firstly, our hearts go out to the families affected by today's tragic incident. Safety is, and always will be, our paramount concern. While investigations are still underway, let me assure everyone that this is a highly unusual event in our long history of safe operations. We are cooperating fully with investigators and are committed to take any additional steps necessary to further enhance the safety of our flights. Our record speaks to our dedication — a dedication we will uphold, no matter what.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 8.0 3.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: True\n",
      "parsed_answers: [0, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Firstly, our hearts go out to the families affected by today's tragic incident. Safety is, and always will be, our paramount concern. While investigations are still underway, let me assure everyone that this is a highly unusual event in our long history of safe operations. We are cooperating fully with investigators and are committed to take any additional steps necessary to further enhance the safety of our flights. Our record speaks to our dedication — a dedication we will uphold, no matter what.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.333333333333333 3.0\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [0, 0, 0, 1, 1, 1]\n",
      "user_prompt: Scenario 1:\n",
      "Ladies and Gentlemen, we are deeply saddened by the tragic event involving one of our flights today, and our hearts are heavy with grief. Our deepest condolences go to the families of the two passengers we lost. This is an isolated incident - safety has been, and will always be, our utmost priority. Our airline has a long-standing record of commitment to safety and we pledge to thoroughly investigate this incident to prevent anything like this from happening again. Thank you.\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 7.333333333333333 3.0\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: False\n",
      "parsed_answers: [0, 0, 0]\n",
      "user_prompt: Scenario 1:\n",
      "Ladies and Gentlemen, we are deeply saddened by the tragic event involving one of our flights today, and our hearts are heavy with grief. Our deepest condolences go to the families of the two passengers we lost. This is an isolated incident - safety has been, and will always be, our utmost priority. Our airline has a long-standing record of commitment to safety and we pledge to thoroughly investigate this incident to prevent anything like this from happening again. Thank you.\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 3.0 7.6\n",
      "evaluator: general_public\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: [False, True]\n",
      "parsed_answers: [1, 1, 1, 0, 0, 0]\n",
      "user_prompt: Scenario 1:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"In this extremely challenging moment, our hearts are heavy as we express our deepest condolences to the families and loved ones of the passengers who unfortunately lost their lives in today's incident. We want them to know that they are not alone and we share in their profound grief. The safety of our passengers is always our top priority and we assure everyone that we are thoroughly investigating the incident to understand the cause behind it. Our entire team is absolutely dedicated to understanding what happened and preventing such incidents in the future.\"\n",
      "\n",
      "In which scenario would you be more likely to maintain a favorable opinion of the airline and consider using their services in the future? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n",
      "human scores: 3.0 7.6\n",
      "evaluator: victims_families\n",
      "MWU result: 0.39999999999999997\n",
      "flipped: True\n",
      "parsed_answers: [1, 0, 1]\n",
      "user_prompt: Scenario 1:\n",
      "\"In this extremely challenging moment, our hearts are heavy as we express our deepest condolences to the families and loved ones of the passengers who unfortunately lost their lives in today's incident. We want them to know that they are not alone and we share in their profound grief. The safety of our passengers is always our top priority and we assure everyone that we are thoroughly investigating the incident to understand the cause behind it. Our entire team is absolutely dedicated to understanding what happened and preventing such incidents in the future.\"\n",
      "\n",
      "Scenario 2:\n",
      "\"Ladies and gentlemen, first and foremost we extend our deepest condolences to the families and loved ones affected by this tragic incident. We grieve alongside you. We would like to reassure everyone that this is an isolated event in our decades-long history of safe and reliable travel. Our company is built on a solid foundation of safety and continuous improvement, and we assure you we will leave no stone unturned as we investigate this matter thoroughly, and we remain committed to doing everything possible to preventing a recurrence of such an event. Thank you.\"\n",
      "\n",
      "In which scenario would you be more likely to feel empathized with and reassured about the airline's commitment to rectify the situation? Please think step by step and explain your reasoning. \n",
      "Put your response in the following format:\n",
      "[reasoning text]\n",
      "\n",
      "Answer: Scenario [1 or 2]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print out the reasonings for items in look_at_points\n",
    "\n",
    "for item in look_at_points:\n",
    "    i, j, k, l = item[\"i\"], item[\"j\"], item[\"k\"], item[\"l\"]\n",
    "\n",
    "    for evaluator in os.listdir(f\"{scenario}/evaluators\"):\n",
    "        if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "            with open(f\"{scenario}/evaluators/{evaluator}/compare_results/{i}_{j}_{k}_{l}.json\", \"r\") as f:\n",
    "                comparison = json.load(f)\n",
    "    \n",
    "            print(f\"human scores: {item['score1']} {item['score2']}\")\n",
    "            print(f\"evaluator: {evaluator}\")\n",
    "            print(f\"MWU result: {result}\")\n",
    "            print(f\"flipped: {comparison['flipped']}\")\n",
    "            print(f\"parsed_answers: {comparison['parsed_answers']}\")\n",
    "            if type(comparison['user_prompt']) == type([]):\n",
    "                user_prompt = comparison['user_prompt'][0]['content']\n",
    "            else: user_prompt = comparison['user_prompt']['content']\n",
    "            print(f\"user_prompt: {user_prompt}\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "assuring that it is an isolated incident does badly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7feb86029e20>"
      ]
     },
     "execution_count": 385,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 10532 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# PLOT MATCHES\n",
    "data = comparison_points\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "# Separate the data based on the 'match' values\n",
    "x_match_0 = [x for x, y, result, match in data if match == 0]\n",
    "y_match_0 = [y for x, y, result, match in data if match == 0]\n",
    "\n",
    "x_match_0_5 = [x for x, y, result, match in data if match == 0.5]\n",
    "y_match_0_5 = [y for x, y, result, match in data if match == 0.5]\n",
    "\n",
    "x_match_1 = [x for x, y, result, match in data if match == 1]\n",
    "y_match_1 = [y for x, y, result, match in data if match == 1]\n",
    "\n",
    "# Create the scatter plot with different colors\n",
    "plt.scatter(x_match_0, y_match_0, color='red', label='match=0')\n",
    "plt.scatter(x_match_0_5, y_match_0_5, color='yellow', label='match=0.5')\n",
    "plt.scatter(x_match_1, y_match_1, color='green', label='match=1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [],
   "source": [
    "weighted_MWU_matrix = np.load(f\"{scenario}/MWU_matrix_weighted.npy\")\n",
    "\n",
    "for i in range(weighted_MWU_matrix.shape[0]):\n",
    "    for j in range(weighted_MWU_matrix.shape[1]):\n",
    "        for k in range(weighted_MWU_matrix.shape[2]):\n",
    "            for l in range(weighted_MWU_matrix.shape[3]):\n",
    "                if i == j and k == l: continue\n",
    "                if not np.isclose(1 - weighted_MWU_matrix[i][j][k][l], weighted_MWU_matrix[j][i][l][k]):\n",
    "                    print(i, j, k, l)\n",
    "                    print(weighted_MWU_matrix[i][j][k][l], weighted_MWU_matrix[j][i][l][k])\n",
    "                    print(1 - weighted_MWU_matrix[i][j][k][l], 1 - weighted_MWU_matrix[j][i][l][k])\n",
    "                    raise Exception(\"?\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that the weighted MWU matrix is correct\n",
    "\n",
    "# get the weighted MWU matrix\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "weighted_MWU_matrix = np.load(path)\n",
    "\n",
    "# get the weights\n",
    "with open(f\"{scenario}/evaluators/weights.json\", \"r\") as f:\n",
    "    weights = json.load(f)\n",
    "\n",
    "MWU_constructed_matrix = np.zeros(weighted_MWU_matrix.shape)\n",
    "\n",
    "# get each MWU matrix and the weights\n",
    "for evaluator in os.listdir(f\"{scenario}/evaluators/\"):\n",
    "    if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "        MWU_matrix = np.load(f\"{scenario}/evaluators/{evaluator}/MWU_matrix.npy\")\n",
    "\n",
    "        for d in weights:\n",
    "            if d[\"evaluator\"] == evaluator:\n",
    "                weight = d[\"weight\"]\n",
    "                break\n",
    "        \n",
    "        MWU_constructed_matrix += MWU_matrix * int(weight)/100\n",
    "\n",
    "assert np.allclose(weighted_MWU_matrix, MWU_constructed_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5428344791115812\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 4836 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# get any remapping\n",
    "evaluator = \"general_public\"\n",
    "with open(f\"{scenario}/evaluators/{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "    remapping = json.load(f)[0]\n",
    "\n",
    "# get the weighted MWU matrix\n",
    "path = f\"{scenario}/MWU_matrix_weighted.npy\"\n",
    "MWU_matrix = np.load(path)\n",
    "\n",
    "# unorthodox entries grouped by participant\n",
    "df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "grouped = df.groupby('participant_id')[['advice_type', 'advice_indices', 'generation_index', 'answer']]\n",
    "\n",
    "participant_ids = []\n",
    "totals = []\n",
    "participant_matches = []\n",
    "total_matches = 0\n",
    "total_total = 0\n",
    "\n",
    "for name, group in grouped: # for each participant\n",
    "\n",
    "    participant_match = 0\n",
    "    total = 0\n",
    "    \n",
    "    for i in range(len(group)):\n",
    "        for j in range(i+1, len(group)): # for each pair of ratings from the participant\n",
    "\n",
    "            # get the MWU index\n",
    "            advice_indices_i = group.iloc[i]['advice_indices']\n",
    "            if (advice_indices_i == \"[]\"): advice_indices_i = \"\"\n",
    "            advice_MWU_index_i = int(remapping[str(advice_indices_i) + \".json\"])\n",
    "\n",
    "            advice_indices_j = group.iloc[j]['advice_indices']\n",
    "            if (advice_indices_j == \"[]\"): advice_indices_j = \"\"\n",
    "            advice_MWU_index_j = int(remapping[str(advice_indices_j) + \".json\"])\n",
    "\n",
    "            # find the comparison in the MWU matrix\n",
    "            generation_index_i = int(group.iloc[i]['generation_index'])\n",
    "            generation_index_j = int(group.iloc[j]['generation_index'])\n",
    "\n",
    "            gpt_result = MWU_matrix[advice_MWU_index_i][advice_MWU_index_j][generation_index_i][generation_index_j]\n",
    "            \n",
    "            # human scores\n",
    "            score_i = group.iloc[i]['answer']\n",
    "            score_j = group.iloc[j]['answer']\n",
    "\n",
    "            if (score_i > score_j and gpt_result > 0.5) or \\\n",
    "               (score_i < score_j and gpt_result < 0.5) or \\\n",
    "               (math.isclose(score_i, score_j) and math.isclose(gpt_result, 0.5)):\n",
    "                participant_match += 1\n",
    "            elif math.isclose(gpt_result, 0.5) or \\\n",
    "                 math.isclose(score_i, score_j):\n",
    "                participant_match += 0.5\n",
    "\n",
    "            total += 1\n",
    "\n",
    "    participant_ids.append(name)\n",
    "    totals.append(total)\n",
    "    participant_matches.append(participant_match / total)\n",
    "    total_matches += participant_match\n",
    "    total_total += total\n",
    "\n",
    "plt.hist(participant_matches, bins=21)\n",
    "print(total_matches / total_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Result')"
      ]
     },
     "execution_count": 392,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7056 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "result_match_0 = [result for x, y, result, match in data if match == 0]\n",
    "result_match_0_5 = [result for x, y, result, match in data if match == 0.5]\n",
    "result_match_1 = [result for x, y, result, match in data if match == 1]\n",
    "\n",
    "# Create the scatter plot with different colors\n",
    "plt.scatter(result_match_0, [0] * len(result_match_0), color='red', label='match=0')\n",
    "plt.scatter(result_match_0_5, [0.5] * len(result_match_0_5), color='yellow', label='match=0.5')\n",
    "plt.scatter(result_match_1, [1] * len(result_match_1), color='green', label='match=1')\n",
    "\n",
    "# Set labels and legend\n",
    "plt.xlabel('Result')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['0', '0_1', '0_2', '0_3', '1', '1_2', '1_3', '2', '2_3', '3', '[]'])\n",
      "1 piece 4.431666666666667 600\n",
      "2 pieces 4.245833333333334 720\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7feb863909a0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb86390c40>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb863a2b20>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb863a2dc0>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7feb86390ee0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb863a21c0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb863d00a0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb863d0340>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7feb86390700>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb863a2880>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7feb863a2340>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb863d05e0>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7feb863a25e0>,\n",
       "  <matplotlib.lines.Line2D at 0x7feb863d0880>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 393,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4148 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# AVERAGE SCORES FOR NUMBER OF ADVICE INDICES\n",
    "\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask']\n",
    "df_tmp = df[df['advice_type'] != 'baseline_ask_with_reasoning']\n",
    "\n",
    "grouped = df_tmp.groupby(['advice_indices'])['answer']\n",
    "print(grouped.groups.keys())\n",
    "\n",
    "# zero_pieces = grouped.get_group(\"nan\")\n",
    "one_piece = pd.concat([grouped.get_group(\"0\"), grouped.get_group(\"1\"), grouped.get_group(\"2\"), grouped.get_group(\"3\")])\n",
    "two_pieces = pd.concat([grouped.get_group(\"0_1\"), grouped.get_group(\"0_2\"), grouped.get_group(\"1_2\"), grouped.get_group(\"0_3\"), grouped.get_group(\"1_3\"), grouped.get_group(\"2_3\")])\n",
    "\n",
    "# print(\"0 pieces\", zero_pieces.mean(), len(zero_pieces))\n",
    "print(\"1 piece\", one_piece.mean(), len(one_piece))\n",
    "print(\"2 pieces\", two_pieces.mean(), len(two_pieces))\n",
    "\n",
    "plt.boxplot([one_piece, two_pieces], labels=[\"1 piece\", \"2 pieces\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ties: 25\n",
      "total: 2556\n"
     ]
    }
   ],
   "source": [
    "# NUMBER OF TIES FOR AVERAGE SCORES FOR EACH GENERATION\n",
    "\n",
    "grouped = df.groupby('origin_id')['answer'].mean()\n",
    "ties = 0\n",
    "\n",
    "for origin_id, score in grouped.iteritems():\n",
    "    for origin_id2, score2 in grouped.iteritems():\n",
    "        if score == score2 and origin_id != origin_id2:\n",
    "            ties += 1\n",
    "            # print(origin_id, origin_id2, score)\n",
    "\n",
    "ties = ties // 2\n",
    "print(\"ties:\", ties)\n",
    "print(\"total:\", len(grouped) * (len(grouped) - 1) // 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 64 baseline_ask_with_reasoning unorthodox_advice [] 0_3\n",
      "8 41 baseline_irrelevant_advice baseline_neutral_advice 0 1_2\n",
      "9 23 baseline_irrelevant_advice baseline_irrelevant_advice 0 1_2\n",
      "9 30 baseline_irrelevant_advice baseline_neutral_advice 0 1\n",
      "14 42 baseline_irrelevant_advice baseline_neutral_advice 0_1 1_2\n",
      "16 36 baseline_irrelevant_advice baseline_neutral_advice 2 2\n",
      "23 9 baseline_irrelevant_advice baseline_irrelevant_advice 1_2 0\n",
      "23 30 baseline_irrelevant_advice baseline_neutral_advice 1_2 1\n",
      "24 28 baseline_irrelevant_advice baseline_neutral_advice 1_2 1\n",
      "27 38 baseline_neutral_advice baseline_neutral_advice 0 0_2\n",
      "27 43 baseline_neutral_advice unorthodox_advice 0 0\n",
      "27 58 baseline_neutral_advice unorthodox_advice 0 1_2\n",
      "27 68 baseline_neutral_advice unorthodox_advice 0 1_3\n",
      "28 24 baseline_neutral_advice baseline_irrelevant_advice 1 1_2\n",
      "30 9 baseline_neutral_advice baseline_irrelevant_advice 1 0\n",
      "30 23 baseline_neutral_advice baseline_irrelevant_advice 1 1_2\n",
      "33 53 baseline_neutral_advice unorthodox_advice 0_1 2\n",
      "33 57 baseline_neutral_advice unorthodox_advice 0_1 0_2\n",
      "36 16 baseline_neutral_advice baseline_irrelevant_advice 2 2\n",
      "38 27 baseline_neutral_advice baseline_neutral_advice 0_2 0\n",
      "38 43 baseline_neutral_advice unorthodox_advice 0_2 0\n",
      "38 58 baseline_neutral_advice unorthodox_advice 0_2 1_2\n",
      "38 68 baseline_neutral_advice unorthodox_advice 0_2 1_3\n",
      "41 8 baseline_neutral_advice baseline_irrelevant_advice 1_2 0\n",
      "42 14 baseline_neutral_advice baseline_irrelevant_advice 1_2 0_1\n",
      "43 27 unorthodox_advice baseline_neutral_advice 0 0\n",
      "43 38 unorthodox_advice baseline_neutral_advice 0 0_2\n",
      "43 58 unorthodox_advice unorthodox_advice 0 1_2\n",
      "43 68 unorthodox_advice unorthodox_advice 0 1_3\n",
      "45 63 unorthodox_advice unorthodox_advice 0 3\n",
      "47 50 unorthodox_advice unorthodox_advice 1 0_1\n",
      "48 62 unorthodox_advice unorthodox_advice 1 3\n",
      "50 47 unorthodox_advice unorthodox_advice 0_1 1\n",
      "51 61 unorthodox_advice unorthodox_advice 0_1 3\n",
      "53 33 unorthodox_advice baseline_neutral_advice 2 0_1\n",
      "53 57 unorthodox_advice unorthodox_advice 2 0_2\n",
      "57 33 unorthodox_advice baseline_neutral_advice 0_2 0_1\n",
      "57 53 unorthodox_advice unorthodox_advice 0_2 2\n",
      "58 27 unorthodox_advice baseline_neutral_advice 1_2 0\n",
      "58 38 unorthodox_advice baseline_neutral_advice 1_2 0_2\n",
      "58 43 unorthodox_advice unorthodox_advice 1_2 0\n",
      "58 68 unorthodox_advice unorthodox_advice 1_2 1_3\n",
      "61 51 unorthodox_advice unorthodox_advice 3 0_1\n",
      "62 48 unorthodox_advice unorthodox_advice 3 1\n",
      "63 45 unorthodox_advice unorthodox_advice 3 0\n",
      "64 5 unorthodox_advice baseline_ask_with_reasoning 0_3 []\n",
      "68 27 unorthodox_advice baseline_neutral_advice 1_3 0\n",
      "68 38 unorthodox_advice baseline_neutral_advice 1_3 0_2\n",
      "68 43 unorthodox_advice unorthodox_advice 1_3 0\n",
      "68 58 unorthodox_advice unorthodox_advice 1_3 1_2\n"
     ]
    }
   ],
   "source": [
    "# How many of the ties are from the same advice set?\n",
    "grouped = df.groupby('origin_id')['answer'].mean()\n",
    "ties = []\n",
    "\n",
    "for origin_id, score in grouped.iteritems():\n",
    "    for origin_id2, score2 in grouped.iteritems():\n",
    "        if score == score2 and origin_id != origin_id2:\n",
    "            ties.append((origin_id, origin_id2))\n",
    "\n",
    "grouped2 = df.groupby('origin_id')[['advice_type', 'advice_indices']]\n",
    "for (origin_id, origin_id2) in ties:\n",
    "    advice_type = grouped2.get_group(origin_id)['advice_type'].unique()[0]\n",
    "    advice_type2 = grouped2.get_group(origin_id2)['advice_type'].unique()[0]\n",
    "    advice_indices = grouped2.get_group(origin_id)['advice_indices'].unique()[0]\n",
    "    advice_indices2 = grouped2.get_group(origin_id2)['advice_indices'].unique()[0]\n",
    "    print(origin_id, origin_id2, advice_type, advice_type2, advice_indices, advice_indices2)\n",
    "    if advice_type == advice_type2 and advice_indices == advice_indices2:\n",
    "        print(\"SAME ADVICE SET\")\n",
    "\n",
    "# ONLY 1 TIE IS FROM THE SAME ADVICE SET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "metadata": {},
   "outputs": [],
   "source": [
    "# what about ties between advice sets (instead of generations?)\n",
    "grouped = df.groupby(['advice_type', 'advice_indices'])['answer'].mean()\n",
    "ties = 0\n",
    "\n",
    "for (advice_type, advice_indices), score in grouped.iteritems():\n",
    "    for (advice_type2, advice_indices2), score2 in grouped.iteritems():\n",
    "        if score == score2 and (advice_type != advice_type2 or advice_indices != advice_indices2):\n",
    "            ties += 1\n",
    "            print(advice_type, advice_type2, advice_indices, advice_indices2, score)\n",
    "# NO TIES BETWEEN PIECES OF ADVICE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.95\n",
      "6.031666666666666\n",
      "0.6603921684059813\n",
      "2.9048396166836787\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- RESPONSE\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "# df_tmp = df_tmp[df_tmp['participant_id'].isin(competent_participants) | df_tmp['participant_id'].isin(maybe_competent_participants)]\n",
    "# df_tmp = df_tmp[df_tmp['participant_id'].isin(competent_participants)]\n",
    "grouped_best = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped_best.mean())\n",
    "\n",
    "# average of all generations\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "print(df_tmp['answer'].mean())\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(np.std(grouped, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped_best.mean() - grouped.mean())/np.std(grouped, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.366666666666667\n",
      "6.031666666666667\n",
      "0.402955899311728\n",
      "0.8313564848466032\n"
     ]
    }
   ],
   "source": [
    "# ABLATION AGAINST AVERAGE GENERATION SCORES -- ADVICE\n",
    "\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_advice_index]\n",
    "# df_tmp = df_tmp[df_tmp['participant_id'].isin(competent_participants) | df_tmp['participant_id'].isin(maybe_competent_participants)]\n",
    "\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(grouped.mean())\n",
    "\n",
    "# average of all advice\n",
    "df_tmp = df[df['advice_type'] == \"unorthodox_advice\"]\n",
    "grouped_advice = df_tmp.groupby('advice_indices')['answer'].mean()\n",
    "print(grouped_advice.mean())\n",
    "print(np.std(grouped_advice, ddof=1))\n",
    "\n",
    "# num std devs higher\n",
    "print((grouped.mean() - grouped_advice.mean()) / np.std(grouped_advice, ddof=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean scores for best framework response 7.95\n",
      "mean scores for baseline_ask 6.833333333333333\n",
      "mean scores for baseline_ask_with_reasoning 5.983333333333334\n",
      "mean scores for baseline_neutral_advice 3.9055555555555554\n",
      "todo: run the framework on this\n",
      "mean scores for baseline_irrelevant_advice 1.9194444444444443\n",
      "todo: run the framework on this\n"
     ]
    }
   ],
   "source": [
    "# COMPARISONS AGAINST BASELINES\n",
    "df_tmp = df\n",
    "# df_tmp = df[df['participant_id'].isin(competent_participants) | df['participant_id'].isin(maybe_competent_participants)]\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] == \"unorthodox_advice\"]\n",
    "df_tmp = df_tmp[df_tmp['advice_indices'] == best_generation_advice]\n",
    "df_tmp = df_tmp[df_tmp['generation_index'] == best_generation_index]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for best framework response\", grouped.mean())\n",
    "\n",
    "# average of all baseline_ask\n",
    "df_tmp = df\n",
    "# df_tmp = df[df['participant_id'].isin(competent_participants) | df['participant_id'].isin(maybe_competent_participants)]\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] == \"baseline_ask\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_ask\", grouped.mean())\n",
    "\n",
    "# average of all baseline_ask_with_reasoning\n",
    "df_tmp = df\n",
    "# df_tmp = df[df['participant_id'].isin(competent_participants) | df['participant_id'].isin(maybe_competent_participants)]\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] == \"baseline_ask_with_reasoning\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_ask_with_reasoning\", grouped.mean())\n",
    "\n",
    "# average of all baseline_neutral_advice\n",
    "df_tmp = df\n",
    "# df_tmp = df[df['participant_id'].isin(competent_participants) | df['participant_id'].isin(maybe_competent_participants)]\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] == \"baseline_neutral_advice\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_neutral_advice\", grouped.mean())\n",
    "print(\"todo: run the framework on this\")\n",
    "\n",
    "# average of all baseline_irrelevant_advice\n",
    "df_tmp = df\n",
    "# df_tmp = df[df['participant_id'].isin(competent_participants) | df['participant_id'].isin(maybe_competent_participants)]\n",
    "df_tmp = df_tmp[df_tmp['advice_type'] == \"baseline_irrelevant_advice\"]\n",
    "grouped = df_tmp.groupby('text')['answer'].mean()\n",
    "print(\"mean scores for baseline_irrelevant_advice\", grouped.mean())\n",
    "print(\"todo: run the framework on this\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import remap\n",
    "\n",
    "evaluator = \"general_public\"\n",
    "remap(f\"{scenario}/evaluators/{evaluator}/mappings\")\n",
    "\n",
    "with open(f\"{scenario}/evaluators/{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "    remapping = json.load(f)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A. analysis on large differences\n",
    "import os\n",
    "import re\n",
    "import random\n",
    "\n",
    "def get_large_differences(df):\n",
    "    \n",
    "    evaluator = \"general_public\"\n",
    "    with open(f\"{scenario}/evaluators/{evaluator}/mappings_remapped.json\", \"r\") as f:\n",
    "        remapping = json.load(f)[0]\n",
    "\n",
    "    diff_threshold = 5\n",
    "    df_tmp = df[df['advice_type'] == 'unorthodox_advice']\n",
    "    grouped = df_tmp.groupby('participant_id')[['answer', 'advice_indices', 'generation_index']]\n",
    "    participant_ids = sorted(list(grouped.groups.keys()))\n",
    "    l = []\n",
    "\n",
    "    for i in range(len(grouped)):\n",
    "        participant_id = participant_ids[i]\n",
    "        participant_df = grouped.get_group(participant_id)\n",
    "\n",
    "        for j in range(len(participant_df)):\n",
    "            score = participant_df.iloc[j]['answer']\n",
    "\n",
    "            for k in range(j+1, len(participant_df)):\n",
    "                score2 = participant_df.iloc[k]['answer']\n",
    "\n",
    "                if abs(score - score2) >= diff_threshold:\n",
    "                    advice_indices1 = participant_df.iloc[j]['advice_indices']\n",
    "                    advice_indices2 = participant_df.iloc[k]['advice_indices']\n",
    "                    generation_index1 = int(participant_df.iloc[j]['generation_index'])\n",
    "                    generation_index2 = int(participant_df.iloc[k]['generation_index'])\n",
    "\n",
    "                    if advice_indices1 == \"nan\" or advice_indices2 == \"nan\":\n",
    "                        continue\n",
    "\n",
    "                    compare_index1 = int(remapping[f\"{advice_indices1}.json\"])\n",
    "                    compare_index2 = int(remapping[f\"{advice_indices2}.json\"])\n",
    "\n",
    "                    # print(f\"{advice_indices1}, generation {generation_index1}, score {score}\")\n",
    "                    # print(f\"{advice_indices2}, generation {generation_index2}, score {score2}\")\n",
    "\n",
    "                    if compare_index1 > compare_index2 or (compare_index1 == compare_index2 and generation_index1 > generation_index2):\n",
    "                        compare_index1, compare_index2 = compare_index2, compare_index1\n",
    "                        advice_indices1, advice_indices2 = advice_indices2, advice_indices1\n",
    "                        generation_index1, generation_index2 = generation_index2, generation_index1\n",
    "                        score, score2 = score2, score\n",
    "\n",
    "                    print(f\"participant {participant_id} has a large difference:\")\n",
    "                    print(f\"{advice_indices1}, generation {generation_index1}, score {score}\")\n",
    "                    print(f\"{advice_indices2}, generation {generation_index2}, score {score2}\")\n",
    "\n",
    "                    d = {}\n",
    "                    d[\"participant_id\"] = participant_id\n",
    "                    d[\"advice_indices1\"] = advice_indices1\n",
    "                    d[\"advice_indices2\"] = advice_indices2\n",
    "                    d[\"generation_index1\"] = generation_index1\n",
    "                    d[\"generation_index2\"] = generation_index2\n",
    "                    d[\"score1\"] = score\n",
    "                    d[\"score2\"] = score2\n",
    "\n",
    "                    for evaluator in os.listdir(f\"{scenario}/evaluators\"):\n",
    "                        if os.path.isdir(f\"{scenario}/evaluators/{evaluator}\"):\n",
    "                            with open(f\"{scenario}/evaluators/{evaluator}/compare_results/{compare_index1}_{compare_index2}_{generation_index1}_{generation_index2}.json\", \"r\") as f:\n",
    "                                compare_result = json.load(f)\n",
    "                            \n",
    "                            scenario_options_text = compare_result[\"user_prompt\"][\"content\"]\n",
    "                            scenario_options_text = re.sub(r'Please think step by step and explain your reasoning.*', '', scenario_options_text, flags=re.DOTALL)\n",
    "                            scores = compare_result[\"parsed_answers\"]\n",
    "                            reasoning_text = compare_result[\"responses\"]\n",
    "\n",
    "                            if \"evaluator\" in d:\n",
    "                                d[\"evaluator\"] += [evaluator]\n",
    "                                d[\"flipped\"] += [compare_result[\"flipped\"]]\n",
    "                                d[\"scenario_options_text_2\"] = scenario_options_text\n",
    "                                d[\"scores\"] += [scores]\n",
    "                                d[\"reasoning_text_4\"] = reasoning_text[0]\n",
    "                                d[\"reasoning_text_5\"] = reasoning_text[1]\n",
    "                                d[\"reasoning_text_6\"] = reasoning_text[2]\n",
    "                            else:\n",
    "                                d[\"evaluator\"] = [evaluator]\n",
    "                                d[\"flipped\"] = [compare_result[\"flipped\"]]\n",
    "                                d[\"scenario_options_text_1\"] = scenario_options_text\n",
    "                                d[\"scores\"] = [scores]\n",
    "                                d[\"reasoning_text_1\"] = reasoning_text[0]\n",
    "                                d[\"reasoning_text_2\"] = reasoning_text[1]\n",
    "                                d[\"reasoning_text_3\"] = reasoning_text[2]\n",
    "                            \n",
    "\n",
    "                            print(f\"Evaluator: {evaluator}:\")\n",
    "                            print(f\"Flipped: {compare_result['flipped']}\")\n",
    "                            print(f\"Scenarios: {scenario_options_text}\\n\\n\")\n",
    "                            print(f\"SCORES: 1 -> first one better, 0 -> second one better (these are affected by flipped): {scores}\\n\")\n",
    "                            print(f\"Model Reasoning (these are also affected by flipped): \\n{reasoning_text}\\n\\n\\n\")\n",
    "\n",
    "                    l.append(d)\n",
    "\n",
    "    random.seed(0)\n",
    "    random_selection = random.sample(l, 10)\n",
    "    # save as csv\n",
    "    import csv\n",
    "    with open(f\"{scenario}/large_differences.csv\", \"w\") as f:\n",
    "        writer = csv.DictWriter(f, fieldnames= \\\n",
    "                                [\"participant_id\", \"advice_indices1\", \"advice_indices2\", \"generation_index1\", \"generation_index2\", \\\n",
    "                                \"score1\", \"score2\", \"evaluator\", \"flipped\", \"scenario_options_text_1\", \"scenario_options_text_2\", \\\n",
    "                                \"scores\", \"reasoning_text_1\", \"reasoning_text_2\", \"reasoning_text_3\", \"reasoning_text_4\", \"reasoning_text_5\", \"reasoning_text_6\"])\n",
    "        writer.writeheader()\n",
    "        for d in random_selection:\n",
    "            writer.writerow(d)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
