{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/marinatenhave/miniconda3/envs/uropdinamopy39/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import re\n",
    "import base64\n",
    "import requests\n",
    "import json, os\n",
    "import pandas as pd\n",
    "import path\n",
    "import os\n",
    "import sys\n",
    "from pathlib import Path\n",
    "\n",
    "# set directory to top folder level to address relative imports\n",
    "directory = Path(os.getcwd()).absolute()\n",
    "sys.path.append(str(directory.parent.parent.parent))\n",
    "\n",
    "from AI2Thor.env_new import AI2ThorEnv\n",
    "from AI2Thor.base_env import convert_dict_to_string\n",
    "from AI2Thor.object_actions import get_closest_feasible_action, get_closest_object_id\n",
    "\n",
    "# initialize environment parameters\n",
    "class Config:\n",
    "    def __init__(self):\n",
    "        self.num_agents = 2\n",
    "        self.scene = \"FloorPlan2\"\n",
    "        self.scene_name = \"FloorPlan2\"\n",
    "        self.model = \"gpt-4\"\n",
    "        self.use_langchain = False\n",
    "        self.use_strict_format = True\n",
    "        self.use_obs_summariser = False\n",
    "        self.use_act_summariser = False\n",
    "        self.use_action_failure = True\n",
    "        self.use_shared_subtask = True\n",
    "        self.use_separate_subtask = False\n",
    "        self.use_future_message = True\n",
    "        self.forceAction = False\n",
    "        self.use_memory = True\n",
    "        self.use_plan = True\n",
    "        self.use_separate_memory = False\n",
    "        self.use_shared_memory = True\n",
    "        self.temperature = 0.7\n",
    "        \n",
    "config = Config()\n",
    "# save a json file with your openai api key in your\n",
    "# home folder as {\"my_openai_api_key\": \"INSERT API HERE\"}\n",
    "with open(os.path.expanduser(\"~\") + \"/openai_key.json\") as json_file:\n",
    "    key = json.load(json_file)\n",
    "    api_key = key[\"my_openai_api_key\"]\n",
    "headers = {\"Content-Type\": \"application/json\", \"Authorization\": f\"Bearer {api_key}\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prompts for planner, actor and planner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# planner\n",
    "agent_name = [\"Alice\", \"Bob\"]\n",
    "PLANNER_PROMPT = f\"\"\"You are an excellent planner who is tasked with helping 2 embodied robots named {agent_name[0]} and {agent_name[1]} carry out a task. Both robots have a partially observable view of the environment. Hence they have to explore around in the environment to do the task.\n",
    "\n",
    "You will get a description of the task robots are supposed to do. You will get an image of the environment from {agent_name[0]}'s perspective and {agent_name[1]}'s perspective as the observation input. To help you with detecting objects in the image, you will also get a list objects each agent is able to see in the environment. Here the objects are named as \"<object_name>_<object_id>\". \n",
    "So, along with the image inputs you will get the following information:\n",
    "### INPUT FORMAT ###\n",
    "{{Task: description of the task the robots are supposed to do, \n",
    "{agent_name[0]}'s observation: list of objects the {agent_name[0]} is observing,\n",
    "{agent_name[1]}'s observation: list of objects the {agent_name[1]} is observing,\n",
    "Robots' open subtasks: list of subtasks the robots are supposed to carry out to finish the task. If no plan has been already created, this will be None.\n",
    "Robots' completed subtasks: list of subtasks the robots have already completed. If no subtasks have been completed, this will be None.\n",
    "Robots' combined memory: description of robots' combined memory}}\n",
    "\n",
    "Reason over the robots' task, image inputs, observations, open subtasks, completed subtasks and memory, and then output the following:\n",
    "* Reason: The reason for why new subtasks need to be added.\n",
    "* Subtasks: A list of open subtasks the robots are supposed to take to complete the task. Remember, as you get new information about the environment, you can modify this list. You can keep the same plan if you think it is still valid. Do not include the subtasks that have already been completed.\n",
    "The \"Plan\" should be in a list format where the actions are listed sequentially. For example: [\"locate the apple\", \"transport the apple to the fridge\", \"transport the book to the table\", \"\"]\n",
    "Your output should be in the form of a python dictionary as shown below.\n",
    "Example output: {{\"reason\": \"since the subtask list is empty, the robots need to transport the apple to the fridge and transport the book to the table\", \"plan\": [\"transport the apple to the fridge\", \"transport the book to the table\"]}}\n",
    "\n",
    "Ensure that the subtasks are not generic statements like \"explore the environment\" or \"do the task\". They should be specific to the task at hand.\n",
    "Do not assign subtasks to any particular robot. Try not to modify the subtasks that already exist in the open subtasks list. Rather add new subtasks to the list.\n",
    "\n",
    "* NOTE: DO NOT OUTPUT ANYTHING EXTRA OTHER THAN WHAT HAS BEEN SPECIFIED\n",
    "Let's work this out in a step by step way to be sure we have the right answer.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# verifier\n",
    "# moves subtasks from open to completed\n",
    "agent_name = [\"Alice\", \"Bob\"]\n",
    "VERIFIER_PROMPT = f\"\"\"You are an excellent planner who is tasked with helping 2 embodied robots named {agent_name[0]} and {agent_name[1]} carry out a task. Both robots have a partially observable view of the environment. Hence they have to explore around in the environment to do the task.\n",
    "\n",
    "You will get a description of the task robots are supposed to do. You will get an image of the environment from {agent_name[0]}'s perspective and {agent_name[1]}'s perspective as the observation input. To help you with detecting objects in the image, you will also get a list objects each agent is able to see in the environment. Here the objects are named as \"<object_name>_<object_id>\". \n",
    "So, along with the image inputs you will get the following information:\n",
    "### INPUT FORMAT ###\n",
    "{{Task: description of the task the robots are supposed to do, \n",
    "{agent_name[0]}'s observation: list of objects the {agent_name[0]} is observing,\n",
    "{agent_name[0]}'s state: description of {agent_name[0]}'s state,\n",
    "{agent_name[0]}'s previous action: the action {agent_name[0]} took in the previous step and if it was successful,\n",
    "{agent_name[1]}'s observation: list of objects the {agent_name[1]} is observing,\n",
    "{agent_name[0]}'s state: description of {agent_name[0]}'s state,\n",
    "{agent_name[1]}'s previous action: the action {agent_name[1]} took in the previous step,\n",
    "Robots' open subtasks: list of open subtasks the robots in the previous step. If no plan has been already created, this will be None.\n",
    "Robots' completed subtasks: list of subtasks the robots have already completed. If no subtasks have been completed, this will be None.\n",
    "Robots' combined memory: description of robots' combined memory}}\n",
    "\n",
    "Reason over the robots' task, image inputs, observations, previous actions, open subtasks, completed subtasks and memory, and then output the following:\n",
    "* Reason: The reason for why you think a particular subtask should be moved from the open subtasks list to the completed subtasks list.\n",
    "* Completed Subtasks: The list of subtasks that have been completed by the robots. Note that you can add subtasks to this list only if they have been successfully completed and were in the open subtask list. If no subtasks have been completed at the current step, return an empty list.\n",
    "The \"Completed Subtasks\" should be in a list format where the completed subtasks are listed. For example: [\"locate the apple\", \"transport the apple to the fridge\", \"transport the book to the table\"]}}\n",
    "Your output should be in the form of a python dictionary as shown below.\n",
    "Example output: {{\"reason\": \"Alice placed the apple in the fridge in the previous step and was successful and Bob picked up the the book from the table. Hence Alice has completed the subtask of transporting the apple to the fridge but Bob has still not completed the subtask of transporting the book to the table\", \"completed subtasks\": [\"transport the apple to the fridge\"]}}\n",
    "\n",
    "* NOTE: DO NOT OUTPUT ANYTHING EXTRA OTHER THAN WHAT HAS BEEN SPECIFIED\n",
    "Let's work this out in a step by step way to be sure we have the right answer.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# action planner\n",
    "agent_name = [\"Alice\", \"Bob\"]\n",
    "ACTION_PROMPT = f\"\"\"You are an excellent planner who is tasked with helping 2 embodied robots named {agent_name[0]} and {agent_name[1]} carry out a task. Both robots have a partially observable view of the environment. Hence they have to explore around in the environment to do the task.\n",
    "They can perform the following actions: [\"navigate to object <object_id>\", \"rotate in <rotation> direction\", \"pick up object <object_id>\", \"put object on <receptacle_id>\", \"open object <object_id>\", \"close object <object_id>\", \"slice object <object_id>\", “toggle object <object_id> on”, “toggle object <object_id> off”, \"clean object <object_id>\", \"look up by angle <angle>\", \"look down by angle <angle>\", “move in <translation> direction\", \"stay idle\", \"Done\"]\n",
    "Here \"Done\" is used when all the robots have completed the main task. Only use it when you think all the subtasks are complete.\n",
    "\"stay idle\" is used when you want the robot to stay idle for one time step. This could be used to wait for the other robot to complete its subtask. Use it only when you think it is necessary.\n",
    "Here <rotation> can be one of [\"Right\", \"Left\"].\n",
    "Here <angle> is the angle in degrees and can only be one of [30, 60, 90, 120, 150, 180].\n",
    "Here <translation> can be one of [\"Ahead\", \"Back\", \"Left\", \"Right”].\n",
    "\n",
    "You need to suggest the action that each robot should take at the current time step.\n",
    "\n",
    "You will get a description of the task robots are supposed to do. You will get an image of the environment from {agent_name[0]}'s perspective and {agent_name[1]}'s perspective as the observation input.\n",
    "To help you with detecting objects in the image, you will also get a list objects each agent is able to see in the environment. Here the objects are named as \"<object_name>_<object_id>\". \n",
    "So, along with the image inputs you will get the following information:\n",
    "### INPUT FORMAT ###\n",
    "{{Task: description of the task the robots are supposed to do, \n",
    "{agent_name[0]}'s observation: list of objects the {agent_name[0]} is observing,\n",
    "{agent_name[0]}'s state: description of {agent_name[0]}'s state,\n",
    "{agent_name[0]}'s previous action: description of what {agent_name[0]} did in the previous time step and whether it was successful,\n",
    "{agent_name[0]}'s previous failures: if {agent_name[0]}'s few previous actions failed, description of what failed,\n",
    "{agent_name[1]}'s observation: list of objects the {agent_name[1]} is observing,\n",
    "{agent_name[1]}'s state: description of {agent_name[1]}'s state,\n",
    "{agent_name[1]}'s previous action: description of what {agent_name[1]} did in the previous time step and whether it was successful,\n",
    "{agent_name[1]}'s previous failures: if {agent_name[1]}'s few previous actions failed, description of what failed,\n",
    "Robots' open subtasks: list of subtasks  supposed to carry out to finish the task. If no plan has been already created, this will be None.\n",
    "Robots' completed subtasks: list of subtasks the robots have already completed. If no subtasks have been completed, this will be None.\n",
    "Robots' subtask: description of the subtasks the robots were trying to complete in the previous step,\n",
    "Robots' combined memory: description of robot's combined memory}}\n",
    "\n",
    "First of all you are supposed to reason over the image inputs, the robots' observations, previous actions, previous failures, previous memory, subtasks and the available actions the robots can perform, and think step by step and then output the following things:\n",
    "* Failure reason: If any robot's previous action failed, think why it failed and output the reason for failure. If the previous action was successful, output \"None\".\n",
    "* Memory: Whatever important information about the scene you think you should remember for the future as a memory. Remember that this memory will be used in future steps to carry out the task. So, you should not include information that is not relevant to the task. You can also include information that is already present in its memory if you think it might be useful in the future.\n",
    "* Reason: The reasoning for what each robot is supposed to do next\n",
    "* Subtask: The subtask each robot should currently try to solve, choose this from the list of open subtasks.\n",
    "* Action: The actions the robots are supposed to take just in the next step such that they make progress towards completing the task. Make sure that this suggested actions make these robots more efficient in completing the task as compared only one agent solving the task. \n",
    "Your output should just be in the form of a python dictionary as shown below.\n",
    "Example output: \n",
    "{{\"failure reason\": \"Bob failed to pickup lettuce earlier because it was far away from it so it needs to navigate closer to it\", \n",
    "\"memory\": \"Alice saw a refrigerator next to the table when Alice was at co-ordinates (0.5, -1.5) and was facing north. Bob saw a lettuce on the countertop next to the refrigerator when Bob was at co-ordinates (1, 0) and was facing north.\", \n",
    "\"reason\": \"Alice can see an an apple and can pick it and place it on the countertop to progress towards completing the task. Bob can see the lettuce but is far away from it to pick it up so it should go closer to it before picking it up.\", \n",
    "\"subtask\": \"Alice is currently trying to pick up the apple and Bob is currently trying to get navigate to the lettuce\", \n",
    "\"{agent_name[0]}'s action\": pick up Apple_1, \n",
    "\"{agent_name[1]}'s action\": navigate to Lettuce_1}}\n",
    "Note that the output should just be a dictionary similar to the example output.\n",
    "\n",
    "### Important Notes ###\n",
    "* The robots can hold only one object at a time.\n",
    "For example: If {agent_name} is holding an apple, it cannot pick up another object until it puts the apple down.\n",
    "* Even if the robot can see objects, it might not be able to interact with them if they are too far away. Hence you will need to make the robot move closer to the objects they want to interact with.\n",
    "For example: An action like \"pick up <object_id>\" is feasible only if robot can see the object and is close enough to it. So you will have to move closer to it before you can pick it up.\n",
    "So if a particular action fails, you will have to choose a different action for the robot. \n",
    "* If you open an object, please ensure that you close it before you move to a different place.\n",
    "* Opening object like drawers, cabinets, fridge can block the path of the robot. So open objects only when you think it is necessary.\n",
    "\n",
    "* NOTE: DO NOT OUTPUT ANYTHING EXTRA OTHER THAN WHAT HAS BEEN SPECIFIED\n",
    "Let's work this out in a step by step way to be sure we have the right answer.\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize the environment with the config and the task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting Config Task to 0, with task name 'Put the bread, lettuce, and tomato in the fridge'. Floorplan unknown.\n",
      "Setting Config FloorPlan to 0, with name 'FloorPlan1'\n",
      "Setting Config FloorPlan to 1, with name 'FloorPlan2'\n",
      "Setting Config FloorPlan to 2, with name 'FloorPlan3'\n",
      "Setting Config FloorPlan to 3, with name 'FloorPlan4'\n",
      "Setting Config FloorPlan to 4, with name 'FloorPlan5'\n",
      "Setting Config Task to 1, with task name 'Put the computer, book, and remotecontrol on the sofa'. Floorplan unknown.\n",
      "Setting Config FloorPlan to 0, with name 'FloorPlan201'\n",
      "Setting Config FloorPlan to 1, with name 'FloorPlan202'\n",
      "Setting Config FloorPlan to 2, with name 'FloorPlan203'\n",
      "Setting Config FloorPlan to 3, with name 'FloorPlan209'\n",
      "Setting Config FloorPlan to 4, with name 'FloorPlan224'\n",
      "Setting Config Task to 2, with task name 'Put the butter knife, bowl, and mug on the countertop'. Floorplan unknown.\n",
      "Setting Config FloorPlan to 0, with name 'FloorPlan1'\n",
      "Setting Config FloorPlan to 1, with name 'FloorPlan2'\n",
      "Setting Config FloorPlan to 2, with name 'FloorPlan3'\n",
      "Setting Config FloorPlan to 3, with name 'FloorPlan4'\n",
      "Setting Config FloorPlan to 4, with name 'FloorPlan5'\n",
      "Setting Config Task to 3, with task name 'Put the plate, mug, and bowl in the fridge'. Floorplan unknown.\n",
      "Setting Config FloorPlan to 0, with name 'FloorPlan1'\n",
      "Setting Config FloorPlan to 1, with name 'FloorPlan2'\n",
      "Setting Config FloorPlan to 2, with name 'FloorPlan3'\n",
      "Setting Config FloorPlan to 3, with name 'FloorPlan4'\n",
      "Setting Config FloorPlan to 4, with name 'FloorPlan5'\n",
      "Setting Config Task to 4, with task name 'Put the remotecontrol, keys, and watch in the box'. Floorplan unknown.\n",
      "Setting Config FloorPlan to 0, with name 'FloorPlan201'\n",
      "Setting Config FloorPlan to 1, with name 'FloorPlan202'\n",
      "Setting Config FloorPlan to 2, with name 'FloorPlan207'\n",
      "Setting Config FloorPlan to 3, with name 'FloorPlan209'\n",
      "Setting Config FloorPlan to 4, with name 'FloorPlan215'\n",
      "Setting Config FloorPlan to 5, with name 'FloorPlan226'\n",
      "Setting Config FloorPlan to 6, with name 'FloorPlan228'\n",
      "Setting Config Task to 5, with task name 'Put the vase, tissue box, and remote control on the table'. Floorplan unknown.\n",
      "Setting Config FloorPlan to 0, with name 'FloorPlan201'\n",
      "Setting Config FloorPlan to 1, with name 'FloorPlan203'\n",
      "Setting Config FloorPlan to 2, with name 'FloorPlan216'\n",
      "Setting Config FloorPlan to 3, with name 'FloorPlan219'\n",
      "Setting Config FloorPlan to 4, with name 'FloorPlan229'\n",
      "Setting Config Task to 6, with task name 'Slice the bread, lettuce, tomato, and egg'. Floorplan unknown.\n",
      "Setting Config FloorPlan to 0, with name 'FloorPlan1'\n",
      "Setting Config FloorPlan to 1, with name 'FloorPlan2'\n",
      "Setting Config FloorPlan to 2, with name 'FloorPlan3'\n",
      "Setting Config FloorPlan to 3, with name 'FloorPlan4'\n",
      "Setting Config FloorPlan to 4, with name 'FloorPlan5'\n",
      "Setting Config Task to 7, with task name 'Turn off the faucet and light if either is on'. Floorplan unknown.\n",
      "Setting Config FloorPlan to 0, with name 'FloorPlan1'\n",
      "Setting Config FloorPlan to 1, with name 'FloorPlan2'\n",
      "Setting Config FloorPlan to 2, with name 'FloorPlan3'\n",
      "Setting Config FloorPlan to 3, with name 'FloorPlan4'\n",
      "Setting Config FloorPlan to 4, with name 'FloorPlan5'\n",
      "Setting Config Task to 8, with task name 'Wash the bowl, mug, pot, and pan'. Floorplan unknown.\n",
      "Setting Config FloorPlan to 0, with name 'FloorPlan1'\n",
      "Setting Config FloorPlan to 1, with name 'FloorPlan2'\n",
      "Setting Config FloorPlan to 2, with name 'FloorPlan3'\n",
      "Setting Config FloorPlan to 3, with name 'FloorPlan4'\n",
      "Setting Config FloorPlan to 4, with name 'FloorPlan5'\n"
     ]
    }
   ],
   "source": [
    "from AI2Thor.baselines.utils.auto_config import AutoConfig\n",
    "\n",
    "auto=AutoConfig()\n",
    "amt_tasks=auto.get_amt_tasks()\n",
    "for task_index in range(amt_tasks):\n",
    "\n",
    "    auto.set_task(task_index) # set task index\n",
    "\n",
    "    amt_floorplans=auto.get_amt_floorplans(task_index)\n",
    "    for fp_index in range(amt_floorplans):\n",
    "        auto.set_floorplan(fp_index) # set floorplan\n",
    "        timeout=auto.get_task_timeout()\n",
    "        cf = auto.config()\n",
    "        # print(auto.task_string(), timeout, '\\n')\n",
    "        #print(cf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________\n",
      "Pre-initializing environment with closest task: 1_put_bread_lettuce_tomato_fridge\n",
      "__________________________________________________\n",
      "__________________________________________________\n",
      "Subtasks to complete:\n",
      "NavigateTo(Bread)\n",
      "PickupObject(Bread)\n",
      "NavigateTo(Fridge, Bread)\n",
      "PutObject(Fridge, Bread)\n",
      "NavigateTo(Tomato)\n",
      "PickupObject(Tomato)\n",
      "NavigateTo(Fridge, Tomato)\n",
      "PutObject(Fridge, Tomato)\n",
      "NavigateTo(Lettuce)\n",
      "PickupObject(Lettuce)\n",
      "NavigateTo(Fridge, Lettuce)\n",
      "PutObject(Fridge, Lettuce)\n",
      "OpenObject(Fridge)\n",
      "CloseObject(Fridge)\n",
      "__________________________________________________\n",
      "Preinitializing the environment\n",
      "__________________________________________________\n"
     ]
    }
   ],
   "source": [
    "env = AI2ThorEnv(config)\n",
    "# task = 'Put the butter knife, bowl, and mug in the countertop'\n",
    "task = 'put bread, lettuce, and tomato in fridge'\n",
    "d = env.reset(task=task)\n",
    "# d = env.step(['Move(Ahead)', ])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/marinatenhave/uropmultiagent/SayCanPlaygroundOrig/AI2Thor/baselines/utils/logging.py:65: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
      "  self.df = pd.concat([self.df, row])\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for /: 'str' and 'str'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 7\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m# random template - step everytime that you complete one loop (see act llm jupyter file notebook)\u001b[39;00m\n\u001b[1;32m      6\u001b[0m logger\u001b[38;5;241m.\u001b[39mlog_step(step\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, preaction\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnill\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnill\u001b[39m\u001b[38;5;124m'\u001b[39m], action\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnill\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnill\u001b[39m\u001b[38;5;124m'\u001b[39m], success\u001b[38;5;241m=\u001b[39m[\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;28;01mTrue\u001b[39;00m], coverage\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m.1\u001b[39m, transport_rate\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m.1\u001b[39m, finished\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m----> 7\u001b[0m \u001b[43mlogger\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msummarize\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/uropmultiagent/SayCanPlaygroundOrig/AI2Thor/baselines/utils/logging.py:260\u001b[0m, in \u001b[0;36mLogger.summarize\u001b[0;34m(self, num_agents)\u001b[0m\n\u001b[1;32m    257\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetrics\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msuccess_rate\u001b[39m\u001b[38;5;124m'\u001b[39m : sr, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtransport_rate\u001b[39m\u001b[38;5;124m'\u001b[39m : tr, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcoverage\u001b[39m\u001b[38;5;124m'\u001b[39m : c, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124maverage_steps\u001b[39m\u001b[38;5;124m'\u001b[39m : st, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbalance\u001b[39m\u001b[38;5;124m'\u001b[39m : bl}\n\u001b[1;32m    258\u001b[0m main_dict\u001b[38;5;241m=\u001b[39m{}\n\u001b[0;32m--> 260\u001b[0m tsk_dirs\u001b[38;5;241m=\u001b[39mglob\u001b[38;5;241m.\u001b[39mglob(\u001b[38;5;28mstr\u001b[39m( \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbaseline_path\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m/\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mactions\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m*\u001b[39m\u001b[38;5;124m\"\u001b[39m) ))\n\u001b[1;32m    261\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m tsk_dir \u001b[38;5;129;01min\u001b[39;00m tsk_dirs:\n\u001b[1;32m    262\u001b[0m     tsk_dict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mdict\u001b[39m([(k, []) \u001b[38;5;28;01mfor\u001b[39;00m k \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetrics\u001b[38;5;241m.\u001b[39mkeys()])\n",
      "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for /: 'str' and 'str'"
     ]
    }
   ],
   "source": [
    "# logger\n",
    "from AI2Thor.baselines.utils.logging import Logger\n",
    "baseline_name = \"llamar\"\n",
    "logger=Logger(baseline_name, env, 'llamar')\n",
    "# random template - step everytime that you complete one loop (see act llm jupyter file notebook)\n",
    "logger.log_step(step=1, preaction=['nill','nill'], action=['nill', 'nill'], success=[True, True], coverage=.1, transport_rate=.1, finished=False)\n",
    "logger.summarize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "log_step() missing 1 required positional argument: 'preaction'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[17], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mlogger\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlog_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstep\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maction\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mnill\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mnill\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msuccess\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcoverage\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m.1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtransport_rate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m.1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfinished\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mTypeError\u001b[0m: log_step() missing 1 required positional argument: 'preaction'"
     ]
    }
   ],
   "source": [
    "logger.log_step(step=1, preaction=['nill', 'nill'], action=['nill', 'nill'], success=[True, True], coverage=.1, transport_rate=.1, finished=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'1_put_knife_bowl_mug_countertop': {'success_rate': [0],\n",
       "  'transport_rate': [0.8],\n",
       "  'coverage': [0.7],\n",
       "  'average_steps': [1]},\n",
       " '1_put_bread_lettuce_tomato_fridge': {'success_rate': [0, 1],\n",
       "  'transport_rate': [0.1, 0.9],\n",
       "  'coverage': [0.1, 0.3],\n",
       "  'average_steps': [1, 1]}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logger.summarize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'success_rate': 0.25,\n",
       " 'transport_rate': 0.65,\n",
       " 'coverage': 0.44999999999999996,\n",
       " 'average_steps': 1.0}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logger.get_overall_average(logger.summarize())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Step</th>\n",
       "      <th>Action</th>\n",
       "      <th>Success</th>\n",
       "      <th>Coverage</th>\n",
       "      <th>Transport Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[nill, nill]</td>\n",
       "      <td>[False, False]</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>[nill, nill]</td>\n",
       "      <td>[True, True]</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Step        Action         Success  Coverage  Transport Rate\n",
       "0    0  [nill, nill]  [False, False]       0.0             0.0\n",
       "0    1  [nill, nill]    [True, True]       0.7             0.8"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#logger.df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0. , 0.7])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#(logger.df['Coverage']).to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#env.checker.check_success()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'results/multiple_modules/*'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(logger.baseline_path / '*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['AlignOrientation(0,270,False)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Rotate(Right)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Rotate(Left)', 'Move(Ahead)', 'LookDown(30)', 'LookDown(30)', 'AlignOrientation(0,275,False)'] [True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True]\n",
      "['AlignOrientation(0,270,False)', 'Rotate(Left)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Move(Ahead)', 'Rotate(Left)', 'Move(Ahead)', 'Move(Ahead)', 'LookDown(30)', 'Rotate(Left)', 'AlignOrientation(0,15,False)'] [True, True]\n"
     ]
    }
   ],
   "source": [
    "d = env.step(['NavigateTo(Fridge_1)', 'NavigateTo(Tomato_1)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Code to load prompts into the openai api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: [0, 0]\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'agent_name' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 163\u001b[0m\n\u001b[1;32m    159\u001b[0m         repeat_str \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m reason_str\n\u001b[1;32m    161\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m repeated_action, repeat_str\n\u001b[0;32m--> 163\u001b[0m \u001b[43mprint_relevant_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minput_dict\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[7], line 102\u001b[0m, in \u001b[0;36mprint_relevant_info\u001b[0;34m(input_dict)\u001b[0m\n\u001b[1;32m    100\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mprint_relevant_info\u001b[39m(input_dict):\n\u001b[1;32m    101\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mStep: \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mstr\u001b[39m(env\u001b[38;5;241m.\u001b[39mstep_num))\n\u001b[0;32m--> 102\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00magent_name[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    103\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mState: \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m input_dict[\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00magent_name[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms state\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[1;32m    104\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPrevious action: \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m input_dict[\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00magent_name[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms previous action\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n",
      "\u001b[0;31mNameError\u001b[0m: name 'agent_name' is not defined"
     ]
    }
   ],
   "source": [
    "previous_action = []*config.num_agents\n",
    "previous_success = [True, True] # initialize with True\n",
    "def encode_image(image_path:str):\n",
    "    with open(image_path, \"rb\") as image_file:\n",
    "        return base64.b64encode(image_file.read()).decode(\"utf-8\")\n",
    "    \n",
    "def prepare_prompt(module_name:str, addendum:str):\n",
    "    \"\"\"module_name: str \n",
    "        choose from planner, verifier, action\n",
    "    \"\"\"\n",
    "    # Choose the appropriate prompt based on what module is being called\n",
    "    # user_prompt = convert_dict_to_string(env.input_dict)\n",
    "    if module_name == \"action\":\n",
    "        system_prompt = ACTION_PROMPT\n",
    "        user_prompt = convert_dict_to_string(env.get_action_llm_input())\n",
    "    elif module_name == \"planner\":\n",
    "        system_prompt = PLANNER_PROMPT\n",
    "        user_prompt = convert_dict_to_string(env.get_planner_llm_input())\n",
    "    elif module_name == \"verifier\":\n",
    "        system_prompt = VERIFIER_PROMPT\n",
    "        user_prompt = convert_dict_to_string(env.get_verifier_llm_input())\n",
    "    user_prompt += addendum\n",
    "    return system_prompt, user_prompt\n",
    "\n",
    "def prepare_payload(module_name:str, addendum:str=\"\"):\n",
    "    \"\"\"# payload consists of \n",
    "    * the image from each agent's perspective\n",
    "    * the system prompt (which is constant)\n",
    "    * the user prompt (which changes based on the state)\n",
    "    This is then sent to the openai api to get the response (action or plan or verification of the plan)\n",
    "    \"\"\"\n",
    "    system_prompt, user_prompt = prepare_prompt(module_name, addendum)\n",
    "    base64_image = []\n",
    "    image_path = env.get_frame(0)\n",
    "    base64_image.append(encode_image(image_path))\n",
    "    image_path = env.get_frame(1)\n",
    "    base64_image.append(encode_image(image_path))\n",
    "    payload = {\n",
    "        \"model\": \"gpt-4-vision-preview\",\n",
    "        \"messages\": [\n",
    "            {\n",
    "                \"role\": \"system\",\n",
    "                \"content\": [\n",
    "                    {\"type\": \"text\", \"text\": system_prompt},\n",
    "                ],\n",
    "            },\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": [\n",
    "                    {\"type\": \"text\", \"text\": user_prompt},\n",
    "                    {\n",
    "                        \"type\": \"image_url\",\n",
    "                        \"image_url\": {\"url\": f\"data:image/jpeg;base64,{base64_image[0]}\"},\n",
    "                    },\n",
    "                    {\n",
    "                        \"type\": \"image_url\",\n",
    "                        \"image_url\": {\"url\": f\"data:image/jpeg;base64,{base64_image[1]}\"},\n",
    "                    },\n",
    "                ],\n",
    "            }\n",
    "        ],\n",
    "        \"max_tokens\": 1000,\n",
    "        \"temperature\": config.temperature,\n",
    "    }\n",
    "    return payload\n",
    "\n",
    "\n",
    "def get_action(response):\n",
    "    response_dict = response.json()\n",
    "    # convert the string to a dict\n",
    "    # json_acceptable_string = response_dict[\"choices\"][0][\"message\"][\"content\"].replace(\"'\", \"\\\"\").replace(\"\\n\", \"\").replace(\"json\", \"\").replace(\"`\", \"\")\n",
    "    output = response_dict[\"choices\"][0][\"message\"][\"content\"]\n",
    "    json_match = re.search(r'```json(.*?)```', output, re.DOTALL)\n",
    "    python_match = re.search(r'```python(.*?)```', output, re.DOTALL)\n",
    "    tilde_match = re.search(r'```(.*?)```', output, re.DOTALL)\n",
    "    if json_match:\n",
    "        print(\"Got JSON TYPE OUTPUT\")\n",
    "        json_data = json_match.group(1)\n",
    "    elif python_match:\n",
    "        print(\"Got JSON TYPE OUTPUT\")\n",
    "        json_data = python_match.group(1)\n",
    "    elif tilde_match:\n",
    "        print(\"Got JSON TYPE OUTPUT\")\n",
    "        json_data = tilde_match.group(1)\n",
    "    else:\n",
    "        print(\"Got NORMAL TYPE OUTPUT\")\n",
    "        json_data = output\n",
    "    print(json_data)\n",
    "    out_dict = json.loads(json_data)\n",
    "    return out_dict\n",
    "\n",
    "def get_gpt_response(action_or_planner:str, addendum:str=\"\"):\n",
    "    payload = prepare_payload(action_or_planner, addendum)\n",
    "    response = requests.post(\n",
    "    \"https://api.openai.com/v1/chat/completions\", headers=headers, json=payload\n",
    ")\n",
    "    return response\n",
    "\n",
    "\n",
    "def print_relevant_info(input_dict):\n",
    "    print(\"Step: \" + str(env.step_num))\n",
    "    print(f\"{agent_name[0]}\")\n",
    "    print(\"State: \" + input_dict[f\"{agent_name[0]}'s state\"])\n",
    "    print(\"Previous action: \" + input_dict[f\"{agent_name[0]}'s previous action\"])\n",
    "    print(\"Previous failures: \" + input_dict[f\"{agent_name[0]}'s previous failures\"])\n",
    "    print(f\"{agent_name[1]}\")\n",
    "    print(\"State: \" + input_dict[f\"{agent_name[1]}'s state\"])\n",
    "    print(\"Previous action: \" + input_dict[f\"{agent_name[1]}'s previous action\"])\n",
    "    print(\"Previous failures: \" + input_dict[f\"{agent_name[1]}'s previous failures\"])\n",
    "    if config.use_shared_subtask:\n",
    "        print(\"Subtask: \", input_dict[f\"Robots' subtasks\"])\n",
    "    elif config.use_separate_subtask:\n",
    "        print(f\"{agent_name[0]}'s subtask: \" + input_dict[f\"{agent_name[0]}'s subtask\"])\n",
    "        print(f\"{agent_name[1]}'s subtask: \" + input_dict[f\"{agent_name[1]}'s subtask\"])\n",
    "    if config.use_shared_memory:\n",
    "        print(\"Memory: \" + input_dict[\"Robots' combined memory\"])\n",
    "    elif config.use_separate_memory:\n",
    "        print(f\"{agent_name[0]}'s memory: \" + input_dict[f\"{agent_name[0]}'s memory\"])\n",
    "        print(f\"{agent_name[1]}'s memory: \" + input_dict[f\"{agent_name[1]}'s memory\"])\n",
    "\n",
    "def action_checker(actions):\n",
    "    \"\"\"\n",
    "    Get closest valid action\n",
    "\n",
    "    The action output from the model is in natural language.\n",
    "    This function will find the env feasible action which has the closest embedding \n",
    "    to the natural language action output from the model.\n",
    "    Eg: \"pick up the apple\" -> \"PickupObject(Apple_1)\"\n",
    "    \"\"\"\n",
    "    checked_actions = []\n",
    "    for act in actions:\n",
    "        act = get_closest_feasible_action(act)\n",
    "        action_type = act.split(\"(\")[0]\n",
    "        if action_type in ['PickupObject', 'PutObject', 'OpenObject', 'CloseObject', 'SliceObject', 'NavigateTo']:\n",
    "            act = get_closest_object_id(act, env.object_dict)\n",
    "        checked_actions.append(act)\n",
    "    return checked_actions\n",
    "\n",
    "def check_repeat_actions(actions):\n",
    "    agent_name = env.agent_names\n",
    "    repeated_action = False\n",
    "    repeat_str = \"\"\n",
    "    repeat_agent_names = [] # to store names of agents who repeated actions\n",
    "    for i in range(len(actions)):\n",
    "        if actions[i] == previous_action[i]:\n",
    "            repeated_action = True\n",
    "            success_str = \"successful\" if previous_success[i] else \"unsuccessful\"\n",
    "            repeat_str += f\"\\nYou predicted {agent_name[i]}'s action as {actions[i]} which is the same as the previous action {agent_name[i]} took. {agent_name[i]} was {success_str} in carrying out that action.\"\n",
    "            repeat_agent_names.append(agent_name[i])\n",
    "    \n",
    "    if repeated_action:\n",
    "        reason_str = f\"\\nPlease reason over the image inputs and the observations to suggest a different action for \"\n",
    "        for i in range(len(repeat_agent_names)):\n",
    "            if i == 0:\n",
    "                reason_str += f\"{repeat_agent_names[i]}\"\n",
    "            else:\n",
    "                reason_str += f\" and {repeat_agent_names[i]}\"\n",
    "        reason_str += \".\"\n",
    "        repeat_str += reason_str\n",
    "    \n",
    "    return repeated_action, repeat_str\n",
    "        \n",
    "print_relevant_info(env.input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "[encoded data, 131324 characters]",
      "image/png": "[encoded data, 1752868 characters]",
      "text/plain": [
       "<PIL.Image.Image image mode=RGB size=1000x1000>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from PIL import Image\n",
    "\n",
    "def frame(agent_id=0):\n",
    "    return (Image.fromarray(env.controller.last_event.events[agent_id].frame))\n",
    "\n",
    "frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Where is all the information about the environment stored?\n",
    "Check `print(convert_dict_to_string(env.input_dict)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(convert_dict_to_string(env.input_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(convert_dict_to_string(env.get_action_llm_input()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the inital plan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Got JSON TYPE OUTPUT\n",
      "\n",
      "{\n",
      "  \"reason\": \"The mug has been located, but the butter knife and bowl still need to be found. The mug can be moved to the designated countertop area, and the robots need to continue searching for the other items.\",\n",
      "  \"plan\": [\n",
      "    \"transport Mug_1 to the designated countertop area\",\n",
      "    \"locate the butter knife\",\n",
      "    \"transport the butter knife to the designated countertop area\",\n",
      "    \"locate the bowl\",\n",
      "    \"transport the bowl to the designated countertop area\"\n",
      "  ]\n",
      "}\n",
      "\n",
      "{'reason': 'The mug has been located, but the butter knife and bowl still need to be found. The mug can be moved to the designated countertop area, and the robots need to continue searching for the other items.', 'plan': ['transport Mug_1 to the designated countertop area', 'locate the butter knife', 'transport the butter knife to the designated countertop area', 'locate the bowl', 'transport the bowl to the designated countertop area']}\n"
     ]
    }
   ],
   "source": [
    "response = get_gpt_response(action_or_planner=\"planner\")\n",
    "outdict = get_action(response)\n",
    "print(outdict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Update the plan in the env.input_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_plan(open_subtasks, completed_subtasks):\n",
    "    env.open_subtasks = open_subtasks\n",
    "    env.closed_subtasks = completed_subtasks\n",
    "    env.input_dict[\"Robots' open subtasks\"] = env.open_subtasks\n",
    "    env.input_dict[\"Robots' completed subtasks\"] = env.closed_subtasks\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Actor module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Task': 'Put the butter knife, bowl, and mug in the countertop',\n",
       " \"Alice's observation\": \"['Stove_1', 'Drawer_1', 'Cabinet_1', 'StoveBurner_1', 'Cabinet_2', 'Cabinet_3', 'StoveKnob_1', 'Sink_1', 'Cabinet_4', 'Cabinet_5', 'Mug_1', 'StoveBurner_2', 'StoveKnob_2', 'Cabinet_6', 'UpperCabinets_1', 'StoveBurner_3', 'StoveKnob_3', 'Window_1', 'StandardCounterHeightWidth_1', 'Floor_1', 'Wall_1', 'StandardIslandHeight_1', 'Stove_2', 'Window_2', 'StoveKnob_4', 'Cabinet_7', 'Ceiling_1', 'StoveBurner_4', 'CeilingLight_1', 'CeilingLight_2', 'Faucet_1', 'CeilingLight_3', 'Dishwasher_1', 'LightFixture_1', 'LightFixture2_1', 'Pot_1', 'SaltShaker_1', 'Fridge_1', 'SoapBottle_1', 'HousePlant_1', 'Pan_1', 'Tomato_1', 'PepperShaker_1', 'Apple_1', 'Toaster_1', 'Lettuce_1', 'Spatula_1', 'Microwave_1', 'Potato_1', 'Bread_1', 'CoffeeMachine_1', 'PaperTowelRoll_1', 'Stool_1', 'CounterTop_1', 'CounterTop_2', 'CounterTop_3']\",\n",
       " \"Alice's state\": 'I am at co-ordinates: (1.5, -1.5) facing west and I am holding nothing',\n",
       " \"Alice's previous action\": 'I have not taken any action yet',\n",
       " \"Alice's previous failures\": 'None',\n",
       " \"Bob's observation\": \"['Cabinet_2', 'Sink_1', 'UpperCabinets_1', 'StandardCounterHeightWidth_1', 'Wall_1', 'Window_2', 'Fridge_1', 'Toaster_1', 'CoffeeMachine_1', 'PaperTowelRoll_1', 'CounterTop_1']\",\n",
       " \"Bob's state\": 'I am at co-ordinates: (-1.0, 0.0) facing west and I am holding nothing',\n",
       " \"Bob's previous action\": 'I have not taken any action yet',\n",
       " \"Bob's previous failures\": 'None',\n",
       " \"Robots' open subtasks\": 'None',\n",
       " \"Robots' completed subtasks\": 'None',\n",
       " \"Robots' combined memory\": 'Memory is empty because no actions taken yet'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.get_action_llm_input()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Task': 'Put the butter knife, bowl, and mug in the countertop',\n",
       " \"Alice's observation\": \"['Stove_1', 'Drawer_1', 'Cabinet_1', 'StoveBurner_1', 'Cabinet_2', 'Cabinet_3', 'StoveKnob_1', 'Sink_1', 'Cabinet_4', 'Cabinet_5', 'Mug_1', 'StoveBurner_2', 'StoveKnob_2', 'Cabinet_6', 'UpperCabinets_1', 'StoveBurner_3', 'StoveKnob_3', 'Window_1', 'StandardCounterHeightWidth_1', 'Floor_1', 'Wall_1', 'StandardIslandHeight_1', 'Stove_2', 'Window_2', 'StoveKnob_4', 'Cabinet_7', 'Ceiling_1', 'StoveBurner_4', 'CeilingLight_1', 'CeilingLight_2', 'Faucet_1', 'CeilingLight_3', 'Dishwasher_1', 'LightFixture_1', 'LightFixture2_1', 'Pot_1', 'SaltShaker_1', 'Fridge_1', 'SoapBottle_1', 'HousePlant_1', 'Pan_1', 'Tomato_1', 'PepperShaker_1', 'Apple_1', 'Toaster_1', 'Lettuce_1', 'Spatula_1', 'Microwave_1', 'Potato_1', 'Bread_1', 'CoffeeMachine_1', 'PaperTowelRoll_1', 'Stool_1', 'CounterTop_1', 'CounterTop_2', 'CounterTop_3']\",\n",
       " \"Alice's state\": 'I am at co-ordinates: (1.5, -1.5) facing west and I am holding nothing',\n",
       " \"Alice's previous action\": 'I have not taken any action yet',\n",
       " \"Alice's previous failures\": 'None',\n",
       " \"Bob's observation\": \"['Cabinet_2', 'Sink_1', 'UpperCabinets_1', 'StandardCounterHeightWidth_1', 'Wall_1', 'Window_2', 'Fridge_1', 'Toaster_1', 'CoffeeMachine_1', 'PaperTowelRoll_1', 'CounterTop_1']\",\n",
       " \"Bob's state\": 'I am at co-ordinates: (-1.0, 0.0) facing west and I am holding nothing',\n",
       " \"Bob's previous action\": 'I have not taken any action yet',\n",
       " \"Bob's previous failures\": 'None',\n",
       " \"Robots' open subtasks\": ['transport Mug_1 to the designated countertop area',\n",
       "  'locate the butter knife',\n",
       "  'transport the butter knife to the designated countertop area',\n",
       "  'locate the bowl',\n",
       "  'transport the bowl to the designated countertop area'],\n",
       " \"Robots' completed subtasks\": None,\n",
       " \"Robots' combined memory\": 'Memory is empty because no actions taken yet'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# env.open_subtasks = outdict[\"plan\"]\n",
    "# env.closed_subtasks = None\n",
    "# env.input_dict[\"Robots' open subtasks\"] = env.open_subtasks\n",
    "# env.input_dict[\"Robots' completed subtasks\"] = env.closed_subtasks\n",
    "update_plan(outdict[\"plan\"], None)\n",
    "env.get_action_llm_input()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_action_llm_output(outdict):\n",
    "    action = []\n",
    "    action.append(outdict[f\"{agent_name[0]}'s action\"])\n",
    "    action.append(outdict[f\"{agent_name[1]}'s action\"])\n",
    "    reason = outdict[\"reason\"]\n",
    "    subtask = outdict[\"subtask\"]\n",
    "    memory = outdict[\"memory\"]\n",
    "    failure_reason = outdict[\"failure reason\"]\n",
    "    return action, reason, subtask, memory, failure_reason\n",
    "\n",
    "def print_stuff(action, reason, subtask, memory, failure_reason):\n",
    "    print(\"Step: \" + str(env.step_num))\n",
    "    print(f'Action: {action}')\n",
    "    action = action_checker(action)\n",
    "    print(f'Closest Action: {action}')\n",
    "    print(f'Reason: {reason}')\n",
    "    print(f'Subtask: {subtask}')\n",
    "    print(f'Memory: {memory}')\n",
    "    print(f'Failure Reason: {failure_reason}')\n",
    "    return action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Got NORMAL TYPE OUTPUT\n",
      "Based on the information provided and the images, we can derive the following actions for Alice and Bob to complete their tasks efficiently:\n",
      "\n",
      "Alice can see the Mug_1 which is one of the items that needs to be transported to the designated countertop area. Since Alice is not holding anything, she can pick up the mug.\n",
      "\n",
      "Bob's current observation does not include the butter knife or bowl, which are part of the open subtasks. Bob needs to locate these items. Since Bob is facing west and is near the countertop and cabinets, Bob can start by looking for the butter knife and bowl in the nearby cabinets. Bob could either navigate or rotate to get a better view of the area.\n",
      "\n",
      "Here's the plan for the next actions:\n",
      "\n",
      "{\"failure reason\": \"None\", \n",
      "\"memory\": \"Alice spotted Mug_1 on the countertop. Bob is near the counter and cabinets and needs to locate the butter knife and bowl.\", \n",
      "\"reason\": \"Alice can pick up Mug_1 since it is within her view and she is not holding anything. Bob needs to locate the butter knife and bowl, so he should continue exploring the kitchen, possibly by opening nearby cabinets.\", \n",
      "\"subtask\": \"Alice is currently trying to transport Mug_1 to the designated countertop area, and Bob is trying to locate the butter knife and bowl.\", \n",
      "\"Alice's action\": \"pick up Mug_1\", \n",
      "\"Bob's action\": \"rotate in Right direction\"}\n"
     ]
    },
    {
     "ename": "JSONDecodeError",
     "evalue": "Expecting value: line 1 column 1 (char 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[15], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m response \u001b[38;5;241m=\u001b[39m get_gpt_response(action_or_planner\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124maction\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m outdict \u001b[38;5;241m=\u001b[39m \u001b[43mget_action\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      3\u001b[0m action, reason, subtask, memory, failure_reason \u001b[38;5;241m=\u001b[39m process_action_llm_output(outdict)\n\u001b[1;32m      4\u001b[0m action \u001b[38;5;241m=\u001b[39m print_stuff(action, reason, subtask, memory, failure_reason)\n",
      "Cell \u001b[0;32mIn[6], line 89\u001b[0m, in \u001b[0;36mget_action\u001b[0;34m(response)\u001b[0m\n\u001b[1;32m     87\u001b[0m     json_data \u001b[38;5;241m=\u001b[39m output\n\u001b[1;32m     88\u001b[0m \u001b[38;5;28mprint\u001b[39m(json_data)\n\u001b[0;32m---> 89\u001b[0m out_dict \u001b[38;5;241m=\u001b[39m \u001b[43mjson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloads\u001b[49m\u001b[43m(\u001b[49m\u001b[43mjson_data\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     90\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out_dict\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/urop1/lib/python3.9/json/__init__.py:346\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    341\u001b[0m     s \u001b[38;5;241m=\u001b[39m s\u001b[38;5;241m.\u001b[39mdecode(detect_encoding(s), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msurrogatepass\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m    343\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m    344\u001b[0m         parse_int \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m parse_float \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m    345\u001b[0m         parse_constant \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_pairs_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kw):\n\u001b[0;32m--> 346\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_decoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    348\u001b[0m     \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m JSONDecoder\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/urop1/lib/python3.9/json/decoder.py:337\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m    332\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode\u001b[39m(\u001b[38;5;28mself\u001b[39m, s, _w\u001b[38;5;241m=\u001b[39mWHITESPACE\u001b[38;5;241m.\u001b[39mmatch):\n\u001b[1;32m    333\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Return the Python representation of ``s`` (a ``str`` instance\u001b[39;00m\n\u001b[1;32m    334\u001b[0m \u001b[38;5;124;03m    containing a JSON document).\u001b[39;00m\n\u001b[1;32m    335\u001b[0m \n\u001b[1;32m    336\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 337\u001b[0m     obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraw_decode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43midx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_w\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mend\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    338\u001b[0m     end \u001b[38;5;241m=\u001b[39m _w(s, end)\u001b[38;5;241m.\u001b[39mend()\n\u001b[1;32m    339\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m end \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(s):\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/urop1/lib/python3.9/json/decoder.py:355\u001b[0m, in \u001b[0;36mJSONDecoder.raw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m    353\u001b[0m     obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscan_once(s, idx)\n\u001b[1;32m    354\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 355\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m JSONDecodeError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpecting value\u001b[39m\u001b[38;5;124m\"\u001b[39m, s, err\u001b[38;5;241m.\u001b[39mvalue) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    356\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj, end\n",
      "\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
     ]
    }
   ],
   "source": [
    "response = get_gpt_response(action_or_planner=\"action\")\n",
    "outdict = get_action(response)\n",
    "action, reason, subtask, memory, failure_reason = process_action_llm_output(outdict)\n",
    "action = print_stuff(action, reason, subtask, memory, failure_reason)\n",
    "# print(\"Step: \" + str(env.step_num))\n",
    "# print(f'Action: {action}')\n",
    "# action = action_checker(action)\n",
    "# print(f'Closest Action: {action}')\n",
    "# print(f'Reason: {reason}')\n",
    "# print(f'Subtask: {subtask}')\n",
    "# print(f'Memory: {memory}')\n",
    "# print(f'Failure Reason: {failure_reason}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Execute the action in the environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = append_row(df, env.step_num, action, reason, subtask, memory)\n",
    "if config.use_shared_subtask:\n",
    "    env.update_subtask(subtask, 0)\n",
    "if config.use_shared_memory:\n",
    "    env.update_memory(memory, 0)\n",
    "d1, successes = env.step(action)\n",
    "previous_action = action\n",
    "previous_success = successes\n",
    "print_relevant_info(env.input_dict)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Verify the action after execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.get_verifier_llm_input()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = get_gpt_response(action_or_planner=\"verifier\")\n",
    "outdict = get_action(response)\n",
    "outdict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.closed_subtasks = outdict[\"completed subtasks\"]\n",
    "if len(env.closed_subtasks) == 0:\n",
    "    env.closed_subtasks = None\n",
    "env.input_dict[\"Robots' completed subtasks\"] = env.closed_subtasks\n",
    "env.get_planner_llm_input()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = get_gpt_response(action_or_planner=\"planner\")\n",
    "outdict = get_action(response)\n",
    "outdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now repeat from update_plan() cell again.\n",
    "\n",
    "Don't run a for loop for executing the planner, actor, verifier cells. Rather run them manually for now until you get a reasonably successful completion of the task for a decent number of episodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "uropdinamo3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
