#!/bin/bash

# # bc
# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_bc_official_test1/model.pkl \
#                       model.seperate_policy=false model.seperate_target=false \
#                       evaluator.generation_kwargs.include_adv=false \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_bc_test1_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5000/step_rank

# # 10%bc
# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_pbc_official_test1/model_196607.pkl \
#                       model.seperate_policy=false model.seperate_target=false \
#                       evaluator.generation_kwargs.include_adv=false \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_pbc_test1_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5000/step_rank

# # 20%bc
# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_pbc_official_test2/model_196607.pkl \
#                       model.seperate_policy=false model.seperate_target=false \
#                       evaluator.generation_kwargs.include_adv=false \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_pbc_test2_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5000/step_rank

# # 30%bc
# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_pbc_official_test3/model_393215.pkl \
#                       model.seperate_policy=false model.seperate_target=false \
#                       evaluator.generation_kwargs.include_adv=false \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_pbc_test3_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5000/step_rank

# # IQL_tau=0.5
# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test1/model_131071.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=4.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test1_beta4_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5000/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test1/model_131071.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=8.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test1_beta8_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5000/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test1/model_131071.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=16.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test1_beta16_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5000/step_rank

# === BREAK ===

# # IQL_tau=0.7
# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test2/model_131071.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=4.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test2_beta4_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5001/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test2/model_131071.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=8.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test2_beta8_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5001/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test2/model_131071.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=16.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test2_beta16_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5001/step_rank

# # IQL_tau=0.8
# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test3/model_294911.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=4.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test3_beta4_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5001/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test3/model_294911.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=8.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test3_beta8_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5001/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test3/model_294911.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=16.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test3_beta16_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5001/step_rank

# # IQL_tau=0.9
# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test4/model_294911.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=4.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test4_beta4_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5001/step_rank

# === BREAK ===

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test4/model_294911.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=8.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test4_beta8_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5002/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_iql_official_test4/model_294911.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=16.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_iql_test4_beta16_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5002/step_rank

# # CQL_alpha=0.1
# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test1/model_163839.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=4.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test1_beta4_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5002/step_rank

# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test1/model_163839.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=8.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test1_beta8_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5002/step_rank

# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test1/model_163839.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=16.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test1_beta16_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5002/step_rank

# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test1/model_163839.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.include_logits=false \
#                       evaluator.generation_kwargs.adv_weight=1.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test1_betainf_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5002/step_rank

# # CQL_alpha=1.0
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test2/model_229375.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=4.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test2_beta4_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5002/step_rank

# # === BREAK ===

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test2/model_229375.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=8.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test2_beta8_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5003/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test2/model_229375.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=16.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test2_beta16_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5003/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test2/model_229375.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.include_logits=false \
#                       evaluator.generation_kwargs.adv_weight=1.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test2_betainf_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5003/step_rank

# # CQL_alpha=10.0
# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test3/model_262143.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=4.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test3_beta4_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5003/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test3/model_262143.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=8.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test3_beta8_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5003/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test3/model_262143.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.adv_weight=16.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test3_beta16_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5003/step_rank

# # done.
# python eval_policy.py model.load.checkpoint_path=outputs/visual_dialogue/visdial_hard_yn_cql_official_test3/model_262143.pkl \
#                       model.seperate_policy=true model.seperate_target=true \
#                       evaluator.generation_kwargs.include_adv=true \
#                       evaluator.generation_kwargs.include_logits=false \
#                       evaluator.generation_kwargs.adv_weight=1.0 \
#                       eval.log_save_path=outputs/visual_dialogue/evals/visdial_cql_test3_betainf_beam1_eval1.pkl \
#                       evaluator.env.url=http://localhost:5003/step_rank

# # === BREAK ===
