.PHONY: pytest validate-test-profile plot-delay

SHELL = bash

# Set profile PROFILE_OUTPUT to something to enable profiling
PYTHON = PYTHONPATH="$(shell pwd -P)" python3 $(shell if [[ ! -z "$(PROFILE_OUTPUT)" ]]; then echo "-m cProfile -o \"$(PROFILE_OUTPUT)\""; fi)

pytest:
	$(eval PYTHONPATH := $(shell pwd -P))
	PYTHONPATH="$(PYTHONPATH)" pytest -v test/



# An extreme version of the sparse delay, with 50% higher delay.
EXTREME_DELAY_ARG := markov(ord(15,1), ord(3,5,3,shift=22), [[124, 1], [1, 19]])
EXTREME_DELAY_HZN := 24
EXTREME_DELAY_TAG := ExtremeClogL1U23
EXTREME_ASSUMED_DELAY1 := 2
EXTREME_ASSUMED_DELAY2 := 24

EXTREME_RUN_PREFIX := benchmark-v3-tc4

# Same as before, but with a much lower chance of the system clogging up
# and more severe impact if the system clogs up. Now we assume two delays,
# delay 4 or delay 32.
# NOTE: This is a gilbert-elliot process (or Gilbert-Elliot Model)
# https://its.ntia.gov/publications/download/TM-23-565.pdf
EXTREME32_DELAY_ARG := markov(4, 32, [[249, 1], [1, 31]])
EXTREME32_DELAY_HZN := 32
EXTREME32_DELAY_TAG := ExtremeSparseL4U32
EXTREME32_ASSUMED_DELAY1 := 4
EXTREME32_ASSUMED_DELAY2 := 32

EXTREME32_RUN_PREFIX := benchmark-v3-tc7

# An MM1 Queue delay
MM1Q_DELAY_ARG := mm1queue(0.33, 0.75)
MM1Q_DELAY_HZN := 24
MM1Q_DELAY_TAG := MM1Queue_a033_s075
MM1Q_ASSUMED_DELAY1 := 4
MM1Q_ASSUMED_DELAY2 := 16

MM1Q_RUN_PREFIX := benchmark-v3-tc8


TEST_PROFILE ?= NONE

ifeq ($(TEST_PROFILE), extreme)
DELAY_ARG := $(EXTREME_DELAY_ARG)
DELAY_HZN := $(EXTREME_DELAY_HZN)
DELAY_TAG := $(EXTREME_DELAY_TAG)
ASSUMED_DELAY1 := $(EXTREME_ASSUMED_DELAY1)
ASSUMED_DELAY2 := $(EXTREME_ASSUMED_DELAY2)
RUN_PREFIX := $(EXTREME_RUN_PREFIX)
else ifeq ($(TEST_PROFILE), extreme32)
DELAY_ARG := $(EXTREME32_DELAY_ARG)
DELAY_HZN := $(EXTREME32_DELAY_HZN)
DELAY_TAG := $(EXTREME32_DELAY_TAG)
ASSUMED_DELAY1 := $(EXTREME32_ASSUMED_DELAY1)
ASSUMED_DELAY2 := $(EXTREME32_ASSUMED_DELAY2)
RUN_PREFIX := $(EXTREME32_RUN_PREFIX)
else ifeq ($(TEST_PROFILE), mm1q)
DELAY_ARG := $(MM1Q_DELAY_ARG)
DELAY_HZN := $(MM1Q_DELAY_HZN)
DELAY_TAG := $(MM1Q_DELAY_TAG)
ASSUMED_DELAY1 := $(MM1Q_ASSUMED_DELAY1)
ASSUMED_DELAY2 := $(MM1Q_ASSUMED_DELAY2)
RUN_PREFIX := $(MM1Q_RUN_PREFIX)
else ifneq ($(TEST_PROFILE),NONE)
$(error Invalid test profile: $(TEST_PROFILE))
endif


# Validate that the test profile has been set if running a test
validate-test-profile:
	@if [[ -z "$(DELAY_TAG)" ]]; then \
	    echo -e "\033[1;31mNo test profile specified. Specify it with the TEST_PROFILE variable\033[0m"; \
	    exit 1; \
	 fi


RUN_PARAMS := -v --no-resume --no-save "--delay=$(DELAY_TAG)::$(DELAY_ARG)" --horizon=$(DELAY_HZN)

# For testing implementations
#RUN_PARAMS := $(RUN_PARAMS) --trainer-itersteps=1000 --no-save
#RUN_PARAMS := $(RUN_PARAMS) --trainer-random-iterations=0
#RUN_PARAMS := $(RUN_PARAMS) --netdims=4,4 --msz-hid=4 --msz-rec=4

BASELINE_BPQL = $(PYTHON) example/baseline-bpql.py $(RUN_PARAMS)
BASELINE_SAC = $(PYTHON) example/baseline-sac.py $(RUN_PARAMS)


BPQLMDA_PARAMS = --msz-hid=256 \
                 --msz-rec=384 \
                 --mtw=16 \
                 --model-clamp-output \
                 --lr-model=1e-4
BASELINE_BPQLMDA = $(PYTHON) example/baseline-bpql-mda.py $(RUN_PARAMS) $(BPQLMDA_PARAMS)

BPQLMDA_HIGHDIM_PARAMS = --msz-hid=256 \
                         --msz-rec=512 \
                         --mtw=16 \
                         --lr-model=5e-5
BASELINE_BPQLMDA_HIGHDIM = $(PYTHON) example/baseline-bpql-mda.py $(RUN_PARAMS) $(BPQLMDA_HIGHDIM_PARAMS)

BASELINE_MBPAC = $(PYTHON) example/baseline-mbpac.py $(RUN_PARAMS) $(BPQLMDA_PARAMS)
BASELINE_MBPAC_HIGHDIM = $(PYTHON) example/baseline-mbpac.py $(RUN_PARAMS) $(BPQLMDA_HIGHDIM_PARAMS)

BASELINE_MFPAC_MULTIVAR = $(PYTHON) example/baseline-mfpac-multivar.py $(RUN_PARAMS)
BASELINE_MFPAC_TRANSFORMER = $(PYTHON) example/baseline-mfpac-transformer.py $(RUN_PARAMS)



print-baselines:
	@echo -e "\e[1m[BASELINE_BPQL]\e[0m"
	@echo '$(BASELINE_BPQL)'
	@echo -e "\e[1m[BASELINE_BPQLMDA]\e[0m"
	@echo '$(BASELINE_BPQLMDA)'
	@echo -e "\e[1m[BASELINE_SAC]\e[0m"
	@echo '$(BASELINE_SAC)'
	@echo -e "\e[1m[BASELINE_MBPAC]\e[0m"
	@echo '$(BASELINE_MBPAC)'
	@echo -e "\e[1m[BASELINE_MBPAC_HIGHDIM]\e[0m"
	@echo '$(BASELINE_MBPAC_HIGHDIM)'


TAG_BPQL1 := $(DELAY_TAG)-bpql-mem$(ASSUMED_DELAY1)
test-bpql1-%: validate-test-profile
	$(BASELINE_BPQL) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_BPQL1)" --assumed-delay=$(ASSUMED_DELAY1)

TAG_BPQL2 := $(DELAY_TAG)-bpql-mem$(ASSUMED_DELAY2)
test-bpql2-%: validate-test-profile
	$(BASELINE_BPQL) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_BPQL2)" --assumed-delay=$(ASSUMED_DELAY2)

TAG_SAC_AUGMENT1 := $(DELAY_TAG)-sac-aug-mem$(ASSUMED_DELAY1)
test-sacaugment1-%: validate-test-profile
	$(BASELINE_SAC) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_SAC_AUGMENT1)" --memorize=$(ASSUMED_DELAY1)

TAG_SAC_AUGMENT2 := $(DELAY_TAG)-sac-aug-mem$(ASSUMED_DELAY2)
test-sacaugment2-%: validate-test-profile
	$(BASELINE_SAC) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_SAC_AUGMENT2)" --memorize=$(ASSUMED_DELAY2)

TAG_SAC := $(DELAY_TAG)-sac
test-sac-%: validate-test-profile
	$(BASELINE_SAC) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_SAC)"

# New Benchmarks below
TAG_BPQLMDA1 := $(DELAY_TAG)-bpql-mda-mem$(ASSUMED_DELAY1)
TAG_BPQLMDA2 := $(DELAY_TAG)-bpql-mda-mem$(ASSUMED_DELAY2)
test-bpqlmda1-%: validate-test-profile
	$(BASELINE_BPQLMDA) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_BPQLMDA1)" --assumed-delay=$(ASSUMED_DELAY1)
test-bpqlmda2-%: validate-test-profile
	$(BASELINE_BPQLMDA) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_BPQLMDA2)" --assumed-delay=$(ASSUMED_DELAY2)

TAG_BPQLMDA1_HIGHDIM := $(DELAY_TAG)-bpql-mda-highdim-mem$(ASSUMED_DELAY1)
TAG_BPQLMDA2_HIGHDIM := $(DELAY_TAG)-bpql-mda-highdim-mem$(ASSUMED_DELAY2)
test-bpqlmda1_highdim-%: validate-test-profile
	$(BASELINE_BPQLMDA_HIGHDIM) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_BPQLMDA1_HIGHDIM)" --assumed-delay=$(ASSUMED_DELAY1)
test-bpqlmda2_highdim-%: validate-test-profile
	$(BASELINE_BPQLMDA_HIGHDIM) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_BPQLMDA2_HIGHDIM)" --assumed-delay=$(ASSUMED_DELAY2)

TAG_MBPAC_MEMDELAY := $(DELAY_TAG)-mbpac_memdelay
test-mbpac_memdelay-%: validate-test-profile
	$(BASELINE_MBPAC) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_MBPAC_MEMDELAY)" --delay-memorization

TAG_MBPAC_HIGHDIM_MEMDELAY := $(DELAY_TAG)-mbpac-highdim-memdelay
test-mbpac_highdim_memdelay-%: validate-test-profile
	$(BASELINE_MBPAC_HIGHDIM) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_MBPAC_HIGHDIM_MEMDELAY)" --delay-memorization

TAG_MFPAC_MULTIVAR := $(DELAY_TAG)-mfpac_multivar
test-mfpac_multivar-%: validate-test-profile
	$(BASELINE_MFPAC_MULTIVAR) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_MFPAC_MULTIVAR)" --delay-memorization

TAG_MFPAC_TRANSFORMER := $(DELAY_TAG)-mfpac_transformer
test-mfpac_transformer-%: validate-test-profile
	$(BASELINE_MFPAC_TRANSFORMER) $* --trainer-prefix="$(RUN_PREFIX)/$*/$(TAG_MFPAC_TRANSFORMER)" --delay-memorization

# We actually don't run VDPO or Dreamer in our framework, only export the results from their framework
TAG_VDPO1 := $(DELAY_TAG)-vdpo-mem$(ASSUMED_DELAY1)
TAG_VDPO2 := $(DELAY_TAG)-vdpo-mem$(ASSUMED_DELAY2)
TAG_DREAMER := $(DELAY_TAG)-dreamer
TAG_DCAC := $(DELAY_TAG)-dcac


# Use test-noisebpql-withpromille BPQLENV=e BPQLDELAY=d BPQLPROMILLE=p
test-noisebpql-withpromille:
	$(eval TRAINER_PREFIX := noise-eval-v2/$(BPQLENV)/bpql-noise_0.$(shell printf "%03d" $(BPQLPROMILLE))-delay_$(BPQLDELAY))
	$(eval CONFIG_ENV := $(shell \
		if [ "$(BPQLPROMILLE)" == "0" ]; then \
			echo "$(BPQLENV)"; \
		else \
			echo "noisepromille$(BPQLPROMILLE)-$(BPQLENV)"; \
		fi \
	 ))
	@if [ -z "$(BPQLENV)" ]; then echo -e "\e[1;31mError: Must specify environment with \e[1;21mBPQLENV\e[0m"; exit 1; fi
	@if [ -z "$(BPQLDELAY)" ]; then echo -e "\e[1;31mError: Must specify delay with \e[1;21mBPQLDELAY\e[0m"; exit 1; fi
	@if [ -z "$(BPQLPROMILLE)" ]; then echo -e "\e[1;31mError: Must specify noise promille with \e[1;21mBPQLPROMILLE\e[0m"; exit 1; fi
	$(PYTHON) example/baseline-bpql.py \
	    -v --no-resume --no-save \
	    "--delay=$(BPQLDELAY)::$(BPQLDELAY)" \
	    "--horizon=$(BPQLDELAY)" \
	    "--assumed-delay=$(BPQLDELAY)" \
	    "--trainer-prefix=$(TRAINER_PREFIX)" \
	    "$(CONFIG_ENV)"


plot-delay-ticks: validate-test-profile
	$(eval PLOTCMD := $(PYTHON) plot_delay.py)
	$(eval ARGS := $(shell if [[ ! -z "$(output)" ]]; then \
		echo -e "--output-file \"$(DELAY_TAG)-delay-ticks\""; \
	  fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(seed)" ]]; then echo -e "--seed \"$(seed)\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(width)" ]]; then echo -e "--width \"$(width)\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(height)" ]]; then echo -e "--height \"$(height)\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(fontsize)" ]]; then echo -e "--fontsize \"$(fontsize)\""; fi))
	$(PLOTCMD) $(ARGS) '$(DELAY_TAG)::$(DELAY_ARG)'

plot-delay-time-series: validate-test-profile
	$(eval PLOTCMD := $(PYTHON) plot_delay.py --time-series)
	$(eval FILENAME := $(DELAY_TAG)-delay-time-series)
	$(eval FILENAME := $(FILENAME)$(shell if [[ ! -z "$(lowcda)" ]]; then echo -e "-lowcda"; fi))
	$(eval FILENAME := $(FILENAME)$(shell if [[ ! -z "$(highcda)" ]]; then echo -e "-highcda"; fi))
	$(eval ARGS := $(shell if [[ ! -z "$(output)" ]]; then \
		echo -e "--output-file \"$(FILENAME)\""; \
	  fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(seed)" ]]; then echo -e "--seed \"$(seed)\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(width)" ]]; then echo -e "--width \"$(width)\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(height)" ]]; then echo -e "--height \"$(height)\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(fontsize)" ]]; then echo -e "--fontsize \"$(fontsize)\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(lowcda)" ]]; then echo -e "--hbar \"$(ASSUMED_DELAY1)\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(highcda)" ]]; then echo -e "--hbar \"$(ASSUMED_DELAY2)\""; fi))
	$(PLOTCMD) $(ARGS) '$(DELAY_TAG)::$(DELAY_ARG)'



# Generates table for the main body text
table-mainbody:
	$(eval TABLECMD := $(PYTHON) print_joint_latex_table.py --indent=4 )
	$(eval ARGS := $(foreach e,ant humanoid halfcheetah hopper walker2d,$(shell \
		if [[ -f "_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-sac/eval.json" ]]; then \
		    echo -e "\"SAC:$(e):GE_{1,23}:_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-sac/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-sac-aug-mem$(EXTREME_ASSUMED_DELAY2)/eval.json" ]]; then \
		    echo -e "\"SAC w/ CDA:$(e):GE_{1,23}:_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-sac-aug-mem$(EXTREME_ASSUMED_DELAY2)/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-dreamer/eval.json" ]]; then \
		    echo -e "\"Dreamer:$(e):GE_{1,23}:_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-dreamer/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-bpql-mem$(EXTREME_ASSUMED_DELAY2)/eval.json" ]]; then \
		    echo -e "\"BPQL:$(e):GE_{1,23}:_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-bpql-mem$(EXTREME_ASSUMED_DELAY2)/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-vdpo-mem$(EXTREME_ASSUMED_DELAY2)/eval.json" ]]; then \
		    echo -e "\"VDPO:$(e):GE_{1,23}:_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-vdpo-mem$(EXTREME_ASSUMED_DELAY2)/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-dcac/eval.json" ]]; then \
		    echo -e "\"DCAC:$(e):GE_{1,23}:_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-dcac/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-mbpac_memdelay/eval.json" ]]; then \
		    echo -e "\"ACDA:$(e):GE_{1,23}:_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-mbpac_memdelay/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-mbpac-highdim-memdelay/eval.json" ]]; then \
		    echo -e "\"ACDA:$(e):GE_{1,23}:_logs/$(EXTREME_RUN_PREFIX)/noisy-$(e)/$(EXTREME_DELAY_TAG)-mbpac-highdim-memdelay/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-sac/eval.json" ]]; then \
		    echo -e "\"SAC:$(e):GE_{4,32}:_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-sac/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-sac-aug-mem$(EXTREME32_ASSUMED_DELAY2)/eval.json" ]]; then \
		    echo -e "\"SAC w/ CDA:$(e):GE_{4,32}:_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-sac-aug-mem$(EXTREME32_ASSUMED_DELAY2)/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-dreamer/eval.json" ]]; then \
		    echo -e "\"Dreamer:$(e):GE_{4,32}:_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-dreamer/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-bpql-mem$(EXTREME32_ASSUMED_DELAY2)/eval.json" ]]; then \
		    echo -e "\"BPQL:$(e):GE_{4,32}:_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-bpql-mem$(EXTREME32_ASSUMED_DELAY2)/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-vdpo-mem$(EXTREME32_ASSUMED_DELAY2)/eval.json" ]]; then \
		    echo -e "\"VDPO:$(e):GE_{4,32}:_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-vdpo-mem$(EXTREME32_ASSUMED_DELAY2)/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-dcac/eval.json" ]]; then \
		    echo -e "\"DCAC:$(e):GE_{4,32}:_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-dcac/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-mbpac_memdelay/eval.json" ]]; then \
		    echo -e "\"ACDA:$(e):GE_{4,32}:_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-mbpac_memdelay/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-mbpac-highdim-memdelay/eval.json" ]]; then \
		    echo -e "\"ACDA:$(e):GE_{4,32}:_logs/$(EXTREME32_RUN_PREFIX)/noisy-$(e)/$(EXTREME32_DELAY_TAG)-mbpac-highdim-memdelay/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-sac/eval.json" ]]; then \
		    echo -e "\"SAC:$(e):MM1:_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-sac/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-sac-aug-mem$(MM1Q_ASSUMED_DELAY2)/eval.json" ]]; then \
		    echo -e "\"SAC w/ CDA:$(e):MM1:_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-sac-aug-mem$(MM1Q_ASSUMED_DELAY2)/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-dreamer/eval.json" ]]; then \
		    echo -e "\"Dreamer:$(e):MM1:_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-dreamer/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-bpql-mem$(MM1Q_ASSUMED_DELAY2)/eval.json" ]]; then \
		    echo -e "\"BPQL:$(e):MM1:_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-bpql-mem$(MM1Q_ASSUMED_DELAY2)/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-vdpo-mem$(MM1Q_ASSUMED_DELAY2)/eval.json" ]]; then \
		    echo -e "\"VDPO:$(e):MM1:_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-vdpo-mem$(MM1Q_ASSUMED_DELAY2)/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-dcac/eval.json" ]]; then \
		    echo -e "\"DCAC:$(e):MM1:_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-dcac/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-mbpac_memdelay/eval.json" ]]; then \
		    echo -e "\"ACDA:$(e):MM1:_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-mbpac_memdelay/eval.json\""; \
		fi; \
		if [[ -f "_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-mbpac-highdim-memdelay/eval.json" ]]; then \
		    echo -e "\"ACDA:$(e):MM1:_logs/$(MM1Q_RUN_PREFIX)/noisy-$(e)/$(MM1Q_DELAY_TAG)-mbpac-highdim-memdelay/eval.json\""; \
		fi; \
	)))
	$(TABLECMD) $(ARGS)


# This only plots the memorized policy performance
plot-appendix-single-delay-%: validate-test-profile
	$(eval PLOTCMD := $(PYTHON) plot_json.py --plot-std --runavg=5)
	$(eval RUNPATH := _logs/$(RUN_PREFIX)/$*)
	$(eval ARGS := )
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_SAC)/eval.json" ]];                    then echo -e "\"SAC%dotted-red:$(RUNPATH)/$(TAG_SAC)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_SAC_AUGMENT2)/eval.json" ]];           then echo -e "\"SAC w/ CDA%dashed-darkred:$(RUNPATH)/$(TAG_SAC_AUGMENT2)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_DREAMER)/eval.json" ]];                then echo -e "\"Dreamer%dotted-darkpurple:$(RUNPATH)/$(TAG_DREAMER)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_BPQL2)/eval.json" ]];                  then echo -e "\"BPQL %dashed-darkyellow:$(RUNPATH)/$(TAG_BPQL2)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_VDPO2)/eval.json" ]];                  then echo -e "\"VDPO %dashed-darkgreen:$(RUNPATH)/$(TAG_VDPO2)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_DCAC)/eval.json" ]];                   then echo -e "\"DCAC%darkorange:$(RUNPATH)/$(TAG_DCAC)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_MBPAC_MEMDELAY)/eval.json" ]];         then echo -e "\"ACDA%cyan:$(RUNPATH)/$(TAG_MBPAC_MEMDELAY)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json" ]]; then echo -e "\"ACDA%cyan:$(RUNPATH)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell \
	  if [[ ! -z "$(plot_legend)" ]]; then \
	    if [[ ! -z "$(flat_legend)" ]]; then \
	        echo -e "--flat-legend"; \
	    fi; \
	    if [[ ! -z "$(output)" ]]; then \
	        echo -e "--save-legend"; \
	        echo -e " --output-file \"$(DELAY_TAG)-reduced-legend\""; \
	    fi; \
	  else \
	    echo -e "--no-legend"; \
	    if [[ ! -z "$(output)" ]]; then \
	        echo -e " --output-file \"$(DELAY_TAG)-reduced-$*\""; \
	    fi; \
	  fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(fontsize)" ]]; then echo -e "--fontsize \"$(fontsize)\""; fi))
	$(PLOTCMD) $(ARGS)

# Generates accompanying table
table-appendix-single-delay: validate-test-profile
	$(eval TABLECMD := $(PYTHON) print_latex_table.py --indent=4 )
	$(eval RUNPATH := _logs/$(RUN_PREFIX))
	$(eval ARGS := $(foreach e,ant humanoid halfcheetah hopper walker2d,$(shell \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_SAC)/eval.json" ]]; then                    echo -e "\"SAC:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_SAC)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_SAC_AUGMENT2)/eval.json" ]]; then           echo -e "\"SAC w/ CDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_SAC_AUGMENT2)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_DREAMER)/eval.json" ]]; then                echo -e "\"Dreamer:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_DREAMER)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_BPQL2)/eval.json" ]]; then                  echo -e "\"BPQL:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_BPQL2)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_VDPO2)/eval.json" ]]; then                  echo -e "\"VDPO:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_VDPO2)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_DCAC)/eval.json" ]]; then                   echo -e "\"DCAC:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_DCAC)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_MEMDELAY)/eval.json" ]]; then         echo -e "\"ACDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_MEMDELAY)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json" ]]; then echo -e "\"ACDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json\""; fi; \
	)))
	$(TABLECMD) $(ARGS)

# Generates accompanying table
table-appendix-noise-eval-%: validate-test-profile
	$(eval ACDA_TAG := $(shell case "$*" in (ant|humanoid) echo "mbpac-highdim-memdelay" ;; (*) echo "mbpac_memdelay" ;; esac))
	$(PYTHON) print_noise_latex_table.py \
	              --indent=4 \
	              --bm-prefix=$(RUN_PREFIX) \
	              --tag=$(DELAY_TAG) \
	              --noise-percs=0,5,10,15,20,25 \
	              --env-suffix=$* \
	              "BPQL:bpql-mem$(ASSUMED_DELAY2)" \
	              "VDPO:vdpo-mem$(ASSUMED_DELAY2)" \
	              "ACDA:$(ACDA_TAG)"


# This is a plot for the appendix, showing what happens if the CDA is lower than the upper bound.
plot-appendix-lowcda-%: validate-test-profile
	$(eval PLOTCMD := $(PYTHON) plot_json.py --plot-std --runavg=5)
	$(eval RUNPATH := _logs/$(RUN_PREFIX)/$*)
	$(eval ARGS := )
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_BPQL1)/eval.json" ]];                  then echo -e "\"BPQL%dashdot-darkyellow:$(RUNPATH)/$(TAG_BPQL1)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_VDPO1)/eval.json" ]];                  then echo -e "\"VDPO%dashdot-darkgreen:$(RUNPATH)/$(TAG_VDPO1)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_MBPAC_MEMDELAY)/eval.json" ]];         then echo -e "\"ACDA%cyan:$(RUNPATH)/$(TAG_MBPAC_MEMDELAY)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json" ]]; then echo -e "\"ACDA%cyan:$(RUNPATH)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_BPQLMDA1)/eval.json" ]];               then echo -e "\"BPQL w/ MDA%dashdot-cyan-blue:$(RUNPATH)/$(TAG_BPQLMDA1)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_BPQLMDA1_HIGHDIM)/eval.json" ]];       then echo -e "\"BPQL w/ MDA%dashdot-cyan-blue:$(RUNPATH)/$(TAG_BPQLMDA1_HIGHDIM)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell \
	  if [[ ! -z "$(plot_legend)" ]]; then \
	    if [[ ! -z "$(flat_legend)" ]]; then \
	        echo -e "--flat-legend"; \
	    fi; \
	    if [[ ! -z "$(output)" ]]; then \
	        echo -e "--save-legend"; \
	        echo -e " --output-file \"$(DELAY_TAG)-appendix-lowcda-legend\""; \
	    fi; \
	  else \
	    echo -e "--no-legend"; \
	    if [[ ! -z "$(output)" ]]; then \
	        echo -e " --output-file \"$(DELAY_TAG)-appendix-lowcda-$*\""; \
	    fi; \
	  fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(fontsize)" ]]; then echo -e "--fontsize \"$(fontsize)\""; fi))
	$(PLOTCMD) $(ARGS)

# Generates accompanying table for lowcda
table-appendix-lowcda: validate-test-profile
	$(eval TABLECMD := $(PYTHON) print_latex_table.py --indent=4 )
	$(eval RUNPATH := _logs/$(RUN_PREFIX))
	$(eval ARGS := $(foreach e,ant humanoid halfcheetah hopper walker2d,$(shell \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_BPQL1)/eval.json" ]]; then                  echo -e "\"BPQL:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_BPQL1)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_VDPO1)/eval.json" ]]; then                  echo -e "\"VDPO:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_VDPO1)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_MEMDELAY)/eval.json" ]]; then         echo -e "\"ACDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_MEMDELAY)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json" ]]; then echo -e "\"ACDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_BPQLMDA1)/eval.json" ]]; then               echo -e "\"BPQL w/ MDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_BPQLMDA1)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_BPQLMDA1_HIGHDIM)/eval.json" ]]; then       echo -e "\"BPQL w/ MDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_BPQLMDA1_HIGHDIM)/eval.json\""; fi; \
	)))
	$(TABLECMD) $(ARGS)


# This is a plot for the appendix, showing BPQL with our MDA
plot-appendix-highcda-%: validate-test-profile
	$(eval PLOTCMD := $(PYTHON) plot_json.py --plot-std --runavg=5)
	$(eval RUNPATH := _logs/$(RUN_PREFIX)/$*)
	$(eval ARGS := )
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_BPQL2)/eval.json" ]];                  then echo -e "\"BPQL%dashed-darkyellow:$(RUNPATH)/$(TAG_BPQL2)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_VDPO2)/eval.json" ]];                  then echo -e "\"VDPO%dashed-darkgreen:$(RUNPATH)/$(TAG_VDPO2)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_MBPAC_MEMDELAY)/eval.json" ]];         then echo -e "\"ACDA%cyan:$(RUNPATH)/$(TAG_MBPAC_MEMDELAY)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json" ]]; then echo -e "\"ACDA%cyan:$(RUNPATH)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_BPQLMDA2)/eval.json" ]];               then echo -e "\"BPQL w/ MDA%dashed-cyan-blue:$(RUNPATH)/$(TAG_BPQLMDA2)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell if [[ -f "$(RUNPATH)/$(TAG_BPQLMDA2_HIGHDIM)/eval.json" ]];       then echo -e "\"BPQL w/ MDA%dashed-cyan-blue:$(RUNPATH)/$(TAG_BPQLMDA2_HIGHDIM)/eval.json\""; fi))
	$(eval ARGS := $(ARGS) $(shell \
	  if [[ ! -z "$(plot_legend)" ]]; then \
	    if [[ ! -z "$(flat_legend)" ]]; then \
	        echo -e "--flat-legend"; \
	    fi; \
	    if [[ ! -z "$(output)" ]]; then \
	        echo -e "--save-legend"; \
	        echo -e " --output-file \"$(DELAY_TAG)-appendix-highcda-legend\""; \
	    fi; \
	  else \
	    echo -e "--no-legend"; \
	    if [[ ! -z "$(output)" ]]; then \
	        echo -e " --output-file \"$(DELAY_TAG)-appendix-highcda-$*\""; \
	    fi; \
	  fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(fontsize)" ]]; then echo -e "--fontsize \"$(fontsize)\""; fi))
	$(PLOTCMD) $(ARGS)

# Generates accompanying table for highcda
table-appendix-highcda: validate-test-profile
	$(eval TABLECMD := $(PYTHON) print_latex_table.py --indent=4 )
	$(eval RUNPATH := _logs/$(RUN_PREFIX))
	$(eval ARGS := $(foreach e,ant humanoid halfcheetah hopper walker2d,$(shell \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_BPQL2)/eval.json" ]]; then                  echo -e "\"BPQL:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_BPQL2)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_VDPO2)/eval.json" ]]; then                  echo -e "\"VDPO:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_VDPO2)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_MEMDELAY)/eval.json" ]]; then         echo -e "\"ACDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_MEMDELAY)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json" ]]; then echo -e "\"ACDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_MBPAC_HIGHDIM_MEMDELAY)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_BPQLMDA2)/eval.json" ]]; then               echo -e "\"BPQL w/ MDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_BPQLMDA2)/eval.json\""; fi; \
		if [[ -f "$(RUNPATH)/noisy-$(e)/$(TAG_BPQLMDA2_HIGHDIM)/eval.json" ]]; then       echo -e "\"BPQL w/ MDA:$(e):$(RUNPATH)/noisy-$(e)/$(TAG_BPQLMDA2_HIGHDIM)/eval.json\""; fi; \
	)))
	$(TABLECMD) $(ARGS)


plot-appendix-model-free:
	$(eval PLOTCMD := $(PYTHON) plot_json.py --title="$*" --plot-std --runavg=5)
	$(eval RUN_NAME := paper-appendix-compmodelfree)
	$(eval JSON_PREFIX := _logs/paper-appendix-compmodelfree/eval_ExtremeClogL1U23)
	$(eval ARGS := )
	$(eval ARGS := $(ARGS) "MBPAC%cyan:$(JSON_PREFIX)-mbpac-highdim-memdelay.json")
	$(eval ARGS := $(ARGS) "Model-Free Multi-Policy%teal:$(JSON_PREFIX)-mfpac_multivar.json")
	$(eval ARGS := $(ARGS) "Model-Free Transformer%purple:$(JSON_PREFIX)-mfpac_transformer.json")
	$(eval ARGS := $(ARGS) $(shell \
	  if [[ ! -z "$(plot_legend)" ]]; then \
	    if [[ ! -z "$(output)" ]]; then \
	    	echo -e "--save-legend"; \
	    	echo -e " --output-file \"$(RUN_NAME)-legend\""; \
	    fi; \
	  else \
	    echo -e "--no-legend"; \
	    if [[ ! -z "$(output)" ]]; then \
	    	echo -e " --output-file \"$(RUN_NAME)\""; \
	    fi; \
	  fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(fontsize)" ]]; then echo -e "--fontsize \"$(fontsize)\""; fi))
	$(PLOTCMD) $(ARGS)


plot-appendix-bpqlnoise-%:
	$(eval PLOTCMD := $(PYTHON) plot_delaynoise_series.py)
	$(eval ARGS := )
	$(eval ARGS := $(ARGS) --bm-prefix _logs/noise-eval-v2/$*/bpql-noise)
	$(eval ARGS := $(ARGS) --noise-promilles 0,25,50,75,100,150,200)
	$(eval ARGS := $(ARGS) --delays 3,6,9,12,15,18,21,24)
	$(eval ARGS := $(ARGS) $(shell \
	  if [[ ! -z "$(plot_legend)" ]]; then \
	    if [[ ! -z "$(flat_legend)" ]]; then \
	        echo -e "--flat-legend"; \
	    fi; \
	    if [[ ! -z "$(output)" ]]; then \
	        echo -e "--save-legend"; \
	    	echo -e " --output-file \"bpqlnoise-$*-legend\""; \
	    fi; \
	  else \
	    echo -e "--no-legend"; \
	    if [[ ! -z "$(output)" ]]; then \
	    	echo -e " --output-file \"bpqlnoise-$*\""; \
	    fi; \
	  fi))
	$(eval ARGS := $(ARGS) $(shell if [[ ! -z "$(fontsize)" ]]; then echo -e "--fontsize \"$(fontsize)\""; fi))
	$(PLOTCMD) $(ARGS)

table-appendix-bpqlnoise-%:
	$(eval PLOTCMD := $(PYTHON) plot_delaynoise_series.py --gen-table --indent=4)
	$(eval ARGS := )
	$(eval ARGS := $(ARGS) --bm-prefix _logs/noise-eval-v2/$*/bpql-noise)
	$(eval ARGS := $(ARGS) --noise-promilles 0,25,50,75,100,150,200)
	$(eval ARGS := $(ARGS) --delays 3,6,9,12,15,18,21,24)
	$(PLOTCMD) $(ARGS)



# Fetch run data from a remote server
fetch: validate-test-profile
	$(eval SERVER := gullfaxi)
	$(eval FIND_COMMAND := cd ~/GitRepos/latency-env/_logs \
	                    && find $(RUN_PREFIX) -maxdepth 3 -mindepth 3 -name eval.json \
	                     | xargs tar -czvf ${RUN_PREFIX}.tar.gz \
	                    && mv ${RUN_PREFIX}.tar.gz ~/)
	ssh $(SERVER) '$(FIND_COMMAND)' && sync

	mkdir -p _logs
	cd _logs && rm -rf $(RUN_PREFIX) $(RUN_PREFIX).tar.gz

	scp $(SERVER):~/$(RUN_PREFIX).tar.gz _logs/$(RUN_PREFIX).tar.gz
	cd _logs && tar -xzvf $(RUN_PREFIX).tar.gz
