import pandas as pd
import numpy as np

from src.rl_agent import RLAgent
from src.state_representation import StateRepresentation

from src.env_pendulum import EnvironmentPendulum
from src.env_redpillbluepill import EnvironmentRedPillBluePill

from src.rl_experiments import RLExperiments

Red-Pill Blue-Pill (RED CVaR)

# For single tau

# define environment
env = EnvironmentRedPillBluePill() 

# define agent
actions = ['red_pill', 'blue_pill']

states = ['redworld', 'blueworld']      

policy = None

agent = RLAgent(agent_type='q_learning', 
                states=states, 
                actions=actions, 
                policy=policy, 
                use_average_reward=True,
                policy_type='tabular',
                value_type='tabular',
                use_cvar=True,
                initial_avg_reward=0.0, 
                var_quantile=0.25, 
                initial_var_reward=0.0)

# run experiment
rl_experiments = RLExperiments()

step_sizes = {
    'value': 0.02,
    'avg_reward': 0.01,
    'var': 0.01,
}

df_rpbp_cvar = rl_experiments.redpillbluepill(agent, env,
                                              num_runs=50,
                                              max_steps=100000,
                                              epsilon=0.1,
                                              step_size=step_sizes)

100%|███████████████████████████████████████████████████████████████████████████████████| 50/50 [01:15<00:00,  1.51s/it]

# For multiple taus

# define environment
env = EnvironmentRedPillBluePill() 

# define agent
actions = ['red_pill', 'blue_pill']

states = ['redworld', 'blueworld']      

policy = None

tau_experiment_results = {}
for tau in [0.1, 0.25, 0.5, 0.75, 0.85, 0.9]:
    print(tau)
    agent = RLAgent(agent_type='q_learning', 
                    states=states, 
                    actions=actions, 
                    policy=policy, 
                    use_average_reward=True,
                    policy_type='tabular',
                    value_type='tabular',
                    use_cvar=True,
                    initial_avg_reward=0.0, 
                    var_quantile=tau, 
                    initial_var_reward=0.0)

    # run experiment
    rl_experiments = RLExperiments()
    
    step_sizes = {
    'value': 0.02,
    'avg_reward': 0.01,
    'var': 0.1,
    }

    df_rpbp_tau = rl_experiments.redpillbluepill(agent, env,
                                                 num_runs=10,
                                                 max_steps=100000,
                                                 epsilon=0.1,
                                                 step_size=step_sizes)
    
    tau_experiment_results[tau] = df_rpbp_tau

0.1

100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [00:14<00:00,  1.44s/it]

0.25

100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [00:14<00:00,  1.44s/it]

0.5

100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [00:14<00:00,  1.44s/it]

0.75

100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [00:14<00:00,  1.43s/it]

0.85

100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [00:19<00:00,  1.91s/it]

0.9

100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [00:19<00:00,  1.92s/it]

Red-Pill Blue-Pill (Differential)

# define environment
env = EnvironmentRedPillBluePill() 

# define agent
actions = ['red_pill', 'blue_pill']

states = ['redworld', 'blueworld']      

policy = None

agent = RLAgent(agent_type='q_learning', 
                states=states, 
                actions=actions, 
                policy=policy, 
                use_average_reward=True,
                policy_type='tabular',
                value_type='tabular',
                use_cvar=False,
                initial_avg_reward=0.0)

# run experiment
rl_experiments = RLExperiments()

step_sizes = {
    'value': 0.0002,
    'avg_reward': 1,
    'var': 1,
}

df_rpbp_reg = rl_experiments.redpillbluepill(agent, env,
                                              num_runs=50,
                                              max_steps=100000,
                                              epsilon=0.1,
                                              step_size=step_sizes)

100%|███████████████████████████████████████████████████████████████████████████████████| 50/50 [01:39<00:00,  1.99s/it]

Inverted Pendulum (RED CVaR)

# define environment
env = EnvironmentPendulum() 

# define agent with function approximation
num_tiles = 8
num_tilings = 32
iht_size = 4096
state_representation = StateRepresentation(num_tiles=num_tiles,
                                           num_tilings=num_tilings,
                                           iht_size=iht_size,
                                           min_pose = -1*np.pi,
                                           max_pose = np.pi,
                                           min_vel = -1*env.min_max_velocity,
                                           max_vel = env.min_max_velocity,
                                          )

actions = ['accelerate_left', 'dont_accelerate', 'accelerate_right']
states = np.zeros((iht_size, 1))
policy = None

agent = RLAgent(agent_type='td', 
                states=states, 
                actions=actions, 
                policy=policy, 
                use_average_reward=True,
                policy_type='linear',
                policy_softmax_tau = 1.0,
                policy_update_type='stochastic_gradient_descent',
                value_type='linear',
                value_update_type='stochastic_gradient_descent',
                use_cvar=True,
                initial_avg_reward=0.0, 
                var_quantile=0.1, 
                initial_var_reward=0.0)

# run experiment
rl_experiments = RLExperiments()

step_sizes = {
    'value': 0.002,
    'policy': 1,
    'avg_reward': 0.01,
    'var': 0.01,
}

df_pendulum_cvar = rl_experiments.pendulum(agent, env, 
                                           state_representation=state_representation.dynamicstate_tilecoding,
                                           num_runs=50,
                                           max_steps=100000,
                                           step_size=step_sizes)

100%|███████████████████████████████████████████████████████████████████████████████████| 50/50 [10:20<00:00, 12.41s/it]

Inverted Pendulum (Differential)

# define environment
env = EnvironmentPendulum() 

# define agent with function approximation
num_tiles = 8
num_tilings = 32
iht_size = 4096
state_representation = StateRepresentation(num_tiles=num_tiles,
                                           num_tilings=num_tilings,
                                           iht_size=iht_size,
                                           min_pose = -1*np.pi,
                                           max_pose = np.pi,
                                           min_vel = -1*env.min_max_velocity,
                                           max_vel = env.min_max_velocity,
                                          )

actions = ['accelerate_left', 'dont_accelerate', 'accelerate_right']
states = np.zeros((iht_size, 1))
policy = None

agent = RLAgent(agent_type='td', 
                states=states, 
                actions=actions, 
                policy=policy, 
                use_average_reward=True,
                policy_type='linear',
                policy_softmax_tau = 1.0,
                policy_update_type='stochastic_gradient_descent',
                value_type='linear',
                value_update_type='stochastic_gradient_descent',
                use_cvar=False,
                initial_avg_reward=0.0)

# run experiment
rl_experiments = RLExperiments()

step_sizes = {
    'value': 0.002,
    'policy': 1,
    'avg_reward': 0.001,
    'var': 1,
}

df_pendulum_reg = rl_experiments.pendulum(agent, env, 
                                          state_representation=state_representation.dynamicstate_tilecoding,
                                          num_runs=50,
                                          max_steps=100000,
                                          step_size=step_sizes)

100%|███████████████████████████████████████████████████████████████████████████████████| 50/50 [10:21<00:00, 12.42s/it]

Figures

rl_experiments = RLExperiments()
rl_experiments.get_performance_figure(experiment='redpillbluepill',
                                      df_cvar=df_rpbp_cvar, 
                                      df_reg=df_rpbp_reg, 
                                      quantile=0.25, 
                                      rolling_average_amount=1000,
                                      x_max=50000)

100%|███████████████████████████████████████████████████████████████████████████████████| 50/50 [33:33<00:00, 40.27s/it]
100%|███████████████████████████████████████████████████████████████████████████████████| 50/50 [33:23<00:00, 40.07s/it]

rl_experiments = RLExperiments()
rl_experiments.get_performance_figure(experiment='pendulum',
                                      df_cvar=df_pendulum_cvar, 
                                      df_reg=df_pendulum_reg, 
                                      quantile=0.1, 
                                      rolling_average_amount=1000,
                                      x_max=15000)

100%|███████████████████████████████████████████████████████████████████████████████████| 50/50 [32:56<00:00, 39.52s/it]
100%|███████████████████████████████████████████████████████████████████████████████████| 50/50 [32:54<00:00, 39.48s/it]

# define environment
env = EnvironmentRedPillBluePill() 

# define agent
actions = ['red_pill', 'blue_pill']

states = ['redworld', 'blueworld']      

policy = None

agent = RLAgent(agent_type='q_learning', 
                states=states, 
                actions=actions, 
                policy=policy, 
                use_average_reward=True,
                policy_type='tabular',
                value_type='tabular',
                use_cvar=True,
                initial_avg_reward=0.0, 
                var_quantile=0.25, 
                initial_var_reward=-1.0)

# run experiment
rl_experiments = RLExperiments()

step_sizes = {
    'value': 0.02,
    'avg_reward': 0.01,
    'var': 0.01,
}

plots = {
   'case a)': {
       'init_avg_reward': 0.0, 
       'init_var_reward': -1.25,
       'z_max': 100000,
       'color': '#4F1379',
   }, 
   
   'case b)': {
       'init_avg_reward': 1.0, 
       'init_var_reward': -1.0,
       'z_max': 100000,
       'color': '#9A2E7F',
   },
    
    'case c)': {
       'init_avg_reward': 0.0, 
       'init_var_reward': 0.0,
       'z_max': 100000,
       'color': '#D7486C',
   }, 
    
    'case d)': {
       'init_avg_reward': -1.0, 
       'init_var_reward': 0.0,
       'z_max': 100000,
       'color': '#FEB780',
   }, 
}

rl_experiments.get_3d_plot(agent, env,
                           num_runs=1,
                           max_steps=100000,
                           epsilon=0.1,
                           step_size=step_sizes,
                           plots=plots,
                          )

100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [00:01<00:00, 53967.41it/s]
100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [00:01<00:00, 53189.79it/s]
100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [00:01<00:00, 61585.94it/s]
100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [00:01<00:00, 60836.28it/s]

# define environment
env = EnvironmentRedPillBluePill() 

# define agent
actions = ['red_pill', 'blue_pill']

states = ['redworld', 'blueworld']      

policy = None

agent = RLAgent(agent_type='q_learning', 
                states=states, 
                actions=actions, 
                policy=policy, 
                use_average_reward=True,
                policy_type='tabular',
                value_type='tabular',
                use_cvar=True,
                initial_avg_reward=0.0, 
                var_quantile=0.25, 
                initial_var_reward=0.0)

# run experiment
rl_experiments = RLExperiments()

step_sizes = {
    'value': 0.02,
    'avg_reward': 0.01,
    'var': 0.01,
}

rl_experiments.cvar_redpillbluepill_comparison(agent=agent, 
                                               env=env,
                                               experiment='cvar_rpbp_estimates_reg',
                                               # experiment='cvar_rpbp_estimates_hardcoded',
                                               num_runs=1,
                                               max_steps=100000,
                                               epsilon=0.1,
                                               step_size=step_sizes)

100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [00:01<00:00, 57249.67it/s]

# define environment
env = EnvironmentPendulum() 

# define agent with function approximation
num_tiles = 8
num_tilings = 32
iht_size = 4096
state_representation = StateRepresentation(num_tiles=num_tiles,
                                           num_tilings=num_tilings,
                                           iht_size=iht_size,
                                           min_pose = -1*np.pi,
                                           max_pose = np.pi,
                                           min_vel = -1*env.min_max_velocity,
                                           max_vel = env.min_max_velocity,
                                          )

actions = ['accelerate_left', 'dont_accelerate', 'accelerate_right']
states = np.zeros((iht_size, 1))
policy = None

agent = RLAgent(agent_type='td', 
                states=states, 
                actions=actions, 
                policy=policy, 
                use_average_reward=True,
                policy_type='linear',
                policy_softmax_tau = 1.0,
                policy_update_type='stochastic_gradient_descent',
                value_type='linear',
                value_update_type='stochastic_gradient_descent',
                use_cvar=True,
                initial_avg_reward=0.0, 
                var_quantile=0.1, 
                initial_var_reward=0.0)

# run experiment
rl_experiments = RLExperiments()

step_sizes = {
    'value': 0.002,
    'policy': 1,
    'avg_reward': 0.01,
    'var': 0.01,
}

rl_experiments.cvar_pendulum_comparison(agent=agent, 
                                        env=env,
                                        experiment='cvar_pendulum_estimates_reg',
                                        # experiment='cvar_pendulum_estimates_hardcoded',
                                        state_representation=state_representation.dynamicstate_tilecoding,
                                        num_runs=1,
                                        max_steps=500000,
                                        step_size=step_sizes)

100%|█████████████████████████████████████████████████████████████████████████| 500000/500000 [01:03<00:00, 7913.39it/s]

# Compare CVaR values of red and blue policies (estimated using monte carlo)
rl_experiments = RLExperiments()
rl_experiments.get_cvar_by_tau_plot(
    n_samples=100000,
    epsillon=0.1,
)

# tau experiment
rl_experiments = RLExperiments()
rl_experiments.get_tau_results_figure(experiment='rpbp_by_tau',
                                      results_dict=tau_experiment_results, 
                                      n_runs=10, 
                                      rolling_average_amount=1000,
                                      x_max=50000)

100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [03:09<00:00, 18.93s/it]
100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [03:09<00:00, 18.95s/it]
100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [02:41<00:00, 16.16s/it]
100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [02:41<00:00, 16.15s/it]
100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [02:41<00:00, 16.17s/it]
100%|███████████████████████████████████████████████████████████████████████████████████| 10/10 [02:40<00:00, 16.08s/it]