exp_prefix: psro_ma
env_config:
  lib: expground.envs.poker
  env_id: kuhn_poker
  params:
    fixed_player: true
    players: 2
    reward_scale: 1.0

learner_config:
  type: psro
  params:
    seed: 1
    independent_learning: False # trigger centralized learner
    remove_dominated_policy: False
    meta_solver: fictitious_play
    use_learnable_dist: False
    episodic_training: False
    train_every: 10
    learning_mode: off_policy
    ray_mode: True
    evaluation_worker_num: 30
    mixed_at_every_step: False
    centralized_critic_config:
      network: mlp
      layers:
        - units: 128
          activation: ReLU
        - units: 128
          activation: ReLU
      output:
        activation: False
    distribution_training_kwargs:
      train_every: 32 # do not active
      start_from: random # ...
      sample_mode: random # softmax
  stop_conditions:
    max_iteration: 100
  inner_stop_conditions:
    max_episode: 10000

# ============ copy from maddpg
rollout_config:
  type: sequential
  fragment_length: 100
  max_step: 20
  num_simulation: 1000

training_config:
  adv_eps: 0 # 0.001  # 0.001  # for minimax optimization
  actor_lr: 0.0001
  critic_lr: 0.001
  optimizer: Adam
  batch_size: 128
  gamma: 1.0
  tau: 0.01
  grad_norm_clipping: 0.5
  mute_critic: True

sampler_config:
  sampler_type: simple
  params:
    capacity: 10000
    learning_starts: 64

algorithm:
  name: DDPG
  policy: ddpg.DDPG
  trainer: ddpg.DDPGTrainer
  loss: ddpg.DDPGLoss

model_config:
  actor:
    network: mlp
    layers:
      - units: 128
        activation: ReLU
      - units: 128
        activation: ReLU
    output:
      activation: False

  critic:
    network: mlp
    layers:
      - units: 2
        activation: ReLU
      - units: 2
        activation: ReLU
    output:
      activation: False

custom_config:
  use_cuda: False
  # turn on stochastic mode or not
  use_stochastic_decision: False # True
