
{
    'exp_name': 'VDPO',
    'env': 'HalfCheetah-v4',
    'gamma': 0.99,
    'device': device(type='cuda'),
    'seed': 0,
    'buffer_size': 1000000,
    'total_timesteps': 1000000,
    'learn_start': 5000,
    'batch_size': 256,
    'actor_lr': 0.0003,
    'critic_lr': 0.001,
    'alpha_lr': 0.001,
    'target_update_factor': 0.005,
    'actor_freq': 2,
    'target_freq': 1,
    'eval_freq': 10000,
    'eval_num': 10,
    'kl_freq': 1000,
    'belief_lr': 0.0003,
    'embedding_dim': 256,
    'n_steps': 3,
    'delay': 16,
    'delayspec': 'MM1Queue_a033_s075::mm1queue(0.33, 0.75)',
    'noise': 0.25
}
✓ setup
Created Delay Process: MM1Queue(0.33, 0.75)
  1%|          | 9999/1000000 [03:30<8:25:50, 32.62it/s]global step 10000, trans_decision ep_re -261.9553956039553

{"global_step": 10000, "eval_re": [-288.08894685598483, -231.64123350549085, 
-275.23212975474246, -248.7472657304585, -276.4944048457059, -274.7425178740714,
-242.8628204526939, -284.8537395213881, -229.96102120878288, 
-266.92987629023446], "eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 
1000, 1000, 1000]}

  2%|▏         | 19997/1000000 [10:40<8:17:09, 32.85it/s]global step 20000, trans_decision ep_re -74.70379178152737

{"global_step": 20000, "eval_re": [-101.94194550209237, -21.3510677896462, 
48.42230336942268, -197.8948136951078, -94.4620435518982, -107.22486073399145, 
-139.55923942463386, 9.537016690643345, 42.1442742852672, -184.70754146323716], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  3%|▎         | 29997/1000000 [17:40<8:12:23, 32.83it/s]global step 30000, trans_decision ep_re 65.40809382358495

{"global_step": 30000, "eval_re": [209.37526436567356, -10.989273162471143, 
113.5853830001621, 182.177706072305, -105.77910476281632, 190.16216839537955, 
55.75448505241797, -18.352105609187106, 107.06026567142374, -68.91385078703797],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  4%|▍         | 39999/1000000 [24:40<8:05:25, 32.96it/s]global step 40000, trans_decision ep_re 137.9830274314208

{"global_step": 40000, "eval_re": [150.78803623566972, 220.7670524636975, 
188.4794072276875, 190.19272303060808, 206.4776577929538, 44.77027429106796, 
5.604012480048135, 152.15069024243456, 216.22172017146534, 4.378700378575532], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  5%|▍         | 49997/1000000 [31:50<8:01:55, 32.85it/s]global step 50000, trans_decision ep_re 214.77462277113972

{"global_step": 50000, "eval_re": [335.56591441252226, 193.45267642702308, 
168.6232655495929, 78.64150554696374, 373.989125363417, 225.8327127201577, 
271.49547050196, 98.46538314091839, 72.45669189235682, 329.22348215648526], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  6%|▌         | 59998/1000000 [38:50<7:53:53, 33.06it/s]global step 60000, trans_decision ep_re 264.94689090131203

{"global_step": 60000, "eval_re": [363.6260693778392, 253.63843495267756, 
300.72877483072085, 278.025029886593, 202.74906777074918, 232.8692522025554, 
183.98449445858986, 304.11885631060653, 321.5278911663875, 208.2010380564007], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  7%|▋         | 69997/1000000 [45:50<7:50:17, 32.96it/s]global step 70000, trans_decision ep_re 376.63475631412314

{"global_step": 70000, "eval_re": [524.9537197549995, 370.7620378556606, 
310.76651481969157, 342.41302361900705, 362.40295812239685, 366.8853211563917, 
370.5606329393183, 370.94128637952207, 314.68596248771183, 431.9761060065321], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  8%|▊         | 79997/1000000 [53:00<7:44:41, 33.00it/s]global step 80000, trans_decision ep_re 361.10352050747343

{"global_step": 80000, "eval_re": [400.58829316181544, 284.7371119461981, 
270.49611675598334, 356.3655700111287, 464.8595158163497, 332.82894014727765, 
529.2250895491516, 401.9654904686565, 30.503018173129103, 539.466059045044], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  9%|▉         | 89999/1000000 [1:00:00<7:39:11, 33.03it/s]global step 90000, trans_decision ep_re 423.40579948011083

{"global_step": 90000, "eval_re": [142.55978720335295, 409.2738222208207, 
336.69582503757744, 738.0408093752924, 325.5768612553483, 537.697166678604, 
601.0912439488552, 340.67027494278517, 334.52270543478073, 467.929498703691], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 10%|▉         | 99999/1000000 [1:07:00<7:34:02, 33.04it/s]global step 100000, trans_decision ep_re 403.943733369654

{"global_step": 100000, "eval_re": [451.29585523735676, 472.71731363570206, 
445.6389188012364, 390.66896115008745, 399.5054077937648, 352.9170985762084, 
387.9383412865691, 311.64083105776024, 390.88156360849365, 436.23304254936124], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 11%|█         | 109997/1000000 [1:14:10<7:34:11, 32.66it/s]global step 110000, trans_decision ep_re 461.1943722269175

{"global_step": 110000, "eval_re": [329.2261645148934, 742.9910712310774, 
533.2309530614426, 447.52936819208475, 484.224837303295, 348.2889628285693, 
525.0141764280319, 417.7366457705725, 414.5356374084277, 369.16590553077975], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 12%|█▏        | 119997/1000000 [1:21:10<7:24:51, 32.97it/s]global step 120000, trans_decision ep_re 422.2810451466964

{"global_step": 120000, "eval_re": [410.0269668997413, 639.6986357307944, 
351.4661528061116, 388.4659389200111, 411.00471673746716, 417.7595795400266, 
269.30678880188736, 423.8942058711336, 411.06066692441397, 500.12679923537706], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 13%|█▎        | 129997/1000000 [1:28:10<7:19:07, 33.02it/s]global step 130000, trans_decision ep_re 452.6093729599409

{"global_step": 130000, "eval_re": [472.55329093609066, 438.12446835706896, 
477.9128278257546, 338.8483298580049, 424.136808538623, 627.3703927464778, 
350.5172118569078, 352.47023468262773, 534.251073309853, 509.9090914880005], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 14%|█▍        | 139999/1000000 [1:35:10<7:14:53, 32.96it/s]global step 140000, trans_decision ep_re 588.6688696141348

{"global_step": 140000, "eval_re": [503.4253240220194, 489.25602717098974, 
898.8400571651448, 557.2603815148765, 470.42259530067173, 899.637344904667, 
411.8010233626234, 583.2126247347251, 494.1120902777075, 578.7212276879219], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 15%|█▍        | 149997/1000000 [1:42:20<7:10:08, 32.93it/s]global step 150000, trans_decision ep_re 617.6696943389325

{"global_step": 150000, "eval_re": [942.3875589020405, 552.9840021360205, 
498.56134024603324, 425.50996428341125, 593.6753793862384, 550.4206761880015, 
584.9600276244148, 796.791244826153, 419.466976809738, 811.9397729872735], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 16%|█▌        | 159999/1000000 [1:49:20<7:06:47, 32.80it/s]global step 160000, trans_decision ep_re 556.5149653293144

{"global_step": 160000, "eval_re": [664.591373836677, 771.4106220251655, 
416.57596818229985, 589.7292374600613, 398.17457396893377, 624.1103205280417, 
546.4996159396618, 802.4543030784447, 319.64524032801165, 431.958397945846], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 17%|█▋        | 169997/1000000 [1:56:20<7:00:03, 32.93it/s]global step 170000, trans_decision ep_re 484.999468706421

{"global_step": 170000, "eval_re": [639.7840589199229, 423.8701940532646, 
761.9487861622468, 449.9217159645694, 632.7108064312488, 422.82486804833604, 
-107.1533340016558, 654.1574079109938, 566.3571908753167, 405.5729926999663], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 18%|█▊        | 179997/1000000 [2:03:30<6:54:41, 32.96it/s]global step 180000, trans_decision ep_re 599.4799100457914

{"global_step": 180000, "eval_re": [488.92737588175265, 649.9627607041638, 
469.62671839406414, 541.0852844464499, 604.5448869466372, 611.6064858835155, 
551.0997602925495, 725.8994356531344, 863.4458032654755, 488.6005889901708], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 19%|█▉        | 189999/1000000 [2:10:30<6:48:53, 33.02it/s]global step 190000, trans_decision ep_re 631.9079909344746

{"global_step": 190000, "eval_re": [763.4849702982068, 959.8347157554482, 
758.3935552843478, 577.9222627978264, 411.63286233330007, 410.9986736129217, 
698.468631480708, 458.3207051690427, 803.8199928090472, 476.2035398038968], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 20%|█▉        | 199999/1000000 [2:17:30<6:43:22, 33.05it/s]global step 200000, trans_decision ep_re 586.381459259159

{"global_step": 200000, "eval_re": [599.7714328008942, 805.2688170366321, 
920.667754789674, 712.4036311543613, 749.5011385778598, 530.5953550646265, 
420.8416695977876, 403.8367457579119, 365.21297010861076, 355.7150777032319], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 21%|██        | 209997/1000000 [2:24:40<6:39:29, 32.96it/s]global step 210000, trans_decision ep_re 635.1550476837381

{"global_step": 210000, "eval_re": [521.0509832207858, 558.6578360971981, 
793.3391475017787, 427.11605207565714, 835.0361589559999, 495.5953984857002, 
544.6213801190498, 747.771891705401, 496.42203074628367, 931.9395979295269], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 22%|██▏       | 219997/1000000 [2:31:40<6:35:59, 32.83it/s]global step 220000, trans_decision ep_re 527.6639962797424

{"global_step": 220000, "eval_re": [555.1065458846119, 537.387674560895, 
555.3618056478185, 483.58081288790777, 532.2460743040189, 526.9781137883402, 
493.07364508725425, 627.7233474399629, 542.2659762838672, 422.91596691274555], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 23%|██▎       | 229997/1000000 [2:38:40<6:30:05, 32.90it/s]global step 230000, trans_decision ep_re 625.2251763331224

{"global_step": 230000, "eval_re": [502.711963584791, 428.70593976870356, 
413.5311063897622, 725.8515441072647, 1068.4603583359394, 847.1122353876474, 
806.7857946637716, 388.08797288855436, 503.4764023645968, 567.5284458401931], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 24%|██▍       | 239997/1000000 [2:45:50<6:26:16, 32.79it/s]global step 240000, trans_decision ep_re 733.3364223662728

{"global_step": 240000, "eval_re": [953.9870525870973, 570.4188447172319, 
1093.3658828804241, 470.9570110167256, 950.1823404530911, 784.1834785186725, 
539.6708959608397, 447.19812933622575, 1003.4056205956623, 519.994967596758], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 25%|██▍       | 249997/1000000 [2:53:00<6:20:33, 32.85it/s]global step 250000, trans_decision ep_re 530.5520571923837

{"global_step": 250000, "eval_re": [902.327539478244, 469.38862512265945, 
565.1299605152043, 568.6770752693317, 147.12280178445528, 488.23001708145796, 
643.3120482024918, 463.9074260736625, 514.4212572894451, 543.0038211068845], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 26%|██▌       | 259997/1000000 [3:00:00<6:17:20, 32.68it/s]global step 260000, trans_decision ep_re 829.528272063398

{"global_step": 260000, "eval_re": [1171.9995382322254, 771.1537598667729, 
545.3913270530223, 511.0727602214091, 994.1891386681006, 954.8851090894574, 
1039.2652580479207, 596.0439417831263, 787.8281575191357, 923.4537301528092], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 27%|██▋       | 269997/1000000 [3:07:10<6:09:17, 32.95it/s]global step 270000, trans_decision ep_re 447.5321259140642

{"global_step": 270000, "eval_re": [534.026393698755, 443.2469016855263, 
719.5737808534681, -345.79015235474156, 473.5853591319046, 341.6832256790935, 
713.815292296498, 372.9672638704388, 475.63957037615234, 746.5736239035465], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 28%|██▊       | 279997/1000000 [3:14:10<6:05:37, 32.82it/s]global step 280000, trans_decision ep_re 857.3010772644841

{"global_step": 280000, "eval_re": [606.0214710712091, 996.2485821006364, 
1123.9289093966227, 840.6321578847898, 1319.1012235458813, 882.3167664581957, 
485.5414243632368, 674.0692302679392, 588.0304926386043, 1057.120514917724], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 29%|██▉       | 289997/1000000 [3:21:20<6:08:19, 32.13it/s]global step 290000, trans_decision ep_re 518.1150379073573

{"global_step": 290000, "eval_re": [531.4837141570575, 482.5215370117033, 
723.1692661278042, 508.4722447413332, -248.55104296538101, 622.0109841134978, 
688.982851187108, 454.3694442781879, 517.2995669736598, 901.3918134486016], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 30%|██▉       | 299997/1000000 [3:28:20<6:03:00, 32.14it/s]global step 300000, trans_decision ep_re 610.2713645453389

{"global_step": 300000, "eval_re": [636.851016774276, 493.56599300646843, 
508.0344322228875, 436.6372588992357, 1006.5193290660729, 618.576004556878, 
382.30832861750855, 499.92180656765527, 1104.6121568068513, 415.68731893555537],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 31%|███       | 309997/1000000 [3:35:30<5:52:21, 32.64it/s]global step 310000, trans_decision ep_re 588.2663230722446

{"global_step": 310000, "eval_re": [772.2943136600853, 1089.5460772202016, 
629.1236710458029, 688.0315592183855, 658.2902107784626, 451.86566095546533, 
-309.79731932912483, 397.14207402177783, 977.5234664848781, 528.643516666512], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 32%|███▏      | 319997/1000000 [3:42:40<5:45:08, 32.84it/s]global step 320000, trans_decision ep_re 640.9010346320721

{"global_step": 320000, "eval_re": [874.1139022119736, 562.4660025274254, 
711.3769524895347, 675.0705893473761, 619.2275776933916, 778.2220246501277, 
420.8429080472282, 749.8112678640823, 472.9180809136975, 544.961040575883], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 33%|███▎      | 329997/1000000 [3:49:40<5:39:00, 32.94it/s]global step 330000, trans_decision ep_re 681.5689064054105

{"global_step": 330000, "eval_re": [685.9136919957426, 470.15505219285683, 
607.6788130815672, 868.8884620676869, 874.7996965851031, 649.5372926742766, 
616.4285021526628, 701.5074200083927, 823.0198496796816, 517.7602836161361], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 34%|███▍      | 339997/1000000 [3:56:50<5:37:23, 32.60it/s]global step 340000, trans_decision ep_re 578.2368660184358

{"global_step": 340000, "eval_re": [538.4470140216662, 464.2957422196736, 
763.4369908334203, 612.4239955615776, 515.3136954772492, 517.2361964561369, 
742.2710129993162, 553.9782932347927, 438.25004328224486, 636.7156760982808], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 35%|███▍      | 349997/1000000 [4:03:50<5:32:42, 32.56it/s]global step 350000, trans_decision ep_re 639.3179175051289

{"global_step": 350000, "eval_re": [538.6396652546673, 592.2617934768278, 
443.6683908916034, 611.8648738617126, 757.780358341859, 638.1006920627303, 
455.81896672998766, 1166.752514946027, 533.7847281262681, 654.5071913596051], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 36%|███▌      | 359997/1000000 [4:11:00<5:23:28, 32.97it/s]global step 360000, trans_decision ep_re 525.3351959989295

{"global_step": 360000, "eval_re": [405.0950621597054, 583.6518975734444, 
384.7680856282816, 502.502980522263, 580.9752618724915, 468.1753367353823, 
596.0528435921179, 588.7547023840355, 662.7573648918307, 480.61842462974283], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 37%|███▋      | 369997/1000000 [4:18:00<5:26:24, 32.17it/s]global step 370000, trans_decision ep_re 607.3442677219489

{"global_step": 370000, "eval_re": [872.4790478698333, 611.7626869372875, 
454.25428295414343, 435.14137921067925, 401.8297194674995, 653.0917037958036, 
594.5267270584465, 635.8004339491899, 572.9918776233054, 841.564818353301], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 38%|███▊      | 379997/1000000 [4:25:10<5:20:59, 32.19it/s]global step 380000, trans_decision ep_re 604.9853406828024

{"global_step": 380000, "eval_re": [466.854995797885, 885.0884420103175, 
616.4310947163185, 629.019790960722, 502.4197783904221, 521.8901261513957, 
849.1458988684362, 631.5577573265097, 487.7608302959497, 459.6846923100672], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 39%|███▉      | 389997/1000000 [4:32:20<5:09:27, 32.85it/s]global step 390000, trans_decision ep_re 610.4695355325571

{"global_step": 390000, "eval_re": [573.5346813201479, 668.9977902350095, 
413.92779262560026, 487.6509356749667, 479.5907525396731, 814.0837655914855, 
884.825067811571, 603.250677789644, 620.4377290578744, 558.3961626795989], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 40%|███▉      | 399997/1000000 [4:39:20<5:05:02, 32.78it/s]global step 400000, trans_decision ep_re 683.7169530394598

{"global_step": 400000, "eval_re": [564.8378624546579, 805.8909095295081, 
999.1981970795448, 527.2033566350239, 658.4792599450277, 740.3047381918842, 
461.5945210497883, 444.5222227646855, 792.7117054364363, 842.426757308041], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 41%|████      | 409997/1000000 [4:46:30<4:58:27, 32.95it/s]global step 410000, trans_decision ep_re 626.1685253323177

{"global_step": 410000, "eval_re": [451.392592131699, 577.0077473757392, 
908.2738127522363, 637.3170051125826, 780.5911179971246, 701.4960218163517, 
386.91639156079697, 345.3222582736879, 881.7088784992744, 591.6594278036838], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 42%|████▏     | 419997/1000000 [4:53:40<4:59:52, 32.24it/s]global step 420000, trans_decision ep_re 648.0086080949303

{"global_step": 420000, "eval_re": [458.94687110638904, 738.1837891860675, 
458.51593945761135, 771.3752473263118, 585.1600928688871, 1052.5014370044962, 
830.9720140759455, 556.8545815500736, 443.6543759521121, 583.9217324214098], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 43%|████▎     | 429997/1000000 [5:00:50<4:54:47, 32.23it/s]global step 430000, trans_decision ep_re 642.3752737555051

{"global_step": 430000, "eval_re": [738.9709648371299, 662.5647133106719, 
422.4660294216516, 559.713482285318, 833.8818557936524, 513.6876643566612, 
858.6980306396832, 753.9414681452284, 546.8553229254825, 532.973205839572], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 44%|████▍     | 439997/1000000 [5:08:00<4:51:55, 31.97it/s]global step 440000, trans_decision ep_re 843.5932485602865

{"global_step": 440000, "eval_re": [938.9365905680139, 1188.3621677456451, 
458.1164809931548, 837.5218897489042, 1011.0458233457725, 776.3823338455239, 
460.53733056931793, 851.234511881296, 1028.082628391685, 885.7127285135505], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 45%|████▍     | 449997/1000000 [5:15:10<4:47:24, 31.89it/s]global step 450000, trans_decision ep_re 714.0969797643811

{"global_step": 450000, "eval_re": [1066.8334804587705, 803.1177961449916, 
719.8788144395729, 549.8901756627191, 599.3905929563691, 536.5353284638785, 
444.66650223921215, 1089.1222876602078, 764.7934158330585, 566.7414037850311], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 46%|████▌     | 459997/1000000 [5:22:10<4:38:58, 32.26it/s]global step 460000, trans_decision ep_re 711.12554667351

{"global_step": 460000, "eval_re": [724.9885404353292, 806.7282904335635, 
577.2817900982369, 712.7123769012278, 571.1506291108195, 708.567022550199, 
649.6412077881789, 678.7581475114619, 1066.6141452751242, 614.8133166309595], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 47%|████▋     | 469997/1000000 [5:29:20<4:33:09, 32.34it/s]global step 470000, trans_decision ep_re 939.3989688375156

{"global_step": 470000, "eval_re": [1039.8073539540278, 678.4217015523715, 
1105.9703978652597, 896.634275919241, 1453.462403573351, 661.7010483643633, 
605.1833394174497, 881.0494960906595, 1191.5080591322373, 880.2516125061949], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 48%|████▊     | 479997/1000000 [5:36:30<4:27:18, 32.42it/s]global step 480000, trans_decision ep_re 774.9758336473585

{"global_step": 480000, "eval_re": [872.8683617963301, 412.23486807192415, 
754.4860225465154, 473.44569447138167, 1076.598448309849, 657.6569668148807, 
947.4187839421418, 943.9207749329186, 1057.664635831672, 553.4637797559712], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 49%|████▉     | 489997/1000000 [5:43:40<4:20:28, 32.63it/s]global step 490000, trans_decision ep_re 630.0774147815704

{"global_step": 490000, "eval_re": [575.1293536852909, 679.0676465765113, 
737.4462444244979, 569.0923650623404, 792.25694886591, 457.3297594931544, 
498.21341047896163, 636.0287017974618, 862.2287592890756, 493.9809581424999], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 50%|████▉     | 499997/1000000 [5:50:50<4:17:40, 32.34it/s]global step 500000, trans_decision ep_re 559.6552512945802

{"global_step": 500000, "eval_re": [363.25198458070565, 380.90822440750975, 
626.4706575577524, 452.41095525465965, 722.6354261558976, 456.2189364532909, 
563.3856552380851, 591.9470912421128, 592.9309042348689, 846.3926778209203], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 51%|█████     | 509997/1000000 [5:58:00<4:14:44, 32.06it/s]global step 510000, trans_decision ep_re 734.9031477285602

{"global_step": 510000, "eval_re": [689.3917504981612, 736.9213577536289, 
809.2365147439436, 515.5868483674714, 387.45896377754343, 995.1326760044327, 
1023.261881924226, 762.4488794186686, 772.3125446865098, 657.2800601110164], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 52%|█████▏    | 519997/1000000 [6:05:10<4:02:32, 32.98it/s]global step 520000, trans_decision ep_re 803.1396929338014

{"global_step": 520000, "eval_re": [902.9658819823165, 934.0803625079942, 
458.22080508551085, 1090.927920529407, 573.8077894434156, 1100.719771203925, 
471.50476331108194, 566.4121293817631, 1348.3335779481204, 584.423927944478], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 53%|█████▎    | 529997/1000000 [6:12:20<4:00:42, 32.54it/s]global step 530000, trans_decision ep_re 702.9523994401557

{"global_step": 530000, "eval_re": [472.518680297726, 575.6663996517156, 
510.0883551172999, 461.9217032495926, 734.159542982186, 523.7403004052752, 
812.9340488013435, 1158.74079122205, 971.1648485963829, 808.5893240779853], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 54%|█████▍    | 539997/1000000 [6:19:20<3:54:34, 32.68it/s]global step 540000, trans_decision ep_re 550.0152579111406

{"global_step": 540000, "eval_re": [801.8314231520543, 482.82180792397776, 
621.0167667311463, 455.23337288680585, 521.0133777411065, 615.7299431658485, 
704.4430522357806, 475.2164626727918, 408.849862983454, 413.99650961843935], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 55%|█████▍    | 549997/1000000 [6:26:30<3:50:17, 32.57it/s]global step 550000, trans_decision ep_re 637.3435106341784

{"global_step": 550000, "eval_re": [612.4683213463403, 802.9614525704297, 
750.5878883812213, 357.94791978458994, 725.6892842877106, 776.0694650866525, 
452.5432866939672, 714.4749162533983, 632.819765798511, 547.8728061389638], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 56%|█████▌    | 559997/1000000 [6:33:30<3:44:19, 32.69it/s]global step 560000, trans_decision ep_re 625.3516894403384

{"global_step": 560000, "eval_re": [618.1065151644258, 635.4886040773313, 
546.2129260063922, 920.5449419396086, 436.1350258973279, 627.2910437186115, 
418.98968404602016, 585.3818949240081, 853.9573509506918, 611.4089076789671], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 57%|█████▋    | 569997/1000000 [6:40:40<3:41:58, 32.29it/s]global step 570000, trans_decision ep_re 651.9934006433953

{"global_step": 570000, "eval_re": [533.5454742844142, 705.2283415925643, 
692.6491830384791, 521.7317228431697, 518.0189965066687, 701.544401155381, 
543.6412504103826, 934.3866350335402, 844.0333288765522, 525.1546726928015], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 58%|█████▊    | 579997/1000000 [6:47:40<3:32:11, 32.99it/s]global step 580000, trans_decision ep_re 628.8582602445404

{"global_step": 580000, "eval_re": [544.5888371026724, 650.787111794603, 
639.6337824576976, 625.6072825077474, 437.0258288347116, 925.7010419812158, 
549.2377005589747, 670.1357564098715, 360.62489421422447, 885.2403665836845], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 59%|█████▉    | 589997/1000000 [6:54:50<3:28:31, 32.77it/s]global step 590000, trans_decision ep_re 660.4640492807341

{"global_step": 590000, "eval_re": [600.5544898596263, 727.9068496517268, 
855.4020198062229, 619.7185074159231, 741.946015523628, 709.7805951998993, 
697.8089213679809, 506.4538520512884, 522.9123233239669, 622.1569186070784], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 60%|█████▉    | 599997/1000000 [7:01:50<3:21:45, 33.04it/s]global step 600000, trans_decision ep_re 749.9901215210764

{"global_step": 600000, "eval_re": [796.2382567980576, 906.3250444381666, 
468.72950461308824, 656.3741955625417, 849.302226716276, 681.6565947062815, 
1018.0810648866533, 630.2616557099232, 714.7399858536725, 778.1926859261029], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 61%|██████    | 609997/1000000 [7:08:50<3:18:01, 32.82it/s]global step 610000, trans_decision ep_re 652.6665160974392

{"global_step": 610000, "eval_re": [749.1651468738988, 524.7972938357024, 
507.2800471091125, 461.4969211041699, 887.560963028428, 676.0375978013849, 
746.9441865461969, 687.6166794921736, 532.2577221768275, 753.5086030064963], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 62%|██████▏   | 619999/1000000 [7:16:00<3:16:40, 32.20it/s]global step 620000, trans_decision ep_re 708.8975466347896

{"global_step": 620000, "eval_re": [685.6114025052295, 1014.0486170632335, 
576.8299045359021, 902.0076047753847, 721.3170711957574, 521.666601761827, 
647.8940422301848, 707.9591465079639, 680.2902992684436, 631.350776503969], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 63%|██████▎   | 629997/1000000 [7:23:10<3:09:31, 32.54it/s]global step 630000, trans_decision ep_re 654.444544936699

{"global_step": 630000, "eval_re": [584.7244475143233, 456.6173418637944, 
914.9944700684018, 577.0775083153139, 521.9783021449218, 458.3002412971367, 
629.6040911603588, 742.4452655535063, 876.3832705117701, 782.3205109374628], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 64%|██████▍   | 639997/1000000 [7:30:10<3:03:23, 32.72it/s]global step 640000, trans_decision ep_re 547.0318620765108

{"global_step": 640000, "eval_re": [605.0457687431306, 621.0376884863542, 
472.17203356318305, 459.2016897843774, 432.8560134404329, 496.2304586899356, 
755.6641906360032, 442.06631993837334, 491.4599491938067, 694.5845082895113], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 65%|██████▍   | 649997/1000000 [7:37:20<2:57:28, 32.87it/s]global step 650000, trans_decision ep_re 868.9584022105122

{"global_step": 650000, "eval_re": [953.7855741364915, 697.6740319391848, 
777.9460201329384, 979.1034489433171, 1057.232211902815, 966.2677175586726, 
1160.535480407986, 838.9602330038716, 489.9915518368717, 768.0877522429739], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 66%|██████▌   | 659997/1000000 [7:44:20<2:51:56, 32.96it/s]global step 660000, trans_decision ep_re 640.4490954196267

{"global_step": 660000, "eval_re": [687.4111620868462, 578.3264961707386, 
912.801868759834, 642.5992705650494, 646.4051180883506, 573.9570626438743, 
672.2708105821591, 606.2608255962348, 582.9516510554921, 501.5066886476877], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 67%|██████▋   | 669997/1000000 [7:51:30<2:49:33, 32.44it/s]global step 670000, trans_decision ep_re 634.6566986156175

{"global_step": 670000, "eval_re": [505.1994080994956, 520.0187724974893, 
680.8074126029287, 679.9827849383955, 574.1047700858105, 516.5007503224095, 
872.9065741506727, 793.8603380517792, 663.039644072001, 540.1465313351935], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 68%|██████▊   | 679997/1000000 [7:58:30<2:44:35, 32.40it/s]global step 680000, trans_decision ep_re 608.5845932454398

{"global_step": 680000, "eval_re": [628.1933129785081, 1022.3681313491518, 
469.5714875587454, 543.4466803281842, 468.91115341856374, 648.4649887553255, 
580.0215909170192, 621.4163711582863, 541.6616996298695, 561.790516360744], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 69%|██████▉   | 689997/1000000 [8:05:30<2:38:00, 32.70it/s]global step 690000, trans_decision ep_re 514.6643554986813

{"global_step": 690000, "eval_re": [693.3462439426461, 808.5044544206245, 
722.1350880940728, 469.18705810204693, 480.9095242784536, -284.8669460323628, 
729.5516485907082, 403.62265212888656, 689.7240057813668, 434.52982568037066], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 70%|██████▉   | 699997/1000000 [8:12:40<2:32:47, 32.72it/s]global step 700000, trans_decision ep_re 772.1511561759071

{"global_step": 700000, "eval_re": [1033.47898130983, 558.058042124991, 
749.4047449161279, 1065.8706928474946, 773.5205902725137, 722.8424012600285, 
581.729552476451, 789.5541858722103, 465.54528482445227, 981.507085854972], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 71%|███████   | 709997/1000000 [8:19:40<2:26:38, 32.96it/s]global step 710000, trans_decision ep_re 514.1250731672237

{"global_step": 710000, "eval_re": [631.850350578034, 518.360929751489, 
589.8380659546661, 609.9027933452242, 579.2788814424, 419.1409662240399, 
501.05393704477297, 541.5293558587407, 504.57759207013635, 245.71785940273344], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 72%|███████▏  | 719997/1000000 [8:26:50<2:21:50, 32.90it/s]global step 720000, trans_decision ep_re 511.2549072733098

{"global_step": 720000, "eval_re": [484.79628894994903, 439.42187106836155, 
626.1253589407305, 615.979114287016, 632.2421308464942, 474.94746278323294, 
288.354739673295, 389.27131503411823, 552.8055028522755, 608.6052882976247], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 73%|███████▎  | 729997/1000000 [8:33:50<2:17:11, 32.80it/s]global step 730000, trans_decision ep_re 694.635271813946

{"global_step": 730000, "eval_re": [684.9034136746703, 983.4942050536147, 
580.2164743509519, 592.782153765263, 850.9361426576421, 611.4606590196557, 
840.1095533907221, 421.4154881301521, 692.3644240262462, 688.6702040705416], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 74%|███████▍  | 739997/1000000 [8:41:00<2:14:19, 32.26it/s]global step 740000, trans_decision ep_re 573.046806977783

{"global_step": 740000, "eval_re": [621.8129687380102, 441.75036954720395, 
590.927296630372, 478.20416732564166, 493.16170709303964, 858.0366355625351, 
492.04969046498525, 441.34981916035593, 671.4516382322977, 641.7237770233888], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 75%|███████▍  | 749997/1000000 [8:48:00<2:07:07, 32.78it/s]global step 750000, trans_decision ep_re 687.065985767419

{"global_step": 750000, "eval_re": [796.2739666843426, 483.9101258269764, 
890.0747188363215, 382.29322344046864, 714.8330849925331, 884.735890732067, 
557.3054665154438, 625.8345723621095, 579.5947379556063, 955.8040703283202], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 76%|███████▌  | 759997/1000000 [8:55:10<2:02:10, 32.74it/s]global step 760000, trans_decision ep_re 513.5119955003345

{"global_step": 760000, "eval_re": [425.45548184312753, 570.225099594744, 
358.1909565800974, 686.7815541068477, 383.53164812099163, 630.0538785409227, 
414.78063195969173, 587.7787788782416, 526.9798077997435, 551.342117578937], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 77%|███████▋  | 769997/1000000 [9:02:10<1:56:21, 32.94it/s]global step 770000, trans_decision ep_re 532.4595858342982

{"global_step": 770000, "eval_re": [375.6667645205142, 549.0655013490124, 
521.8317372498723, 607.8564592108462, 700.4695266563347, 498.9177474446981, 
723.6885575534129, 846.2500829057966, -397.8359033025272, 898.685384755022], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 78%|███████▊  | 779997/1000000 [9:09:10<1:51:25, 32.91it/s]global step 780000, trans_decision ep_re 720.9952501536621

{"global_step": 780000, "eval_re": [804.0220756705484, 606.5358579443781, 
681.8486442565871, 732.0982101590107, 1053.305601627612, 653.9427506441966, 
757.3712934166365, 707.0053083822073, 593.8658797780903, 619.9568796573545], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 79%|███████▉  | 789999/1000000 [9:16:20<1:46:38, 32.82it/s]global step 790000, trans_decision ep_re 653.0440500291014

{"global_step": 790000, "eval_re": [912.3140700427573, 479.36134382543435, 
702.4144939249951, 613.3855466172326, 575.1573480442842, 657.6250610201075, 
440.15251138134835, 657.018151746469, 925.2044111022822, 567.8075625861028], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 80%|███████▉  | 799997/1000000 [9:23:20<1:41:45, 32.76it/s]global step 800000, trans_decision ep_re 596.5156275542164

{"global_step": 800000, "eval_re": [573.6449662415565, 633.629033485133, 
705.8047455656989, 132.875142371182, 538.7576789333068, 809.8098111869145, 
708.4038024364665, 627.4115572536602, 527.0335280887243, 707.7860099795199], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 81%|████████  | 809997/1000000 [9:30:30<1:38:39, 32.10it/s]global step 810000, trans_decision ep_re 721.373678173167

{"global_step": 810000, "eval_re": [600.936144985092, 844.3676940345376, 
488.13444644662496, 935.9060588829502, 514.885570294064, 626.546598456911, 
576.9937835571947, 826.459661598227, 885.0502823557478, 914.4565411203215], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 82%|████████▏ | 819997/1000000 [9:37:30<1:32:38, 32.38it/s]global step 820000, trans_decision ep_re 674.6226154721134

{"global_step": 820000, "eval_re": [652.0696458690124, 583.8767507284035, 
798.2426235574723, 687.294216924949, 821.2616102886676, 535.1131015075366, 
748.8386623627532, 564.7947592392711, 623.4625212016425, 731.2722630414245], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 83%|████████▎ | 829997/1000000 [9:44:40<1:27:24, 32.41it/s]global step 830000, trans_decision ep_re 561.0290860462138

{"global_step": 830000, "eval_re": [587.5004178960136, 375.5859687043212, 
442.9605509927214, 496.2697769236307, 525.7922676385754, 617.7730339635872, 
753.6538181983833, 832.7953837558597, 562.8234281707824, 415.1362142182637], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 84%|████████▍ | 839997/1000000 [9:51:50<1:22:32, 32.31it/s]global step 840000, trans_decision ep_re 566.7176101175307

{"global_step": 840000, "eval_re": [664.1631442600531, 737.958884141728, 
408.8934469394957, 523.5706971668588, 550.3429807683256, 686.3955175004289, 
592.1174355279761, 442.2048593998251, 463.80918910018954, 597.7199463704259], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 85%|████████▍ | 849997/1000000 [9:59:00<1:16:02, 32.88it/s]global step 850000, trans_decision ep_re 478.0484985232944

{"global_step": 850000, "eval_re": [528.33818068348, 269.7849364359083, 
573.624189792006, 682.460875978874, 731.6996560944417, 396.15264231292537, 
360.93842132313495, 532.2109369452697, 345.1903837344125, 360.08476193249226], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 86%|████████▌ | 859997/1000000 [10:06:00<1:11:01, 32.85it/s]global step 860000, trans_decision ep_re 639.4540092535328

{"global_step": 860000, "eval_re": [762.6054807984706, 591.6060678915276, 
480.05018997225824, 766.8418708580264, 831.0199926808118, 564.0911103610252, 
514.8132584542927, 633.3673650747569, 596.9384863421329, 653.2062701020271], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 87%|████████▋ | 869997/1000000 [10:13:10<1:07:35, 32.06it/s]global step 870000, trans_decision ep_re 397.32110499551123

{"global_step": 870000, "eval_re": [519.5716376379702, 441.19630015250414, 
510.17242052049664, 270.7108804036812, 495.63712187088856, 405.0974297261641, 
393.5221852631031, 425.4194551705496, 103.03000670411882, 408.85361250563585], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 88%|████████▊ | 879997/1000000 [10:20:10<1:01:40, 32.43it/s]global step 880000, trans_decision ep_re 631.9533186205201

{"global_step": 880000, "eval_re": [542.1475114688768, 629.5517332266527, 
640.2570991265451, 620.3605837860373, 606.1125655343463, 456.7147539892001, 
366.8841278827771, 1000.0238972787208, 678.2654211385875, 779.2154927734575], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 89%|████████▉ | 889997/1000000 [10:27:20<57:09, 32.07it/s]global step 890000, trans_decision ep_re 719.7944262325063

{"global_step": 890000, "eval_re": [819.2534296665901, 404.9268830174725, 
1026.5191552504414, 850.9001656620185, 727.2454416184953, 877.5938710185152, 
727.6885565251873, 402.3006616706762, 785.4190671098113, 576.0970307858549], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 90%|████████▉ | 899997/1000000 [10:34:30<50:44, 32.84it/s]global step 900000, trans_decision ep_re 503.73947245852526

{"global_step": 900000, "eval_re": [532.335283918212, 383.04465523475875, 
462.6206248511252, 645.4372800272092, 686.2274583220695, 404.88651553826423, 
635.0016656537995, 204.11380448874294, 596.8480447673785, 486.87939178369226], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 91%|█████████ | 909997/1000000 [10:41:30<45:30, 32.96it/s]global step 910000, trans_decision ep_re 504.7631338382853

{"global_step": 910000, "eval_re": [591.6765779420678, 551.0751333503325, 
428.55918900404436, 518.6944914296797, 634.6157002176423, 418.84560508409703, 
538.0146260658358, 460.7750003937543, 345.2485971575704, 560.1264177378278], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 92%|█████████▏| 919997/1000000 [10:48:40<40:24, 32.99it/s]global step 920000, trans_decision ep_re 653.3435411123041

{"global_step": 920000, "eval_re": [747.5453959123461, 550.8320541323948, 
696.0388621059072, 724.612058960904, 682.7257524947267, 693.9644703127487, 
688.7307485907477, 594.8057740023428, 410.5800082821903, 743.6002863287318], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 93%|█████████▎| 929997/1000000 [10:55:40<35:39, 32.71it/s]global step 930000, trans_decision ep_re 823.0154952639638

{"global_step": 930000, "eval_re": [771.5263212749334, 622.2885531599985, 
1111.251521244995, 1101.8111138599968, 830.5122381893387, 1106.7804027997443, 
687.3125638038058, 751.8777050528053, 615.916064576618, 630.8784686774019], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 94%|█████████▍| 939997/1000000 [11:02:50<30:35, 32.69it/s]global step 940000, trans_decision ep_re 621.006896239934

{"global_step": 940000, "eval_re": [738.3900236796075, 606.0705504758125, 
527.7520172570239, 702.8148776832314, 667.257514841799, 521.1754436243161, 
717.4834873908899, 606.1224765134988, 574.664768215216, 548.3378027179449], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 95%|█████████▍| 949997/1000000 [11:10:00<25:25, 32.77it/s]global step 950000, trans_decision ep_re 679.6532277528302

{"global_step": 950000, "eval_re": [634.7457687719156, 893.8043221629998, 
527.1114800497619, 684.7645737543213, 411.6817718237255, 661.8057566215, 
828.6130871861028, 730.0953270927528, 756.3553893839013, 667.55480068132], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 96%|█████████▌| 959998/1000000 [11:17:00<20:20, 32.79it/s]global step 960000, trans_decision ep_re 582.8201790406614

{"global_step": 960000, "eval_re": [747.9412692819689, 607.7057383710812, 
195.0984042495106, 522.0673604015316, 433.5836570332099, 568.8890075328915, 
663.80648059443, 578.7502338005086, 725.1090931479614, 785.2505459935207], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 97%|█████████▋| 969997/1000000 [11:24:00<15:13, 32.86it/s]global step 970000, trans_decision ep_re 624.4986449787446

{"global_step": 970000, "eval_re": [530.9534692824009, 662.2209829162248, 
739.7000768217522, 565.6253833260149, 639.0976933011854, 656.0795288116428, 
555.3734433770649, 663.1877323563157, 552.6625456523176, 680.0855939425269], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 98%|█████████▊| 979998/1000000 [11:31:10<10:08, 32.89it/s]global step 980000, trans_decision ep_re 611.5429030400538

{"global_step": 980000, "eval_re": [933.4323536328429, 592.5603911524696, 
477.0260446298234, 869.6761233524352, 517.8104408828742, 585.831314113958, 
673.3422721270094, 397.75472231361624, 578.3821521492292, 489.6132160462806], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 99%|█████████▉| 989997/1000000 [11:38:10<05:03, 32.93it/s]global step 990000, trans_decision ep_re 572.2577936846026

{"global_step": 990000, "eval_re": [682.3809786535624, 549.3020057543154, 
676.5085771133456, 544.342289828642, 630.9442539284653, 322.4540171573305, 
549.7572624824813, 606.5630025125145, 517.8107876199931, 642.5147617953756], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

100%|█████████▉| 999997/1000000 [11:45:20<00:00, 32.95it/s]global step 1000000, trans_decision ep_re 504.7250030999786

{"global_step": 1000000, "eval_re": [602.7428808603632, 704.9159766402421, 
568.0365300497791, 433.7876315386958, 414.37436342745434, 551.5191885571601, 
9.452431220636619, 574.5609346209368, 713.4306335410967, 474.4294605434209], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

100%|██████████| 1000000/1000000 [11:45:40<00:00, 23.62it/s]
