
{
    'exp_name': 'VDPO',
    'env': 'HalfCheetah-v4',
    'gamma': 0.99,
    'device': device(type='cuda'),
    'seed': 0,
    'buffer_size': 1000000,
    'total_timesteps': 1000000,
    'learn_start': 5000,
    'batch_size': 256,
    'actor_lr': 0.0003,
    'critic_lr': 0.001,
    'alpha_lr': 0.001,
    'target_update_factor': 0.005,
    'actor_freq': 2,
    'target_freq': 1,
    'eval_freq': 10000,
    'eval_num': 10,
    'kl_freq': 1000,
    'belief_lr': 0.0003,
    'embedding_dim': 256,
    'n_steps': 3,
    'delay': 32,
    'delayspec': 'ExtremeSparseL4U32::markov(4, 32, [[249, 1], [1, 31]])'
}
✓ setup
Created Delay Process: Markovian(ConstantDelay4, ConstantDelay32, [[0.996, 
0.004], [0.03125, 0.96875]])
  1%|          | 9999/1000000 [03:30<7:41:14, 35.77it/s]global step 10000, trans_decision ep_re -42.642817860003625

{"global_step": 10000, "eval_re": [-10.789635190638272, -28.04212009154307, 
-45.430830074143906, -61.21333689263718, -45.546449902802394, 
-93.71131179005977, -39.01589758015313, -27.79269147188396, -39.843441160511325,
-35.0424644456632], "eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000,
1000, 1000]}

  2%|▏         | 19996/1000000 [10:10<7:32:15, 36.11it/s]global step 20000, trans_decision ep_re 332.494003405378

{"global_step": 20000, "eval_re": [429.9969047398494, 489.78788408598757, 
176.67918991320164, 194.14515231125634, 507.7320922942095, 112.4908795115233, 
375.10003733970973, 529.0882647751196, 548.1446341926982, -38.22500510977518], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  3%|▎         | 29996/1000000 [17:00<7:25:10, 36.31it/s]global step 30000, trans_decision ep_re 638.2767044339257

{"global_step": 30000, "eval_re": [933.7248436209276, 748.4276416385026, 
726.0834078524323, 548.3364322815856, 506.86349541437437, 461.115754533262, 
697.8356211471684, 1039.4395116017379, 490.5604252324852, 230.37991101677974], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  4%|▍         | 39996/1000000 [23:40<7:20:55, 36.29it/s]global step 40000, trans_decision ep_re 886.2827813048673

{"global_step": 40000, "eval_re": [708.304424635499, 1026.0944075229133, 
920.1936016670544, 932.2949986738752, 953.0826228036624, 993.0246411739156, 
643.5902243805617, 855.9664264379984, 884.5236177878245, 945.7528479653674], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  5%|▍         | 49996/1000000 [30:30<7:21:08, 35.89it/s]global step 50000, trans_decision ep_re 1190.8331331881234

{"global_step": 50000, "eval_re": [1076.3107327027135, 1364.2038549683061, 
1258.5564765156073, 1286.7871248771376, 1233.098315956739, 1029.7902212297515, 
964.7658677699825, 1130.0749278212265, 1213.0666782142525, 1351.677131825517], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  6%|▌         | 59999/1000000 [37:10<7:11:17, 36.32it/s]global step 60000, trans_decision ep_re 1384.0836020918462

{"global_step": 60000, "eval_re": [1249.7718979734595, 1135.1464740535507, 
2049.6779805892197, 1895.0887678151867, 1138.6547842381551, 1378.0410542212976, 
1539.4485680710793, 1162.388855062369, 1256.564696129111, 1036.0529427650308], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  7%|▋         | 69996/1000000 [44:00<7:09:43, 36.07it/s]global step 70000, trans_decision ep_re 1278.4715483309549

{"global_step": 70000, "eval_re": [1228.1368510056286, 1368.8594003108385, 
1168.3885591587198, 1889.9601198583996, 1115.9409059254378, 1189.4770786441402, 
1481.101366781879, 1115.7906290067492, 1094.515058079832, 1132.545514537923], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  8%|▊         | 79996/1000000 [50:40<7:04:51, 36.09it/s]global step 80000, trans_decision ep_re 1550.3477273448434

{"global_step": 80000, "eval_re": [1604.9948664648427, 1388.7905356303422, 
1461.7025041421277, 1660.624990144307, 1076.1846923113956, 1626.3439848898242, 
2686.9161242987516, 1497.240206575492, 1099.7012119630913, 1400.9781570282612], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

  9%|▉         | 89999/1000000 [57:30<6:58:48, 36.21it/s]global step 90000, trans_decision ep_re 1446.4945260869358

{"global_step": 90000, "eval_re": [1993.8979767823544, 1604.3782098074455, 
1671.1067201691421, 2182.829766496143, 1640.7856558976773, 1143.6482118909378, 
240.61438540802433, 1363.7411348616206, 1342.9043771294616, 1281.0388224265507],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 10%|▉         | 99996/1000000 [1:04:10<6:52:53, 36.33it/s]global step 100000, trans_decision ep_re 1362.8227833578449

{"global_step": 100000, "eval_re": [1405.9433030922266, 818.513029082893, 
2114.046990337019, 1520.811700169667, 1326.2805312733851, 701.7309355235799, 
1486.877663237084, 1618.246420825842, 1537.1277397005226, 1098.6495203362283], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 11%|█         | 109996/1000000 [1:10:50<6:47:12, 36.43it/s]global step 110000, trans_decision ep_re 1817.032322650009

{"global_step": 110000, "eval_re": [1172.393042259462, 2943.433674592539, 
1448.94406066192, 2597.518752090054, 1437.8392729372822, 1418.9672823776332, 
1848.1410324320598, 1763.8541893745207, 1903.206932211401, 1636.0249875632185], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 12%|█▏        | 119996/1000000 [1:17:40<8:50:38, 27.64it/s]global step 120000, trans_decision ep_re 1866.2690038019505

{"global_step": 120000, "eval_re": [1934.724894789081, 1343.9789781465129, 
1466.1183585089136, 3703.187861038665, 2579.8237385780913, 1384.9877635127636, 
2235.930226124211, 1230.5509126464308, 1448.9141926055845, 1334.4731120692525], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 13%|█▎        | 129996/1000000 [1:24:30<6:41:44, 36.09it/s]global step 130000, trans_decision ep_re 2769.4089951334086

{"global_step": 130000, "eval_re": [3674.463408418186, 1900.680959965522, 
3338.161297135299, 3670.6776936403367, 3594.029348184166, 2216.0128365162536, 
1092.6480932429065, 3722.863071502795, 2042.0757344509864, 2442.477508277633], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 14%|█▍        | 139999/1000000 [1:31:32<6:34:42, 36.31it/s]global step 140000, trans_decision ep_re 2744.8604491271967

{"global_step": 140000, "eval_re": [3887.953427556775, 2205.223071903972, 
2084.074743440713, 3950.7158758250303, 3112.068753239171, 2639.85723454375, 
3357.6454091414284, 2294.4658409896642, 2829.825599342312, 1086.7745352891525], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 15%|█▍        | 149999/1000000 [1:38:22<6:33:27, 36.01it/s]global step 150000, trans_decision ep_re 1658.5984742823482

{"global_step": 150000, "eval_re": [1393.9916311758348, 1630.834968886763, 
1656.0024540890054, 1809.1231215223672, 1675.0730223304931, 1347.0851957652078, 
1887.612945720113, 1717.143427132581, 1960.9368008059232, 1508.1811753951918], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 16%|█▌        | 159996/1000000 [1:45:02<6:26:10, 36.25it/s]global step 160000, trans_decision ep_re 1431.358293187072

{"global_step": 160000, "eval_re": [1568.5646153156554, 1644.1337858708212, 
1319.7033517862421, 1565.061390501821, 1319.934350564564, 1336.794450654635, 
1421.530728311461, 1340.3882430739977, 1401.8007916274437, 1395.6712241640776], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 17%|█▋        | 169996/1000000 [1:51:52<6:22:19, 36.18it/s]global step 170000, trans_decision ep_re 1751.694966639793

{"global_step": 170000, "eval_re": [1725.332695434614, 2910.2090759404127, 
2446.548447011565, 1300.6635301241236, 1382.3859185328515, 1360.7751686351548, 
1812.0852317198041, 1114.7146536955504, 1353.104167650794, 2111.1307776530575], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 18%|█▊        | 179996/1000000 [1:58:32<6:17:28, 36.21it/s]global step 180000, trans_decision ep_re 2803.241336943328

{"global_step": 180000, "eval_re": [4357.433629661244, 2818.9083507482087, 
2210.150471420366, 2075.6466790214345, 1236.8883629953812, 3644.448544584007, 
3994.049174383052, 1769.9364289141593, 3655.145099516405, 2269.8066281890187], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 19%|█▉        | 189996/1000000 [2:05:22<6:11:41, 36.32it/s]global step 190000, trans_decision ep_re 2483.0158476225815

{"global_step": 190000, "eval_re": [3112.345559543035, 1620.6699406758398, 
4137.404991261381, 4050.61741052594, 1824.5076831704512, 1246.9928432628592, 
1682.8275607676803, 2209.8249361916073, 3112.1202870175875, 1832.8472638094365],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 20%|█▉        | 199996/1000000 [2:12:13<6:07:31, 36.28it/s]global step 200000, trans_decision ep_re 3114.909056952754

{"global_step": 200000, "eval_re": [3991.212773338075, 3976.2857546288196, 
1722.6891100966116, 3959.1207399594055, 2125.3532545709922, 3174.7294815505948, 
3893.337713248111, 2244.170750871334, 1998.4549041750372, 4063.73608708856], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 21%|██        | 209996/1000000 [2:18:53<6:02:52, 36.28it/s]global step 210000, trans_decision ep_re 2195.3225784212846

{"global_step": 210000, "eval_re": [2673.265660337015, 1203.683739579758, 
2169.7192589893093, 2775.808106196014, 2063.4212340967315, 2504.8286344439693, 
1566.8385098573838, 1695.9923833757568, 1896.9607397809116, 3402.707517555997], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 22%|██▏       | 219996/1000000 [2:25:43<5:58:40, 36.25it/s]global step 220000, trans_decision ep_re 2098.7303044542105

{"global_step": 220000, "eval_re": [1277.4054949525996, 1317.4559083998834, 
1911.6022506550953, 1233.5750359596695, 4312.2746502171185, 1711.6452679361682, 
1383.04686069177, 1374.5111799908143, 3096.3578338761704, 3369.4285618628196], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 23%|██▎       | 229996/1000000 [2:32:23<5:52:56, 36.36it/s]global step 230000, trans_decision ep_re 2086.4766211091805

{"global_step": 230000, "eval_re": [1741.2868979445034, 2378.77776392472, 
1188.7604435315836, 1247.2758929625759, 1975.612069513107, 3923.980878913344, 
1033.9687311390803, 2939.23926513318, 2893.6683593960674, 1542.1959086336458], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 24%|██▍       | 239999/1000000 [2:39:13<5:47:45, 36.42it/s]global step 240000, trans_decision ep_re 2915.815818810842

{"global_step": 240000, "eval_re": [1710.4953201098126, 3455.6094618484067, 
3886.3232663338617, 4010.414584091882, 1611.7599683403882, 2554.8619105963226, 
3989.43981389213, 3399.192158385666, 1990.5555750144401, 2549.5061294955035], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 25%|██▍       | 249996/1000000 [2:45:53<5:44:43, 36.26it/s]global step 250000, trans_decision ep_re 2471.6163589853595

{"global_step": 250000, "eval_re": [2340.7837610741744, 2092.5314352346513, 
1534.9958639228155, 3716.2507625140465, 2026.1521903192936, 3092.819440864906, 
2506.5528932183793, 3829.086168896768, 1368.1386271551053, 2208.852446653454], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 26%|██▌       | 259996/1000000 [2:52:43<5:40:30, 36.22it/s]global step 260000, trans_decision ep_re 2342.093406120824

{"global_step": 260000, "eval_re": [2473.526328695819, 2630.393994220473, 
1944.9079871929735, 1876.1143121053467, 2457.50714514247, 3659.8597844113583, 
2471.285447989086, 1362.4886208328219, 1841.2250930773653, 2703.6253475405288], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 27%|██▋       | 269996/1000000 [2:59:23<5:36:43, 36.13it/s]global step 270000, trans_decision ep_re 2134.055290037778

{"global_step": 270000, "eval_re": [2705.126351355529, 2044.9562197547712, 
1788.247969377551, 1951.943099185774, 1453.610933608954, 1241.8385082155203, 
2174.7753171018717, 4258.351473056164, 2227.403726143562, 1494.2993025780804], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 28%|██▊       | 279996/1000000 [3:06:03<5:33:06, 36.03it/s]global step 280000, trans_decision ep_re 2422.4813135466147

{"global_step": 280000, "eval_re": [1468.2189560788656, 2592.5785521554353, 
1559.1499187796958, 3125.08314431162, 2314.8322010670036, 4060.737560593324, 
1376.063894464093, 1175.0947096073714, 2506.7837809007706, 4046.2704175079616], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 29%|██▉       | 289996/1000000 [3:12:53<5:25:30, 36.35it/s]global step 290000, trans_decision ep_re 3138.517644919306

{"global_step": 290000, "eval_re": [4430.297330101169, 4453.931019921651, 
1356.714660514362, 2329.9016123778893, 2396.428335966426, 1246.6768335735496, 
4556.039402305355, 4496.484759255417, 4442.851799333685, 1675.850695843555], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 30%|██▉       | 299996/1000000 [3:19:33<5:19:54, 36.47it/s]global step 300000, trans_decision ep_re 2745.7506136861657

{"global_step": 300000, "eval_re": [2767.2954202421256, 3162.8185229902742, 
2589.0726317020035, 1217.0296955219658, 4388.225112143943, 3804.5381766639, 
1465.172352191066, 4500.820343255646, 1227.3019931061572, 2335.2318890445736], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 31%|███       | 309996/1000000 [3:26:23<5:18:59, 36.05it/s]global step 310000, trans_decision ep_re 3664.296780212405

{"global_step": 310000, "eval_re": [2547.651348203534, 4202.344710691168, 
4391.4417300392415, 3587.783214973152, 4087.872734100453, 4202.786170954898, 
4142.722456566342, 1320.3017609349224, 4193.789122572214, 3966.274553088122], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 32%|███▏      | 319996/1000000 [3:33:03<5:11:22, 36.40it/s]global step 320000, trans_decision ep_re 1949.58326994472

{"global_step": 320000, "eval_re": [1797.9144846439551, 1461.7484080928364, 
1490.3958345806384, 2775.435492163073, 2252.8919035487565, 2042.6272610539718, 
1566.8721134874597, 2034.9884313754421, 2135.38254330294, 1937.5762271981246], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 33%|███▎      | 329996/1000000 [3:39:43<5:07:42, 36.29it/s]global step 330000, trans_decision ep_re 2177.7535575220572

{"global_step": 330000, "eval_re": [1424.6305620332926, 1253.9770496532942, 
3355.779371927764, 1189.2207953764187, 2623.4523060036013, 1629.3977129633574, 
4422.952975488998, 1886.2783550023405, 1623.8868374877206, 2367.959609283786], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 34%|███▍      | 339996/1000000 [3:46:33<5:02:50, 36.32it/s]global step 340000, trans_decision ep_re 1664.7837611653001

{"global_step": 340000, "eval_re": [1409.6875264531172, 1243.7155968936543, 
1408.9779789371682, 1996.1981193581225, 2226.2044508680806, 2099.203268370635, 
1600.0530212223146, 1522.9783646823864, 1682.1081053347461, 1458.7111795327764],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 35%|███▍      | 349996/1000000 [3:53:13<4:57:56, 36.36it/s]global step 350000, trans_decision ep_re 1372.033715300709

{"global_step": 350000, "eval_re": [1458.9337266670182, 1549.1378512234617, 
1134.347453988741, 1771.0555227830398, 304.2579051569246, 1825.4596080813105, 
1672.4283218913079, 1441.6615119330968, 1114.2227237821871, 1448.8325275000027],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 36%|███▌      | 359996/1000000 [4:00:03<4:54:06, 36.27it/s]global step 360000, trans_decision ep_re 1970.7414966915617

{"global_step": 360000, "eval_re": [2959.472643728831, 2418.335651888633, 
1303.0051519938504, 2848.3791035045865, 1836.5935830349752, 1201.2487805059013, 
1732.0346597903804, 1576.1085517562221, 1421.3179000840016, 2410.9189406282358],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 37%|███▋      | 369996/1000000 [4:06:43<4:52:51, 35.85it/s]global step 370000, trans_decision ep_re 1456.5541521008156

{"global_step": 370000, "eval_re": [535.5295647524923, 1642.1386586449935, 
1733.0472872832745, 1612.5145096524639, 1174.4100530867818, 1461.831383543053, 
1544.9606518867026, 1592.0381437300832, 1376.211736189132, 1892.8595322391795], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 38%|███▊      | 379996/1000000 [4:13:23<4:45:56, 36.14it/s]global step 380000, trans_decision ep_re 3277.2042302827736

{"global_step": 380000, "eval_re": [2259.1965317388654, 4436.689264006359, 
4512.997724500681, 1304.4341073405792, 4453.040171560894, 2766.8757054862162, 
3447.7382387449748, 4077.3227872669445, 4416.339371583386, 1097.4084005988313], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 39%|███▉      | 389996/1000000 [4:20:13<4:41:04, 36.17it/s]global step 390000, trans_decision ep_re 2135.8258164095996

{"global_step": 390000, "eval_re": [1805.12824302363, 2198.8884546314557, 
1303.5830877997832, 3566.9789850572192, 1597.816194184872, 1112.1336315637716, 
1384.7559111478163, 2248.857503443201, 2049.841372111888, 4090.27478113236], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 40%|███▉      | 399999/1000000 [4:26:53<4:34:12, 36.47it/s]global step 400000, trans_decision ep_re 2855.8534470811987

{"global_step": 400000, "eval_re": [4509.016500222107, 1349.4768401250562, 
2572.2632203748585, 2369.18450755124, 4259.741697291533, 3299.2171774086605, 
1262.7737672031037, 2819.3012192244573, 4075.555385456246, 2042.0041559547244], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 41%|████      | 409996/1000000 [4:33:43<4:32:46, 36.05it/s]global step 410000, trans_decision ep_re 1647.1552576895351

{"global_step": 410000, "eval_re": [1613.530558728477, 1388.7202794833834, 
1715.1348402014753, 1835.7873373437778, 1846.3949226858442, 1879.2231305734304, 
1295.6626779557357, 1523.7866325611624, 1417.3970528457803, 1955.9151445162843],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 42%|████▏     | 419996/1000000 [4:40:23<4:27:12, 36.18it/s]global step 420000, trans_decision ep_re 1687.9246856738123

{"global_step": 420000, "eval_re": [1574.4386241684037, 2100.098582818466, 
1205.9969622692408, 2117.1227851996496, 2018.833403473092, 1609.8320962361183, 
1531.7196156349435, 1186.9175789329734, 2372.0969435036077, 1162.190264501626], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 43%|████▎     | 429996/1000000 [4:47:03<4:22:07, 36.24it/s]global step 430000, trans_decision ep_re 1739.799259427681

{"global_step": 430000, "eval_re": [1186.7081270116607, 1184.8173715832502, 
2058.6539386997515, 1573.2816345859167, 1453.1013792086464, 2587.082225291219, 
2105.2379768210417, 1247.7193565440966, 2368.6369049059663, 1632.7536796252596],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 44%|████▍     | 439996/1000000 [4:53:53<4:18:27, 36.11it/s]global step 440000, trans_decision ep_re 2056.2382724318936

{"global_step": 440000, "eval_re": [1734.694374877102, 2210.024639997388, 
1840.6723632495296, 2419.2938345765015, 1662.55186021293, 1388.6276906634948, 
1890.8578131942256, 1227.533899067874, 3145.365843225219, 3042.7604052546685], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 45%|████▍     | 449996/1000000 [5:00:34<4:12:01, 36.37it/s]global step 450000, trans_decision ep_re 1538.312951231992

{"global_step": 450000, "eval_re": [1559.3598600774314, 1937.7096659062406, 
1314.8878240168717, 1659.2149059904582, 1271.690346639612, 1598.9537834698858, 
1246.5967132079672, 1513.6311498356347, 1927.4950120658257, 1353.5902511099919],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 46%|████▌     | 459996/1000000 [5:07:24<4:07:40, 36.34it/s]global step 460000, trans_decision ep_re 2377.1541452357314

{"global_step": 460000, "eval_re": [2459.793933785744, 1780.6259784964031, 
2778.4836105185627, 2779.1600087698253, 1637.5305713019682, 2995.5710364674046, 
2836.463747786465, 1595.939179338566, 2269.346569223471, 2638.6268166689024], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 47%|████▋     | 469996/1000000 [5:14:04<4:04:31, 36.12it/s]global step 470000, trans_decision ep_re 1674.9781478920336

{"global_step": 470000, "eval_re": [1441.8933333965058, 1712.9768626324553, 
2491.2136077461055, 1384.1032128297215, 2252.2805607328705, 1593.9575182342487, 
1407.1385076714846, 1537.383754005304, 1520.7564331814797, 1408.0776884901597], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 48%|████▊     | 479996/1000000 [5:20:44<3:59:40, 36.16it/s]global step 480000, trans_decision ep_re 3489.8331685580497

{"global_step": 480000, "eval_re": [4596.168067434227, 2708.321956823353, 
3956.4785636197034, 2572.587096250348, 3118.799230796346, 4624.32636951549, 
4104.1436528358245, 4583.54405262541, 2023.202254674949, 2610.760441004846], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 49%|████▉     | 489996/1000000 [5:27:34<3:54:38, 36.23it/s]global step 490000, trans_decision ep_re 1557.2865478313497

{"global_step": 490000, "eval_re": [1529.9603591597609, 1245.84374504243, 
2130.868373199951, 1612.123723071949, 1203.712347422411, 1268.8522633337757, 
1490.0778746692927, 1103.883253963412, 1889.3670113704613, 2098.176527080053], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 50%|████▉     | 499996/1000000 [5:34:14<3:49:03, 36.38it/s]global step 500000, trans_decision ep_re 2333.608682520116

{"global_step": 500000, "eval_re": [1320.7156452108786, 1764.3150820681906, 
3532.083745721423, 3441.4044820433774, 2016.587569289467, 3267.4585148223605, 
2873.750310874318, 1253.3826125167097, 2370.8180424446678, 1495.5708202097655], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 51%|█████     | 509996/1000000 [5:41:04<3:44:50, 36.32it/s]global step 510000, trans_decision ep_re 1965.426510580622

{"global_step": 510000, "eval_re": [3456.135731073346, 1428.1461131888595, 
1604.7387372898584, 5.2186713692792965, 4123.673849552272, 1708.2772284794441, 
1689.324606297788, 2573.889600622871, 1835.1544217442022, 1229.7061461883018], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 52%|█████▏    | 519996/1000000 [5:47:44<3:41:18, 36.15it/s]global step 520000, trans_decision ep_re 2333.146146549124

{"global_step": 520000, "eval_re": [2153.8584579792946, 1419.268582134267, 
3254.838918767943, 3256.7385883788356, 3004.2330763399573, 3010.69984684323, 
1431.2185228741234, 1241.6329892923459, 1292.0204026473978, 3266.952080233841], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 53%|█████▎    | 529996/1000000 [5:54:34<3:36:38, 36.16it/s]global step 530000, trans_decision ep_re 1987.1571900172971

{"global_step": 530000, "eval_re": [1939.6547812447436, 2421.388681535741, 
3185.70574111851, 1736.7837724254155, 1647.2564934999796, 1577.109803919562, 
1319.5941151991694, 2676.5371920357807, 1614.885732470105, 1752.6555867239633], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 54%|█████▍    | 539996/1000000 [6:01:14<3:31:00, 36.33it/s]global step 540000, trans_decision ep_re 1941.7598649468182

{"global_step": 540000, "eval_re": [2132.7917759368606, 1420.2566150659104, 
1928.8590316032685, 2000.3488276150083, 2040.296999678655, 2480.245984931901, 
2221.468810496778, 1470.1162705209993, 1480.61027564404, 2242.6040579747596], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 55%|█████▍    | 549999/1000000 [6:07:54<3:26:56, 36.24it/s]global step 550000, trans_decision ep_re 1896.714471072144

{"global_step": 550000, "eval_re": [1899.8844774809374, 1506.5972893390208, 
2013.5277373335612, 1580.4457595391602, 1424.0227645598247, 1472.6455631844362, 
2357.603067265716, 1395.5458154091646, 3873.65832328861, 1443.213913321008], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 56%|█████▌    | 559996/1000000 [6:14:44<3:22:36, 36.20it/s]global step 560000, trans_decision ep_re 1647.350042752476

{"global_step": 560000, "eval_re": [1567.6640856049162, 1180.789650951936, 
1469.6730648129492, 3063.9621606373476, 1878.2882789684058, 1836.0924524277643, 
1332.7277006201957, 1679.9369768982604, 1186.7952602381476, 1277.5707963648356],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 57%|█████▋    | 569996/1000000 [6:21:24<3:18:50, 36.04it/s]global step 570000, trans_decision ep_re 1647.7009954364719

{"global_step": 570000, "eval_re": [1250.2193487892089, 2312.7074480705123, 
1261.9664455791583, 2264.747681640438, 1629.6748120121658, 1308.1378103477907, 
1443.9056449663983, 2342.0509259023183, 1218.1761774331553, 1445.4236596235723],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 58%|█████▊    | 579996/1000000 [6:28:14<3:14:38, 35.96it/s]global step 580000, trans_decision ep_re 1824.8506410995408

{"global_step": 580000, "eval_re": [1986.9422825443032, 2092.8190439263108, 
1843.9079203259803, 2889.424959877663, 1239.9567270316727, 2137.785973923789, 
1422.4984172277607, 1283.6550674770726, 1255.6259379615176, 2095.890080699336], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 59%|█████▉    | 589996/1000000 [6:34:54<3:09:56, 35.98it/s]global step 590000, trans_decision ep_re 2274.1026002263175

{"global_step": 590000, "eval_re": [2551.610704551466, 2065.854081567417, 
2735.873077994134, 2226.178805545311, 1654.5081745984999, 1959.8581850344406, 
2464.9753615029776, 2601.9584848803765, 1550.6086903970465, 2929.6004361915047],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 60%|█████▉    | 599996/1000000 [6:41:34<3:03:47, 36.27it/s]global step 600000, trans_decision ep_re 1869.3164953042574

{"global_step": 600000, "eval_re": [2936.13675711063, 1382.9150656649751, 
2447.9550060371553, 1416.0597668514733, 1282.2835925709142, 1712.1251111045804, 
1314.6812794583514, 1267.7209385005967, 2376.0669870640013, 2557.220448679899], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 61%|██████    | 609996/1000000 [6:48:24<2:59:01, 36.31it/s]global step 610000, trans_decision ep_re 1690.7971888364204

{"global_step": 610000, "eval_re": [1441.6419304607432, 2101.1222040413368, 
1655.266777680109, 2102.0486175994292, 1512.1337954504932, 1073.2191214636125, 
1273.768937761584, 2129.994117084425, 2040.0743329946936, 1578.7020538277802], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 62%|██████▏   | 619996/1000000 [6:55:04<2:54:02, 36.39it/s]global step 620000, trans_decision ep_re 1908.7526747827524

{"global_step": 620000, "eval_re": [2193.924037008791, 2984.2843053972365, 
1269.7746158469563, 1515.7153000556143, 1351.5499195678847, 1579.9215475681099, 
2623.683779875608, 1582.4871829618605, 2993.426272256181, 992.759787289284], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 63%|██████▎   | 629996/1000000 [7:01:54<2:52:00, 35.85it/s]global step 630000, trans_decision ep_re 1856.1974332119753

{"global_step": 630000, "eval_re": [1378.7465944929763, 1973.2774134246647, 
1634.871837384867, 1968.6616728217402, 2182.8291823156706, 2785.025769701183, 
1569.3754744570433, 1202.9554235678784, 2069.149913520728, 1797.0810504330038], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 64%|██████▍   | 639996/1000000 [7:08:34<2:45:41, 36.21it/s]global step 640000, trans_decision ep_re 2054.6515102615335

{"global_step": 640000, "eval_re": [1490.578080996995, 1975.226509331814, 
2336.9378164849827, 2342.388051146708, 2274.0713943995147, 1934.7499337122845, 
1873.6864892151723, 2260.4862935750734, 1998.631761872454, 2059.75877188034], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 65%|██████▍   | 649996/1000000 [7:15:14<2:42:23, 35.92it/s]global step 650000, trans_decision ep_re 1815.675787136165

{"global_step": 650000, "eval_re": [1281.0694568962642, 2383.948102517408, 
11.637980290884643, 2461.563900552132, 1635.5054795221392, 3257.2358068635826, 
1288.7702909180991, 1777.4123967039982, 1850.2207322795803, 2209.3937248175607],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 66%|██████▌   | 659996/1000000 [7:22:04<2:36:54, 36.12it/s]global step 660000, trans_decision ep_re 1993.6059643818476

{"global_step": 660000, "eval_re": [1938.14856483579, 1674.567554876718, 
2316.9567537179705, 1480.891808825956, 1786.262248260541, 2238.2365157289087, 
2454.9759482342797, 1670.0152601932043, 2446.7147600107733, 1929.290229134335], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 67%|██████▋   | 669999/1000000 [7:28:44<2:32:21, 36.10it/s]global step 670000, trans_decision ep_re 1461.403400986709

{"global_step": 670000, "eval_re": [1342.2099744859956, 1460.9042549979324, 
1443.9276834704403, 1546.9418896864918, 2067.670068206467, 1152.8859502627756, 
2444.6187010674603, 1357.3534199087012, 295.35515761316225, 1502.166910167662], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 68%|██████▊   | 679996/1000000 [7:35:34<2:27:25, 36.18it/s]global step 680000, trans_decision ep_re 2453.1633869302573

{"global_step": 680000, "eval_re": [1830.7612642671943, 3097.361282666281, 
2142.889905132633, 2740.306911906928, 2314.354674605053, 3121.281909326982, 
1962.09342923924, 2812.7945856976303, 1625.4636048631876, 2884.326301597442], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 69%|██████▉   | 689999/1000000 [7:42:14<2:23:43, 35.95it/s]global step 690000, trans_decision ep_re 1770.4524986311903

{"global_step": 690000, "eval_re": [1369.8590361478202, 1209.582285338201, 
1612.2121514806136, 2267.4966570498227, 2371.9836648537403, 1808.9942367128551, 
2346.686136877345, 1858.0530421619628, 1374.149366811527, 1485.5084088780118], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 70%|██████▉   | 699996/1000000 [7:49:04<2:17:57, 36.24it/s]global step 700000, trans_decision ep_re 1731.7630472341946

{"global_step": 700000, "eval_re": [1654.380782628366, 2066.4366363264235, 
1604.7835092921105, 1789.9661212880299, 1860.2522330211764, 1764.3977885891243, 
1346.2334887384554, 2506.295631471809, 1320.4303883612208, 1404.4538926252308], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 71%|███████   | 709996/1000000 [7:55:44<2:13:24, 36.23it/s]global step 710000, trans_decision ep_re 1784.9800120777375

{"global_step": 710000, "eval_re": [2065.1116594897712, 2175.672767808775, 
1958.6875026011799, 2136.590429430685, 2707.749794290486, 1378.6212340480033, 
1293.1989523464804, 1503.0845869250954, 1196.4696341236765, 1434.6135597132195],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 72%|███████▏  | 719996/1000000 [8:02:24<2:08:29, 36.32it/s]global step 720000, trans_decision ep_re 1640.7502502958116

{"global_step": 720000, "eval_re": [1734.598366085519, 1733.9746125900406, 
1211.1261091569975, 1824.914344886229, 1474.9676093913843, 2266.3463524277913, 
1746.5273328586597, 1534.1386622155096, 1238.4822892719276, 1642.4268240740555],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 73%|███████▎  | 729996/1000000 [8:09:14<2:04:45, 36.07it/s]global step 730000, trans_decision ep_re 1602.295147173238

{"global_step": 730000, "eval_re": [2436.3655171472406, 1364.2190235076753, 
1407.1127008854382, 1485.3189972346904, 1476.4684815809396, 1662.3127180838678, 
1235.1101890147033, 1389.6286241596238, 1797.6831825878592, 1768.7320375303432],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 74%|███████▍  | 739996/1000000 [8:15:54<1:59:29, 36.27it/s]global step 740000, trans_decision ep_re 1735.2082757638368

{"global_step": 740000, "eval_re": [2479.7864219920325, 1654.099095120019, 
2534.70354197304, 1626.2919244811799, 1410.8463118654645, 1281.347117974965, 
2506.36626122054, 1295.2236127560695, 1315.1351557615, 1248.2833144935576], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 75%|███████▍  | 749996/1000000 [8:22:34<1:55:10, 36.18it/s]global step 750000, trans_decision ep_re 1394.0265206262004

{"global_step": 750000, "eval_re": [1318.1190967068737, 1902.0145860361954, 
1381.666700317472, 1381.1631658299998, 1316.5023282143554, 526.3676368268708, 
1490.4577769347961, 1395.9383069569026, 1298.6599910640225, 1929.375617374516], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 76%|███████▌  | 759996/1000000 [8:29:24<1:50:00, 36.36it/s]global step 760000, trans_decision ep_re 1992.0875497829177

{"global_step": 760000, "eval_re": [1229.1909156043876, 2872.911107200754, 
1614.4216126294684, 1733.0340388493685, 2214.0396979740967, 2996.7800231146884, 
1292.7623843852105, 2763.070511748676, 1323.444480701846, 1881.220725620681], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 77%|███████▋  | 769996/1000000 [8:36:04<1:45:26, 36.35it/s]global step 770000, trans_decision ep_re 1481.1189556300565

{"global_step": 770000, "eval_re": [1454.9843020441563, 1277.0749650314003, 
1698.7252420315724, 1332.3057509627197, 2039.1197934678387, 1499.3524409351212, 
1839.3467262868814, 1212.7542163322628, 1214.4021094557993, 1243.1240097528107],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 78%|███████▊  | 779996/1000000 [8:42:54<1:41:17, 36.20it/s]global step 780000, trans_decision ep_re 1703.993362156978

{"global_step": 780000, "eval_re": [1365.6246154585074, 1653.7362757386843, 
1368.2147660318078, 1490.4878203123735, 1396.87820466374, 1678.9051691793777, 
1447.9806778928457, 2581.2528121082414, 2592.121064263982, 1464.7322159202217], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 79%|███████▉  | 789996/1000000 [8:49:34<1:37:00, 36.08it/s]global step 790000, trans_decision ep_re 1112.1353111921326

{"global_step": 790000, "eval_re": [1160.9973536667517, 1453.3705760911234, 
1441.3962689593238, 1352.7544360392342, 1362.5733194655745, 1353.0418442539317, 
1391.9357141277812, 1529.583772963749, -0.28206423438903183, 75.98189058824467],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 80%|███████▉  | 799996/1000000 [8:56:14<1:32:03, 36.21it/s]global step 800000, trans_decision ep_re 1612.870396567261

{"global_step": 800000, "eval_re": [1316.8815674808066, 1630.023068744138, 
1223.60056645968, 1619.100091752515, 1456.7305395567762, 1428.4012638936658, 
2079.4106641443022, 1875.2748458451592, 2384.1484347970554, 1115.1329229985113],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 81%|████████  | 809996/1000000 [9:03:04<1:27:56, 36.01it/s]global step 810000, trans_decision ep_re 1766.7310878088206

{"global_step": 810000, "eval_re": [2351.4600736659863, 1441.6673342052736, 
1219.941467671335, 2575.0812475447683, 1515.698473555588, 1816.6171339041698, 
1166.5522025707442, 1777.3132821711195, 2220.7546201545124, 1582.225042644709], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 82%|████████▏ | 819996/1000000 [9:09:44<1:22:31, 36.35it/s]global step 820000, trans_decision ep_re 1408.7048390521297

{"global_step": 820000, "eval_re": [1446.4416610438793, 1292.643429020917, 
1257.9099538500138, 1487.0463557105613, 1883.3646676514606, 1186.5257903757833, 
1336.3256101612435, 1211.5907432425163, 1539.4614915398538, 1445.7386879250676],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 83%|████████▎ | 829996/1000000 [9:16:34<1:18:47, 35.96it/s]global step 830000, trans_decision ep_re 1464.8951236993012

{"global_step": 830000, "eval_re": [1676.3761117455144, 1607.4255902327736, 
1311.04907237274, 1847.958919727483, 1525.8773887579775, 1631.8987141373354, 
1185.6250014926566, 1217.9881561127813, 1231.087513993595, 1413.664768420158], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 84%|████████▍ | 839996/1000000 [9:23:24<1:13:36, 36.23it/s]global step 840000, trans_decision ep_re 1380.8518178909117

{"global_step": 840000, "eval_re": [1822.4507745126411, 1407.9927061990902, 
1163.3601449172158, 1237.9419396462695, 1638.11449013219, 1136.5064850455774, 
1622.5325545947849, 1222.128380262465, 1220.1678917489767, 1337.3228118499053], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 85%|████████▍ | 849996/1000000 [9:30:14<1:09:02, 36.21it/s]global step 850000, trans_decision ep_re 1729.1585750363279

{"global_step": 850000, "eval_re": [1387.1124692015014, 1870.734472106214, 
3161.8936421174803, 1546.6365772502684, 1329.9808438750658, 1413.6297680605155, 
2357.4461114882447, 1465.9129001757506, 1469.561943396534, 1288.677022691708], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 86%|████████▌ | 859996/1000000 [9:37:04<1:04:31, 36.16it/s]global step 860000, trans_decision ep_re 1758.9788715233906

{"global_step": 860000, "eval_re": [1277.6192160413575, 1388.705040628937, 
2505.2224287412737, 1483.032214299422, 2351.314596908755, 1639.9102191994566, 
1944.9788868374062, 1730.644747489028, 1289.0959333729777, 1979.2654317152906], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 87%|████████▋ | 869996/1000000 [9:43:54<59:42, 36.29it/s]global step 870000, trans_decision ep_re 1540.7883262984778

{"global_step": 870000, "eval_re": [1155.203296841429, 2328.915559050131, 
1220.8814865993882, 1142.9186748482323, 1686.8112336217812, 2199.2075416822495, 
1783.8614987311873, 1348.991920627633, 1239.8577648969951, 1301.2342860857514], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 88%|████████▊ | 879996/1000000 [9:50:34<55:14, 36.21it/s]global step 880000, trans_decision ep_re 1604.0043908079338

{"global_step": 880000, "eval_re": [1589.6661154067979, 1408.8077812117754, 
1445.809673815554, 1408.850890125256, 2215.003617047458, 1293.3143117930344, 
1348.53588811314, 2255.365601498052, 1299.226345252206, 1775.4636838160636], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 89%|████████▉ | 889996/1000000 [9:57:24<51:01, 35.93it/s]global step 890000, trans_decision ep_re 1474.60517148351

{"global_step": 890000, "eval_re": [1672.3101223676017, 1593.134161052922, 
1377.5556791365423, 1343.3176380164186, 1658.4503101157934, 1612.2432108317205, 
1231.4801541397014, 1041.773083014844, 1361.6291592061668, 1854.158196953391], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 90%|████████▉ | 899996/1000000 [10:04:14<46:11, 36.08it/s]global step 900000, trans_decision ep_re 1376.1259833116371

{"global_step": 900000, "eval_re": [1280.565540029585, -67.81033570824333, 
1235.037652587535, 2277.767099566616, 1758.6986643345263, 1395.7511349010974, 
1271.6955550948312, 1217.2607559871872, 1889.5085749957384, 1502.7851913274988],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 91%|█████████ | 909996/1000000 [10:11:04<41:23, 36.24it/s]global step 910000, trans_decision ep_re 1544.1760387814443

{"global_step": 910000, "eval_re": [1699.2348374955093, 1553.2860899211055, 
1880.3562712143205, 1234.5977250883095, 1247.904981216478, 1814.4667759659653, 
1218.4338807461443, 1321.6911687620272, 1992.263587090194, 1479.5250703143888], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 92%|█████████▏| 919996/1000000 [10:18:05<37:00, 36.03it/s]global step 920000, trans_decision ep_re 1614.1313575273878

{"global_step": 920000, "eval_re": [1380.7987680423612, 1430.8570099848898, 
1777.0420390683687, 1796.4065200772686, 1267.5254276775559, 1709.477067652414, 
1322.7043725042793, 1161.003435300155, 2574.901686528693, 1720.5972484378908], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 93%|█████████▎| 929996/1000000 [10:24:56<32:12, 36.23it/s]global step 930000, trans_decision ep_re 1842.1573392506587

{"global_step": 930000, "eval_re": [2035.3582272604772, 2220.715711563586, 
1661.4681722436064, 2714.5457744572764, 1356.7874927443445, 1485.9235369658568, 
1898.5663749293833, 1231.8169786469534, 2124.847766195411, 1691.5433574996925], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 94%|█████████▍| 939996/1000000 [10:31:46<27:29, 36.37it/s]global step 940000, trans_decision ep_re 1987.586367184742

{"global_step": 940000, "eval_re": [2171.850075891299, 2359.4019040052685, 
2158.4308917899853, 2090.2975005417597, 2069.9752530224055, 2218.65933398158, 
2271.890322084105, 1837.4261835990105, 1472.9227321053236, 1225.0094748266845], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 95%|█████████▍| 949996/1000000 [10:38:36<24:04, 34.63it/s]global step 950000, trans_decision ep_re 1677.5418011530874

{"global_step": 950000, "eval_re": [1527.6527034353867, 2524.041136406095, 
1539.1611872156116, 1475.7659379517688, 1750.643869456248, 1233.637071076281, 
1472.6990073829686, 1784.4570775711463, 1558.9844767094646, 1908.375544325903], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 96%|█████████▌| 959996/1000000 [10:45:26<18:20, 36.34it/s]global step 960000, trans_decision ep_re 1587.9377895550297

{"global_step": 960000, "eval_re": [2481.0900455190717, 1244.8811674068195, 
1569.742336282447, 1553.4035783498464, 2305.785042642805, 1208.2269928902774, 
1397.9470824466518, 1431.1762076439247, 1337.8451861354706, 1349.280256232981], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 97%|█████████▋| 969996/1000000 [10:52:16<13:48, 36.22it/s]global step 970000, trans_decision ep_re 1319.0490737272355

{"global_step": 970000, "eval_re": [1405.1007896246983, 1503.6697395364154, 
1294.47740056815, 1327.0158353096274, 1186.484307498274, 1426.3193439389945, 
1232.722964320402, 1263.5330306066853, 1372.1743247818526, 1178.9930010872554], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 98%|█████████▊| 979996/1000000 [10:59:06<09:09, 36.38it/s]global step 980000, trans_decision ep_re 1763.7397545773351

{"global_step": 980000, "eval_re": [1202.8231341508802, 1797.5026670485877, 
1151.414868520672, 3574.331970773522, 1175.7980731268178, 1629.6200244076965, 
1844.684144093779, 1432.8895542020607, 2224.1171338461418, 1604.2159756031956], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

 99%|█████████▉| 989996/1000000 [11:05:57<04:36, 36.12it/s]global step 990000, trans_decision ep_re 1993.1499143220303

{"global_step": 990000, "eval_re": [2073.033442004368, 1478.4275608360724, 
1647.7847966190648, 2286.611931523505, 1702.6551466832302, 1795.5768614155531, 
2130.3064987672255, 2811.5874540397663, 1505.7807652110785, 2499.734686120437], 
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

100%|█████████▉| 999996/1000000 [11:12:48<00:00, 36.14it/s]global step 1000000, trans_decision ep_re 1646.8262506420604

{"global_step": 1000000, "eval_re": [1297.1066313208191, 2033.398253459876, 
1656.2707273318, 1885.4191731162891, 1564.3737033726838, 1490.3803891432335, 
1594.5316596438804, 1775.6018050385642, 1933.9373192329913, 1237.2428447604652],
"eval_len": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]}

100%|██████████| 1000000/1000000 [11:13:12<00:00, 24.76it/s]
