
{
    'exp_name': 'VDPO',
    'env': 'Hopper-v4',
    'gamma': 0.99,
    'device': device(type='cuda'),
    'seed': 0,
    'buffer_size': 1000000,
    'total_timesteps': 1000000,
    'learn_start': 5000,
    'batch_size': 256,
    'actor_lr': 0.0003,
    'critic_lr': 0.001,
    'alpha_lr': 0.001,
    'target_update_factor': 0.005,
    'actor_freq': 2,
    'target_freq': 1,
    'eval_freq': 10000,
    'eval_num': 10,
    'kl_freq': 1000,
    'belief_lr': 0.0003,
    'embedding_dim': 256,
    'n_steps': 3,
    'delay': 16,
    'delayspec': 'MM1Queue_a033_s075::mm1queue(0.33, 0.75)'
}
✓ setup
Created Delay Process: MM1Queue(0.33, 0.75)
  1%|          | 9999/1000000 [02:47<5:54:20, 46.57it/s]global step 10000, trans_decision ep_re 178.5846902531024

{"global_step": 10000, "eval_re": [201.7228611280455, 171.45847696793825, 
203.73066351276879, 218.01839368948512, 178.53912982537307, 91.26211305923754, 
205.89275144190836, 150.7148993547778, 144.38395256644276, 220.12366098504683], 
"eval_len": [101, 89, 99, 110, 94, 71, 105, 85, 89, 111]}

  2%|▏         | 19996/1000000 [08:02<7:36:00, 35.82it/s]global step 20000, trans_decision ep_re 231.4140460223804

{"global_step": 20000, "eval_re": [321.89771027897046, 266.7677905925084, 
271.7324388899342, 330.9468624054531, 238.34151963752228, 69.42960444530645, 
340.4257222749525, 248.9072982838303, 159.0186358574369, 66.67287755788959], 
"eval_len": [145, 138, 148, 159, 126, 62, 167, 122, 95, 58]}

  3%|▎         | 29998/1000000 [13:08<5:52:39, 45.84it/s]global step 30000, trans_decision ep_re 236.97665133811475

{"global_step": 30000, "eval_re": [181.9090563366142, 203.70804422455177, 
226.77164707667703, 240.45635114021678, 247.6507007926097, 238.36457657328464, 
398.5190105042757, 173.72849071973874, 248.92579859705214, 209.73283741612642], 
"eval_len": [149, 181, 164, 181, 178, 173, 323, 129, 198, 137]}

  4%|▍         | 39996/1000000 [18:30<5:48:24, 45.92it/s]global step 40000, trans_decision ep_re 298.60441863881914

{"global_step": 40000, "eval_re": [246.23277739377184, 347.61381525115854, 
344.41336250874986, 465.6839884113805, 331.63620500313897, 241.66467645773363, 
222.89996588410142, 244.57356432406917, 273.99502872113055, 267.3308024329573], 
"eval_len": [180, 236, 292, 338, 280, 165, 178, 176, 209, 209]}

  5%|▍         | 49997/1000000 [23:42<5:33:41, 47.45it/s]global step 50000, trans_decision ep_re 347.4614060268302

{"global_step": 50000, "eval_re": [1113.9217993983737, 330.4625126877675, 
370.8193756509016, 253.13846887801424, 434.06628654760397, 49.75338765569191, 
420.32145436049615, 167.94837769976638, 149.16241828239296, 185.01997910729355],
"eval_len": [770, 235, 289, 188, 312, 47, 308, 124, 111, 146]}

  6%|▌         | 59998/1000000 [28:53<7:23:38, 35.31it/s]global step 60000, trans_decision ep_re 375.21730492119457

{"global_step": 60000, "eval_re": [279.32306858591664, 473.9636817242578, 
544.8194854556571, 550.6929357224884, 146.94251512315518, 515.6220120561318, 
242.8461399009931, 561.5109307954999, 164.8055970747131, 271.64668277313274], 
"eval_len": [145, 203, 231, 228, 92, 214, 135, 235, 106, 144]}

  7%|▋         | 69996/1000000 [34:10<5:38:03, 45.85it/s]global step 70000, trans_decision ep_re 653.1990769691305

{"global_step": 70000, "eval_re": [1040.1310738330012, 271.6964939102144, 
214.51732680398905, 274.7065594000176, 1064.7348010752432, 862.4070927222326, 
1075.4980575051118, 627.1024342538873, 304.8336424760649, 796.3632877115435], 
"eval_len": [370, 142, 117, 143, 353, 327, 366, 260, 153, 317]}

  8%|▊         | 79996/1000000 [39:12<5:43:44, 44.61it/s]global step 80000, trans_decision ep_re 676.1745670860752

{"global_step": 80000, "eval_re": [357.4292678717177, 1174.0679405879, 
555.0433002233813, 155.73587956926764, 354.5497884818772, 556.109603133264, 
555.7313912157556, 1724.6478292584022, 1038.4198389542546, 290.0108315649313], 
"eval_len": [179, 431, 243, 99, 175, 248, 250, 601, 348, 158]}

  9%|▉         | 89999/1000000 [44:30<5:47:01, 43.71it/s]global step 90000, trans_decision ep_re 542.8603673566766

{"global_step": 90000, "eval_re": [880.8011678982438, 438.3674487300035, 
392.3367621618492, 390.07893156544304, 392.0264140256424, 901.5814018087027, 
818.1350934166114, 110.655710602717, 659.9066254904167, 444.714117867137], 
"eval_len": [322, 203, 176, 184, 177, 333, 293, 78, 263, 194]}

 10%|▉         | 99995/1000000 [49:30<5:15:21, 47.57it/s]global step 100000, trans_decision ep_re 488.12909487957296

{"global_step": 100000, "eval_re": [1293.3511059372513, 184.9040602168701, 
307.76973542591315, 226.1929363912518, 1461.7503917680533, 311.31504923200015, 
226.83287809543398, 321.441308885698, 321.82898552188544, 225.90449732137242], 
"eval_len": [456, 112, 151, 120, 477, 157, 116, 158, 151, 122]}

 11%|█         | 109998/1000000 [54:33<6:36:03, 37.45it/s]global step 110000, trans_decision ep_re 260.4234078918704

{"global_step": 110000, "eval_re": [95.26424470881217, 1053.5963979492112, 
164.25089124862674, 456.06851938799707, 220.26162504508474, 107.83659535185211, 
104.46917852053762, 139.99691857022813, 206.04682187630556, 56.442886260048645],
"eval_len": [65, 379, 92, 194, 120, 78, 76, 84, 114, 50]}

 12%|█▏        | 119995/1000000 [59:51<5:33:10, 44.02it/s]global step 120000, trans_decision ep_re 94.83768055218893

{"global_step": 120000, "eval_re": [76.31213201649707, 95.38150481044352, 
101.91708154613387, 82.09506360605707, 98.62563928691499, 82.54387286295865, 
182.18261333667078, 71.49652572928129, 80.04656967711912, 77.77580264981297], 
"eval_len": [61, 68, 69, 65, 67, 62, 101, 57, 57, 61]}

 13%|█▎        | 129999/1000000 [1:05:10<5:19:30, 45.38it/s]global step 130000, trans_decision ep_re 780.8672555065634

{"global_step": 130000, "eval_re": [1106.4398108454802, 929.7915181385108, 
2039.7320150325165, 2087.6282915733404, 77.6135340159728, 72.31610173634871, 
76.08465219009415, 184.73745924604518, 410.3611967199095, 823.9679755674161], 
"eval_len": [385, 303, 634, 646, 62, 54, 60, 98, 177, 285]}

 14%|█▍        | 139996/1000000 [1:10:30<6:21:16, 37.59it/s]global step 140000, trans_decision ep_re 1010.4891026295945

{"global_step": 140000, "eval_re": [101.83710616533652, 1278.3330331460074, 
1217.8875434458253, 1490.5954761233336, 1088.9101492728396, 87.77675230011486, 
1277.8442243685809, 877.8547952284928, 1696.4246934472621, 987.427252798152], 
"eval_len": [74, 412, 408, 474, 352, 67, 414, 302, 545, 323]}

 15%|█▍        | 149997/1000000 [1:15:40<4:58:56, 47.39it/s]global step 150000, trans_decision ep_re 668.4235566279567

{"global_step": 150000, "eval_re": [1774.8565845725934, 480.9736254857964, 
275.4430651571102, 135.83633956474972, 1201.769559778036, 80.32747681335782, 
294.69937618581037, 1106.3426628550872, 1240.5095353016986, 93.47734056532693], 
"eval_len": [576, 204, 136, 84, 378, 60, 138, 365, 416, 64]}

 16%|█▌        | 159999/1000000 [1:20:43<6:17:11, 37.12it/s]global step 160000, trans_decision ep_re 532.2035785302097

{"global_step": 160000, "eval_re": [1009.2923118084537, 1397.1294926749458, 
301.9837328849834, 140.03527804863128, 122.02726883126577, 956.4565897156575, 
657.0554291797008, 295.7920907428386, 319.0782020745979, 123.18538934102153], 
"eval_len": [331, 418, 141, 84, 78, 317, 236, 136, 148, 74]}

 17%|█▋        | 169995/1000000 [1:26:01<5:58:19, 38.61it/s]global step 170000, trans_decision ep_re 749.6425716850107

{"global_step": 170000, "eval_re": [453.0747554162061, 964.2491037603322, 
474.08587577515686, 1120.7902106698039, 1131.9294091563415, 424.04766003626327, 
426.2906921961454, 1097.064505075569, 1324.1840630783454, 80.70944168594335], 
"eval_len": [186, 349, 196, 344, 371, 183, 180, 357, 412, 65]}

 18%|█▊        | 179997/1000000 [1:31:09<4:46:02, 47.78it/s]global step 180000, trans_decision ep_re 276.10046354423736

{"global_step": 180000, "eval_re": [50.32545043783716, 756.0516630623569, 
388.075782361339, 88.74271182433147, 308.41626983053067, 160.3918744365397, 
398.8733071006747, 74.67075201027072, 444.947251958835, 90.50957241965806], 
"eval_len": [49, 285, 175, 63, 150, 98, 174, 61, 193, 62]}

 19%|█▉        | 189998/1000000 [1:36:12<4:44:57, 47.38it/s]global step 190000, trans_decision ep_re 83.19474857852431

{"global_step": 190000, "eval_re": [60.32476434727644, 40.155852915376784, 
63.96795248605105, 52.50861245780144, 94.03171768190465, 125.72429391908581, 
85.34762663329352, 49.40102410896364, 177.68749342596772, 82.79814780952212], 
"eval_len": [50, 37, 51, 44, 65, 75, 60, 43, 96, 59]}

 20%|█▉        | 199996/1000000 [1:41:11<4:47:52, 46.32it/s]global step 200000, trans_decision ep_re 771.1068062034176

{"global_step": 200000, "eval_re": [950.7431397791458, 1090.8059887663012, 
824.0887576349124, 914.325802676098, 83.96610383757746, 390.4273306903538, 
1657.2481568931466, 67.74532623029204, 856.4073834685732, 875.3100720577759], 
"eval_len": [317, 349, 268, 289, 63, 168, 525, 53, 268, 299]}

 21%|██        | 209999/1000000 [1:46:40<4:50:23, 45.34it/s]global step 210000, trans_decision ep_re 704.4412278515093

{"global_step": 210000, "eval_re": [58.853450876919645, 905.3837589061884, 
650.5737763599466, 37.511208692615895, 51.48355728474006, 1692.048001686288, 
1207.0227873342747, 828.7459527261833, 639.0220490041307, 973.7677356438056], 
"eval_len": [50, 309, 245, 37, 41, 526, 384, 304, 234, 327]}

 22%|██▏       | 219997/1000000 [1:52:00<5:45:36, 37.61it/s]global step 220000, trans_decision ep_re 1077.6600782974501

{"global_step": 220000, "eval_re": [1484.286692190167, 2195.135684079852, 
221.6127188757934, 1278.9411640529818, 915.5097516687957, 959.4983606944612, 
1246.362091647864, 971.6477247973419, 941.4644288660708, 562.1421661011713], 
"eval_len": [472, 690, 115, 391, 294, 301, 375, 303, 304, 216]}

 23%|██▎       | 229996/1000000 [1:56:53<5:44:57, 37.20it/s]global step 230000, trans_decision ep_re 655.8824106433625

{"global_step": 230000, "eval_re": [927.0835829145863, 2045.705488529958, 
127.32395047847145, 479.4926985998289, 396.83736229789696, 257.05891218784313, 
1399.1600827131958, 147.42828064504585, 303.4859497586533, 475.2477983081461], 
"eval_len": [306, 649, 74, 194, 172, 120, 465, 88, 147, 190]}

 24%|██▍       | 239999/1000000 [2:02:20<4:39:14, 45.36it/s]global step 240000, trans_decision ep_re 1459.8755627387152

{"global_step": 240000, "eval_re": [1515.5301777081138, 1605.180587240731, 
1774.513816408683, 142.62414314841675, 393.86020533784176, 3020.4250881328153, 
2997.7984081964923, 1524.1853094597527, 997.8990565749336, 626.7388351793718], 
"eval_len": [516, 572, 592, 89, 170, 1000, 1000, 487, 360, 243]}

 25%|██▍       | 249997/1000000 [2:07:30<4:36:19, 45.24it/s]global step 250000, trans_decision ep_re 681.3695188599844

{"global_step": 250000, "eval_re": [953.8696194323302, 53.271080395584015, 
43.64254993430716, 40.64538186085208, 40.00832513103556, 793.1983017250241, 
1100.0422378326668, 281.01799716120246, 1462.0715219347276, 2045.9281731921144],
"eval_len": [317, 45, 38, 37, 35, 287, 348, 134, 486, 680]}

 26%|██▌       | 259998/1000000 [2:12:22<4:16:37, 48.06it/s]global step 260000, trans_decision ep_re 443.7724626551468

{"global_step": 260000, "eval_re": [918.2188630071834, 926.7006800504992, 
241.67216172746976, 126.3391619217692, 979.9114061889351, 208.38098321760174, 
191.89634325624246, 207.1018793042925, 435.7358540645874, 201.76729381288754], 
"eval_len": [318, 296, 121, 75, 295, 109, 105, 103, 181, 107]}

 27%|██▋       | 269997/1000000 [2:17:40<4:14:30, 47.81it/s]global step 270000, trans_decision ep_re 1110.7947025977435

{"global_step": 270000, "eval_re": [1160.6555267874674, 361.65458681771867, 
2713.3071120429963, 1186.5969132787575, 1292.6930667617396, 174.13794839378608, 
2451.9627883818407, 950.8316344814905, 644.0354803702346, 172.0719686614056], 
"eval_len": [367, 152, 876, 372, 407, 99, 791, 331, 243, 102]}

 28%|██▊       | 279997/1000000 [2:22:50<4:24:52, 45.30it/s]global step 280000, trans_decision ep_re 1154.5455272492086

{"global_step": 280000, "eval_re": [1771.15784314077, 466.18119788958614, 
458.39113185703883, 474.55790353448344, 1310.6266469692114, 1352.0388933049874, 
993.9824236095392, 1424.4027420103298, 152.1772943475824, 3141.9391958285587], 
"eval_len": [580, 193, 194, 193, 422, 458, 347, 440, 92, 1000]}

 29%|██▉       | 289999/1000000 [2:27:51<4:21:03, 45.33it/s]global step 290000, trans_decision ep_re 759.2447794567364

{"global_step": 290000, "eval_re": [1275.0131479824583, 734.2052976784872, 
859.1786044167925, 742.861490508121, 76.61446396606219, 457.44873175262444, 
502.9298875379469, 1003.8081073168077, 1448.3385277698421, 492.04953563822124], 
"eval_len": [397, 267, 308, 274, 55, 192, 198, 357, 428, 204]}

 30%|██▉       | 299995/1000000 [2:33:00<4:17:56, 45.23it/s]global step 300000, trans_decision ep_re 1002.2343051149722

{"global_step": 300000, "eval_re": [344.58935509130026, 646.2406610196483, 
1654.6556769131196, 41.05405591957427, 393.6024832793745, 1559.5765627559565, 
399.7770656489888, 1483.4179729067346, 645.0482681549025, 2854.380949460124], 
"eval_len": [150, 245, 552, 40, 175, 471, 172, 428, 249, 851]}

 31%|███       | 309996/1000000 [2:37:44<4:19:49, 44.26it/s]global step 310000, trans_decision ep_re 161.97151156723163

{"global_step": 310000, "eval_re": [111.12938144155741, 139.78935279520832, 
120.12328674923822, 470.67869569969463, 137.65643806398785, 126.48500374650526, 
134.37518171271378, 133.43588659777785, 123.44518322426983, 122.5967056413634], 
"eval_len": [70, 82, 75, 211, 85, 76, 82, 75, 76, 79]}

 32%|███▏      | 319998/1000000 [2:42:51<3:56:02, 48.01it/s]global step 320000, trans_decision ep_re 1348.0912137499822

{"global_step": 320000, "eval_re": [2005.100533681648, 2412.5786383503246, 
2236.6858579373807, 168.10525828432117, 1206.7080263087234, 594.0438169426349, 
186.24210905522457, 3021.615597226389, 948.6049496167792, 701.2273500963955], 
"eval_len": [664, 762, 689, 94, 411, 231, 100, 1000, 332, 263]}

 33%|███▎      | 329996/1000000 [2:47:41<3:52:50, 47.96it/s]global step 330000, trans_decision ep_re 448.37397052747485

{"global_step": 330000, "eval_re": [170.26221968962565, 440.30385894321483, 
936.2124159060738, 434.6614493360677, 189.6439469924222, 397.20689521636416, 
657.6813102116911, 424.90319696948694, 648.3462556886168, 184.51815632118542], 
"eval_len": [92, 183, 294, 175, 104, 162, 233, 168, 230, 98]}

 34%|███▍      | 339995/1000000 [2:52:15<3:52:22, 47.34it/s]global step 340000, trans_decision ep_re 1070.4879182975087

{"global_step": 340000, "eval_re": [175.86615895553638, 1196.0941550009088, 
3227.209673495891, 705.1445921078767, 268.88037710992586, 1203.1094999970785, 
891.9825087083955, 1690.8447349659748, 1182.1553321160804, 163.59215051741975], 
"eval_len": [96, 396, 1000, 259, 126, 371, 320, 508, 370, 89]}

 35%|███▍      | 349999/1000000 [2:56:56<3:48:32, 47.40it/s]global step 350000, trans_decision ep_re 392.31963993809575

{"global_step": 350000, "eval_re": [290.1685402132079, 309.8607791353442, 
499.0943249041841, 142.9246761661571, 720.8985193800827, 219.60269376321585, 
261.90184805493357, 937.4219677976353, 286.2931271415293, 255.02992282466755], 
"eval_len": [138, 149, 197, 82, 249, 109, 124, 321, 130, 119]}

 36%|███▌      | 359998/1000000 [3:01:30<3:42:59, 47.83it/s]global step 360000, trans_decision ep_re 338.3971033088689

{"global_step": 360000, "eval_re": [131.97596123394976, 764.2108901598227, 
274.8363851144895, 511.76567551415246, 259.0942783891707, 492.78739208634664, 
217.42404705002184, 243.85263766225947, 251.5645754458664, 236.4591904326099], 
"eval_len": [79, 268, 134, 200, 122, 193, 115, 119, 127, 118]}

 37%|███▋      | 369995/1000000 [3:06:04<3:41:59, 47.30it/s]global step 370000, trans_decision ep_re 280.8415136352556

{"global_step": 370000, "eval_re": [177.98792100428042, 470.60889782989614, 
281.82455836825676, 298.59151033726266, 250.36619494858581, 282.89531327133585, 
256.99282967894754, 288.3613754562717, 224.41455645087646, 276.37197900684276], 
"eval_len": [97, 196, 142, 138, 124, 145, 128, 134, 114, 127]}

 38%|███▊      | 379999/1000000 [3:10:37<3:38:16, 47.34it/s]global step 380000, trans_decision ep_re 692.8002620579784

{"global_step": 380000, "eval_re": [958.8717140500942, 85.36546815023964, 
922.3272073286495, 306.7373949019565, 983.3092930659574, 389.7694634723351, 
440.7833592978438, 1185.836013425298, 733.6820238117978, 921.3206830756112], 
"eval_len": [296, 62, 294, 141, 338, 158, 179, 336, 264, 321]}

 39%|███▉      | 389997/1000000 [3:15:31<3:51:35, 43.90it/s]global step 390000, trans_decision ep_re 750.5218422040723

{"global_step": 390000, "eval_re": [353.74883941615604, 1153.8024081947303, 
118.71628044124911, 1351.9278712966359, 110.11443905257823, 523.0211362938082, 
1219.150791566578, 1088.1402340456286, 1147.0245082230142, 439.5719135103444], 
"eval_len": [167, 410, 78, 473, 75, 210, 402, 371, 376, 186]}

 40%|███▉      | 399996/1000000 [3:20:19<3:31:26, 47.29it/s]global step 400000, trans_decision ep_re 472.7441694432255

{"global_step": 400000, "eval_re": [537.3418647772828, 87.22879305430746, 
189.91240798378416, 72.19692662868094, 278.0000790092226, 3014.0970301335183, 
92.98277708641915, 271.6060980210106, 83.9513743268837, 100.12434341114509], 
"eval_len": [212, 64, 101, 55, 135, 871, 65, 130, 68, 70]}

 41%|████      | 409996/1000000 [3:24:53<3:25:31, 47.85it/s]global step 410000, trans_decision ep_re 442.3816770595404

{"global_step": 410000, "eval_re": [418.273158271582, 192.35421100585958, 
423.89434326744055, 88.60320746071669, 544.2367454852059, 753.2891104475422, 
725.2387357059239, 743.5607110664878, 70.16007509148096, 464.2064727931646], 
"eval_len": [177, 107, 176, 60, 213, 266, 257, 256, 53, 187]}

 42%|████▏     | 419997/1000000 [3:29:29<3:24:46, 47.21it/s]global step 420000, trans_decision ep_re 332.1917833500942

{"global_step": 420000, "eval_re": [232.86783801451745, 244.385261838412, 
254.96153058589533, 255.63601790254657, 238.5276507521516, 243.13121019739236, 
252.89980972312335, 227.76616491780086, 1121.5880722762454, 250.15427729285753],
"eval_len": [108, 121, 119, 122, 117, 117, 127, 110, 352, 119]}

 43%|████▎     | 429997/1000000 [3:34:04<3:20:29, 47.39it/s]global step 430000, trans_decision ep_re 709.0821817620388

{"global_step": 430000, "eval_re": [204.65762071625295, 1128.9947986536124, 
1004.6473321889736, 203.93415699592816, 185.2214002563091, 695.5247655503821, 
886.456946621875, 663.1709303292436, 1070.834667098129, 1047.379199209681], 
"eval_len": [112, 348, 308, 109, 102, 248, 290, 241, 321, 315]}

 44%|████▍     | 439999/1000000 [3:38:42<3:16:50, 47.42it/s]global step 440000, trans_decision ep_re 553.7200921601036

{"global_step": 440000, "eval_re": [741.0375395600041, 1031.9832036179068, 
372.6234572835993, 688.9505027578581, 407.4141411761737, 810.486391734095, 
612.8051300996055, 74.66774606611898, 68.17549733594535, 729.0573119697278], 
"eval_len": [263, 329, 156, 250, 163, 286, 219, 47, 46, 266]}

 45%|████▍     | 449997/1000000 [3:43:19<3:13:58, 47.26it/s]global step 450000, trans_decision ep_re 628.4152789716615

{"global_step": 450000, "eval_re": [737.0719027501267, 726.1812881167798, 
195.55177075901915, 755.8818759946791, 956.5725998693633, 151.2086682585259, 
79.40811460367242, 1216.4593337872363, 709.2813234059848, 756.535912171226], 
"eval_len": [244, 250, 102, 255, 309, 85, 56, 346, 243, 253]}

 46%|████▌     | 459998/1000000 [3:47:55<3:07:07, 48.10it/s]global step 460000, trans_decision ep_re 914.7273453041056

{"global_step": 460000, "eval_re": [699.7674561659965, 1156.3048975979896, 
933.1872694806339, 374.53077888312316, 756.3616316346429, 377.9794501184679, 
1733.0816207001612, 1392.003378479993, 121.08219718723426, 1602.9747727928138], 
"eval_len": [244, 326, 284, 159, 253, 152, 473, 410, 84, 467]}

 47%|████▋     | 469995/1000000 [3:52:32<3:06:16, 47.42it/s]global step 470000, trans_decision ep_re 456.6970155081428

{"global_step": 470000, "eval_re": [700.913320775649, 704.4074985812657, 
732.4665006961719, 409.48408538426025, 434.68147209906556, 304.8541394320167, 
327.8879718611884, 88.26481962119443, 435.6941894321435, 428.31615719847224], 
"eval_len": [243, 252, 250, 171, 171, 136, 147, 56, 181, 176]}

 48%|████▊     | 479998/1000000 [3:57:08<3:00:26, 48.03it/s]global step 480000, trans_decision ep_re 381.7809111679497

{"global_step": 480000, "eval_re": [312.25630759620265, 386.98330979062666, 
302.60443616133807, 84.04784670454315, 278.94562562158006, 309.23360739004227, 
729.1826466837279, 863.1154623016998, 301.92350874767084, 249.51636068206568], 
"eval_len": [138, 163, 136, 58, 128, 134, 257, 287, 138, 120]}

 49%|████▉     | 489995/1000000 [4:01:43<2:58:59, 47.49it/s]global step 490000, trans_decision ep_re 655.2148198378203

{"global_step": 490000, "eval_re": [22.15783989625593, 128.23558367283056, 
964.6759978765875, 1256.856273644148, 1007.658673047984, 126.01514756279835, 
189.14016482421383, 988.1542202165988, 201.27919282344982, 1667.9751048133364], 
"eval_len": [22, 81, 305, 384, 335, 79, 103, 317, 108, 492]}

 50%|████▉     | 499998/1000000 [4:06:19<2:53:47, 47.95it/s]global step 500000, trans_decision ep_re 566.2729946215785

{"global_step": 500000, "eval_re": [925.9130726484401, 439.3281850339917, 
1178.990545843414, 200.87964633973843, 735.2411053984368, 465.98852667241067, 
66.58849097856974, 746.8242953975421, 749.1540551857738, 153.82202271746726], 
"eval_len": [293, 176, 346, 103, 260, 177, 51, 258, 249, 93]}

 51%|█████     | 509996/1000000 [4:10:54<2:50:08, 48.00it/s]global step 510000, trans_decision ep_re 599.9438657737867

{"global_step": 510000, "eval_re": [197.12397120243332, 561.5384210360423, 
697.4356012041642, 1387.5119714415046, 680.6628497536788, 426.9372823637198, 
245.37483927761352, 267.5617487042427, 967.8933020717759, 567.3986706826921], 
"eval_len": [108, 205, 252, 400, 245, 177, 121, 127, 301, 212]}

 52%|█████▏    | 519999/1000000 [4:15:42<2:50:00, 47.06it/s]global step 520000, trans_decision ep_re 708.7959200769662

{"global_step": 520000, "eval_re": [1004.955284253849, 715.67878650178, 
350.68591481076567, 91.78341300740578, 1489.4470514540865, 90.80134744354476, 
1272.6116395605966, 1716.5043634023893, 144.73769607694032, 210.75370425830437],
"eval_len": [332, 255, 154, 63, 479, 66, 351, 460, 87, 108]}

 53%|█████▎    | 529996/1000000 [4:20:06<2:43:56, 47.78it/s]global step 530000, trans_decision ep_re 522.7960234110118

{"global_step": 530000, "eval_re": [211.1364245364499, 89.94048655154685, 
949.7935366514987, 1147.8765689188092, 894.1906972604913, 1030.394274302962, 
81.53193119707194, 97.7492341759157, 274.22395044437536, 451.12313007099664], 
"eval_len": [109, 66, 300, 352, 326, 313, 59, 71, 123, 190]}

 54%|█████▍    | 539999/1000000 [4:24:41<2:42:13, 47.26it/s]global step 540000, trans_decision ep_re 386.13244170919813

{"global_step": 540000, "eval_re": [208.57779553795154, 1310.594116117109, 
561.6960922391407, 195.42519759504876, 366.9824027941793, 332.8757436075685, 
163.96273282871167, 300.0171635889178, 126.67935284499114, 294.5138199383628], 
"eval_len": [102, 392, 203, 100, 162, 152, 90, 132, 78, 126]}

 55%|█████▍    | 549996/1000000 [4:29:13<2:36:06, 48.05it/s]global step 550000, trans_decision ep_re 198.13942622626797

{"global_step": 550000, "eval_re": [464.38447511177384, 334.4221492081489, 
47.40872089214188, 43.10425935538245, 30.10114225791465, 249.41649893811592, 
44.118316161485545, 34.62603589880385, 695.1737762954867, 38.638888143426165], 
"eval_len": [180, 143, 40, 37, 29, 113, 36, 30, 245, 35]}

 56%|█████▌    | 559999/1000000 [4:33:45<2:35:27, 47.17it/s]global step 560000, trans_decision ep_re 429.03686769063387

{"global_step": 560000, "eval_re": [343.35429084957013, 234.2933062556242, 
239.4116176359202, 646.5654416627597, 1113.3034013859487, 660.7395087372105, 
50.49543649131125, 876.4870174104584, 62.161041414939916, 63.55761506259612], 
"eval_len": [136, 115, 115, 233, 330, 227, 41, 294, 49, 53]}

 57%|█████▋    | 569999/1000000 [4:38:19<2:31:47, 47.22it/s]global step 570000, trans_decision ep_re 597.1330430568099

{"global_step": 570000, "eval_re": [1379.3133494977174, 1028.7800256454793, 
413.0761076221034, 931.3275015074162, 79.1594846970438, 139.9231675720562, 
134.77224705846015, 48.25635023348652, 383.14390155846235, 1433.5782951758745], 
"eval_len": [406, 302, 169, 298, 61, 83, 78, 42, 161, 409]}

 58%|█████▊    | 579998/1000000 [4:42:56<2:24:15, 48.52it/s]global step 580000, trans_decision ep_re 633.4827181653766

{"global_step": 580000, "eval_re": [88.52819454287057, 1191.6494541250936, 
230.3901172709624, 590.8567871567009, 71.61040022360999, 899.094501207065, 
212.69866131036272, 453.59705522889715, 746.8639023454275, 1849.5381082427764], 
"eval_len": [65, 341, 115, 229, 49, 286, 105, 183, 269, 574]}

 59%|█████▉    | 589996/1000000 [4:47:28<2:21:22, 48.33it/s]global step 590000, trans_decision ep_re 101.16951346397344

{"global_step": 590000, "eval_re": [118.58653570361034, 66.30729760308792, 
82.74338838772803, 70.33807355182066, 187.6202257000941, 111.5526414407474, 
116.49457160761136, 85.4500903177351, 75.2662951055107, 97.33601522178883], 
"eval_len": [148, 95, 112, 100, 217, 141, 146, 115, 105, 127]}

 60%|█████▉    | 599996/1000000 [4:51:57<2:17:37, 48.44it/s]global step 600000, trans_decision ep_re 571.3070057543994

{"global_step": 600000, "eval_re": [468.074180164372, 955.9377608044454, 
128.01944088034142, 931.430930585482, 229.48871588489843, 624.2683547118371, 
1762.6366785962964, 227.70746912492294, 265.2946559309316, 120.21187086046652], 
"eval_len": [188, 311, 79, 294, 113, 234, 487, 119, 123, 70]}

 61%|██████    | 609996/1000000 [4:56:29<2:14:59, 48.15it/s]global step 610000, trans_decision ep_re 307.1680529589815

{"global_step": 610000, "eval_re": [83.13207757952611, 426.46161704582863, 
43.724893302236175, 333.0595947967872, 107.06616382122643, 880.1387213816339, 
64.61238855283275, 52.605678414881844, 78.03763249923097, 1002.8417621956311], 
"eval_len": [63, 164, 41, 146, 67, 301, 52, 47, 60, 322]}

 62%|██████▏   | 619999/1000000 [5:00:59<2:12:53, 47.66it/s]global step 620000, trans_decision ep_re 799.1334330430809

{"global_step": 620000, "eval_re": [166.4545434395539, 712.8390836318707, 
361.2318802097021, 1282.2999424896288, 1397.9849228614669, 1919.4861540045363, 
179.36352858010213, 344.0991344847383, 177.41685934733155, 1450.1582813818789], 
"eval_len": [96, 256, 158, 378, 393, 521, 103, 147, 97, 420]}

 63%|██████▎   | 629997/1000000 [5:05:42<2:08:23, 48.03it/s]global step 630000, trans_decision ep_re 965.1748018034529

{"global_step": 630000, "eval_re": [1375.5881040674328, 1418.858008917164, 
697.0477106646407, 1333.4794520082398, 404.93578649523783, 1867.5813110233667, 
468.94905590610375, 403.5389594321695, 1538.2647318866054, 143.50489763356825], 
"eval_len": [386, 406, 246, 380, 165, 528, 181, 164, 434, 78]}

 64%|██████▍   | 639995/1000000 [5:10:06<2:06:09, 47.56it/s]global step 640000, trans_decision ep_re 615.4673990034095

{"global_step": 640000, "eval_re": [168.53471828979056, 735.0540585733836, 
203.12441589038355, 495.8884239281796, 1258.6182018038523, 742.2152107796533, 
1022.0873028455613, 68.89787367748492, 685.6338567970289, 774.6199274487758], 
"eval_len": [96, 251, 103, 186, 396, 249, 323, 59, 233, 269]}

 65%|██████▍   | 649998/1000000 [5:14:38<2:00:59, 48.21it/s]global step 650000, trans_decision ep_re 855.5292683164868

{"global_step": 650000, "eval_re": [2085.217742513716, 310.7455615290216, 
1213.5093753197282, 229.6955431504877, 880.3586855405057, 302.73245945373884, 
155.9792321279841, 1714.2566728427466, 93.0143920649579, 1569.7830186219808], 
"eval_len": [570, 144, 368, 119, 295, 138, 85, 473, 70, 465]}

 66%|██████▌   | 659995/1000000 [5:19:12<1:58:02, 48.01it/s]global step 660000, trans_decision ep_re 870.164130125484

{"global_step": 660000, "eval_re": [472.9240903458644, 966.381862320601, 
711.7280265338297, 1646.4079051420756, 1529.594678648251, 198.4368023200482, 
1088.844666035054, 648.8094634119888, 1252.5121141119268, 186.00169238519914], 
"eval_len": [192, 329, 256, 477, 481, 101, 315, 227, 376, 93]}

 67%|██████▋   | 669996/1000000 [5:23:46<1:54:35, 48.00it/s]global step 670000, trans_decision ep_re 475.1004230904073

{"global_step": 670000, "eval_re": [99.8082524296128, 280.04610164016714, 
514.9240321782383, 515.9692077343088, 57.808725239292094, 67.68474455380398, 
817.6430594087453, 1229.503094589249, 746.4107841007935, 421.2062290298615], 
"eval_len": [72, 129, 197, 200, 47, 52, 256, 343, 258, 168]}

 68%|██████▊   | 679997/1000000 [5:28:17<1:51:21, 47.89it/s]global step 680000, trans_decision ep_re 526.5966737443454

{"global_step": 680000, "eval_re": [143.38448606017917, 156.08674193377854, 
118.77853197195537, 1332.04127526175, 455.29219616114926, 244.5954820287584, 
149.15586684470483, 173.0078632596376, 1232.709972620484, 1260.9143213010561], 
"eval_len": [85, 95, 81, 405, 177, 123, 84, 94, 355, 398]}

 69%|██████▉   | 689997/1000000 [5:32:49<1:47:55, 47.87it/s]global step 690000, trans_decision ep_re 700.9650699836704

{"global_step": 690000, "eval_re": [230.89528617688305, 390.7116673529942, 
1503.428348494051, 875.676574185913, 605.9649045629656, 1534.3617599189497, 
279.68572003617055, 70.66505110821639, 357.6068812076464, 1160.6545067929137], 
"eval_len": [110, 159, 458, 283, 213, 456, 128, 55, 152, 320]}

 70%|██████▉   | 699997/1000000 [5:37:22<1:45:01, 47.60it/s]global step 700000, trans_decision ep_re 372.05092705671655

{"global_step": 700000, "eval_re": [779.7196073710677, 869.6013916796894, 
272.2039199961361, 56.65044416303221, 404.60819041948815, 287.4053982160405, 
586.8403681694921, 47.535901181778506, 286.26754052409814, 129.67650884634293], 
"eval_len": [248, 259, 127, 49, 167, 133, 220, 44, 135, 77]}

 71%|███████   | 709995/1000000 [5:42:02<1:41:13, 47.75it/s]global step 710000, trans_decision ep_re 1015.3034148866343

{"global_step": 710000, "eval_re": [3023.300704199452, 78.61129975531112, 
742.0650455577974, 3239.8391756353503, 705.0563644071702, 75.52006374837734, 
2042.6056142197829, 91.0802034811752, 73.95059904570797, 81.00507881621871], 
"eval_len": [906, 63, 259, 933, 240, 63, 615, 66, 56, 58]}

 72%|███████▏  | 719997/1000000 [5:46:29<1:37:55, 47.66it/s]global step 720000, trans_decision ep_re 592.5331447989981

{"global_step": 720000, "eval_re": [1325.5201028485785, 634.3584614592207, 
194.82253724431177, 778.267041238812, 1442.2371776923972, 57.82483575121934, 
81.87573454173267, 1021.1887980519459, 61.08021573981, 328.1565434219542], 
"eval_len": [378, 233, 104, 273, 402, 47, 62, 328, 51, 149]}

 73%|███████▎  | 729996/1000000 [5:51:02<1:33:11, 48.29it/s]global step 730000, trans_decision ep_re 267.92740244582745

{"global_step": 730000, "eval_re": [213.35591353618827, 421.2776133463809, 
438.8015638127263, 368.2616711679935, 130.43894044193917, 182.89632681023727, 
130.0466237131914, 370.20240837074107, 122.91322285419297, 301.0797404046841], 
"eval_len": [111, 172, 177, 156, 78, 93, 80, 155, 76, 134]}

 74%|███████▍  | 739999/1000000 [5:55:33<1:30:28, 47.90it/s]global step 740000, trans_decision ep_re 349.2715718677174

{"global_step": 740000, "eval_re": [1848.9133693778456, 77.76905121066936, 
157.8539301779491, 159.59556054426292, 835.4165020967052, 69.01192332975036, 
59.25337592941053, 68.6636668450127, 137.61050789043358, 78.62783127513457], 
"eval_len": [527, 59, 89, 88, 290, 56, 52, 55, 81, 60]}

 75%|███████▍  | 749997/1000000 [6:00:04<1:26:30, 48.16it/s]global step 750000, trans_decision ep_re 767.808917898688

{"global_step": 750000, "eval_re": [469.9266028908582, 213.78739095626878, 
1876.3865744673635, 224.5821021099251, 1526.5659434595907, 743.625726505394, 
222.88369948692306, 82.43177452528624, 81.47018418318501, 2236.429180402087], 
"eval_len": [182, 114, 527, 114, 449, 255, 111, 60, 61, 640]}

 76%|███████▌  | 759999/1000000 [6:04:39<1:24:21, 47.42it/s]global step 760000, trans_decision ep_re 521.0111803714499

{"global_step": 760000, "eval_re": [1121.6837348629235, 370.72094907276676, 
202.47754783970308, 220.94019813430035, 175.42905353464636, 220.44421438368224, 
1673.2465363220738, 723.9937531346995, 121.55172035694508, 379.6240960727581], 
"eval_len": [372, 155, 112, 111, 91, 114, 467, 254, 79, 162]}

 77%|███████▋  | 769999/1000000 [6:09:22<1:20:26, 47.66it/s]global step 770000, trans_decision ep_re 857.7080728054025

{"global_step": 770000, "eval_re": [1202.69295252058, 462.2681113914167, 
1749.1233996238432, 167.5158392405202, 176.19698534577338, 422.0523103385271, 
1536.3707364893403, 1738.7969199345948, 238.38268815204643, 883.6807850173818], 
"eval_len": [380, 179, 505, 90, 104, 164, 425, 489, 119, 286]}

 78%|███████▊  | 779998/1000000 [6:13:46<1:15:52, 48.33it/s]global step 780000, trans_decision ep_re 145.19244292359275

{"global_step": 780000, "eval_re": [115.71834165096278, 103.99237649439583, 
97.57949026804938, 103.96528191456488, 563.7640787990304, 86.88893704351737, 
92.92974190307694, 91.48422074114896, 115.0369734865091, 80.56498693467182], 
"eval_len": [76, 71, 71, 68, 205, 62, 64, 62, 75, 65]}

 79%|███████▉  | 789999/1000000 [6:18:16<1:13:11, 47.82it/s]global step 790000, trans_decision ep_re 951.8473790688798

{"global_step": 790000, "eval_re": [1291.9789679569121, 1487.565929581799, 
102.61752420546085, 174.18703168501898, 1327.3602613620599, 1162.4949810665623, 
468.4393093497476, 1064.1569765296647, 1441.5342364902453, 998.1385724613264], 
"eval_len": [361, 428, 69, 95, 362, 368, 184, 342, 404, 333]}

 80%|███████▉  | 799997/1000000 [6:23:02<1:09:42, 47.82it/s]global step 800000, trans_decision ep_re 921.7679276441388

{"global_step": 800000, "eval_re": [117.07672948377831, 990.9331558409607, 
101.06803646115446, 3217.139027126295, 153.6898377595191, 2184.056970419396, 
831.7762378242737, 84.359175028455, 547.7944675663866, 989.7856389311718], 
"eval_len": [68, 312, 69, 882, 85, 635, 262, 63, 209, 311]}

 81%|████████  | 809995/1000000 [6:27:28<1:06:15, 47.80it/s]global step 810000, trans_decision ep_re 525.6150677202229

{"global_step": 810000, "eval_re": [395.0139476849569, 1454.1360125168826, 
284.9354915507285, 949.6938533140215, 453.13523153991906, 377.7490429959194, 
407.9518960577059, 105.56577063485233, 752.3606239780933, 75.60880692915033], 
"eval_len": [173, 406, 130, 315, 201, 159, 173, 66, 259, 62]}

 82%|████████▏ | 819995/1000000 [6:32:03<1:03:14, 47.44it/s]global step 820000, trans_decision ep_re 502.3299437284278

{"global_step": 820000, "eval_re": [1326.0032424177045, 612.5832175199282, 
1059.7295277634473, 259.51414816308005, 228.7427588812715, 489.40530101303307, 
239.0115177626713, 219.18276193672438, 79.15660504188251, 509.9703567845341], 
"eval_len": [368, 228, 311, 124, 111, 190, 114, 108, 57, 199]}

 83%|████████▎ | 829998/1000000 [6:36:36<58:48, 48.18it/s]global step 830000, trans_decision ep_re 391.14620780327175

{"global_step": 830000, "eval_re": [225.1294180031579, 378.31785996756025, 
122.509650844693, 199.31448217563738, 202.03133843391797, 628.2804829843001, 
355.18243749127913, 1345.615645559711, 223.97047121037778, 231.11029136208356], 
"eval_len": [112, 156, 80, 101, 99, 226, 152, 385, 111, 121]}

 84%|████████▍ | 839999/1000000 [6:41:09<1:16:49, 34.71it/s]global step 840000, trans_decision ep_re 712.1494081791379

{"global_step": 840000, "eval_re": [1307.8599080040242, 463.7680943878376, 
1025.9502774284563, 97.8401845782188, 1526.717180654103, 99.05177067299203, 
1119.1322585736991, 1301.4069469648011, 96.12141696136302, 83.64604356588382], 
"eval_len": [372, 180, 325, 70, 417, 68, 342, 380, 67, 64]}

 85%|████████▍ | 849997/1000000 [6:45:43<51:51, 48.21it/s]global step 850000, trans_decision ep_re 672.0901975185463

{"global_step": 850000, "eval_re": [21.51746775869756, 199.49957526107784, 
202.87948900358097, 204.09461145179, 1105.5713892662263, 1355.5949539878063, 
1520.081348190412, 370.52775859504135, 1381.695952077307, 359.4394295935246], 
"eval_len": [24, 106, 108, 99, 319, 369, 424, 148, 392, 149]}

 86%|████████▌ | 859997/1000000 [6:50:17<49:01, 47.60it/s]global step 860000, trans_decision ep_re 249.31960346925774

{"global_step": 860000, "eval_re": [1431.4569096523173, 159.24506117798063, 
162.621743705189, 137.3848761497202, 43.87345938012357, 97.42915126172068, 
144.20112159655193, 68.25354063202782, 108.22918265719396, 140.50098847975232], 
"eval_len": [395, 91, 91, 81, 41, 68, 84, 52, 71, 84]}

 87%|████████▋ | 869998/1000000 [6:54:50<45:03, 48.08it/s]global step 870000, trans_decision ep_re 51.297989362644685

{"global_step": 870000, "eval_re": [230.70721159685164, 33.07504677807388, 
28.799270110774913, 30.050834131874502, 25.051755482334233, 25.2263774019814, 
32.684947163578286, 28.451514261391992, 39.625854701240286, 39.30708199834567], 
"eval_len": [114, 32, 31, 30, 30, 27, 30, 29, 34, 37]}

 88%|████████▊ | 879998/1000000 [6:59:19<41:41, 47.97it/s]global step 880000, trans_decision ep_re 366.65812478258874

{"global_step": 880000, "eval_re": [84.52270262250302, 92.25329195545245, 
77.64419411377925, 1051.0757013527204, 728.0201767180395, 185.5922707160909, 
76.01276585543634, 90.0464077809028, 1179.9988168066716, 101.41491990429152], 
"eval_len": [63, 68, 59, 308, 252, 92, 58, 61, 334, 67]}

 89%|████████▉ | 889996/1000000 [7:03:49<37:55, 48.34it/s]global step 890000, trans_decision ep_re 242.3365056196582

{"global_step": 890000, "eval_re": [142.42334088533258, 77.18655479793999, 
745.9061461587119, 430.60779309456694, 119.45401900035058, 117.35589040283307, 
412.9880199569687, 108.1865261730965, 92.99327127205292, 176.26349445472903], 
"eval_len": [84, 60, 257, 179, 72, 77, 167, 75, 64, 100]}

 90%|████████▉ | 899998/1000000 [7:08:21<34:16, 48.62it/s]global step 900000, trans_decision ep_re 424.32161852796736

{"global_step": 900000, "eval_re": [799.674233087078, 245.62548975373252, 
710.432157096438, 460.15695021526244, 74.96423131959143, 719.9948104535553, 
254.23426966304032, 495.6807298414619, 239.28874466099006, 243.1645691885233], 
"eval_len": [255, 114, 241, 174, 58, 252, 131, 199, 117, 118]}

 91%|█████████ | 909995/1000000 [7:12:53<31:24, 47.77it/s]global step 910000, trans_decision ep_re 575.8128253679145

{"global_step": 910000, "eval_re": [52.932289900377185, 1289.757735672451, 
1144.4934852641873, 599.9365401429262, 1082.2766788731835, 182.40513047184808, 
119.13460013991202, 561.6599492679062, 433.2854791442999, 292.24636480205317], 
"eval_len": [47, 400, 317, 208, 345, 96, 73, 204, 170, 128]}

 92%|█████████▏| 919999/1000000 [7:17:25<27:56, 47.71it/s]global step 920000, trans_decision ep_re 754.6544510728338

{"global_step": 920000, "eval_re": [1246.3923335923091, 860.1732550941801, 
550.8875285521011, 851.4234291959724, 64.75999222999727, 548.8208236544184, 
1694.2139662405264, 545.8677178744723, 1158.9921704259607, 25.01329386840092], 
"eval_len": [382, 253, 192, 253, 51, 203, 471, 204, 331, 28]}

 93%|█████████▎| 929997/1000000 [7:21:58<24:31, 47.58it/s]global step 930000, trans_decision ep_re 572.8457675291733

{"global_step": 930000, "eval_re": [207.89984181806759, 32.8220957647766, 
65.07808670800517, 1078.075200208122, 187.74607167577955, 627.7379386657615, 
642.3913764689619, 693.8307473722745, 1565.8070387255036, 627.0692778844801], 
"eval_len": [101, 37, 50, 350, 94, 220, 232, 244, 463, 219]}

 94%|█████████▍| 939999/1000000 [7:26:30<21:04, 47.46it/s]global step 940000, trans_decision ep_re 95.06357185148872

{"global_step": 940000, "eval_re": [84.32319244726959, 27.87972146819999, 
28.32870094047337, 36.596279636452586, 40.2887047291358, 32.43609806927933, 
201.0697642085473, 28.067149940403056, 446.03758097722516, 25.608526097901116], 
"eval_len": [59, 31, 31, 36, 35, 33, 100, 30, 174, 28]}

 95%|█████████▍| 949999/1000000 [7:30:58<17:26, 47.76it/s]global step 950000, trans_decision ep_re 261.68398781194753

{"global_step": 950000, "eval_re": [908.0516526922875, 19.887440840294637, 
192.91326840260197, 173.5253509607932, 172.6609091125771, 194.85322528398413, 
179.08663579793605, 436.80191772043537, 183.87015205698077, 155.18932525158465],
"eval_len": [288, 23, 104, 95, 96, 108, 102, 170, 102, 91]}

 96%|█████████▌| 959997/1000000 [7:35:28<14:00, 47.58it/s]global step 960000, trans_decision ep_re 658.3812293007684

{"global_step": 960000, "eval_re": [630.0481776549256, 632.8125008048077, 
1497.650307735681, 109.52395649994676, 629.2213087419958, 318.53317429190525, 
776.5082356154514, 626.0187194307234, 1234.8173775050004, 128.67853472724636], 
"eval_len": [237, 223, 468, 73, 224, 157, 266, 226, 392, 81]}

 97%|█████████▋| 969998/1000000 [7:40:01<10:23, 48.14it/s]global step 970000, trans_decision ep_re 335.1932620636059

{"global_step": 970000, "eval_re": [1021.624630945362, 238.81874167159464, 
312.56735914352316, 337.1787292487076, 121.69434291373393, 461.4359776627155, 
213.50667550397074, 137.78357771190454, 342.09755840938357, 165.2250274251637], 
"eval_len": [325, 128, 154, 139, 74, 183, 140, 84, 163, 91]}

 98%|█████████▊| 979995/1000000 [7:44:35<07:02, 47.35it/s]global step 980000, trans_decision ep_re 388.17357209001216

{"global_step": 980000, "eval_re": [27.738401805307216, 193.46340816993114, 
51.66442734635969, 75.68921126253989, 140.27171833749352, 1658.2954964836674, 
1183.6762082828154, 313.3775471452985, 104.62295211495798, 132.9363499517508], 
"eval_len": [28, 101, 51, 57, 81, 468, 323, 134, 72, 79]}

 99%|█████████▉| 989995/1000000 [7:49:08<03:31, 47.42it/s]global step 990000, trans_decision ep_re 514.4552431760991

{"global_step": 990000, "eval_re": [799.6503889965765, 47.39741451366244, 
392.5828012354361, 161.6783628875717, 198.8674973204955, 1323.681918113417, 
1321.6747091801044, 428.92289105540215, 219.18538799984248, 250.911060458483], 
"eval_len": [280, 41, 163, 86, 113, 368, 381, 167, 107, 121]}

100%|█████████▉| 999999/1000000 [7:53:44<00:00, 47.18it/s]global step 1000000, trans_decision ep_re 210.26092763239504

{"global_step": 1000000, "eval_re": [24.181703176191625, 723.399621484832, 
92.25752209485948, 304.9969369312348, 22.382554712754064, 646.7202446993508, 
72.08401891100937, 40.76731881534536, 60.23490056431224, 115.58445493406059], 
"eval_len": [25, 247, 65, 132, 24, 228, 61, 37, 45, 69]}

100%|██████████| 1000000/1000000 [7:53:52<00:00, 35.17it/s]
