QOJDoudizhuModel / model /report.json
xcx0902's picture
Upload folder using huggingface_hub (part 4)
f115597 verified
Raw History Blame Contribute Delete
68.3 kB
{ "status": "complete", "device": "mps", "config": { "data": "/Users/xcx0902/Documents/Github/qojddz/data/qoj-history", "output": "/Users/xcx0902/Documents/Github/qojddz/QOJDoudizhuModel/model", "teacher_dir": "/Users/xcx0902/Documents/Github/qojddz/models/douzero-baselines/douzero_ADP", "device": "mps", "width": 128, "warmup_games": 32, "offline_epochs": 2, "games_per_round": 12, "ppo_epochs": 2, "batch_states": 16, "max_batch_actions": 4096, "candidate_cap": 64, "max_history_games": 0, "holdout": 0.1, "learning_rate": 0.0001, "ppo_learning_rate": 2e-05, "anchor_weight": 0.02, "league_size": 4, "validation_interval": 16, "validation_deals": 12, "test_deals": 48, "seed": 20261002, "validation_seed": 30261002, "test_seed": 40261002, "mode": "mixed", "teacher_sha256": { "landlord": "6f2971813495e9c509cbd4a101213c49587472942fa0f018d04c139a6a647c2d", "landlord_up": "7fb7c495ff095db2ddadd8f9f8fd53cad840211578dabc83bbcc8d75085f843e", "landlord_down": "c173dbe30258f19b6392c30ab1e330b6b3090bbd9ccdc89a9b18123a6ab6e2d9" }, "source_sha256": { "Douzero_Resnet/douzero/dmc/models.py": "b873ebcceb8907ccf5e1b75751bef640da944d5c9d6b100b2e5fb632150c18fa", "Douzero_Resnet/douzero/env/env.py": "4ba02e8e181db4b84704ab205b94691618bca4d71d8e5ce03c61d2c060f4724a", "QOJDoudizhuModel/__init__.py": "f9a452c634683d816ff3a33d679ec510b3a6fb9ebaa4f266d39181322895ea91", "QOJDoudizhuModel/dataset.py": "43bd492ea3190917dafc46cfbaf12a505977976130ab101fcecafed34a4fab73", "QOJDoudizhuModel/evaluate.py": "543441b8bb404da0f5c39842d2dd4e04b14875a6a7a052c93fe690b0489f5e72", "QOJDoudizhuModel/features.py": "cc7d076b67b9a59fb8b1b5d4f3e23fa0fe9313e6ed7d2ccbbc763ebd99563dc6", "QOJDoudizhuModel/learning.py": "2276f3c9af8bb7d1ddab6e85e33a9d1f4325426dc60f2d51dacd3b113cf92f5f", "QOJDoudizhuModel/network.py": "14450e8d72f40afcd0a814bd609d5f0dab23579055c06bfaa239fb5ec4a154dc", "QOJDoudizhuModel/policy.py": "275892378b80d64e2d469ced9fd27968eafaa699cf265ddd0ade214f798ccdf3", "QOJDoudizhuModel/storage.py": "c9621015b6dd6c35418ee3fd31a73ea71a545ef51320ec811ccc9e053b943417", "QOJDoudizhuModel/train.py": "4495d1fa4911cb6d92f39bd9c3691a16fc74fa7ef320986cac67acf923631239", "qojbot/adapter.py": "64922460bf74a2d2413fe08ecfcc3539b1de5656d7295e154fe98946e48593e9", "qojbot/curriculum.py": "b005956942bde93e683b9d37e444e6652801f0d6b691a8bbe631291379e88bfa", "qojbot/douzero.py": "84495d88a2ac67a01a659db0b9b039f5b36d4d4fcd1849712ea9b0912a6c3864", "qojbot/game.py": "6f8b77ec661dcbaaf2042df24e0453dd3263d7c9cc4df799595b746738ef5ff1", "qojbot/history.py": "27eeeffd080c476b0b72245278495bda9ec9ed744fe6b85340f7f2742f8c6ea9", "qojbot/history_training.py": "74ff26534168f4f98652cf6d43c268ab2b3211e3e27f04a9fad9e1a857759b61", "qojbot/model_benchmark.py": "03de89831e54857840af92bdd47325a13cee48d73035fb45b98508f85e89e9e4", "qojbot/policy.py": "cb406eb98f43517dbec106d8bc62f6edac00bd2b0c9dace00c85315d1f6008ec", "qojbot/rules.py": "8fb9ee4ff9de21c58fd667a8e4c22a8c3db9b919e711029638f24340541e76f0" } }, "progress": { "offline_epoch": 2, "round": 128, "updates": 28490, "games": 1536, "best_score": 0.1111111111111111, "best_stage": "round-80", "stage": "ppo" }, "dataset": { "fingerprint": "13e5709daa6e5e8d777c104cebd6e7ec75115874095319ee7d570c7a02277878", "counts": { "train": 180278, "validation": 21260, "test": 21165 }, "valid_games": 7266, "rejected_games": 0 }, "metrics": [ { "stage": "baseline", "validation_score": 0, "best_score": 0 }, { "epoch": 1, "offline_validation": { "samples": 21260, "loss": 0.6506123136262902 } }, { "stage": "offline-1", "validation_score": -0.28703703703703703, "best_score": 0 }, { "epoch": 2, "offline_validation": { "samples": 21260, "loss": 0.6081839078849818 } }, { "stage": "offline-2", "validation_score": -0.3194444444444444, "best_score": 0 }, { "round": 1, "ppo": { "loss": 0.6096090006251489, "policy": -6.413908414943244e-06, "anchor_kl": 7.90482844772742e-09, "value": 0.036875955224718135, "belief": 0.18955422481221537, "approx_kl": 7.245289061659126e-09, "samples": 744, "actions": 5118 }, "samples": 372, "early_kl_stop": false, "seconds": 13.777114041999084 }, { "round": 2, "ppo": { "loss": 0.7068826687820529, "policy": -1.512260192180904e-05, "anchor_kl": 3.437007834763857e-08, "value": 0.2472581016332421, "belief": 0.18990855674944762, "approx_kl": 1.4541383748305014e-08, "samples": 818, "actions": 5844 }, "samples": 409, "early_kl_stop": false, "seconds": 14.79054187499969 }, { "round": 3, "ppo": { "loss": 0.5746077012246961, "policy": -2.8213972498656836e-06, "anchor_kl": 6.720765208616969e-08, "value": 0.05978727498489201, "belief": 0.16814449467476766, "approx_kl": 6.611203881585674e-09, "samples": 628, "actions": 4364 }, "samples": 314, "early_kl_stop": false, "seconds": 10.995700875000693 }, { "round": 4, "ppo": { "loss": 0.7064850827625819, "policy": -5.684750420706613e-06, "anchor_kl": 7.034261264899604e-08, "value": 0.29715588314192637, "belief": 0.19041749055896487, "approx_kl": 5.7688781256765815e-09, "samples": 700, "actions": 5268 }, "samples": 350, "early_kl_stop": false, "seconds": 11.179604124999969 }, { "round": 5, "ppo": { "loss": 0.4973030815994929, "policy": -3.669824865129259e-06, "anchor_kl": 3.650177238985937e-08, "value": 0.04242017335361904, "belief": 0.17779199672596796, "approx_kl": 6.338906666941613e-09, "samples": 630, "actions": 5460 }, "samples": 315, "early_kl_stop": false, "seconds": 12.243349374999525 }, { "round": 6, "ppo": { "loss": 0.7072747652868615, "policy": -1.0951750003660208e-05, "anchor_kl": 8.232433277687201e-08, "value": 0.3120166570306509, "belief": 0.21283678541023784, "approx_kl": 8.379001855107483e-09, "samples": 642, "actions": 5184 }, "samples": 321, "early_kl_stop": false, "seconds": 10.581966499999908 }, { "round": 7, "ppo": { "loss": 0.5007647565643853, "policy": -3.92063160429798e-06, "anchor_kl": 4.188005594193752e-08, "value": 0.0452385424944823, "belief": 0.17448846047784522, "approx_kl": 4.451894905009661e-09, "samples": 646, "actions": 4872 }, "samples": 323, "early_kl_stop": false, "seconds": 11.247399666999627 }, { "round": 8, "ppo": { "loss": 0.7336269403281419, "policy": -1.206394770871038e-05, "anchor_kl": 7.404081089297822e-08, "value": 0.3467860811430475, "belief": 0.20613388328448587, "approx_kl": 7.673288169114487e-09, "samples": 736, "actions": 5334 }, "samples": 368, "early_kl_stop": false, "seconds": 11.015911458000119 }, { "round": 9, "ppo": { "loss": 0.47186622778071635, "policy": -1.0547758657721024e-06, "anchor_kl": 4.439529380404736e-08, "value": 0.03989364779731141, "belief": 0.16794568262522733, "approx_kl": 3.5013013220095257e-09, "samples": 632, "actions": 4436 }, "samples": 316, "early_kl_stop": false, "seconds": 10.26273824999953 }, { "round": 10, "ppo": { "loss": 0.6806026247569493, "policy": -4.7181120940617155e-06, "anchor_kl": 6.471287774851524e-08, "value": 0.2556270485717271, "belief": 0.2068670101250921, "approx_kl": 6.162694490592051e-09, "samples": 700, "actions": 5124 }, "samples": 350, "early_kl_stop": false, "seconds": 10.902530540999578 }, { "round": 11, "ppo": { "loss": 0.5056302774915457, "policy": -3.7598555964679773e-06, "anchor_kl": 7.633922281995053e-08, "value": 0.1115345879092107, "belief": 0.17971833724746464, "approx_kl": 4.475017943949399e-09, "samples": 718, "actions": 4840 }, "samples": 359, "early_kl_stop": false, "seconds": 11.55123149999963 }, { "round": 12, "ppo": { "loss": 0.6970414358174081, "policy": -8.141695279774703e-06, "anchor_kl": 6.057163387252657e-08, "value": 0.32165614548787047, "belief": 0.1738200538896826, "approx_kl": 7.2696938017342395e-09, "samples": 762, "actions": 5020 }, "samples": 381, "early_kl_stop": false, "seconds": 12.158407832999728 }, { "round": 13, "ppo": { "loss": 0.4656745560717683, "policy": -7.082642961283024e-06, "anchor_kl": 9.857341799706634e-08, "value": 0.08241950598589722, "belief": 0.19632076463696002, "approx_kl": 4.920096290079157e-09, "samples": 714, "actions": 5332 }, "samples": 357, "early_kl_stop": false, "seconds": 11.450703499998781 }, { "round": 14, "ppo": { "loss": 0.7353519941564133, "policy": -1.5121337782021518e-05, "anchor_kl": 1.5578324776153064e-07, "value": 0.35178173452721795, "belief": 0.2095364790635177, "approx_kl": 8.72990627908808e-09, "samples": 562, "actions": 4248 }, "samples": 281, "early_kl_stop": false, "seconds": 8.611376917000598 }, { "round": 15, "ppo": { "loss": 0.4514082455951156, "policy": -1.05175105008212e-06, "anchor_kl": 1.5782193353667633e-07, "value": 0.06449757902459664, "belief": 0.1848393997911251, "approx_kl": 5.644379242785259e-09, "samples": 660, "actions": 4744 }, "samples": 330, "early_kl_stop": false, "seconds": 10.69090587500068 }, { "round": 16, "ppo": { "loss": 0.7103247732827158, "policy": -1.015182998445299e-05, "anchor_kl": 1.5786771848467533e-07, "value": 0.38205278025131034, "belief": 0.1923854390178064, "approx_kl": 7.3941368050836085e-09, "samples": 792, "actions": 5176 }, "samples": 396, "early_kl_stop": false, "seconds": 11.948139209000146 }, { "stage": "round-16", "validation_score": 0, "best_score": 0 }, { "round": 17, "ppo": { "loss": 0.4145474828098115, "policy": -1.8430613605669757e-06, "anchor_kl": 2.0271443627368867e-07, "value": 0.03184943552540001, "belief": 0.16915904803399268, "approx_kl": 5.0936581036232035e-09, "samples": 716, "actions": 4584 }, "samples": 358, "early_kl_stop": false, "seconds": 12.558681834001618 }, { "round": 18, "ppo": { "loss": 0.475287568412329, "policy": -9.91148372460445e-06, "anchor_kl": 2.0761337970298493e-07, "value": 0.10663812479505967, "belief": 0.18374744341377133, "approx_kl": 6.666308957892151e-09, "samples": 646, "actions": 4170 }, "samples": 323, "early_kl_stop": false, "seconds": 11.107545708000544 }, { "round": 19, "ppo": { "loss": 0.41852791612053214, "policy": -7.75610781057761e-06, "anchor_kl": 1.6542085957021186e-07, "value": 0.0493800065089618, "belief": 0.17066317468240697, "approx_kl": 7.737654942103432e-09, "samples": 558, "actions": 3590 }, "samples": 279, "early_kl_stop": false, "seconds": 10.488347459000579 }, { "round": 20, "ppo": { "loss": 0.6844861578906891, "policy": -1.1180909727350136e-05, "anchor_kl": 1.5635325618926525e-07, "value": 0.3954741277791172, "belief": 0.15828846649110662, "approx_kl": 8.139651619387678e-09, "samples": 692, "actions": 4942 }, "samples": 346, "early_kl_stop": false, "seconds": 11.797273582998969 }, { "round": 21, "ppo": { "loss": 0.4500025973363888, "policy": -2.5963673562359957e-06, "anchor_kl": 1.929114569519212e-07, "value": 0.05820000217370461, "belief": 0.18219689650038268, "approx_kl": 5.239403683915979e-09, "samples": 652, "actions": 4524 }, "samples": 326, "early_kl_stop": false, "seconds": 11.49547100000018 }, { "round": 22, "ppo": { "loss": 0.7681089490652084, "policy": -1.0709033200615331e-05, "anchor_kl": 2.1916747303557235e-07, "value": 0.4591146346769835, "belief": 0.19132567001016518, "approx_kl": 7.015554920623177e-09, "samples": 608, "actions": 4888 }, "samples": 304, "early_kl_stop": false, "seconds": 10.822109708000426 }, { "round": 23, "ppo": { "loss": 0.40222113164695533, "policy": -9.560101741069073e-07, "anchor_kl": 2.916251102416592e-07, "value": 0.027196657921917534, "belief": 0.18415737444081823, "approx_kl": 5.326158291584737e-09, "samples": 740, "actions": 4644 }, "samples": 370, "early_kl_stop": false, "seconds": 13.510235124998871 }, { "round": 24, "ppo": { "loss": 0.5657669985368197, "policy": -1.536928790651327e-05, "anchor_kl": 2.6901714229756296e-07, "value": 0.2247238093036778, "belief": 0.17170743196768246, "approx_kl": 9.014550844693244e-09, "samples": 686, "actions": 4640 }, "samples": 343, "early_kl_stop": false, "seconds": 11.803076375001183 }, { "round": 25, "ppo": { "loss": 0.4207793133354735, "policy": -4.408174547655829e-06, "anchor_kl": 2.4625710394481767e-07, "value": 0.06494351939951208, "belief": 0.1688026307129312, "approx_kl": 5.828794739335831e-09, "samples": 696, "actions": 4458 }, "samples": 348, "early_kl_stop": false, "seconds": 11.788431583998317 }, { "round": 26, "ppo": { "loss": 0.6766984295597416, "policy": -1.4638865383159513e-05, "anchor_kl": 3.726572820931281e-07, "value": 0.3843160573619719, "belief": 0.18269463213995232, "approx_kl": 1.492326974161301e-08, "samples": 674, "actions": 4224 }, "samples": 337, "early_kl_stop": false, "seconds": 11.810041875000024 }, { "round": 27, "ppo": { "loss": 0.46670324161000876, "policy": -2.955693913542706e-06, "anchor_kl": 4.2403026682173417e-07, "value": 0.04940832633039226, "belief": 0.17209890905929648, "approx_kl": 8.731149137020111e-09, "samples": 736, "actions": 5102 }, "samples": 368, "early_kl_stop": false, "seconds": 12.626642000001084 }, { "round": 28, "ppo": { "loss": 0.6171821735531805, "policy": -3.44719567922111e-06, "anchor_kl": 4.5364405597746826e-07, "value": 0.267722678198097, "belief": 0.18771274772820865, "approx_kl": 9.30756421344642e-09, "samples": 658, "actions": 4386 }, "samples": 329, "early_kl_stop": false, "seconds": 11.240961958999833 }, { "round": 29, "ppo": { "loss": 0.36211739046367136, "policy": 2.434882983135113e-06, "anchor_kl": 4.0252915808141216e-07, "value": 0.020389392292444786, "belief": 0.16770132419865902, "approx_kl": 7.841687193434968e-09, "samples": 762, "actions": 5656 }, "samples": 381, "early_kl_stop": false, "seconds": 14.572274166999705 }, { "round": 30, "ppo": { "loss": 0.5314487383720723, "policy": -1.0197149946334514e-05, "anchor_kl": 5.324699265337528e-07, "value": 0.17108771902449588, "belief": 0.16798705655209561, "approx_kl": 6.742180978998225e-09, "samples": 752, "actions": 5318 }, "samples": 376, "early_kl_stop": false, "seconds": 13.822133792000386 }, { "round": 31, "ppo": { "loss": 0.35934976078801273, "policy": -9.589498827916494e-07, "anchor_kl": 4.0940930091790685e-07, "value": 0.02441366005512475, "belief": 0.177646416499748, "approx_kl": 4.144674687652114e-09, "samples": 644, "actions": 4160 }, "samples": 322, "early_kl_stop": false, "seconds": 11.23210808299882 }, { "round": 32, "ppo": { "loss": 0.6082724892887575, "policy": -1.6579577397920724e-05, "anchor_kl": 6.348831176268654e-07, "value": 0.30482786761433756, "belief": 0.1782855314822665, "approx_kl": 2.5400155706850658e-08, "samples": 754, "actions": 4534 }, "samples": 377, "early_kl_stop": false, "seconds": 12.344354167000347 }, { "stage": "round-32", "validation_score": 0, "best_score": 0 }, { "round": 33, "ppo": { "loss": 0.4110652687559498, "policy": -5.785984675970104e-06, "anchor_kl": 4.265400542021591e-07, "value": 0.0410347248523024, "belief": 0.18865951113506038, "approx_kl": 8.761139785335143e-09, "samples": 722, "actions": 4404 }, "samples": 361, "early_kl_stop": false, "seconds": 12.533811249999417 }, { "round": 34, "ppo": { "loss": 0.5980861756527042, "policy": -1.8012712630864136e-05, "anchor_kl": 7.641329581966435e-07, "value": 0.21156757999396186, "belief": 0.16195918424136438, "approx_kl": 1.0296137615640636e-08, "samples": 686, "actions": 4290 }, "samples": 343, "early_kl_stop": false, "seconds": 11.388618041999507 }, { "round": 35, "ppo": { "loss": 0.3813615984116976, "policy": -3.079793249551779e-06, "anchor_kl": 4.7050650085333943e-07, "value": 0.039199434930230946, "belief": 0.15614726436455795, "approx_kl": 1.1257617083052169e-08, "samples": 638, "actions": 4064 }, "samples": 319, "early_kl_stop": false, "seconds": 12.395912207999572 }, { "round": 36, "ppo": { "loss": 0.44144202081056744, "policy": -3.308485067643962e-06, "anchor_kl": 8.077923327361562e-07, "value": 0.10417472649324726, "belief": 0.16949331329746561, "approx_kl": 1.2113334058405279e-08, "samples": 910, "actions": 5560 }, "samples": 455, "early_kl_stop": false, "seconds": 14.918387833000452 }, { "round": 37, "ppo": { "loss": 0.38974281125947047, "policy": -7.097070154390837e-06, "anchor_kl": 8.048940145499728e-07, "value": 0.042678142995818666, "belief": 0.19153615244125066, "approx_kl": 1.2557536951805415e-08, "samples": 608, "actions": 4494 }, "samples": 304, "early_kl_stop": false, "seconds": 10.627120999999534 }, { "round": 38, "ppo": { "loss": 0.48714203984811694, "policy": -2.7481455978523233e-05, "anchor_kl": 1.2120043689724903e-06, "value": 0.1461424847731426, "belief": 0.18469569730775554, "approx_kl": 3.277833333434194e-08, "samples": 706, "actions": 5004 }, "samples": 353, "early_kl_stop": false, "seconds": 12.43266337499881 }, { "round": 39, "ppo": { "loss": 0.38307077174564613, "policy": -4.572928259053181e-06, "anchor_kl": 7.081261157471784e-07, "value": 0.04606193087073331, "belief": 0.16849355514823777, "approx_kl": 1.1204674686353231e-08, "samples": 776, "actions": 4846 }, "samples": 388, "early_kl_stop": false, "seconds": 13.449488665999525 }, { "round": 40, "ppo": { "loss": 0.5262456282421395, "policy": 4.147489865620931e-07, "anchor_kl": 1.3560064040957289e-06, "value": 0.21186314192083147, "belief": 0.160752795268724, "approx_kl": 2.338654465145535e-08, "samples": 648, "actions": 3852 }, "samples": 324, "early_kl_stop": false, "seconds": 11.356789458999629 }, { "round": 41, "ppo": { "loss": 0.3233040434330009, "policy": -6.4942639234454135e-06, "anchor_kl": 1.2638358138839215e-06, "value": 0.022661873919152937, "belief": 0.18542867309825364, "approx_kl": 1.5236870493999747e-08, "samples": 688, "actions": 4738 }, "samples": 344, "early_kl_stop": false, "seconds": 12.288893042001291 }, { "round": 42, "ppo": { "loss": 0.5624391911826896, "policy": -1.6695715090226845e-05, "anchor_kl": 2.080275386407594e-06, "value": 0.25843902410575625, "belief": 0.1745237613306243, "approx_kl": 2.4556055576843624e-08, "samples": 676, "actions": 4548 }, "samples": 338, "early_kl_stop": false, "seconds": 12.878628624999692 }, { "round": 43, "ppo": { "loss": 0.3365613258815742, "policy": -1.0817370436213039e-05, "anchor_kl": 1.1168936586349523e-06, "value": 0.030387414862645103, "belief": 0.1661001393297413, "approx_kl": 1.9213772024191062e-08, "samples": 666, "actions": 4408 }, "samples": 333, "early_kl_stop": false, "seconds": 10.541936749999877 }, { "round": 44, "ppo": { "loss": 0.5321836865047891, "policy": -3.764137086097909e-05, "anchor_kl": 2.9577220010598015e-06, "value": 0.2598895319740088, "belief": 0.18796809536500894, "approx_kl": 5.183238176888054e-08, "samples": 718, "actions": 5266 }, "samples": 359, "early_kl_stop": false, "seconds": 11.89054808400033 }, { "round": 45, "ppo": { "loss": 0.3169401465470974, "policy": -5.40049746632576e-06, "anchor_kl": 1.6661800887399854e-06, "value": 0.02257745574013545, "belief": 0.1751642852792373, "approx_kl": 2.1973481556012933e-08, "samples": 650, "actions": 4512 }, "samples": 325, "early_kl_stop": false, "seconds": 12.425301500001297 }, { "round": 46, "ppo": { "loss": 0.4726611711894688, "policy": 2.0256670920745186e-06, "anchor_kl": 4.657239732414777e-06, "value": 0.15922424181000047, "belief": 0.1575612850163294, "approx_kl": 1.3864052522441614e-07, "samples": 736, "actions": 4918 }, "samples": 368, "early_kl_stop": false, "seconds": 12.673876999999266 }, { "round": 47, "ppo": { "loss": 0.34339073732495307, "policy": -3.315955400466919e-06, "anchor_kl": 3.5903056220831787e-06, "value": 0.03340241340920329, "belief": 0.18684057295322418, "approx_kl": 2.6277266442775726e-08, "samples": 800, "actions": 5604 }, "samples": 400, "early_kl_stop": false, "seconds": 13.744887875000131 }, { "round": 48, "ppo": { "loss": 0.4472398462906555, "policy": -7.632019082505499e-05, "anchor_kl": 4.938072775921363e-06, "value": 0.17398852723926755, "belief": 0.16885405775140877, "approx_kl": 3.4820700282285434e-07, "samples": 796, "actions": 6114 }, "samples": 398, "early_kl_stop": false, "seconds": 12.834215458000472 }, { "stage": "round-48", "validation_score": 0, "best_score": 0 }, { "round": 49, "ppo": { "loss": 0.34123810755511735, "policy": -6.886256253061119e-06, "anchor_kl": 2.413566996083709e-06, "value": 0.04466785203116986, "belief": 0.18237228645868828, "approx_kl": 9.25437530125577e-08, "samples": 652, "actions": 4650 }, "samples": 326, "early_kl_stop": false, "seconds": 12.363830709000467 }, { "round": 50, "ppo": { "loss": 0.5865871643827808, "policy": -2.7519416424535935e-05, "anchor_kl": 3.3522525434877495e-06, "value": 0.3599334239238693, "belief": 0.2010598269201094, "approx_kl": 3.731298833457629e-08, "samples": 620, "actions": 5572 }, "samples": 310, "early_kl_stop": false, "seconds": 11.4006315000006 }, { "round": 51, "ppo": { "loss": 0.2936433883403215, "policy": -1.555992959161187e-05, "anchor_kl": 2.6047153865869463e-06, "value": 0.02284379205797752, "belief": 0.16407825727048897, "approx_kl": 4.880461919201588e-08, "samples": 674, "actions": 4114 }, "samples": 337, "early_kl_stop": false, "seconds": 11.227483000000575 }, { "round": 52, "ppo": { "loss": 0.580580355300278, "policy": -5.579209718547884e-05, "anchor_kl": 7.015099707460336e-06, "value": 0.3388769540263981, "belief": 0.1964623341306311, "approx_kl": 2.048298960826436e-07, "samples": 610, "actions": 5118 }, "samples": 305, "early_kl_stop": false, "seconds": 11.562775374999546 }, { "round": 53, "ppo": { "loss": 0.34909863736240837, "policy": -1.6654051883875976e-05, "anchor_kl": 5.647575844651802e-06, "value": 0.04144460778705209, "belief": 0.16607045979179136, "approx_kl": 4.035567773286175e-07, "samples": 684, "actions": 4734 }, "samples": 342, "early_kl_stop": false, "seconds": 12.04103487499924 }, { "round": 54, "ppo": { "loss": 0.4801193990825135, "policy": 1.897221362149274e-05, "anchor_kl": 1.0426627127737537e-05, "value": 0.2515830339656936, "belief": 0.16157769485388274, "approx_kl": 1.6608011023497876e-07, "samples": 648, "actions": 4188 }, "samples": 324, "early_kl_stop": false, "seconds": 10.955311959000028 }, { "round": 55, "ppo": { "loss": 0.33880770859840026, "policy": -8.983532678706484e-07, "anchor_kl": 4.6613218676073645e-06, "value": 0.041650012896167043, "belief": 0.17418281858816198, "approx_kl": 1.6539996372405873e-07, "samples": 764, "actions": 5036 }, "samples": 382, "early_kl_stop": false, "seconds": 12.291360667000845 }, { "round": 56, "ppo": { "loss": 0.3577471317783479, "policy": -6.787971623482243e-05, "anchor_kl": 1.0546501102076965e-05, "value": 0.11469673012853951, "belief": 0.1837316474286459, "approx_kl": 3.301200547045277e-07, "samples": 744, "actions": 4696 }, "samples": 372, "early_kl_stop": false, "seconds": 13.374362332999226 }, { "round": 57, "ppo": { "loss": 0.24586398757057762, "policy": -1.5991421854072582e-05, "anchor_kl": 3.586896036039101e-06, "value": 0.01050807167826821, "belief": 0.18081036714908202, "approx_kl": 5.333933764666882e-07, "samples": 732, "actions": 4348 }, "samples": 366, "early_kl_stop": false, "seconds": 12.680850708000435 }, { "round": 58, "ppo": { "loss": 0.5195657474769129, "policy": -0.00013414927431055018, "anchor_kl": 1.6918938913354274e-05, "value": 0.2318935191510497, "belief": 0.18951642746860917, "approx_kl": 1.296831445919501e-06, "samples": 740, "actions": 5372 }, "samples": 370, "early_kl_stop": false, "seconds": 12.74253866700019 }, { "round": 59, "ppo": { "loss": 0.2837424045228588, "policy": 2.0169824148419366e-06, "anchor_kl": 4.597281753271317e-06, "value": 0.028238169037623592, "belief": 0.15750182316880806, "approx_kl": 7.934245972295516e-08, "samples": 838, "actions": 5790 }, "samples": 419, "early_kl_stop": false, "seconds": 14.392572541000845 }, { "round": 60, "ppo": { "loss": 0.5961791160977334, "policy": -2.185398960683126e-05, "anchor_kl": 2.4300583297745944e-05, "value": 0.3769649881043125, "belief": 0.18047486220203568, "approx_kl": 4.113815499006336e-07, "samples": 586, "actions": 4832 }, "samples": 293, "early_kl_stop": false, "seconds": 10.175231332999829 }, { "round": 61, "ppo": { "loss": 0.35493865515202727, "policy": 1.047273625188799e-05, "anchor_kl": 8.441716765935156e-06, "value": 0.051015214916707866, "belief": 0.17028672463858305, "approx_kl": 2.6443549216992324e-07, "samples": 670, "actions": 4726 }, "samples": 335, "early_kl_stop": false, "seconds": 11.279638666999745 }, { "round": 62, "ppo": { "loss": 0.438005578788844, "policy": -8.143599532524088e-05, "anchor_kl": 2.7439442205969968e-05, "value": 0.21756295531413444, "belief": 0.169233654053579, "approx_kl": 3.693887023452446e-07, "samples": 726, "actions": 4814 }, "samples": 363, "early_kl_stop": false, "seconds": 12.171465166999042 }, { "round": 63, "ppo": { "loss": 0.3954891061283046, "policy": -1.5345419415776034e-07, "anchor_kl": 1.1120399401265758e-05, "value": 0.06797796497063607, "belief": 0.186421021680284, "approx_kl": 2.721949301151015e-07, "samples": 644, "actions": 5850 }, "samples": 322, "early_kl_stop": false, "seconds": 12.266677999999956 }, { "round": 64, "ppo": { "loss": 0.3904332318328152, "policy": -5.4430927852210875e-05, "anchor_kl": 3.2671306903422604e-05, "value": 0.20021946596819606, "belief": 0.17855914469965067, "approx_kl": 1.9783823234371294e-07, "samples": 618, "actions": 5246 }, "samples": 309, "early_kl_stop": false, "seconds": 10.581776041999547 }, { "stage": "round-64", "validation_score": 0.0, "best_score": 0 }, { "round": 65, "ppo": { "loss": 0.34275312338556563, "policy": -5.304142832756043e-05, "anchor_kl": 1.3476908173158466e-05, "value": 0.038649801830095906, "belief": 0.1711451340998922, "approx_kl": 4.834761576993124e-07, "samples": 560, "actions": 4626 }, "samples": 280, "early_kl_stop": false, "seconds": 9.933415999999852 }, { "round": 66, "ppo": { "loss": 0.5831509653501715, "policy": -0.00013008055511427803, "anchor_kl": 6.008888392527319e-05, "value": 0.3692891448736191, "belief": 0.17865905247948652, "approx_kl": 8.78031496987021e-07, "samples": 652, "actions": 4690 }, "samples": 326, "early_kl_stop": false, "seconds": 11.50466249999954 }, { "round": 67, "ppo": { "loss": 0.3743048288719091, "policy": -3.231311701748469e-05, "anchor_kl": 2.3590948455408852e-05, "value": 0.06629837212497242, "belief": 0.16117030785304226, "approx_kl": 2.6484007296496876e-07, "samples": 584, "actions": 4476 }, "samples": 292, "early_kl_stop": false, "seconds": 10.336333833998651 }, { "round": 68, "ppo": { "loss": 0.4869099936177654, "policy": -3.496543053657778e-05, "anchor_kl": 8.470489979851165e-05, "value": 0.2953557124060969, "belief": 0.17782552477813537, "approx_kl": 5.657694513027147e-07, "samples": 620, "actions": 4694 }, "samples": 310, "early_kl_stop": false, "seconds": 11.88318179199996 }, { "round": 69, "ppo": { "loss": 0.2945056479735516, "policy": 7.04672549050452e-06, "anchor_kl": 1.0583656817869702e-05, "value": 0.020521677794522353, "belief": 0.1674058177882449, "approx_kl": 2.8727761899043286e-06, "samples": 742, "actions": 4958 }, "samples": 371, "early_kl_stop": false, "seconds": 12.252957582999443 }, { "round": 70, "ppo": { "loss": 0.3915925869345665, "policy": -6.831899285316468e-05, "anchor_kl": 0.00011334842345604556, "value": 0.23209055479615925, "belief": 0.16792187541723252, "approx_kl": 1.2517254799604416e-06, "samples": 800, "actions": 6076 }, "samples": 400, "early_kl_stop": false, "seconds": 12.923589957999866 }, { "round": 71, "ppo": { "loss": 0.3826256521009459, "policy": -6.071613927192876e-05, "anchor_kl": 1.2787871376894158e-05, "value": 0.05962082007762246, "belief": 0.16569291517652315, "approx_kl": 1.0422417180334691e-06, "samples": 812, "actions": 5466 }, "samples": 406, "early_kl_stop": false, "seconds": 13.955563375000565 }, { "round": 72, "ppo": { "loss": 0.4265236663686879, "policy": 1.6686670920428107e-05, "anchor_kl": 0.00017388023524664972, "value": 0.22102363083292456, "belief": 0.17785958805504967, "approx_kl": 7.53050998729818e-07, "samples": 680, "actions": 4276 }, "samples": 340, "early_kl_stop": false, "seconds": 12.55478941599904 }, { "round": 73, "ppo": { "loss": 0.378459846830938, "policy": -3.6385066183914703e-06, "anchor_kl": 9.211648486156213e-06, "value": 0.05419353914792201, "belief": 0.17937817988523924, "approx_kl": 3.868365192262496e-07, "samples": 502, "actions": 3632 }, "samples": 251, "early_kl_stop": false, "seconds": 8.862942750000002 }, { "round": 74, "ppo": { "loss": 0.3485002202598814, "policy": -2.7795776757657978e-05, "anchor_kl": 0.0001766743325822775, "value": 0.14963258749404243, "belief": 0.16381675046246957, "approx_kl": 7.00424952290395e-07, "samples": 794, "actions": 6510 }, "samples": 397, "early_kl_stop": false, "seconds": 13.56521666700064 }, { "round": 75, "ppo": { "loss": 0.35814486965862896, "policy": 1.7066825583099276e-06, "anchor_kl": 2.2214967613336387e-05, "value": 0.05959861931382915, "belief": 0.1695217853493489, "approx_kl": 3.7455458860546227e-07, "samples": 686, "actions": 4792 }, "samples": 343, "early_kl_stop": false, "seconds": 12.255302000001393 }, { "round": 76, "ppo": { "loss": 0.3954085357160271, "policy": -0.00011970708469320389, "anchor_kl": 0.00016261522593509029, "value": 0.20011233289319566, "belief": 0.15590915259395405, "approx_kl": 2.9612207832567685e-06, "samples": 866, "actions": 5276 }, "samples": 433, "early_kl_stop": false, "seconds": 14.003025374999197 }, { "round": 77, "ppo": { "loss": 0.27609392059045834, "policy": -3.7646412358179197e-06, "anchor_kl": 4.027771148765089e-05, "value": 0.02651905828247686, "belief": 0.16065571547209562, "approx_kl": 1.1882550293169086e-07, "samples": 728, "actions": 5014 }, "samples": 364, "early_kl_stop": false, "seconds": 11.347284749999744 }, { "round": 78, "ppo": { "loss": 0.340189452631404, "policy": -0.00014670902680338852, "anchor_kl": 0.00025312614806128584, "value": 0.13193262727135688, "belief": 0.18354489268927737, "approx_kl": 3.6847011763673036e-06, "samples": 754, "actions": 5164 }, "samples": 377, "early_kl_stop": false, "seconds": 12.144811541000308 }, { "round": 79, "ppo": { "loss": 0.3679338090880188, "policy": -0.0001308871154299753, "anchor_kl": 6.459625239836394e-05, "value": 0.04989206793154785, "belief": 0.1647585396192031, "approx_kl": 3.193262507441736e-06, "samples": 668, "actions": 5042 }, "samples": 334, "early_kl_stop": false, "seconds": 11.908317416999125 }, { "round": 80, "ppo": { "loss": 0.43565517090834105, "policy": 8.47511358194418e-06, "anchor_kl": 0.00038359645615499533, "value": 0.2243327272313458, "belief": 0.1883853268060651, "approx_kl": 1.1166785629140125e-06, "samples": 572, "actions": 4552 }, "samples": 286, "early_kl_stop": false, "seconds": 10.777723042001526 }, { "stage": "round-80", "validation_score": 0.1111111111111111, "best_score": 0.1111111111111111 }, { "round": 81, "ppo": { "loss": 0.3356556016438288, "policy": -0.00014610612825642253, "anchor_kl": 9.667154312806898e-05, "value": 0.03682451994907587, "belief": 0.1720303328226981, "approx_kl": 8.358967915385557e-06, "samples": 798, "actions": 5124 }, "samples": 399, "early_kl_stop": false, "seconds": 12.609279540998614 }, { "round": 82, "ppo": { "loss": 0.37927412499523944, "policy": -0.00019073366042866044, "anchor_kl": 0.0003109703150756114, "value": 0.16629281104098223, "belief": 0.1526840714895108, "approx_kl": 2.5615126549996665e-06, "samples": 734, "actions": 5324 }, "samples": 367, "early_kl_stop": false, "seconds": 13.529078916999424 }, { "round": 83, "ppo": { "loss": 0.39411171260128264, "policy": -4.0987853184107354e-05, "anchor_kl": 5.48244669350456e-05, "value": 0.06572314456974858, "belief": 0.18081660284322895, "approx_kl": 1.3552946898059708e-06, "samples": 634, "actions": 5480 }, "samples": 317, "early_kl_stop": false, "seconds": 11.482346208000308 }, { "round": 84, "ppo": { "loss": 0.30880907928788814, "policy": -3.0472555330821447e-05, "anchor_kl": 0.00036147287037592476, "value": 0.13682159392175253, "belief": 0.17294827330035048, "approx_kl": 5.547378163833123e-06, "samples": 770, "actions": 5432 }, "samples": 385, "early_kl_stop": false, "seconds": 13.711109957999724 }, { "round": 85, "ppo": { "loss": 0.3387542598711168, "policy": -2.438374807052218e-05, "anchor_kl": 4.9871751852544115e-05, "value": 0.04394774893130289, "belief": 0.14167695080510098, "approx_kl": 9.300880305538971e-07, "samples": 774, "actions": 4690 }, "samples": 387, "early_kl_stop": false, "seconds": 12.889784666000196 }, { "round": 86, "ppo": { "loss": 0.41163976999938995, "policy": -6.557237409967982e-05, "anchor_kl": 0.0007077401097917974, "value": 0.22551353830854778, "belief": 0.17630811351911294, "approx_kl": 2.454149078084422e-06, "samples": 702, "actions": 5126 }, "samples": 351, "early_kl_stop": false, "seconds": 12.396800458000143 }, { "round": 87, "ppo": { "loss": 0.34400616867362327, "policy": -9.595812444791854e-05, "anchor_kl": 5.800044749942017e-05, "value": 0.04073433218957306, "belief": 0.16642953563223845, "approx_kl": 2.02467770361529e-06, "samples": 636, "actions": 4174 }, "samples": 318, "early_kl_stop": false, "seconds": 10.97333562499989 }, { "round": 88, "ppo": { "loss": 0.4092904344838571, "policy": -0.00034284167350294756, "anchor_kl": 0.0008234951424444802, "value": 0.21973183241119998, "belief": 0.16854444437007854, "approx_kl": 1.827805218386677e-05, "samples": 748, "actions": 5264 }, "samples": 374, "early_kl_stop": false, "seconds": 11.60650270899896 }, { "round": 89, "ppo": { "loss": 0.3253878716390796, "policy": -0.00035819631719261134, "anchor_kl": 0.00010547310511570207, "value": 0.05541942094592235, "belief": 0.1769923947364183, "approx_kl": 6.815343376089582e-05, "samples": 654, "actions": 4046 }, "samples": 327, "early_kl_stop": false, "seconds": 10.91758533400025 }, { "round": 90, "ppo": { "loss": 0.6327342875301838, "policy": -2.1951505914330482e-05, "anchor_kl": 0.0009915007950288175, "value": 0.4912377721630037, "belief": 0.15295682412882647, "approx_kl": 1.9275816157460213e-05, "samples": 768, "actions": 5972 }, "samples": 384, "early_kl_stop": false, "seconds": 13.869618249998894 }, { "round": 91, "ppo": { "loss": 0.3556676037149665, "policy": -1.3154046035107272e-05, "anchor_kl": 5.4806435447767115e-05, "value": 0.05520120692197923, "belief": 0.1537196492339358, "approx_kl": 1.6389920573129441e-06, "samples": 810, "actions": 4898 }, "samples": 405, "early_kl_stop": false, "seconds": 12.123014708000483 }, { "round": 92, "ppo": { "loss": 0.41995771950609806, "policy": -0.0010509431283100847, "anchor_kl": 0.0012472778811285937, "value": 0.2322406245084277, "belief": 0.17884088685088828, "approx_kl": 4.941588625884973e-05, "samples": 754, "actions": 4848 }, "samples": 377, "early_kl_stop": false, "seconds": 12.457418958001654 }, { "round": 93, "ppo": { "loss": 0.3193457246716343, "policy": 3.072591985005941e-05, "anchor_kl": 3.661856026698856e-05, "value": 0.016679451939886184, "belief": 0.1863095563516688, "approx_kl": 3.9138275184738105e-06, "samples": 536, "actions": 4406 }, "samples": 268, "early_kl_stop": false, "seconds": 9.149040291000347 }, { "round": 94, "ppo": { "loss": 0.4869650430653406, "policy": 0.00034056599859310234, "anchor_kl": 0.0017566097495032211, "value": 0.2983889042035393, "belief": 0.1654304706532022, "approx_kl": 1.3045189679478822e-05, "samples": 736, "actions": 4904 }, "samples": 368, "early_kl_stop": false, "seconds": 12.340036457999304 }, { "round": 95, "ppo": { "loss": 0.24433126356846616, "policy": 1.7145260313126255e-05, "anchor_kl": 4.128377478711484e-05, "value": 0.01969073502926488, "belief": 0.1729898624279708, "approx_kl": 2.4888990800711047e-06, "samples": 662, "actions": 5166 }, "samples": 331, "early_kl_stop": false, "seconds": 11.468624625000302 }, { "round": 96, "ppo": { "loss": 0.5214908684541053, "policy": 0.0001427928024023509, "anchor_kl": 0.0017794997025393808, "value": 0.2633387984003315, "belief": 0.15658825086402403, "approx_kl": 8.424948846148314e-06, "samples": 682, "actions": 4586 }, "samples": 341, "early_kl_stop": false, "seconds": 11.182884333000402 }, { "stage": "round-96", "validation_score": -0.018518518518518517, "best_score": 0.1111111111111111 }, { "round": 97, "ppo": { "loss": 0.34531797412075577, "policy": 2.8380275338510925e-05, "anchor_kl": 7.351795537470946e-05, "value": 0.051311631675196596, "belief": 0.18933653690015215, "approx_kl": 9.982628575580978e-07, "samples": 632, "actions": 4570 }, "samples": 316, "early_kl_stop": false, "seconds": 10.337659125001665 }, { "round": 98, "ppo": { "loss": 0.4338683062568522, "policy": -0.0004095885398064979, "anchor_kl": 0.002013979783792972, "value": 0.23872747772536979, "belief": 0.18480563524477076, "approx_kl": 1.6880685986195258e-05, "samples": 694, "actions": 4328 }, "samples": 347, "early_kl_stop": false, "seconds": 11.577839583000241 }, { "round": 99, "ppo": { "loss": 0.3033090577053186, "policy": -5.861807359976117e-05, "anchor_kl": 5.2345305340885844e-05, "value": 0.026015598049644713, "belief": 0.1773267354542455, "approx_kl": 6.567835575392434e-06, "samples": 626, "actions": 4330 }, "samples": 313, "early_kl_stop": false, "seconds": 11.12299879099919 }, { "round": 100, "ppo": { "loss": 0.4080846024249096, "policy": -0.0001961120537349156, "anchor_kl": 0.0022023555281086137, "value": 0.23520829079684263, "belief": 0.1673012669689907, "approx_kl": 4.606552838538745e-06, "samples": 644, "actions": 4940 }, "samples": 322, "early_kl_stop": false, "seconds": 12.195894792001127 }, { "round": 101, "ppo": { "loss": 0.3402062629089981, "policy": -0.0001016933409894099, "anchor_kl": 3.564764289175863e-05, "value": 0.043113647492938355, "belief": 0.17361833870899482, "approx_kl": 5.414996479378372e-06, "samples": 610, "actions": 4672 }, "samples": 305, "early_kl_stop": false, "seconds": 11.71478904200012 }, { "round": 102, "ppo": { "loss": 0.4368515668910033, "policy": 3.565076013901226e-05, "anchor_kl": 0.0027091569881856296, "value": 0.3164478076615901, "belief": 0.162978403366072, "approx_kl": 8.404278867302146e-06, "samples": 622, "actions": 4840 }, "samples": 311, "early_kl_stop": false, "seconds": 10.691602750001039 }, { "round": 103, "ppo": { "loss": 0.3552584761004907, "policy": -0.0001177035791929378, "anchor_kl": 9.241847713615984e-05, "value": 0.04873414401373396, "belief": 0.17849230964318463, "approx_kl": 3.1725828365359308e-06, "samples": 602, "actions": 4242 }, "samples": 301, "early_kl_stop": false, "seconds": 10.366369958999712 }, { "round": 104, "ppo": { "loss": 0.4965393871685023, "policy": 0.00012195463116624867, "anchor_kl": 0.0024942560178947875, "value": 0.3229594750074761, "belief": 0.1760816677095662, "approx_kl": 7.807546672461461e-06, "samples": 774, "actions": 5670 }, "samples": 387, "early_kl_stop": false, "seconds": 13.570977583000058 }, { "round": 105, "ppo": { "loss": 0.37534060903725897, "policy": -0.0002499394199991096, "anchor_kl": 0.0001261404865597088, "value": 0.05964503185544417, "belief": 0.17562885976292456, "approx_kl": 2.13830571886763e-05, "samples": 734, "actions": 5278 }, "samples": 367, "early_kl_stop": false, "seconds": 12.465651916998468 }, { "round": 106, "ppo": { "loss": 0.4683390755368316, "policy": -0.0003325201409018558, "anchor_kl": 0.0026041505844541066, "value": 0.299419431420772, "belief": 0.1645106368738672, "approx_kl": 1.3711791404563447e-05, "samples": 736, "actions": 4946 }, "samples": 368, "early_kl_stop": false, "seconds": 11.565004083000531 }, { "round": 107, "ppo": { "loss": 0.34041170207918553, "policy": -0.0005603602345007053, "anchor_kl": 0.00011582454197037883, "value": 0.03907139850915342, "belief": 0.18404790547446928, "approx_kl": 1.9217476880800425e-05, "samples": 690, "actions": 4750 }, "samples": 345, "early_kl_stop": false, "seconds": 10.711245207998218 }, { "round": 108, "ppo": { "loss": 0.31643603928387165, "policy": -0.0006388432473084177, "anchor_kl": 0.0015548839724396257, "value": 0.11581156027673835, "belief": 0.14602519974042552, "approx_kl": 2.4578999726096307e-05, "samples": 852, "actions": 4982 }, "samples": 426, "early_kl_stop": false, "seconds": 12.624377916999947 }, { "round": 109, "ppo": { "loss": 0.41706447629376525, "policy": 3.010743598598257e-05, "anchor_kl": 0.00026375673407561065, "value": 0.07326589667880505, "belief": 0.17197070642357867, "approx_kl": 2.519922858903072e-05, "samples": 786, "actions": 4854 }, "samples": 393, "early_kl_stop": false, "seconds": 12.301625165999212 }, { "round": 110, "ppo": { "loss": 0.5399146613901888, "policy": -0.0006182552177525131, "anchor_kl": 0.0017811788464572622, "value": 0.3689870618989129, "belief": 0.15504824803293457, "approx_kl": 7.262826516959795e-05, "samples": 676, "actions": 4302 }, "samples": 338, "early_kl_stop": false, "seconds": 11.317028084000412 }, { "round": 111, "ppo": { "loss": 0.34010358206261465, "policy": -3.0333756754391883e-06, "anchor_kl": 0.0002369805050281578, "value": 0.04271988483867671, "belief": 0.1511777491689859, "approx_kl": 9.781113121567703e-06, "samples": 738, "actions": 4732 }, "samples": 369, "early_kl_stop": false, "seconds": 12.83973129199876 }, { "round": 112, "ppo": { "loss": 0.44609734215145946, "policy": -0.0007560285725003126, "anchor_kl": 0.0024683292296264017, "value": 0.2831444618610379, "belief": 0.18499336387016183, "approx_kl": 5.407489821709675e-05, "samples": 638, "actions": 4658 }, "samples": 319, "early_kl_stop": false, "seconds": 10.197487417000957 }, { "stage": "round-112", "validation_score": -0.14814814814814814, "best_score": 0.1111111111111111 }, { "round": 113, "ppo": { "loss": 0.33866592713340166, "policy": -0.0005075645909084838, "anchor_kl": 0.0008595443668583707, "value": 0.05133357316957286, "belief": 0.17374140637155386, "approx_kl": 3.1789591902505135e-05, "samples": 722, "actions": 4738 }, "samples": 361, "early_kl_stop": false, "seconds": 12.270911915999022 }, { "round": 114, "ppo": { "loss": 0.5620451294529585, "policy": -0.0006662419548741093, "anchor_kl": 0.001864743273315217, "value": 0.4355825890361527, "belief": 0.1618800018856555, "approx_kl": 5.39054795179838e-05, "samples": 648, "actions": 4568 }, "samples": 324, "early_kl_stop": false, "seconds": 11.61586399999942 }, { "round": 115, "ppo": { "loss": 0.2769248927563009, "policy": -0.00023774901850962302, "anchor_kl": 0.0012342270958675136, "value": 0.025878139129433205, "belief": 0.15994935751381054, "approx_kl": 7.808252014728438e-05, "samples": 710, "actions": 5114 }, "samples": 355, "early_kl_stop": false, "seconds": 12.580273624998881 }, { "round": 116, "ppo": { "loss": 0.41718435603497406, "policy": 0.0001657941703046306, "anchor_kl": 0.0030641262683581845, "value": 0.24900373749518662, "belief": 0.1672750874684098, "approx_kl": 2.524191565979063e-05, "samples": 712, "actions": 5110 }, "samples": 356, "early_kl_stop": false, "seconds": 12.06315154100048 }, { "round": 117, "ppo": { "loss": 0.2638221539676508, "policy": 2.9759790330009666e-05, "anchor_kl": 0.0002550705964451364, "value": 0.020250021590836307, "belief": 0.1575875001396725, "approx_kl": 1.0406030690758252e-05, "samples": 748, "actions": 4900 }, "samples": 374, "early_kl_stop": false, "seconds": 12.482599584000127 }, { "round": 118, "ppo": { "loss": 0.36037773623188857, "policy": -0.00026612541962912494, "anchor_kl": 0.002822671970466291, "value": 0.18728303943430027, "belief": 0.16524556531579954, "approx_kl": 8.529843926071415e-06, "samples": 614, "actions": 4100 }, "samples": 307, "early_kl_stop": false, "seconds": 11.482779834001121 }, { "round": 119, "ppo": { "loss": 0.3194208438462107, "policy": -0.00011915793464128727, "anchor_kl": 0.00046892446554542546, "value": 0.038718836425062846, "belief": 0.1664550324822023, "approx_kl": 1.2534303248973476e-05, "samples": 606, "actions": 4828 }, "samples": 303, "early_kl_stop": false, "seconds": 10.831750625000495 }, { "round": 120, "ppo": { "loss": 0.45402732561192205, "policy": -0.00014295010797439083, "anchor_kl": 0.002388134316330956, "value": 0.3051282127538035, "belief": 0.20637029289237915, "approx_kl": 6.103152108769263e-06, "samples": 620, "actions": 4844 }, "samples": 310, "early_kl_stop": false, "seconds": 11.298984999999448 }, { "round": 121, "ppo": { "loss": 0.31969300885167384, "policy": 3.362703137099743e-05, "anchor_kl": 0.0006050472216480153, "value": 0.03405880063979162, "belief": 0.17631620251470143, "approx_kl": 3.005769556491739e-05, "samples": 576, "actions": 4194 }, "samples": 288, "early_kl_stop": false, "seconds": 10.598979041998973 }, { "round": 122, "ppo": { "loss": 0.3936293887893359, "policy": 0.00010861063003540039, "anchor_kl": 0.0023350342309568077, "value": 0.24468207788467408, "belief": 0.17327190375328064, "approx_kl": 2.172292026565022e-05, "samples": 750, "actions": 4810 }, "samples": 375, "early_kl_stop": false, "seconds": 12.397454458001448 }, { "round": 123, "ppo": { "loss": 0.40020292123158774, "policy": 0.00012424548467000326, "anchor_kl": 0.0008227153700621178, "value": 0.0696490394944946, "belief": 0.1861157548427582, "approx_kl": 3.3805581369961146e-06, "samples": 600, "actions": 3804 }, "samples": 300, "early_kl_stop": false, "seconds": 10.550259790999917 }, { "round": 124, "ppo": { "loss": 0.5404799008369445, "policy": -0.0003094242622747141, "anchor_kl": 0.002050886542805101, "value": 0.3512840117075864, "belief": 0.16589895853224923, "approx_kl": 1.972065690150724e-05, "samples": 850, "actions": 5676 }, "samples": 425, "early_kl_stop": false, "seconds": 13.108403333000751 }, { "round": 125, "ppo": { "loss": 0.3008581376477574, "policy": -0.00012702067916312915, "anchor_kl": 0.0006583722161450894, "value": 0.03483712505758478, "belief": 0.15185303529829122, "approx_kl": 2.5193269751714855e-06, "samples": 712, "actions": 4690 }, "samples": 356, "early_kl_stop": false, "seconds": 12.364789666999059 }, { "round": 126, "ppo": { "loss": 0.41094049983299696, "policy": 0.0003777313308838086, "anchor_kl": 0.002345845087611218, "value": 0.2242793250709581, "belief": 0.16237300525491055, "approx_kl": 2.1257483137596194e-05, "samples": 780, "actions": 5522 }, "samples": 390, "early_kl_stop": false, "seconds": 12.932946124999944 }, { "round": 127, "ppo": { "loss": 0.25967057983433534, "policy": -8.965945485884768e-05, "anchor_kl": 0.0002667132705148087, "value": 0.022752964844322068, "belief": 0.15495971662138924, "approx_kl": 1.539345026999561e-05, "samples": 788, "actions": 5086 }, "samples": 394, "early_kl_stop": false, "seconds": 12.023093874999176 }, { "round": 128, "ppo": { "loss": 0.3811246964226787, "policy": -0.00010877350966135661, "anchor_kl": 0.00215097206455539, "value": 0.1816894531560441, "belief": 0.15920284328361353, "approx_kl": 2.46294269648691e-05, "samples": 768, "actions": 5194 }, "samples": 384, "early_kl_stop": false, "seconds": 12.689505292000831 }, { "stage": "round-128", "validation_score": -0.09259259259259259, "best_score": 0.1111111111111111 } ], "limitation": "holdout imitation loss is not playing strength; best selected on paired validation", "test": { "games": 576, "selection_score": -0.10648148148148148, "summary": { "bidding": { "games": 288, "deals": 48, "mean_bid": 2.1875, "total_redeals": 12, "lstm": { "player_games": 432, "mean_score": 0.12962962962962962, "win_rate": 0.4722222222222222, "mean_score_95pct_cluster_bootstrap": [ -0.08333333333333333, 0.3888888888888889 ], "by_role": { "landlord": { "player_games": 144, "mean_score": 2.0277777777777777, "win_rate": 0.6805555555555556 }, "landlord_down": { "player_games": 144, "mean_score": -0.8472222222222222, "win_rate": 0.3611111111111111 }, "landlord_up": { "player_games": 144, "mean_score": -0.7916666666666666, "win_rate": 0.375 } }, "by_composition": { "1_vs_2": { "player_games": 144, "mean_score": 0.19444444444444445, "win_rate": 0.4861111111111111 }, "2_vs_1": { "player_games": 288, "mean_score": 0.09722222222222222, "win_rate": 0.4652777777777778 } }, "bid_counts": { "0": 204, "1": 45, "2": 84, "3": 54 }, "decision_seconds": { "count": 4738, "mean": 0.0024290908678962714, "p50": 0.0017553750003571622, "p95": 0.004693791997851804, "p99": 0.007003999999142252, "max": 0.03849849999824073 } }, "resnet": { "player_games": 432, "mean_score": -0.12962962962962962, "win_rate": 0.4305555555555556, "mean_score_95pct_cluster_bootstrap": [ -0.3888888888888889, 0.08333333333333333 ], "by_role": { "landlord": { "player_games": 144, "mean_score": 1.4722222222222223, "win_rate": 0.6111111111111112 }, "landlord_down": { "player_games": 144, "mean_score": -0.9027777777777778, "win_rate": 0.3472222222222222 }, "landlord_up": { "player_games": 144, "mean_score": -0.9583333333333334, "win_rate": 0.3333333333333333 } }, "by_composition": { "1_vs_2": { "player_games": 144, "mean_score": -0.19444444444444445, "win_rate": 0.4166666666666667 }, "2_vs_1": { "player_games": 288, "mean_score": -0.09722222222222222, "win_rate": 0.4375 } }, "bid_counts": { "0": 204, "1": 45, "2": 84, "3": 54 }, "decision_seconds": { "count": 4752, "mean": 0.014777978487191135, "p50": 0.011539833998540416, "p95": 0.024261999998998363, "p99": 0.04513658299401868, "max": 0.18392479199974332 } }, "pure_team_matchups": { "lstm_landlord": { "games": 48, "landlord_win_rate": 0.6875, "landlord_mean_score": 2.0833333333333335 }, "resnet_landlord": { "games": 48, "landlord_win_rate": 0.6041666666666666, "landlord_mean_score": 1.4166666666666667 } } }, "fixed": { "games": 288, "deals": 48, "mean_bid": 1, "total_redeals": 0, "lstm": { "player_games": 432, "mean_score": 0.08333333333333333, "win_rate": 0.5277777777777778, "mean_score_95pct_cluster_bootstrap": [ -0.018518518518518517, 0.2037037037037037 ], "by_role": { "landlord": { "player_games": 144, "mean_score": -0.125, "win_rate": 0.5 }, "landlord_down": { "player_games": 144, "mean_score": 0.1875, "win_rate": 0.5416666666666666 }, "landlord_up": { "player_games": 144, "mean_score": 0.1875, "win_rate": 0.5416666666666666 } }, "by_composition": { "1_vs_2": { "player_games": 144, "mean_score": 0.125, "win_rate": 0.5416666666666666 }, "2_vs_1": { "player_games": 288, "mean_score": 0.0625, "win_rate": 0.5208333333333334 } }, "bid_counts": { "0": 0, "1": 0, "2": 0, "3": 0 }, "decision_seconds": { "count": 4631, "mean": 0.00239102467029075, "p50": 0.0017541249981150031, "p95": 0.004621833999408409, "p99": 0.006799167000281159, "max": 0.03820970800006762 } }, "resnet": { "player_games": 432, "mean_score": -0.08333333333333333, "win_rate": 0.4930555555555556, "mean_score_95pct_cluster_bootstrap": [ -0.2037037037037037, 0.018518518518518517 ], "by_role": { "landlord": { "player_games": 144, "mean_score": -0.5, "win_rate": 0.4375 }, "landlord_down": { "player_games": 144, "mean_score": 0.125, "win_rate": 0.5208333333333334 }, "landlord_up": { "player_games": 144, "mean_score": 0.125, "win_rate": 0.5208333333333334 } }, "by_composition": { "1_vs_2": { "player_games": 144, "mean_score": -0.125, "win_rate": 0.4791666666666667 }, "2_vs_1": { "player_games": 288, "mean_score": -0.0625, "win_rate": 0.5 } }, "bid_counts": { "0": 0, "1": 0, "2": 0, "3": 0 }, "decision_seconds": { "count": 4628, "mean": 0.015213161093120063, "p50": 0.01167416599491844, "p95": 0.024690166996151675, "p99": 0.08202295900264289, "max": 0.18462649999855785 } }, "pure_team_matchups": { "lstm_landlord": { "games": 48, "landlord_win_rate": 0.5, "landlord_mean_score": -0.125 }, "resnet_landlord": { "games": 48, "landlord_win_rate": 0.4375, "landlord_mean_score": -0.5 } } } } }, "offline_test": { "samples": 21165, "loss": 0.7485360586471431 } }
{
"status": "complete",
"device": "mps",
"config": {
"data": "/Users/xcx0902/Documents/Github/qojddz/data/qoj-history",
"output": "/Users/xcx0902/Documents/Github/qojddz/QOJDoudizhuModel/model",
"teacher_dir": "/Users/xcx0902/Documents/Github/qojddz/models/douzero-baselines/douzero_ADP",
"device": "mps",
"width": 128,
"warmup_games": 32,
"offline_epochs": 2,
"games_per_round": 12,
"ppo_epochs": 2,
"batch_states": 16,
"max_batch_actions": 4096,
"candidate_cap": 64,
"max_history_games": 0,
"holdout": 0.1,
"learning_rate": 0.0001,
"ppo_learning_rate": 2e-05,
"anchor_weight": 0.02,
"league_size": 4,
"validation_interval": 16,
"validation_deals": 12,
"test_deals": 48,
"seed": 20261002,
"validation_seed": 30261002,
"test_seed": 40261002,
"mode": "mixed",
"teacher_sha256": {
"landlord": "6f2971813495e9c509cbd4a101213c49587472942fa0f018d04c139a6a647c2d",
"landlord_up": "7fb7c495ff095db2ddadd8f9f8fd53cad840211578dabc83bbcc8d75085f843e",
"landlord_down": "c173dbe30258f19b6392c30ab1e330b6b3090bbd9ccdc89a9b18123a6ab6e2d9"
},
"source_sha256": {
"Douzero_Resnet/douzero/dmc/models.py": "b873ebcceb8907ccf5e1b75751bef640da944d5c9d6b100b2e5fb632150c18fa",
"Douzero_Resnet/douzero/env/env.py": "4ba02e8e181db4b84704ab205b94691618bca4d71d8e5ce03c61d2c060f4724a",
"QOJDoudizhuModel/__init__.py": "f9a452c634683d816ff3a33d679ec510b3a6fb9ebaa4f266d39181322895ea91",
"QOJDoudizhuModel/dataset.py": "43bd492ea3190917dafc46cfbaf12a505977976130ab101fcecafed34a4fab73",
"QOJDoudizhuModel/evaluate.py": "543441b8bb404da0f5c39842d2dd4e04b14875a6a7a052c93fe690b0489f5e72",
"QOJDoudizhuModel/features.py": "cc7d076b67b9a59fb8b1b5d4f3e23fa0fe9313e6ed7d2ccbbc763ebd99563dc6",
"QOJDoudizhuModel/learning.py": "2276f3c9af8bb7d1ddab6e85e33a9d1f4325426dc60f2d51dacd3b113cf92f5f",
"QOJDoudizhuModel/network.py": "14450e8d72f40afcd0a814bd609d5f0dab23579055c06bfaa239fb5ec4a154dc",
"QOJDoudizhuModel/policy.py": "275892378b80d64e2d469ced9fd27968eafaa699cf265ddd0ade214f798ccdf3",
"QOJDoudizhuModel/storage.py": "c9621015b6dd6c35418ee3fd31a73ea71a545ef51320ec811ccc9e053b943417",
"QOJDoudizhuModel/train.py": "4495d1fa4911cb6d92f39bd9c3691a16fc74fa7ef320986cac67acf923631239",
"qojbot/adapter.py": "64922460bf74a2d2413fe08ecfcc3539b1de5656d7295e154fe98946e48593e9",
"qojbot/curriculum.py": "b005956942bde93e683b9d37e444e6652801f0d6b691a8bbe631291379e88bfa",
"qojbot/douzero.py": "84495d88a2ac67a01a659db0b9b039f5b36d4d4fcd1849712ea9b0912a6c3864",
"qojbot/game.py": "6f8b77ec661dcbaaf2042df24e0453dd3263d7c9cc4df799595b746738ef5ff1",
"qojbot/history.py": "27eeeffd080c476b0b72245278495bda9ec9ed744fe6b85340f7f2742f8c6ea9",
"qojbot/history_training.py": "74ff26534168f4f98652cf6d43c268ab2b3211e3e27f04a9fad9e1a857759b61",
"qojbot/model_benchmark.py": "03de89831e54857840af92bdd47325a13cee48d73035fb45b98508f85e89e9e4",
"qojbot/policy.py": "cb406eb98f43517dbec106d8bc62f6edac00bd2b0c9dace00c85315d1f6008ec",
"qojbot/rules.py": "8fb9ee4ff9de21c58fd667a8e4c22a8c3db9b919e711029638f24340541e76f0"
}
},
"progress": {
"offline_epoch": 2,
"round": 128,
"updates": 28490,
"games": 1536,
"best_score": 0.1111111111111111,
"best_stage": "round-80",
"stage": "ppo"
},
"dataset": {
"fingerprint": "13e5709daa6e5e8d777c104cebd6e7ec75115874095319ee7d570c7a02277878",
"counts": {
"train": 180278,
"validation": 21260,
"test": 21165
},
"valid_games": 7266,
"rejected_games": 0
},
"metrics": [
{
"stage": "baseline",
"validation_score": 0,
"best_score": 0
},
{
"epoch": 1,
"offline_validation": {
"samples": 21260,
"loss": 0.6506123136262902
}
},
{
"stage": "offline-1",
"validation_score": -0.28703703703703703,
"best_score": 0
},
{
"epoch": 2,
"offline_validation": {
"samples": 21260,
"loss": 0.6081839078849818
}