Download model/report.json from xcx0902/QOJDoudizhuModel: direct link, hf CLI and curl.
- Browser
- Download file 68.3 kB
-
https://huggingface.co/xcx0902/QOJDoudizhuModel/resolve/main/model/report.json
- Command line
-
hf download hf://xcx0902/QOJDoudizhuModel/model/report.json
-
curl -L -o report.json https://huggingface.co/xcx0902/QOJDoudizhuModel/resolve/main/model/report.json
68.3 kB
{
"status": "complete",
"device": "mps",
"config": {
"data": "/Users/xcx0902/Documents/Github/qojddz/data/qoj-history",
"output": "/Users/xcx0902/Documents/Github/qojddz/QOJDoudizhuModel/model",
"teacher_dir": "/Users/xcx0902/Documents/Github/qojddz/models/douzero-baselines/douzero_ADP",
"device": "mps",
"width": 128,
"warmup_games": 32,
"offline_epochs": 2,
"games_per_round": 12,
"ppo_epochs": 2,
"batch_states": 16,
"max_batch_actions": 4096,
"candidate_cap": 64,
"max_history_games": 0,
"holdout": 0.1,
"learning_rate": 0.0001,
"ppo_learning_rate": 2e-05,
"anchor_weight": 0.02,
"league_size": 4,
"validation_interval": 16,
"validation_deals": 12,
"test_deals": 48,
"seed": 20261002,
"validation_seed": 30261002,
"test_seed": 40261002,
"mode": "mixed",
"teacher_sha256": {
"landlord": "6f2971813495e9c509cbd4a101213c49587472942fa0f018d04c139a6a647c2d",
"landlord_up": "7fb7c495ff095db2ddadd8f9f8fd53cad840211578dabc83bbcc8d75085f843e",
"landlord_down": "c173dbe30258f19b6392c30ab1e330b6b3090bbd9ccdc89a9b18123a6ab6e2d9"
},
"source_sha256": {
"Douzero_Resnet/douzero/dmc/models.py": "b873ebcceb8907ccf5e1b75751bef640da944d5c9d6b100b2e5fb632150c18fa",
"Douzero_Resnet/douzero/env/env.py": "4ba02e8e181db4b84704ab205b94691618bca4d71d8e5ce03c61d2c060f4724a",
"QOJDoudizhuModel/__init__.py": "f9a452c634683d816ff3a33d679ec510b3a6fb9ebaa4f266d39181322895ea91",
"QOJDoudizhuModel/dataset.py": "43bd492ea3190917dafc46cfbaf12a505977976130ab101fcecafed34a4fab73",
"QOJDoudizhuModel/evaluate.py": "543441b8bb404da0f5c39842d2dd4e04b14875a6a7a052c93fe690b0489f5e72",
"QOJDoudizhuModel/features.py": "cc7d076b67b9a59fb8b1b5d4f3e23fa0fe9313e6ed7d2ccbbc763ebd99563dc6",
"QOJDoudizhuModel/learning.py": "2276f3c9af8bb7d1ddab6e85e33a9d1f4325426dc60f2d51dacd3b113cf92f5f",
"QOJDoudizhuModel/network.py": "14450e8d72f40afcd0a814bd609d5f0dab23579055c06bfaa239fb5ec4a154dc",
"QOJDoudizhuModel/policy.py": "275892378b80d64e2d469ced9fd27968eafaa699cf265ddd0ade214f798ccdf3",
"QOJDoudizhuModel/storage.py": "c9621015b6dd6c35418ee3fd31a73ea71a545ef51320ec811ccc9e053b943417",
"QOJDoudizhuModel/train.py": "4495d1fa4911cb6d92f39bd9c3691a16fc74fa7ef320986cac67acf923631239",
"qojbot/adapter.py": "64922460bf74a2d2413fe08ecfcc3539b1de5656d7295e154fe98946e48593e9",
"qojbot/curriculum.py": "b005956942bde93e683b9d37e444e6652801f0d6b691a8bbe631291379e88bfa",
"qojbot/douzero.py": "84495d88a2ac67a01a659db0b9b039f5b36d4d4fcd1849712ea9b0912a6c3864",
"qojbot/game.py": "6f8b77ec661dcbaaf2042df24e0453dd3263d7c9cc4df799595b746738ef5ff1",
"qojbot/history.py": "27eeeffd080c476b0b72245278495bda9ec9ed744fe6b85340f7f2742f8c6ea9",
"qojbot/history_training.py": "74ff26534168f4f98652cf6d43c268ab2b3211e3e27f04a9fad9e1a857759b61",
"qojbot/model_benchmark.py": "03de89831e54857840af92bdd47325a13cee48d73035fb45b98508f85e89e9e4",
"qojbot/policy.py": "cb406eb98f43517dbec106d8bc62f6edac00bd2b0c9dace00c85315d1f6008ec",
"qojbot/rules.py": "8fb9ee4ff9de21c58fd667a8e4c22a8c3db9b919e711029638f24340541e76f0"
}
},
"progress": {
"offline_epoch": 2,
"round": 128,
"updates": 28490,
"games": 1536,
"best_score": 0.1111111111111111,
"best_stage": "round-80",
"stage": "ppo"
},
"dataset": {
"fingerprint": "13e5709daa6e5e8d777c104cebd6e7ec75115874095319ee7d570c7a02277878",
"counts": {
"train": 180278,
"validation": 21260,
"test": 21165
},
"valid_games": 7266,
"rejected_games": 0
},
"metrics": [
{
"stage": "baseline",
"validation_score": 0,
"best_score": 0
},
{
"epoch": 1,
"offline_validation": {
"samples": 21260,
"loss": 0.6506123136262902
}
},
{
"stage": "offline-1",
"validation_score": -0.28703703703703703,
"best_score": 0
},
{
"epoch": 2,
"offline_validation": {
"samples": 21260,
"loss": 0.6081839078849818
}
},
{
"stage": "offline-2",
"validation_score": -0.3194444444444444,
"best_score": 0
},
{
"round": 1,
"ppo": {
"loss": 0.6096090006251489,
"policy": -6.413908414943244e-06,
"anchor_kl": 7.90482844772742e-09,
"value": 0.036875955224718135,
"belief": 0.18955422481221537,
"approx_kl": 7.245289061659126e-09,
"samples": 744,
"actions": 5118
},
"samples": 372,
"early_kl_stop": false,
"seconds": 13.777114041999084
},
{
"round": 2,
"ppo": {
"loss": 0.7068826687820529,
"policy": -1.512260192180904e-05,
"anchor_kl": 3.437007834763857e-08,
"value": 0.2472581016332421,
"belief": 0.18990855674944762,
"approx_kl": 1.4541383748305014e-08,
"samples": 818,
"actions": 5844
},
"samples": 409,
"early_kl_stop": false,
"seconds": 14.79054187499969
},
{
"round": 3,
"ppo": {
"loss": 0.5746077012246961,
"policy": -2.8213972498656836e-06,
"anchor_kl": 6.720765208616969e-08,
"value": 0.05978727498489201,
"belief": 0.16814449467476766,
"approx_kl": 6.611203881585674e-09,
"samples": 628,
"actions": 4364
},
"samples": 314,
"early_kl_stop": false,
"seconds": 10.995700875000693
},
{
"round": 4,
"ppo": {
"loss": 0.7064850827625819,
"policy": -5.684750420706613e-06,
"anchor_kl": 7.034261264899604e-08,
"value": 0.29715588314192637,
"belief": 0.19041749055896487,
"approx_kl": 5.7688781256765815e-09,
"samples": 700,
"actions": 5268
},
"samples": 350,
"early_kl_stop": false,
"seconds": 11.179604124999969
},
{
"round": 5,
"ppo": {
"loss": 0.4973030815994929,
"policy": -3.669824865129259e-06,
"anchor_kl": 3.650177238985937e-08,
"value": 0.04242017335361904,
"belief": 0.17779199672596796,
"approx_kl": 6.338906666941613e-09,
"samples": 630,
"actions": 5460
},
"samples": 315,
"early_kl_stop": false,
"seconds": 12.243349374999525
},
{
"round": 6,
"ppo": {
"loss": 0.7072747652868615,
"policy": -1.0951750003660208e-05,
"anchor_kl": 8.232433277687201e-08,
"value": 0.3120166570306509,
"belief": 0.21283678541023784,
"approx_kl": 8.379001855107483e-09,
"samples": 642,
"actions": 5184
},
"samples": 321,
"early_kl_stop": false,
"seconds": 10.581966499999908
},
{
"round": 7,
"ppo": {
"loss": 0.5007647565643853,
"policy": -3.92063160429798e-06,
"anchor_kl": 4.188005594193752e-08,
"value": 0.0452385424944823,
"belief": 0.17448846047784522,
"approx_kl": 4.451894905009661e-09,
"samples": 646,
"actions": 4872
},
"samples": 323,
"early_kl_stop": false,
"seconds": 11.247399666999627
},
{
"round": 8,
"ppo": {
"loss": 0.7336269403281419,
"policy": -1.206394770871038e-05,
"anchor_kl": 7.404081089297822e-08,
"value": 0.3467860811430475,
"belief": 0.20613388328448587,
"approx_kl": 7.673288169114487e-09,
"samples": 736,
"actions": 5334
},
"samples": 368,
"early_kl_stop": false,
"seconds": 11.015911458000119
},
{
"round": 9,
"ppo": {
"loss": 0.47186622778071635,
"policy": -1.0547758657721024e-06,
"anchor_kl": 4.439529380404736e-08,
"value": 0.03989364779731141,
"belief": 0.16794568262522733,
"approx_kl": 3.5013013220095257e-09,
"samples": 632,
"actions": 4436
},
"samples": 316,
"early_kl_stop": false,
"seconds": 10.26273824999953
},
{
"round": 10,
"ppo": {
"loss": 0.6806026247569493,
"policy": -4.7181120940617155e-06,
"anchor_kl": 6.471287774851524e-08,
"value": 0.2556270485717271,
"belief": 0.2068670101250921,
"approx_kl": 6.162694490592051e-09,
"samples": 700,
"actions": 5124
},
"samples": 350,
"early_kl_stop": false,
"seconds": 10.902530540999578
},
{
"round": 11,
"ppo": {
"loss": 0.5056302774915457,
"policy": -3.7598555964679773e-06,
"anchor_kl": 7.633922281995053e-08,
"value": 0.1115345879092107,
"belief": 0.17971833724746464,
"approx_kl": 4.475017943949399e-09,
"samples": 718,
"actions": 4840
},
"samples": 359,
"early_kl_stop": false,
"seconds": 11.55123149999963
},
{
"round": 12,
"ppo": {
"loss": 0.6970414358174081,
"policy": -8.141695279774703e-06,
"anchor_kl": 6.057163387252657e-08,
"value": 0.32165614548787047,
"belief": 0.1738200538896826,
"approx_kl": 7.2696938017342395e-09,
"samples": 762,
"actions": 5020
},
"samples": 381,
"early_kl_stop": false,
"seconds": 12.158407832999728
},
{
"round": 13,
"ppo": {
"loss": 0.4656745560717683,
"policy": -7.082642961283024e-06,
"anchor_kl": 9.857341799706634e-08,
"value": 0.08241950598589722,
"belief": 0.19632076463696002,
"approx_kl": 4.920096290079157e-09,
"samples": 714,
"actions": 5332
},
"samples": 357,
"early_kl_stop": false,
"seconds": 11.450703499998781
},
{
"round": 14,
"ppo": {
"loss": 0.7353519941564133,
"policy": -1.5121337782021518e-05,
"anchor_kl": 1.5578324776153064e-07,
"value": 0.35178173452721795,
"belief": 0.2095364790635177,
"approx_kl": 8.72990627908808e-09,
"samples": 562,
"actions": 4248
},
"samples": 281,
"early_kl_stop": false,
"seconds": 8.611376917000598
},
{
"round": 15,
"ppo": {
"loss": 0.4514082455951156,
"policy": -1.05175105008212e-06,
"anchor_kl": 1.5782193353667633e-07,
"value": 0.06449757902459664,
"belief": 0.1848393997911251,
"approx_kl": 5.644379242785259e-09,
"samples": 660,
"actions": 4744
},
"samples": 330,
"early_kl_stop": false,
"seconds": 10.69090587500068
},
{
"round": 16,
"ppo": {
"loss": 0.7103247732827158,
"policy": -1.015182998445299e-05,
"anchor_kl": 1.5786771848467533e-07,
"value": 0.38205278025131034,
"belief": 0.1923854390178064,
"approx_kl": 7.3941368050836085e-09,
"samples": 792,
"actions": 5176
},
"samples": 396,
"early_kl_stop": false,
"seconds": 11.948139209000146
},
{
"stage": "round-16",
"validation_score": 0,
"best_score": 0
},
{
"round": 17,
"ppo": {
"loss": 0.4145474828098115,
"policy": -1.8430613605669757e-06,
"anchor_kl": 2.0271443627368867e-07,
"value": 0.03184943552540001,
"belief": 0.16915904803399268,
"approx_kl": 5.0936581036232035e-09,
"samples": 716,
"actions": 4584
},
"samples": 358,
"early_kl_stop": false,
"seconds": 12.558681834001618
},
{
"round": 18,
"ppo": {
"loss": 0.475287568412329,
"policy": -9.91148372460445e-06,
"anchor_kl": 2.0761337970298493e-07,
"value": 0.10663812479505967,
"belief": 0.18374744341377133,
"approx_kl": 6.666308957892151e-09,
"samples": 646,
"actions": 4170
},
"samples": 323,
"early_kl_stop": false,
"seconds": 11.107545708000544
},
{
"round": 19,
"ppo": {
"loss": 0.41852791612053214,
"policy": -7.75610781057761e-06,
"anchor_kl": 1.6542085957021186e-07,
"value": 0.0493800065089618,
"belief": 0.17066317468240697,
"approx_kl": 7.737654942103432e-09,
"samples": 558,
"actions": 3590
},
"samples": 279,
"early_kl_stop": false,
"seconds": 10.488347459000579
},
{
"round": 20,
"ppo": {
"loss": 0.6844861578906891,
"policy": -1.1180909727350136e-05,
"anchor_kl": 1.5635325618926525e-07,
"value": 0.3954741277791172,
"belief": 0.15828846649110662,
"approx_kl": 8.139651619387678e-09,
"samples": 692,
"actions": 4942
},
"samples": 346,
"early_kl_stop": false,
"seconds": 11.797273582998969
},
{
"round": 21,
"ppo": {
"loss": 0.4500025973363888,
"policy": -2.5963673562359957e-06,
"anchor_kl": 1.929114569519212e-07,
"value": 0.05820000217370461,
"belief": 0.18219689650038268,
"approx_kl": 5.239403683915979e-09,
"samples": 652,
"actions": 4524
},
"samples": 326,
"early_kl_stop": false,
"seconds": 11.49547100000018
},
{
"round": 22,
"ppo": {
"loss": 0.7681089490652084,
"policy": -1.0709033200615331e-05,
"anchor_kl": 2.1916747303557235e-07,
"value": 0.4591146346769835,
"belief": 0.19132567001016518,
"approx_kl": 7.015554920623177e-09,
"samples": 608,
"actions": 4888
},
"samples": 304,
"early_kl_stop": false,
"seconds": 10.822109708000426
},
{
"round": 23,
"ppo": {
"loss": 0.40222113164695533,
"policy": -9.560101741069073e-07,
"anchor_kl": 2.916251102416592e-07,
"value": 0.027196657921917534,
"belief": 0.18415737444081823,
"approx_kl": 5.326158291584737e-09,
"samples": 740,
"actions": 4644
},
"samples": 370,
"early_kl_stop": false,
"seconds": 13.510235124998871
},
{
"round": 24,
"ppo": {
"loss": 0.5657669985368197,
"policy": -1.536928790651327e-05,
"anchor_kl": 2.6901714229756296e-07,
"value": 0.2247238093036778,
"belief": 0.17170743196768246,
"approx_kl": 9.014550844693244e-09,
"samples": 686,
"actions": 4640
},
"samples": 343,
"early_kl_stop": false,
"seconds": 11.803076375001183
},
{
"round": 25,
"ppo": {
"loss": 0.4207793133354735,
"policy": -4.408174547655829e-06,
"anchor_kl": 2.4625710394481767e-07,
"value": 0.06494351939951208,
"belief": 0.1688026307129312,
"approx_kl": 5.828794739335831e-09,
"samples": 696,
"actions": 4458
},
"samples": 348,
"early_kl_stop": false,
"seconds": 11.788431583998317
},
{
"round": 26,
"ppo": {
"loss": 0.6766984295597416,
"policy": -1.4638865383159513e-05,
"anchor_kl": 3.726572820931281e-07,
"value": 0.3843160573619719,
"belief": 0.18269463213995232,
"approx_kl": 1.492326974161301e-08,
"samples": 674,
"actions": 4224
},
"samples": 337,
"early_kl_stop": false,
"seconds": 11.810041875000024
},
{
"round": 27,
"ppo": {
"loss": 0.46670324161000876,
"policy": -2.955693913542706e-06,
"anchor_kl": 4.2403026682173417e-07,
"value": 0.04940832633039226,
"belief": 0.17209890905929648,
"approx_kl": 8.731149137020111e-09,
"samples": 736,
"actions": 5102
},
"samples": 368,
"early_kl_stop": false,
"seconds": 12.626642000001084
},
{
"round": 28,
"ppo": {
"loss": 0.6171821735531805,
"policy": -3.44719567922111e-06,
"anchor_kl": 4.5364405597746826e-07,
"value": 0.267722678198097,
"belief": 0.18771274772820865,
"approx_kl": 9.30756421344642e-09,
"samples": 658,
"actions": 4386
},
"samples": 329,
"early_kl_stop": false,
"seconds": 11.240961958999833
},
{
"round": 29,
"ppo": {
"loss": 0.36211739046367136,
"policy": 2.434882983135113e-06,
"anchor_kl": 4.0252915808141216e-07,
"value": 0.020389392292444786,
"belief": 0.16770132419865902,
"approx_kl": 7.841687193434968e-09,
"samples": 762,
"actions": 5656
},
"samples": 381,
"early_kl_stop": false,
"seconds": 14.572274166999705
},
{
"round": 30,
"ppo": {
"loss": 0.5314487383720723,
"policy": -1.0197149946334514e-05,
"anchor_kl": 5.324699265337528e-07,
"value": 0.17108771902449588,
"belief": 0.16798705655209561,
"approx_kl": 6.742180978998225e-09,
"samples": 752,
"actions": 5318
},
"samples": 376,
"early_kl_stop": false,
"seconds": 13.822133792000386
},
{
"round": 31,
"ppo": {
"loss": 0.35934976078801273,
"policy": -9.589498827916494e-07,
"anchor_kl": 4.0940930091790685e-07,
"value": 0.02441366005512475,
"belief": 0.177646416499748,
"approx_kl": 4.144674687652114e-09,
"samples": 644,
"actions": 4160
},
"samples": 322,
"early_kl_stop": false,
"seconds": 11.23210808299882
},
{
"round": 32,
"ppo": {
"loss": 0.6082724892887575,
"policy": -1.6579577397920724e-05,
"anchor_kl": 6.348831176268654e-07,
"value": 0.30482786761433756,
"belief": 0.1782855314822665,
"approx_kl": 2.5400155706850658e-08,
"samples": 754,
"actions": 4534
},
"samples": 377,
"early_kl_stop": false,
"seconds": 12.344354167000347
},
{
"stage": "round-32",
"validation_score": 0,
"best_score": 0
},
{
"round": 33,
"ppo": {
"loss": 0.4110652687559498,
"policy": -5.785984675970104e-06,
"anchor_kl": 4.265400542021591e-07,
"value": 0.0410347248523024,
"belief": 0.18865951113506038,
"approx_kl": 8.761139785335143e-09,
"samples": 722,
"actions": 4404
},
"samples": 361,
"early_kl_stop": false,
"seconds": 12.533811249999417
},
{
"round": 34,
"ppo": {
"loss": 0.5980861756527042,
"policy": -1.8012712630864136e-05,
"anchor_kl": 7.641329581966435e-07,
"value": 0.21156757999396186,
"belief": 0.16195918424136438,
"approx_kl": 1.0296137615640636e-08,
"samples": 686,
"actions": 4290
},
"samples": 343,
"early_kl_stop": false,
"seconds": 11.388618041999507
},
{
"round": 35,
"ppo": {
"loss": 0.3813615984116976,
"policy": -3.079793249551779e-06,
"anchor_kl": 4.7050650085333943e-07,
"value": 0.039199434930230946,
"belief": 0.15614726436455795,
"approx_kl": 1.1257617083052169e-08,
"samples": 638,
"actions": 4064
},
"samples": 319,
"early_kl_stop": false,
"seconds": 12.395912207999572
},
{
"round": 36,
"ppo": {
"loss": 0.44144202081056744,
"policy": -3.308485067643962e-06,
"anchor_kl": 8.077923327361562e-07,
"value": 0.10417472649324726,
"belief": 0.16949331329746561,
"approx_kl": 1.2113334058405279e-08,
"samples": 910,
"actions": 5560
},
"samples": 455,
"early_kl_stop": false,
"seconds": 14.918387833000452
},
{
"round": 37,
"ppo": {
"loss": 0.38974281125947047,
"policy": -7.097070154390837e-06,
"anchor_kl": 8.048940145499728e-07,
"value": 0.042678142995818666,
"belief": 0.19153615244125066,
"approx_kl": 1.2557536951805415e-08,
"samples": 608,
"actions": 4494
},
"samples": 304,
"early_kl_stop": false,
"seconds": 10.627120999999534
},
{
"round": 38,
"ppo": {
"loss": 0.48714203984811694,
"policy": -2.7481455978523233e-05,
"anchor_kl": 1.2120043689724903e-06,
"value": 0.1461424847731426,
"belief": 0.18469569730775554,
"approx_kl": 3.277833333434194e-08,
"samples": 706,
"actions": 5004
},
"samples": 353,
"early_kl_stop": false,
"seconds": 12.43266337499881
},
{
"round": 39,
"ppo": {
"loss": 0.38307077174564613,
"policy": -4.572928259053181e-06,
"anchor_kl": 7.081261157471784e-07,
"value": 0.04606193087073331,
"belief": 0.16849355514823777,
"approx_kl": 1.1204674686353231e-08,
"samples": 776,
"actions": 4846
},
"samples": 388,
"early_kl_stop": false,
"seconds": 13.449488665999525
},
{
"round": 40,
"ppo": {
"loss": 0.5262456282421395,
"policy": 4.147489865620931e-07,
"anchor_kl": 1.3560064040957289e-06,
"value": 0.21186314192083147,
"belief": 0.160752795268724,
"approx_kl": 2.338654465145535e-08,
"samples": 648,
"actions": 3852
},
"samples": 324,
"early_kl_stop": false,
"seconds": 11.356789458999629
},
{
"round": 41,
"ppo": {
"loss": 0.3233040434330009,
"policy": -6.4942639234454135e-06,
"anchor_kl": 1.2638358138839215e-06,
"value": 0.022661873919152937,
"belief": 0.18542867309825364,
"approx_kl": 1.5236870493999747e-08,
"samples": 688,
"actions": 4738
},
"samples": 344,
"early_kl_stop": false,
"seconds": 12.288893042001291
},
{
"round": 42,
"ppo": {
"loss": 0.5624391911826896,
"policy": -1.6695715090226845e-05,
"anchor_kl": 2.080275386407594e-06,
"value": 0.25843902410575625,
"belief": 0.1745237613306243,
"approx_kl": 2.4556055576843624e-08,
"samples": 676,
"actions": 4548
},
"samples": 338,
"early_kl_stop": false,
"seconds": 12.878628624999692
},
{
"round": 43,
"ppo": {
"loss": 0.3365613258815742,
"policy": -1.0817370436213039e-05,
"anchor_kl": 1.1168936586349523e-06,
"value": 0.030387414862645103,
"belief": 0.1661001393297413,
"approx_kl": 1.9213772024191062e-08,
"samples": 666,
"actions": 4408
},
"samples": 333,
"early_kl_stop": false,
"seconds": 10.541936749999877
},
{
"round": 44,
"ppo": {
"loss": 0.5321836865047891,
"policy": -3.764137086097909e-05,
"anchor_kl": 2.9577220010598015e-06,
"value": 0.2598895319740088,
"belief": 0.18796809536500894,
"approx_kl": 5.183238176888054e-08,
"samples": 718,
"actions": 5266
},
"samples": 359,
"early_kl_stop": false,
"seconds": 11.89054808400033
},
{
"round": 45,
"ppo": {
"loss": 0.3169401465470974,
"policy": -5.40049746632576e-06,
"anchor_kl": 1.6661800887399854e-06,
"value": 0.02257745574013545,
"belief": 0.1751642852792373,
"approx_kl": 2.1973481556012933e-08,
"samples": 650,
"actions": 4512
},
"samples": 325,
"early_kl_stop": false,
"seconds": 12.425301500001297
},
{
"round": 46,
"ppo": {
"loss": 0.4726611711894688,
"policy": 2.0256670920745186e-06,
"anchor_kl": 4.657239732414777e-06,
"value": 0.15922424181000047,
"belief": 0.1575612850163294,
"approx_kl": 1.3864052522441614e-07,
"samples": 736,
"actions": 4918
},
"samples": 368,
"early_kl_stop": false,
"seconds": 12.673876999999266
},
{
"round": 47,
"ppo": {
"loss": 0.34339073732495307,
"policy": -3.315955400466919e-06,
"anchor_kl": 3.5903056220831787e-06,
"value": 0.03340241340920329,
"belief": 0.18684057295322418,
"approx_kl": 2.6277266442775726e-08,
"samples": 800,
"actions": 5604
},
"samples": 400,
"early_kl_stop": false,
"seconds": 13.744887875000131
},
{
"round": 48,
"ppo": {
"loss": 0.4472398462906555,
"policy": -7.632019082505499e-05,
"anchor_kl": 4.938072775921363e-06,
"value": 0.17398852723926755,
"belief": 0.16885405775140877,
"approx_kl": 3.4820700282285434e-07,
"samples": 796,
"actions": 6114
},
"samples": 398,
"early_kl_stop": false,
"seconds": 12.834215458000472
},
{
"stage": "round-48",
"validation_score": 0,
"best_score": 0
},
{
"round": 49,
"ppo": {
"loss": 0.34123810755511735,
"policy": -6.886256253061119e-06,
"anchor_kl": 2.413566996083709e-06,
"value": 0.04466785203116986,
"belief": 0.18237228645868828,
"approx_kl": 9.25437530125577e-08,
"samples": 652,
"actions": 4650
},
"samples": 326,
"early_kl_stop": false,
"seconds": 12.363830709000467
},
{
"round": 50,
"ppo": {
"loss": 0.5865871643827808,
"policy": -2.7519416424535935e-05,
"anchor_kl": 3.3522525434877495e-06,
"value": 0.3599334239238693,
"belief": 0.2010598269201094,
"approx_kl": 3.731298833457629e-08,
"samples": 620,
"actions": 5572
},
"samples": 310,
"early_kl_stop": false,
"seconds": 11.4006315000006
},
{
"round": 51,
"ppo": {
"loss": 0.2936433883403215,
"policy": -1.555992959161187e-05,
"anchor_kl": 2.6047153865869463e-06,
"value": 0.02284379205797752,
"belief": 0.16407825727048897,
"approx_kl": 4.880461919201588e-08,
"samples": 674,
"actions": 4114
},
"samples": 337,
"early_kl_stop": false,
"seconds": 11.227483000000575
},
{
"round": 52,
"ppo": {
"loss": 0.580580355300278,
"policy": -5.579209718547884e-05,
"anchor_kl": 7.015099707460336e-06,
"value": 0.3388769540263981,
"belief": 0.1964623341306311,
"approx_kl": 2.048298960826436e-07,
"samples": 610,
"actions": 5118
},
"samples": 305,
"early_kl_stop": false,
"seconds": 11.562775374999546
},
{
"round": 53,
"ppo": {
"loss": 0.34909863736240837,
"policy": -1.6654051883875976e-05,
"anchor_kl": 5.647575844651802e-06,
"value": 0.04144460778705209,
"belief": 0.16607045979179136,
"approx_kl": 4.035567773286175e-07,
"samples": 684,
"actions": 4734
},
"samples": 342,
"early_kl_stop": false,
"seconds": 12.04103487499924
},
{
"round": 54,
"ppo": {
"loss": 0.4801193990825135,
"policy": 1.897221362149274e-05,
"anchor_kl": 1.0426627127737537e-05,
"value": 0.2515830339656936,
"belief": 0.16157769485388274,
"approx_kl": 1.6608011023497876e-07,
"samples": 648,
"actions": 4188
},
"samples": 324,
"early_kl_stop": false,
"seconds": 10.955311959000028
},
{
"round": 55,
"ppo": {
"loss": 0.33880770859840026,
"policy": -8.983532678706484e-07,
"anchor_kl": 4.6613218676073645e-06,
"value": 0.041650012896167043,
"belief": 0.17418281858816198,
"approx_kl": 1.6539996372405873e-07,
"samples": 764,
"actions": 5036
},
"samples": 382,
"early_kl_stop": false,
"seconds": 12.291360667000845
},
{
"round": 56,
"ppo": {
"loss": 0.3577471317783479,
"policy": -6.787971623482243e-05,
"anchor_kl": 1.0546501102076965e-05,
"value": 0.11469673012853951,
"belief": 0.1837316474286459,
"approx_kl": 3.301200547045277e-07,
"samples": 744,
"actions": 4696
},
"samples": 372,
"early_kl_stop": false,
"seconds": 13.374362332999226
},
{
"round": 57,
"ppo": {
"loss": 0.24586398757057762,
"policy": -1.5991421854072582e-05,
"anchor_kl": 3.586896036039101e-06,
"value": 0.01050807167826821,
"belief": 0.18081036714908202,
"approx_kl": 5.333933764666882e-07,
"samples": 732,
"actions": 4348
},
"samples": 366,
"early_kl_stop": false,
"seconds": 12.680850708000435
},
{
"round": 58,
"ppo": {
"loss": 0.5195657474769129,
"policy": -0.00013414927431055018,
"anchor_kl": 1.6918938913354274e-05,
"value": 0.2318935191510497,
"belief": 0.18951642746860917,
"approx_kl": 1.296831445919501e-06,
"samples": 740,
"actions": 5372
},
"samples": 370,
"early_kl_stop": false,
"seconds": 12.74253866700019
},
{
"round": 59,
"ppo": {
"loss": 0.2837424045228588,
"policy": 2.0169824148419366e-06,
"anchor_kl": 4.597281753271317e-06,
"value": 0.028238169037623592,
"belief": 0.15750182316880806,
"approx_kl": 7.934245972295516e-08,
"samples": 838,
"actions": 5790
},
"samples": 419,
"early_kl_stop": false,
"seconds": 14.392572541000845
},
{
"round": 60,
"ppo": {
"loss": 0.5961791160977334,
"policy": -2.185398960683126e-05,
"anchor_kl": 2.4300583297745944e-05,
"value": 0.3769649881043125,
"belief": 0.18047486220203568,
"approx_kl": 4.113815499006336e-07,
"samples": 586,
"actions": 4832
},
"samples": 293,
"early_kl_stop": false,
"seconds": 10.175231332999829
},
{
"round": 61,
"ppo": {
"loss": 0.35493865515202727,
"policy": 1.047273625188799e-05,
"anchor_kl": 8.441716765935156e-06,
"value": 0.051015214916707866,
"belief": 0.17028672463858305,
"approx_kl": 2.6443549216992324e-07,
"samples": 670,
"actions": 4726
},
"samples": 335,
"early_kl_stop": false,
"seconds": 11.279638666999745
},
{
"round": 62,
"ppo": {
"loss": 0.438005578788844,
"policy": -8.143599532524088e-05,
"anchor_kl": 2.7439442205969968e-05,
"value": 0.21756295531413444,
"belief": 0.169233654053579,
"approx_kl": 3.693887023452446e-07,
"samples": 726,
"actions": 4814
},
"samples": 363,
"early_kl_stop": false,
"seconds": 12.171465166999042
},
{
"round": 63,
"ppo": {
"loss": 0.3954891061283046,
"policy": -1.5345419415776034e-07,
"anchor_kl": 1.1120399401265758e-05,
"value": 0.06797796497063607,
"belief": 0.186421021680284,
"approx_kl": 2.721949301151015e-07,
"samples": 644,
"actions": 5850
},
"samples": 322,
"early_kl_stop": false,
"seconds": 12.266677999999956
},
{
"round": 64,
"ppo": {
"loss": 0.3904332318328152,
"policy": -5.4430927852210875e-05,
"anchor_kl": 3.2671306903422604e-05,
"value": 0.20021946596819606,
"belief": 0.17855914469965067,
"approx_kl": 1.9783823234371294e-07,
"samples": 618,
"actions": 5246
},
"samples": 309,
"early_kl_stop": false,
"seconds": 10.581776041999547
},
{
"stage": "round-64",
"validation_score": 0.0,
"best_score": 0
},
{
"round": 65,
"ppo": {
"loss": 0.34275312338556563,
"policy": -5.304142832756043e-05,
"anchor_kl": 1.3476908173158466e-05,
"value": 0.038649801830095906,
"belief": 0.1711451340998922,
"approx_kl": 4.834761576993124e-07,
"samples": 560,
"actions": 4626
},
"samples": 280,
"early_kl_stop": false,
"seconds": 9.933415999999852
},
{
"round": 66,
"ppo": {
"loss": 0.5831509653501715,
"policy": -0.00013008055511427803,
"anchor_kl": 6.008888392527319e-05,
"value": 0.3692891448736191,
"belief": 0.17865905247948652,
"approx_kl": 8.78031496987021e-07,
"samples": 652,
"actions": 4690
},
"samples": 326,
"early_kl_stop": false,
"seconds": 11.50466249999954
},
{
"round": 67,
"ppo": {
"loss": 0.3743048288719091,
"policy": -3.231311701748469e-05,
"anchor_kl": 2.3590948455408852e-05,
"value": 0.06629837212497242,
"belief": 0.16117030785304226,
"approx_kl": 2.6484007296496876e-07,
"samples": 584,
"actions": 4476
},
"samples": 292,
"early_kl_stop": false,
"seconds": 10.336333833998651
},
{
"round": 68,
"ppo": {
"loss": 0.4869099936177654,
"policy": -3.496543053657778e-05,
"anchor_kl": 8.470489979851165e-05,
"value": 0.2953557124060969,
"belief": 0.17782552477813537,
"approx_kl": 5.657694513027147e-07,
"samples": 620,
"actions": 4694
},
"samples": 310,
"early_kl_stop": false,
"seconds": 11.88318179199996
},
{
"round": 69,
"ppo": {
"loss": 0.2945056479735516,
"policy": 7.04672549050452e-06,
"anchor_kl": 1.0583656817869702e-05,
"value": 0.020521677794522353,
"belief": 0.1674058177882449,
"approx_kl": 2.8727761899043286e-06,
"samples": 742,
"actions": 4958
},
"samples": 371,
"early_kl_stop": false,
"seconds": 12.252957582999443
},
{
"round": 70,
"ppo": {
"loss": 0.3915925869345665,
"policy": -6.831899285316468e-05,
"anchor_kl": 0.00011334842345604556,
"value": 0.23209055479615925,
"belief": 0.16792187541723252,
"approx_kl": 1.2517254799604416e-06,
"samples": 800,
"actions": 6076
},
"samples": 400,
"early_kl_stop": false,
"seconds": 12.923589957999866
},
{
"round": 71,
"ppo": {
"loss": 0.3826256521009459,
"policy": -6.071613927192876e-05,
"anchor_kl": 1.2787871376894158e-05,
"value": 0.05962082007762246,
"belief": 0.16569291517652315,
"approx_kl": 1.0422417180334691e-06,
"samples": 812,
"actions": 5466
},
"samples": 406,
"early_kl_stop": false,
"seconds": 13.955563375000565
},
{
"round": 72,
"ppo": {
"loss": 0.4265236663686879,
"policy": 1.6686670920428107e-05,
"anchor_kl": 0.00017388023524664972,
"value": 0.22102363083292456,
"belief": 0.17785958805504967,
"approx_kl": 7.53050998729818e-07,
"samples": 680,
"actions": 4276
},
"samples": 340,
"early_kl_stop": false,
"seconds": 12.55478941599904
},
{
"round": 73,
"ppo": {
"loss": 0.378459846830938,
"policy": -3.6385066183914703e-06,
"anchor_kl": 9.211648486156213e-06,
"value": 0.05419353914792201,
"belief": 0.17937817988523924,
"approx_kl": 3.868365192262496e-07,
"samples": 502,
"actions": 3632
},
"samples": 251,
"early_kl_stop": false,
"seconds": 8.862942750000002
},
{
"round": 74,
"ppo": {
"loss": 0.3485002202598814,
"policy": -2.7795776757657978e-05,
"anchor_kl": 0.0001766743325822775,
"value": 0.14963258749404243,
"belief": 0.16381675046246957,
"approx_kl": 7.00424952290395e-07,
"samples": 794,
"actions": 6510
},
"samples": 397,
"early_kl_stop": false,
"seconds": 13.56521666700064
},
{
"round": 75,
"ppo": {
"loss": 0.35814486965862896,
"policy": 1.7066825583099276e-06,
"anchor_kl": 2.2214967613336387e-05,
"value": 0.05959861931382915,
"belief": 0.1695217853493489,
"approx_kl": 3.7455458860546227e-07,
"samples": 686,
"actions": 4792
},
"samples": 343,
"early_kl_stop": false,
"seconds": 12.255302000001393
},
{
"round": 76,
"ppo": {
"loss": 0.3954085357160271,
"policy": -0.00011970708469320389,
"anchor_kl": 0.00016261522593509029,
"value": 0.20011233289319566,
"belief": 0.15590915259395405,
"approx_kl": 2.9612207832567685e-06,
"samples": 866,
"actions": 5276
},
"samples": 433,
"early_kl_stop": false,
"seconds": 14.003025374999197
},
{
"round": 77,
"ppo": {
"loss": 0.27609392059045834,
"policy": -3.7646412358179197e-06,
"anchor_kl": 4.027771148765089e-05,
"value": 0.02651905828247686,
"belief": 0.16065571547209562,
"approx_kl": 1.1882550293169086e-07,
"samples": 728,
"actions": 5014
},
"samples": 364,
"early_kl_stop": false,
"seconds": 11.347284749999744
},
{
"round": 78,
"ppo": {
"loss": 0.340189452631404,
"policy": -0.00014670902680338852,
"anchor_kl": 0.00025312614806128584,
"value": 0.13193262727135688,
"belief": 0.18354489268927737,
"approx_kl": 3.6847011763673036e-06,
"samples": 754,
"actions": 5164
},
"samples": 377,
"early_kl_stop": false,
"seconds": 12.144811541000308
},
{
"round": 79,
"ppo": {
"loss": 0.3679338090880188,
"policy": -0.0001308871154299753,
"anchor_kl": 6.459625239836394e-05,
"value": 0.04989206793154785,
"belief": 0.1647585396192031,
"approx_kl": 3.193262507441736e-06,
"samples": 668,
"actions": 5042
},
"samples": 334,
"early_kl_stop": false,
"seconds": 11.908317416999125
},
{
"round": 80,
"ppo": {
"loss": 0.43565517090834105,
"policy": 8.47511358194418e-06,
"anchor_kl": 0.00038359645615499533,
"value": 0.2243327272313458,
"belief": 0.1883853268060651,
"approx_kl": 1.1166785629140125e-06,
"samples": 572,
"actions": 4552
},
"samples": 286,
"early_kl_stop": false,
"seconds": 10.777723042001526
},
{
"stage": "round-80",
"validation_score": 0.1111111111111111,
"best_score": 0.1111111111111111
},
{
"round": 81,
"ppo": {
"loss": 0.3356556016438288,
"policy": -0.00014610612825642253,
"anchor_kl": 9.667154312806898e-05,
"value": 0.03682451994907587,
"belief": 0.1720303328226981,
"approx_kl": 8.358967915385557e-06,
"samples": 798,
"actions": 5124
},
"samples": 399,
"early_kl_stop": false,
"seconds": 12.609279540998614
},
{
"round": 82,
"ppo": {
"loss": 0.37927412499523944,
"policy": -0.00019073366042866044,
"anchor_kl": 0.0003109703150756114,
"value": 0.16629281104098223,
"belief": 0.1526840714895108,
"approx_kl": 2.5615126549996665e-06,
"samples": 734,
"actions": 5324
},
"samples": 367,
"early_kl_stop": false,
"seconds": 13.529078916999424
},
{
"round": 83,
"ppo": {
"loss": 0.39411171260128264,
"policy": -4.0987853184107354e-05,
"anchor_kl": 5.48244669350456e-05,
"value": 0.06572314456974858,
"belief": 0.18081660284322895,
"approx_kl": 1.3552946898059708e-06,
"samples": 634,
"actions": 5480
},
"samples": 317,
"early_kl_stop": false,
"seconds": 11.482346208000308
},
{
"round": 84,
"ppo": {
"loss": 0.30880907928788814,
"policy": -3.0472555330821447e-05,
"anchor_kl": 0.00036147287037592476,
"value": 0.13682159392175253,
"belief": 0.17294827330035048,
"approx_kl": 5.547378163833123e-06,
"samples": 770,
"actions": 5432
},
"samples": 385,
"early_kl_stop": false,
"seconds": 13.711109957999724
},
{
"round": 85,
"ppo": {
"loss": 0.3387542598711168,
"policy": -2.438374807052218e-05,
"anchor_kl": 4.9871751852544115e-05,
"value": 0.04394774893130289,
"belief": 0.14167695080510098,
"approx_kl": 9.300880305538971e-07,
"samples": 774,
"actions": 4690
},
"samples": 387,
"early_kl_stop": false,
"seconds": 12.889784666000196
},
{
"round": 86,
"ppo": {
"loss": 0.41163976999938995,
"policy": -6.557237409967982e-05,
"anchor_kl": 0.0007077401097917974,
"value": 0.22551353830854778,
"belief": 0.17630811351911294,
"approx_kl": 2.454149078084422e-06,
"samples": 702,
"actions": 5126
},
"samples": 351,
"early_kl_stop": false,
"seconds": 12.396800458000143
},
{
"round": 87,
"ppo": {
"loss": 0.34400616867362327,
"policy": -9.595812444791854e-05,
"anchor_kl": 5.800044749942017e-05,
"value": 0.04073433218957306,
"belief": 0.16642953563223845,
"approx_kl": 2.02467770361529e-06,
"samples": 636,
"actions": 4174
},
"samples": 318,
"early_kl_stop": false,
"seconds": 10.97333562499989
},
{
"round": 88,
"ppo": {
"loss": 0.4092904344838571,
"policy": -0.00034284167350294756,
"anchor_kl": 0.0008234951424444802,
"value": 0.21973183241119998,
"belief": 0.16854444437007854,
"approx_kl": 1.827805218386677e-05,
"samples": 748,
"actions": 5264
},
"samples": 374,
"early_kl_stop": false,
"seconds": 11.60650270899896
},
{
"round": 89,
"ppo": {
"loss": 0.3253878716390796,
"policy": -0.00035819631719261134,
"anchor_kl": 0.00010547310511570207,
"value": 0.05541942094592235,
"belief": 0.1769923947364183,
"approx_kl": 6.815343376089582e-05,
"samples": 654,
"actions": 4046
},
"samples": 327,
"early_kl_stop": false,
"seconds": 10.91758533400025
},
{
"round": 90,
"ppo": {
"loss": 0.6327342875301838,
"policy": -2.1951505914330482e-05,
"anchor_kl": 0.0009915007950288175,
"value": 0.4912377721630037,
"belief": 0.15295682412882647,
"approx_kl": 1.9275816157460213e-05,
"samples": 768,
"actions": 5972
},
"samples": 384,
"early_kl_stop": false,
"seconds": 13.869618249998894
},
{
"round": 91,
"ppo": {
"loss": 0.3556676037149665,
"policy": -1.3154046035107272e-05,
"anchor_kl": 5.4806435447767115e-05,
"value": 0.05520120692197923,
"belief": 0.1537196492339358,
"approx_kl": 1.6389920573129441e-06,
"samples": 810,
"actions": 4898
},
"samples": 405,
"early_kl_stop": false,
"seconds": 12.123014708000483
},
{
"round": 92,
"ppo": {
"loss": 0.41995771950609806,
"policy": -0.0010509431283100847,
"anchor_kl": 0.0012472778811285937,
"value": 0.2322406245084277,
"belief": 0.17884088685088828,
"approx_kl": 4.941588625884973e-05,
"samples": 754,
"actions": 4848
},
"samples": 377,
"early_kl_stop": false,
"seconds": 12.457418958001654
},
{
"round": 93,
"ppo": {
"loss": 0.3193457246716343,
"policy": 3.072591985005941e-05,
"anchor_kl": 3.661856026698856e-05,
"value": 0.016679451939886184,
"belief": 0.1863095563516688,
"approx_kl": 3.9138275184738105e-06,
"samples": 536,
"actions": 4406
},
"samples": 268,
"early_kl_stop": false,
"seconds": 9.149040291000347
},
{
"round": 94,
"ppo": {
"loss": 0.4869650430653406,
"policy": 0.00034056599859310234,
"anchor_kl": 0.0017566097495032211,
"value": 0.2983889042035393,
"belief": 0.1654304706532022,
"approx_kl": 1.3045189679478822e-05,
"samples": 736,
"actions": 4904
},
"samples": 368,
"early_kl_stop": false,
"seconds": 12.340036457999304
},
{
"round": 95,
"ppo": {
"loss": 0.24433126356846616,
"policy": 1.7145260313126255e-05,
"anchor_kl": 4.128377478711484e-05,
"value": 0.01969073502926488,
"belief": 0.1729898624279708,
"approx_kl": 2.4888990800711047e-06,
"samples": 662,
"actions": 5166
},
"samples": 331,
"early_kl_stop": false,
"seconds": 11.468624625000302
},
{
"round": 96,
"ppo": {
"loss": 0.5214908684541053,
"policy": 0.0001427928024023509,
"anchor_kl": 0.0017794997025393808,
"value": 0.2633387984003315,
"belief": 0.15658825086402403,
"approx_kl": 8.424948846148314e-06,
"samples": 682,
"actions": 4586
},
"samples": 341,
"early_kl_stop": false,
"seconds": 11.182884333000402
},
{
"stage": "round-96",
"validation_score": -0.018518518518518517,
"best_score": 0.1111111111111111
},
{
"round": 97,
"ppo": {
"loss": 0.34531797412075577,
"policy": 2.8380275338510925e-05,
"anchor_kl": 7.351795537470946e-05,
"value": 0.051311631675196596,
"belief": 0.18933653690015215,
"approx_kl": 9.982628575580978e-07,
"samples": 632,
"actions": 4570
},
"samples": 316,
"early_kl_stop": false,
"seconds": 10.337659125001665
},
{
"round": 98,
"ppo": {
"loss": 0.4338683062568522,
"policy": -0.0004095885398064979,
"anchor_kl": 0.002013979783792972,
"value": 0.23872747772536979,
"belief": 0.18480563524477076,
"approx_kl": 1.6880685986195258e-05,
"samples": 694,
"actions": 4328
},
"samples": 347,
"early_kl_stop": false,
"seconds": 11.577839583000241
},
{
"round": 99,
"ppo": {
"loss": 0.3033090577053186,
"policy": -5.861807359976117e-05,
"anchor_kl": 5.2345305340885844e-05,
"value": 0.026015598049644713,
"belief": 0.1773267354542455,
"approx_kl": 6.567835575392434e-06,
"samples": 626,
"actions": 4330
},
"samples": 313,
"early_kl_stop": false,
"seconds": 11.12299879099919
},
{
"round": 100,
"ppo": {
"loss": 0.4080846024249096,
"policy": -0.0001961120537349156,
"anchor_kl": 0.0022023555281086137,
"value": 0.23520829079684263,
"belief": 0.1673012669689907,
"approx_kl": 4.606552838538745e-06,
"samples": 644,
"actions": 4940
},
"samples": 322,
"early_kl_stop": false,
"seconds": 12.195894792001127
},
{
"round": 101,
"ppo": {
"loss": 0.3402062629089981,
"policy": -0.0001016933409894099,
"anchor_kl": 3.564764289175863e-05,
"value": 0.043113647492938355,
"belief": 0.17361833870899482,
"approx_kl": 5.414996479378372e-06,
"samples": 610,
"actions": 4672
},
"samples": 305,
"early_kl_stop": false,
"seconds": 11.71478904200012
},
{
"round": 102,
"ppo": {
"loss": 0.4368515668910033,
"policy": 3.565076013901226e-05,
"anchor_kl": 0.0027091569881856296,
"value": 0.3164478076615901,
"belief": 0.162978403366072,
"approx_kl": 8.404278867302146e-06,
"samples": 622,
"actions": 4840
},
"samples": 311,
"early_kl_stop": false,
"seconds": 10.691602750001039
},
{
"round": 103,
"ppo": {
"loss": 0.3552584761004907,
"policy": -0.0001177035791929378,
"anchor_kl": 9.241847713615984e-05,
"value": 0.04873414401373396,
"belief": 0.17849230964318463,
"approx_kl": 3.1725828365359308e-06,
"samples": 602,
"actions": 4242
},
"samples": 301,
"early_kl_stop": false,
"seconds": 10.366369958999712
},
{
"round": 104,
"ppo": {
"loss": 0.4965393871685023,
"policy": 0.00012195463116624867,
"anchor_kl": 0.0024942560178947875,
"value": 0.3229594750074761,
"belief": 0.1760816677095662,
"approx_kl": 7.807546672461461e-06,
"samples": 774,
"actions": 5670
},
"samples": 387,
"early_kl_stop": false,
"seconds": 13.570977583000058
},
{
"round": 105,
"ppo": {
"loss": 0.37534060903725897,
"policy": -0.0002499394199991096,
"anchor_kl": 0.0001261404865597088,
"value": 0.05964503185544417,
"belief": 0.17562885976292456,
"approx_kl": 2.13830571886763e-05,
"samples": 734,
"actions": 5278
},
"samples": 367,
"early_kl_stop": false,
"seconds": 12.465651916998468
},
{
"round": 106,
"ppo": {
"loss": 0.4683390755368316,
"policy": -0.0003325201409018558,
"anchor_kl": 0.0026041505844541066,
"value": 0.299419431420772,
"belief": 0.1645106368738672,
"approx_kl": 1.3711791404563447e-05,
"samples": 736,
"actions": 4946
},
"samples": 368,
"early_kl_stop": false,
"seconds": 11.565004083000531
},
{
"round": 107,
"ppo": {
"loss": 0.34041170207918553,
"policy": -0.0005603602345007053,
"anchor_kl": 0.00011582454197037883,
"value": 0.03907139850915342,
"belief": 0.18404790547446928,
"approx_kl": 1.9217476880800425e-05,
"samples": 690,
"actions": 4750
},
"samples": 345,
"early_kl_stop": false,
"seconds": 10.711245207998218
},
{
"round": 108,
"ppo": {
"loss": 0.31643603928387165,
"policy": -0.0006388432473084177,
"anchor_kl": 0.0015548839724396257,
"value": 0.11581156027673835,
"belief": 0.14602519974042552,
"approx_kl": 2.4578999726096307e-05,
"samples": 852,
"actions": 4982
},
"samples": 426,
"early_kl_stop": false,
"seconds": 12.624377916999947
},
{
"round": 109,
"ppo": {
"loss": 0.41706447629376525,
"policy": 3.010743598598257e-05,
"anchor_kl": 0.00026375673407561065,
"value": 0.07326589667880505,
"belief": 0.17197070642357867,
"approx_kl": 2.519922858903072e-05,
"samples": 786,
"actions": 4854
},
"samples": 393,
"early_kl_stop": false,
"seconds": 12.301625165999212
},
{
"round": 110,
"ppo": {
"loss": 0.5399146613901888,
"policy": -0.0006182552177525131,
"anchor_kl": 0.0017811788464572622,
"value": 0.3689870618989129,
"belief": 0.15504824803293457,
"approx_kl": 7.262826516959795e-05,
"samples": 676,
"actions": 4302
},
"samples": 338,
"early_kl_stop": false,
"seconds": 11.317028084000412
},
{
"round": 111,
"ppo": {
"loss": 0.34010358206261465,
"policy": -3.0333756754391883e-06,
"anchor_kl": 0.0002369805050281578,
"value": 0.04271988483867671,
"belief": 0.1511777491689859,
"approx_kl": 9.781113121567703e-06,
"samples": 738,
"actions": 4732
},
"samples": 369,
"early_kl_stop": false,
"seconds": 12.83973129199876
},
{
"round": 112,
"ppo": {
"loss": 0.44609734215145946,
"policy": -0.0007560285725003126,
"anchor_kl": 0.0024683292296264017,
"value": 0.2831444618610379,
"belief": 0.18499336387016183,
"approx_kl": 5.407489821709675e-05,
"samples": 638,
"actions": 4658
},
"samples": 319,
"early_kl_stop": false,
"seconds": 10.197487417000957
},
{
"stage": "round-112",
"validation_score": -0.14814814814814814,
"best_score": 0.1111111111111111
},
{
"round": 113,
"ppo": {
"loss": 0.33866592713340166,
"policy": -0.0005075645909084838,
"anchor_kl": 0.0008595443668583707,
"value": 0.05133357316957286,
"belief": 0.17374140637155386,
"approx_kl": 3.1789591902505135e-05,
"samples": 722,
"actions": 4738
},
"samples": 361,
"early_kl_stop": false,
"seconds": 12.270911915999022
},
{
"round": 114,
"ppo": {
"loss": 0.5620451294529585,
"policy": -0.0006662419548741093,
"anchor_kl": 0.001864743273315217,
"value": 0.4355825890361527,
"belief": 0.1618800018856555,
"approx_kl": 5.39054795179838e-05,
"samples": 648,
"actions": 4568
},
"samples": 324,
"early_kl_stop": false,
"seconds": 11.61586399999942
},
{
"round": 115,
"ppo": {
"loss": 0.2769248927563009,
"policy": -0.00023774901850962302,
"anchor_kl": 0.0012342270958675136,
"value": 0.025878139129433205,
"belief": 0.15994935751381054,
"approx_kl": 7.808252014728438e-05,
"samples": 710,
"actions": 5114
},
"samples": 355,
"early_kl_stop": false,
"seconds": 12.580273624998881
},
{
"round": 116,
"ppo": {
"loss": 0.41718435603497406,
"policy": 0.0001657941703046306,
"anchor_kl": 0.0030641262683581845,
"value": 0.24900373749518662,
"belief": 0.1672750874684098,
"approx_kl": 2.524191565979063e-05,
"samples": 712,
"actions": 5110
},
"samples": 356,
"early_kl_stop": false,
"seconds": 12.06315154100048
},
{
"round": 117,
"ppo": {
"loss": 0.2638221539676508,
"policy": 2.9759790330009666e-05,
"anchor_kl": 0.0002550705964451364,
"value": 0.020250021590836307,
"belief": 0.1575875001396725,
"approx_kl": 1.0406030690758252e-05,
"samples": 748,
"actions": 4900
},
"samples": 374,
"early_kl_stop": false,
"seconds": 12.482599584000127
},
{
"round": 118,
"ppo": {
"loss": 0.36037773623188857,
"policy": -0.00026612541962912494,
"anchor_kl": 0.002822671970466291,
"value": 0.18728303943430027,
"belief": 0.16524556531579954,
"approx_kl": 8.529843926071415e-06,
"samples": 614,
"actions": 4100
},
"samples": 307,
"early_kl_stop": false,
"seconds": 11.482779834001121
},
{
"round": 119,
"ppo": {
"loss": 0.3194208438462107,
"policy": -0.00011915793464128727,
"anchor_kl": 0.00046892446554542546,
"value": 0.038718836425062846,
"belief": 0.1664550324822023,
"approx_kl": 1.2534303248973476e-05,
"samples": 606,
"actions": 4828
},
"samples": 303,
"early_kl_stop": false,
"seconds": 10.831750625000495
},
{
"round": 120,
"ppo": {
"loss": 0.45402732561192205,
"policy": -0.00014295010797439083,
"anchor_kl": 0.002388134316330956,
"value": 0.3051282127538035,
"belief": 0.20637029289237915,
"approx_kl": 6.103152108769263e-06,
"samples": 620,
"actions": 4844
},
"samples": 310,
"early_kl_stop": false,
"seconds": 11.298984999999448
},
{
"round": 121,
"ppo": {
"loss": 0.31969300885167384,
"policy": 3.362703137099743e-05,
"anchor_kl": 0.0006050472216480153,
"value": 0.03405880063979162,
"belief": 0.17631620251470143,
"approx_kl": 3.005769556491739e-05,
"samples": 576,
"actions": 4194
},
"samples": 288,
"early_kl_stop": false,
"seconds": 10.598979041998973
},
{
"round": 122,
"ppo": {
"loss": 0.3936293887893359,
"policy": 0.00010861063003540039,
"anchor_kl": 0.0023350342309568077,
"value": 0.24468207788467408,
"belief": 0.17327190375328064,
"approx_kl": 2.172292026565022e-05,
"samples": 750,
"actions": 4810
},
"samples": 375,
"early_kl_stop": false,
"seconds": 12.397454458001448
},
{
"round": 123,
"ppo": {
"loss": 0.40020292123158774,
"policy": 0.00012424548467000326,
"anchor_kl": 0.0008227153700621178,
"value": 0.0696490394944946,
"belief": 0.1861157548427582,
"approx_kl": 3.3805581369961146e-06,
"samples": 600,
"actions": 3804
},
"samples": 300,
"early_kl_stop": false,
"seconds": 10.550259790999917
},
{
"round": 124,
"ppo": {
"loss": 0.5404799008369445,
"policy": -0.0003094242622747141,
"anchor_kl": 0.002050886542805101,
"value": 0.3512840117075864,
"belief": 0.16589895853224923,
"approx_kl": 1.972065690150724e-05,
"samples": 850,
"actions": 5676
},
"samples": 425,
"early_kl_stop": false,
"seconds": 13.108403333000751
},
{
"round": 125,
"ppo": {
"loss": 0.3008581376477574,
"policy": -0.00012702067916312915,
"anchor_kl": 0.0006583722161450894,
"value": 0.03483712505758478,
"belief": 0.15185303529829122,
"approx_kl": 2.5193269751714855e-06,
"samples": 712,
"actions": 4690
},
"samples": 356,
"early_kl_stop": false,
"seconds": 12.364789666999059
},
{
"round": 126,
"ppo": {
"loss": 0.41094049983299696,
"policy": 0.0003777313308838086,
"anchor_kl": 0.002345845087611218,
"value": 0.2242793250709581,
"belief": 0.16237300525491055,
"approx_kl": 2.1257483137596194e-05,
"samples": 780,
"actions": 5522
},
"samples": 390,
"early_kl_stop": false,
"seconds": 12.932946124999944
},
{
"round": 127,
"ppo": {
"loss": 0.25967057983433534,
"policy": -8.965945485884768e-05,
"anchor_kl": 0.0002667132705148087,
"value": 0.022752964844322068,
"belief": 0.15495971662138924,
"approx_kl": 1.539345026999561e-05,
"samples": 788,
"actions": 5086
},
"samples": 394,
"early_kl_stop": false,
"seconds": 12.023093874999176
},
{
"round": 128,
"ppo": {
"loss": 0.3811246964226787,
"policy": -0.00010877350966135661,
"anchor_kl": 0.00215097206455539,
"value": 0.1816894531560441,
"belief": 0.15920284328361353,
"approx_kl": 2.46294269648691e-05,
"samples": 768,
"actions": 5194
},
"samples": 384,
"early_kl_stop": false,
"seconds": 12.689505292000831
},
{
"stage": "round-128",
"validation_score": -0.09259259259259259,
"best_score": 0.1111111111111111
}
],
"limitation": "holdout imitation loss is not playing strength; best selected on paired validation",
"test": {
"games": 576,
"selection_score": -0.10648148148148148,
"summary": {
"bidding": {
"games": 288,
"deals": 48,
"mean_bid": 2.1875,
"total_redeals": 12,
"lstm": {
"player_games": 432,
"mean_score": 0.12962962962962962,
"win_rate": 0.4722222222222222,
"mean_score_95pct_cluster_bootstrap": [
-0.08333333333333333,
0.3888888888888889
],
"by_role": {
"landlord": {
"player_games": 144,
"mean_score": 2.0277777777777777,
"win_rate": 0.6805555555555556
},
"landlord_down": {
"player_games": 144,
"mean_score": -0.8472222222222222,
"win_rate": 0.3611111111111111
},
"landlord_up": {
"player_games": 144,
"mean_score": -0.7916666666666666,
"win_rate": 0.375
}
},
"by_composition": {
"1_vs_2": {
"player_games": 144,
"mean_score": 0.19444444444444445,
"win_rate": 0.4861111111111111
},
"2_vs_1": {
"player_games": 288,
"mean_score": 0.09722222222222222,
"win_rate": 0.4652777777777778
}
},
"bid_counts": {
"0": 204,
"1": 45,
"2": 84,
"3": 54
},
"decision_seconds": {
"count": 4738,
"mean": 0.0024290908678962714,
"p50": 0.0017553750003571622,
"p95": 0.004693791997851804,
"p99": 0.007003999999142252,
"max": 0.03849849999824073
}
},
"resnet": {
"player_games": 432,
"mean_score": -0.12962962962962962,
"win_rate": 0.4305555555555556,
"mean_score_95pct_cluster_bootstrap": [
-0.3888888888888889,
0.08333333333333333
],
"by_role": {
"landlord": {
"player_games": 144,
"mean_score": 1.4722222222222223,
"win_rate": 0.6111111111111112
},
"landlord_down": {
"player_games": 144,
"mean_score": -0.9027777777777778,
"win_rate": 0.3472222222222222
},
"landlord_up": {
"player_games": 144,
"mean_score": -0.9583333333333334,
"win_rate": 0.3333333333333333
}
},
"by_composition": {
"1_vs_2": {
"player_games": 144,
"mean_score": -0.19444444444444445,
"win_rate": 0.4166666666666667
},
"2_vs_1": {
"player_games": 288,
"mean_score": -0.09722222222222222,
"win_rate": 0.4375
}
},
"bid_counts": {
"0": 204,
"1": 45,
"2": 84,
"3": 54
},
"decision_seconds": {
"count": 4752,
"mean": 0.014777978487191135,
"p50": 0.011539833998540416,
"p95": 0.024261999998998363,
"p99": 0.04513658299401868,
"max": 0.18392479199974332
}
},
"pure_team_matchups": {
"lstm_landlord": {
"games": 48,
"landlord_win_rate": 0.6875,
"landlord_mean_score": 2.0833333333333335
},
"resnet_landlord": {
"games": 48,
"landlord_win_rate": 0.6041666666666666,
"landlord_mean_score": 1.4166666666666667
}
}
},
"fixed": {
"games": 288,
"deals": 48,
"mean_bid": 1,
"total_redeals": 0,
"lstm": {
"player_games": 432,
"mean_score": 0.08333333333333333,
"win_rate": 0.5277777777777778,
"mean_score_95pct_cluster_bootstrap": [
-0.018518518518518517,
0.2037037037037037
],
"by_role": {
"landlord": {
"player_games": 144,
"mean_score": -0.125,
"win_rate": 0.5
},
"landlord_down": {
"player_games": 144,
"mean_score": 0.1875,
"win_rate": 0.5416666666666666
},
"landlord_up": {
"player_games": 144,
"mean_score": 0.1875,
"win_rate": 0.5416666666666666
}
},
"by_composition": {
"1_vs_2": {
"player_games": 144,
"mean_score": 0.125,
"win_rate": 0.5416666666666666
},
"2_vs_1": {
"player_games": 288,
"mean_score": 0.0625,
"win_rate": 0.5208333333333334
}
},
"bid_counts": {
"0": 0,
"1": 0,
"2": 0,
"3": 0
},
"decision_seconds": {
"count": 4631,
"mean": 0.00239102467029075,
"p50": 0.0017541249981150031,
"p95": 0.004621833999408409,
"p99": 0.006799167000281159,
"max": 0.03820970800006762
}
},
"resnet": {
"player_games": 432,
"mean_score": -0.08333333333333333,
"win_rate": 0.4930555555555556,
"mean_score_95pct_cluster_bootstrap": [
-0.2037037037037037,
0.018518518518518517
],
"by_role": {
"landlord": {
"player_games": 144,
"mean_score": -0.5,
"win_rate": 0.4375
},
"landlord_down": {
"player_games": 144,
"mean_score": 0.125,
"win_rate": 0.5208333333333334
},
"landlord_up": {
"player_games": 144,
"mean_score": 0.125,
"win_rate": 0.5208333333333334
}
},
"by_composition": {
"1_vs_2": {
"player_games": 144,
"mean_score": -0.125,
"win_rate": 0.4791666666666667
},
"2_vs_1": {
"player_games": 288,
"mean_score": -0.0625,
"win_rate": 0.5
}
},
"bid_counts": {
"0": 0,
"1": 0,
"2": 0,
"3": 0
},
"decision_seconds": {
"count": 4628,
"mean": 0.015213161093120063,
"p50": 0.01167416599491844,
"p95": 0.024690166996151675,
"p99": 0.08202295900264289,
"max": 0.18462649999855785
}
},
"pure_team_matchups": {
"lstm_landlord": {
"games": 48,
"landlord_win_rate": 0.5,
"landlord_mean_score": -0.125
},
"resnet_landlord": {
"games": 48,
"landlord_win_rate": 0.4375,
"landlord_mean_score": -0.5
}
}
}
}
},
"offline_test": {
"samples": 21165,
"loss": 0.7485360586471431
}
}
{
"status": "complete",
"device": "mps",
"config": {
"data": "/Users/xcx0902/Documents/Github/qojddz/data/qoj-history",
"output": "/Users/xcx0902/Documents/Github/qojddz/QOJDoudizhuModel/model",
"teacher_dir": "/Users/xcx0902/Documents/Github/qojddz/models/douzero-baselines/douzero_ADP",
"device": "mps",
"width": 128,
"warmup_games": 32,
"offline_epochs": 2,
"games_per_round": 12,
"ppo_epochs": 2,
"batch_states": 16,
"max_batch_actions": 4096,
"candidate_cap": 64,
"max_history_games": 0,
"holdout": 0.1,
"learning_rate": 0.0001,
"ppo_learning_rate": 2e-05,
"anchor_weight": 0.02,
"league_size": 4,
"validation_interval": 16,
"validation_deals": 12,
"test_deals": 48,
"seed": 20261002,
"validation_seed": 30261002,
"test_seed": 40261002,
"mode": "mixed",
"teacher_sha256": {
"landlord": "6f2971813495e9c509cbd4a101213c49587472942fa0f018d04c139a6a647c2d",
"landlord_up": "7fb7c495ff095db2ddadd8f9f8fd53cad840211578dabc83bbcc8d75085f843e",
"landlord_down": "c173dbe30258f19b6392c30ab1e330b6b3090bbd9ccdc89a9b18123a6ab6e2d9"
},
"source_sha256": {
"Douzero_Resnet/douzero/dmc/models.py": "b873ebcceb8907ccf5e1b75751bef640da944d5c9d6b100b2e5fb632150c18fa",
"Douzero_Resnet/douzero/env/env.py": "4ba02e8e181db4b84704ab205b94691618bca4d71d8e5ce03c61d2c060f4724a",
"QOJDoudizhuModel/__init__.py": "f9a452c634683d816ff3a33d679ec510b3a6fb9ebaa4f266d39181322895ea91",
"QOJDoudizhuModel/dataset.py": "43bd492ea3190917dafc46cfbaf12a505977976130ab101fcecafed34a4fab73",
"QOJDoudizhuModel/evaluate.py": "543441b8bb404da0f5c39842d2dd4e04b14875a6a7a052c93fe690b0489f5e72",
"QOJDoudizhuModel/features.py": "cc7d076b67b9a59fb8b1b5d4f3e23fa0fe9313e6ed7d2ccbbc763ebd99563dc6",
"QOJDoudizhuModel/learning.py": "2276f3c9af8bb7d1ddab6e85e33a9d1f4325426dc60f2d51dacd3b113cf92f5f",
"QOJDoudizhuModel/network.py": "14450e8d72f40afcd0a814bd609d5f0dab23579055c06bfaa239fb5ec4a154dc",
"QOJDoudizhuModel/policy.py": "275892378b80d64e2d469ced9fd27968eafaa699cf265ddd0ade214f798ccdf3",
"QOJDoudizhuModel/storage.py": "c9621015b6dd6c35418ee3fd31a73ea71a545ef51320ec811ccc9e053b943417",
"QOJDoudizhuModel/train.py": "4495d1fa4911cb6d92f39bd9c3691a16fc74fa7ef320986cac67acf923631239",
"qojbot/adapter.py": "64922460bf74a2d2413fe08ecfcc3539b1de5656d7295e154fe98946e48593e9",
"qojbot/curriculum.py": "b005956942bde93e683b9d37e444e6652801f0d6b691a8bbe631291379e88bfa",
"qojbot/douzero.py": "84495d88a2ac67a01a659db0b9b039f5b36d4d4fcd1849712ea9b0912a6c3864",
"qojbot/game.py": "6f8b77ec661dcbaaf2042df24e0453dd3263d7c9cc4df799595b746738ef5ff1",
"qojbot/history.py": "27eeeffd080c476b0b72245278495bda9ec9ed744fe6b85340f7f2742f8c6ea9",
"qojbot/history_training.py": "74ff26534168f4f98652cf6d43c268ab2b3211e3e27f04a9fad9e1a857759b61",
"qojbot/model_benchmark.py": "03de89831e54857840af92bdd47325a13cee48d73035fb45b98508f85e89e9e4",
"qojbot/policy.py": "cb406eb98f43517dbec106d8bc62f6edac00bd2b0c9dace00c85315d1f6008ec",
"qojbot/rules.py": "8fb9ee4ff9de21c58fd667a8e4c22a8c3db9b919e711029638f24340541e76f0"
}
},
"progress": {
"offline_epoch": 2,
"round": 128,
"updates": 28490,
"games": 1536,
"best_score": 0.1111111111111111,
"best_stage": "round-80",
"stage": "ppo"
},
"dataset": {
"fingerprint": "13e5709daa6e5e8d777c104cebd6e7ec75115874095319ee7d570c7a02277878",
"counts": {
"train": 180278,
"validation": 21260,
"test": 21165
},
"valid_games": 7266,
"rejected_games": 0
},
"metrics": [
{
"stage": "baseline",
"validation_score": 0,
"best_score": 0
},
{
"epoch": 1,
"offline_validation": {
"samples": 21260,
"loss": 0.6506123136262902
}
},
{
"stage": "offline-1",
"validation_score": -0.28703703703703703,
"best_score": 0
},
{
"epoch": 2,
"offline_validation": {
"samples": 21260,
"loss": 0.6081839078849818
}