Save exact-resume scratch checkpoint probe_006 before its BigSolDB probe

probe_006/atomic_preprocessor.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09ee1beeb2b04754e26e52db432f690b0cc4a55b659ab6752a6d499f713d2043
+size 19537832

probe_006/config.json

@@ -0,0 +1,33 @@
+{
+  "epochs": 10,
+  "probes_per_epoch": 5,
+  "seed": 42026,
+  "architecture_source_commit": "20df6fe9c61154e351d677cb0f77fd716d00beae",
+  "initialization": "Random learned Rem3Di weights; no published or PCQM/COSMO checkpoint loaded",
+  "learning_rate_max": 0.0001,
+  "weight_decay": 0.001,
+  "learning_rate_uncertainty": 0.001,
+  "gradient_clip": 1.0,
+  "noise_sigma": 0.3,
+  "denoising": "Exact author atomic embedding denoising loss and gradient paths, on PCQM",
+  "uncertainty": "sum(L_t/(2*exp(s_t))+s_t/2); raw denoising loss; initial log variance ln(1088)-0.5 for denoising, -0.5 for COSMO",
+  "uncertainty_initialization": "Equivalent initial precision to author-denoising per channel; no modification of raw denoising function",
+  "task_order": [
+    "rem3di_denoising",
+    "cosmo"
+  ],
+  "cosmo_head": "640 -> 256 SiLU -> 155",
+  "pcqm_batch_size": 128,
+  "pcqm_atom_cap": 5250,
+  "cosmo_batch_size": 96,
+  "cosmo_atom_cap": 4000,
+  "epoch_definition": "Maximum of complete PCQM and COSMO loader lengths; shorter loader cycles; all retained training molecules seen each epoch",
+  "scheduler": "OneCycleLR max_lr=1e-4, total optimizer steps computed from ten exact deterministic epoch plans",
+  "decorrelation": false,
+  "combisolv": false,
+  "mace": "Frozen MACE-POLAR-1-M; raw 4096-channel float32 cache",
+  "normalization": "Fresh train-only population invariant mean/std across PCQM and COSMO atoms",
+  "probe": "Frozen pooled 640; solute640+solvent640+Temperature_K; exact BigSolDB rows; full75731-row Kumo Large context, 4 estimators, seed42",
+  "probe_schedule": "ceil(k * epoch_steps / 5), k=1..5, each epoch; fifty test probes; no test-based selection",
+  "checkpoint_policy": "All fifty probe points include all weights, optimizer, scheduler and RNG; verified Hugging Face backup before probing"
+}

probe_006/cosmo_head.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4fcd02f17b24a782f1bff3e85d2b6449b28516b3a1f2bd45078688e79f85952
+size 818235

probe_006/cosmo_label_transform.json

@@ -0,0 +1,319 @@
+{
+  "mean": [
+    8.388609302506075e-07,
+    8.177936762676325e-06,
+    2.093551106918719e-05,
+    3.2799945778375235e-05,
+    8.079647897075607e-05,
+    0.00028593167046552306,
+    0.0013247879576240857,
+    0.004911009903662613,
+    0.011972004240760965,
+    0.02519924983721102,
+    0.05170354532086424,
+    0.09800470302606258,
+    0.16136536322515166,
+    0.24276781948811774,
+    0.35190012806998444,
+    0.5058363824821164,
+    0.7376728874925607,
+    1.079937829228189,
+    1.585597292616362,
+    2.1727961301157115,
+    2.5681275931951375,
+    2.8615992011079157,
+    3.085695557386356,
+    3.1463694036869296,
+    3.1017587386259997,
+    3.0281585992264777,
+    2.870755267688759,
+    2.6345205944607564,
+    2.4498347991972524,
+    2.2885653265418706,
+    2.0242288487184594,
+    1.711372277266878,
+    1.3611100719514313,
+    1.1184139825757506,
+    0.9493885599335725,
+    0.8190656240168543,
+    0.6854897852152764,
+    0.4807500726657983,
+    0.26500685221188475,
+    0.11846122851381499,
+    0.04194921103893719,
+    0.015010926529055155,
+    0.0052818421311684235,
+    0.0015414647022698948,
+    0.0003659878860339974,
+    6.642230429539629e-05,
+    1.3615537474360663e-05,
+    1.094322173399213e-05,
+    9.427612339493122e-06,
+    3.743626086553491e-06,
+    5.638919142925201e-07,
+    9.423005000227588e-07,
+    1.1441121678440357e-05,
+    4.664364545223993e-05,
+    0.00012604296359064056,
+    0.0005190436846488184,
+    0.003430448946674064,
+    0.019182132767525627,
+    0.055865439145762426,
+    0.07756995862057466,
+    0.08973915699345283,
+    0.11007139824798137,
+    0.12384428755469998,
+    0.12116300539177713,
+    0.11369018357993191,
+    0.10361484585220103,
+    0.09044244058954115,
+    0.0745410602979547,
+    0.05773971037710788,
+    0.041780073643036264,
+    0.02786654950518393,
+    0.016632237140395512,
+    0.008549525176370483,
+    0.003535849141210067,
+    0.0010380697027366956,
+    0.00015317522290193833,
+    0.0,
+    0.001742221641205026,
+    0.007301519607067941,
+    0.017175679559386748,
+    0.03186570830891406,
+    0.051321857700535045,
+    0.0743486305857352,
+    0.0981567121561718,
+    0.12126263248901792,
+    0.14351820586935335,
+    0.16309458826418485,
+    0.16636493281314874,
+    0.12828499395789633,
+    0.09239210646599323,
+    0.08051793491647082,
+    0.03711639251723531,
+    0.0020701015800011725,
+    0.00014635810399557018,
+    4.085478464250703e-05,
+    9.313651640138456e-06,
+    3.822372201745394e-06,
+    1.5908470070494528e-06,
+    3.4818475695660627e-07,
+    0.0,
+    0.0,
+    0.0,
+    7.184148646239562e-07,
+    1.3630640712144018e-05,
+    6.023465682074327e-05,
+    0.00015470631248008695,
+    0.00045331952246500777,
+    0.001813059350819177,
+    0.006404232364622423,
+    0.01668153637234231,
+    0.04495299581974657,
+    0.09102377394692583,
+    0.15262216852353822,
+    0.2216090929758519,
+    0.26690885764260674,
+    0.27742989078399527,
+    0.2675435090414338,
+    0.24939016402658284,
+    0.22216836995660932,
+    0.1763647211917833,
+    0.12873690017545603,
+    0.08572932745724374,
+    0.050387852265173484,
+    0.02591023156521495,
+    0.011376053233313772,
+    0.003812555502702103,
+    0.0006774999475051197,
+    0.0,
+    0.013389393413463032,
+    0.06396947387967701,
+    0.15713137140339634,
+    0.27079739782770523,
+    0.38424125910978074,
+    0.5236806913163866,
+    0.6652570583595291,
+    0.7859406753453589,
+    0.8755402768924716,
+    0.962475114412502,
+    1.0273617209230694,
+    0.9410559772757581,
+    0.6879079278394965,
+    0.4026525071693016,
+    0.21610681648989294,
+    0.12835062094639127,
+    0.06865609860817891,
+    0.02986094715983099,
+    0.010542547078815662,
+    0.002525452268036032,
+    0.00015031742690276706,
+    1.8465715549648715e-05,
+    2.768806094724967e-06,
+    1.1904319231572592e-06,
+    2.980087826597099e-07,
+    5.494981557237649,
+    5.416589856550859
+  ],
+  "scale": [
+    0.00028219344764356824,
+    0.0011011888633900837,
+    0.002032794049394733,
+    0.0021744372628592756,
+    0.0035370464125024426,
+    0.0058414822542558545,
+    0.010300543693129138,
+    0.0195059605959777,
+    0.03473629330863863,
+    0.05673391716994678,
+    0.09008313021446317,
+    0.140318218407688,
+    0.2050931388737851,
+    0.2822467994285733,
+    0.37420099153549075,
+    0.47123146505679603,
+    0.5741080807206825,
+    0.6786372101273207,
+    0.73326159744217,
+    0.6770884458858979,
+    0.5566054270808432,
+    0.43403996609811285,
+    0.39721845736090716,
+    0.4484266507126092,
+    0.46233920634848663,
+    0.4626227511704608,
+    0.4486601485528492,
+    0.4278095297290624,
+    0.5403342411465548,
+    0.5725947835737405,
+    0.5727288655406647,
+    0.5326233133333143,
+    0.44462112891649164,
+    0.42650465913732244,
+    0.41200164205579987,
+    0.4010066974424893,
+    0.38568778794864655,
+    0.31737336143368144,
+    0.2347226338677458,
+    0.14878637619757537,
+    0.07213814709548623,
+    0.03655831279574978,
+    0.017272437591659993,
+    0.008072515484241326,
+    0.004169572435250142,
+    0.00273783662215128,
+    0.0024361080766053477,
+    0.0023486443342219344,
+    0.0021046181137050254,
+    0.0011070186359565528,
+    0.0002996281855087881,
+    0.0004696189406328482,
+    0.0020666132859492397,
+    0.0049305727401950045,
+    0.008699673184506481,
+    0.01666072225772805,
+    0.04227377695140685,
+    0.10348653179413272,
+    0.19116421436713862,
+    0.22758633950022059,
+    0.23248226345479975,
+    0.24582953356230133,
+    0.255614853307072,
+    0.24359883691663753,
+    0.22652509402818907,
+    0.20640650051593623,
+    0.18182468559969245,
+    0.15409415304072574,
+    0.12525210811900375,
+    0.09679679216411825,
+    0.06964573198793922,
+    0.04528746751586639,
+    0.025542641545468682,
+    0.011794670388920288,
+    0.003979556463034284,
+    0.0007018856011055862,
+    1.0,
+    0.003784754469215642,
+    0.015673306348323625,
+    0.03633026080503415,
+    0.06610212980233189,
+    0.10389356551963261,
+    0.14673390529733085,
+    0.1893109285501493,
+    0.23034729554183123,
+    0.27164469723530893,
+    0.31168606990055486,
+    0.3303851963843814,
+    0.29237606472907185,
+    0.2722433705623426,
+    0.27238277888113904,
+    0.16614356670083044,
+    0.02788814290945399,
+    0.006743962356432955,
+    0.0032215021961893016,
+    0.0012657011706010831,
+    0.0009981786007495917,
+    0.000547071296946782,
+    0.0001200691240582884,
+    1.0,
+    1.0,
+    1.0,
+    0.00026967614295216605,
+    0.0016174434150032575,
+    0.004924205504038004,
+    0.008826260677460942,
+    0.016376118249718345,
+    0.0340881730555313,
+    0.0726013963425638,
+    0.11160454916185913,
+    0.1820008391053674,
+    0.25657196427401247,
+    0.3198342566167783,
+    0.3780597469288841,
+    0.4054791604761226,
+    0.3996737544257664,
+    0.3776824847221058,
+    0.3437008458802379,
+    0.30539006111526956,
+    0.2549901113548528,
+    0.19916321470008305,
+    0.14235207496087326,
+    0.08882274545714894,
+    0.048186036191444015,
+    0.022519855178205792,
+    0.00812797096866809,
+    0.0015847190395135223,
+    1.0,
+    0.018782481833647034,
+    0.09019472396122012,
+    0.18595288675734614,
+    0.23038954588382266,
+    0.2483786269502534,
+    0.31032101164495307,
+    0.3753120388869143,
+    0.42589821965931074,
+    0.4473171619878703,
+    0.497676856170505,
+    0.5693176915783906,
+    0.5897962272695908,
+    0.5689045798121833,
+    0.4745775350431732,
+    0.3341915416128173,
+    0.2561288444249746,
+    0.1776543673606147,
+    0.1132377370652306,
+    0.06230069912570992,
+    0.02683724093738618,
+    0.006469618034965119,
+    0.0024636088601022147,
+    0.0009860941271745965,
+    0.00062422490255574,
+    0.00016149919964486597,
+    0.18170383328418652,
+    0.20633700063992041
+  ],
+  "train_rows": 352834,
+  "training_only": true,
+  "transform": "log1p(sigma bin areas), ln(A), ln(V), population standardization"
+}

probe_006/data_manifest.json

@@ -0,0 +1,43 @@
+{
+  "verified": true,
+  "repository": "Queimo/rem3di-scratch-pcqm-cosmo-data",
+  "revision": "e05015b38b7ba4ce29a4196d89a6bd16740f50fb",
+  "files_sha256": {
+    "pcqm.parquet": "75f5b766bf3d5f4b9869b6d78ebb57157fb0fcf75200821b25042b2483340dbf",
+    "cosmo.parquet": "2f9ce4571fa2c3d1879819527c54b8dd2be38ccf4d93cba7efb0268b7f39ada9",
+    "bigsoldb_fixed_geometries.parquet": "5907c1cb3cca494f52ea8225c694542da4c963c201e4a212a9bc75758f2a2d41",
+    "cosmo_label_transform.json": "c65b87dd94b73a93321d91ed257a4ec5d6c133378f1fc7b8d91807d6a4db5825",
+    "normalization.pt": "b96e193c7f2d2da8c9a60bffb2ee23b16d16afd6c00fb8067a8822b0d3afdb87",
+    "normalization.json": "b1d1c4546e921a0b5c546108c642e71be26d947938a94272550d2badab963df9",
+    "README.md": "e28386cff454ee8f0f42b080a01e60700a8a8312f75a98311a40d160c2e46470",
+    "pcqm_identity_quarantine.csv": "73124afda2ad6be75b7304b00b112ae8c6dd0fe7e0c84ae261cfe01bc6217bae",
+    "architecture.yaml": "8f0c7abafec8c58789d6082730a4f92c309c5faca03ed416e3a17dd16e7562b3",
+    "split_validation.json": "864684edee9daa72d04dc8e3175f253bcb2c15628d8aa78f9dc0b5a12d5b62df"
+  },
+  "passed": true,
+  "pcqm_rows": 500000,
+  "pcqm_splits": {
+    "train": 489511,
+    "validation": 10403,
+    "cross_objective_holdout": 69,
+    "benchmark_overlap": 16,
+    "identity_quarantine": 1
+  },
+  "cosmo_rows": 360259,
+  "cosmo_splits": {
+    "train": 352834,
+    "validation": 7082,
+    "new_holdout": 168,
+    "cross_objective_holdout": 92,
+    "benchmark_overlap": 83
+  },
+  "benchmark_test_solute_groups": 126,
+  "no_pretraining_train_benchmark_overlap": true,
+  "no_cross_objective_train_validation_group_overlap": true,
+  "pcqm_composition_validated_all_rows": true,
+  "pcqm_invalid_identity_rows": 1,
+  "all_train_compositions_and_charges_validated": true,
+  "geometry": "Original PCQM SDF coordinates and original COSMO PM6 coordinates; no regeneration",
+  "initial_rem3di_checkpoint": null,
+  "combisolv_training": false
+}

probe_006/denoising_decoder.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da8a0c07f5a8098a566331e8eafa41ca1f0f6cd6b0330f6e99657e5b74e80de3
+size 352590523

probe_006/encoder.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd88fa89064299554ea3504b4dc8108663481cf88ce9d62240f00ff9dea83435
+size 310115859

probe_006/geometric_preprocessor.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08a3a1eb22072c1257355549d6413f7e92939fb620b7238d861db72937783881
+size 6632

probe_006/manifest.json

@@ -0,0 +1,58 @@
+{
+  "point": 6,
+  "global_step": 4590,
+  "files": {
+    "encoder.pth": {
+      "bytes": 310115859,
+      "sha256": "bd88fa89064299554ea3504b4dc8108663481cf88ce9d62240f00ff9dea83435"
+    },
+    "atomic_preprocessor.pth": {
+      "bytes": 19537832,
+      "sha256": "09ee1beeb2b04754e26e52db432f690b0cc4a55b659ab6752a6d499f713d2043"
+    },
+    "geometric_preprocessor.pth": {
+      "bytes": 6632,
+      "sha256": "08a3a1eb22072c1257355549d6413f7e92939fb620b7238d861db72937783881"
+    },
+    "denoising_decoder.pth": {
+      "bytes": 352590523,
+      "sha256": "da8a0c07f5a8098a566331e8eafa41ca1f0f6cd6b0330f6e99657e5b74e80de3"
+    },
+    "cosmo_head.pth": {
+      "bytes": 818235,
+      "sha256": "d4fcd02f17b24a782f1bff3e85d2b6449b28516b3a1f2bd45078688e79f85952"
+    },
+    "task_weighting.pth": {
+      "bytes": 1690,
+      "sha256": "76fd204c2ce76557b8cbba043cc531d41dd0ece6e33974291225302248ca00c1"
+    },
+    "training_state.pt": {
+      "bytes": 1363636431,
+      "sha256": "07ecc7037d6f7be23833e8b82422ceac52a9c42b631a2e08b9050a35bef4af25"
+    },
+    "metrics.json": {
+      "bytes": 1204,
+      "sha256": "49bb274f1cdee7b5b440ace1a7ed79c38183f2fccbb8647b984069d3cbf5d973"
+    },
+    "config.json": {
+      "bytes": 1817,
+      "sha256": "cf011354b42108cf0a79c022e58f71f269db4e4e36bd0b287e434481a717842d"
+    },
+    "post_training_architecture_config.yaml": {
+      "bytes": 1488,
+      "sha256": "8f0c7abafec8c58789d6082730a4f92c309c5faca03ed416e3a17dd16e7562b3"
+    },
+    "normalization.pt": {
+      "bytes": 21141,
+      "sha256": "b96e193c7f2d2da8c9a60bffb2ee23b16d16afd6c00fb8067a8822b0d3afdb87"
+    },
+    "cosmo_label_transform.json": {
+      "bytes": 7849,
+      "sha256": "c65b87dd94b73a93321d91ed257a4ec5d6c133378f1fc7b8d91807d6a4db5825"
+    },
+    "data_manifest.json": {
+      "bytes": 1954,
+      "sha256": "b5836841eaafebb2eecd2404de002a074aeab3666b391781b14b844e68c23198"
+    }
+  }
+}

probe_006/metrics.json

@@ -0,0 +1,39 @@
+{
+  "point": 6,
+  "epoch": 2,
+  "fifth": 1,
+  "batch_start": 0,
+  "batch_end": 765,
+  "batches": 3825,
+  "epoch_position": 1.2,
+  "nominal_epoch": 1.2,
+  "global_step": 4590,
+  "train": {
+    "denoising_raw": 335.4945885951223,
+    "denoising_per_channel": 0.3083589968705168,
+    "descriptor_std_pcqm": 0.17670656675411986,
+    "weighted_denoising": 3.4580084267784565,
+    "cosmo": 0.19930383975501934,
+    "descriptor_std_cosmo": 0.5501785549852584,
+    "weighted_cosmo": -0.21738123036677542,
+    "cosmo_sigma": 0.43830191914746963,
+    "cosmo_area": 0.0816238478757441,
+    "cosmo_volume": 0.07798573344799818,
+    "gradient_norm": 2.999256024252266,
+    "total": 3.2406271964116815,
+    "log_variance_denoising": 6.301948267494152,
+    "log_variance_cosmo": -0.9513802801082337
+  },
+  "validation": {
+    "pcqm_rows": 10403,
+    "pcqm_loss": 223.44083747794082,
+    "pcqm_descriptor_channel_std": 0.1922706597128588,
+    "pcqm_target_batch_std": 0.9198758489114331,
+    "pcqm_loss_per_channel": 0.20536841679957796,
+    "cosmo_rows": 7082,
+    "cosmo_loss": 0.14549201687146163,
+    "cosmo_descriptor_channel_std": 0.6755814245117883
+  },
+  "seconds": 493.73204084899953,
+  "epoch_coverage": null
+}

probe_006/normalization.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b96e193c7f2d2da8c9a60bffb2ee23b16d16afd6c00fb8067a8822b0d3afdb87
+size 21141

probe_006/post_training_architecture_config.yaml

@@ -0,0 +1,62 @@
+decoder_config:
+  N_layers: 4
+  attention_layer_config:
+    dim_feedforward: 4096
+    dropout: 0.3
+    embedding_dim: 1088
+    num_heads: 8
+  d_descriptor: 640
+  d_geo: 16
+  d_pair: 64
+embedding_preprocess_config:
+  chiral_embedding_dimension: 64
+  equivariant_rms_normalization: true
+  gated: true
+  input_dimension: 4096
+  input_equivariant_dimension: 3072
+  input_invariant_dimension: 1024
+  input_irreps: 512x0e+512x1o+512x0e+512x1o
+  invariant_irreps: 512x0e+512x0e
+  invariant_normalization_config:
+    kind: precomputed_normalization
+  output_irreps: 512x0e+512x0e+64x0o
+  output_irreps_dim: 1088
+  pseudoscalar_dimension: 64
+  pseudoscalar_irrep: 64x0o
+  pseudoscalars: true
+encoder_config:
+  N_layers: 4
+  attention_layer_config:
+    dim_feedforward: 4096
+    dropout: 0.3
+    embedding_dim: 1088
+    num_heads: 8
+  d_geo: 16
+  d_pair: 64
+global_aggregator_config:
+  aggregator_type_config:
+    aggregator_type: pma_attention
+    attn_dropout: 0.3
+    head_dim: 320
+    num_heads: 8
+    num_seeds: 4
+    reduction: mean
+    use_mlp: true
+  global_molecular_descriptor_dropout: 0.0
+  input_dim: 1088
+  output_dim: 640
+kind: encoder_decoder
+mace_config:
+  compute_forces: false
+  compute_stress: false
+  device: cuda
+  dtype: float32
+  enable_cueq: true
+  model_path: /home/fb638/research/mace_models/MACE-POLAR-1-M.model
+positional_encoding_config:
+  N_radial_basis_functions: 16
+  basis_function_config:
+    eps: 1e-08
+    kind: bessel
+  d_projection: 64
+  distance_cutoff: 32.0

probe_006/task_weighting.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76fd204c2ce76557b8cbba043cc531d41dd0ece6e33974291225302248ca00c1
+size 1690

probe_006/training_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07ecc7037d6f7be23833e8b82422ceac52a9c42b631a2e08b9050a35bef4af25
+size 1363636431