run records: final.json, log, probes
Browse files
log.jsonl
CHANGED
|
@@ -321,3 +321,4 @@
|
|
| 321 |
{"event": "eval", "step": 6000, "phase": 3, "bpb": {"en": 1.88486, "tr": 1.99774, "fi": 2.10921}, "bytes_per_chunk": {"en": 4.873, "tr": 5.239, "fi": 4.739}, "bpb_mean": 1.99727, "frozen_inventory": true, "t": 1851.6}
|
| 322 |
{"event": "reps_dumped", "dirs": ["/local/scratch/sas245/beetle-hnet/runs/ali_mem-skel_s2/probe/reps/th0.30", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-skel_s2/probe/reps/th0.40", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-skel_s2/probe/reps/th0.50", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-skel_s2/probe/reps/th0.60", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-skel_s2/probe/reps/th0.70"], "t": 1854.3}
|
| 323 |
{"event": "final", "run": "ali_mem-skel_s2", "arch": "ali", "objective": "ratio", "condition": "A", "step": 6000, "params": 22785088, "seed": 2, "per_lang": {"en": {"cpc": 0.2089294949146699, "cps": 28.40625, "bpb": 1.8891463736054175, "bytes_per_chunk": 4.814840664891174, "reads_per_chunk": 0.5980989989898062}, "tr": {"cpc": 0.21543390598243384, "cps": 30.46875, "bpb": 1.9904279993433465, "bytes_per_chunk": 5.251176859437912, "reads_per_chunk": 0.5799963943029988}, "fi": {"cpc": 0.23294880654935943, "cps": 34.234375, "bpb": 2.126474477598491, "bytes_per_chunk": 4.7085956514320095, "reads_per_chunk": 0.6858918515990552}}, "bpb_mean": 2.002016283515752, "gini_cpc": 0.024361139564611722, "gini_cps": 0.04172959668848253, "bpic": 4.76019466353415, "mean_boundary_entropy": 0.11170578002929688, "dechunk": "ema", "router_noise": 0.0, "transitions": [{"step": 0, "phase": 1, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 2400, "phase": 2, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 4200, "phase": 3, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}], "probe_overhead_sec_total": 331.3731119632721, "probe_overhead_seq_frac": 0.499765625, "wall_sec": 1857.3115632534027, "inventory": {"size": 65536, "admitted": 65536, "staged": 0, "budget_B": 65536, "staging_capacity": 0, "collisions": 0, "dropped": 0, "type_rejected": 0, "config_keys": 0, "admitted_surface": 0, "staged_surface": 0, "admitted_split": 0, "staged_split": 0, "admitted_merge": 0, "staged_merge": 0, "admitted_skel": 65536, "staged_skel": 0, "mean_count": 361.39111328125, "n_util_obs": 1300, "util_count_spearman": 0.2596489470935999}, "ali_params": 268096, "ali_config": {"enabled": true, "budget_B": 65536, "staging_capacity": 262144, "policy": "frequency", "value_mode": "learned", "candidate_types": ["skel"], "gate": "vector", "retrieval": true, "memory_scope": "persistent", "read_dropout": 0.1, "write_rule": "mean", "eta_min": 0.0, "consolidate_every": 50, "probe_seqs": 2, "probe_reps": 8, "horizon": 32, "predictor": "mlp", "lambda_util": 0.1, "max_span": 64, "byte_classes_path": "/local/scratch/sas245/beetle-hnet/scratch/data/byte_classes.json", "gate_hidden": 0, "type_dim": 16, "gate_bias_init": -2.0}, "util_vs_frequency_spearman": 0.2596489470935999, "paired_vs_dropout": {"n": 2730, "pearson": -0.006880562740271094, "spearman": -0.002931798169149572, "mean_abs_discrepancy": 1.059653639793396, "mean_paired": 0.044733691960573196, "mean_dropout": 0.055564913898706436}, "predictor_calibration": {"n": 6592, "all": {"n": 6592, "pearson": 0.1363131604910194, "spearman": 0.1383622981261577, "mae": 0.0987764224410057, "reliability": [{"pred": 0.002392061287537217, "obs": 0.010740666650235653, "n": 660}, {"pred": 0.012135072611272335, "obs": 0.018849022686481476, "n": 659}, {"pred": 0.014819816686213017, "obs": 0.02018786035478115, "n": 659}, {"pred": 0.01694437488913536, "obs": 0.022722948342561722, "n": 659}, {"pred": 0.018824057653546333, "obs": 0.021593667566776276, "n": 659}, {"pred": 0.02109544351696968, "obs": 0.03046458587050438, "n": 659}, {"pred": 0.024069782346487045, "obs": 0.036752745509147644, "n": 659}, {"pred": 0.02879488095641136, "obs": 0.026973681524395943, "n": 659}, {"pred": 0.036858052015304565, "obs": 0.0521981343626976, "n": 659}, {"pred": 0.06617799401283264, "obs": 0.08191295713186264, "n": 660}]}, "admitted": {"n": 6592, "pearson": 0.1363131604910194, "spearman": 0.1383622981261577, "mae": 0.0987764224410057, "reliability": [{"pred": 0.002392061287537217, "obs": 0.010740666650235653, "n": 660}, {"pred": 0.012135072611272335, "obs": 0.018849022686481476, "n": 659}, {"pred": 0.014819816686213017, "obs": 0.02018786035478115, "n": 659}, {"pred": 0.01694437488913536, "obs": 0.022722948342561722, "n": 659}, {"pred": 0.018824057653546333, "obs": 0.021593667566776276, "n": 659}, {"pred": 0.02109544351696968, "obs": 0.03046458587050438, "n": 659}, {"pred": 0.024069782346487045, "obs": 0.036752745509147644, "n": 659}, {"pred": 0.02879488095641136, "obs": 0.026973681524395943, "n": 659}, {"pred": 0.036858052015304565, "obs": 0.0521981343626976, "n": 659}, {"pred": 0.06617799401283264, "obs": 0.08191295713186264, "n": 660}]}, "staged": {"n": 0, "pearson": NaN, "spearman": NaN, "mae": NaN}}, "t": 1857.4}
|
|
|
|
|
|
| 321 |
{"event": "eval", "step": 6000, "phase": 3, "bpb": {"en": 1.88486, "tr": 1.99774, "fi": 2.10921}, "bytes_per_chunk": {"en": 4.873, "tr": 5.239, "fi": 4.739}, "bpb_mean": 1.99727, "frozen_inventory": true, "t": 1851.6}
|
| 322 |
{"event": "reps_dumped", "dirs": ["/local/scratch/sas245/beetle-hnet/runs/ali_mem-skel_s2/probe/reps/th0.30", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-skel_s2/probe/reps/th0.40", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-skel_s2/probe/reps/th0.50", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-skel_s2/probe/reps/th0.60", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-skel_s2/probe/reps/th0.70"], "t": 1854.3}
|
| 323 |
{"event": "final", "run": "ali_mem-skel_s2", "arch": "ali", "objective": "ratio", "condition": "A", "step": 6000, "params": 22785088, "seed": 2, "per_lang": {"en": {"cpc": 0.2089294949146699, "cps": 28.40625, "bpb": 1.8891463736054175, "bytes_per_chunk": 4.814840664891174, "reads_per_chunk": 0.5980989989898062}, "tr": {"cpc": 0.21543390598243384, "cps": 30.46875, "bpb": 1.9904279993433465, "bytes_per_chunk": 5.251176859437912, "reads_per_chunk": 0.5799963943029988}, "fi": {"cpc": 0.23294880654935943, "cps": 34.234375, "bpb": 2.126474477598491, "bytes_per_chunk": 4.7085956514320095, "reads_per_chunk": 0.6858918515990552}}, "bpb_mean": 2.002016283515752, "gini_cpc": 0.024361139564611722, "gini_cps": 0.04172959668848253, "bpic": 4.76019466353415, "mean_boundary_entropy": 0.11170578002929688, "dechunk": "ema", "router_noise": 0.0, "transitions": [{"step": 0, "phase": 1, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 2400, "phase": 2, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 4200, "phase": 3, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}], "probe_overhead_sec_total": 331.3731119632721, "probe_overhead_seq_frac": 0.499765625, "wall_sec": 1857.3115632534027, "inventory": {"size": 65536, "admitted": 65536, "staged": 0, "budget_B": 65536, "staging_capacity": 0, "collisions": 0, "dropped": 0, "type_rejected": 0, "config_keys": 0, "admitted_surface": 0, "staged_surface": 0, "admitted_split": 0, "staged_split": 0, "admitted_merge": 0, "staged_merge": 0, "admitted_skel": 65536, "staged_skel": 0, "mean_count": 361.39111328125, "n_util_obs": 1300, "util_count_spearman": 0.2596489470935999}, "ali_params": 268096, "ali_config": {"enabled": true, "budget_B": 65536, "staging_capacity": 262144, "policy": "frequency", "value_mode": "learned", "candidate_types": ["skel"], "gate": "vector", "retrieval": true, "memory_scope": "persistent", "read_dropout": 0.1, "write_rule": "mean", "eta_min": 0.0, "consolidate_every": 50, "probe_seqs": 2, "probe_reps": 8, "horizon": 32, "predictor": "mlp", "lambda_util": 0.1, "max_span": 64, "byte_classes_path": "/local/scratch/sas245/beetle-hnet/scratch/data/byte_classes.json", "gate_hidden": 0, "type_dim": 16, "gate_bias_init": -2.0}, "util_vs_frequency_spearman": 0.2596489470935999, "paired_vs_dropout": {"n": 2730, "pearson": -0.006880562740271094, "spearman": -0.002931798169149572, "mean_abs_discrepancy": 1.059653639793396, "mean_paired": 0.044733691960573196, "mean_dropout": 0.055564913898706436}, "predictor_calibration": {"n": 6592, "all": {"n": 6592, "pearson": 0.1363131604910194, "spearman": 0.1383622981261577, "mae": 0.0987764224410057, "reliability": [{"pred": 0.002392061287537217, "obs": 0.010740666650235653, "n": 660}, {"pred": 0.012135072611272335, "obs": 0.018849022686481476, "n": 659}, {"pred": 0.014819816686213017, "obs": 0.02018786035478115, "n": 659}, {"pred": 0.01694437488913536, "obs": 0.022722948342561722, "n": 659}, {"pred": 0.018824057653546333, "obs": 0.021593667566776276, "n": 659}, {"pred": 0.02109544351696968, "obs": 0.03046458587050438, "n": 659}, {"pred": 0.024069782346487045, "obs": 0.036752745509147644, "n": 659}, {"pred": 0.02879488095641136, "obs": 0.026973681524395943, "n": 659}, {"pred": 0.036858052015304565, "obs": 0.0521981343626976, "n": 659}, {"pred": 0.06617799401283264, "obs": 0.08191295713186264, "n": 660}]}, "admitted": {"n": 6592, "pearson": 0.1363131604910194, "spearman": 0.1383622981261577, "mae": 0.0987764224410057, "reliability": [{"pred": 0.002392061287537217, "obs": 0.010740666650235653, "n": 660}, {"pred": 0.012135072611272335, "obs": 0.018849022686481476, "n": 659}, {"pred": 0.014819816686213017, "obs": 0.02018786035478115, "n": 659}, {"pred": 0.01694437488913536, "obs": 0.022722948342561722, "n": 659}, {"pred": 0.018824057653546333, "obs": 0.021593667566776276, "n": 659}, {"pred": 0.02109544351696968, "obs": 0.03046458587050438, "n": 659}, {"pred": 0.024069782346487045, "obs": 0.036752745509147644, "n": 659}, {"pred": 0.02879488095641136, "obs": 0.026973681524395943, "n": 659}, {"pred": 0.036858052015304565, "obs": 0.0521981343626976, "n": 659}, {"pred": 0.06617799401283264, "obs": 0.08191295713186264, "n": 660}]}, "staged": {"n": 0, "pearson": NaN, "spearman": NaN, "mae": NaN}}, "t": 1857.4}
|
| 324 |
+
{"event": "hub_done", "repo": "AdaptiveChunking/ali_mem-skel_s2", "pushed": [0, 1, 2, 4, 6, 8, 16, 32, 64, 128, 256, 512, 1000, 1024, 2048, 3000, 4096, 6000], "failed": [], "ok": true, "t": 1873.0}
|