diffcider-typed-decisions: Qwen3-0.6B-diffusion-mdlm-v0.1 lora · accuracy 0.767 (test) · code 123ad8b+
45f08f6 verified | { | |
| "sysone_version": "0.3.0", | |
| "backend": "sysone", | |
| "name": "diffcider-typed-decisions", | |
| "encoder": { | |
| "id": "dllm-hub/Qwen3-0.6B-diffusion-mdlm-v0.1", | |
| "revision": "c8d24a3f4adaeef46881b450e1bf7d1005203bd7", | |
| "source": "transformers", | |
| "hidden_size": 1024, | |
| "family": "a2d-qwen3", | |
| "modality": "text", | |
| "dtype": "float32", | |
| "weights": "adapter" | |
| }, | |
| "template": { | |
| "start": null, | |
| "sep": null, | |
| "end": null, | |
| "mask": "<|mask|>", | |
| "image": null, | |
| "row": "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\nState:\n{state}\n\nQuestion:\n{instructions}\n\n{options}<|im_end|>\n", | |
| "option": "{text}: {mask}", | |
| "max_len": 4096, | |
| "head_max_len": 2048, | |
| "option_tokens": 512, | |
| "truncate": "right", | |
| "option_sep": "\n", | |
| "kinds": { | |
| "choice": { | |
| "header": "For each option, mark Yes if it answers the question, otherwise No.\n", | |
| "option": "{text}: {mask}", | |
| "sep": "\n" | |
| }, | |
| "score": { | |
| "header": "For each option, mark Yes if it answers the question, otherwise No.\n", | |
| "option": "{text}: {mask}", | |
| "sep": "\n" | |
| }, | |
| "noul": { | |
| "header": "Answer Yes or No.\nAnswer:", | |
| "option": "{mask}", | |
| "sep": "" | |
| } | |
| }, | |
| "name": "jev" | |
| }, | |
| "tfms": [], | |
| "head": { | |
| "width": 1024, | |
| "layers": 0, | |
| "scorer": "yesno", | |
| "dropout": 0.1, | |
| "nhead": 16, | |
| "standardize": false, | |
| "readout": "anchor", | |
| "query": null, | |
| "query_width": null, | |
| "pair_dim": 256, | |
| "act": false, | |
| "regime": "lora" | |
| }, | |
| "temperature": { | |
| "choice": 0.9453, | |
| "choice:3-5": 0.9453, | |
| "score": 1.0186, | |
| "score:3-5": 1.0186, | |
| "noul": 1.0234, | |
| "noul:2": 1.0234 | |
| }, | |
| "act_threshold": null, | |
| "act_score": null, | |
| "question_types": [ | |
| "choice", | |
| "score", | |
| "noul" | |
| ], | |
| "metrics": { | |
| "epoch": 3.532544378698225, | |
| "loss": 0.9801202416419983, | |
| "accuracy": 0.6925, | |
| "soft_accuracy": 0.45718836379426847, | |
| "brier": 0.06207205604909272, | |
| "kl": 0.10056160501177605, | |
| "tv": 0.16701407909774973, | |
| "ece": 0.14945683625768713, | |
| "score_mae": 0.2234294522603933, | |
| "within_one": 1.0, | |
| "n": 400, | |
| "accuracy_choice": 0.7, | |
| "accuracy_score": 0.60625, | |
| "accuracy_noul": 0.85, | |
| "n_unanswerable": 0, | |
| "runtime": 22.1624, | |
| "samples_per_second": 18.049, | |
| "steps_per_second": 0.587 | |
| }, | |
| "training": { | |
| "recorded": "2026-10-06T00:27:16+00:00", | |
| "code": { | |
| "git": null, | |
| "sysone_git": { | |
| "repo": "sysonelib", | |
| "commit": "123ad8bac647e2a14555881ea19e4a6a159b27e6", | |
| "branch": "main", | |
| "remote": "https://github.com/sgaseretto/sysonelib.git", | |
| "dirty": true, | |
| "changed": [ | |
| "bs/01_template.ipynb", | |
| "nbs/02_data.ipynb", | |
| "nbs/03_models.ipynb", | |
| "nbs/04_losses.ipynb", | |
| "nbs/05_learner.ipynb", | |
| "nbs/06_inference.ipynb", | |
| "nbs/40_cloud.ipynb", | |
| "nbs/50_cli.ipynb", | |
| "nbs/fixtures/make_fixtures.py", | |
| "sysone/__init__.py", | |
| "sysone/_modidx.py", | |
| "sysone/cli.py", | |
| "sysone/cloud.py", | |
| "sysone/data.py", | |
| "sysone/inference.py", | |
| "sysone/learner.py", | |
| "sysone/losses.py", | |
| "sysone/models.py", | |
| "sysone/template.py", | |
| "nbs/24_mdlm.ipynb", | |
| "nbs/fixtures/tiny-a2d-qwen3/", | |
| "sysone/mdlm.py" | |
| ], | |
| "n_changed": 22, | |
| "diff_sha256": "77eb5a356ea0493da3c37d8984638a514afb742071229bdac7f2f56fc62a864e" | |
| }, | |
| "entry": { | |
| "name": "run_b.py", | |
| "sha256": "64a038d242e574bd2a514812e9949b0ae3fa80217728556ee1ac623b4dfd9ae4" | |
| } | |
| }, | |
| "environment": { | |
| "python": "3.13.15", | |
| "implementation": "CPython", | |
| "os": "Linux 6.18.48+", | |
| "packages": { | |
| "sysone": "0.3.0", | |
| "torch": "2.11.0+cu128", | |
| "transformers": "5.16.1", | |
| "peft": "0.20.0", | |
| "accelerate": "1.14.0", | |
| "datasets": "4.8.5", | |
| "huggingface_hub": "1.29.0", | |
| "tokenizers": "0.23.1", | |
| "safetensors": "0.8.0", | |
| "numpy": "2.1.3", | |
| "fastcore": "2.2.32", | |
| "plum-dispatch": "2.10.1" | |
| }, | |
| "cuda": "12.8", | |
| "cudnn": 91900 | |
| }, | |
| "hardware": { | |
| "accelerator": { | |
| "kind": "cuda", | |
| "devices": [ | |
| "Tesla T4", | |
| "Tesla T4" | |
| ], | |
| "memory_gb": [ | |
| 14.6, | |
| 14.6 | |
| ], | |
| "capability": "7.5" | |
| }, | |
| "host": { | |
| "cpu": "Intel(R) Xeon(R) CPU @ 2.00GHz", | |
| "cores": 4, | |
| "memory_gb": 31.3, | |
| "system": "Linux", | |
| "machine": "x86_64" | |
| }, | |
| "run": { | |
| "backend": "kaggle", | |
| "job": "mdlm-lora-check", | |
| "machine": "NvidiaTeslaT4" | |
| } | |
| }, | |
| "data": { | |
| "source": { | |
| "kind": "hub", | |
| "id": "LocalLLaMA/typed-decisions", | |
| "config": "all", | |
| "split": "train", | |
| "valid": "test[:80]", | |
| "revision": null, | |
| "sha": "d0e2f0c42fef86cc15d1688d25a19f5ba7c85b18", | |
| "note": "read with datasets.load_dataset and passed as records; the dataset's last commit before the run" | |
| }, | |
| "splits": { | |
| "train": { | |
| "cases": 1080, | |
| "fingerprint": "89a925d7f54eeaea" | |
| }, | |
| "valid": { | |
| "cases": 80, | |
| "fingerprint": "0bff619936359396" | |
| }, | |
| "calib": { | |
| "cases": 120, | |
| "fingerprint": "11352d26c9dc187b" | |
| } | |
| }, | |
| "template": "jev", | |
| "transforms": [], | |
| "sampler": null, | |
| "seed": 0 | |
| }, | |
| "model": { | |
| "encoder": "dllm-hub/Qwen3-0.6B-diffusion-mdlm-v0.1", | |
| "regime": "lora", | |
| "lora": { | |
| "peft_type": "LORA", | |
| "r": 16, | |
| "lora_alpha": 32, | |
| "lora_dropout": 0.05, | |
| "target_modules": [ | |
| "down_proj", | |
| "gate_proj", | |
| "k_proj", | |
| "o_proj", | |
| "q_proj", | |
| "up_proj", | |
| "v_proj" | |
| ], | |
| "bias": "none", | |
| "use_rslora": false, | |
| "use_dora": false, | |
| "modules_to_save": null | |
| }, | |
| "params": { | |
| "encoder": 606142464, | |
| "head": 4097, | |
| "trainable": 10096641 | |
| } | |
| }, | |
| "training": { | |
| "preset": "t4", | |
| "loss": "soft_ce_rps=0.25", | |
| "lr": [ | |
| 0.0002, | |
| 0.001 | |
| ], | |
| "seed": 0, | |
| "args": { | |
| "lr_scheduler_type": "cosine", | |
| "warmup_steps": 0.1, | |
| "weight_decay": 0.01, | |
| "gradient_accumulation_steps": 4, | |
| "fp16": true, | |
| "gradient_checkpointing": true, | |
| "logging_steps": 10, | |
| "disable_tqdm": false, | |
| "report_to": [], | |
| "per_device_eval_batch_size": 16, | |
| "save_strategy": "no", | |
| "seed": 0, | |
| "accelerator_config": "AcceleratorConfig(split_batches=False, dispatch_batches=None, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False)", | |
| "remove_unused_columns": false, | |
| "label_names": [ | |
| "target" | |
| ], | |
| "train_sampling_strategy": "group_by_length", | |
| "length_column_name": "n_tokens", | |
| "debug": [] | |
| }, | |
| "fits": [ | |
| { | |
| "started": "2026-10-05T22:55:01+00:00", | |
| "regime": "lora", | |
| "epochs": 1, | |
| "max_steps": 300, | |
| "lr_head": 0.001, | |
| "lr_encoder": 0.0002, | |
| "schedule": "cosine", | |
| "warmup": 0.25, | |
| "batch_size": 8, | |
| "grad_accum": 4, | |
| "precision": "fp16", | |
| "device": "cuda", | |
| "steps": 300, | |
| "seconds": 5246.0, | |
| "final_loss": 0.8778909683227539, | |
| "peak_memory_gb": 7.69 | |
| } | |
| ], | |
| "history": { | |
| "loss": [ | |
| 1.4344513893127442, | |
| 1.249254322052002, | |
| 1.1781773567199707, | |
| 1.163340187072754, | |
| 1.0988536834716798, | |
| 1.0159993171691895, | |
| 1.1186190605163575, | |
| 0.9268903732299805, | |
| 1.0303213119506835, | |
| 0.8802448272705078, | |
| 1.0308435440063477, | |
| 0.87939453125, | |
| 0.9772283554077148, | |
| 0.9342096328735352, | |
| 0.9400447845458985, | |
| 0.9097446441650391, | |
| 0.9203708648681641, | |
| 0.9839879035949707, | |
| 0.8999062538146972, | |
| 0.896394920349121, | |
| 0.8628653526306153, | |
| 0.9104316711425782, | |
| 0.8542628288269043, | |
| 0.9121370315551758, | |
| 0.772752046585083, | |
| 0.8732522010803223, | |
| 0.7740259647369385, | |
| 0.9276440620422364, | |
| 0.750361967086792, | |
| 0.8778909683227539 | |
| ], | |
| "eval": [ | |
| { | |
| "epoch": 1.0, | |
| "loss": 1.09174382686615, | |
| "accuracy": 0.5425, | |
| "soft_accuracy": 0.4193413073577042, | |
| "brier": 0.141350766370905, | |
| "kl": 0.2354086200846709, | |
| "tv": 0.24462439980862877, | |
| "ece": 0.10866832427761491, | |
| "score_mae": 0.278731628334489, | |
| "within_one": 0.99375, | |
| "n": 400, | |
| "accuracy_choice": 0.38125, | |
| "accuracy_score": 0.60625, | |
| "accuracy_noul": 0.7375, | |
| "n_unanswerable": 0, | |
| "runtime": 22.1301, | |
| "samples_per_second": 18.075, | |
| "steps_per_second": 0.587 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "loss": 1.0160019397735596, | |
| "accuracy": 0.67, | |
| "soft_accuracy": 0.4682727722546358, | |
| "brier": 0.07573289376790472, | |
| "kl": 0.11924152387670522, | |
| "tv": 0.17498002941222868, | |
| "ece": 0.08067372087327374, | |
| "score_mae": 0.2732784988754318, | |
| "within_one": 0.99375, | |
| "n": 400, | |
| "accuracy_choice": 0.675, | |
| "accuracy_score": 0.6125, | |
| "accuracy_noul": 0.775, | |
| "n_unanswerable": 0, | |
| "runtime": 22.208, | |
| "samples_per_second": 18.012, | |
| "steps_per_second": 0.585 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "loss": 0.9821739792823792, | |
| "accuracy": 0.675, | |
| "soft_accuracy": 0.4534864358013278, | |
| "brier": 0.06318044447460737, | |
| "kl": 0.10197306195815968, | |
| "tv": 0.1695142981192557, | |
| "ece": 0.1388826772323706, | |
| "score_mae": 0.2302615860129519, | |
| "within_one": 1.0, | |
| "n": 400, | |
| "accuracy_choice": 0.66875, | |
| "accuracy_score": 0.59375, | |
| "accuracy_noul": 0.85, | |
| "n_unanswerable": 0, | |
| "runtime": 21.9708, | |
| "samples_per_second": 18.206, | |
| "steps_per_second": 0.592 | |
| }, | |
| { | |
| "epoch": 3.532544378698225, | |
| "loss": 0.9801202416419983, | |
| "accuracy": 0.6925, | |
| "soft_accuracy": 0.45718836379426847, | |
| "brier": 0.06207205604909272, | |
| "kl": 0.10056160501177605, | |
| "tv": 0.16701407909774973, | |
| "ece": 0.14945683625768713, | |
| "score_mae": 0.2234294522603933, | |
| "within_one": 1.0, | |
| "n": 400, | |
| "accuracy_choice": 0.7, | |
| "accuracy_score": 0.60625, | |
| "accuracy_noul": 0.85, | |
| "n_unanswerable": 0, | |
| "runtime": 22.1624, | |
| "samples_per_second": 18.049, | |
| "steps_per_second": 0.587 | |
| } | |
| ] | |
| } | |
| }, | |
| "tracking": null | |
| }, | |
| "evaluations": [ | |
| { | |
| "split": "test", | |
| "dataset": "LocalLLaMA/typed-decisions", | |
| "config": "all", | |
| "n": 2000, | |
| "metrics": { | |
| "accuracy": 0.7675, | |
| "accuracy_choice": 0.7517, | |
| "accuracy_score": 0.7238, | |
| "accuracy_noul": 0.8417, | |
| "ece": 0.1496, | |
| "brier": 0.0536 | |
| }, | |
| "note": "calibrated, on a Kaggle T4" | |
| } | |
| ] | |
| } |