{
  "sysone_version": "0.3.0",
  "backend": "sysone",
  "name": "diffcider-typed-decisions",
  "encoder": {
    "id": "dllm-hub/Qwen3-0.6B-diffusion-mdlm-v0.1",
    "revision": "c8d24a3f4adaeef46881b450e1bf7d1005203bd7",
    "source": "transformers",
    "hidden_size": 1024,
    "family": "a2d-qwen3",
    "modality": "text",
    "dtype": "float32",
    "weights": "adapter"
  },
  "template": {
    "start": null,
    "sep": null,
    "end": null,
    "mask": "<|mask|>",
    "image": null,
    "row": "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\nState:\n{state}\n\nQuestion:\n{instructions}\n\n{options}<|im_end|>\n",
    "option": "{text}: {mask}",
    "max_len": 4096,
    "head_max_len": 2048,
    "option_tokens": 512,
    "truncate": "right",
    "option_sep": "\n",
    "kinds": {
      "choice": {
        "header": "For each option, mark Yes if it answers the question, otherwise No.\n",
        "option": "{text}: {mask}",
        "sep": "\n"
      },
      "score": {
        "header": "For each option, mark Yes if it answers the question, otherwise No.\n",
        "option": "{text}: {mask}",
        "sep": "\n"
      },
      "noul": {
        "header": "Answer Yes or No.\nAnswer:",
        "option": "{mask}",
        "sep": ""
      }
    },
    "name": "jev"
  },
  "tfms": [],
  "head": {
    "width": 1024,
    "layers": 0,
    "scorer": "yesno",
    "dropout": 0.1,
    "nhead": 16,
    "standardize": false,
    "readout": "anchor",
    "query": null,
    "query_width": null,
    "pair_dim": 256,
    "act": false,
    "regime": "lora"
  },
  "temperature": {
    "choice": 0.9453,
    "choice:3-5": 0.9453,
    "score": 1.0186,
    "score:3-5": 1.0186,
    "noul": 1.0234,
    "noul:2": 1.0234
  },
  "act_threshold": null,
  "act_score": null,
  "question_types": [
    "choice",
    "score",
    "noul"
  ],
  "metrics": {
    "epoch": 3.532544378698225,
    "loss": 0.9801202416419983,
    "accuracy": 0.6925,
    "soft_accuracy": 0.45718836379426847,
    "brier": 0.06207205604909272,
    "kl": 0.10056160501177605,
    "tv": 0.16701407909774973,
    "ece": 0.14945683625768713,
    "score_mae": 0.2234294522603933,
    "within_one": 1.0,
    "n": 400,
    "accuracy_choice": 0.7,
    "accuracy_score": 0.60625,
    "accuracy_noul": 0.85,
    "n_unanswerable": 0,
    "runtime": 22.1624,
    "samples_per_second": 18.049,
    "steps_per_second": 0.587
  },
  "training": {
    "recorded": "2026-10-06T00:27:16+00:00",
    "code": {
      "git": null,
      "sysone_git": {
        "repo": "sysonelib",
        "commit": "123ad8bac647e2a14555881ea19e4a6a159b27e6",
        "branch": "main",
        "remote": "https://github.com/sgaseretto/sysonelib.git",
        "dirty": true,
        "changed": [
          "bs/01_template.ipynb",
          "nbs/02_data.ipynb",
          "nbs/03_models.ipynb",
          "nbs/04_losses.ipynb",
          "nbs/05_learner.ipynb",
          "nbs/06_inference.ipynb",
          "nbs/40_cloud.ipynb",
          "nbs/50_cli.ipynb",
          "nbs/fixtures/make_fixtures.py",
          "sysone/__init__.py",
          "sysone/_modidx.py",
          "sysone/cli.py",
          "sysone/cloud.py",
          "sysone/data.py",
          "sysone/inference.py",
          "sysone/learner.py",
          "sysone/losses.py",
          "sysone/models.py",
          "sysone/template.py",
          "nbs/24_mdlm.ipynb",
          "nbs/fixtures/tiny-a2d-qwen3/",
          "sysone/mdlm.py"
        ],
        "n_changed": 22,
        "diff_sha256": "77eb5a356ea0493da3c37d8984638a514afb742071229bdac7f2f56fc62a864e"
      },
      "entry": {
        "name": "run_b.py",
        "sha256": "64a038d242e574bd2a514812e9949b0ae3fa80217728556ee1ac623b4dfd9ae4"
      }
    },
    "environment": {
      "python": "3.13.15",
      "implementation": "CPython",
      "os": "Linux 6.18.48+",
      "packages": {
        "sysone": "0.3.0",
        "torch": "2.11.0+cu128",
        "transformers": "5.16.1",
        "peft": "0.20.0",
        "accelerate": "1.14.0",
        "datasets": "4.8.5",
        "huggingface_hub": "1.29.0",
        "tokenizers": "0.23.1",
        "safetensors": "0.8.0",
        "numpy": "2.1.3",
        "fastcore": "2.2.32",
        "plum-dispatch": "2.10.1"
      },
      "cuda": "12.8",
      "cudnn": 91900
    },
    "hardware": {
      "accelerator": {
        "kind": "cuda",
        "devices": [
          "Tesla T4",
          "Tesla T4"
        ],
        "memory_gb": [
          14.6,
          14.6
        ],
        "capability": "7.5"
      },
      "host": {
        "cpu": "Intel(R) Xeon(R) CPU @ 2.00GHz",
        "cores": 4,
        "memory_gb": 31.3,
        "system": "Linux",
        "machine": "x86_64"
      },
      "run": {
        "backend": "kaggle",
        "job": "mdlm-lora-check",
        "machine": "NvidiaTeslaT4"
      }
    },
    "data": {
      "source": {
        "kind": "hub",
        "id": "LocalLLaMA/typed-decisions",
        "config": "all",
        "split": "train",
        "valid": "test[:80]",
        "revision": null,
        "sha": "d0e2f0c42fef86cc15d1688d25a19f5ba7c85b18",
        "note": "read with datasets.load_dataset and passed as records; the dataset's last commit before the run"
      },
      "splits": {
        "train": {
          "cases": 1080,
          "fingerprint": "89a925d7f54eeaea"
        },
        "valid": {
          "cases": 80,
          "fingerprint": "0bff619936359396"
        },
        "calib": {
          "cases": 120,
          "fingerprint": "11352d26c9dc187b"
        }
      },
      "template": "jev",
      "transforms": [],
      "sampler": null,
      "seed": 0
    },
    "model": {
      "encoder": "dllm-hub/Qwen3-0.6B-diffusion-mdlm-v0.1",
      "regime": "lora",
      "lora": {
        "peft_type": "LORA",
        "r": 16,
        "lora_alpha": 32,
        "lora_dropout": 0.05,
        "target_modules": [
          "down_proj",
          "gate_proj",
          "k_proj",
          "o_proj",
          "q_proj",
          "up_proj",
          "v_proj"
        ],
        "bias": "none",
        "use_rslora": false,
        "use_dora": false,
        "modules_to_save": null
      },
      "params": {
        "encoder": 606142464,
        "head": 4097,
        "trainable": 10096641
      }
    },
    "training": {
      "preset": "t4",
      "loss": "soft_ce_rps=0.25",
      "lr": [
        0.0002,
        0.001
      ],
      "seed": 0,
      "args": {
        "lr_scheduler_type": "cosine",
        "warmup_steps": 0.1,
        "weight_decay": 0.01,
        "gradient_accumulation_steps": 4,
        "fp16": true,
        "gradient_checkpointing": true,
        "logging_steps": 10,
        "disable_tqdm": false,
        "report_to": [],
        "per_device_eval_batch_size": 16,
        "save_strategy": "no",
        "seed": 0,
        "accelerator_config": "AcceleratorConfig(split_batches=False, dispatch_batches=None, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False)",
        "remove_unused_columns": false,
        "label_names": [
          "target"
        ],
        "train_sampling_strategy": "group_by_length",
        "length_column_name": "n_tokens",
        "debug": []
      },
      "fits": [
        {
          "started": "2026-10-05T22:55:01+00:00",
          "regime": "lora",
          "epochs": 1,
          "max_steps": 300,
          "lr_head": 0.001,
          "lr_encoder": 0.0002,
          "schedule": "cosine",
          "warmup": 0.25,
          "batch_size": 8,
          "grad_accum": 4,
          "precision": "fp16",
          "device": "cuda",
          "steps": 300,
          "seconds": 5246.0,
          "final_loss": 0.8778909683227539,
          "peak_memory_gb": 7.69
        }
      ],
      "history": {
        "loss": [
          1.4344513893127442,
          1.249254322052002,
          1.1781773567199707,
          1.163340187072754,
          1.0988536834716798,
          1.0159993171691895,
          1.1186190605163575,
          0.9268903732299805,
          1.0303213119506835,
          0.8802448272705078,
          1.0308435440063477,
          0.87939453125,
          0.9772283554077148,
          0.9342096328735352,
          0.9400447845458985,
          0.9097446441650391,
          0.9203708648681641,
          0.9839879035949707,
          0.8999062538146972,
          0.896394920349121,
          0.8628653526306153,
          0.9104316711425782,
          0.8542628288269043,
          0.9121370315551758,
          0.772752046585083,
          0.8732522010803223,
          0.7740259647369385,
          0.9276440620422364,
          0.750361967086792,
          0.8778909683227539
        ],
        "eval": [
          {
            "epoch": 1.0,
            "loss": 1.09174382686615,
            "accuracy": 0.5425,
            "soft_accuracy": 0.4193413073577042,
            "brier": 0.141350766370905,
            "kl": 0.2354086200846709,
            "tv": 0.24462439980862877,
            "ece": 0.10866832427761491,
            "score_mae": 0.278731628334489,
            "within_one": 0.99375,
            "n": 400,
            "accuracy_choice": 0.38125,
            "accuracy_score": 0.60625,
            "accuracy_noul": 0.7375,
            "n_unanswerable": 0,
            "runtime": 22.1301,
            "samples_per_second": 18.075,
            "steps_per_second": 0.587
          },
          {
            "epoch": 2.0,
            "loss": 1.0160019397735596,
            "accuracy": 0.67,
            "soft_accuracy": 0.4682727722546358,
            "brier": 0.07573289376790472,
            "kl": 0.11924152387670522,
            "tv": 0.17498002941222868,
            "ece": 0.08067372087327374,
            "score_mae": 0.2732784988754318,
            "within_one": 0.99375,
            "n": 400,
            "accuracy_choice": 0.675,
            "accuracy_score": 0.6125,
            "accuracy_noul": 0.775,
            "n_unanswerable": 0,
            "runtime": 22.208,
            "samples_per_second": 18.012,
            "steps_per_second": 0.585
          },
          {
            "epoch": 3.0,
            "loss": 0.9821739792823792,
            "accuracy": 0.675,
            "soft_accuracy": 0.4534864358013278,
            "brier": 0.06318044447460737,
            "kl": 0.10197306195815968,
            "tv": 0.1695142981192557,
            "ece": 0.1388826772323706,
            "score_mae": 0.2302615860129519,
            "within_one": 1.0,
            "n": 400,
            "accuracy_choice": 0.66875,
            "accuracy_score": 0.59375,
            "accuracy_noul": 0.85,
            "n_unanswerable": 0,
            "runtime": 21.9708,
            "samples_per_second": 18.206,
            "steps_per_second": 0.592
          },
          {
            "epoch": 3.532544378698225,
            "loss": 0.9801202416419983,
            "accuracy": 0.6925,
            "soft_accuracy": 0.45718836379426847,
            "brier": 0.06207205604909272,
            "kl": 0.10056160501177605,
            "tv": 0.16701407909774973,
            "ece": 0.14945683625768713,
            "score_mae": 0.2234294522603933,
            "within_one": 1.0,
            "n": 400,
            "accuracy_choice": 0.7,
            "accuracy_score": 0.60625,
            "accuracy_noul": 0.85,
            "n_unanswerable": 0,
            "runtime": 22.1624,
            "samples_per_second": 18.049,
            "steps_per_second": 0.587
          }
        ]
      }
    },
    "tracking": null
  },
  "evaluations": [
    {
      "split": "test",
      "dataset": "LocalLLaMA/typed-decisions",
      "config": "all",
      "n": 2000,
      "metrics": {
        "accuracy": 0.7675,
        "accuracy_choice": 0.7517,
        "accuracy_score": 0.7238,
        "accuracy_noul": 0.8417,
        "ece": 0.1496,
        "brier": 0.0536
      },
      "note": "calibrated, on a Kaggle T4"
    }
  ]
}