Update chatcode/train.py

chatcode/train.py

@@ -13,7 +13,11 @@ def read_data(folder):
     manifest = json.loads((folder / "manifest.json").read_text(encoding="utf-8"))
     if manifest["config"] != CONFIG:
         raise ValueError("Data manifest does not match the pinned experiment configuration.")
-    records, seen = {}, set()
+    if "article_examples_sha256" in manifest:
+        current = hashlib.sha256(Path(__file__).with_name("article_examples.json").read_bytes()).hexdigest()
+        if manifest["article_examples_sha256"] != current:
+            raise ValueError("Article behavior examples changed after preparation.")
+    records, seen, groups = {}, set(), {}
     for split in ["train", "validation", "test"]:
         raw = (folder / f"{split}.jsonl").read_bytes()
         if hashlib.sha256(raw).hexdigest() != manifest["sha256"][split]:
@@ -26,6 +30,10 @@ def read_data(folder):
             if key != row["prompt_hash"] or key in seen:
                 raise ValueError("Duplicate or inconsistent prompts across dataset splits.")
             seen.add(key)
+            group = row.get("group_id", key)
+            if group in groups and groups[group] != split:
+                raise ValueError("A source packet appears in multiple dataset splits.")
+            groups[group] = split
     return records
 
 
@@ -92,16 +100,16 @@ def train(args):
     shutil.copy2(Path(args.data) / "manifest.json", destination / "data_manifest.json")
     report = {"config": CONFIG, "baseline_validation": baseline, "final_validation": final,
               "training": trained.metrics, "counts": {k: len(v) for k, v in records.items()},
-              "status": "candidate_requires_answer_and_code_evaluation", "max_steps": args.max_steps}
+              "status": "candidate_requires_chat_code_and_article_evaluation", "max_steps": args.max_steps}
     (destination / "training_report.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
     (destination / "README.md").write_text(
         "---\nlicense: gpl-3.0\nlibrary_name: peft\npipeline_tag: text-generation\n"
         f"base_model: {CONFIG['base_model']}\nbase_model_relation: adapter\n"
         "datasets:\n- HuggingFaceTB/smoltalk\ntags:\n- lora\n- code\n- experimental\n---\n\n"
-        "# PocketVision Chat & Code candidate\n\nText-only QLoRA adapter. "
-        "This candidate has validation-loss measurements only; no claim of better chat or coding accuracy. "
+        "# PocketVision Chat, Code & Writing candidate\n\nText-only QLoRA adapter. "
+        "This candidate has validation-loss measurements only; no claim of better chat, coding or factual article quality. "
         "See training_report.json and data_manifest.json. "
-        "Training data: Smol-Magpie-Ultra (Apache-2.0) and the Self-OSS-Instruct subset "
+        "Training data: Smol-Magpie-Ultra, Smol-rewrite, Smol-summarize (Apache-2.0), authored fictional article behavior examples (GPL-3.0-only), and the Self-OSS-Instruct subset "
         "(original: bigcode/self-oss-instruct-sc2-exec-filter-50k, ODC-BY). "
         "Original base weights remain Apache-2.0; project code and adapter use GPL-3.0-only.\n",
         encoding="utf-8")