Update chatcode/data.py

chatcode/data.py

@@ -8,6 +8,7 @@ import random
 import re
 
 from .runtime import CONFIG
+from .articles import behavior_examples
 
 
 def normalize_messages(row):
@@ -111,6 +112,19 @@ def prepare(args):
             raise RuntimeError(f"Not enough usable data for {source['config']}: {dict(counts)}")
         stats[source["config"]] = {"selected": dict(counts), "skipped": dict(skipped)}
         print(source["config"], dict(counts), flush=True)
+    behavior_path = Path(__file__).with_name("article_examples.json")
+    for split, rows in behavior_examples().items():
+        for row in rows:
+            encode_example(tokenizer, row["messages"], CONFIG["max_sequence_length"])
+            key = prompt_key(row["messages"])
+            if key in seen:
+                raise ValueError("Article behavior prompt duplicates a selected Hub prompt.")
+            seen.add(key)
+            row.update({"id": hashlib.sha256(json.dumps(row["messages"], sort_keys=True).encode()).hexdigest(),
+                        "prompt_hash": key})
+            result[split].append(row)
+    stats["authored-article-behavior"] = {"selected": {k: len(v) for k, v in behavior_examples().items()},
+                                          "license": "GPL-3.0-only", "fictional": True}
     output.mkdir(parents=True, exist_ok=True)
     hashes = {}
     for split, rows in result.items():
@@ -119,8 +133,9 @@ def prepare(args):
         (output / f"{split}.jsonl").write_bytes(content.encode("utf-8"))
         hashes[split] = hashlib.sha256(content.encode()).hexdigest()
     manifest = {"config": CONFIG, "counts": {k: len(v) for k, v in result.items()},
+                "article_examples_sha256": hashlib.sha256(behavior_path.read_bytes()).hexdigest(),
                 "sources": stats, "sha256": hashes,
-                "split_policy": "Hash of normalized first user question; official source test splits unused.",
+                "split_policy": "Hub: hash of normalized first user question. Article behavior: source-packet groups. Official source test splits unused.",
                 "limitations": ["Synthetic data may contain mistakes.",
                     "Exact prompt deduplication is not semantic or repository-level decontamination.",
                     "The pretrained base may already have seen these data.",