File size: 5,840 Bytes
031e289 d5cc31c 031e289 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 | """Stage a verified public release using an explicit allowlist, without credentials."""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import shutil
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from pathlib import Path
from codelock_affect.vectors import VectorBank
ROOT_FILES = (
"README.md",
"LICENSE",
"NOTICE",
"pyproject.toml",
"requirements.txt",
"Makefile",
"MANIFEST.in",
".gitignore",
".gitattributes",
"adapter-release.json",
"adapter_config.json",
"adapter_model.safetensors",
"added_tokens.json",
"chat_template.jinja",
"merges.txt",
"special_tokens_map.json",
"tokenizer.json",
"tokenizer_config.json",
"vocab.json",
)
PUBLIC_DIRS = (
"codelock_affect",
"scripts",
"tests",
"configs",
"data",
"docs",
"vectors",
"results",
)
SUFFIXES = {".py", ".json", ".jsonl", ".md", ".txt", ".sh", ".safetensors", ".xml"}
def release_files(root):
paths = [root / name for name in ROOT_FILES if (root / name).is_file()]
for directory in PUBLIC_DIRS:
paths.extend(
path
for path in (root / directory).rglob("*")
if path.is_file()
and path.suffix in SUFFIXES
and not any(
part.startswith(".") or part == "__pycache__"
for part in path.relative_to(root).parts
)
)
return sorted(paths, key=lambda path: path.relative_to(root).as_posix())
def build_release(root, output=None):
root = Path(root).resolve()
bank = VectorBank.load(root / "vectors/multichannel")
if set(bank.vectors) != {"pain", "fear", "anxiety", "relief"}:
raise ValueError("The v0.2 release requires all four channels")
data_hash = hashlib.sha256(
(root / "data/affect_pairs.jsonl").read_bytes()
).hexdigest()
if any(
vector.meta.get("dataset_file_sha256") != data_hash
for vector in bank.vectors.values()
):
raise ValueError("Calibration vectors do not match the current dataset")
suites = ET.parse(root / "results/v2/tests.xml").getroot().iter("testsuite")
counts = [dict(suite.attrib) for suite in suites]
if not counts or any(
int(suite.get("failures", 0)) or int(suite.get("errors", 0)) for suite in counts
):
raise ValueError("A passing test report is required")
replay = json.loads((root / "results/v2/replay.json").read_text(encoding="utf-8"))
if not replay.get("complete") or any(
"generation_context" not in row for row in replay.get("steps", [])
):
raise ValueError("The event-grounded replay is incomplete")
evaluation = json.loads(
(root / "results/v2/evaluation.json").read_text(encoding="utf-8")
)
if not evaluation.get("complete"):
raise ValueError("Calibration evaluation must be complete")
for seed in (7, 23, 41):
result = json.loads(
(root / f"results/v2/recommended-seed{seed}.json").read_text(
encoding="utf-8"
)
)
if not result.get("complete"):
raise ValueError(f"Recommended evaluation seed {seed} is incomplete")
stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
output = (
Path(output).resolve()
if output
else root / ".local" / "release-staging" / stamp
)
if output.exists() and any(output.iterdir()):
raise ValueError(
"Release output must be empty; an earlier bundle will not be overwritten"
)
# Keep generated files outside the public allowlist to avoid recursive staging.
if output.is_relative_to(root) and output.relative_to(root).parts[0] in PUBLIC_DIRS:
raise ValueError("Release staging cannot be inside a public source directory")
output.mkdir(parents=True, exist_ok=True)
manifest = {
"version": "0.2.0",
"created_utc": datetime.now(timezone.utc).isoformat(),
"base_model": "Qwen/Qwen3-8B",
"contains_base_weights": False,
"files": {},
}
token_pattern = re.compile(
r"(?:hf_[A-Za-z0-9]{20,}|tsk_[A-Za-z0-9_-]{20,}|-----BEGIN (?:OPENSSH |RSA |EC )?PRIVATE KEY-----)"
)
for path in release_files(root):
if path.is_symlink() or not path.resolve().is_relative_to(root):
raise ValueError(
f"Public file must remain inside the source directory: {path.name}"
)
payload = path.read_bytes()
if path.suffix != ".safetensors" and token_pattern.search(
payload.decode("utf-8", errors="replace")
):
raise ValueError(
f"Credential-like content found in {path.relative_to(root)}; refusing publication"
)
relative = path.relative_to(root)
target = output / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(path, target)
manifest["files"][relative.as_posix()] = {
"sha256": hashlib.sha256(payload).hexdigest(),
"bytes": len(payload),
}
(output / "release-manifest.json").write_bytes(
(json.dumps(manifest, indent=2) + "\n").encode("utf-8")
)
return output, manifest
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--output")
args = p.parse_args()
root = Path(__file__).resolve().parents[1]
output, manifest = build_release(root, args.output)
print(
json.dumps(
{
"staging": str(output),
"files": len(manifest["files"]),
"bytes": sum(row["bytes"] for row in manifest["files"].values()),
},
indent=2,
)
)
if __name__ == "__main__":
main()
|