File size: 7,278 Bytes
788274d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | """`uvw` command line: environment, weight hashing and frozen-config generation."""
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import asdict
from pathlib import Path
from uvw import RUNS
def _cmd_env(args: argparse.Namespace) -> int:
from uvw.provenance import environment
print(json.dumps(environment(), indent=2))
return 0
def _cmd_hash(args: argparse.Namespace) -> int:
from uvw.models import load_pipeline, load_registry, weight_hashes
variant = load_registry()[args.variant]
print(json.dumps({"variant": variant.name, **weight_hashes(load_pipeline(variant))}, indent=2))
return 0
def _cmd_generate(args: argparse.Namespace) -> int:
"""Generate for a JSONL file of {prompt_id, prompt, seed}; save images and a records file."""
from uvw.generate import GenItem, generate, load_inference_config
from uvw.models import load_pipeline, load_registry, weight_hashes
from uvw.provenance import RunRecord, config_hash
cfg = load_inference_config()
if args.steps:
cfg["num_inference_steps"] = args.steps
variant = load_registry()[args.variant]
rows = [json.loads(line) for line in Path(args.prompts).read_text().splitlines() if line.strip()]
if args.limit:
rows = rows[: args.limit]
record = RunRecord(job="generate", variants=[variant.name], configs={"inference": config_hash(cfg)})
out = Path(args.out or RUNS / record.run_id)
(out / "images").mkdir(parents=True, exist_ok=True)
pipe = load_pipeline(variant)
record.weights = {f"{variant.name}/{k}": v for k, v in weight_hashes(pipe).items()}
items = [GenItem(r["prompt_id"], r.get("text") or r["prompt"], int(r["seed"])) for r in rows]
with open(out / "generations.jsonl", "w") as f:
for g in generate(pipe, items, cfg, record.cost):
name = f"{g.item.prompt_id}_s{g.item.seed}.png"
if not args.no_images:
g.image.save(out / "images" / name)
f.write(json.dumps({**asdict(g.item), "image": name, "image_sha256": g.image_sha256,
"config_hash": g.config_hash, "variant": variant.name}) + "\n")
record.outputs = [str(out / "generations.jsonl")]
record.finish()
print(record.write(out))
return 0
def _cmd_splits(args: argparse.Namespace) -> int:
"""Build sources → template-level splits → leakage report → data/splits/."""
from uvw import ROOT
from uvw.data import leakage
from uvw.data.sources import build_all
from uvw.data.splits import assign, write_splits
out = Path(args.out or ROOT / "data" / "splits")
out.mkdir(parents=True, exist_ok=True)
df, excluded = build_all()
emb = leakage.clip_text_embeddings(df.text.tolist())
df["group_id"] = leakage.group_templates(df, emb, threshold=args.threshold).values
df["_row"] = range(len(df))
df = assign(df)
emb, df = emb[df._row.to_numpy()], df.drop(columns="_row")
overlap = leakage.template_overlap(df)
exact = leakage.exact_duplicates(df)
near = leakage.near_duplicates(df, emb, threshold=args.threshold)
near.to_csv(out / "near_duplicates.csv", index=False)
report = {"rows": len(df), "excluded": len(excluded), "template_overlap": len(overlap),
"exact_cross_split_duplicates": len(exact), "near_cross_split_duplicates": len(near),
"threshold": args.threshold}
if len(exact) or len(near):
raise SystemExit(f"Cross-split duplicates remain: {report}")
if len(overlap):
print(overlap.head())
raise SystemExit("Template overlap across splits: refusing to write.")
manifest = write_splits(df, out)
(out / "excluded.jsonl").write_text("".join(json.dumps(e) + "\n" for e in excluded))
exact.to_csv(out / "exact_duplicates.csv", index=False)
(out / "leakage_report.json").write_text(json.dumps(report, indent=2))
print(json.dumps(report, indent=2))
for name, info in sorted(manifest["files"].items()):
print(f"{name:28s} rows={info['rows']:6d} templates={info['templates']:5d} {info['by_role']}")
return 0
def _cmd_evaluate(args: argparse.Namespace) -> int:
from uvw.evaluate import evaluate
for concept in args.concepts:
out = evaluate(args.variant, concept, args.split, n_seeds=args.seeds, limit=args.limit,
out=Path(args.out) / f"{args.variant}-{concept}-{args.split}" if args.out else None,
keep_images=args.keep_images, edits=Path(args.edits) if args.edits else None)
print(out)
return 0
def _cmd_edit_agent(args: argparse.Namespace) -> int:
"""Run the editor agent locally (ChatGPT plan via atbot) against a remote tool server."""
from uvw import RUNS
from uvw.harness.agent import run_editor
from uvw.harness.server import RemoteSession
session = RemoteSession(args.url, RUNS / "edits-local" / args.run_id)
summary = run_editor(session, editor_model=args.editor, model_name=args.model_name)
print(json.dumps(summary, indent=2))
return 0
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="uvw")
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("env", help="Print the environment record")
p = sub.add_parser("hash", help="Hash a variant's weights")
p.add_argument("variant")
p = sub.add_parser("generate", help="Generate images with the frozen inference config")
p.add_argument("variant")
p.add_argument("prompts", help="JSONL with prompt_id, text (or prompt), seed")
p.add_argument("--out")
p.add_argument("--limit", type=int)
p.add_argument("--steps", type=int, help="Override steps (smoke tests only)")
p.add_argument("--no-images", action="store_true", help="Keep hashes and scores only")
p = sub.add_parser("splits", help="Build frozen dev/val/test prompt splits")
p.add_argument("--out")
p.add_argument("--threshold", type=float, default=0.95)
p = sub.add_parser("evaluate", help="Generate + score a variant on concept splits")
p.add_argument("variant")
p.add_argument("concepts", nargs="+", choices=["nudity", "vangogh", "church", "coco"])
p.add_argument("--split", default="dev", choices=["dev", "val"]) # test: evaluator only (003)
p.add_argument("--seeds", type=int, default=1)
p.add_argument("--limit", type=int, help="Max prompts per role (pilots)")
p.add_argument("--out")
p.add_argument("--keep-images", default="none", choices=["none", "benign", "all"])
p.add_argument("--edits", help="edited_tensors.safetensors from a committed candidate")
p = sub.add_parser("edit-agent", help="Run the editor agent against a tool server")
p.add_argument("--url", default="http://127.0.0.1:8765")
p.add_argument("--run-id", required=True)
p.add_argument("--editor", default="gpt-6-astra")
p.add_argument("--model-name", default="Stable Diffusion v1.4")
args = parser.parse_args(argv)
commands = {"edit-agent": _cmd_edit_agent, "evaluate": _cmd_evaluate, "env": _cmd_env, "hash": _cmd_hash, "generate": _cmd_generate, "splits": _cmd_splits}
return commands[args.command](args)
if __name__ == "__main__":
sys.exit(main())
|