beno-rohman's picture
Publish WDC product embedding checkpoint, training source, and evaluation
a360597 verified
Raw History Blame Contribute Delete
4.67 kB
"""Explicit execution gate for resource-intensive pipeline commands."""
import argparse
import json
from pathlib import Path
def load_config(path):
path = Path(path)
config = json.loads(path.read_text(encoding="utf-8"))
if isinstance(config, dict) and "extends" in config:
parent = load_config(path.parent / config.pop("extends"))
config = {**parent, **{key: {**parent[key], **value} if isinstance(value, dict) and isinstance(parent.get(key), dict) else value for key, value in config.items()}}
if not isinstance(config, dict) or isinstance(config.get("seed"), bool) or not isinstance(config.get("seed"), int):
raise ValueError("Config must be an object with an integer seed")
for section in ("prepare", "pairs", "evaluate", "train", "search"):
if not isinstance(config.get(section), dict):
raise ValueError(f"Missing config section: {section}")
positive = {
"prepare": ["batch_size", "threads"],
"pairs": ["max_train_pairs", "max_queries", "max_corpus", "threads"],
"evaluate": ["batch_size", "query_chunk_size", "max_seq_length"],
"train": ["max_pairs", "batch_size", "mini_batch_size", "max_seq_length", "epochs", "learning_rate"],
"search": ["batch_size", "max_seq_length", "top_k", "max_candidates"],
}
for section, fields in positive.items():
for field in fields:
value = config[section][field]
if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
raise ValueError(f"{section}.{field} must be positive")
if field not in {"epochs", "learning_rate"} and not isinstance(value, int):
raise ValueError(f"{section}.{field} must be an integer")
if config["prepare"]["max_rows"] is not None and (not isinstance(config["prepare"]["max_rows"], int) or config["prepare"]["max_rows"] <= 0):
raise ValueError("prepare.max_rows must be null or a positive integer")
if config["prepare"]["description_characters"] < 0:
raise ValueError("description_characters must be nonnegative")
if config["pairs"]["max_corpus"] < config["pairs"]["max_queries"]:
raise ValueError("max_corpus must be >= max_queries")
if not isinstance(config["pairs"]["require_shared_gtin"], bool):
raise ValueError("require_shared_gtin must be a boolean")
if config["evaluate"]["backend"] not in {"lexical", "embedding"}:
raise ValueError("Evaluation backend must be lexical or embedding")
if not 0 <= config["train"]["warmup_ratio"] <= 1:
raise ValueError("warmup_ratio must be between 0 and 1")
if config["train"]["mini_batch_size"] > config["train"]["batch_size"]:
raise ValueError("mini_batch_size must not exceed batch_size")
return config
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("command", choices=("prepare", "pairs", "evaluate", "train", "search"))
parser.add_argument("--config", type=Path, default=Path("configs/local.json"))
parser.add_argument("--execute", action="store_true", help="Explicitly authorize this resource-intensive stage")
parser.add_argument("--allow-download", action="store_true", help="Allow model downloads for this execution (default: cached/local files only)")
args = parser.parse_args(argv)
try:
config = load_config(args.config)
section = config[args.command]
if not args.execute:
print(json.dumps({"status": "dry_run", "command": args.command, "seed": config["seed"], "config": section, "note": "No corpus scan, optional-library import, model download, or GPU work performed. Add --execute only when resources are available."}, indent=2))
return
if args.command == "prepare":
from .prepare import prepare
result = prepare(section, config["seed"])
elif args.command == "pairs":
from .pairs import build_datasets
result = build_datasets(section, config["seed"])
elif args.command == "evaluate":
from .evaluate import evaluate
result = evaluate(section, args.allow_download)
elif args.command == "train":
from .train import train
result = train(section, config["seed"], args.allow_download)
else:
from .search import search
result = search(section, args.allow_download)
print(json.dumps(result, indent=2))
except (ValueError, OSError, KeyError, ImportError, RuntimeError) as exc:
parser.exit(1, f"Error: {exc}\n")
if __name__ == "__main__":
main()