code/scripts/eval.py · viai957/align at main

Add files using upload-large-folder tool

bcc8d35 verified 16 days ago

3.92 kB

"""Thin eval entry: single / paired / golden.
    python scripts/eval.py configs/eval.toml --mode=single --model.path=...
ORDER OF OPERATIONS IS LOAD-BEARING
-----------------------------------
1. `apply_resource_guards` runs ONLY when we are not inside a multiprocessing
   child. The verifier's worker pool uses 'spawn', which RE-IMPORTS this
   module in every worker; a resource guard or an exclusive flock at module
   scope therefore executes in each child and kills it on startup. Every
   future then dies with BrokenProcessPool -- which is precisely the
   documented cause of the "all answers incorrect" masquerade the verifier
   raises on. The guard is the fix; `_in_multiprocessing_child()` is what
   makes it safe.
2. Banner next: resolved config, dataset hashes, MDE, model hash, backend.
   A run that cannot answer its own question does not begin.
3. Then dispatch.
"""

from __future__ import annotations

import argparse
import os
import sys

REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if REPO_ROOT not in sys.path:
    sys.path.insert(0, REPO_ROOT)

from align.utils.guard import _in_multiprocessing_child  # noqa: E402

# See docstring point 1. This condition is not an optimization.
if not _in_multiprocessing_child():
    from align.utils.guard import apply_resource_guards

    apply_resource_guards(cpu_threads=8, heavy=False, label="eval")


def build_parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(description="nmo evaluation entry")
    p.add_argument("config", nargs="?", default="configs/eval.toml")
    p.add_argument("--mode", choices=("single", "paired", "golden"), default="single")
    p.add_argument(
        "--out", default=None, help="report path; defaults to outputs/eval/ under the repo"
    )
    p.add_argument(
        "overrides", nargs="*", default=[], help="dotted overrides, e.g. eval.limit=8"
    )
    return p


def default_out_dir() -> str:
    """Reports land under /apps -- asserted, never assumed."""
    from align.utils.guard import assert_paths_off_root

    out = os.path.join(REPO_ROOT, "outputs", "eval")
    os.makedirs(out, exist_ok=True)
    assert_paths_off_root([out])
    return out


def main(argv=None) -> int:
    args = build_parser().parse_args(argv)

    from align.config import load_toml

    # Schemaless on purpose (eval.toml has no dataclass): plain nested dict,
    # dotted overrides TOML-literal parsed. Same shape OmegaConf.to_object gave.
    cfg = load_toml(args.config, list(args.overrides))
    out_dir = args.out or default_out_dir()

    if args.mode == "golden":
        from align.eval.golden import run_golden_regression

        banner: dict = {}
        path = os.path.join(out_dir, "checkpoint-200-golden.json")
        # `run_golden_regression` writes the report ITSELF, before gating, so
        # a failing gate still leaves the evidence. Writing again here would
        # be dead code on the path that matters: the gate raises, and anything
        # after this call never runs.
        try:
            run_golden_regression(banner=banner, out_path=path)
        finally:
            # U01-9: name the eval source that produced the report (also when the gate raised), so
            # `record_golden_from_report` cannot re-certify it after align/eval/ changes.
            if os.path.isfile(path):
                from align.train.stage_gates import stamp_golden_report

                stamp_golden_report(path)
        print(f"golden gate PASSED; report at {path}")
        return 0

    raise SystemExit(
        f"mode {args.mode!r} needs a generation backend, which arrives "
        "with section-04 (nmo model) / section-09 (engine). The harness "
        "itself is complete and tested: run_eval takes a generate_fn. "
        "Use --mode=golden, which uses plain HF generation by design."
    )


if __name__ == "__main__":
    raise SystemExit(main())

	"""Thin eval entry: single / paired / golden.

	python scripts/eval.py configs/eval.toml --mode=single --model.path=...

	ORDER OF OPERATIONS IS LOAD-BEARING
	-----------------------------------
	1. `apply_resource_guards` runs ONLY when we are not inside a multiprocessing
	child. The verifier's worker pool uses 'spawn', which RE-IMPORTS this
	module in every worker; a resource guard or an exclusive flock at module
	scope therefore executes in each child and kills it on startup. Every
	future then dies with BrokenProcessPool -- which is precisely the
	documented cause of the "all answers incorrect" masquerade the verifier
	raises on. The guard is the fix; `_in_multiprocessing_child()` is what
	makes it safe.
	2. Banner next: resolved config, dataset hashes, MDE, model hash, backend.
	A run that cannot answer its own question does not begin.
	3. Then dispatch.
	"""

	from __future__ import annotations

	import argparse
	import os
	import sys

	REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
	if REPO_ROOT not in sys.path:
	sys.path.insert(0, REPO_ROOT)

	from align.utils.guard import _in_multiprocessing_child # noqa: E402

	# See docstring point 1. This condition is not an optimization.
	if not _in_multiprocessing_child():
	from align.utils.guard import apply_resource_guards

	apply_resource_guards(cpu_threads=8, heavy=False, label="eval")


	def build_parser() -> argparse.ArgumentParser:
	p = argparse.ArgumentParser(description="nmo evaluation entry")
	p.add_argument("config", nargs="?", default="configs/eval.toml")
	p.add_argument("--mode", choices=("single", "paired", "golden"), default="single")
	p.add_argument(
	"--out", default=None, help="report path; defaults to outputs/eval/ under the repo"
	)
	p.add_argument(
	"overrides", nargs="*", default=[], help="dotted overrides, e.g. eval.limit=8"
	)
	return p


	def default_out_dir() -> str:
	"""Reports land under /apps -- asserted, never assumed."""
	from align.utils.guard import assert_paths_off_root

	out = os.path.join(REPO_ROOT, "outputs", "eval")
	os.makedirs(out, exist_ok=True)
	assert_paths_off_root([out])
	return out


	def main(argv=None) -> int:
	args = build_parser().parse_args(argv)

	from align.config import load_toml

	# Schemaless on purpose (eval.toml has no dataclass): plain nested dict,
	# dotted overrides TOML-literal parsed. Same shape OmegaConf.to_object gave.
	cfg = load_toml(args.config, list(args.overrides))
	out_dir = args.out or default_out_dir()

	if args.mode == "golden":
	from align.eval.golden import run_golden_regression

	banner: dict = {}
	path = os.path.join(out_dir, "checkpoint-200-golden.json")
	# `run_golden_regression` writes the report ITSELF, before gating, so
	# a failing gate still leaves the evidence. Writing again here would
	# be dead code on the path that matters: the gate raises, and anything
	# after this call never runs.
	try:
	run_golden_regression(banner=banner, out_path=path)
	finally:
	# U01-9: name the eval source that produced the report (also when the gate raised), so
	# `record_golden_from_report` cannot re-certify it after align/eval/ changes.
	if os.path.isfile(path):
	from align.train.stage_gates import stamp_golden_report

	stamp_golden_report(path)
	print(f"golden gate PASSED; report at {path}")
	return 0

	raise SystemExit(
	f"mode {args.mode!r} needs a generation backend, which arrives "
	"with section-04 (nmo model) / section-09 (engine). The harness "
	"itself is complete and tested: run_eval takes a generate_fn. "
	"Use --mode=golden, which uses plain HF generation by design."
	)


	if __name__ == "__main__":
	raise SystemExit(main())