| import json |
| from pathlib import Path |
| from typing import Any |
|
|
| from br_sovereign_llm.cli import main |
|
|
|
|
| def test_validate_sources_command(capsys: Any) -> None: |
| assert main(["validate-sources"]) == 0 |
| output = json.loads(capsys.readouterr().out) |
|
|
| assert output["ok"] is True |
| assert output["source_count"] >= 10 |
| assert "wikimedia_ptwiki" in output["scientific_source_ids"] |
|
|
|
|
| def test_validate_project_command_covers_all_registries(capsys: Any) -> None: |
| assert main(["validate-project"]) == 0 |
| output = json.loads(capsys.readouterr().out) |
|
|
| assert output["ok"] is True |
| assert output["benchmarks"]["benchmark_count"] == 10 |
| assert output["configurations"]["configuration_count"] == 19 |
| assert output["tokenizer"]["vocabulary_size"] == 32768 |
| assert len(output["model_ladder"]) == 6 |
|
|
|
|
| def test_project_root_allows_installed_cli_from_another_directory( |
| tmp_path: Path, capsys: Any, monkeypatch: Any |
| ) -> None: |
| project_root = Path(__file__).resolve().parents[2] |
| monkeypatch.chdir(tmp_path) |
|
|
| assert main(["--project-root", str(project_root), "validate-project"]) == 0 |
| output = json.loads(capsys.readouterr().out) |
| assert output["configurations"]["configuration_count"] == 19 |
|
|
|
|
| def test_cli_reports_invalid_registry_without_traceback( |
| tmp_path: Path, capsys: Any |
| ) -> None: |
| missing = tmp_path / "missing.yaml" |
|
|
| assert main(["validate-benchmarks", "--registry", str(missing)]) == 2 |
| captured = capsys.readouterr() |
| error = json.loads(captured.err) |
|
|
| assert captured.out == "" |
| assert error["ok"] is False |
| assert error["error_type"] == "BenchmarkRegistryError" |
|
|
|
|
| def test_template_fingerprint_discloses_unresolved_inputs(capsys: Any) -> None: |
| assert ( |
| main( |
| [ |
| "fingerprint", |
| "--study", |
| "configs/experiments/scaling-study.yaml", |
| "--runtime", |
| "configs/runtime/local-cpu.yaml", |
| ] |
| ) |
| == 0 |
| ) |
| output = json.loads(capsys.readouterr().out) |
|
|
| assert output["fingerprint_kind"] == "configuration_template" |
| assert output["scientific_fingerprint_ready"] is False |
| assert "$" + "{TOKENIZER_ID}" in output["study_unresolved_placeholders"] |
|
|
|
|
| def test_evidence_report_keeps_every_metric_unmeasured( |
| tmp_path: Path, capsys: Any |
| ) -> None: |
| output_path = tmp_path / "evidence.json" |
|
|
| assert main(["evidence-report", "--output", str(output_path)]) == 0 |
| command_output = json.loads(capsys.readouterr().out) |
| report = json.loads(output_path.read_text(encoding="utf-8")) |
|
|
| assert command_output["eligible_for_public_claims"] is False |
| assert report["eligible_for_public_claims"] is False |
| assert all(item["state"] == "not_measured" for item in report["evidence"]) |
|
|
|
|
| def test_tokenizer_manifest_uses_content_addressed_input_identifier( |
| tmp_path: Path, capsys: Any |
| ) -> None: |
| assert ( |
| main( |
| [ |
| "train-tokenizer", |
| "--output", |
| str(tmp_path / "tokenizer"), |
| ] |
| ) |
| == 0 |
| ) |
| capsys.readouterr() |
| manifest_text = (tmp_path / "tokenizer/tokenizer-manifest.json").read_text( |
| encoding="utf-8" |
| ) |
| manifest = json.loads(manifest_text) |
|
|
| assert set(manifest["input_hashes"]) == { |
| "urn:sha256:6896439e772f212c2848846f1f908410a606d81cd8845583024c16496a07979f" |
| } |
| assert "/Users/" not in manifest_text |
|
|
