Blame for Makefile

PYTHON ?= /usr/local/opt/python@3.11/bin/python3.11

.PHONY: install test lint serve ingest ingest-k8s evaluate-fast evaluate-full benchmark evaluate-langchain calibrate evaluate-judges stats-table epochs epochs-dry-run epochs-dry-run-k8s evaluate-stats plots preflight-deploy canary-report docker modal-deploy modal-stop vllm-up benchmark-all k8s-dev k8s-prod tf-plan tf-validate

install:
	$(PYTHON) -m pip install -e ".[dev]"

test:
	$(PYTHON) -m pytest tests/ -v --tb=short

lint:
	ruff check agent_bench/ stats/ stats_adapters/ tests/ scripts/run_epochs.py scripts/run_canary_eval.py scripts/make_plots.py
	ruff format --check stats/ stats_adapters/ tests/stats/ scripts/run_epochs.py scripts/run_canary_eval.py scripts/make_plots.py
	mypy agent_bench/ stats/ stats_adapters/ scripts/run_epochs.py scripts/run_canary_eval.py scripts/make_plots.py --ignore-missing-imports

serve:
	$(PYTHON) -m uvicorn agent_bench.serving.app:create_app --factory --reload --port 8000

ingest:
	$(PYTHON) scripts/ingest.py --config configs/tasks/tech_docs.yaml

ingest-k8s:  ## Ingest Kubernetes docs into .cache/store_k8s
	$(PYTHON) scripts/ingest.py --doc-dir data/k8s_docs --store-path .cache/store_k8s

evaluate-fast:
	$(PYTHON) scripts/evaluate.py --config configs/default.yaml --mode deterministic

evaluate-full:
	$(PYTHON) scripts/evaluate.py --config configs/default.yaml --mode full

benchmark:
	$(PYTHON) scripts/benchmark.py --output docs/benchmark_report.md

evaluate-langchain:
	$(PYTHON) scripts/run_langchain_eval.py --provider openai

calibrate:  ## Run full calibration pipeline (system outputs → all rows → strict κ table). Costs ~$2 in API calls.
	$(PYTHON) scripts/run_calibration.py generate-outputs
	@for cfg in configs/calibration/rows/*.yaml; do \
		echo "==> running judges for $$cfg"; \
		$(PYTHON) scripts/run_calibration.py run-judges --row-config=$$cfg || exit 1; \
	done
	$(PYTHON) scripts/run_calibration.py build-table --strict

evaluate-judges:  ## Re-run all rows + build-table against existing system_outputs (no regeneration). Costs ~$1.
	@for cfg in configs/calibration/rows/*.yaml; do \
		echo "==> running judges for $$cfg"; \
		$(PYTHON) scripts/run_calibration.py run-judges --row-config=$$cfg || exit 1; \
	done
	$(PYTHON) scripts/run_calibration.py build-table --strict

stats-table:  ## Convert legacy results JSON to validated long CSV (free, offline)
	$(PYTHON) -m stats_adapters.from_results_json --legacy \
		--input results/fastapi_postedit.json \
		--golden agent_bench/evaluation/datasets/tech_docs_golden.json \
		--config-id custom-openai-legacy \
		--out-dir results/long

epochs:  ## PAID, HUMAN-RUN: repeat eval k times per config. Usage: make epochs K=5 CONFIGS=custom-openai,custom-anthropic CONFIRM_PAID=1
	@test "$(CONFIRM_PAID)" = "1" || (echo "Refusing: paid target. Set CONFIRM_PAID=1 to run. Costs real API money." && exit 1)
	$(PYTHON) scripts/run_epochs.py --k $(K) --configs $(CONFIGS) --allow-paid

epochs-dry-run:  ## Free pre-spend gate: validate the WP5 configs (corpora, golden, store, keys) with no API calls
	$(PYTHON) scripts/run_epochs.py --k 1 --dry-run \
		--configs custom-openai,custom-anthropic,langchain-openai,langchain-anthropic

epochs-dry-run-k8s:  ## Free pre-spend gate for the UNMEASURED k8s configs (corpus/golden/store/keys); no API calls
	$(PYTHON) scripts/run_epochs.py --k 1 --dry-run \
		--configs custom-openai-k8s,custom-anthropic-k8s

evaluate-stats:  ## Regenerate docs/_generated/stats_report.md from results/long (free, offline)
	$(PYTHON) -m stats.report --tables results/long --out docs/_generated/stats_report.md

plots:  ## Regenerate README figures from the stats report (needs `pip install -e .[plots]`)
	$(PYTHON) scripts/make_plots.py generate

preflight-deploy:  ## Refuse unresolved placeholder tokens on deploy surfaces (run before any push to the hf remote)
	$(PYTHON) scripts/preflight_deploy.py

canary-report:  ## Regenerate docs/_generated/canary_detection.md from the committed canary fixtures (free, offline; verdicts are simulated)
	$(PYTHON) scripts/run_canary_eval.py build-report \
		--canaries tests/stats/fixtures/canary/canaries.json \
		--predictions tests/stats/fixtures/canary/predictions.json

docker:
	docker-compose -f docker/docker-compose.yaml up --build

## --- Infrastructure ---

modal-deploy:  ## Deploy vLLM on Modal (prints endpoint URL)
	@command -v modal >/dev/null 2>&1 || { echo "Error: modal CLI not found. Run: pip install -e '.[modal]' && modal setup"; exit 1; }
	modal deploy modal/serve_vllm.py

modal-stop:  ## Stop Modal deployment
	@command -v modal >/dev/null 2>&1 || { echo "Error: modal CLI not found. Run: pip install -e '.[modal]' && modal setup"; exit 1; }
	modal app stop agent-bench-vllm

vllm-up:  ## Start local vLLM via Docker Compose (requires NVIDIA GPU)
	docker compose -f docker/docker-compose.vllm.yml up --build

benchmark-all:  ## Run provider comparison (requires Modal deployment + API keys)
	$(PYTHON) modal/run_benchmark.py --base-url $(MODAL_VLLM_URL)

k8s-dev:  ## Deploy to minikube (dev values, set MODAL_VLLM_URL first)
	@test -n "$(MODAL_VLLM_URL)" || (echo "Error: MODAL_VLLM_URL is not set" && exit 1)
	helm install agent-bench k8s/helm/agent-bench/ -f k8s/helm/agent-bench/values-dev.yaml \
		--set provider.selfhosted.modalEndpoint=$(MODAL_VLLM_URL)

k8s-prod:  ## Deploy via Helm (prod values, set MODAL_VLLM_URL first)
	@test -n "$(MODAL_VLLM_URL)" || (echo "Error: MODAL_VLLM_URL is not set" && exit 1)
	helm install agent-bench k8s/helm/agent-bench/ -f k8s/helm/agent-bench/values-prod.yaml \
		--set provider.selfhosted.modalEndpoint=$(MODAL_VLLM_URL)

tf-plan:  ## Run terraform plan (no apply)
	cd terraform && terraform plan

tf-validate:  ## Validate terraform syntax
	cd terraform && terraform validate