Upload folder using huggingface_hub

81f30c1 verified 10 days ago

6.36 kB

#!/usr/bin/env bash
# Cadena completa y reanudable del experimento principal (reemplaza run_eval_v1 -> run_full_index -> run_morning):
#   1. eval-v1       60 problemas × 5 configs, tabla de muestra de Mathlib
#   2. eval-v1-full  `full` y `no_critic` con el índice completo (278 160, ANN IVF_SQ); `no_rag` y las líneas
#                    base no dependen del índice y se copian de eval-v1
#   3. cierre        informes, §6.3 del informe final, resumen en data/eval/results/chain_summary.md, métricas QR con el índice ANN
# Cada etapa se reanuda sola hasta 4 veces (caída del proceso, red) y se corta limpio si la key no tiene crédito;
# volver a lanzar el mismo comando continúa donde quedó.
# Uso: setsid nohup bash scripts/run_chain.sh > data/eval/results/chain.log 2>&1 < /dev/null &
set -u
cd "$(dirname "$0")/.."
export PATH="$HOME/.elan/bin:$PATH" PYTHONUNBUFFERED=1
F() { grep --line-buffered -v "Warning\|Loading\|it/s\|prompt name\|_fork_exec\|WARN lance"; }
step() { echo "=== $1 $(date '+%F %H:%M') mem_avail=$(free -m | awk '/Mem:/{print $7}')MB ==="; }
PROBLEMS=data/eval/problems/eval_v1.jsonl
CONFIGS=full,no_rag,no_critic,baseline_a,baseline_b

key_left() {
  uv run python - <<'PY' 2>/dev/null || echo 0
import httpx
key = [l.split("=", 1)[1].strip() for l in open(".env") if l.startswith("OPENROUTER_API_KEY=")][0]
d = httpx.get("https://openrouter.ai/api/v1/auth/key", headers={"Authorization": "Bearer " + key}, timeout=30).json()["data"]
print(round(d.get("limit_remaining") or 0, 3))
PY
}

pending() {  # pares (config, problema) sin resultado válido en data/eval/runs/$1
  uv run python - "$1" "$PROBLEMS" "$CONFIGS" <<'PY'
import json, sys
from pathlib import Path
run, problems, configs = sys.argv[1:4]
probs = [json.loads(l)["problem_id"] for l in open(problems, encoding="utf-8") if l.strip()]
p = Path("data/eval/runs") / run / "results.jsonl"
rows = [json.loads(l) for l in p.read_text(encoding="utf-8").splitlines() if l.strip()] if p.exists() else []
done = {(r["config"], r["problem_id"]) for r in rows if r["status"] != "error"}
print(sum((c, q) not in done for c in configs.split(",") for q in probs))
PY
}

run_until_done() {  # $1 run-id
  for i in 1 2 3 4; do
    n=$(pending "$1"); [ "$n" = "0" ] && return 0
    left=$(key_left)
    awk "BEGIN{exit !($left > 0.02)}" || { echo "[chain] sin crédito en la key (US\$$left): $n pares pendientes en $1"; return 1; }
    step "$1 intento $i: $n pares pendientes, key US\$$left"
    uv run python -m ragsolver.orchestrator.run --problems "$PROBLEMS" --configs "$CONFIGS" --run-id "$1" --workers 6 2>&1 | F
  done
  [ "$(pending "$1")" = "0" ]
}

# 1. eval-v1 -------------------------------------------------------------------------------------------
run_until_done eval-v1; ok1=$?
step "REPORT eval-v1"
uv run python scripts/eval_report.py --run-id eval-v1 --problems "$PROBLEMS" 2>&1 | F

# 2. eval-v1-full ----------------------------------------------------------------------------------------
ok2=1
if [ $ok1 = 0 ]; then
  step "eval-v1-full: índice ANN y copia de configs independientes del índice"
  uv run python - <<'PY' 2>&1 | F
import json, lancedb
from pathlib import Path
from ragsolver.rag.index import build_ann_index
t = lancedb.connect("data/index/lancedb").open_table("mathlib_full")
if not any("IVF" in str(i.index_type).upper() for i in t.list_indices()):
    build_ann_index(t); print("índice ANN construido")
src = [json.loads(l) for l in open("data/eval/runs/eval-v1/results.jsonl", encoding="utf-8") if l.strip()]
dst_p = Path("data/eval/runs/eval-v1-full/results.jsonl"); dst_p.parent.mkdir(parents=True, exist_ok=True)
dst = [json.loads(l) for l in dst_p.read_text(encoding="utf-8").splitlines() if l.strip()] if dst_p.exists() else []
have = {(r["config"], r["problem_id"]) for r in dst}
new = [r for r in src if r["config"] in ("no_rag", "baseline_a", "baseline_b") and (r["config"], r["problem_id"]) not in have]
with dst_p.open("a", encoding="utf-8") as f:
    f.writelines(json.dumps(r, ensure_ascii=False) + "\n" for r in new)
print(f"copiadas de eval-v1: {len(new)} (ya había {len(dst)})")
PY
  RAGSOLVER_MATHLIB_TABLE=mathlib_full run_until_done eval-v1-full; ok2=$?
  step "REPORT eval-v1-full"
  RAGSOLVER_MATHLIB_TABLE=mathlib_full uv run python scripts/eval_report.py --run-id eval-v1-full --problems "$PROBLEMS" --out docs/eval_report_full_index.md 2>&1 | F
fi

# 3. cierre -------------------------------------------------------------------------------------------------
step "cierre: §6.3 del informe final y PROGRESS"
uv run python scripts/fill_final_report.py --run-id eval-v1 --full-index-run-id eval-v1-full --problems "$PROBLEMS" 2>&1 | F
uv run python - "$ok1" "$ok2" <<'PY'
import json, sys, time
from pathlib import Path
ok1, ok2 = sys.argv[1:3]
L = [f"\n## Resultado automático de la cadena ({time.strftime('%Y-%m-%d %H:%M')})\n", f"eval-v1 completo: {'sí' if ok1 == '0' else 'NO'} · eval-v1-full completo: {'sí' if ok2 == '0' else 'NO'}\n", "| corrida | config | n | pass@1 | Wilson 95 % | llamadas LLM | tokens |", "|---|---|---|---|---|---|---|"]
for run in ("eval-v1", "eval-v1-full"):
    p = Path("data/eval/runs") / run / "summary.json"
    if p.exists():
        for c, s in json.loads(p.read_text(encoding="utf-8")).items():
            L.append(f"| {run} | {c} | {s['n']} | {s['pass@1']} | {s['pass@1_wilson95']} | {s['llm_calls_mean']} | {s['tokens_mean']} |")
L += ["", "Detalle: `docs/eval_report.md`, `docs/eval_report_full_index.md`, `docs/final_report.md` §6.3. Log: `data/eval/results/chain.log`.", ""]
# docs/PLAN_CORRECCIONES.md Fase 3, tarea 5: el resumen automático ya no se agrega a
# docs/PROGRESS.md (documentación editada a mano; auto-escribirla la desincroniza).
Path("data/eval/results/chain_summary.md").open("a", encoding="utf-8").write("\n".join(L))
print("data/eval/results/chain_summary.md actualizado")
PY
if [ ! -f data/eval/results/fase1_full_ann_mathlibqr_100q.json ]; then
  step "MathlibQR 100q sobre el índice completo con ANN (sin costo de API)"
  uv run python -m ragsolver.rag.evaluate --table mathlib_full --sample 100 --out data/eval/results/fase1_full_ann_mathlibqr_100q.json 2>&1 | F | tail -2
  uv run python scripts/eval_report.py --run-id eval-v1 --problems "$PROBLEMS" 2>&1 | F
fi
echo "CHAIN DONE $(date '+%F %H:%M') eval-v1=$ok1 eval-v1-full=$ok2"

	#!/usr/bin/env bash
	# Cadena completa y reanudable del experimento principal (reemplaza run_eval_v1 -> run_full_index -> run_morning):
	# 1. eval-v1 60 problemas × 5 configs, tabla de muestra de Mathlib
	# 2. eval-v1-full `full` y `no_critic` con el índice completo (278 160, ANN IVF_SQ); `no_rag` y las líneas
	# base no dependen del índice y se copian de eval-v1
	# 3. cierre informes, §6.3 del informe final, resumen en data/eval/results/chain_summary.md, métricas QR con el índice ANN
	# Cada etapa se reanuda sola hasta 4 veces (caída del proceso, red) y se corta limpio si la key no tiene crédito;
	# volver a lanzar el mismo comando continúa donde quedó.
	# Uso: setsid nohup bash scripts/run_chain.sh > data/eval/results/chain.log 2>&1 < /dev/null &
	set -u
	cd "$(dirname "$0")/.."
	export PATH="$HOME/.elan/bin:$PATH" PYTHONUNBUFFERED=1
	F() { grep --line-buffered -v "Warning\\|Loading\\|it/s\\|prompt name\\|_fork_exec\\|WARN lance"; }
	step() { echo "=== $1 $(date '+%F %H:%M') mem_avail=$(free -m \| awk '/Mem:/{print $7}')MB ==="; }
	PROBLEMS=data/eval/problems/eval_v1.jsonl
	CONFIGS=full,no_rag,no_critic,baseline_a,baseline_b

	key_left() {
	uv run python - <<'PY' 2>/dev/null \|\| echo 0
	import httpx
	key = [l.split("=", 1)[1].strip() for l in open(".env") if l.startswith("OPENROUTER_API_KEY=")][0]
	d = httpx.get("https://openrouter.ai/api/v1/auth/key", headers={"Authorization": "Bearer " + key}, timeout=30).json()["data"]
	print(round(d.get("limit_remaining") or 0, 3))
	PY
	}

	pending() { # pares (config, problema) sin resultado válido en data/eval/runs/$1
	uv run python - "$1" "$PROBLEMS" "$CONFIGS" <<'PY'
	import json, sys
	from pathlib import Path
	run, problems, configs = sys.argv[1:4]
	probs = [json.loads(l)["problem_id"] for l in open(problems, encoding="utf-8") if l.strip()]
	p = Path("data/eval/runs") / run / "results.jsonl"
	rows = [json.loads(l) for l in p.read_text(encoding="utf-8").splitlines() if l.strip()] if p.exists() else []
	done = {(r["config"], r["problem_id"]) for r in rows if r["status"] != "error"}
	print(sum((c, q) not in done for c in configs.split(",") for q in probs))
	PY
	}

	run_until_done() { # $1 run-id
	for i in 1 2 3 4; do
	n=$(pending "$1"); [ "$n" = "0" ] && return 0
	left=$(key_left)
	awk "BEGIN{exit !($left > 0.02)}" \|\| { echo "[chain] sin crédito en la key (US\$$left): $n pares pendientes en $1"; return 1; }
	step "$1 intento $i: $n pares pendientes, key US\$$left"
	uv run python -m ragsolver.orchestrator.run --problems "$PROBLEMS" --configs "$CONFIGS" --run-id "$1" --workers 6 2>&1 \| F
	done
	[ "$(pending "$1")" = "0" ]
	}

	# 1. eval-v1 -------------------------------------------------------------------------------------------
	run_until_done eval-v1; ok1=$?
	step "REPORT eval-v1"
	uv run python scripts/eval_report.py --run-id eval-v1 --problems "$PROBLEMS" 2>&1 \| F

	# 2. eval-v1-full ----------------------------------------------------------------------------------------
	ok2=1
	if [ $ok1 = 0 ]; then
	step "eval-v1-full: índice ANN y copia de configs independientes del índice"
	uv run python - <<'PY' 2>&1 \| F
	import json, lancedb
	from pathlib import Path
	from ragsolver.rag.index import build_ann_index
	t = lancedb.connect("data/index/lancedb").open_table("mathlib_full")
	if not any("IVF" in str(i.index_type).upper() for i in t.list_indices()):
	build_ann_index(t); print("índice ANN construido")
	src = [json.loads(l) for l in open("data/eval/runs/eval-v1/results.jsonl", encoding="utf-8") if l.strip()]
	dst_p = Path("data/eval/runs/eval-v1-full/results.jsonl"); dst_p.parent.mkdir(parents=True, exist_ok=True)
	dst = [json.loads(l) for l in dst_p.read_text(encoding="utf-8").splitlines() if l.strip()] if dst_p.exists() else []
	have = {(r["config"], r["problem_id"]) for r in dst}
	new = [r for r in src if r["config"] in ("no_rag", "baseline_a", "baseline_b") and (r["config"], r["problem_id"]) not in have]
	with dst_p.open("a", encoding="utf-8") as f:
	f.writelines(json.dumps(r, ensure_ascii=False) + "\n" for r in new)
	print(f"copiadas de eval-v1: {len(new)} (ya había {len(dst)})")
	PY
	RAGSOLVER_MATHLIB_TABLE=mathlib_full run_until_done eval-v1-full; ok2=$?
	step "REPORT eval-v1-full"
	RAGSOLVER_MATHLIB_TABLE=mathlib_full uv run python scripts/eval_report.py --run-id eval-v1-full --problems "$PROBLEMS" --out docs/eval_report_full_index.md 2>&1 \| F
	fi

	# 3. cierre -------------------------------------------------------------------------------------------------
	step "cierre: §6.3 del informe final y PROGRESS"
	uv run python scripts/fill_final_report.py --run-id eval-v1 --full-index-run-id eval-v1-full --problems "$PROBLEMS" 2>&1 \| F
	uv run python - "$ok1" "$ok2" <<'PY'
	import json, sys, time
	from pathlib import Path
	ok1, ok2 = sys.argv[1:3]
	L = [f"\n## Resultado automático de la cadena ({time.strftime('%Y-%m-%d %H:%M')})\n", f"eval-v1 completo: {'sí' if ok1 == '0' else 'NO'} · eval-v1-full completo: {'sí' if ok2 == '0' else 'NO'}\n", "\| corrida \| config \| n \| pass@1 \| Wilson 95 % \| llamadas LLM \| tokens \|", "\|---\|---\|---\|---\|---\|---\|---\|"]
	for run in ("eval-v1", "eval-v1-full"):
	p = Path("data/eval/runs") / run / "summary.json"
	if p.exists():
	for c, s in json.loads(p.read_text(encoding="utf-8")).items():
	L.append(f"\| {run} \| {c} \| {s['n']} \| {s['pass@1']} \| {s['pass@1_wilson95']} \| {s['llm_calls_mean']} \| {s['tokens_mean']} \|")
	L += ["", "Detalle: `docs/eval_report.md`, `docs/eval_report_full_index.md`, `docs/final_report.md` §6.3. Log: `data/eval/results/chain.log`.", ""]
	# docs/PLAN_CORRECCIONES.md Fase 3, tarea 5: el resumen automático ya no se agrega a
	# docs/PROGRESS.md (documentación editada a mano; auto-escribirla la desincroniza).
	Path("data/eval/results/chain_summary.md").open("a", encoding="utf-8").write("\n".join(L))
	print("data/eval/results/chain_summary.md actualizado")
	PY
	if [ ! -f data/eval/results/fase1_full_ann_mathlibqr_100q.json ]; then
	step "MathlibQR 100q sobre el índice completo con ANN (sin costo de API)"
	uv run python -m ragsolver.rag.evaluate --table mathlib_full --sample 100 --out data/eval/results/fase1_full_ann_mathlibqr_100q.json 2>&1 \| F \| tail -2
	uv run python scripts/eval_report.py --run-id eval-v1 --problems "$PROBLEMS" 2>&1 \| F
	fi
	echo "CHAIN DONE $(date '+%F %H:%M') eval-v1=$ok1 eval-v1-full=$ok2"