| """Assemble the Hugging Face Dataset folder (hf_dataset/) from results/. Uploading is a separate, manual step. | |
| Usage: uv run python scripts/export_hf.py | |
| Upload (after review): hf upload bunyamineymen/rag-gate-stats-results hf_dataset --repo-type dataset | |
| hf upload bunyamineymen/rag-gate-stats space --repo-type space | |
| hf upload bunyamineymen/rag-gate-stats-code <git archive of HEAD> (model repo, code only) | |
| """ | |
| import json | |
| import shutil | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parents[1] | |
| RES, OUT = ROOT / "results", ROOT / "hf_dataset" | |
| TABLES = ["per_query", "threshold", "pairs", "contrasts", "latency"] | |
| CARD = """--- | |
| license: cc-by-sa-4.0 | |
| pretty_name: RAG regression gate statistics (project 01) | |
| language: [en] | |
| tags: [rag, retrieval, evaluation, statistics, beir, regression-testing] | |
| configs: | |
| {configs} | |
| --- | |
| # RAG regression gate statistics — results | |
| Per-query retrieval metrics and simulated PASS/FAIL gate decisions for small golden sets, | |
| produced by project 01 of a RAG evaluation series. No corpus or query text is redistributed; | |
| rows reference BEIR query ids only. | |
| ## Source data | |
| | Dataset | HF ids | License (as checked 2026-10-14) | | |
| |---|---|---| | |
| | NFCorpus | `BeIR/nfcorpus`, `BeIR/nfcorpus-qrels` | HF card: CC BY-SA 4.0; original license to be confirmed | | |
| | FiQA-2018 | `BeIR/fiqa`, `BeIR/fiqa-qrels` | HF card: CC BY-SA 4.0; original license to be confirmed | | |
| | SCIDOCS | `BeIR/scidocs`, `BeIR/scidocs-qrels` | HF card: CC BY-SA 4.0; original license to be confirmed | | |
| BEIR's README leaves license responsibility with the user; the per-dataset original terms apply. | |
| ## Tables | |
| | Config | One row per | Key columns | | |
| |---|---|---| | |
| | `per_query` | dataset × system × query | `recall@5`, `recall@10`, `mrr@10`, `ndcg@10` | | |
| | `threshold` | dataset × truth × system × metric × n × d | `k_pass` of `R` golden sets for the absolute-threshold gate G1 (τ = true − d) | | |
| | `pairs` | dataset × truth × scenario × metric × n × gate × param | `k_pass` of `R`, `delta_true`, `sd_d`, `analytic_pass` (G3) | | |
| | `contrasts` | dataset × truth × scenario × metric × n × margin × contrast | mean and 95% CI of per-golden-set gate differences | | |
| | `latency` | system × run × query | `ms` (FiQA, 2 threads) | | |
| Gates: G1 absolute threshold, G2 naive delta, G3 paired bootstrap non-inferiority, G3u unpaired bootstrap, | |
| G3signflip sign-flip test. Truth `population` = all test queries; `split_half` = golden sets drawn from one half, | |
| truth measured on the other half. | |
| ## Reproduction | |
| Code: [bunyamineymen/rag-gate-stats-code](https://huggingface.co/bunyamineymen/rag-gate-stats-code) | |
| `git clone https://huggingface.co/bunyamineymen/rag-gate-stats-code && cd rag-gate-stats-code && uv sync && uv run python scripts/run_all.py && uv run python scripts/latency.py && uv run python scripts/report.py` | |
| - Pre-registration SHA-256: `{prereg}` | |
| - Config SHA-256: `{config}` | |
| - Seed: 20261017 | |
| ## Pre-registered hypotheses | |
| {hyp} | |
| """ | |
| def main(): | |
| env = json.loads((RES / "env.json").read_text()) | |
| hyp = json.loads((RES / "hypotheses.json").read_text()) | |
| (OUT / "data").mkdir(parents=True, exist_ok=True) | |
| present = [t for t in TABLES if (RES / f"{t}.parquet").exists()] | |
| for t in present: | |
| shutil.copy(RES / f"{t}.parquet", OUT / "data" / f"{t}.parquet") | |
| for f in ["env.json", "hypotheses.json"]: | |
| shutil.copy(RES / f, OUT / f) | |
| configs = "\n".join(f"- config_name: {t}\n data_files: data/{t}.parquet" for t in present) | |
| lines = "\n".join(f"- {k}: {'supported' if v['supported'] else 'not supported'}" for k, v in hyp.items()) | |
| (OUT / "README.md").write_text(CARD.format(configs=configs, prereg=env["prereg_sha256"], | |
| config=env["config_sha256"], hyp=lines), encoding="utf-8") | |
| print(f"wrote {OUT} with {present}") | |
| if __name__ == "__main__": | |
| main() | |