Spaces:
Runtime error
Runtime error
File size: 1,908 Bytes
aacc29a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | """CLI: build the NE / code-switch term datastore (paper §4.2 Step 1).
python scribe/training/scripts/build_datastore.py \
--dataset tensorxt/ViMedCSS --limit-per-split 20 \
--output artifacts/retrieval/term_datastore_smoke.json
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "scribe" / "training"))
sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "scribe"))
from gec.cliutil import configure_stdout # noqa: E402
configure_stdout()
from gec.datastore import build_datastore, write_datastore # noqa: E402
from gec.manifest import load_manifest # noqa: E402
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--lexicon", default="data/medical_lexicon.json")
parser.add_argument("--pairs", nargs="*", default=[])
parser.add_argument("--dataset", default=None)
parser.add_argument("--manifest", type=Path)
parser.add_argument(
"--splits",
nargs="+",
default=["train"],
help="governed RAC source splits; training default is train only",
)
parser.add_argument("--limit-per-split", type=int, default=None)
parser.add_argument("--output", default="artifacts/retrieval/term_datastore.json")
args = parser.parse_args()
manifest = load_manifest(args.manifest, require_approved=True) if args.manifest else None
payload = build_datastore(
lexicon_path=Path(args.lexicon),
pair_paths=[Path(p) for p in args.pairs],
dataset=args.dataset,
splits=tuple(args.splits),
limit_per_split=args.limit_per_split,
dataset_manifest=manifest,
)
write_datastore(payload, Path(args.output))
print(f"Wrote {payload['metadata']['term_count']} terms to {args.output}")
if __name__ == "__main__":
main()
|