| # Road to AI 2026 (R2AI) - Multilingual Medical Document Retrieval Configuration | |
| paths: | |
| raw_data_dir: "data/raw" | |
| processed_data_dir: "data/processed" | |
| indices_dir: "data/indices" | |
| outputs_dir: "outputs" | |
| submissions_dir: "outputs/submissions" | |
| crawler: | |
| user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" | |
| timeout_seconds: 15 | |
| max_retries: 3 | |
| concurrency: 10 | |
| ncbi_email: "team@r2ai2026.org" | |
| ncbi_api_key: null # Optional NCBI Entrez API key for higher rate limits | |
| chunking: | |
| max_chunk_size: 800 # Increased from 512 chars (~200 tokens, well within <=1024 tokens recommended by BTC) | |
| chunk_overlap: 128 # Increased from 64 chars to prevent cutting clinical context | |
| min_chunk_size: 60 # Filter out tiny chunks/noise | |
| split_by_sentences: true # Prefer splitting at sentence boundaries | |
| enable_contextual: true # Enrich chunk with document title/context for embedding and reranking | |
| embedding: | |
| model_name: "BAAI/bge-m3" | |
| batch_size: 32 # 32 fits 6GB VRAM; on Colab T4/A100 can be 64-128 | |
| max_length: 512 | |
| normalize_embeddings: true | |
| device: "auto" # "cuda", "cpu", or "auto" | |
| use_fp16: true # Saves ~50% VRAM (fits in 6GB GPU) | |
| return_sparse: true # Generate BGE-M3 native lexical sparse weights for Qdrant | |
| retrieval: | |
| engine: "faiss" # "faiss" (high performance, 1.3GB RAM) or "qdrant" | |
| qdrant_path: "data/indices/qdrant_db" | |
| collection_name: "medical_chunks" | |
| dense_top_k: 150 # Scaled for 1.5M chunks | |
| sparse_top_k: 150 # Scaled for 1.5M chunks | |
| fusion_method: "rrf" # "rrf" (Reciprocal Rank Fusion) | |
| rrf_k: 60 | |
| dense_weight: 0.6 | |
| sparse_weight: 0.4 | |
| hybrid_top_k: 150 # Feed 150 diverse candidates to reranker | |
| reranker: | |
| enabled: true | |
| model_name: "BAAI/bge-reranker-large" | |
| batch_size: 32 # 32 locally, can be 64 on Colab/Kaggle | |
| device: "auto" | |
| top_k_chunks: 75 # Tuned to BTC gold average of ~74.7 chunks | |
| top_k_docs: 60 # Tuned to BTC gold average of ~71.0 docs | |
| score_threshold: -8.0 # Relaxed to capture maximum Recall for Macro F2 | |
| max_chunks_per_doc: 2 # Maintain high diversity across 60 docs | |
| query_translation: | |
| enabled: true | |
| model_name: "Helsinki-NLP/opus-mt-vi-en" # "Helsinki-NLP/opus-mt-vi-en" or "ndhieu1101/medical-bidirectional-machine-translation-checkpoints-511042" | |
| prompt_prefix: "" # e.g. "vi: " for bidirectional T5 models (auto-detected if empty) | |
| device: "auto" | |
| lexicon_path: "data/lexicon/medical_terms.json" | |
| zh_lexicon_path: "data/lexicon/icd10_vi_zh.json" | |
| stopwords_path: "configs/pubmed_stopwords.txt" | |
| pubmed: | |
| enabled: true | |
| max_candidates_per_query: 30 # Increased from 15 to capture more PubMed evidence | |
| source_api: "europe_pmc" # "europe_pmc", "pubtator", "ncbi", or "hybrid" | |
| cache_file: "data/processed/pubmed_cache.jsonl" | |
Xet Storage Details
- Size:
- 2.99 kB
- Xet hash:
- 7d905bffa1008140fb5f60e573fa91e701acb3227b0bc0a142881dd9ba2387d3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.