RAG-api / scripts /fetch_hf_duckdb.py
Minh Be
deploy FastAPI backend
3e1cbf3
Raw History Blame Contribute Delete
2.36 kB
"""Fallback: lấy nội dung văn bản bằng DuckDB đọc parquet HF trực tiếp.
Dùng khi datasets-server /filter trên config `content` (5.6GB) bị 504.
docker compose exec -T api pip install duckdb
docker compose exec -T api python scripts/fetch_hf_duckdb.py
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[0]))
from html_text import html_to_text # noqa: E402
KNOWN_IDS = {"blld2019": "139264", "blld2012": "27615", "nd145_2020": "152668"}
OUT = Path("data/raw")
def _parquet_urls(config: str) -> list[str]:
import httpx
r = httpx.get(
"https://datasets-server.huggingface.co/parquet",
params={"dataset": "th1nhng0/vietnamese-legal-documents", "config": config},
timeout=60,
)
r.raise_for_status()
return [f["url"] for f in r.json().get("parquet_files", [])]
CACHE = OUT / "_content.parquet"
def _ensure_cached() -> None:
"""Tải parquet content về đĩa một lần (784MB) để truy vấn lại khỏi tải lại."""
import httpx
if CACHE.exists() and CACHE.stat().st_size > 100_000_000:
print(f"Dùng parquet đã tải: {CACHE}")
return
url = _parquet_urls("content")[0]
print(f"Tải parquet về {CACHE} ...")
with httpx.stream("GET", url, timeout=None, follow_redirects=True) as r:
r.raise_for_status()
with open(CACHE, "wb") as f:
for chunk in r.iter_bytes(chunk_size=1 << 20):
f.write(chunk)
print(f"Xong: {CACHE.stat().st_size/1e6:.1f} MB")
def main() -> None:
import duckdb
OUT.mkdir(parents=True, exist_ok=True)
_ensure_cached()
con = duckdb.connect()
ids = ",".join(f"'{v}'" for v in KNOWN_IDS.values())
q = f"SELECT id, content_html FROM read_parquet('{CACHE}') WHERE id IN ({ids})"
print("Truy vấn parquet cục bộ...")
rows = con.execute(q).fetchall()
by_id = {r[0]: r[1] for r in rows}
for did, hf_id in KNOWN_IDS.items():
html = by_id.get(hf_id)
if not html:
print(f" [thiếu] {did} ({hf_id})")
continue
text = html_to_text(html)
(OUT / f"{did}.txt").write_text(text, encoding="utf-8")
print(f" LƯU {did}.txt ({len(text):,} ký tự)")
if __name__ == "__main__":
main()