File size: 2,362 Bytes
3e1cbf3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
"""Fallback: lấy nội dung văn bản bằng DuckDB đọc parquet HF trực tiếp.

Dùng khi datasets-server /filter trên config `content` (5.6GB) bị 504.
    docker compose exec -T api pip install duckdb
    docker compose exec -T api python scripts/fetch_hf_duckdb.py
"""

from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[0]))
from html_text import html_to_text  # noqa: E402

KNOWN_IDS = {"blld2019": "139264", "blld2012": "27615", "nd145_2020": "152668"}
OUT = Path("data/raw")


def _parquet_urls(config: str) -> list[str]:
    import httpx

    r = httpx.get(
        "https://datasets-server.huggingface.co/parquet",
        params={"dataset": "th1nhng0/vietnamese-legal-documents", "config": config},
        timeout=60,
    )
    r.raise_for_status()
    return [f["url"] for f in r.json().get("parquet_files", [])]


CACHE = OUT / "_content.parquet"


def _ensure_cached() -> None:
    """Tải parquet content về đĩa một lần (784MB) để truy vấn lại khỏi tải lại."""
    import httpx

    if CACHE.exists() and CACHE.stat().st_size > 100_000_000:
        print(f"Dùng parquet đã tải: {CACHE}")
        return
    url = _parquet_urls("content")[0]
    print(f"Tải parquet về {CACHE} ...")
    with httpx.stream("GET", url, timeout=None, follow_redirects=True) as r:
        r.raise_for_status()
        with open(CACHE, "wb") as f:
            for chunk in r.iter_bytes(chunk_size=1 << 20):
                f.write(chunk)
    print(f"Xong: {CACHE.stat().st_size/1e6:.1f} MB")


def main() -> None:
    import duckdb

    OUT.mkdir(parents=True, exist_ok=True)
    _ensure_cached()
    con = duckdb.connect()
    ids = ",".join(f"'{v}'" for v in KNOWN_IDS.values())
    q = f"SELECT id, content_html FROM read_parquet('{CACHE}') WHERE id IN ({ids})"
    print("Truy vấn parquet cục bộ...")
    rows = con.execute(q).fetchall()
    by_id = {r[0]: r[1] for r in rows}
    for did, hf_id in KNOWN_IDS.items():
        html = by_id.get(hf_id)
        if not html:
            print(f"  [thiếu] {did} ({hf_id})")
            continue
        text = html_to_text(html)
        (OUT / f"{did}.txt").write_text(text, encoding="utf-8")
        print(f"  LƯU {did}.txt ({len(text):,} ký tự)")


if __name__ == "__main__":
    main()