File size: 1,594 Bytes
3e1cbf3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
"""Tìm id văn bản bằng cách tải metadata parquet (nhỏ) về rồi query cục bộ.

Phân biệt bản tiếng Việt với bản dịch tiếng Anh ('Code ...') qua title.
"""

from __future__ import annotations

from pathlib import Path

import httpx

OUT = Path("data/raw")
META = OUT / "_metadata.parquet"
NUMBERS = ["45/2019/QH14", "10/2012/QH13", "145/2020/NĐ-CP"]


def _parquet_url(config: str) -> str:
    r = httpx.get(
        "https://datasets-server.huggingface.co/parquet",
        params={"dataset": "th1nhng0/vietnamese-legal-documents", "config": config},
        timeout=60,
    )
    r.raise_for_status()
    return r.json()["parquet_files"][0]["url"]


def main() -> None:
    import duckdb

    OUT.mkdir(parents=True, exist_ok=True)
    if not (META.exists() and META.stat().st_size > 1_000_000):
        url = _parquet_url("metadata")
        print("Tải metadata parquet...")
        with httpx.stream("GET", url, timeout=None, follow_redirects=True) as r:
            r.raise_for_status()
            with open(META, "wb") as f:
                for c in r.iter_bytes(1 << 20):
                    f.write(c)
        print(f"  {META.stat().st_size/1e6:.1f} MB")

    con = duckdb.connect()
    nums = ",".join(f"'{n}'" for n in NUMBERS)
    rows = con.execute(
        f"SELECT id, so_ky_hieu, loai_van_ban, title FROM read_parquet('{META}') "
        f"WHERE so_ky_hieu IN ({nums}) ORDER BY so_ky_hieu"
    ).fetchall()
    for r in rows:
        print(f"id={r[0]} | so={r[1]} | loai={r[2]} | {(r[3] or '')[:75]}")


if __name__ == "__main__":
    main()