File size: 1,594 Bytes
3e1cbf3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | """Tìm id văn bản bằng cách tải metadata parquet (nhỏ) về rồi query cục bộ.
Phân biệt bản tiếng Việt với bản dịch tiếng Anh ('Code ...') qua title.
"""
from __future__ import annotations
from pathlib import Path
import httpx
OUT = Path("data/raw")
META = OUT / "_metadata.parquet"
NUMBERS = ["45/2019/QH14", "10/2012/QH13", "145/2020/NĐ-CP"]
def _parquet_url(config: str) -> str:
r = httpx.get(
"https://datasets-server.huggingface.co/parquet",
params={"dataset": "th1nhng0/vietnamese-legal-documents", "config": config},
timeout=60,
)
r.raise_for_status()
return r.json()["parquet_files"][0]["url"]
def main() -> None:
import duckdb
OUT.mkdir(parents=True, exist_ok=True)
if not (META.exists() and META.stat().st_size > 1_000_000):
url = _parquet_url("metadata")
print("Tải metadata parquet...")
with httpx.stream("GET", url, timeout=None, follow_redirects=True) as r:
r.raise_for_status()
with open(META, "wb") as f:
for c in r.iter_bytes(1 << 20):
f.write(c)
print(f" {META.stat().st_size/1e6:.1f} MB")
con = duckdb.connect()
nums = ",".join(f"'{n}'" for n in NUMBERS)
rows = con.execute(
f"SELECT id, so_ky_hieu, loai_van_ban, title FROM read_parquet('{META}') "
f"WHERE so_ky_hieu IN ({nums}) ORDER BY so_ky_hieu"
).fetchall()
for r in rows:
print(f"id={r[0]} | so={r[1]} | loai={r[2]} | {(r[3] or '')[:75]}")
if __name__ == "__main__":
main()
|