File size: 1,624 Bytes
3e1cbf3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | """Tính lại embedding cho toàn bộ node bằng model thật (M2).
Chạy sau khi đổi EMBEDDER trong .env và cài requirements-ml:
docker compose exec -e EMBEDDER=vie-v2 -T api python scripts/reembed.py
"""
from __future__ import annotations
from loguru import logger
from sqlalchemy import select, update
from app.config import get_settings
from app.db.base import SessionLocal
from app.db.models import LegalNode
from app.rag.embeddings import embed
def _ctx(n: LegalNode) -> str:
head = f"Điều {n.article_no}" + (f" ({n.heading})" if n.heading else "")
if n.level == "article":
return f"Điều {n.article_no}. {n.heading or ''}. {n.text}".strip()
if n.level == "clause":
return f"{head}, Khoản {n.clause_no}: {n.text}"
return f"{head}, Khoản {n.clause_no}, Điểm {n.point_letter}: {n.text}"
def main() -> None:
s = get_settings()
logger.info("EMBEDDER={} DIM={}", s.embedder, s.embed_dim)
db = SessionLocal()
nodes = list(db.scalars(select(LegalNode)))
logger.info("Tính lại embedding cho {} node...", len(nodes))
batch = 64
done = 0
for i in range(0, len(nodes), batch):
chunk = nodes[i : i + batch]
vecs = embed([_ctx(n) for n in chunk])
for n, v in zip(chunk, vecs):
db.execute(update(LegalNode).where(LegalNode.node_id == n.node_id).values(embedding=v))
db.commit()
done += len(chunk)
if done % 640 == 0 or done == len(nodes):
logger.info(" {}/{}", done, len(nodes))
db.close()
logger.info("Xong.")
if __name__ == "__main__":
main()
|