Spaces:
Runtime error
Runtime error
File size: 3,088 Bytes
ffd36e0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | """Populate the SkillEmbedding table with SBERT vectors for every catalog Skill.
Idempotent. Re-run after any `seed_initial_skills` change to refresh the
embeddings. One-shot script (not a management command) mirroring
`parse_onet_dump.py` style.
Usage:
python backend/scripts/build_skill_embeddings.py
Takes ~15 s for ~70 skills (model load dominates — the encode step itself
is <1 s). Writes via update_or_create keyed on Skill so existing rows get
overwritten with the new vector.
Requires:
* pgvector extension installed on the DB + migration 0003 applied.
* sentence-transformers + torch in the venv (requirements.txt).
* all-MiniLM-L6-v2 on disk or network accessible for first-call download.
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
# Let this script run from anywhere — set up the backend module path first.
BACKEND_DIR = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(BACKEND_DIR))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
import django # noqa: E402
django.setup()
from django.db import transaction # noqa: E402
from apps.skills.models import Skill, SkillEmbedding # noqa: E402
MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
def _encoding_text(skill: Skill) -> str:
"""What we embed for each skill.
Use ``skill_name + " — " + (description or category)`` so the embedding
carries both the exact name AND a short semantic bridge. "Python"
embedded alone sits near "python snake" and "Monty Python" in SBERT
space; "Python — Programming" stays inside the tech cluster.
"""
context = (skill.description or "").strip() or skill.category.strip()
if context:
return f"{skill.skill_name} — {context}"
return skill.skill_name
def main() -> int:
# Lazy import so a failure here is visible, not an ImportError at the top.
from sentence_transformers import SentenceTransformer
skills = list(Skill.objects.all().order_by("id"))
if not skills:
print("No skills in the catalog — run seed_initial_skills first.")
return 1
print(f"Loading {MODEL_NAME}…")
model = SentenceTransformer(MODEL_NAME)
texts = [_encoding_text(s) for s in skills]
print(f"Encoding {len(texts)} skills…")
vectors = model.encode(texts, normalize_embeddings=True, show_progress_bar=False)
created = 0
updated = 0
with transaction.atomic():
for skill, text, vec in zip(skills, texts, vectors):
_, was_created = SkillEmbedding.objects.update_or_create(
skill=skill,
defaults={
"embedding": vec.tolist(),
"source_text": text,
"model_name": MODEL_NAME.rsplit("/", 1)[-1],
},
)
if was_created:
created += 1
else:
updated += 1
print(f"Done: {created} created, {updated} updated, "
f"{len(skills)} total.")
return 0
if __name__ == "__main__":
sys.exit(main())
|