Spaces:
Sleeping
Sleeping
File size: 2,134 Bytes
8d823fc c2e0975 5ebe9b0 c2e0975 5ebe9b0 c2e0975 5ebe9b0 c2e0975 1492d37 5bd2b83 c2e0975 1492d37 8d823fc c2e0975 1492d37 8d823fc b19e375 3f04f41 c2e0975 8d823fc 3f04f41 b19e375 3f04f41 b19e375 3f04f41 b19e375 3f04f41 c2e0975 9fa7dc9 8d823fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | # skill_extraction.py
import json
import spacy
import numpy as np
from sentence_transformers import SentenceTransformer
from spacy.cli import download
# Ensure spaCy model is available
try:
nlp = spacy.load("en_core_web_sm")
except OSError:
download("en_core_web_sm")
nlp = spacy.load("en_core_web_sm")
# Embedding model
model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
# Load skills from JSON
with open("skills_vocab.json", "r", encoding="utf-8") as f:
SKILL_VOCAB = json.load(f)["skills"]
# Precompute embeddings
skill_embeddings = model.encode(SKILL_VOCAB, convert_to_numpy=True, normalize_embeddings=True)
# Map skill -> embedding for quick lookup
SKILL_TO_EMB = {skill: emb for skill, emb in zip(SKILL_VOCAB, skill_embeddings)}
def extract_skills(text, threshold=0.50):
"""
Extract skills from text.
Returns list of (skill, confidence, evidence_sentences)
"""
doc = nlp(text)
sentences = [sent.text.strip() for sent in doc.sents if sent.text.strip()]
if not sentences:
return []
sentence_embeddings = model.encode(sentences, convert_to_numpy=True, normalize_embeddings=True)
skill_confidences = {}
skill_evidence = {}
# Iterate skill-by-skill
for j, skill_emb in enumerate(skill_embeddings):
sims = np.dot(sentence_embeddings, skill_emb) # similarity across sentences
max_sim_idx = int(np.argmax(sims))
max_sim = float(sims[max_sim_idx])
if max_sim >= threshold:
skill = SKILL_VOCAB[j]
skill_confidences[skill] = round(max_sim, 2)
# Take the sentence with highest similarity as evidence
skill_evidence[skill] = sentences[max_sim_idx]
# Return sorted list of (skill, confidence, evidence)
results = [(skill, skill_confidences[skill], skill_evidence[skill]) for skill in skill_confidences]
results.sort(key=lambda x: x[1], reverse=True)
return results
# --- NEW FUNCTION ---
def get_skill_embedding(skill_name):
"""Return the embedding vector for a skill, or None if not in vocab"""
return SKILL_TO_EMB.get(skill_name)
|