File size: 2,134 Bytes
8d823fc
 
c2e0975
5ebe9b0
c2e0975
 
5ebe9b0
 
c2e0975
5ebe9b0
 
 
 
 
 
c2e0975
 
1492d37
5bd2b83
c2e0975
 
1492d37
8d823fc
c2e0975
1492d37
8d823fc
 
 
b19e375
3f04f41
 
 
 
c2e0975
 
 
 
 
 
 
8d823fc
3f04f41
b19e375
3f04f41
b19e375
3f04f41
 
 
 
b19e375
3f04f41
 
 
 
 
 
 
 
 
c2e0975
9fa7dc9
8d823fc
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
# skill_extraction.py

import json
import spacy
import numpy as np
from sentence_transformers import SentenceTransformer
from spacy.cli import download

# Ensure spaCy model is available
try:
    nlp = spacy.load("en_core_web_sm")
except OSError:
    download("en_core_web_sm")
    nlp = spacy.load("en_core_web_sm")

# Embedding model
model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")

# Load skills from JSON
with open("skills_vocab.json", "r", encoding="utf-8") as f:
    SKILL_VOCAB = json.load(f)["skills"]

# Precompute embeddings
skill_embeddings = model.encode(SKILL_VOCAB, convert_to_numpy=True, normalize_embeddings=True)

# Map skill -> embedding for quick lookup
SKILL_TO_EMB = {skill: emb for skill, emb in zip(SKILL_VOCAB, skill_embeddings)}

def extract_skills(text, threshold=0.50):
    """
    Extract skills from text.
    Returns list of (skill, confidence, evidence_sentences)
    """
    doc = nlp(text)
    sentences = [sent.text.strip() for sent in doc.sents if sent.text.strip()]

    if not sentences:
        return []

    sentence_embeddings = model.encode(sentences, convert_to_numpy=True, normalize_embeddings=True)
    skill_confidences = {}
    skill_evidence = {}

    # Iterate skill-by-skill
    for j, skill_emb in enumerate(skill_embeddings):
        sims = np.dot(sentence_embeddings, skill_emb)  # similarity across sentences
        max_sim_idx = int(np.argmax(sims))
        max_sim = float(sims[max_sim_idx])

        if max_sim >= threshold:
            skill = SKILL_VOCAB[j]
            skill_confidences[skill] = round(max_sim, 2)
            # Take the sentence with highest similarity as evidence
            skill_evidence[skill] = sentences[max_sim_idx]

    # Return sorted list of (skill, confidence, evidence)
    results = [(skill, skill_confidences[skill], skill_evidence[skill]) for skill in skill_confidences]
    results.sort(key=lambda x: x[1], reverse=True)
    return results


# --- NEW FUNCTION ---
def get_skill_embedding(skill_name):
    """Return the embedding vector for a skill, or None if not in vocab"""
    return SKILL_TO_EMB.get(skill_name)