Final final final updates to app.py
Browse files
app.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
| 1 |
try:
|
| 2 |
-
import spaces
|
| 3 |
GPU = spaces.GPU
|
| 4 |
except ImportError:
|
| 5 |
-
GPU = lambda f: f
|
| 6 |
|
| 7 |
import gradio as gr
|
| 8 |
import torch
|
|
@@ -32,9 +32,9 @@ NAMES = {"WORKLOAD": "Sleep and workload", "TEAM": "Team dynamics",
|
|
| 32 |
"ADJUSTMENT": "Adjustment and coping", "OUTSIDE_COMMS": "Outside communication"}
|
| 33 |
|
| 34 |
|
| 35 |
-
#
|
| 36 |
|
| 37 |
-
#1. vader baseline: keyword lists + sentiment gate
|
| 38 |
KEYWORDS = {
|
| 39 |
"WORKLOAD": ["exhausted", "exhaustion", "tired", "fatigue", "overworked", "sleep", "schedule", "timeline",
|
| 40 |
"overtime", "demanding", "heavy", "restless", "shift", "tasks", "pressure"],
|
|
@@ -56,7 +56,7 @@ STOP = set(stopwords.words("english"))
|
|
| 56 |
stemmer = SnowballStemmer("english")
|
| 57 |
|
| 58 |
#3. bert-base-uncased + my lora adapter (the adapter files sit next to this app.py)
|
| 59 |
-
|
| 60 |
BERT_THRESHOLDS = {"WORKLOAD": 0.65, "TEAM": 0.90, "ADJUSTMENT": 0.85, "OUTSIDE_COMMS": 0.90}
|
| 61 |
tok = AutoTokenizer.from_pretrained(HERE)
|
| 62 |
base = AutoModelForSequenceClassification.from_pretrained(
|
|
@@ -78,7 +78,7 @@ def vader_flags(seg):
|
|
| 78 |
|
| 79 |
|
| 80 |
def clean_tfidf(text):
|
| 81 |
-
#lowercase, strip punctuation, drop stopwords, stem
|
| 82 |
text = re.sub(r"[^a-z\s]", " ", str(text).lower())
|
| 83 |
return " ".join(stemmer.stem(w) for w in text.split() if w not in STOP and len(w) > 1)
|
| 84 |
|
|
@@ -97,7 +97,7 @@ def bert_probs(segs):
|
|
| 97 |
return np.vstack(out)
|
| 98 |
|
| 99 |
|
| 100 |
-
#
|
| 101 |
|
| 102 |
#five cleaning rules in a fixed order
|
| 103 |
WRAPPED_RE = re.compile(r"\n([A-Z])\n([A-Z][A-Za-z\-]{1,30}:)")
|
|
@@ -194,7 +194,7 @@ def segment(text):
|
|
| 194 |
return [c for c in chunks if len(c.split()) >= MIN_WORDS]
|
| 195 |
|
| 196 |
|
| 197 |
-
#
|
| 198 |
|
| 199 |
def read_input(file, text):
|
| 200 |
if file is None:
|
|
@@ -278,13 +278,11 @@ def analyse(file, text):
|
|
| 278 |
rows = ""
|
| 279 |
for i, c in enumerate(CATEGORIES):
|
| 280 |
counts = [int(v[:, i].sum()), int((t[:, i] >= 0.5).sum()), int((b[:, i] >= BERT_THRESHOLDS[c]).sum())]
|
| 281 |
-
cells = "".join(bar(k, n, col) for k, col in zip(counts, ["#
|
| 282 |
rows += f"<tr><td>{NAMES[c]}</td>{cells}<td>{meter(b[:, i].max(), BERT_THRESHOLDS[c])}</td></tr>"
|
| 283 |
s5 = (f"<h4>5. Classification: all three models</h4>"
|
| 284 |
f"<table><tr><th>Category</th><th>VADER</th><th>TF-IDF</th><th>BERT + LoRA</th><th>BERT top score vs threshold</th></tr>{rows}</table>"
|
| 285 |
-
f"<p class='small'>Segments flagged out of {n}. White line = BERT threshold, the bar fills to the top segment score "
|
| 286 |
-
f"(indigo if it passes). BERT is my final model. A research prototype, it flags passages for a person to read "
|
| 287 |
-
f"and does not diagnose anyone.</p>")
|
| 288 |
|
| 289 |
order = np.argsort(-b.max(axis=1)) #strongest segments first for shap
|
| 290 |
choices = [f"#{k + 1} (BERT max {b[k].max():.2f}) {segs[k][:60]}..." for k in order]
|
|
@@ -313,7 +311,7 @@ def explain(state, choice, category):
|
|
| 313 |
f"<div class='seg' style='line-height:1.9'>{spans}</div>")
|
| 314 |
|
| 315 |
|
| 316 |
-
#
|
| 317 |
|
| 318 |
CSS = """
|
| 319 |
.gradio-container {background: radial-gradient(ellipse at top, #141b33 0%, #05070f 70%) !important;}
|
|
@@ -345,7 +343,7 @@ with gr.Blocks(title="SPACEMIND") as demo:
|
|
| 345 |
#right side: the stages
|
| 346 |
with gr.Column():
|
| 347 |
out1 = gr.Textbox(label="1. Input (word count)")
|
| 348 |
-
out2 = gr.Textbox(label="2. Cleaning
|
| 349 |
out3 = gr.Textbox(label="3. Removing the interviewer", lines=3)
|
| 350 |
out4 = gr.HTML(label="4. Segmenting")
|
| 351 |
out5 = gr.HTML(label="5. Classification")
|
|
|
|
| 1 |
try:
|
| 2 |
+
import spaces
|
| 3 |
GPU = spaces.GPU
|
| 4 |
except ImportError:
|
| 5 |
+
GPU = lambda f: f
|
| 6 |
|
| 7 |
import gradio as gr
|
| 8 |
import torch
|
|
|
|
| 32 |
"ADJUSTMENT": "Adjustment and coping", "OUTSIDE_COMMS": "Outside communication"}
|
| 33 |
|
| 34 |
|
| 35 |
+
# the three models
|
| 36 |
|
| 37 |
+
#1. vader baseline: keyword lists + sentiment gate
|
| 38 |
KEYWORDS = {
|
| 39 |
"WORKLOAD": ["exhausted", "exhaustion", "tired", "fatigue", "overworked", "sleep", "schedule", "timeline",
|
| 40 |
"overtime", "demanding", "heavy", "restless", "shift", "tasks", "pressure"],
|
|
|
|
| 56 |
stemmer = SnowballStemmer("english")
|
| 57 |
|
| 58 |
#3. bert-base-uncased + my lora adapter (the adapter files sit next to this app.py)
|
| 59 |
+
|
| 60 |
BERT_THRESHOLDS = {"WORKLOAD": 0.65, "TEAM": 0.90, "ADJUSTMENT": 0.85, "OUTSIDE_COMMS": 0.90}
|
| 61 |
tok = AutoTokenizer.from_pretrained(HERE)
|
| 62 |
base = AutoModelForSequenceClassification.from_pretrained(
|
|
|
|
| 78 |
|
| 79 |
|
| 80 |
def clean_tfidf(text):
|
| 81 |
+
#lowercase, strip punctuation, drop stopwords, stem
|
| 82 |
text = re.sub(r"[^a-z\s]", " ", str(text).lower())
|
| 83 |
return " ".join(stemmer.stem(w) for w in text.split() if w not in STOP and len(w) > 1)
|
| 84 |
|
|
|
|
| 97 |
return np.vstack(out)
|
| 98 |
|
| 99 |
|
| 100 |
+
#the text pipeline
|
| 101 |
|
| 102 |
#five cleaning rules in a fixed order
|
| 103 |
WRAPPED_RE = re.compile(r"\n([A-Z])\n([A-Z][A-Za-z\-]{1,30}:)")
|
|
|
|
| 194 |
return [c for c in chunks if len(c.split()) >= MIN_WORDS]
|
| 195 |
|
| 196 |
|
| 197 |
+
#running everything
|
| 198 |
|
| 199 |
def read_input(file, text):
|
| 200 |
if file is None:
|
|
|
|
| 278 |
rows = ""
|
| 279 |
for i, c in enumerate(CATEGORIES):
|
| 280 |
counts = [int(v[:, i].sum()), int((t[:, i] >= 0.5).sum()), int((b[:, i] >= BERT_THRESHOLDS[c]).sum())]
|
| 281 |
+
cells = "".join(bar(k, n, col) for k, col in zip(counts, ["#f472b6", "#a78bfa", "#34d399"]))
|
| 282 |
rows += f"<tr><td>{NAMES[c]}</td>{cells}<td>{meter(b[:, i].max(), BERT_THRESHOLDS[c])}</td></tr>"
|
| 283 |
s5 = (f"<h4>5. Classification: all three models</h4>"
|
| 284 |
f"<table><tr><th>Category</th><th>VADER</th><th>TF-IDF</th><th>BERT + LoRA</th><th>BERT top score vs threshold</th></tr>{rows}</table>"
|
| 285 |
+
f"<p class='small'>Segments flagged out of {n}. White line = BERT threshold, the bar fills to the top segment score </p>")
|
|
|
|
|
|
|
| 286 |
|
| 287 |
order = np.argsort(-b.max(axis=1)) #strongest segments first for shap
|
| 288 |
choices = [f"#{k + 1} (BERT max {b[k].max():.2f}) {segs[k][:60]}..." for k in order]
|
|
|
|
| 311 |
f"<div class='seg' style='line-height:1.9'>{spans}</div>")
|
| 312 |
|
| 313 |
|
| 314 |
+
# the page
|
| 315 |
|
| 316 |
CSS = """
|
| 317 |
.gradio-container {background: radial-gradient(ellipse at top, #141b33 0%, #05070f 70%) !important;}
|
|
|
|
| 343 |
#right side: the stages
|
| 344 |
with gr.Column():
|
| 345 |
out1 = gr.Textbox(label="1. Input (word count)")
|
| 346 |
+
out2 = gr.Textbox(label="2. Cleaning")
|
| 347 |
out3 = gr.Textbox(label="3. Removing the interviewer", lines=3)
|
| 348 |
out4 = gr.HTML(label="4. Segmenting")
|
| 349 |
out5 = gr.HTML(label="5. Classification")
|