Instructions to use ssg1/doc-compass with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use ssg1/doc-compass with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="ssg1/doc-compass")# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("ssg1/doc-compass", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Upload the Doc Compass app bundle
Browse files- checkpoints/tfidf_stage1.joblib +1 -1
- doccompass/paths.py +2 -1
- doccompass/patient_comments.py +6 -2
- doccompass/redflags.py +14 -7
checkpoints/tfidf_stage1.joblib
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 1111084
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ba1b5886d760898f605968d3d1fd194db2f6caac588cdf0d1c7cdf4e359a532f
|
| 3 |
size 1111084
|
doccompass/paths.py
CHANGED
|
@@ -6,11 +6,12 @@ from pathlib import Path
|
|
| 6 |
# them would point ROOT at the cache's blob folder instead of the folder holding checkpoints/.
|
| 7 |
ROOT = Path(__file__).absolute().parents[1]
|
| 8 |
PUBLIC = ROOT / "data" / "public" # downloaded and derived public data
|
|
|
|
| 9 |
MANUAL = ROOT / "data" / "manual" # our pool and labels
|
| 10 |
MODELS = ROOT / "checkpoints"
|
| 11 |
|
| 12 |
MEDIQ_POSTS = PUBLIC / "mediq_posts.csv"
|
| 13 |
-
PATIENT_COMMENTS =
|
| 14 |
POOL_IDS = MANUAL / "pool.ids.csv" # committed: which posts we label, and who labels them
|
| 15 |
POOL = MANUAL / "pool.csv" # local only: the same posts with their text
|
| 16 |
DRAFT_LABELS = MANUAL / "draft.labels.csv" # first-pass labels shown in the labelling tool
|
|
|
|
| 6 |
# them would point ROOT at the cache's blob folder instead of the folder holding checkpoints/.
|
| 7 |
ROOT = Path(__file__).absolute().parents[1]
|
| 8 |
PUBLIC = ROOT / "data" / "public" # downloaded and derived public data
|
| 9 |
+
SYNTHETIC = ROOT / "data" / "synthetic" # generated or apparently generated data; never counted as ours
|
| 10 |
MANUAL = ROOT / "data" / "manual" # our pool and labels
|
| 11 |
MODELS = ROOT / "checkpoints"
|
| 12 |
|
| 13 |
MEDIQ_POSTS = PUBLIC / "mediq_posts.csv"
|
| 14 |
+
PATIENT_COMMENTS = SYNTHETIC / "patient_comments.csv"
|
| 15 |
POOL_IDS = MANUAL / "pool.ids.csv" # committed: which posts we label, and who labels them
|
| 16 |
POOL = MANUAL / "pool.csv" # local only: the same posts with their text
|
| 17 |
DRAFT_LABELS = MANUAL / "draft.labels.csv" # first-pass labels shown in the labelling tool
|
doccompass/patient_comments.py
CHANGED
|
@@ -2,6 +2,10 @@
|
|
| 2 |
|
| 3 |
Source: Mendeley Data, DOI 10.17632/2twgjzpn82.2, CC BY 4.0. Each comment has a symptom
|
| 4 |
category; CATEGORY_MAP sends each category to one of our labels, or drops it.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
See docs/label-set.md for the reasoning behind each choice.
|
| 6 |
"""
|
| 7 |
|
|
@@ -16,7 +20,7 @@ import pandas as pd
|
|
| 16 |
|
| 17 |
from .labels import GP, LABELS
|
| 18 |
from .mediq import dedupe_key
|
| 19 |
-
from .paths import
|
| 20 |
|
| 21 |
_URL = "https://data.mendeley.com/public-files/datasets/2twgjzpn82/files/{file_id}/file_downloaded"
|
| 22 |
_FILES = { # name: (Mendeley file id, sha256). We ignore the source's own train/test split.
|
|
@@ -83,7 +87,7 @@ def _download(name: str, folder: Path) -> Path:
|
|
| 83 |
|
| 84 |
def load_patient_comments() -> pd.DataFrame:
|
| 85 |
"""Return unique comments as (text, label) in our label set, with emoji removed."""
|
| 86 |
-
folder =
|
| 87 |
rows = pd.concat([pd.read_excel(_download(name, folder)) for name in _FILES], ignore_index=True)
|
| 88 |
rows["label"] = rows["Patient_Category"].map(lambda category: _MAP_BY_KEY[_category_key(category)])
|
| 89 |
rows = rows.dropna(subset=["label"])
|
|
|
|
| 2 |
|
| 3 |
Source: Mendeley Data, DOI 10.17632/2twgjzpn82.2, CC BY 4.0. Each comment has a symptom
|
| 4 |
category; CATEGORY_MAP sends each category to one of our labels, or drops it.
|
| 5 |
+
|
| 6 |
+
The comments appear to be generated rather than written by real patients (emoji matched to
|
| 7 |
+
each symptom, inserted typos, repeated openings), so we store them under data/synthetic and
|
| 8 |
+
use them for stage-1 training only.
|
| 9 |
See docs/label-set.md for the reasoning behind each choice.
|
| 10 |
"""
|
| 11 |
|
|
|
|
| 20 |
|
| 21 |
from .labels import GP, LABELS
|
| 22 |
from .mediq import dedupe_key
|
| 23 |
+
from .paths import SYNTHETIC
|
| 24 |
|
| 25 |
_URL = "https://data.mendeley.com/public-files/datasets/2twgjzpn82/files/{file_id}/file_downloaded"
|
| 26 |
_FILES = { # name: (Mendeley file id, sha256). We ignore the source's own train/test split.
|
|
|
|
| 87 |
|
| 88 |
def load_patient_comments() -> pd.DataFrame:
|
| 89 |
"""Return unique comments as (text, label) in our label set, with emoji removed."""
|
| 90 |
+
folder = SYNTHETIC / "patient_comments_source"
|
| 91 |
rows = pd.concat([pd.read_excel(_download(name, folder)) for name in _FILES], ignore_index=True)
|
| 92 |
rows["label"] = rows["Patient_Category"].map(lambda category: _MAP_BY_KEY[_category_key(category)])
|
| 93 |
rows = rows.dropna(subset=["label"])
|
doccompass/redflags.py
CHANGED
|
@@ -32,9 +32,10 @@ _CANT = r"(?:can'?t|cannot|can not|unable to|couldn'?t)"
|
|
| 32 |
|
| 33 |
RULES = [
|
| 34 |
_rule("chest pain", "Chest pain or discomfort lasting for two minutes or more", MEDLINEPLUS,
|
| 35 |
-
r"\bchest (?:pain|pressure|tightness|discomfort)\b|\bpain in (?:my|the|his|her) chest\b"
|
|
|
|
| 36 |
_rule("breathing", "Breathing problems (difficulty breathing, shortness of breath)", MEDLINEPLUS,
|
| 37 |
-
rf"\b(?:{_CANT}|struggling to|hard to|trouble|difficulty) breath(?:e|ing)\b|\bshort(?:ness)? of breath\b|\bgasping for (?:air|breath)\b"),
|
| 38 |
_rule("stroke signs", "Sudden numbness or weakness in the face, arm, or leg; sudden confusion or trouble speaking", CDC_STROKE,
|
| 39 |
rf"\bface (?:is |was )?drooping\b|\bslurred speech\b|\b(?:{_CANT}|trouble) (?:speak|speaking|talk|talking)\b"
|
| 40 |
r"|\bsudden(?:ly)? (?:numb|weak|confus)\w*|\b(?:numb|weak)\w* on (?:one|the left|the right|my left|my right) side\b"),
|
|
@@ -44,7 +45,8 @@ RULES = [
|
|
| 44 |
_rule("loss of consciousness", "Fainting or loss of consciousness", MEDLINEPLUS,
|
| 45 |
r"\b(?:passed out|fainted|lost consciousness|unconscious|unresponsive)\b"),
|
| 46 |
_rule("suicidal thoughts", "Feeling of committing suicide or murder", MEDLINEPLUS,
|
| 47 |
-
r"\bsuicid\w*|\bkill(?:ing)? myself\b|\bend(?:ing)? my (?:own )?life\b|\bwant(?:ed)? to die\b|\bdon'?t want to (?:live|be alive)\b"
|
|
|
|
| 48 |
_rule("face or throat swelling", "Swelling of the face, eyes, or tongue", MEDLINEPLUS,
|
| 49 |
r"\b(?:throat|tongue|lips?|face) (?:is |are |was |were )?(?:swelling|swollen|closing)\b"
|
| 50 |
r"|\bswelling (?:of|in) (?:my |the )?(?:face|tongue|throat|lips?)\b|\banaphyla\w*"),
|
|
@@ -54,13 +56,18 @@ RULES = [
|
|
| 54 |
r"\boverdos\w*|(?<!food )\bpoison(?:ed|ing)\b|\b(?:swallowed|drank|ingested) (?:bleach|poison|detergent|antifreeze)\b"),
|
| 55 |
_rule("sudden severe pain", "Sudden, severe pain anywhere in the body; sudden severe headache with no known cause", MEDLINEPLUS,
|
| 56 |
r"\bsudden(?:ly)?\b[^.!?]{0,40}\b(?:severe|excruciating|unbearable|worst)\b[^.!?]{0,25}\b(?:pain|headache)\b"
|
| 57 |
-
r"|\bworst (?:headache|pain) of my life\b"),
|
| 58 |
_rule("sudden vision change", "Sudden trouble seeing", CDC_STROKE,
|
| 59 |
rf"\bsudden(?:ly)? (?:lost|loss of|blurred|blurry|double) vision\b|\bsuddenly {_CANT} see\b|\bsudden(?:ly)? (?:went )?blind\b"),
|
| 60 |
]
|
| 61 |
|
| 62 |
-
# A match is ignored when
|
| 63 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
|
| 65 |
|
| 66 |
def find_flags(text: str) -> list[dict]:
|
|
@@ -68,7 +75,7 @@ def find_flags(text: str) -> list[dict]:
|
|
| 68 |
flags = []
|
| 69 |
for rule in RULES:
|
| 70 |
for match in rule.pattern.finditer(text):
|
| 71 |
-
if _NEGATION.search(text[max(0, match.start() -
|
| 72 |
continue
|
| 73 |
flags.append({"rule": rule.name, "matched": match.group(0), "sign": rule.sign,
|
| 74 |
"source": rule.source[0], "url": rule.source[1]})
|
|
|
|
| 32 |
|
| 33 |
RULES = [
|
| 34 |
_rule("chest pain", "Chest pain or discomfort lasting for two minutes or more", MEDLINEPLUS,
|
| 35 |
+
r"\bchest (?:pain|pressure|tightness|discomfort)\b|\bpain in (?:my|the|his|her) chest\b"
|
| 36 |
+
r"|\bchest (?:hurts|is hurting|feels tight|feels heavy)\b"),
|
| 37 |
_rule("breathing", "Breathing problems (difficulty breathing, shortness of breath)", MEDLINEPLUS,
|
| 38 |
+
rf"\b(?:{_CANT}|can barely|barely able to|struggling to|hard to|trouble|difficulty) breath(?:e|ing)\b|\bshort(?:ness)? of breath\b|\bgasping for (?:air|breath)\b"),
|
| 39 |
_rule("stroke signs", "Sudden numbness or weakness in the face, arm, or leg; sudden confusion or trouble speaking", CDC_STROKE,
|
| 40 |
rf"\bface (?:is |was )?drooping\b|\bslurred speech\b|\b(?:{_CANT}|trouble) (?:speak|speaking|talk|talking)\b"
|
| 41 |
r"|\bsudden(?:ly)? (?:numb|weak|confus)\w*|\b(?:numb|weak)\w* on (?:one|the left|the right|my left|my right) side\b"),
|
|
|
|
| 45 |
_rule("loss of consciousness", "Fainting or loss of consciousness", MEDLINEPLUS,
|
| 46 |
r"\b(?:passed out|fainted|lost consciousness|unconscious|unresponsive)\b"),
|
| 47 |
_rule("suicidal thoughts", "Feeling of committing suicide or murder", MEDLINEPLUS,
|
| 48 |
+
r"\bsuicid\w*|\bkill(?:ing)? myself\b|\bend(?:ing)? my (?:own )?life\b|\bwant(?:ed)? to die\b|\bdon'?t want to (?:live|be alive)\b"
|
| 49 |
+
r"|\bend(?:ing)? it all\b|\btake my (?:own )?life\b"),
|
| 50 |
_rule("face or throat swelling", "Swelling of the face, eyes, or tongue", MEDLINEPLUS,
|
| 51 |
r"\b(?:throat|tongue|lips?|face) (?:is |are |was |were )?(?:swelling|swollen|closing)\b"
|
| 52 |
r"|\bswelling (?:of|in) (?:my |the )?(?:face|tongue|throat|lips?)\b|\banaphyla\w*"),
|
|
|
|
| 56 |
r"\boverdos\w*|(?<!food )\bpoison(?:ed|ing)\b|\b(?:swallowed|drank|ingested) (?:bleach|poison|detergent|antifreeze)\b"),
|
| 57 |
_rule("sudden severe pain", "Sudden, severe pain anywhere in the body; sudden severe headache with no known cause", MEDLINEPLUS,
|
| 58 |
r"\bsudden(?:ly)?\b[^.!?]{0,40}\b(?:severe|excruciating|unbearable|worst)\b[^.!?]{0,25}\b(?:pain|headache)\b"
|
| 59 |
+
r"|\bworst (?:headache|pain) (?:of my life|ever)\b"),
|
| 60 |
_rule("sudden vision change", "Sudden trouble seeing", CDC_STROKE,
|
| 61 |
rf"\bsudden(?:ly)? (?:lost|loss of|blurred|blurry|double) vision\b|\bsuddenly {_CANT} see\b|\bsudden(?:ly)? (?:went )?blind\b"),
|
| 62 |
]
|
| 63 |
|
| 64 |
+
# A match is ignored when a negation governs it: "no chest pain", "I don't want to die",
|
| 65 |
+
# "never had any trouble breathing". Only short filler words may sit between the two, so
|
| 66 |
+
# "not sure why my chest pain started" and "I don't know why but my chest hurts" still flag.
|
| 67 |
+
_NEGATION = re.compile(
|
| 68 |
+
r"\b(?:no|not|never|without|denies|deny|didn'?t|don'?t|doesn'?t|haven'?t|hasn'?t|isn'?t|wasn'?t)"
|
| 69 |
+
r"(?:\s+(?:have|had|has|having|got|get|getting|any|a|an|the|real|really|much|more|been|"
|
| 70 |
+
r"experienced|experiencing|feel|felt|noticed|want|wanted|to|this|that|such)){0,4}\s*$", re.IGNORECASE)
|
| 71 |
|
| 72 |
|
| 73 |
def find_flags(text: str) -> list[dict]:
|
|
|
|
| 75 |
flags = []
|
| 76 |
for rule in RULES:
|
| 77 |
for match in rule.pattern.finditer(text):
|
| 78 |
+
if _NEGATION.search(text[max(0, match.start() - 60):match.start()]):
|
| 79 |
continue
|
| 80 |
flags.append({"rule": rule.name, "matched": match.group(0), "sign": rule.sign,
|
| 81 |
"source": rule.source[0], "url": rule.source[1]})
|