ssg1 commited on
Commit
1b369ff
·
verified ·
1 Parent(s): cc00170

Upload the Doc Compass app bundle

Browse files
checkpoints/tfidf_stage1.joblib CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fb339f36161c91408385df4fecbb5385f98ca73432aa0bba2fce6ada94088331
3
  size 1111084
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ba1b5886d760898f605968d3d1fd194db2f6caac588cdf0d1c7cdf4e359a532f
3
  size 1111084
doccompass/paths.py CHANGED
@@ -6,11 +6,12 @@ from pathlib import Path
6
  # them would point ROOT at the cache's blob folder instead of the folder holding checkpoints/.
7
  ROOT = Path(__file__).absolute().parents[1]
8
  PUBLIC = ROOT / "data" / "public" # downloaded and derived public data
 
9
  MANUAL = ROOT / "data" / "manual" # our pool and labels
10
  MODELS = ROOT / "checkpoints"
11
 
12
  MEDIQ_POSTS = PUBLIC / "mediq_posts.csv"
13
- PATIENT_COMMENTS = PUBLIC / "patient_comments.csv"
14
  POOL_IDS = MANUAL / "pool.ids.csv" # committed: which posts we label, and who labels them
15
  POOL = MANUAL / "pool.csv" # local only: the same posts with their text
16
  DRAFT_LABELS = MANUAL / "draft.labels.csv" # first-pass labels shown in the labelling tool
 
6
  # them would point ROOT at the cache's blob folder instead of the folder holding checkpoints/.
7
  ROOT = Path(__file__).absolute().parents[1]
8
  PUBLIC = ROOT / "data" / "public" # downloaded and derived public data
9
+ SYNTHETIC = ROOT / "data" / "synthetic" # generated or apparently generated data; never counted as ours
10
  MANUAL = ROOT / "data" / "manual" # our pool and labels
11
  MODELS = ROOT / "checkpoints"
12
 
13
  MEDIQ_POSTS = PUBLIC / "mediq_posts.csv"
14
+ PATIENT_COMMENTS = SYNTHETIC / "patient_comments.csv"
15
  POOL_IDS = MANUAL / "pool.ids.csv" # committed: which posts we label, and who labels them
16
  POOL = MANUAL / "pool.csv" # local only: the same posts with their text
17
  DRAFT_LABELS = MANUAL / "draft.labels.csv" # first-pass labels shown in the labelling tool
doccompass/patient_comments.py CHANGED
@@ -2,6 +2,10 @@
2
 
3
  Source: Mendeley Data, DOI 10.17632/2twgjzpn82.2, CC BY 4.0. Each comment has a symptom
4
  category; CATEGORY_MAP sends each category to one of our labels, or drops it.
 
 
 
 
5
  See docs/label-set.md for the reasoning behind each choice.
6
  """
7
 
@@ -16,7 +20,7 @@ import pandas as pd
16
 
17
  from .labels import GP, LABELS
18
  from .mediq import dedupe_key
19
- from .paths import PUBLIC
20
 
21
  _URL = "https://data.mendeley.com/public-files/datasets/2twgjzpn82/files/{file_id}/file_downloaded"
22
  _FILES = { # name: (Mendeley file id, sha256). We ignore the source's own train/test split.
@@ -83,7 +87,7 @@ def _download(name: str, folder: Path) -> Path:
83
 
84
  def load_patient_comments() -> pd.DataFrame:
85
  """Return unique comments as (text, label) in our label set, with emoji removed."""
86
- folder = PUBLIC / "patient_comments_source"
87
  rows = pd.concat([pd.read_excel(_download(name, folder)) for name in _FILES], ignore_index=True)
88
  rows["label"] = rows["Patient_Category"].map(lambda category: _MAP_BY_KEY[_category_key(category)])
89
  rows = rows.dropna(subset=["label"])
 
2
 
3
  Source: Mendeley Data, DOI 10.17632/2twgjzpn82.2, CC BY 4.0. Each comment has a symptom
4
  category; CATEGORY_MAP sends each category to one of our labels, or drops it.
5
+
6
+ The comments appear to be generated rather than written by real patients (emoji matched to
7
+ each symptom, inserted typos, repeated openings), so we store them under data/synthetic and
8
+ use them for stage-1 training only.
9
  See docs/label-set.md for the reasoning behind each choice.
10
  """
11
 
 
20
 
21
  from .labels import GP, LABELS
22
  from .mediq import dedupe_key
23
+ from .paths import SYNTHETIC
24
 
25
  _URL = "https://data.mendeley.com/public-files/datasets/2twgjzpn82/files/{file_id}/file_downloaded"
26
  _FILES = { # name: (Mendeley file id, sha256). We ignore the source's own train/test split.
 
87
 
88
  def load_patient_comments() -> pd.DataFrame:
89
  """Return unique comments as (text, label) in our label set, with emoji removed."""
90
+ folder = SYNTHETIC / "patient_comments_source"
91
  rows = pd.concat([pd.read_excel(_download(name, folder)) for name in _FILES], ignore_index=True)
92
  rows["label"] = rows["Patient_Category"].map(lambda category: _MAP_BY_KEY[_category_key(category)])
93
  rows = rows.dropna(subset=["label"])
doccompass/redflags.py CHANGED
@@ -32,9 +32,10 @@ _CANT = r"(?:can'?t|cannot|can not|unable to|couldn'?t)"
32
 
33
  RULES = [
34
  _rule("chest pain", "Chest pain or discomfort lasting for two minutes or more", MEDLINEPLUS,
35
- r"\bchest (?:pain|pressure|tightness|discomfort)\b|\bpain in (?:my|the|his|her) chest\b"),
 
36
  _rule("breathing", "Breathing problems (difficulty breathing, shortness of breath)", MEDLINEPLUS,
37
- rf"\b(?:{_CANT}|struggling to|hard to|trouble|difficulty) breath(?:e|ing)\b|\bshort(?:ness)? of breath\b|\bgasping for (?:air|breath)\b"),
38
  _rule("stroke signs", "Sudden numbness or weakness in the face, arm, or leg; sudden confusion or trouble speaking", CDC_STROKE,
39
  rf"\bface (?:is |was )?drooping\b|\bslurred speech\b|\b(?:{_CANT}|trouble) (?:speak|speaking|talk|talking)\b"
40
  r"|\bsudden(?:ly)? (?:numb|weak|confus)\w*|\b(?:numb|weak)\w* on (?:one|the left|the right|my left|my right) side\b"),
@@ -44,7 +45,8 @@ RULES = [
44
  _rule("loss of consciousness", "Fainting or loss of consciousness", MEDLINEPLUS,
45
  r"\b(?:passed out|fainted|lost consciousness|unconscious|unresponsive)\b"),
46
  _rule("suicidal thoughts", "Feeling of committing suicide or murder", MEDLINEPLUS,
47
- r"\bsuicid\w*|\bkill(?:ing)? myself\b|\bend(?:ing)? my (?:own )?life\b|\bwant(?:ed)? to die\b|\bdon'?t want to (?:live|be alive)\b"),
 
48
  _rule("face or throat swelling", "Swelling of the face, eyes, or tongue", MEDLINEPLUS,
49
  r"\b(?:throat|tongue|lips?|face) (?:is |are |was |were )?(?:swelling|swollen|closing)\b"
50
  r"|\bswelling (?:of|in) (?:my |the )?(?:face|tongue|throat|lips?)\b|\banaphyla\w*"),
@@ -54,13 +56,18 @@ RULES = [
54
  r"\boverdos\w*|(?<!food )\bpoison(?:ed|ing)\b|\b(?:swallowed|drank|ingested) (?:bleach|poison|detergent|antifreeze)\b"),
55
  _rule("sudden severe pain", "Sudden, severe pain anywhere in the body; sudden severe headache with no known cause", MEDLINEPLUS,
56
  r"\bsudden(?:ly)?\b[^.!?]{0,40}\b(?:severe|excruciating|unbearable|worst)\b[^.!?]{0,25}\b(?:pain|headache)\b"
57
- r"|\bworst (?:headache|pain) of my life\b"),
58
  _rule("sudden vision change", "Sudden trouble seeing", CDC_STROKE,
59
  rf"\bsudden(?:ly)? (?:lost|loss of|blurred|blurry|double) vision\b|\bsuddenly {_CANT} see\b|\bsudden(?:ly)? (?:went )?blind\b"),
60
  ]
61
 
62
- # A match is ignored when one of these appears just before it ("no chest pain").
63
- _NEGATION = re.compile(r"\b(?:no|not|never|without|denies|deny|didn'?t|don'?t have|doesn'?t have|haven'?t)\b[^.!?,;]{0,25}$", re.IGNORECASE)
 
 
 
 
 
64
 
65
 
66
  def find_flags(text: str) -> list[dict]:
@@ -68,7 +75,7 @@ def find_flags(text: str) -> list[dict]:
68
  flags = []
69
  for rule in RULES:
70
  for match in rule.pattern.finditer(text):
71
- if _NEGATION.search(text[max(0, match.start() - 40):match.start()]):
72
  continue
73
  flags.append({"rule": rule.name, "matched": match.group(0), "sign": rule.sign,
74
  "source": rule.source[0], "url": rule.source[1]})
 
32
 
33
  RULES = [
34
  _rule("chest pain", "Chest pain or discomfort lasting for two minutes or more", MEDLINEPLUS,
35
+ r"\bchest (?:pain|pressure|tightness|discomfort)\b|\bpain in (?:my|the|his|her) chest\b"
36
+ r"|\bchest (?:hurts|is hurting|feels tight|feels heavy)\b"),
37
  _rule("breathing", "Breathing problems (difficulty breathing, shortness of breath)", MEDLINEPLUS,
38
+ rf"\b(?:{_CANT}|can barely|barely able to|struggling to|hard to|trouble|difficulty) breath(?:e|ing)\b|\bshort(?:ness)? of breath\b|\bgasping for (?:air|breath)\b"),
39
  _rule("stroke signs", "Sudden numbness or weakness in the face, arm, or leg; sudden confusion or trouble speaking", CDC_STROKE,
40
  rf"\bface (?:is |was )?drooping\b|\bslurred speech\b|\b(?:{_CANT}|trouble) (?:speak|speaking|talk|talking)\b"
41
  r"|\bsudden(?:ly)? (?:numb|weak|confus)\w*|\b(?:numb|weak)\w* on (?:one|the left|the right|my left|my right) side\b"),
 
45
  _rule("loss of consciousness", "Fainting or loss of consciousness", MEDLINEPLUS,
46
  r"\b(?:passed out|fainted|lost consciousness|unconscious|unresponsive)\b"),
47
  _rule("suicidal thoughts", "Feeling of committing suicide or murder", MEDLINEPLUS,
48
+ r"\bsuicid\w*|\bkill(?:ing)? myself\b|\bend(?:ing)? my (?:own )?life\b|\bwant(?:ed)? to die\b|\bdon'?t want to (?:live|be alive)\b"
49
+ r"|\bend(?:ing)? it all\b|\btake my (?:own )?life\b"),
50
  _rule("face or throat swelling", "Swelling of the face, eyes, or tongue", MEDLINEPLUS,
51
  r"\b(?:throat|tongue|lips?|face) (?:is |are |was |were )?(?:swelling|swollen|closing)\b"
52
  r"|\bswelling (?:of|in) (?:my |the )?(?:face|tongue|throat|lips?)\b|\banaphyla\w*"),
 
56
  r"\boverdos\w*|(?<!food )\bpoison(?:ed|ing)\b|\b(?:swallowed|drank|ingested) (?:bleach|poison|detergent|antifreeze)\b"),
57
  _rule("sudden severe pain", "Sudden, severe pain anywhere in the body; sudden severe headache with no known cause", MEDLINEPLUS,
58
  r"\bsudden(?:ly)?\b[^.!?]{0,40}\b(?:severe|excruciating|unbearable|worst)\b[^.!?]{0,25}\b(?:pain|headache)\b"
59
+ r"|\bworst (?:headache|pain) (?:of my life|ever)\b"),
60
  _rule("sudden vision change", "Sudden trouble seeing", CDC_STROKE,
61
  rf"\bsudden(?:ly)? (?:lost|loss of|blurred|blurry|double) vision\b|\bsuddenly {_CANT} see\b|\bsudden(?:ly)? (?:went )?blind\b"),
62
  ]
63
 
64
+ # A match is ignored when a negation governs it: "no chest pain", "I don't want to die",
65
+ # "never had any trouble breathing". Only short filler words may sit between the two, so
66
+ # "not sure why my chest pain started" and "I don't know why but my chest hurts" still flag.
67
+ _NEGATION = re.compile(
68
+ r"\b(?:no|not|never|without|denies|deny|didn'?t|don'?t|doesn'?t|haven'?t|hasn'?t|isn'?t|wasn'?t)"
69
+ r"(?:\s+(?:have|had|has|having|got|get|getting|any|a|an|the|real|really|much|more|been|"
70
+ r"experienced|experiencing|feel|felt|noticed|want|wanted|to|this|that|such)){0,4}\s*$", re.IGNORECASE)
71
 
72
 
73
  def find_flags(text: str) -> list[dict]:
 
75
  flags = []
76
  for rule in RULES:
77
  for match in rule.pattern.finditer(text):
78
+ if _NEGATION.search(text[max(0, match.start() - 60):match.start()]):
79
  continue
80
  flags.append({"rule": rule.name, "matched": match.group(0), "sign": rule.sign,
81
  "source": rule.source[0], "url": rule.source[1]})