andrewbejjani commited on
Commit
4303586
·
1 Parent(s): 510741e

fix degree cleaning

Browse files

modified: src/utils.py

Files changed (1) hide show
  1. src/utils.py +2 -4
src/utils.py CHANGED
@@ -26,13 +26,11 @@ def smart_format(text):
26
  return res.strip()
27
 
28
  def clean_degree_text(text):
29
- """Normalize degree titles before within-school clustering."""
30
  if not isinstance(text, str): return ""
31
- text = re.sub(r'\band\b', '&', text, flags=re.IGNORECASE)
32
- text = re.sub(r'\bet\b', '&', text, flags=re.IGNORECASE)
33
  text = re.sub(r'[^\w\s\-&\+\']', ' ', text)
34
  text = re.sub(r'\s+', ' ', text).strip()
35
- return smart_format(text)
36
 
37
  def normalize_text(text):
38
  """Normalize text for accent-insensitive, case-insensitive comparisons."""
 
26
  return res.strip()
27
 
28
  def clean_degree_text(text):
29
+ """Normalize degree titles before clustering without changing casing."""
30
  if not isinstance(text, str): return ""
 
 
31
  text = re.sub(r'[^\w\s\-&\+\']', ' ', text)
32
  text = re.sub(r'\s+', ' ', text).strip()
33
+ return text
34
 
35
  def normalize_text(text):
36
  """Normalize text for accent-insensitive, case-insensitive comparisons."""