plutoa386 commited on
Commit
cf3d76d
·
1 Parent(s): 0708ccb

added missing functions from old transcriptTest.py to dataloaderImp.py

Browse files
Files changed (1) hide show
  1. dataloaderImp.py +35 -0
dataloaderImp.py CHANGED
@@ -77,6 +77,41 @@ def extract_text_features(cha_path):
77
  print(f"Text extraction failed for {cha_path}: {e}")
78
  return None
79
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  def pull_audio(docID: str):
81
  """
82
  Downloads .wav from Firebase Storage, fetches age/gender from Firestore,
 
77
  print(f"Text extraction failed for {cha_path}: {e}")
78
  return None
79
 
80
+ def text_to_cha_lines(text):
81
+ text = text.replace("...", " ELLIPSIS_MARKER ")
82
+ sentences = []
83
+ current = ""
84
+ for char in text:
85
+ current += char
86
+ if char in ".!?":
87
+ sentences.append(current.strip())
88
+ current = ""
89
+ if current.strip():
90
+ sentences.append(current.strip())
91
+ sentences = [s.replace(" ELLIPSIS_MARKER ", " ... ") for s in sentences]
92
+ return sentences
93
+
94
+ def generate_mor_gra_line(doc):
95
+ mor_tokens = []
96
+ gra_tokens = []
97
+ for i, token in enumerate(doc):
98
+ if token.is_space or token.is_punct:
99
+ continue
100
+ pos = token.pos_.lower() if token.pos_ != 'PROPN' else 'n:prop'
101
+ lemma = token.lemma_.lower()
102
+ pos_map = {
103
+ 'noun': 'n', 'verb': 'v', 'adj': 'adj', 'adv': 'adv',
104
+ 'pron': 'pro', 'det': 'det', 'adp': 'prep', 'conj': 'conj',
105
+ 'num': 'num', 'part': 'part', 'intj': 'co', 'propn': 'n:prop'
106
+ }
107
+ pos_tag = pos_map.get(pos, pos)
108
+ mor_tokens.append(f"{lemma}|{pos_tag}")
109
+ head_idx = token.head.i + 1
110
+ dep = token.dep_.lower()
111
+ gra_tokens.append(f"{head_idx}|{dep}")
112
+ return " ".join(mor_tokens), " ".join(gra_tokens)
113
+
114
+
115
  def pull_audio(docID: str):
116
  """
117
  Downloads .wav from Firebase Storage, fetches age/gender from Firestore,