added missing functions from old transcriptTest.py to dataloaderImp.py
Browse files- dataloaderImp.py +35 -0
dataloaderImp.py
CHANGED
|
@@ -77,6 +77,41 @@ def extract_text_features(cha_path):
|
|
| 77 |
print(f"Text extraction failed for {cha_path}: {e}")
|
| 78 |
return None
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
def pull_audio(docID: str):
|
| 81 |
"""
|
| 82 |
Downloads .wav from Firebase Storage, fetches age/gender from Firestore,
|
|
|
|
| 77 |
print(f"Text extraction failed for {cha_path}: {e}")
|
| 78 |
return None
|
| 79 |
|
| 80 |
+
def text_to_cha_lines(text):
|
| 81 |
+
text = text.replace("...", " ELLIPSIS_MARKER ")
|
| 82 |
+
sentences = []
|
| 83 |
+
current = ""
|
| 84 |
+
for char in text:
|
| 85 |
+
current += char
|
| 86 |
+
if char in ".!?":
|
| 87 |
+
sentences.append(current.strip())
|
| 88 |
+
current = ""
|
| 89 |
+
if current.strip():
|
| 90 |
+
sentences.append(current.strip())
|
| 91 |
+
sentences = [s.replace(" ELLIPSIS_MARKER ", " ... ") for s in sentences]
|
| 92 |
+
return sentences
|
| 93 |
+
|
| 94 |
+
def generate_mor_gra_line(doc):
|
| 95 |
+
mor_tokens = []
|
| 96 |
+
gra_tokens = []
|
| 97 |
+
for i, token in enumerate(doc):
|
| 98 |
+
if token.is_space or token.is_punct:
|
| 99 |
+
continue
|
| 100 |
+
pos = token.pos_.lower() if token.pos_ != 'PROPN' else 'n:prop'
|
| 101 |
+
lemma = token.lemma_.lower()
|
| 102 |
+
pos_map = {
|
| 103 |
+
'noun': 'n', 'verb': 'v', 'adj': 'adj', 'adv': 'adv',
|
| 104 |
+
'pron': 'pro', 'det': 'det', 'adp': 'prep', 'conj': 'conj',
|
| 105 |
+
'num': 'num', 'part': 'part', 'intj': 'co', 'propn': 'n:prop'
|
| 106 |
+
}
|
| 107 |
+
pos_tag = pos_map.get(pos, pos)
|
| 108 |
+
mor_tokens.append(f"{lemma}|{pos_tag}")
|
| 109 |
+
head_idx = token.head.i + 1
|
| 110 |
+
dep = token.dep_.lower()
|
| 111 |
+
gra_tokens.append(f"{head_idx}|{dep}")
|
| 112 |
+
return " ".join(mor_tokens), " ".join(gra_tokens)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
def pull_audio(docID: str):
|
| 116 |
"""
|
| 117 |
Downloads .wav from Firebase Storage, fetches age/gender from Firestore,
|