Spaces:
Runtime error
Runtime error
Dmitry Kirsanov commited on
Commit ·
6e287b6
1
Parent(s): 7b67970
test pass
Browse files- fad.py +46 -21
- understand.py +10 -4
fad.py
CHANGED
|
@@ -36,7 +36,7 @@ class Fad:
|
|
| 36 |
if isinstance(value, int): # any int value is supposed to be a ref to id
|
| 37 |
return self.parentfad.notes[value].get() # so we pass it to the target mark, recursively
|
| 38 |
elif isinstance(value, list):
|
| 39 |
-
return ", ".join([(self.parentfad.notes[v].get() if isinstance(v, int) else v) for v in value]) # do the same for each element in list
|
| 40 |
else:
|
| 41 |
return value
|
| 42 |
|
|
@@ -48,10 +48,10 @@ class Fad:
|
|
| 48 |
def add(self, name, value):
|
| 49 |
if isinstance(value, int): # any int value is supposed to be a ref to id
|
| 50 |
assert(value in self.parentfad.notes.keys()) # so we ensure this id exists
|
| 51 |
-
if
|
|
|
|
|
|
|
| 52 |
self.properties[name].append(value)
|
| 53 |
-
elif self.properties[name] is None:
|
| 54 |
-
self.properties[name] = [value]
|
| 55 |
else:
|
| 56 |
raise("Note: add: trying to add to list (properties) but the existing value is not a list")
|
| 57 |
|
|
@@ -83,7 +83,7 @@ class Fad:
|
|
| 83 |
tend = new.start()
|
| 84 |
id = int(fad[tend + 1:tend + self.idlength + 1])
|
| 85 |
return fad[tstart:tend], fad[tend], id, tend + self.idlength + 1
|
| 86 |
-
return fad[tstart:],
|
| 87 |
|
| 88 |
def get_plaintext(self, fad):
|
| 89 |
texts = []
|
|
@@ -91,7 +91,7 @@ class Fad:
|
|
| 91 |
while True:
|
| 92 |
text, markchar, id, i = self.get_next_mark(fad, i)
|
| 93 |
texts.append(text)
|
| 94 |
-
if i ==
|
| 95 |
break
|
| 96 |
return "".join(texts)
|
| 97 |
|
|
@@ -100,11 +100,11 @@ class Fad:
|
|
| 100 |
newfad = fad
|
| 101 |
while True:
|
| 102 |
text, markchar, id, newi = self.get_next_mark(newfad, i)
|
| 103 |
-
if newi ==
|
| 104 |
break
|
| 105 |
if self.notes[id].type == self.notetypes[notetype]:
|
| 106 |
id_s = f' id="{id}"' if show_id else ''
|
| 107 |
-
properties_s = ''.join([(f' {name}="{str(self.notes[id].get(name))}"' if show_properties else '') for name in self.notes[id].properties.keys()])
|
| 108 |
if markchar == '⌧':
|
| 109 |
newfad = newfad[:newi - self.idlength - 1] + f'<{notetype}{id_s}{properties_s}/>' + newfad[newi:]
|
| 110 |
else:
|
|
@@ -128,8 +128,8 @@ class Fad:
|
|
| 128 |
passed += len(text)
|
| 129 |
if passed >= to:
|
| 130 |
return i + len(text) - (passed - to)
|
| 131 |
-
elif newi ==
|
| 132 |
-
return
|
| 133 |
else:
|
| 134 |
i = newi
|
| 135 |
|
|
@@ -137,7 +137,7 @@ class Fad:
|
|
| 137 |
id = self.newnote(notetype)
|
| 138 |
start_i = self.goto_plaintext_i(fad, start)
|
| 139 |
end_i = self.goto_plaintext_i(fad, end)
|
| 140 |
-
if start_i ==
|
| 141 |
raise Exception("fad: add_span_mark: start or end outside range")
|
| 142 |
newfad = fad[:start_i] + "⌦" + '{:08d}'.format(id) + fad[start_i:end_i] + "⌫" + '{:08d}'.format(id) + fad[end_i:]
|
| 143 |
return newfad
|
|
@@ -145,7 +145,7 @@ class Fad:
|
|
| 145 |
def add_point_mark(self, fad, notetype, start):
|
| 146 |
id = self.newnote(notetype)
|
| 147 |
start_i = self.goto_plaintext_i(fad, start)
|
| 148 |
-
if start_i ==
|
| 149 |
raise Exception("fad: point_mark: start outside range")
|
| 150 |
newfad = fad[:start_i] + "⌧" + '{:08d}'.format(id) + fad[start_i:]
|
| 151 |
return newfad
|
|
@@ -155,14 +155,16 @@ class Fad:
|
|
| 155 |
n = 0
|
| 156 |
while True:
|
| 157 |
text, markchar, id, newi = self.get_next_mark(fad, i)
|
| 158 |
-
if
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
|
|
|
| 164 |
i = newi
|
| 165 |
|
|
|
|
| 166 |
def set_property_to_mark(self, fad, notetype, mark_n, property, value, valuetype, startfrom=0):
|
| 167 |
noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
|
| 168 |
if valuetype != None:
|
|
@@ -175,9 +177,9 @@ class Fad:
|
|
| 175 |
noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
|
| 176 |
if valuetype != None:
|
| 177 |
id = self.get_note_by_value(value, valuetype)
|
| 178 |
-
self.
|
| 179 |
else:
|
| 180 |
-
self.
|
| 181 |
|
| 182 |
|
| 183 |
if __name__ == "__main__":
|
|
@@ -220,5 +222,28 @@ if __name__ == "__main__":
|
|
| 220 |
|
| 221 |
f.set_property_to_mark(fad, "span", 3, "actors", "actor2", "actor")
|
| 222 |
|
| 223 |
-
print(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True))
|
| 224 |
assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6">56789</span id="4">B123456789<span id="5" actors="actor2">C1234</span id="6">56789</span id="5">')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
if isinstance(value, int): # any int value is supposed to be a ref to id
|
| 37 |
return self.parentfad.notes[value].get() # so we pass it to the target mark, recursively
|
| 38 |
elif isinstance(value, list):
|
| 39 |
+
return ", ".join([(self.parentfad.notes[v].get() if isinstance(v, int) else v) for v in value if v]) # do the same for each element in list
|
| 40 |
else:
|
| 41 |
return value
|
| 42 |
|
|
|
|
| 48 |
def add(self, name, value):
|
| 49 |
if isinstance(value, int): # any int value is supposed to be a ref to id
|
| 50 |
assert(value in self.parentfad.notes.keys()) # so we ensure this id exists
|
| 51 |
+
if not name in self.properties.keys() or self.properties[name] is None:
|
| 52 |
+
self.properties[name] = [value] if value is not None else []
|
| 53 |
+
elif isinstance(self.properties[name], list):
|
| 54 |
self.properties[name].append(value)
|
|
|
|
|
|
|
| 55 |
else:
|
| 56 |
raise("Note: add: trying to add to list (properties) but the existing value is not a list")
|
| 57 |
|
|
|
|
| 83 |
tend = new.start()
|
| 84 |
id = int(fad[tend + 1:tend + self.idlength + 1])
|
| 85 |
return fad[tstart:tend], fad[tend], id, tend + self.idlength + 1
|
| 86 |
+
return fad[tstart:], None, None, None # no marks till the end
|
| 87 |
|
| 88 |
def get_plaintext(self, fad):
|
| 89 |
texts = []
|
|
|
|
| 91 |
while True:
|
| 92 |
text, markchar, id, i = self.get_next_mark(fad, i)
|
| 93 |
texts.append(text)
|
| 94 |
+
if i == None:
|
| 95 |
break
|
| 96 |
return "".join(texts)
|
| 97 |
|
|
|
|
| 100 |
newfad = fad
|
| 101 |
while True:
|
| 102 |
text, markchar, id, newi = self.get_next_mark(newfad, i)
|
| 103 |
+
if newi == None:
|
| 104 |
break
|
| 105 |
if self.notes[id].type == self.notetypes[notetype]:
|
| 106 |
id_s = f' id="{id}"' if show_id else ''
|
| 107 |
+
properties_s = ''.join([(f' {name}="{str(self.notes[id].get(name))}"' if show_properties and self.notes[id].get(name) else '') for name in self.notes[id].properties.keys()]) # show if a property's value is not [], None, or 0
|
| 108 |
if markchar == '⌧':
|
| 109 |
newfad = newfad[:newi - self.idlength - 1] + f'<{notetype}{id_s}{properties_s}/>' + newfad[newi:]
|
| 110 |
else:
|
|
|
|
| 128 |
passed += len(text)
|
| 129 |
if passed >= to:
|
| 130 |
return i + len(text) - (passed - to)
|
| 131 |
+
elif newi == None:
|
| 132 |
+
return None # could not reach to
|
| 133 |
else:
|
| 134 |
i = newi
|
| 135 |
|
|
|
|
| 137 |
id = self.newnote(notetype)
|
| 138 |
start_i = self.goto_plaintext_i(fad, start)
|
| 139 |
end_i = self.goto_plaintext_i(fad, end)
|
| 140 |
+
if start_i == None or end_i == None:
|
| 141 |
raise Exception("fad: add_span_mark: start or end outside range")
|
| 142 |
newfad = fad[:start_i] + "⌦" + '{:08d}'.format(id) + fad[start_i:end_i] + "⌫" + '{:08d}'.format(id) + fad[end_i:]
|
| 143 |
return newfad
|
|
|
|
| 145 |
def add_point_mark(self, fad, notetype, start):
|
| 146 |
id = self.newnote(notetype)
|
| 147 |
start_i = self.goto_plaintext_i(fad, start)
|
| 148 |
+
if start_i == None:
|
| 149 |
raise Exception("fad: point_mark: start outside range")
|
| 150 |
newfad = fad[:start_i] + "⌧" + '{:08d}'.format(id) + fad[start_i:]
|
| 151 |
return newfad
|
|
|
|
| 155 |
n = 0
|
| 156 |
while True:
|
| 157 |
text, markchar, id, newi = self.get_next_mark(fad, i)
|
| 158 |
+
if markchar != '⌫':
|
| 159 |
+
if newi == None:
|
| 160 |
+
return None
|
| 161 |
+
if self.notes[id].type == self.notetypes[notetype]:
|
| 162 |
+
n += 1
|
| 163 |
+
if n == mark_n:
|
| 164 |
+
return id
|
| 165 |
i = newi
|
| 166 |
|
| 167 |
+
|
| 168 |
def set_property_to_mark(self, fad, notetype, mark_n, property, value, valuetype, startfrom=0):
|
| 169 |
noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
|
| 170 |
if valuetype != None:
|
|
|
|
| 177 |
noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
|
| 178 |
if valuetype != None:
|
| 179 |
id = self.get_note_by_value(value, valuetype)
|
| 180 |
+
self.notes[noteid].add(property, id)
|
| 181 |
else:
|
| 182 |
+
self.notes[noteid].add(property, value)
|
| 183 |
|
| 184 |
|
| 185 |
if __name__ == "__main__":
|
|
|
|
| 222 |
|
| 223 |
f.set_property_to_mark(fad, "span", 3, "actors", "actor2", "actor")
|
| 224 |
|
|
|
|
| 225 |
assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6">56789</span id="4">B123456789<span id="5" actors="actor2">C1234</span id="6">56789</span id="5">')
|
| 226 |
+
|
| 227 |
+
f.set_property_to_mark(fad, "span", 3, "actors", "actor1", "actor")
|
| 228 |
+
f.set_property_to_mark(fad, "span", 2, "actors", "actor1", "actor")
|
| 229 |
+
|
| 230 |
+
assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6" actors="actor1">56789</span id="4">B123456789<span id="5" actors="actor1">C1234</span id="6">56789</span id="5">')
|
| 231 |
+
|
| 232 |
+
f.set_property_to_mark(fad, "span", 3, "actors", None, "actor")
|
| 233 |
+
|
| 234 |
+
assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6" actors="actor1">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
|
| 235 |
+
|
| 236 |
+
f.add_property_to_mark(fad, "span", 1, "audiences", None, "audience")
|
| 237 |
+
f.add_property_to_mark(fad, "span", 1, "audiences", "audience1", "audience")
|
| 238 |
+
f.add_property_to_mark(fad, "span", 1, "audiences", "audience2", "audience")
|
| 239 |
+
|
| 240 |
+
assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1" audiences="audience1, audience2">A1234<span id="6" actors="actor1">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
|
| 241 |
+
|
| 242 |
+
f.add_property_to_mark(fad, "span", 1, "audiences", "audience3", "audience")
|
| 243 |
+
f.add_property_to_mark(fad, "span", 2, "audiences", "audience3", "audience")
|
| 244 |
+
|
| 245 |
+
#print(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True))
|
| 246 |
+
assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1" audiences="audience1, audience2, audience3">A1234<span id="6" actors="actor1" audiences="audience3">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
|
| 247 |
+
|
| 248 |
+
f.set_property_to_mark(fad, "span", 2, "audiences", None, "audience")
|
| 249 |
+
assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1" audiences="audience1, audience2, audience3">A1234<span id="6" actors="actor1">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
|
understand.py
CHANGED
|
@@ -170,7 +170,7 @@ Here's the document:
|
|
| 170 |
self.values.put("actors jackard distance", jackard["actors"])
|
| 171 |
|
| 172 |
# also store concepts in the fad marks
|
| 173 |
-
self.fad.
|
| 174 |
|
| 175 |
print(f"===Reconciled: Concepts: {self.values.get('concepts')}, Actors: {self.values.get('actors')}")
|
| 176 |
|
|
@@ -196,7 +196,7 @@ Output the line "ACTORS:" followed by your list of actors, one per line.
|
|
| 196 |
self.values.put("actors, generalized", generalized)
|
| 197 |
|
| 198 |
# also store concepts in the fad marks
|
| 199 |
-
self.fad.
|
| 200 |
|
| 201 |
|
| 202 |
def split_sentences(self):
|
|
@@ -207,7 +207,7 @@ Output the line "ACTORS:" followed by your list of actors, one per line.
|
|
| 207 |
print("ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source")
|
| 208 |
i = 0
|
| 209 |
for sent in sents:
|
| 210 |
-
newfad = fad.
|
| 211 |
self.values.put("document", newfad)
|
| 212 |
print(f"=== Sentence-split FAD : {newfad}")
|
| 213 |
# todo: pass this through LLM to verify and correct sentence splittings
|
|
@@ -253,7 +253,13 @@ Output just the document adding the actors attributes for sentences, but otherw
|
|
| 253 |
|
| 254 |
sents = re.split('</sentence>', annotated0)
|
| 255 |
for i, sent in enumerate(sents):
|
| 256 |
-
match = re.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 257 |
|
| 258 |
|
| 259 |
def split_document(self):
|
|
|
|
| 170 |
self.values.put("actors jackard distance", jackard["actors"])
|
| 171 |
|
| 172 |
# also store concepts in the fad marks
|
| 173 |
+
self.fad.add_notes_from_list(reconciled["concepts"], "concept")
|
| 174 |
|
| 175 |
print(f"===Reconciled: Concepts: {self.values.get('concepts')}, Actors: {self.values.get('actors')}")
|
| 176 |
|
|
|
|
| 196 |
self.values.put("actors, generalized", generalized)
|
| 197 |
|
| 198 |
# also store concepts in the fad marks
|
| 199 |
+
self.fad.add_notes_from_list(generalized, "actor")
|
| 200 |
|
| 201 |
|
| 202 |
def split_sentences(self):
|
|
|
|
| 207 |
print("ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source")
|
| 208 |
i = 0
|
| 209 |
for sent in sents:
|
| 210 |
+
newfad = fad.add_span_mark(newfad, "sentence", i, i+len(sent) - 1)
|
| 211 |
self.values.put("document", newfad)
|
| 212 |
print(f"=== Sentence-split FAD : {newfad}")
|
| 213 |
# todo: pass this through LLM to verify and correct sentence splittings
|
|
|
|
| 253 |
|
| 254 |
sents = re.split('</sentence>', annotated0)
|
| 255 |
for i, sent in enumerate(sents):
|
| 256 |
+
match = re.match('<sentence[^>]+actors="([^"]+)"')
|
| 257 |
+
if match:
|
| 258 |
+
actors = match.groups(1)
|
| 259 |
+
for actor in re.split(",\s+", actors):
|
| 260 |
+
self.fad.add_property_to_mark(self.values.get("document"), "sentence", i, "actors", actor, "actor")
|
| 261 |
+
else:
|
| 262 |
+
raise(f"annotate_sentences_with_actors: Bad markup from LLM: {sent}")
|
| 263 |
|
| 264 |
|
| 265 |
def split_document(self):
|