Dmitry Kirsanov commited on
Commit
6e287b6
·
1 Parent(s): 7b67970

test pass

Browse files
Files changed (2) hide show
  1. fad.py +46 -21
  2. understand.py +10 -4
fad.py CHANGED
@@ -36,7 +36,7 @@ class Fad:
36
  if isinstance(value, int): # any int value is supposed to be a ref to id
37
  return self.parentfad.notes[value].get() # so we pass it to the target mark, recursively
38
  elif isinstance(value, list):
39
- return ", ".join([(self.parentfad.notes[v].get() if isinstance(v, int) else v) for v in value]) # do the same for each element in list
40
  else:
41
  return value
42
 
@@ -48,10 +48,10 @@ class Fad:
48
  def add(self, name, value):
49
  if isinstance(value, int): # any int value is supposed to be a ref to id
50
  assert(value in self.parentfad.notes.keys()) # so we ensure this id exists
51
- if isinstance(self.properties[name], list):
 
 
52
  self.properties[name].append(value)
53
- elif self.properties[name] is None:
54
- self.properties[name] = [value]
55
  else:
56
  raise("Note: add: trying to add to list (properties) but the existing value is not a list")
57
 
@@ -83,7 +83,7 @@ class Fad:
83
  tend = new.start()
84
  id = int(fad[tend + 1:tend + self.idlength + 1])
85
  return fad[tstart:tend], fad[tend], id, tend + self.idlength + 1
86
- return fad[tstart:], -1, -1, -1 # no marks till the end
87
 
88
  def get_plaintext(self, fad):
89
  texts = []
@@ -91,7 +91,7 @@ class Fad:
91
  while True:
92
  text, markchar, id, i = self.get_next_mark(fad, i)
93
  texts.append(text)
94
- if i == -1:
95
  break
96
  return "".join(texts)
97
 
@@ -100,11 +100,11 @@ class Fad:
100
  newfad = fad
101
  while True:
102
  text, markchar, id, newi = self.get_next_mark(newfad, i)
103
- if newi == -1:
104
  break
105
  if self.notes[id].type == self.notetypes[notetype]:
106
  id_s = f' id="{id}"' if show_id else ''
107
- properties_s = ''.join([(f' {name}="{str(self.notes[id].get(name))}"' if show_properties else '') for name in self.notes[id].properties.keys()])
108
  if markchar == '⌧':
109
  newfad = newfad[:newi - self.idlength - 1] + f'<{notetype}{id_s}{properties_s}/>' + newfad[newi:]
110
  else:
@@ -128,8 +128,8 @@ class Fad:
128
  passed += len(text)
129
  if passed >= to:
130
  return i + len(text) - (passed - to)
131
- elif newi == -1:
132
- return -1 # could not reach to
133
  else:
134
  i = newi
135
 
@@ -137,7 +137,7 @@ class Fad:
137
  id = self.newnote(notetype)
138
  start_i = self.goto_plaintext_i(fad, start)
139
  end_i = self.goto_plaintext_i(fad, end)
140
- if start_i == -1 or end_i == -1:
141
  raise Exception("fad: add_span_mark: start or end outside range")
142
  newfad = fad[:start_i] + "⌦" + '{:08d}'.format(id) + fad[start_i:end_i] + "⌫" + '{:08d}'.format(id) + fad[end_i:]
143
  return newfad
@@ -145,7 +145,7 @@ class Fad:
145
  def add_point_mark(self, fad, notetype, start):
146
  id = self.newnote(notetype)
147
  start_i = self.goto_plaintext_i(fad, start)
148
- if start_i == -1:
149
  raise Exception("fad: point_mark: start outside range")
150
  newfad = fad[:start_i] + "⌧" + '{:08d}'.format(id) + fad[start_i:]
151
  return newfad
@@ -155,14 +155,16 @@ class Fad:
155
  n = 0
156
  while True:
157
  text, markchar, id, newi = self.get_next_mark(fad, i)
158
- if newi == -1:
159
- return None
160
- if self.notes[id].type == self.notetypes[notetype]:
161
- n += 1
162
- if n == mark_n:
163
- return id
 
164
  i = newi
165
 
 
166
  def set_property_to_mark(self, fad, notetype, mark_n, property, value, valuetype, startfrom=0):
167
  noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
168
  if valuetype != None:
@@ -175,9 +177,9 @@ class Fad:
175
  noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
176
  if valuetype != None:
177
  id = self.get_note_by_value(value, valuetype)
178
- self.marks[noteid].set(property, id)
179
  else:
180
- self.marks[noteid].set(property, value)
181
 
182
 
183
  if __name__ == "__main__":
@@ -220,5 +222,28 @@ if __name__ == "__main__":
220
 
221
  f.set_property_to_mark(fad, "span", 3, "actors", "actor2", "actor")
222
 
223
- print(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True))
224
  assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6">56789</span id="4">B123456789<span id="5" actors="actor2">C1234</span id="6">56789</span id="5">')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
36
  if isinstance(value, int): # any int value is supposed to be a ref to id
37
  return self.parentfad.notes[value].get() # so we pass it to the target mark, recursively
38
  elif isinstance(value, list):
39
+ return ", ".join([(self.parentfad.notes[v].get() if isinstance(v, int) else v) for v in value if v]) # do the same for each element in list
40
  else:
41
  return value
42
 
 
48
  def add(self, name, value):
49
  if isinstance(value, int): # any int value is supposed to be a ref to id
50
  assert(value in self.parentfad.notes.keys()) # so we ensure this id exists
51
+ if not name in self.properties.keys() or self.properties[name] is None:
52
+ self.properties[name] = [value] if value is not None else []
53
+ elif isinstance(self.properties[name], list):
54
  self.properties[name].append(value)
 
 
55
  else:
56
  raise("Note: add: trying to add to list (properties) but the existing value is not a list")
57
 
 
83
  tend = new.start()
84
  id = int(fad[tend + 1:tend + self.idlength + 1])
85
  return fad[tstart:tend], fad[tend], id, tend + self.idlength + 1
86
+ return fad[tstart:], None, None, None # no marks till the end
87
 
88
  def get_plaintext(self, fad):
89
  texts = []
 
91
  while True:
92
  text, markchar, id, i = self.get_next_mark(fad, i)
93
  texts.append(text)
94
+ if i == None:
95
  break
96
  return "".join(texts)
97
 
 
100
  newfad = fad
101
  while True:
102
  text, markchar, id, newi = self.get_next_mark(newfad, i)
103
+ if newi == None:
104
  break
105
  if self.notes[id].type == self.notetypes[notetype]:
106
  id_s = f' id="{id}"' if show_id else ''
107
+ properties_s = ''.join([(f' {name}="{str(self.notes[id].get(name))}"' if show_properties and self.notes[id].get(name) else '') for name in self.notes[id].properties.keys()]) # show if a property's value is not [], None, or 0
108
  if markchar == '⌧':
109
  newfad = newfad[:newi - self.idlength - 1] + f'<{notetype}{id_s}{properties_s}/>' + newfad[newi:]
110
  else:
 
128
  passed += len(text)
129
  if passed >= to:
130
  return i + len(text) - (passed - to)
131
+ elif newi == None:
132
+ return None # could not reach to
133
  else:
134
  i = newi
135
 
 
137
  id = self.newnote(notetype)
138
  start_i = self.goto_plaintext_i(fad, start)
139
  end_i = self.goto_plaintext_i(fad, end)
140
+ if start_i == None or end_i == None:
141
  raise Exception("fad: add_span_mark: start or end outside range")
142
  newfad = fad[:start_i] + "⌦" + '{:08d}'.format(id) + fad[start_i:end_i] + "⌫" + '{:08d}'.format(id) + fad[end_i:]
143
  return newfad
 
145
  def add_point_mark(self, fad, notetype, start):
146
  id = self.newnote(notetype)
147
  start_i = self.goto_plaintext_i(fad, start)
148
+ if start_i == None:
149
  raise Exception("fad: point_mark: start outside range")
150
  newfad = fad[:start_i] + "⌧" + '{:08d}'.format(id) + fad[start_i:]
151
  return newfad
 
155
  n = 0
156
  while True:
157
  text, markchar, id, newi = self.get_next_mark(fad, i)
158
+ if markchar != '⌫':
159
+ if newi == None:
160
+ return None
161
+ if self.notes[id].type == self.notetypes[notetype]:
162
+ n += 1
163
+ if n == mark_n:
164
+ return id
165
  i = newi
166
 
167
+
168
  def set_property_to_mark(self, fad, notetype, mark_n, property, value, valuetype, startfrom=0):
169
  noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
170
  if valuetype != None:
 
177
  noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
178
  if valuetype != None:
179
  id = self.get_note_by_value(value, valuetype)
180
+ self.notes[noteid].add(property, id)
181
  else:
182
+ self.notes[noteid].add(property, value)
183
 
184
 
185
  if __name__ == "__main__":
 
222
 
223
  f.set_property_to_mark(fad, "span", 3, "actors", "actor2", "actor")
224
 
 
225
  assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6">56789</span id="4">B123456789<span id="5" actors="actor2">C1234</span id="6">56789</span id="5">')
226
+
227
+ f.set_property_to_mark(fad, "span", 3, "actors", "actor1", "actor")
228
+ f.set_property_to_mark(fad, "span", 2, "actors", "actor1", "actor")
229
+
230
+ assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6" actors="actor1">56789</span id="4">B123456789<span id="5" actors="actor1">C1234</span id="6">56789</span id="5">')
231
+
232
+ f.set_property_to_mark(fad, "span", 3, "actors", None, "actor")
233
+
234
+ assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6" actors="actor1">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
235
+
236
+ f.add_property_to_mark(fad, "span", 1, "audiences", None, "audience")
237
+ f.add_property_to_mark(fad, "span", 1, "audiences", "audience1", "audience")
238
+ f.add_property_to_mark(fad, "span", 1, "audiences", "audience2", "audience")
239
+
240
+ assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1" audiences="audience1, audience2">A1234<span id="6" actors="actor1">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
241
+
242
+ f.add_property_to_mark(fad, "span", 1, "audiences", "audience3", "audience")
243
+ f.add_property_to_mark(fad, "span", 2, "audiences", "audience3", "audience")
244
+
245
+ #print(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True))
246
+ assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1" audiences="audience1, audience2, audience3">A1234<span id="6" actors="actor1" audiences="audience3">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
247
+
248
+ f.set_property_to_mark(fad, "span", 2, "audiences", None, "audience")
249
+ assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1" audiences="audience1, audience2, audience3">A1234<span id="6" actors="actor1">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
understand.py CHANGED
@@ -170,7 +170,7 @@ Here's the document:
170
  self.values.put("actors jackard distance", jackard["actors"])
171
 
172
  # also store concepts in the fad marks
173
- self.fad.add_marks_from_list(reconciled["concepts"], "concept")
174
 
175
  print(f"===Reconciled: Concepts: {self.values.get('concepts')}, Actors: {self.values.get('actors')}")
176
 
@@ -196,7 +196,7 @@ Output the line "ACTORS:" followed by your list of actors, one per line.
196
  self.values.put("actors, generalized", generalized)
197
 
198
  # also store concepts in the fad marks
199
- self.fad.add_marks_from_list(generalized, "actor")
200
 
201
 
202
  def split_sentences(self):
@@ -207,7 +207,7 @@ Output the line "ACTORS:" followed by your list of actors, one per line.
207
  print("ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source")
208
  i = 0
209
  for sent in sents:
210
- newfad = fad.envelope_marks(newfad, "sentence", i, i+len(sent) - 1)
211
  self.values.put("document", newfad)
212
  print(f"=== Sentence-split FAD : {newfad}")
213
  # todo: pass this through LLM to verify and correct sentence splittings
@@ -253,7 +253,13 @@ Output just the document adding the actors attributes for sentences, but otherw
253
 
254
  sents = re.split('</sentence>', annotated0)
255
  for i, sent in enumerate(sents):
256
- match = re.search('<sentence[^>]+actors="([^"]+)"')
 
 
 
 
 
 
257
 
258
 
259
  def split_document(self):
 
170
  self.values.put("actors jackard distance", jackard["actors"])
171
 
172
  # also store concepts in the fad marks
173
+ self.fad.add_notes_from_list(reconciled["concepts"], "concept")
174
 
175
  print(f"===Reconciled: Concepts: {self.values.get('concepts')}, Actors: {self.values.get('actors')}")
176
 
 
196
  self.values.put("actors, generalized", generalized)
197
 
198
  # also store concepts in the fad marks
199
+ self.fad.add_notes_from_list(generalized, "actor")
200
 
201
 
202
  def split_sentences(self):
 
207
  print("ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source")
208
  i = 0
209
  for sent in sents:
210
+ newfad = fad.add_span_mark(newfad, "sentence", i, i+len(sent) - 1)
211
  self.values.put("document", newfad)
212
  print(f"=== Sentence-split FAD : {newfad}")
213
  # todo: pass this through LLM to verify and correct sentence splittings
 
253
 
254
  sents = re.split('</sentence>', annotated0)
255
  for i, sent in enumerate(sents):
256
+ match = re.match('<sentence[^>]+actors="([^"]+)"')
257
+ if match:
258
+ actors = match.groups(1)
259
+ for actor in re.split(",\s+", actors):
260
+ self.fad.add_property_to_mark(self.values.get("document"), "sentence", i, "actors", actor, "actor")
261
+ else:
262
+ raise(f"annotate_sentences_with_actors: Bad markup from LLM: {sent}")
263
 
264
 
265
  def split_document(self):