Dmitry Kirsanov commited on
Commit
5d42088
·
1 Parent(s): ab4f636
Files changed (1) hide show
  1. understand.py +6 -5
understand.py CHANGED
@@ -181,12 +181,13 @@ Output the line "ACTORS:" followed by your list of actors, one per line.
181
  newfad = self.values.get(src)
182
  plaintext = self.fad.get_plaintext(newfad)
183
  sents = nltk.tokenize.sent_tokenize(plaintext)
184
- if "".join(sents) != plaintext:
185
- print("ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source")
186
  i = 0
187
  for sent in sents:
188
- newfad = self.fad.add_span_mark(newfad, "sentence", i, i+len(sent) - 1)
189
- i += len(sent)
 
 
 
190
  self.values.put(dst, newfad)
191
  print(f"=== Sentence-split FAD : {newfad}")
192
  # todo: pass this through LLM to verify and correct sentence splittings
@@ -201,7 +202,7 @@ Output the line "ACTORS:" followed by your list of actors, one per line.
201
  vals = match.group(1)
202
  for val in re.split(",\s+", vals):
203
  print(f"property {property} val {val} notetype {notetype}")
204
- self.fad.add_property_to_mark(self.values.get("document"), "sentence", i, property, val, notetype)
205
  else:
206
  raise(Exception(f"annotate_sentences_with_{property}: Bad markup from LLM: {sent}"))
207
 
 
181
  newfad = self.values.get(src)
182
  plaintext = self.fad.get_plaintext(newfad)
183
  sents = nltk.tokenize.sent_tokenize(plaintext)
 
 
184
  i = 0
185
  for sent in sents:
186
+ start = plaintext.find(plaintext, start=i)
187
+ if start == -1:
188
+ raise(Exception("ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source"))
189
+ newfad = self.fad.add_span_mark(newfad, "sentence", start, start+len(sent) - 1)
190
+ i = start+len(sent)
191
  self.values.put(dst, newfad)
192
  print(f"=== Sentence-split FAD : {newfad}")
193
  # todo: pass this through LLM to verify and correct sentence splittings
 
202
  vals = match.group(1)
203
  for val in re.split(",\s+", vals):
204
  print(f"property {property} val {val} notetype {notetype}")
205
+ self.fad.add_property_to_mark(self.values.get("document"), "sentence", i + 1, property, val, notetype)
206
  else:
207
  raise(Exception(f"annotate_sentences_with_{property}: Bad markup from LLM: {sent}"))
208