Dmitry Kirsanov commited on
Commit
7aa69aa
·
1 Parent(s): 62825e9

transferring markup with possible newlines added is tricky

Browse files
Files changed (2) hide show
  1. fad.py +4 -2
  2. understand.py +42 -4
fad.py CHANGED
@@ -226,7 +226,7 @@ class Fad:
226
  return "⌧" + '{:08d}'.format(id)
227
 
228
 
229
- def __goto_plaintext_i(self, fad, to):
230
  # fixme: suboptimal, can skip faster
231
  passed = 0
232
  i = 0
@@ -234,6 +234,8 @@ class Fad:
234
  text, markchar, id, newi = self.__next_mark(fad, i)
235
  passed += len(text)
236
  if passed >= to:
 
 
237
  return i + len(text) - (passed - to)
238
  elif newi == None:
239
  return None # could not reach to
@@ -242,7 +244,7 @@ class Fad:
242
 
243
  def add_span_mark(self, notetype, start, end, src=defaultnote) -> Note:
244
  fad = self.get_value_by_type(src)
245
- start_i = self.__goto_plaintext_i(fad, start)
246
  end_i = self.__goto_plaintext_i(fad, end)
247
  if start_i == None or end_i == None:
248
  raise Exception(f"Fad: {inspect.stack()[0][3]}: start {start} or end {end} outside range {len(self.__plaintext(fad))}")
 
226
  return "⌧" + '{:08d}'.format(id)
227
 
228
 
229
+ def __goto_plaintext_i(self, fad, to, endskip=None):
230
  # fixme: suboptimal, can skip faster
231
  passed = 0
232
  i = 0
 
234
  text, markchar, id, newi = self.__next_mark(fad, i)
235
  passed += len(text)
236
  if passed >= to:
237
+ if endskip and passed == to and self.notes[id].typeid == notetypes[endskip]:
238
+ return i + len(text) + idlength + 1
239
  return i + len(text) - (passed - to)
240
  elif newi == None:
241
  return None # could not reach to
 
244
 
245
  def add_span_mark(self, notetype, start, end, src=defaultnote) -> Note:
246
  fad = self.get_value_by_type(src)
247
+ start_i = self.__goto_plaintext_i(fad, start, endskip=notetype)
248
  end_i = self.__goto_plaintext_i(fad, end)
249
  if start_i == None or end_i == None:
250
  raise Exception(f"Fad: {inspect.stack()[0][3]}: start {start} or end {end} outside range {len(self.__plaintext(fad))}")
understand.py CHANGED
@@ -210,19 +210,22 @@ Output the line "ACTORS:" followed by your list of actors, one per line.
210
  if len(clean) <= len(self.fad.get_plaintext()):
211
  pattern = p
212
  break
213
- pattern_o = pattern.replace('/?', '')
214
  pattern_c = pattern
 
215
  pattern_c_o = pattern_c + pattern_o
216
  chunks = re.split(pattern_c_o, llmtext)
 
217
  t = 0
218
  for i, chunk in enumerate(chunks):
219
  if f"{span}>" in chunk:
220
  text = re.sub(pattern_o, '', chunk)
221
  text = re.sub(pattern_c, '', text)
222
- self.fad.add_span_mark(span, t, t + len(text))
223
  else:
224
  text = chunk
225
- t += len(text)
 
 
226
 
227
  def annotate_sentences_with_actors(self, src="document", dst="document with actors"):
228
  markup = self.fad.get_plaintext_with_markup(self.values.get(src), "sentence")
@@ -542,4 +545,39 @@ Here's the derivative document:
542
  if __name__ == "__main__":
543
  d = Document(llm.Debug())
544
  d.ingest("samples/submission.html")
545
- print(d.get_understanding())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
210
  if len(clean) <= len(self.fad.get_plaintext()):
211
  pattern = p
212
  break
213
+ print(f"pattern /{pattern}/")
214
  pattern_c = pattern
215
+ pattern_o = pattern.replace('/?', '')
216
  pattern_c_o = pattern_c + pattern_o
217
  chunks = re.split(pattern_c_o, llmtext)
218
+ print(f"pattern_c_o /{pattern_c_o}/")
219
  t = 0
220
  for i, chunk in enumerate(chunks):
221
  if f"{span}>" in chunk:
222
  text = re.sub(pattern_o, '', chunk)
223
  text = re.sub(pattern_c, '', text)
 
224
  else:
225
  text = chunk
226
+ self.fad.add_span_mark(span, t, t + len(text) + 1)
227
+ t += len(text) + 1
228
+ print(f"text /{text}/")
229
 
230
  def annotate_sentences_with_actors(self, src="document", dst="document with actors"):
231
  markup = self.fad.get_plaintext_with_markup(self.values.get(src), "sentence")
 
545
  if __name__ == "__main__":
546
  d = Document(llm.Debug())
547
  d.ingest("samples/submission.html")
548
+
549
+ #print(d.get_understanding())
550
+
551
+ s = """<document>
552
+
553
+ <section>
554
+ The process of submitting a scientific paper to a journal involves several sequential stages, starting with the preparation of the manuscript. Researchers meticulously compile and format their findings according to the specific guidelines outlined by the journal of their choice. Once the manuscript is ready, authors initiate the submission process, typically done through an online system or by email.
555
+ </section>
556
+
557
+ <section>
558
+ The journey of the manuscript begins with an initial editorial review conducted by journal editors. This preliminary assessment aims to ensure that the submission aligns with the journal's scope, adheres to ethical standards, and meets the specified formatting requirements. Following this, the manuscript undergoes a critical phase known as peer review. Editors select expert reviewers in the relevant field to evaluate the manuscript's quality, methodology, and contribution to scientific knowledge.
559
+ </section>
560
+
561
+ <section>
562
+ Based on the feedback received from reviewers, authors may be required to make revisions to their manuscript. These revisions could range from minor adjustments to substantial changes. After the authors have made the necessary modifications, the manuscript returns to the editorial team for a final decision.
563
+ </section>
564
+
565
+ <section>
566
+ The editorial decision involves a comprehensive evaluation of the revised manuscript, taking into account the recommendations provided by the reviewers. Depending on the assessment, the decision may be acceptance, further revisions, or rejection. If revisions are necessary, authors diligently modify their manuscript and resubmit it for another round of evaluation.
567
+ </section>
568
+
569
+ <section>
570
+ Once the manuscript successfully passes through the review and revision stages, it moves on to the final steps before publication. The accepted manuscript undergoes copyediting and formatting to ensure adherence to the journal's style and standards. Authors are then provided with the final proofs for a meticulous review and approval before the paper is officially published.
571
+ </section>
572
+
573
+ <section>
574
+ The publication process extends beyond the journal's platform, as the paper may be indexed in various databases and repositories, enhancing its visibility and accessibility to the broader scientific community. Throughout this intricate process, several key actors play crucial roles, including the authors, editors, reviewers, copyeditors, and publishers, all working collaboratively to contribute to the dissemination of valuable scientific knowledge.
575
+ </section>
576
+
577
+ </document>"""
578
+
579
+ val = d.parseoutput("section", s, title="section")
580
+
581
+ d.create_spans_in_fad(val, "section")
582
+
583
+ print(d.fad.get_plaintext_with_markup('section'))