Spaces:
Runtime error
Runtime error
File size: 26,122 Bytes
2faa652 7c55da8 7b67970 57ea3cc 2faa652 fbee5da 097e81a 4c58022 7b67970 46e2995 097e81a 0eecb35 b72e4e6 097e81a d1d70b6 4c58022 7b67970 097e81a 9a5a6dc 097e81a 9a5a6dc bf23f98 d1d70b6 097e81a 43e002c 097e81a 9a5a6dc f5d28e2 9a5a6dc 4c58022 43e002c 9a5a6dc fbee5da 9a5a6dc 4c58022 097e81a bf23f98 4c58022 ed32639 43e002c d1d70b6 ed32639 d1d70b6 fbee5da 9a5a6dc ed32639 4c58022 bf23f98 4c58022 ae112b0 4c58022 bf23f98 fbee5da 43e002c fbee5da ed32639 d199a11 ed32639 9a5a6dc fbee5da 9a5a6dc bf23f98 ed32639 bf23f98 43e002c bf23f98 3692d5a bf23f98 4c58022 bf23f98 f09d6be bf23f98 45fc24e 93bd36d fbee5da 43e002c fbee5da ed32639 bf23f98 ed32639 bf23f98 f09d6be bf23f98 f09d6be bf23f98 f09d6be 9334bee bf23f98 f09d6be 9334bee bf23f98 f09d6be bf23f98 ed32639 4c58022 3692d5a 50c7d58 9334bee 4c58022 3692d5a 43e002c 082d0f1 7b67970 d1983b7 46e2995 4c58022 fbee5da 43e002c fbee5da ed32639 b72e4e6 ed32639 b72e4e6 199ce86 b72e4e6 ed32639 4c58022 b72e4e6 7b67970 43e002c 7b67970 43e002c 7b67970 8552f4e 5d42088 8552f4e 43e002c 5d42088 43e002c 7b67970 43e002c 4c58022 be97aaf 43e002c ce1ed78 43e002c 7b67970 43e002c 3bfd8ed 62825e9 3bfd8ed 7aa69aa 3bfd8ed 43e002c 3bfd8ed 62825e9 c4d6264 7aa69aa 2faa652 4c58022 74b9dcf d66a76e a9fdece fbee5da 43e002c fbee5da ed32639 2faa652 ed32639 2faa652 7b67970 2faa652 eb2c877 2faa652 7b67970 2faa652 7b67970 2faa652 eb2c877 7b67970 eb2c877 2faa652 7b67970 2faa652 fbee5da 4c58022 0a2597f 4c58022 7b67970 43e002c 7b67970 2faa652 fbee5da 4c58022 74b9dcf 4c58022 43e002c 78b4826 ed32639 93bd36d ed32639 4c58022 43e002c 4c58022 93bd36d 4c58022 43e002c 4c58022 ed32639 4c58022 ed32639 93bd36d fbee5da 78b4826 e32cd6d 78b4826 fbee5da ed32639 fbee5da 4c58022 78b4826 2faa652 4c58022 43e002c 024fcd4 46e2995 df3e641 43e002c 46e2995 d1d70b6 4c58022 43e002c d26894c 43e002c 4c58022 086165e 4c58022 7b67970 4c58022 7b67970 4c58022 decdef2 43e002c 46e2995 4c58022 43e002c 46e2995 097e81a 43e002c 7aa69aa d26894c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 | import re
from lxml import etree
import nltk.tokenize
import nltk
nltk.download('punkt')
import prompts
import parse
import llm
import llm_parsing
import fad
from versioneddict import VersionedDict
newline = "\n"
class Document():
def __init__(self, llm) -> None:
self.LLM = llm
# TODO: move all understandings to this object instead of self.values
# for this, need to implement timestamps and is_newer
self.fad = fad.Fad()
def query(self, s):
try:
answer, truncated = self.LLM.send_question(s)
# log
print(f"===Question: \"\"\"{s}\"\"\"")
print(f"===Answer: \"\"\"{answer}\"\"\"")
print(f"===Truncated? {truncated}")
return answer.strip()
except Exception as e:
print (e)
return None
def query_n(self, n, s):
return [self.query(s) for i in range(n)]
def query_document_type(self, src="document", dst="document type"):
ret = self.query(prompts.get_prompt_recursive(
f"""#document-intro
#q-document-type
<document>{self.values.get(src)}</document>"""))
self.fad.newnote(type=dst, value=ret)
def query_summary(self, src="document", dst="document summary"):
ret = self.query(prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}.
#q-document-summary
<document>{self.values.get(src)}</document>"""))
self.fad.newnote(type=dst, value=ret)
def query_concepts_and_actors(self, src="document", dst=[]):
concepts = dst[0]
actors = dst[1]
two_attempts = [{actors: [], concepts: []}, {actors: [], concepts: []}]
ret = self.query_n(2, prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
#q-concepts-actors
Here's the document:
<document>{self.fad.get_plaintext(self.values.get(src))}</document>"""))
two_attempts[0][concepts] = llm_parsing.parse_list(ret[0], concepts)
two_attempts[1][concepts] = llm_parsing.parse_list(ret[1], concepts)
two_attempts[0][actors] = llm_parsing.parse_list(ret[0], actors)
two_attempts[1][actors] = llm_parsing.parse_list(ret[1], actors)
print(f"===Reconcile: {two_attempts}")
#reconcile two lists
reconciled = {actors: [], concepts: []}
jackard = {actors: [], concepts: []}
for what in [
{'plural': "concepts", 'singular': "concept", 'full': "concepts that this document talks about"},
{'plural': "actors", 'singular': "actor", 'full': "actors described or implied in the document"}
]:
set1 = set(two_attempts[0][what['plural']])
set2 = set(two_attempts[1][what['plural']])
jackard[what['plural']] = len(set1.intersection(set2)) / len(set1.union(set2))
if set1 != set2:
print(f"===Two attempts of {what['plural']} are different, jaccard similarity: {jackard[what['plural']]}")
ret = self.query(prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
Two different readers have read this document and created different lists of the {what['full']}. Your goal is to read these lists and the entire document carefully and then create a new combined list of the document's {what['plural']}.
Compare the two lists:
- If they have the same {what['singular']} but expressed differently, add a better expression of that {what['singular']} to your final list.
- If any of the {what['plural']} in these lists is actually irrelevant or confusing, drop it.
- If you see some {what['singular']} in the document that both lists missed, add it.
First list of {what['plural']}:
{newline.join(["- " + x for x in two_attempts[0][what['plural']]])}
Second list of {what['plural']}:
{newline.join(["- " + x for x in two_attempts[1][what['plural']]])}
Output the line "{what['plural'].upper()}:" followed by your combined list of {what['plural']}, one per line.
Here's the document:
<document>{self.fad.get_plaintext(self.values.get(src))}</document>"""))
reconciled[what['plural']] = llm_parsing.parse_lists(ret, what['plural'])
print(f"===Reconciled {what['plural']}: {reconciled[what['plural']]}")
else:
reconciled[what['plural']] = two_attempts[0][what['plural']] # they are the same, so we take the first one
# finally store in the values
self.values.put(concepts, reconciled[concepts])
self.values.put(actors, reconciled[actors])
self.values.put("concepts jackard distance", jackard[concepts])
self.values.put("actors jackard distance", jackard[actors])
# also store concepts in the fad notes
self.fad.add_from_list(reconciled[concepts], "concept")
self.fad.add_from_list(reconciled[actors], "actor")
print("store concepts in the fad notes")
print(reconciled[concepts])
print(reconciled[actors])
print(f"===Reconciled: Concepts: {newline.join(self.fad.get_values_by_type('concept'))}, Actors: {newline.join(self.fad.get_values_by_type('actor'))}")
def generalize_actors(self, src="actors", dst="actors, generalized"):
ret = self.query(prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
A reader of this document has compiled a list of actors mentioned or implied in the document. Here is their list:
{newline.join(["- " + x for x in self.values.get("actors")])}
Your goal is to generalize this list, replacing groups of actors playing the same or similar roles with their generic term. If an actor is not very relevant to the document (that is, you cannot see this document as written by, for, or about this actor), then drop it altogether. Also, drop any inanimate actors, leaving only categories of persons. Overall, pare this list down as much as possible, leaving only the most important categories of actors.
Output the line "ACTORS:" followed by your list of actors, one per line.
<document>{self.fad.get_plaintext(self.values.get("document"))}</document>"""))
generalized = llm_parsing.parse_lists(ret, "actors")
self.values.put("actors, generalized", generalized)
# also store concepts in the fad marks
self.fad.delete_by_type("actor")
self.fad.add_from_list(generalized, "actor")
def split_sentences(self, src="document"):
plaintext = self.fad.get_plaintext(src)
sents = nltk.tokenize.sent_tokenize(plaintext)
i = 0
for sent in sents:
start = plaintext.find(sent, i)
if start == -1:
raise(Exception(f"ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source: can't find '{sent}' in '{plaintext[i:]}'"))
self.fad.add_span_mark("sentence", start, start+len(sent), src=src)
i = start+len(sent)
print(f"=== Sentence-split FAD : {self.fad.get_plaintext_with_markup('sentence', src=src)}")
# todo: pass this through LLM to verify and correct sentence splittings
def annotate_spans_in_fad(self, llmtext, span, property, notetype):
sents = re.split(f'</{span}>', llmtext)
print(property, notetype)
for i, sent in enumerate(sents):
print(i, sent)
if f'<{span}' in sent:
match = re.match(f'[^<]*<{span}[^>]+{property}="([^"]+)"', sent)
if match:
vals = match.group(1)
for val in re.split(",\s+", vals):
print(f"property {property} val {val} notetype {notetype}")
self.fad.add_property_to_mark(span, i + 1, property, val, notetype)
def create_spans_in_fad(self, llmtext, span):
pattern = ""
for p in [f'</?{span}>', f'\n?</?{span}>\n?', f'\n*</?{span}>\n*']:
clean = re.sub(p, '', llmtext)
if len(clean) <= len(self.fad.get_plaintext()):
pattern = p
break
pattern_c = pattern
pattern_o = pattern.replace('/?', '')
pattern_c_o = pattern_c + pattern_o
chunks = re.split(pattern_c_o, llmtext)
t = 0
for i, chunk in enumerate(chunks):
if f"{span}>" in chunk:
text = re.sub(pattern_o, '', chunk)
text = re.sub(pattern_c, '', text)
else:
text = chunk
self.fad.add_span_mark(span, t, t + len(text) + 1)
t += len(text) + 1
def annotate_sentences_with_actors(self, src="document", dst="document with actors"):
markup = self.fad.get_plaintext_with_markup(self.values.get(src), "sentence")
print(f"=== Sentence markup: {markup}")
ret = self.query_n(2, prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
A reader of this document has compiled a list of actors (parties that do something) mentioned or implied in the document. Here is their list:
{newline.join(["- " + x for x in self.values.get("actors, generalized")])}
Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended actor(s) by picking one or more parties from this list. For each sentence, ask yourself: which of the parties from the list are the subjects? Who does the action described here? If there's no actor, leave the sentence unmarked.
Add the relevant partes, separated by commas, inside the "actors" attribute of each <sentence> tag. For example, suppose you have a document describing the game of Sudoku, and your parties are "Players" and "Creators". Here are example sentences with audience markup:
<example>
<sentence>Sudoku is a popular logic puzzle with numbers with simple rules that even beginners can easily master.</sentence> <sentence actors="Creators">You should not create a puzzle with more than one possible solution, as that would usually be considered defective.</sentence> <sentence actors="Players">If you have completed finding all positions for a number, draw an X over that number in the list outside.</sentence>
</example>
Output just the document adding the actors attributes for sentences, but otherwise unchanged:
<document>{markup}</document>"""))
annotated0, annotated1 = llm_parsing.extract_document_from_response(ret[0]), llm_parsing.extract_document_from_response(ret[1])
diff = newline.join([str(x) for x in zip(annotated0.split('</sentence>'), annotated1.split('</sentence>')) if x[0] != x[1]])
self.values.put(dst, annotated1)
self.values.put(f"{dst}, diff", diff)
#todo: reconcile two lists
self.annotate_spans_in_fad(annotated0, "sentence", "actors", "actor")
def annotate_sentences_with_audiences(self, src="document", dst="document with audiences"):
# EXPERIMENTAL! we use the actors list but here interpret it as audiences
markup = self.fad.get_plaintext_with_markup(self.values.get(src), "sentence")
print(f"=== Sentence markup: {markup}")
ret = self.query_n(2, prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
A reader of this document has compiled a list of audiences (parties that parts of this document are addressed to) mentioned or implied in the document. Here is their list:
{newline.join(["- " + x for x in self.values.get("actors, generalized")])}
Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended audience(s) by picking one or more parties from this list. For each sentence, ask yourself: who, from the list, would want to hear this? Who needs to know this? If the intended audience is unclear, leave the sentence unmarked.
Add the relevant partes, separated by commas, inside the "audiences" attribute of each <sentence> tag. For example, suppose you have a document describing the game of Sudoku, and your parties are "Players" and "Creators". Here are example sentences with audience markup:
<example>
<sentence audiences="players, creators">Sudoku is a popular logic puzzle with numbers with simple rules that even beginners can easily master.</sentence> <sentence actors="creators">You should not create a puzzle with more than one possible solution, as that would usually be considered defective.</sentence> <sentence actors="players">If you have completed finding all positions for a number, draw an X over that number in the list outside.</sentence>
</example>
Output just the document adding the "audiences" attributes for sentences, but otherwise unchanged:
<document>{markup}</document>"""))
annotated0, annotated1 = llm_parsing.extract_document_from_response(ret[0]), llm_parsing.extract_document_from_response(ret[1])
diff = newline.join([str(x) for x in zip(annotated0.split('</sentence>'), annotated1.split('</sentence>')) if x[0] != x[1]])
self.values.put(dst, annotated1)
self.values.put(f"{dst}, diff", diff)
#todo: reconcile two lists
self.annotate_spans_in_fad(annotated0, "sentence", "audiences", "actor") # not an error, we use actors but tell llm it's audiences
def extract_and_cleanup_for_audience(self, src, audience, dst):
#derivative_string = f"<document>{self.values.get(src)}</document>"
#derivative = newline.join(etree.fromstring(derivative_string).xpath(f".//sentence[contains(@audience,'{audience}')]/text()"))
derivative = newline.join(self.fad.get_spans_by_property_value(self.values.get(src), 'sentence', name="audiences", values=[audience]))
ret = self.query(prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
An editor was tasked with creating a derivative document that is addressed to {audience}. The editor compiled all sentences from the document that have information relevant for {audience}. These sentences are dumped below the main document inside <derivative-document for="{audience}">...</derivative-document>.
Your goal is to edit the derivative document for structure and consistency. Rearrange its sentences in a way that makes sense for a reader. Group sentences into paragraphs. If necessary, write, or adapt from the original document, connecting sentences. Then, remove as many instances of "{audience}" as you can: since this entire document is now addressed to {audience}, you can just say "you" most of the time.
Output just your revision inside <derivative-document>...</derivative-document>.
Here's the original document:
<document>{self.fad.get_plaintext(self.values.get(src))}</document>
Here's the derivative document:
<derivative-document for="{audience}">{derivative}</derivative-document>
"""))
self.values.put(dst.format(audience=audience), llm_parsing.extract_document_from_response(ret))
def formatinput(self, src, selectors=None):
if selectors:
for selector in selectors.split(","):
what = selector.split('-')[0]
which = selector.split('-')[1]
match which:
case 'all':
material = self.fad.get_plaintext_with_markup(what, src=src, show_properties=False) # todo: add a way to select properties
case _:
# todo
pass
else:
material = self.fad.get_plaintext(src)
match src:
case "document":
return f"<document>{material}</document>"
case _:
return material
def parseoutput(self, outputformat, val, title=None, type=None):
match outputformat:
case "singlevalue":
return val
case "document" | "sentence" | "section":
return llm_parsing.extract_document_from_response(val)
case "list":
list = llm_parsing.parse_list(val, title)
return list
def derive(self,
sideinfo = None,
src = "document",
selectors = None, # can be a comma-separated list of selectors applied in turn
target = "what-this-is",
outputformat="singlevalue"
):
if self.fad.get(src) == None or not(self.fad.is_newer(src, target)):
print(f"No {src}, or {src} already has up-to-date (newer) {target}, skip query")
return # we already have an up-to-date target
##### constructing prompt
## GENERAL: what is the task we're working on here
general_prompt = prompts.get_prompt(f"#{src}-general")
## SIDEINFO: information not in the document which is needed for the current task
sideinfo_prompt = ''
if sideinfo and self.fad.get(sideinfo):
sideinfo_prompt = prompts.get_prompt(f"#{src}-{sideinfo}").format(value=self.fad.get_values_by_type_string(sideinfo))
## INTRO: introducing the document or part we're working on
intro_prompt = prompts.get_prompt(f"{src}-intro")
## SUMMARIES: already-extracted summary-like information bits about the document or part
summaries = []
summaries_prompt = ''
for s in fad.summaries: # a set list for now
if self.fad.get_note(src).get(s): # if this src has this summary
summaries.append(prompts.get_prompt(f'#{src}-{s}').format(name=src, value=self.fad.get_note(src).get(s)))
if summaries:
summaries_prompt = " ".join(summaries) + "\n\n"
## SELECT: explaining which part of the document we have here
select_prompt = ''
if selectors:
for selector in selectors.split(','):
select_prompt += prompts.get_prompt(f"#select-{selector})") + " "
## Q: question or task, defined by our target
q = f"q-{outputformat}-{target}"
q_prompt = prompts.get_prompt(q)
## MATERIAL: document or part that we're working on; comes in the end because it is often the largest part of the prompt
material_prompt = prompts.get_prompt("#material").format(name=src, value=self.formatinput(src, selectors))
## run the query
ret = self.query(f"""{general_prompt}{sideinfo_prompt}{intro_prompt}{summaries_prompt}{q_prompt}{material_prompt}""")
what = fad.plurals[target] if outputformat == 'list' or outputformat == 'sentence' else target
val = self.parseoutput(outputformat, ret, title=what)
match outputformat:
case 'singlevalue':
# the whole output is a single return value
self.fad.get_note(src).set_property_newnote(name=target, value=val, valuetype=target)
case 'list':
# the output is a list
self.fad.add_from_list(val, target)
case 'sentence' | 'section':
if target == outputformat:
# the output is newly created spans
self.create_spans_in_fad(val, outputformat)
else:
# the output is added attributes in XML markup passed as input
self.annotate_spans_in_fad(val, outputformat, what, target)
pass
case 'spans':
# the output is XML markup (elements only) added by LLM
pass
def ingest(self, filename):
soup = parse.parse(filename)
self.fad.add("document", soup.get_text())
def understand(self):
# fact finders:
self.derive(
src="document",
target="what-this-is",
outputformat="singlevalue"
)
self.derive(
src="document",
target="summary",
outputformat="singlevalue"
)
self.derive(
src="document",
target="concept",
outputformat="list"
)
self.derive(
src="document",
target="actor",
outputformat="list"
)
self.split_sentences(src="document") # not LLM!
self.derive(
sideinfo="actor",
src="document",
selectors="sentence-all",
target="actor",
outputformat="sentence"
)
print(self.fad.get_plaintext_with_markup("sentence", show_properties=True, properties_to_show="actors"))
self.derive(
src="document",
target="section",
outputformat="section"
)
print(self.fad.get_plaintext_with_markup("section"))
"""
if self.values.is_newer("document", "document type"):
self.query_document_type(src="document", dst="document type")
if self.values.is_newer("document", "document summary"):
self.query_summary(src="document", dst="document summary")
if self.values.is_newer("document", "concepts"):
self.query_concepts_and_actors(src="document", dst=['concepts', 'actors'])
self.generalize_actors(src="actors", dst="actors, generalized")
if self.values.is_newer("document", "sentences"):
self.split_sentences(src="document", dst="document")
self.annotate_sentences_with_actors( src="document", dst="document with actors")
self.annotate_sentences_with_audiences(src="document", dst="document with audiences")
self.extract_and_cleanup_for_audience(src='document', audience=self.fad.get_values_by_type("actor")[0], dst="document for {audience}")
"""
def get_understanding(self):
self.understand()
return (
self.fad.get_value_by_type("what-this-is"),
self.fad.get_value_by_type("summary"),
"=== Concepts:\n"+self.fad.get_values_by_type_string("concept"),
"=== Actors:\n"+self.fad.get_values_by_type_string("actor"),
"=== Document annotated:\n"+self.fad.get_plaintext_with_markup("sentence", show_properties=True, properties_to_show="actors"),
f"=== Document broken into sections {self.fad.get_plaintext_with_markup('section')}"
)
if __name__ == "__main__":
d = Document(llm.Debug())
d.ingest("samples/submission.html")
#print(d.get_understanding())
s = """<document>
<section>
The process of submitting a scientific paper to a journal involves several sequential stages, starting with the preparation of the manuscript. Researchers meticulously compile and format their findings according to the specific guidelines outlined by the journal of their choice. Once the manuscript is ready, authors initiate the submission process, typically done through an online system or by email.
</section>
<section>
The journey of the manuscript begins with an initial editorial review conducted by journal editors. This preliminary assessment aims to ensure that the submission aligns with the journal's scope, adheres to ethical standards, and meets the specified formatting requirements. Following this, the manuscript undergoes a critical phase known as peer review. Editors select expert reviewers in the relevant field to evaluate the manuscript's quality, methodology, and contribution to scientific knowledge.
</section>
<section>
Based on the feedback received from reviewers, authors may be required to make revisions to their manuscript. These revisions could range from minor adjustments to substantial changes. After the authors have made the necessary modifications, the manuscript returns to the editorial team for a final decision.
</section>
<section>
The editorial decision involves a comprehensive evaluation of the revised manuscript, taking into account the recommendations provided by the reviewers. Depending on the assessment, the decision may be acceptance, further revisions, or rejection. If revisions are necessary, authors diligently modify their manuscript and resubmit it for another round of evaluation.
</section>
<section>
Once the manuscript successfully passes through the review and revision stages, it moves on to the final steps before publication. The accepted manuscript undergoes copyediting and formatting to ensure adherence to the journal's style and standards. Authors are then provided with the final proofs for a meticulous review and approval before the paper is officially published.
</section>
<section>
The publication process extends beyond the journal's platform, as the paper may be indexed in various databases and repositories, enhancing its visibility and accessibility to the broader scientific community. Throughout this intricate process, several key actors play crucial roles, including the authors, editors, reviewers, copyeditors, and publishers, all working collaboratively to contribute to the dissemination of valuable scientific knowledge.
</section>
</document>"""
val = d.parseoutput("section", s, title="section")
d.create_spans_in_fad(val, "section")
print(d.fad.get_plaintext_with_markup('section'))
|