DU / understand.py
Dmitry Kirsanov
fix
d26894c
Raw History Blame Contribute Delete
26.1 kB
import re
from lxml import etree
import nltk.tokenize
import nltk
nltk.download('punkt')
import prompts
import parse
import llm
import llm_parsing
import fad
from versioneddict import VersionedDict
newline = "\n"
class Document():
def __init__(self, llm) -> None:
self.LLM = llm
# TODO: move all understandings to this object instead of self.values
# for this, need to implement timestamps and is_newer
self.fad = fad.Fad()
def query(self, s):
try:
answer, truncated = self.LLM.send_question(s)
# log
print(f"===Question: \"\"\"{s}\"\"\"")
print(f"===Answer: \"\"\"{answer}\"\"\"")
print(f"===Truncated? {truncated}")
return answer.strip()
except Exception as e:
print (e)
return None
def query_n(self, n, s):
return [self.query(s) for i in range(n)]
def query_document_type(self, src="document", dst="document type"):
ret = self.query(prompts.get_prompt_recursive(
f"""#document-intro
#q-document-type
<document>{self.values.get(src)}</document>"""))
self.fad.newnote(type=dst, value=ret)
def query_summary(self, src="document", dst="document summary"):
ret = self.query(prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}.
#q-document-summary
<document>{self.values.get(src)}</document>"""))
self.fad.newnote(type=dst, value=ret)
def query_concepts_and_actors(self, src="document", dst=[]):
concepts = dst[0]
actors = dst[1]
two_attempts = [{actors: [], concepts: []}, {actors: [], concepts: []}]
ret = self.query_n(2, prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
#q-concepts-actors
Here's the document:
<document>{self.fad.get_plaintext(self.values.get(src))}</document>"""))
two_attempts[0][concepts] = llm_parsing.parse_list(ret[0], concepts)
two_attempts[1][concepts] = llm_parsing.parse_list(ret[1], concepts)
two_attempts[0][actors] = llm_parsing.parse_list(ret[0], actors)
two_attempts[1][actors] = llm_parsing.parse_list(ret[1], actors)
print(f"===Reconcile: {two_attempts}")
#reconcile two lists
reconciled = {actors: [], concepts: []}
jackard = {actors: [], concepts: []}
for what in [
{'plural': "concepts", 'singular': "concept", 'full': "concepts that this document talks about"},
{'plural': "actors", 'singular': "actor", 'full': "actors described or implied in the document"}
]:
set1 = set(two_attempts[0][what['plural']])
set2 = set(two_attempts[1][what['plural']])
jackard[what['plural']] = len(set1.intersection(set2)) / len(set1.union(set2))
if set1 != set2:
print(f"===Two attempts of {what['plural']} are different, jaccard similarity: {jackard[what['plural']]}")
ret = self.query(prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
Two different readers have read this document and created different lists of the {what['full']}. Your goal is to read these lists and the entire document carefully and then create a new combined list of the document's {what['plural']}.
Compare the two lists:
- If they have the same {what['singular']} but expressed differently, add a better expression of that {what['singular']} to your final list.
- If any of the {what['plural']} in these lists is actually irrelevant or confusing, drop it.
- If you see some {what['singular']} in the document that both lists missed, add it.
First list of {what['plural']}:
{newline.join(["- " + x for x in two_attempts[0][what['plural']]])}
Second list of {what['plural']}:
{newline.join(["- " + x for x in two_attempts[1][what['plural']]])}
Output the line "{what['plural'].upper()}:" followed by your combined list of {what['plural']}, one per line.
Here's the document:
<document>{self.fad.get_plaintext(self.values.get(src))}</document>"""))
reconciled[what['plural']] = llm_parsing.parse_lists(ret, what['plural'])
print(f"===Reconciled {what['plural']}: {reconciled[what['plural']]}")
else:
reconciled[what['plural']] = two_attempts[0][what['plural']] # they are the same, so we take the first one
# finally store in the values
self.values.put(concepts, reconciled[concepts])
self.values.put(actors, reconciled[actors])
self.values.put("concepts jackard distance", jackard[concepts])
self.values.put("actors jackard distance", jackard[actors])
# also store concepts in the fad notes
self.fad.add_from_list(reconciled[concepts], "concept")
self.fad.add_from_list(reconciled[actors], "actor")
print("store concepts in the fad notes")
print(reconciled[concepts])
print(reconciled[actors])
print(f"===Reconciled: Concepts: {newline.join(self.fad.get_values_by_type('concept'))}, Actors: {newline.join(self.fad.get_values_by_type('actor'))}")
def generalize_actors(self, src="actors", dst="actors, generalized"):
ret = self.query(prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
A reader of this document has compiled a list of actors mentioned or implied in the document. Here is their list:
{newline.join(["- " + x for x in self.values.get("actors")])}
Your goal is to generalize this list, replacing groups of actors playing the same or similar roles with their generic term. If an actor is not very relevant to the document (that is, you cannot see this document as written by, for, or about this actor), then drop it altogether. Also, drop any inanimate actors, leaving only categories of persons. Overall, pare this list down as much as possible, leaving only the most important categories of actors.
Output the line "ACTORS:" followed by your list of actors, one per line.
<document>{self.fad.get_plaintext(self.values.get("document"))}</document>"""))
generalized = llm_parsing.parse_lists(ret, "actors")
self.values.put("actors, generalized", generalized)
# also store concepts in the fad marks
self.fad.delete_by_type("actor")
self.fad.add_from_list(generalized, "actor")
def split_sentences(self, src="document"):
plaintext = self.fad.get_plaintext(src)
sents = nltk.tokenize.sent_tokenize(plaintext)
i = 0
for sent in sents:
start = plaintext.find(sent, i)
if start == -1:
raise(Exception(f"ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source: can't find '{sent}' in '{plaintext[i:]}'"))
self.fad.add_span_mark("sentence", start, start+len(sent), src=src)
i = start+len(sent)
print(f"=== Sentence-split FAD : {self.fad.get_plaintext_with_markup('sentence', src=src)}")
# todo: pass this through LLM to verify and correct sentence splittings
def annotate_spans_in_fad(self, llmtext, span, property, notetype):
sents = re.split(f'</{span}>', llmtext)
print(property, notetype)
for i, sent in enumerate(sents):
print(i, sent)
if f'<{span}' in sent:
match = re.match(f'[^<]*<{span}[^>]+{property}="([^"]+)"', sent)
if match:
vals = match.group(1)
for val in re.split(",\s+", vals):
print(f"property {property} val {val} notetype {notetype}")
self.fad.add_property_to_mark(span, i + 1, property, val, notetype)
def create_spans_in_fad(self, llmtext, span):
pattern = ""
for p in [f'</?{span}>', f'\n?</?{span}>\n?', f'\n*</?{span}>\n*']:
clean = re.sub(p, '', llmtext)
if len(clean) <= len(self.fad.get_plaintext()):
pattern = p
break
pattern_c = pattern
pattern_o = pattern.replace('/?', '')
pattern_c_o = pattern_c + pattern_o
chunks = re.split(pattern_c_o, llmtext)
t = 0
for i, chunk in enumerate(chunks):
if f"{span}>" in chunk:
text = re.sub(pattern_o, '', chunk)
text = re.sub(pattern_c, '', text)
else:
text = chunk
self.fad.add_span_mark(span, t, t + len(text) + 1)
t += len(text) + 1
def annotate_sentences_with_actors(self, src="document", dst="document with actors"):
markup = self.fad.get_plaintext_with_markup(self.values.get(src), "sentence")
print(f"=== Sentence markup: {markup}")
ret = self.query_n(2, prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
A reader of this document has compiled a list of actors (parties that do something) mentioned or implied in the document. Here is their list:
{newline.join(["- " + x for x in self.values.get("actors, generalized")])}
Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended actor(s) by picking one or more parties from this list. For each sentence, ask yourself: which of the parties from the list are the subjects? Who does the action described here? If there's no actor, leave the sentence unmarked.
Add the relevant partes, separated by commas, inside the "actors" attribute of each <sentence> tag. For example, suppose you have a document describing the game of Sudoku, and your parties are "Players" and "Creators". Here are example sentences with audience markup:
<example>
<sentence>Sudoku is a popular logic puzzle with numbers with simple rules that even beginners can easily master.</sentence> <sentence actors="Creators">You should not create a puzzle with more than one possible solution, as that would usually be considered defective.</sentence> <sentence actors="Players">If you have completed finding all positions for a number, draw an X over that number in the list outside.</sentence>
</example>
Output just the document adding the actors attributes for sentences, but otherwise unchanged:
<document>{markup}</document>"""))
annotated0, annotated1 = llm_parsing.extract_document_from_response(ret[0]), llm_parsing.extract_document_from_response(ret[1])
diff = newline.join([str(x) for x in zip(annotated0.split('</sentence>'), annotated1.split('</sentence>')) if x[0] != x[1]])
self.values.put(dst, annotated1)
self.values.put(f"{dst}, diff", diff)
#todo: reconcile two lists
self.annotate_spans_in_fad(annotated0, "sentence", "actors", "actor")
def annotate_sentences_with_audiences(self, src="document", dst="document with audiences"):
# EXPERIMENTAL! we use the actors list but here interpret it as audiences
markup = self.fad.get_plaintext_with_markup(self.values.get(src), "sentence")
print(f"=== Sentence markup: {markup}")
ret = self.query_n(2, prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
A reader of this document has compiled a list of audiences (parties that parts of this document are addressed to) mentioned or implied in the document. Here is their list:
{newline.join(["- " + x for x in self.values.get("actors, generalized")])}
Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended audience(s) by picking one or more parties from this list. For each sentence, ask yourself: who, from the list, would want to hear this? Who needs to know this? If the intended audience is unclear, leave the sentence unmarked.
Add the relevant partes, separated by commas, inside the "audiences" attribute of each <sentence> tag. For example, suppose you have a document describing the game of Sudoku, and your parties are "Players" and "Creators". Here are example sentences with audience markup:
<example>
<sentence audiences="players, creators">Sudoku is a popular logic puzzle with numbers with simple rules that even beginners can easily master.</sentence> <sentence actors="creators">You should not create a puzzle with more than one possible solution, as that would usually be considered defective.</sentence> <sentence actors="players">If you have completed finding all positions for a number, draw an X over that number in the list outside.</sentence>
</example>
Output just the document adding the "audiences" attributes for sentences, but otherwise unchanged:
<document>{markup}</document>"""))
annotated0, annotated1 = llm_parsing.extract_document_from_response(ret[0]), llm_parsing.extract_document_from_response(ret[1])
diff = newline.join([str(x) for x in zip(annotated0.split('</sentence>'), annotated1.split('</sentence>')) if x[0] != x[1]])
self.values.put(dst, annotated1)
self.values.put(f"{dst}, diff", diff)
#todo: reconcile two lists
self.annotate_spans_in_fad(annotated0, "sentence", "audiences", "actor") # not an error, we use actors but tell llm it's audiences
def extract_and_cleanup_for_audience(self, src, audience, dst):
#derivative_string = f"<document>{self.values.get(src)}</document>"
#derivative = newline.join(etree.fromstring(derivative_string).xpath(f".//sentence[contains(@audience,'{audience}')]/text()"))
derivative = newline.join(self.fad.get_spans_by_property_value(self.values.get(src), 'sentence', name="audiences", values=[audience]))
ret = self.query(prompts.get_prompt_recursive(
f"""#document-intro
This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
{self.fad.get_values_by_type("document summary")[0]}
An editor was tasked with creating a derivative document that is addressed to {audience}. The editor compiled all sentences from the document that have information relevant for {audience}. These sentences are dumped below the main document inside <derivative-document for="{audience}">...</derivative-document>.
Your goal is to edit the derivative document for structure and consistency. Rearrange its sentences in a way that makes sense for a reader. Group sentences into paragraphs. If necessary, write, or adapt from the original document, connecting sentences. Then, remove as many instances of "{audience}" as you can: since this entire document is now addressed to {audience}, you can just say "you" most of the time.
Output just your revision inside <derivative-document>...</derivative-document>.
Here's the original document:
<document>{self.fad.get_plaintext(self.values.get(src))}</document>
Here's the derivative document:
<derivative-document for="{audience}">{derivative}</derivative-document>
"""))
self.values.put(dst.format(audience=audience), llm_parsing.extract_document_from_response(ret))
def formatinput(self, src, selectors=None):
if selectors:
for selector in selectors.split(","):
what = selector.split('-')[0]
which = selector.split('-')[1]
match which:
case 'all':
material = self.fad.get_plaintext_with_markup(what, src=src, show_properties=False) # todo: add a way to select properties
case _:
# todo
pass
else:
material = self.fad.get_plaintext(src)
match src:
case "document":
return f"<document>{material}</document>"
case _:
return material
def parseoutput(self, outputformat, val, title=None, type=None):
match outputformat:
case "singlevalue":
return val
case "document" | "sentence" | "section":
return llm_parsing.extract_document_from_response(val)
case "list":
list = llm_parsing.parse_list(val, title)
return list
def derive(self,
sideinfo = None,
src = "document",
selectors = None, # can be a comma-separated list of selectors applied in turn
target = "what-this-is",
outputformat="singlevalue"
):
if self.fad.get(src) == None or not(self.fad.is_newer(src, target)):
print(f"No {src}, or {src} already has up-to-date (newer) {target}, skip query")
return # we already have an up-to-date target
##### constructing prompt
## GENERAL: what is the task we're working on here
general_prompt = prompts.get_prompt(f"#{src}-general")
## SIDEINFO: information not in the document which is needed for the current task
sideinfo_prompt = ''
if sideinfo and self.fad.get(sideinfo):
sideinfo_prompt = prompts.get_prompt(f"#{src}-{sideinfo}").format(value=self.fad.get_values_by_type_string(sideinfo))
## INTRO: introducing the document or part we're working on
intro_prompt = prompts.get_prompt(f"{src}-intro")
## SUMMARIES: already-extracted summary-like information bits about the document or part
summaries = []
summaries_prompt = ''
for s in fad.summaries: # a set list for now
if self.fad.get_note(src).get(s): # if this src has this summary
summaries.append(prompts.get_prompt(f'#{src}-{s}').format(name=src, value=self.fad.get_note(src).get(s)))
if summaries:
summaries_prompt = " ".join(summaries) + "\n\n"
## SELECT: explaining which part of the document we have here
select_prompt = ''
if selectors:
for selector in selectors.split(','):
select_prompt += prompts.get_prompt(f"#select-{selector})") + " "
## Q: question or task, defined by our target
q = f"q-{outputformat}-{target}"
q_prompt = prompts.get_prompt(q)
## MATERIAL: document or part that we're working on; comes in the end because it is often the largest part of the prompt
material_prompt = prompts.get_prompt("#material").format(name=src, value=self.formatinput(src, selectors))
## run the query
ret = self.query(f"""{general_prompt}{sideinfo_prompt}{intro_prompt}{summaries_prompt}{q_prompt}{material_prompt}""")
what = fad.plurals[target] if outputformat == 'list' or outputformat == 'sentence' else target
val = self.parseoutput(outputformat, ret, title=what)
match outputformat:
case 'singlevalue':
# the whole output is a single return value
self.fad.get_note(src).set_property_newnote(name=target, value=val, valuetype=target)
case 'list':
# the output is a list
self.fad.add_from_list(val, target)
case 'sentence' | 'section':
if target == outputformat:
# the output is newly created spans
self.create_spans_in_fad(val, outputformat)
else:
# the output is added attributes in XML markup passed as input
self.annotate_spans_in_fad(val, outputformat, what, target)
pass
case 'spans':
# the output is XML markup (elements only) added by LLM
pass
def ingest(self, filename):
soup = parse.parse(filename)
self.fad.add("document", soup.get_text())
def understand(self):
# fact finders:
self.derive(
src="document",
target="what-this-is",
outputformat="singlevalue"
)
self.derive(
src="document",
target="summary",
outputformat="singlevalue"
)
self.derive(
src="document",
target="concept",
outputformat="list"
)
self.derive(
src="document",
target="actor",
outputformat="list"
)
self.split_sentences(src="document") # not LLM!
self.derive(
sideinfo="actor",
src="document",
selectors="sentence-all",
target="actor",
outputformat="sentence"
)
print(self.fad.get_plaintext_with_markup("sentence", show_properties=True, properties_to_show="actors"))
self.derive(
src="document",
target="section",
outputformat="section"
)
print(self.fad.get_plaintext_with_markup("section"))
"""
if self.values.is_newer("document", "document type"):
self.query_document_type(src="document", dst="document type")
if self.values.is_newer("document", "document summary"):
self.query_summary(src="document", dst="document summary")
if self.values.is_newer("document", "concepts"):
self.query_concepts_and_actors(src="document", dst=['concepts', 'actors'])
self.generalize_actors(src="actors", dst="actors, generalized")
if self.values.is_newer("document", "sentences"):
self.split_sentences(src="document", dst="document")
self.annotate_sentences_with_actors( src="document", dst="document with actors")
self.annotate_sentences_with_audiences(src="document", dst="document with audiences")
self.extract_and_cleanup_for_audience(src='document', audience=self.fad.get_values_by_type("actor")[0], dst="document for {audience}")
"""
def get_understanding(self):
self.understand()
return (
self.fad.get_value_by_type("what-this-is"),
self.fad.get_value_by_type("summary"),
"=== Concepts:\n"+self.fad.get_values_by_type_string("concept"),
"=== Actors:\n"+self.fad.get_values_by_type_string("actor"),
"=== Document annotated:\n"+self.fad.get_plaintext_with_markup("sentence", show_properties=True, properties_to_show="actors"),
f"=== Document broken into sections {self.fad.get_plaintext_with_markup('section')}"
)
if __name__ == "__main__":
d = Document(llm.Debug())
d.ingest("samples/submission.html")
#print(d.get_understanding())
s = """<document>
<section>
The process of submitting a scientific paper to a journal involves several sequential stages, starting with the preparation of the manuscript. Researchers meticulously compile and format their findings according to the specific guidelines outlined by the journal of their choice. Once the manuscript is ready, authors initiate the submission process, typically done through an online system or by email.
</section>
<section>
The journey of the manuscript begins with an initial editorial review conducted by journal editors. This preliminary assessment aims to ensure that the submission aligns with the journal's scope, adheres to ethical standards, and meets the specified formatting requirements. Following this, the manuscript undergoes a critical phase known as peer review. Editors select expert reviewers in the relevant field to evaluate the manuscript's quality, methodology, and contribution to scientific knowledge.
</section>
<section>
Based on the feedback received from reviewers, authors may be required to make revisions to their manuscript. These revisions could range from minor adjustments to substantial changes. After the authors have made the necessary modifications, the manuscript returns to the editorial team for a final decision.
</section>
<section>
The editorial decision involves a comprehensive evaluation of the revised manuscript, taking into account the recommendations provided by the reviewers. Depending on the assessment, the decision may be acceptance, further revisions, or rejection. If revisions are necessary, authors diligently modify their manuscript and resubmit it for another round of evaluation.
</section>
<section>
Once the manuscript successfully passes through the review and revision stages, it moves on to the final steps before publication. The accepted manuscript undergoes copyediting and formatting to ensure adherence to the journal's style and standards. Authors are then provided with the final proofs for a meticulous review and approval before the paper is officially published.
</section>
<section>
The publication process extends beyond the journal's platform, as the paper may be indexed in various databases and repositories, enhancing its visibility and accessibility to the broader scientific community. Throughout this intricate process, several key actors play crucial roles, including the authors, editors, reviewers, copyeditors, and publishers, all working collaboratively to contribute to the dissemination of valuable scientific knowledge.
</section>
</document>"""
val = d.parseoutput("section", s, title="section")
d.create_spans_in_fad(val, "section")
print(d.fad.get_plaintext_with_markup('section'))