Spaces:
Runtime error
Runtime error
Download understand.py from PrecisionContent/DU: direct link, hf CLI and curl.
- Browser
- Download file 26.1 kB
-
https://huggingface.co/spaces/PrecisionContent/DU/resolve/main/understand.py
- Command line
-
hf download hf://spaces/PrecisionContent/DU/understand.py
-
curl -L -o understand.py https://huggingface.co/spaces/PrecisionContent/DU/resolve/main/understand.py
26.1 kB
| import re | |
| from lxml import etree | |
| import nltk.tokenize | |
| import nltk | |
| nltk.download('punkt') | |
| import prompts | |
| import parse | |
| import llm | |
| import llm_parsing | |
| import fad | |
| from versioneddict import VersionedDict | |
| newline = "\n" | |
| class Document(): | |
| def __init__(self, llm) -> None: | |
| self.LLM = llm | |
| # TODO: move all understandings to this object instead of self.values | |
| # for this, need to implement timestamps and is_newer | |
| self.fad = fad.Fad() | |
| def query(self, s): | |
| try: | |
| answer, truncated = self.LLM.send_question(s) | |
| # log | |
| print(f"===Question: \"\"\"{s}\"\"\"") | |
| print(f"===Answer: \"\"\"{answer}\"\"\"") | |
| print(f"===Truncated? {truncated}") | |
| return answer.strip() | |
| except Exception as e: | |
| print (e) | |
| return None | |
| def query_n(self, n, s): | |
| return [self.query(s) for i in range(n)] | |
| def query_document_type(self, src="document", dst="document type"): | |
| ret = self.query(prompts.get_prompt_recursive( | |
| f"""#document-intro | |
| #q-document-type | |
| <document>{self.values.get(src)}</document>""")) | |
| self.fad.newnote(type=dst, value=ret) | |
| def query_summary(self, src="document", dst="document summary"): | |
| ret = self.query(prompts.get_prompt_recursive( | |
| f"""#document-intro | |
| This document is a {self.fad.get_values_by_type("document type")[0]}. | |
| #q-document-summary | |
| <document>{self.values.get(src)}</document>""")) | |
| self.fad.newnote(type=dst, value=ret) | |
| def query_concepts_and_actors(self, src="document", dst=[]): | |
| concepts = dst[0] | |
| actors = dst[1] | |
| two_attempts = [{actors: [], concepts: []}, {actors: [], concepts: []}] | |
| ret = self.query_n(2, prompts.get_prompt_recursive( | |
| f"""#document-intro | |
| This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document: | |
| {self.fad.get_values_by_type("document summary")[0]} | |
| #q-concepts-actors | |
| Here's the document: | |
| <document>{self.fad.get_plaintext(self.values.get(src))}</document>""")) | |
| two_attempts[0][concepts] = llm_parsing.parse_list(ret[0], concepts) | |
| two_attempts[1][concepts] = llm_parsing.parse_list(ret[1], concepts) | |
| two_attempts[0][actors] = llm_parsing.parse_list(ret[0], actors) | |
| two_attempts[1][actors] = llm_parsing.parse_list(ret[1], actors) | |
| print(f"===Reconcile: {two_attempts}") | |
| #reconcile two lists | |
| reconciled = {actors: [], concepts: []} | |
| jackard = {actors: [], concepts: []} | |
| for what in [ | |
| {'plural': "concepts", 'singular': "concept", 'full': "concepts that this document talks about"}, | |
| {'plural': "actors", 'singular': "actor", 'full': "actors described or implied in the document"} | |
| ]: | |
| set1 = set(two_attempts[0][what['plural']]) | |
| set2 = set(two_attempts[1][what['plural']]) | |
| jackard[what['plural']] = len(set1.intersection(set2)) / len(set1.union(set2)) | |
| if set1 != set2: | |
| print(f"===Two attempts of {what['plural']} are different, jaccard similarity: {jackard[what['plural']]}") | |
| ret = self.query(prompts.get_prompt_recursive( | |
| f"""#document-intro | |
| This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document: | |
| {self.fad.get_values_by_type("document summary")[0]} | |
| Two different readers have read this document and created different lists of the {what['full']}. Your goal is to read these lists and the entire document carefully and then create a new combined list of the document's {what['plural']}. | |
| Compare the two lists: | |
| - If they have the same {what['singular']} but expressed differently, add a better expression of that {what['singular']} to your final list. | |
| - If any of the {what['plural']} in these lists is actually irrelevant or confusing, drop it. | |
| - If you see some {what['singular']} in the document that both lists missed, add it. | |
| First list of {what['plural']}: | |
| {newline.join(["- " + x for x in two_attempts[0][what['plural']]])} | |
| Second list of {what['plural']}: | |
| {newline.join(["- " + x for x in two_attempts[1][what['plural']]])} | |
| Output the line "{what['plural'].upper()}:" followed by your combined list of {what['plural']}, one per line. | |
| Here's the document: | |
| <document>{self.fad.get_plaintext(self.values.get(src))}</document>""")) | |
| reconciled[what['plural']] = llm_parsing.parse_lists(ret, what['plural']) | |
| print(f"===Reconciled {what['plural']}: {reconciled[what['plural']]}") | |
| else: | |
| reconciled[what['plural']] = two_attempts[0][what['plural']] # they are the same, so we take the first one | |
| # finally store in the values | |
| self.values.put(concepts, reconciled[concepts]) | |
| self.values.put(actors, reconciled[actors]) | |
| self.values.put("concepts jackard distance", jackard[concepts]) | |
| self.values.put("actors jackard distance", jackard[actors]) | |
| # also store concepts in the fad notes | |
| self.fad.add_from_list(reconciled[concepts], "concept") | |
| self.fad.add_from_list(reconciled[actors], "actor") | |
| print("store concepts in the fad notes") | |
| print(reconciled[concepts]) | |
| print(reconciled[actors]) | |
| print(f"===Reconciled: Concepts: {newline.join(self.fad.get_values_by_type('concept'))}, Actors: {newline.join(self.fad.get_values_by_type('actor'))}") | |
| def generalize_actors(self, src="actors", dst="actors, generalized"): | |
| ret = self.query(prompts.get_prompt_recursive( | |
| f"""#document-intro | |
| This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document: | |
| {self.fad.get_values_by_type("document summary")[0]} | |
| A reader of this document has compiled a list of actors mentioned or implied in the document. Here is their list: | |
| {newline.join(["- " + x for x in self.values.get("actors")])} | |
| Your goal is to generalize this list, replacing groups of actors playing the same or similar roles with their generic term. If an actor is not very relevant to the document (that is, you cannot see this document as written by, for, or about this actor), then drop it altogether. Also, drop any inanimate actors, leaving only categories of persons. Overall, pare this list down as much as possible, leaving only the most important categories of actors. | |
| Output the line "ACTORS:" followed by your list of actors, one per line. | |
| <document>{self.fad.get_plaintext(self.values.get("document"))}</document>""")) | |
| generalized = llm_parsing.parse_lists(ret, "actors") | |
| self.values.put("actors, generalized", generalized) | |
| # also store concepts in the fad marks | |
| self.fad.delete_by_type("actor") | |
| self.fad.add_from_list(generalized, "actor") | |
| def split_sentences(self, src="document"): | |
| plaintext = self.fad.get_plaintext(src) | |
| sents = nltk.tokenize.sent_tokenize(plaintext) | |
| i = 0 | |
| for sent in sents: | |
| start = plaintext.find(sent, i) | |
| if start == -1: | |
| raise(Exception(f"ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source: can't find '{sent}' in '{plaintext[i:]}'")) | |
| self.fad.add_span_mark("sentence", start, start+len(sent), src=src) | |
| i = start+len(sent) | |
| print(f"=== Sentence-split FAD : {self.fad.get_plaintext_with_markup('sentence', src=src)}") | |
| # todo: pass this through LLM to verify and correct sentence splittings | |
| def annotate_spans_in_fad(self, llmtext, span, property, notetype): | |
| sents = re.split(f'</{span}>', llmtext) | |
| print(property, notetype) | |
| for i, sent in enumerate(sents): | |
| print(i, sent) | |
| if f'<{span}' in sent: | |
| match = re.match(f'[^<]*<{span}[^>]+{property}="([^"]+)"', sent) | |
| if match: | |
| vals = match.group(1) | |
| for val in re.split(",\s+", vals): | |
| print(f"property {property} val {val} notetype {notetype}") | |
| self.fad.add_property_to_mark(span, i + 1, property, val, notetype) | |
| def create_spans_in_fad(self, llmtext, span): | |
| pattern = "" | |
| for p in [f'</?{span}>', f'\n?</?{span}>\n?', f'\n*</?{span}>\n*']: | |
| clean = re.sub(p, '', llmtext) | |
| if len(clean) <= len(self.fad.get_plaintext()): | |
| pattern = p | |
| break | |
| pattern_c = pattern | |
| pattern_o = pattern.replace('/?', '') | |
| pattern_c_o = pattern_c + pattern_o | |
| chunks = re.split(pattern_c_o, llmtext) | |
| t = 0 | |
| for i, chunk in enumerate(chunks): | |
| if f"{span}>" in chunk: | |
| text = re.sub(pattern_o, '', chunk) | |
| text = re.sub(pattern_c, '', text) | |
| else: | |
| text = chunk | |
| self.fad.add_span_mark(span, t, t + len(text) + 1) | |
| t += len(text) + 1 | |
| def annotate_sentences_with_actors(self, src="document", dst="document with actors"): | |
| markup = self.fad.get_plaintext_with_markup(self.values.get(src), "sentence") | |
| print(f"=== Sentence markup: {markup}") | |
| ret = self.query_n(2, prompts.get_prompt_recursive( | |
| f"""#document-intro | |
| This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document: | |
| {self.fad.get_values_by_type("document summary")[0]} | |
| A reader of this document has compiled a list of actors (parties that do something) mentioned or implied in the document. Here is their list: | |
| {newline.join(["- " + x for x in self.values.get("actors, generalized")])} | |
| Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended actor(s) by picking one or more parties from this list. For each sentence, ask yourself: which of the parties from the list are the subjects? Who does the action described here? If there's no actor, leave the sentence unmarked. | |
| Add the relevant partes, separated by commas, inside the "actors" attribute of each <sentence> tag. For example, suppose you have a document describing the game of Sudoku, and your parties are "Players" and "Creators". Here are example sentences with audience markup: | |
| <example> | |
| <sentence>Sudoku is a popular logic puzzle with numbers with simple rules that even beginners can easily master.</sentence> <sentence actors="Creators">You should not create a puzzle with more than one possible solution, as that would usually be considered defective.</sentence> <sentence actors="Players">If you have completed finding all positions for a number, draw an X over that number in the list outside.</sentence> | |
| </example> | |
| Output just the document adding the actors attributes for sentences, but otherwise unchanged: | |
| <document>{markup}</document>""")) | |
| annotated0, annotated1 = llm_parsing.extract_document_from_response(ret[0]), llm_parsing.extract_document_from_response(ret[1]) | |
| diff = newline.join([str(x) for x in zip(annotated0.split('</sentence>'), annotated1.split('</sentence>')) if x[0] != x[1]]) | |
| self.values.put(dst, annotated1) | |
| self.values.put(f"{dst}, diff", diff) | |
| #todo: reconcile two lists | |
| self.annotate_spans_in_fad(annotated0, "sentence", "actors", "actor") | |
| def annotate_sentences_with_audiences(self, src="document", dst="document with audiences"): | |
| # EXPERIMENTAL! we use the actors list but here interpret it as audiences | |
| markup = self.fad.get_plaintext_with_markup(self.values.get(src), "sentence") | |
| print(f"=== Sentence markup: {markup}") | |
| ret = self.query_n(2, prompts.get_prompt_recursive( | |
| f"""#document-intro | |
| This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document: | |
| {self.fad.get_values_by_type("document summary")[0]} | |
| A reader of this document has compiled a list of audiences (parties that parts of this document are addressed to) mentioned or implied in the document. Here is their list: | |
| {newline.join(["- " + x for x in self.values.get("actors, generalized")])} | |
| Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended audience(s) by picking one or more parties from this list. For each sentence, ask yourself: who, from the list, would want to hear this? Who needs to know this? If the intended audience is unclear, leave the sentence unmarked. | |
| Add the relevant partes, separated by commas, inside the "audiences" attribute of each <sentence> tag. For example, suppose you have a document describing the game of Sudoku, and your parties are "Players" and "Creators". Here are example sentences with audience markup: | |
| <example> | |
| <sentence audiences="players, creators">Sudoku is a popular logic puzzle with numbers with simple rules that even beginners can easily master.</sentence> <sentence actors="creators">You should not create a puzzle with more than one possible solution, as that would usually be considered defective.</sentence> <sentence actors="players">If you have completed finding all positions for a number, draw an X over that number in the list outside.</sentence> | |
| </example> | |
| Output just the document adding the "audiences" attributes for sentences, but otherwise unchanged: | |
| <document>{markup}</document>""")) | |
| annotated0, annotated1 = llm_parsing.extract_document_from_response(ret[0]), llm_parsing.extract_document_from_response(ret[1]) | |
| diff = newline.join([str(x) for x in zip(annotated0.split('</sentence>'), annotated1.split('</sentence>')) if x[0] != x[1]]) | |
| self.values.put(dst, annotated1) | |
| self.values.put(f"{dst}, diff", diff) | |
| #todo: reconcile two lists | |
| self.annotate_spans_in_fad(annotated0, "sentence", "audiences", "actor") # not an error, we use actors but tell llm it's audiences | |
| def extract_and_cleanup_for_audience(self, src, audience, dst): | |
| #derivative_string = f"<document>{self.values.get(src)}</document>" | |
| #derivative = newline.join(etree.fromstring(derivative_string).xpath(f".//sentence[contains(@audience,'{audience}')]/text()")) | |
| derivative = newline.join(self.fad.get_spans_by_property_value(self.values.get(src), 'sentence', name="audiences", values=[audience])) | |
| ret = self.query(prompts.get_prompt_recursive( | |
| f"""#document-intro | |
| This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document: | |
| {self.fad.get_values_by_type("document summary")[0]} | |
| An editor was tasked with creating a derivative document that is addressed to {audience}. The editor compiled all sentences from the document that have information relevant for {audience}. These sentences are dumped below the main document inside <derivative-document for="{audience}">...</derivative-document>. | |
| Your goal is to edit the derivative document for structure and consistency. Rearrange its sentences in a way that makes sense for a reader. Group sentences into paragraphs. If necessary, write, or adapt from the original document, connecting sentences. Then, remove as many instances of "{audience}" as you can: since this entire document is now addressed to {audience}, you can just say "you" most of the time. | |
| Output just your revision inside <derivative-document>...</derivative-document>. | |
| Here's the original document: | |
| <document>{self.fad.get_plaintext(self.values.get(src))}</document> | |
| Here's the derivative document: | |
| <derivative-document for="{audience}">{derivative}</derivative-document> | |
| """)) | |
| self.values.put(dst.format(audience=audience), llm_parsing.extract_document_from_response(ret)) | |
| def formatinput(self, src, selectors=None): | |
| if selectors: | |
| for selector in selectors.split(","): | |
| what = selector.split('-')[0] | |
| which = selector.split('-')[1] | |
| match which: | |
| case 'all': | |
| material = self.fad.get_plaintext_with_markup(what, src=src, show_properties=False) # todo: add a way to select properties | |
| case _: | |
| # todo | |
| pass | |
| else: | |
| material = self.fad.get_plaintext(src) | |
| match src: | |
| case "document": | |
| return f"<document>{material}</document>" | |
| case _: | |
| return material | |
| def parseoutput(self, outputformat, val, title=None, type=None): | |
| match outputformat: | |
| case "singlevalue": | |
| return val | |
| case "document" | "sentence" | "section": | |
| return llm_parsing.extract_document_from_response(val) | |
| case "list": | |
| list = llm_parsing.parse_list(val, title) | |
| return list | |
| def derive(self, | |
| sideinfo = None, | |
| src = "document", | |
| selectors = None, # can be a comma-separated list of selectors applied in turn | |
| target = "what-this-is", | |
| outputformat="singlevalue" | |
| ): | |
| if self.fad.get(src) == None or not(self.fad.is_newer(src, target)): | |
| print(f"No {src}, or {src} already has up-to-date (newer) {target}, skip query") | |
| return # we already have an up-to-date target | |
| ##### constructing prompt | |
| ## GENERAL: what is the task we're working on here | |
| general_prompt = prompts.get_prompt(f"#{src}-general") | |
| ## SIDEINFO: information not in the document which is needed for the current task | |
| sideinfo_prompt = '' | |
| if sideinfo and self.fad.get(sideinfo): | |
| sideinfo_prompt = prompts.get_prompt(f"#{src}-{sideinfo}").format(value=self.fad.get_values_by_type_string(sideinfo)) | |
| ## INTRO: introducing the document or part we're working on | |
| intro_prompt = prompts.get_prompt(f"{src}-intro") | |
| ## SUMMARIES: already-extracted summary-like information bits about the document or part | |
| summaries = [] | |
| summaries_prompt = '' | |
| for s in fad.summaries: # a set list for now | |
| if self.fad.get_note(src).get(s): # if this src has this summary | |
| summaries.append(prompts.get_prompt(f'#{src}-{s}').format(name=src, value=self.fad.get_note(src).get(s))) | |
| if summaries: | |
| summaries_prompt = " ".join(summaries) + "\n\n" | |
| ## SELECT: explaining which part of the document we have here | |
| select_prompt = '' | |
| if selectors: | |
| for selector in selectors.split(','): | |
| select_prompt += prompts.get_prompt(f"#select-{selector})") + " " | |
| ## Q: question or task, defined by our target | |
| q = f"q-{outputformat}-{target}" | |
| q_prompt = prompts.get_prompt(q) | |
| ## MATERIAL: document or part that we're working on; comes in the end because it is often the largest part of the prompt | |
| material_prompt = prompts.get_prompt("#material").format(name=src, value=self.formatinput(src, selectors)) | |
| ## run the query | |
| ret = self.query(f"""{general_prompt}{sideinfo_prompt}{intro_prompt}{summaries_prompt}{q_prompt}{material_prompt}""") | |
| what = fad.plurals[target] if outputformat == 'list' or outputformat == 'sentence' else target | |
| val = self.parseoutput(outputformat, ret, title=what) | |
| match outputformat: | |
| case 'singlevalue': | |
| # the whole output is a single return value | |
| self.fad.get_note(src).set_property_newnote(name=target, value=val, valuetype=target) | |
| case 'list': | |
| # the output is a list | |
| self.fad.add_from_list(val, target) | |
| case 'sentence' | 'section': | |
| if target == outputformat: | |
| # the output is newly created spans | |
| self.create_spans_in_fad(val, outputformat) | |
| else: | |
| # the output is added attributes in XML markup passed as input | |
| self.annotate_spans_in_fad(val, outputformat, what, target) | |
| pass | |
| case 'spans': | |
| # the output is XML markup (elements only) added by LLM | |
| pass | |
| def ingest(self, filename): | |
| soup = parse.parse(filename) | |
| self.fad.add("document", soup.get_text()) | |
| def understand(self): | |
| # fact finders: | |
| self.derive( | |
| src="document", | |
| target="what-this-is", | |
| outputformat="singlevalue" | |
| ) | |
| self.derive( | |
| src="document", | |
| target="summary", | |
| outputformat="singlevalue" | |
| ) | |
| self.derive( | |
| src="document", | |
| target="concept", | |
| outputformat="list" | |
| ) | |
| self.derive( | |
| src="document", | |
| target="actor", | |
| outputformat="list" | |
| ) | |
| self.split_sentences(src="document") # not LLM! | |
| self.derive( | |
| sideinfo="actor", | |
| src="document", | |
| selectors="sentence-all", | |
| target="actor", | |
| outputformat="sentence" | |
| ) | |
| print(self.fad.get_plaintext_with_markup("sentence", show_properties=True, properties_to_show="actors")) | |
| self.derive( | |
| src="document", | |
| target="section", | |
| outputformat="section" | |
| ) | |
| print(self.fad.get_plaintext_with_markup("section")) | |
| """ | |
| if self.values.is_newer("document", "document type"): | |
| self.query_document_type(src="document", dst="document type") | |
| if self.values.is_newer("document", "document summary"): | |
| self.query_summary(src="document", dst="document summary") | |
| if self.values.is_newer("document", "concepts"): | |
| self.query_concepts_and_actors(src="document", dst=['concepts', 'actors']) | |
| self.generalize_actors(src="actors", dst="actors, generalized") | |
| if self.values.is_newer("document", "sentences"): | |
| self.split_sentences(src="document", dst="document") | |
| self.annotate_sentences_with_actors( src="document", dst="document with actors") | |
| self.annotate_sentences_with_audiences(src="document", dst="document with audiences") | |
| self.extract_and_cleanup_for_audience(src='document', audience=self.fad.get_values_by_type("actor")[0], dst="document for {audience}") | |
| """ | |
| def get_understanding(self): | |
| self.understand() | |
| return ( | |
| self.fad.get_value_by_type("what-this-is"), | |
| self.fad.get_value_by_type("summary"), | |
| "=== Concepts:\n"+self.fad.get_values_by_type_string("concept"), | |
| "=== Actors:\n"+self.fad.get_values_by_type_string("actor"), | |
| "=== Document annotated:\n"+self.fad.get_plaintext_with_markup("sentence", show_properties=True, properties_to_show="actors"), | |
| f"=== Document broken into sections {self.fad.get_plaintext_with_markup('section')}" | |
| ) | |
| if __name__ == "__main__": | |
| d = Document(llm.Debug()) | |
| d.ingest("samples/submission.html") | |
| #print(d.get_understanding()) | |
| s = """<document> | |
| <section> | |
| The process of submitting a scientific paper to a journal involves several sequential stages, starting with the preparation of the manuscript. Researchers meticulously compile and format their findings according to the specific guidelines outlined by the journal of their choice. Once the manuscript is ready, authors initiate the submission process, typically done through an online system or by email. | |
| </section> | |
| <section> | |
| The journey of the manuscript begins with an initial editorial review conducted by journal editors. This preliminary assessment aims to ensure that the submission aligns with the journal's scope, adheres to ethical standards, and meets the specified formatting requirements. Following this, the manuscript undergoes a critical phase known as peer review. Editors select expert reviewers in the relevant field to evaluate the manuscript's quality, methodology, and contribution to scientific knowledge. | |
| </section> | |
| <section> | |
| Based on the feedback received from reviewers, authors may be required to make revisions to their manuscript. These revisions could range from minor adjustments to substantial changes. After the authors have made the necessary modifications, the manuscript returns to the editorial team for a final decision. | |
| </section> | |
| <section> | |
| The editorial decision involves a comprehensive evaluation of the revised manuscript, taking into account the recommendations provided by the reviewers. Depending on the assessment, the decision may be acceptance, further revisions, or rejection. If revisions are necessary, authors diligently modify their manuscript and resubmit it for another round of evaluation. | |
| </section> | |
| <section> | |
| Once the manuscript successfully passes through the review and revision stages, it moves on to the final steps before publication. The accepted manuscript undergoes copyediting and formatting to ensure adherence to the journal's style and standards. Authors are then provided with the final proofs for a meticulous review and approval before the paper is officially published. | |
| </section> | |
| <section> | |
| The publication process extends beyond the journal's platform, as the paper may be indexed in various databases and repositories, enhancing its visibility and accessibility to the broader scientific community. Throughout this intricate process, several key actors play crucial roles, including the authors, editors, reviewers, copyeditors, and publishers, all working collaboratively to contribute to the dissemination of valuable scientific knowledge. | |
| </section> | |
| </document>""" | |
| val = d.parseoutput("section", s, title="section") | |
| d.create_spans_in_fad(val, "section") | |
| print(d.fad.get_plaintext_with_markup('section')) | |