Spaces:
Runtime error
Runtime error
Dmitry Kirsanov commited on
Commit ·
024fcd4
1
Parent(s): 86f0692
extract concepts and principles
Browse files- app.py +3 -0
- prompts.py +5 -5
- understand.py +48 -1
- versioneddict.py +12 -0
app.py
CHANGED
|
@@ -31,6 +31,9 @@ gradio_app = gr.Interface(
|
|
| 31 |
gr.Textbox(label="Document, annotated with actors", lines=18),
|
| 32 |
gr.Textbox(label="Document, annotated with actors, diff of 2 attempts", lines=18),
|
| 33 |
gr.Textbox(label="Document rewritten for one audience", lines=18),
|
|
|
|
|
|
|
|
|
|
| 34 |
]
|
| 35 |
)
|
| 36 |
|
|
|
|
| 31 |
gr.Textbox(label="Document, annotated with actors", lines=18),
|
| 32 |
gr.Textbox(label="Document, annotated with actors, diff of 2 attempts", lines=18),
|
| 33 |
gr.Textbox(label="Document rewritten for one audience", lines=18),
|
| 34 |
+
gr.Textbox(label="Extracted concepts", lines=18),
|
| 35 |
+
gr.Textbox(label="Extracted principles", lines=18),
|
| 36 |
+
gr.Textbox(label="Document remainder", lines=18),
|
| 37 |
]
|
| 38 |
)
|
| 39 |
|
prompts.py
CHANGED
|
@@ -84,7 +84,9 @@ Use task information to answer:
|
|
| 84 |
|
| 85 |
|
| 86 |
"information-types":
|
| 87 |
-
"""#intro-
|
|
|
|
|
|
|
| 88 |
|
| 89 |
#recognize-information-types
|
| 90 |
|
|
@@ -164,6 +166,8 @@ Write just one line with the description.""",
|
|
| 164 |
|
| 165 |
|
| 166 |
"fulldocument-intro": """Below is a document inside <document></document>. Please read this document carefully. Remember everything you know about the concepts it touches.""",
|
|
|
|
|
|
|
| 167 |
"q-document-summary": """Give a one-paragraph summary of the document's content. Write just one paragraph with the summary.""",
|
| 168 |
|
| 169 |
|
|
@@ -187,10 +191,6 @@ ACTORS:
|
|
| 187 |
""",
|
| 188 |
|
| 189 |
|
| 190 |
-
"": """""",
|
| 191 |
-
"": """""",
|
| 192 |
-
"": """""",
|
| 193 |
-
|
| 194 |
|
| 195 |
|
| 196 |
|
|
|
|
| 84 |
|
| 85 |
|
| 86 |
"information-types":
|
| 87 |
+
"""#intro-structured-writing
|
| 88 |
+
|
| 89 |
+
#intro-information-types
|
| 90 |
|
| 91 |
#recognize-information-types
|
| 92 |
|
|
|
|
| 166 |
|
| 167 |
|
| 168 |
"fulldocument-intro": """Below is a document inside <document></document>. Please read this document carefully. Remember everything you know about the concepts it touches.""",
|
| 169 |
+
|
| 170 |
+
|
| 171 |
"q-document-summary": """Give a one-paragraph summary of the document's content. Write just one paragraph with the summary.""",
|
| 172 |
|
| 173 |
|
|
|
|
| 191 |
""",
|
| 192 |
|
| 193 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
|
| 195 |
|
| 196 |
|
understand.py
CHANGED
|
@@ -90,7 +90,7 @@ This document is a {self.values.get("document type")}.
|
|
| 90 |
|
| 91 |
|
| 92 |
def parse_document(self, ret):
|
| 93 |
-
return re.sub(r'^<document>(.*)</document>$', r'\1', ret, flags=re.DOTALL)
|
| 94 |
|
| 95 |
def query_concepts(self):
|
| 96 |
two_attempts = [{"actors": [], "concepts": []}, {"actors": [], "concepts": []}]
|
|
@@ -253,6 +253,49 @@ Here's the derivative document:
|
|
| 253 |
self.values.put(f"""FAD for {self.values.get("actors, generalized")[0]}""", self.parse_document(ret))
|
| 254 |
|
| 255 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 256 |
def ingest(self, filename):
|
| 257 |
soup = parse.parse(filename)
|
| 258 |
self.values.put("document (soup)", soup)
|
|
@@ -269,6 +312,7 @@ Here's the derivative document:
|
|
| 269 |
self.generalize_actors()
|
| 270 |
self.annotate_actors()
|
| 271 |
self.split_document()
|
|
|
|
| 272 |
|
| 273 |
def get_understanding(self):
|
| 274 |
self.understand()
|
|
@@ -282,6 +326,9 @@ Here's the derivative document:
|
|
| 282 |
"=== Document annotated:\n"+self.values.get("FAD with audience"),
|
| 283 |
"=== Document annotated, diff of 2 attempts:\n"+self.values.get("FAD with audience, diff"),
|
| 284 |
f"=== Document rewritten for {self.values.get('actors, generalized')[0]}:\n"+self.values.get(f"""FAD for {self.values.get("actors, generalized")[0]}"""),
|
|
|
|
|
|
|
|
|
|
| 285 |
)
|
| 286 |
|
| 287 |
|
|
|
|
| 90 |
|
| 91 |
|
| 92 |
def parse_document(self, ret):
|
| 93 |
+
return re.sub(r'^<([a-z]+-)?document>(.*)</([a-z]+-)?document>$', r'\1', ret, flags=re.DOTALL)
|
| 94 |
|
| 95 |
def query_concepts(self):
|
| 96 |
two_attempts = [{"actors": [], "concepts": []}, {"actors": [], "concepts": []}]
|
|
|
|
| 253 |
self.values.put(f"""FAD for {self.values.get("actors, generalized")[0]}""", self.parse_document(ret))
|
| 254 |
|
| 255 |
|
| 256 |
+
def chunk_document_for(self, audience):
|
| 257 |
+
ret = self.query(prompts.get_prompt_recursive(
|
| 258 |
+
f"""#information-types
|
| 259 |
+
|
| 260 |
+
Let's start with concepts and principles. #fulldocument-intro
|
| 261 |
+
|
| 262 |
+
Please read through the document and extract useful concept and principle chunks from it.
|
| 263 |
+
|
| 264 |
+
Each concept or principle can be anything from a single sentence or several paragraphs in length. It must have the same information as is contained in the original document, but you can edit or reword it so it reads better on its own. Give each one a title prefixed with "CONCEPT:" or "PRINCIPLE:".
|
| 265 |
+
|
| 266 |
+
Output just your revision inside <revised-document>...</revised-document>.
|
| 267 |
+
|
| 268 |
+
<document>{self.values.get(f"""FAD""")}</document>
|
| 269 |
+
"""))
|
| 270 |
+
titles = re.finditer("(CONCEPT:)|(PRINCIPLE:)", ret).tolist()
|
| 271 |
+
for i, t in enumerate(titles):
|
| 272 |
+
start = t.start(0)
|
| 273 |
+
end = -1 if i == len(titles) - 1 else titles[i+1].start(0)
|
| 274 |
+
chunk = ret[start:end]
|
| 275 |
+
if chunk.startswith('CONCEPT'):
|
| 276 |
+
self.values.append("Concepts", chunk)
|
| 277 |
+
elif chunk.startswith('PRINCIPLE'):
|
| 278 |
+
self.values.append("Principles", chunk)
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
concepts = ["<excerpt>"+s+"</excerpt>\n\n" for s in self.values.get("Concepts")]
|
| 282 |
+
principles = ["<excerpt>"+s+"</excerpt>\n\n" for s in self.values.get("Principles")]
|
| 283 |
+
ret = self.query("""#fulldocument-intro
|
| 284 |
+
|
| 285 |
+
Below, there are a few excerpts from this document. Your goal is to go through the document and erase parts of it that is duplicated in these excertps. In other words, please do a subtraction: new document = document - excerpts. Note that the excertps may not literally match the text of the document; use your judgement to decide what parts of the document are redundant, given the excertps, and erase them.
|
| 286 |
+
|
| 287 |
+
<document>{self.values.get("FAD")}</document>
|
| 288 |
+
|
| 289 |
+
{concepts}
|
| 290 |
+
{principles}
|
| 291 |
+
""")
|
| 292 |
+
self.values.put(f"""FAD minus concepts, principles""", self.parse_document(ret))
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
|
| 299 |
def ingest(self, filename):
|
| 300 |
soup = parse.parse(filename)
|
| 301 |
self.values.put("document (soup)", soup)
|
|
|
|
| 312 |
self.generalize_actors()
|
| 313 |
self.annotate_actors()
|
| 314 |
self.split_document()
|
| 315 |
+
self.chunk_document_for(self.values.get('actors, generalized')[0])
|
| 316 |
|
| 317 |
def get_understanding(self):
|
| 318 |
self.understand()
|
|
|
|
| 326 |
"=== Document annotated:\n"+self.values.get("FAD with audience"),
|
| 327 |
"=== Document annotated, diff of 2 attempts:\n"+self.values.get("FAD with audience, diff"),
|
| 328 |
f"=== Document rewritten for {self.values.get('actors, generalized')[0]}:\n"+self.values.get(f"""FAD for {self.values.get("actors, generalized")[0]}"""),
|
| 329 |
+
f"=== Extracted concepts:\n"+self.values.get(f"""Concepts"""),
|
| 330 |
+
f"=== Extracted principles:\n"+self.values.get(f"""Principles"""),
|
| 331 |
+
f"=== Document remainder:\n"+self.values.get(f"""FAD minus concepts, principles"""),
|
| 332 |
)
|
| 333 |
|
| 334 |
|
versioneddict.py
CHANGED
|
@@ -17,6 +17,18 @@ class VersionedDict:
|
|
| 17 |
def touch(self, name):
|
| 18 |
timestamp = time.time()
|
| 19 |
self.data[name] = (self.get(name), timestamp)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
def is_newer(self, name1, name2):
|
| 22 |
timestamp1 = self.get_timestamp(name1)
|
|
|
|
| 17 |
def touch(self, name):
|
| 18 |
timestamp = time.time()
|
| 19 |
self.data[name] = (self.get(name), timestamp)
|
| 20 |
+
|
| 21 |
+
def append(self, name, value):
|
| 22 |
+
timestamp = time.time()
|
| 23 |
+
if name in self.data:
|
| 24 |
+
current_value, _ = self.data[name]
|
| 25 |
+
if isinstance(current_value, list):
|
| 26 |
+
current_value.append(value)
|
| 27 |
+
self.data[name] = (current_value, timestamp)
|
| 28 |
+
else:
|
| 29 |
+
raise ValueError(f"Value for key '{name}' is not a list")
|
| 30 |
+
else:
|
| 31 |
+
self.data[name] = ([value], timestamp)
|
| 32 |
|
| 33 |
def is_newer(self, name1, name2):
|
| 34 |
timestamp1 = self.get_timestamp(name1)
|