Dmitry Kirsanov commited on
Commit
024fcd4
·
1 Parent(s): 86f0692

extract concepts and principles

Browse files
Files changed (4) hide show
  1. app.py +3 -0
  2. prompts.py +5 -5
  3. understand.py +48 -1
  4. versioneddict.py +12 -0
app.py CHANGED
@@ -31,6 +31,9 @@ gradio_app = gr.Interface(
31
  gr.Textbox(label="Document, annotated with actors", lines=18),
32
  gr.Textbox(label="Document, annotated with actors, diff of 2 attempts", lines=18),
33
  gr.Textbox(label="Document rewritten for one audience", lines=18),
 
 
 
34
  ]
35
  )
36
 
 
31
  gr.Textbox(label="Document, annotated with actors", lines=18),
32
  gr.Textbox(label="Document, annotated with actors, diff of 2 attempts", lines=18),
33
  gr.Textbox(label="Document rewritten for one audience", lines=18),
34
+ gr.Textbox(label="Extracted concepts", lines=18),
35
+ gr.Textbox(label="Extracted principles", lines=18),
36
+ gr.Textbox(label="Document remainder", lines=18),
37
  ]
38
  )
39
 
prompts.py CHANGED
@@ -84,7 +84,9 @@ Use task information to answer:
84
 
85
 
86
  "information-types":
87
- """#intro-information-types
 
 
88
 
89
  #recognize-information-types
90
 
@@ -164,6 +166,8 @@ Write just one line with the description.""",
164
 
165
 
166
  "fulldocument-intro": """Below is a document inside <document></document>. Please read this document carefully. Remember everything you know about the concepts it touches.""",
 
 
167
  "q-document-summary": """Give a one-paragraph summary of the document's content. Write just one paragraph with the summary.""",
168
 
169
 
@@ -187,10 +191,6 @@ ACTORS:
187
  """,
188
 
189
 
190
- "": """""",
191
- "": """""",
192
- "": """""",
193
-
194
 
195
 
196
 
 
84
 
85
 
86
  "information-types":
87
+ """#intro-structured-writing
88
+
89
+ #intro-information-types
90
 
91
  #recognize-information-types
92
 
 
166
 
167
 
168
  "fulldocument-intro": """Below is a document inside <document></document>. Please read this document carefully. Remember everything you know about the concepts it touches.""",
169
+
170
+
171
  "q-document-summary": """Give a one-paragraph summary of the document's content. Write just one paragraph with the summary.""",
172
 
173
 
 
191
  """,
192
 
193
 
 
 
 
 
194
 
195
 
196
 
understand.py CHANGED
@@ -90,7 +90,7 @@ This document is a {self.values.get("document type")}.
90
 
91
 
92
  def parse_document(self, ret):
93
- return re.sub(r'^<document>(.*)</document>$', r'\1', ret, flags=re.DOTALL)
94
 
95
  def query_concepts(self):
96
  two_attempts = [{"actors": [], "concepts": []}, {"actors": [], "concepts": []}]
@@ -253,6 +253,49 @@ Here's the derivative document:
253
  self.values.put(f"""FAD for {self.values.get("actors, generalized")[0]}""", self.parse_document(ret))
254
 
255
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
256
  def ingest(self, filename):
257
  soup = parse.parse(filename)
258
  self.values.put("document (soup)", soup)
@@ -269,6 +312,7 @@ Here's the derivative document:
269
  self.generalize_actors()
270
  self.annotate_actors()
271
  self.split_document()
 
272
 
273
  def get_understanding(self):
274
  self.understand()
@@ -282,6 +326,9 @@ Here's the derivative document:
282
  "=== Document annotated:\n"+self.values.get("FAD with audience"),
283
  "=== Document annotated, diff of 2 attempts:\n"+self.values.get("FAD with audience, diff"),
284
  f"=== Document rewritten for {self.values.get('actors, generalized')[0]}:\n"+self.values.get(f"""FAD for {self.values.get("actors, generalized")[0]}"""),
 
 
 
285
  )
286
 
287
 
 
90
 
91
 
92
  def parse_document(self, ret):
93
+ return re.sub(r'^<([a-z]+-)?document>(.*)</([a-z]+-)?document>$', r'\1', ret, flags=re.DOTALL)
94
 
95
  def query_concepts(self):
96
  two_attempts = [{"actors": [], "concepts": []}, {"actors": [], "concepts": []}]
 
253
  self.values.put(f"""FAD for {self.values.get("actors, generalized")[0]}""", self.parse_document(ret))
254
 
255
 
256
+ def chunk_document_for(self, audience):
257
+ ret = self.query(prompts.get_prompt_recursive(
258
+ f"""#information-types
259
+
260
+ Let's start with concepts and principles. #fulldocument-intro
261
+
262
+ Please read through the document and extract useful concept and principle chunks from it.
263
+
264
+ Each concept or principle can be anything from a single sentence or several paragraphs in length. It must have the same information as is contained in the original document, but you can edit or reword it so it reads better on its own. Give each one a title prefixed with "CONCEPT:" or "PRINCIPLE:".
265
+
266
+ Output just your revision inside <revised-document>...</revised-document>.
267
+
268
+ <document>{self.values.get(f"""FAD""")}</document>
269
+ """))
270
+ titles = re.finditer("(CONCEPT:)|(PRINCIPLE:)", ret).tolist()
271
+ for i, t in enumerate(titles):
272
+ start = t.start(0)
273
+ end = -1 if i == len(titles) - 1 else titles[i+1].start(0)
274
+ chunk = ret[start:end]
275
+ if chunk.startswith('CONCEPT'):
276
+ self.values.append("Concepts", chunk)
277
+ elif chunk.startswith('PRINCIPLE'):
278
+ self.values.append("Principles", chunk)
279
+
280
+
281
+ concepts = ["<excerpt>"+s+"</excerpt>\n\n" for s in self.values.get("Concepts")]
282
+ principles = ["<excerpt>"+s+"</excerpt>\n\n" for s in self.values.get("Principles")]
283
+ ret = self.query("""#fulldocument-intro
284
+
285
+ Below, there are a few excerpts from this document. Your goal is to go through the document and erase parts of it that is duplicated in these excertps. In other words, please do a subtraction: new document = document - excerpts. Note that the excertps may not literally match the text of the document; use your judgement to decide what parts of the document are redundant, given the excertps, and erase them.
286
+
287
+ <document>{self.values.get("FAD")}</document>
288
+
289
+ {concepts}
290
+ {principles}
291
+ """)
292
+ self.values.put(f"""FAD minus concepts, principles""", self.parse_document(ret))
293
+
294
+
295
+
296
+
297
+
298
+
299
  def ingest(self, filename):
300
  soup = parse.parse(filename)
301
  self.values.put("document (soup)", soup)
 
312
  self.generalize_actors()
313
  self.annotate_actors()
314
  self.split_document()
315
+ self.chunk_document_for(self.values.get('actors, generalized')[0])
316
 
317
  def get_understanding(self):
318
  self.understand()
 
326
  "=== Document annotated:\n"+self.values.get("FAD with audience"),
327
  "=== Document annotated, diff of 2 attempts:\n"+self.values.get("FAD with audience, diff"),
328
  f"=== Document rewritten for {self.values.get('actors, generalized')[0]}:\n"+self.values.get(f"""FAD for {self.values.get("actors, generalized")[0]}"""),
329
+ f"=== Extracted concepts:\n"+self.values.get(f"""Concepts"""),
330
+ f"=== Extracted principles:\n"+self.values.get(f"""Principles"""),
331
+ f"=== Document remainder:\n"+self.values.get(f"""FAD minus concepts, principles"""),
332
  )
333
 
334
 
versioneddict.py CHANGED
@@ -17,6 +17,18 @@ class VersionedDict:
17
  def touch(self, name):
18
  timestamp = time.time()
19
  self.data[name] = (self.get(name), timestamp)
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
  def is_newer(self, name1, name2):
22
  timestamp1 = self.get_timestamp(name1)
 
17
  def touch(self, name):
18
  timestamp = time.time()
19
  self.data[name] = (self.get(name), timestamp)
20
+
21
+ def append(self, name, value):
22
+ timestamp = time.time()
23
+ if name in self.data:
24
+ current_value, _ = self.data[name]
25
+ if isinstance(current_value, list):
26
+ current_value.append(value)
27
+ self.data[name] = (current_value, timestamp)
28
+ else:
29
+ raise ValueError(f"Value for key '{name}' is not a list")
30
+ else:
31
+ self.data[name] = ([value], timestamp)
32
 
33
  def is_newer(self, name1, name2):
34
  timestamp1 = self.get_timestamp(name1)