Dmitry Kirsanov commited on
Commit
7b67970
·
1 Parent(s): 215f84e

start of fad class, fails test at this time

Browse files
Files changed (2) hide show
  1. fad.py +224 -0
  2. understand.py +73 -42
fad.py ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+
3
+ class Fad:
4
+
5
+ idlength = 8
6
+
7
+ notetypes = {
8
+ # generic
9
+ "span": 1,
10
+ "point": 2,
11
+
12
+ "sentence": 3,
13
+
14
+ "concept": 4,
15
+ "actor": 5,
16
+ "audience": 6,
17
+ }
18
+
19
+ notes = {}
20
+
21
+ pattern_marks = re.compile("[⌦⌫⌧]")
22
+
23
+ highest_note_id = 0
24
+
25
+ class Note:
26
+ def __init__(self, parentfad, id, type, value=None) -> None:
27
+ self.id = id
28
+ self.type = type
29
+ self.parentfad = parentfad
30
+ self.properties = {}
31
+ if value is not None:
32
+ self.properties["value"] = value
33
+
34
+ def get(self, name=None):
35
+ value = self.properties[name if name is not None else "value"]
36
+ if isinstance(value, int): # any int value is supposed to be a ref to id
37
+ return self.parentfad.notes[value].get() # so we pass it to the target mark, recursively
38
+ elif isinstance(value, list):
39
+ return ", ".join([(self.parentfad.notes[v].get() if isinstance(v, int) else v) for v in value]) # do the same for each element in list
40
+ else:
41
+ return value
42
+
43
+ def set(self, name, value):
44
+ if isinstance(value, int): # any int value is supposed to be a ref to id
45
+ assert(value in self.parentfad.notes.keys()) # so we ensure this id exists
46
+ self.properties[name] = value
47
+
48
+ def add(self, name, value):
49
+ if isinstance(value, int): # any int value is supposed to be a ref to id
50
+ assert(value in self.parentfad.notes.keys()) # so we ensure this id exists
51
+ if isinstance(self.properties[name], list):
52
+ self.properties[name].append(value)
53
+ elif self.properties[name] is None:
54
+ self.properties[name] = [value]
55
+ else:
56
+ raise("Note: add: trying to add to list (properties) but the existing value is not a list")
57
+
58
+ def add_notes_from_list(self, values, type):
59
+ for value in values:
60
+ self.newnote(type, value)
61
+
62
+ def get_note_by_value(self, value, valuetype):
63
+ for id, note in self.notes.items():
64
+ if note.type == self.notetypes[valuetype] and note.properties["value"] == value:
65
+ return id
66
+ return None
67
+
68
+ def get_notes_by_type_and_value(self, type, value):
69
+ ret = []
70
+ for id, m in self.notes.items():
71
+ if self.notetypes[type] == m.type:
72
+ if value == m.properties["value"]:
73
+ ret.append(id)
74
+ return ret
75
+
76
+
77
+
78
+
79
+ def get_next_mark(self, fad, start=0):
80
+ tstart = start
81
+ new = self.pattern_marks.search(fad, start)
82
+ if new != None:
83
+ tend = new.start()
84
+ id = int(fad[tend + 1:tend + self.idlength + 1])
85
+ return fad[tstart:tend], fad[tend], id, tend + self.idlength + 1
86
+ return fad[tstart:], -1, -1, -1 # no marks till the end
87
+
88
+ def get_plaintext(self, fad):
89
+ texts = []
90
+ i = 0
91
+ while True:
92
+ text, markchar, id, i = self.get_next_mark(fad, i)
93
+ texts.append(text)
94
+ if i == -1:
95
+ break
96
+ return "".join(texts)
97
+
98
+ def get_plaintext_with_markup(self, fad, notetype, show_id=False, show_properties=True):
99
+ i = 0
100
+ newfad = fad
101
+ while True:
102
+ text, markchar, id, newi = self.get_next_mark(newfad, i)
103
+ if newi == -1:
104
+ break
105
+ if self.notes[id].type == self.notetypes[notetype]:
106
+ id_s = f' id="{id}"' if show_id else ''
107
+ properties_s = ''.join([(f' {name}="{str(self.notes[id].get(name))}"' if show_properties else '') for name in self.notes[id].properties.keys()])
108
+ if markchar == '⌧':
109
+ newfad = newfad[:newi - self.idlength - 1] + f'<{notetype}{id_s}{properties_s}/>' + newfad[newi:]
110
+ else:
111
+ newfad = newfad[:newi - self.idlength - 1] + f'<{"/" if markchar == "⌫" else ""}{notetype}{id_s}{properties_s if markchar == "⌦" else ""}>' + newfad[newi:]
112
+ i = newi
113
+ return self.get_plaintext(newfad)
114
+
115
+ def newnote(self, type, value=None):
116
+ id = self.highest_note_id + 1
117
+ type_code = self.notetypes[type]
118
+ self.highest_note_id += 1
119
+ self.notes[id] = self.Note(self, id, type_code, value)
120
+ return id
121
+
122
+ def goto_plaintext_i(self, fad, to):
123
+ # fixme: suboptimal, can skip faster
124
+ passed = 0
125
+ i = 0
126
+ while True:
127
+ text, markchar, id, newi = self.get_next_mark(fad, i)
128
+ passed += len(text)
129
+ if passed >= to:
130
+ return i + len(text) - (passed - to)
131
+ elif newi == -1:
132
+ return -1 # could not reach to
133
+ else:
134
+ i = newi
135
+
136
+ def add_span_mark(self, fad, notetype, start, end):
137
+ id = self.newnote(notetype)
138
+ start_i = self.goto_plaintext_i(fad, start)
139
+ end_i = self.goto_plaintext_i(fad, end)
140
+ if start_i == -1 or end_i == -1:
141
+ raise Exception("fad: add_span_mark: start or end outside range")
142
+ newfad = fad[:start_i] + "⌦" + '{:08d}'.format(id) + fad[start_i:end_i] + "⌫" + '{:08d}'.format(id) + fad[end_i:]
143
+ return newfad
144
+
145
+ def add_point_mark(self, fad, notetype, start):
146
+ id = self.newnote(notetype)
147
+ start_i = self.goto_plaintext_i(fad, start)
148
+ if start_i == -1:
149
+ raise Exception("fad: point_mark: start outside range")
150
+ newfad = fad[:start_i] + "⌧" + '{:08d}'.format(id) + fad[start_i:]
151
+ return newfad
152
+
153
+ def get_note_for_mark_n(self, fad, notetype, mark_n, startfrom=0):
154
+ i = startfrom
155
+ n = 0
156
+ while True:
157
+ text, markchar, id, newi = self.get_next_mark(fad, i)
158
+ if newi == -1:
159
+ return None
160
+ if self.notes[id].type == self.notetypes[notetype]:
161
+ n += 1
162
+ if n == mark_n:
163
+ return id
164
+ i = newi
165
+
166
+ def set_property_to_mark(self, fad, notetype, mark_n, property, value, valuetype, startfrom=0):
167
+ noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
168
+ if valuetype != None:
169
+ id = self.get_note_by_value(value, valuetype)
170
+ self.notes[noteid].set(property, id)
171
+ else:
172
+ self.notes[noteid].set(property, value)
173
+
174
+ def add_property_to_mark(self, fad, notetype, mark_n, property, value, valuetype, startfrom=0):
175
+ noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
176
+ if valuetype != None:
177
+ id = self.get_note_by_value(value, valuetype)
178
+ self.marks[noteid].set(property, id)
179
+ else:
180
+ self.marks[noteid].set(property, value)
181
+
182
+
183
+ if __name__ == "__main__":
184
+ # tests
185
+
186
+ f = Fad()
187
+ orig = "A123456789B123456789C123456789"
188
+ fad = orig
189
+ fad = f.add_point_mark(fad, "point", 1)
190
+ fad = f.add_point_mark(fad, "point", 1)
191
+ fad = f.add_point_mark(fad, "point", 0)
192
+
193
+ #print(fad)
194
+ assert(fad == "⌧00000003A⌧00000002⌧00000001123456789B123456789C123456789")
195
+ #print(f.get_plaintext(fad))
196
+ assert(f.get_plaintext(fad) == orig)
197
+ #print(f.get_plaintext_with_markup(fad, "point", show_id=True))
198
+ assert(f.get_plaintext_with_markup(fad, "point", show_id=True) == '<point id="3"/>A<point id="2"/><point id="1"/>123456789B123456789C123456789')
199
+
200
+ fad = f.add_span_mark(fad, "span", 0, 10)
201
+ fad = f.add_span_mark(fad, "span", 20, 30)
202
+ fad = f.add_span_mark(fad, "span", 5, 25)
203
+
204
+ #print(fad)
205
+ assert(fad == "⌦00000004⌧00000003A⌧00000002⌧000000011234⌦0000000656789⌫00000004B123456789⌦00000005C1234⌫0000000656789⌫00000005")
206
+ #print(f.get_plaintext(fad))
207
+ assert(f.get_plaintext(fad) == orig)
208
+ #print(f.get_plaintext_with_markup(fad, "span"))
209
+ assert(f.get_plaintext_with_markup(fad, "span", show_id=True) == '<span id="4">A1234<span id="6">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
210
+ #print(f.get_plaintext_with_markup(fad, "point"))
211
+ assert(f.get_plaintext_with_markup(fad, "point", show_id=True) == '<point id="3"/>A<point id="2"/><point id="1"/>123456789B123456789C123456789')
212
+
213
+ f.add_notes_from_list(["actor1", "actor2"], "actor")
214
+ f.add_notes_from_list(["audience1", "audience2", "audience3"], "audience")
215
+
216
+ f.set_property_to_mark(fad, "span", 1, "actors", "actor1", "actor")
217
+
218
+ #print(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True))
219
+ assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
220
+
221
+ f.set_property_to_mark(fad, "span", 3, "actors", "actor2", "actor")
222
+
223
+ print(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True))
224
+ assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6">56789</span id="4">B123456789<span id="5" actors="actor2">C1234</span id="6">56789</span id="5">')
understand.py CHANGED
@@ -1,12 +1,13 @@
1
  import re
2
  from lxml import etree
3
- from nltk.tokenize import sent_tokenize
4
  import nltk
5
  nltk.download('punkt')
6
 
7
  import prompts
8
  import parse
9
  import llm
 
10
  from versioneddict import VersionedDict
11
 
12
  newline = "\n"
@@ -26,6 +27,7 @@ class Document():
26
  def __init__(self, llm) -> None:
27
  self.LLM = llm
28
  self.values = VersionedDict()
 
29
 
30
  def query(self, s):
31
  try:
@@ -43,13 +45,13 @@ class Document():
43
  return [self.query(s) for i in range(n)]
44
 
45
 
46
- def query_doctype(self):
47
  self.values.put("document type", self.query(prompts.get_prompt_recursive(
48
  f"""#fulldocument-intro
49
 
50
  #q-document-type
51
 
52
- <document>{self.values.get("FAD")}</document>""")))
53
 
54
 
55
  def query_summary(self):
@@ -60,7 +62,7 @@ This document is a {self.values.get("document type")}.
60
 
61
  #q-document-summary
62
 
63
- <document>{self.values.get("FAD")}</document>""")))
64
 
65
 
66
  def parse_lists(self, ret, what):
@@ -89,13 +91,13 @@ This document is a {self.values.get("document type")}.
89
  return retlist
90
 
91
 
92
- def parse_document(self, ret):
93
  r = re.sub(r'^<derivative-document>(.*)</derivative-document>$', r'\1', ret, flags=re.DOTALL)
94
  r = re.sub(r'^<revised-document>(.*)</revised-document>$', r'\1', r, flags=re.DOTALL)
95
  r = re.sub(r'^<document>(.*)</document>$', r'\1', r, flags=re.DOTALL)
96
  return r
97
 
98
- def query_concepts(self):
99
  two_attempts = [{"actors": [], "concepts": []}, {"actors": [], "concepts": []}]
100
 
101
  ret = self.query_n(2, prompts.get_prompt_recursive(
@@ -109,7 +111,7 @@ This document is a {self.values.get("document type")}. Here's a summary of the d
109
 
110
  Here's the document:
111
 
112
- <document>{self.values.get("FAD")}</document>"""))
113
 
114
  two_attempts[0]["concepts"] = self.parse_lists(ret[0], "concepts")
115
  two_attempts[1]["concepts"] = self.parse_lists(ret[1], "concepts")
@@ -155,7 +157,7 @@ Output the line "{what['plural'].upper()}:" followed by your combined list of {w
155
 
156
  Here's the document:
157
 
158
- <document>{self.values.get("FAD")}</document>"""))
159
  reconciled[what['plural']] = self.parse_lists(ret, what['plural'])
160
  print(f"===Reconciled {what['plural']}: {reconciled[what['plural']]}")
161
  else:
@@ -167,6 +169,9 @@ Here's the document:
167
  self.values.put("concepts jackard distance", jackard["concepts"])
168
  self.values.put("actors jackard distance", jackard["actors"])
169
 
 
 
 
170
  print(f"===Reconciled: Concepts: {self.values.get('concepts')}, Actors: {self.values.get('actors')}")
171
 
172
 
@@ -186,14 +191,34 @@ Your goal is to generalize this list, replacing groups of actors playing the sam
186
 
187
  Output the line "ACTORS:" followed by your list of actors, one per line.
188
 
189
- <document>{self.values.get("FAD")}</document>"""))
190
  generalized = self.parse_lists(ret, "actors")
191
  self.values.put("actors, generalized", generalized)
192
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
193
 
194
- def annotate_actors(self):
195
- sents = sent_tokenize(self.values.get("FAD"))
196
- markup = "".join([f'<sentence audience="">{x}</sentence>' for x in sents])
 
197
 
198
  print(f"=== Sentence markup: {markup}")
199
 
@@ -204,30 +229,36 @@ This document is a {self.values.get("document type")}. Here's a summary of the d
204
 
205
  {self.values.get("summary")}
206
 
207
- A reader of this document has compiled a list of actors (parties) mentioned or implied in the document. Here is their list:
208
 
209
  {newline.join(["- " + x for x in self.values.get("actors, generalized")])}
210
 
211
- Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended audience by picking one or more parties from this list. For each sentence, ask yourself: which of the parties from the list would want to know this sentence's information? Use only the parties from the list.
212
 
213
- Add the relevant partes, separated by commas, inside the "audience" attribute of each <sentence> tag. For example, suppose you have a document describing the game of Sudoku, and your parties are "Players" and "Creators". Here are example sentences with audience markup:
214
 
215
  <example>
216
- <sentence audience="Creators,Players">Sudoku is a popular logic puzzle with numbers with simple rules that even beginners can easily master.</sentence> <sentence audience="Creators">A presented puzzle with more than one possible solution would usually be considered defective.</sentence> <sentence audience="Players">If you have completed finding all positions for a number, draw an X over that number in the list outside.</sentence>
217
  </example>
218
 
219
- Output just the document with the audience markup sentence, but otherwise unchanged:
220
 
221
  <document>{markup}</document>"""))
222
- annotated0, annotated1 = self.parse_document(ret[0]), self.parse_document(ret[1])
223
  diff = newline.join([str(x) for x in zip(annotated0.split('</sentence>'), annotated1.split('</sentence>')) if x[0] != x[1]])
224
- self.values.put("FAD with audience", annotated1)
225
- self.values.put("FAD with audience, diff", diff)
 
 
 
 
 
 
 
226
 
227
  def split_document(self):
228
  audience = self.values.get("actors, generalized")[0]
229
- derivative_string = f"<document>{self.values.get('FAD with audience')}</document>"
230
- sentences = re.findall(f'.*?(?=\</sentence>|$)', self.values.get('FAD with audience'))
231
 
232
  derivative = newline.join(etree.fromstring(derivative_string).xpath(f".//sentence[contains(@audience,'{audience}')]/text()"))
233
  ret = self.query(prompts.get_prompt_recursive(
@@ -246,14 +277,14 @@ Output just your revision inside <derivative-document>...</derivative-document>.
246
 
247
  Here's the original document:
248
 
249
- <document>{self.values.get("FAD")}</document>
250
 
251
  Here's the derivative document:
252
 
253
  <derivative-document for="{audience}">{derivative}</derivative-document>
254
 
255
  """))
256
- self.values.put(f"""FAD for {self.values.get("actors, generalized")[0]}""", self.parse_document(ret))
257
 
258
 
259
  def chunk_document_for(self, audience):
@@ -268,7 +299,7 @@ Each concept or principle can be anything from a single sentence or several para
268
 
269
  Output just your revision inside <revised-document>...</revised-document>.
270
 
271
- <document>{self.values.get(f"FAD")}</document>
272
  """))
273
  titles = list(re.finditer("(CONCEPT:)|(PRINCIPLE:)", ret))
274
  for i, t in enumerate(titles):
@@ -287,35 +318,35 @@ Output just your revision inside <revised-document>...</revised-document>.
287
 
288
  Below, there are a few excerpts from this document. Your goal is to go through the document and erase parts of it that is duplicated in these excertps. In other words, please do a subtraction: new document = document - excerpts. Note that the excertps may not literally match the text of the document; use your judgement to decide what parts of the document are redundant, given the excertps, and erase them.
289
 
290
- <document>{self.values.get("FAD")}</document>
291
 
292
  {concepts}
293
  {principles}
294
  """))
295
- self.values.put(f"""FAD minus concepts, principles""", self.parse_document(ret))
296
 
297
 
298
 
299
 
300
-
301
-
302
  def ingest(self, filename):
303
  soup = parse.parse(filename)
304
  self.values.put("document (soup)", soup)
305
- self.values.put("FAD", soup.get_text())
306
 
307
  def understand(self):
308
  # fact finders:
309
- if self.values.is_newer("FAD", "document type"):
310
- self.query_doctype()
311
- if self.values.is_newer("FAD", "summary"):
312
  self.query_summary()
313
- if self.values.is_newer("FAD", "concepts"):
314
- self.query_concepts()
315
  self.generalize_actors()
316
- self.annotate_actors()
317
- self.split_document()
318
- self.chunk_document_for(self.values.get('actors, generalized')[0])
 
 
319
 
320
  def get_understanding(self):
321
  self.understand()
@@ -326,12 +357,12 @@ Below, there are a few excerpts from this document. Your goal is to go through t
326
  "=== Actors:\n"+newline.join(["- " + x for x in self.values.get("actors")]),
327
  self.values.get("actors jackard distance"),
328
  "=== Actors, generalized:\n"+newline.join(["- " + x for x in self.values.get("actors, generalized")]),
329
- "=== Document annotated:\n"+self.values.get("FAD with audience"),
330
- "=== Document annotated, diff of 2 attempts:\n"+self.values.get("FAD with audience, diff"),
331
- f"=== Document rewritten for {self.values.get('actors, generalized')[0]}:\n"+self.values.get(f"""FAD for {self.values.get("actors, generalized")[0]}"""),
332
  f"=== Extracted concepts:\n"+"".join(self.values.get(f"""Concepts""")),
333
  f"=== Extracted principles:\n"+"".join(self.values.get(f"""Principles""")),
334
- f"=== Document remainder:\n"+self.values.get(f"""FAD minus concepts, principles"""),
335
  )
336
 
337
 
 
1
  import re
2
  from lxml import etree
3
+ import nltk.tokenize
4
  import nltk
5
  nltk.download('punkt')
6
 
7
  import prompts
8
  import parse
9
  import llm
10
+ import fad
11
  from versioneddict import VersionedDict
12
 
13
  newline = "\n"
 
27
  def __init__(self, llm) -> None:
28
  self.LLM = llm
29
  self.values = VersionedDict()
30
+ self.fad = fad.Fad()
31
 
32
  def query(self, s):
33
  try:
 
45
  return [self.query(s) for i in range(n)]
46
 
47
 
48
+ def query_document_type(self):
49
  self.values.put("document type", self.query(prompts.get_prompt_recursive(
50
  f"""#fulldocument-intro
51
 
52
  #q-document-type
53
 
54
+ <document>{self.values.get("document")}</document>""")))
55
 
56
 
57
  def query_summary(self):
 
62
 
63
  #q-document-summary
64
 
65
+ <document>{self.values.get("document")}</document>""")))
66
 
67
 
68
  def parse_lists(self, ret, what):
 
91
  return retlist
92
 
93
 
94
+ def extract_document_from_response(self, ret):
95
  r = re.sub(r'^<derivative-document>(.*)</derivative-document>$', r'\1', ret, flags=re.DOTALL)
96
  r = re.sub(r'^<revised-document>(.*)</revised-document>$', r'\1', r, flags=re.DOTALL)
97
  r = re.sub(r'^<document>(.*)</document>$', r'\1', r, flags=re.DOTALL)
98
  return r
99
 
100
+ def query_concepts_and_actors(self):
101
  two_attempts = [{"actors": [], "concepts": []}, {"actors": [], "concepts": []}]
102
 
103
  ret = self.query_n(2, prompts.get_prompt_recursive(
 
111
 
112
  Here's the document:
113
 
114
+ <document>{self.values.get("document")}</document>"""))
115
 
116
  two_attempts[0]["concepts"] = self.parse_lists(ret[0], "concepts")
117
  two_attempts[1]["concepts"] = self.parse_lists(ret[1], "concepts")
 
157
 
158
  Here's the document:
159
 
160
+ <document>{self.values.get("document")}</document>"""))
161
  reconciled[what['plural']] = self.parse_lists(ret, what['plural'])
162
  print(f"===Reconciled {what['plural']}: {reconciled[what['plural']]}")
163
  else:
 
169
  self.values.put("concepts jackard distance", jackard["concepts"])
170
  self.values.put("actors jackard distance", jackard["actors"])
171
 
172
+ # also store concepts in the fad marks
173
+ self.fad.add_marks_from_list(reconciled["concepts"], "concept")
174
+
175
  print(f"===Reconciled: Concepts: {self.values.get('concepts')}, Actors: {self.values.get('actors')}")
176
 
177
 
 
191
 
192
  Output the line "ACTORS:" followed by your list of actors, one per line.
193
 
194
+ <document>{self.values.get("document")}</document>"""))
195
  generalized = self.parse_lists(ret, "actors")
196
  self.values.put("actors, generalized", generalized)
197
 
198
+ # also store concepts in the fad marks
199
+ self.fad.add_marks_from_list(generalized, "actor")
200
+
201
+
202
+ def split_sentences(self):
203
+ newfad = self.values.get("document")
204
+ plaintext = fad.get_plaintext(newfad)
205
+ sents = nltk.tokenize.sent_tokenize(plaintext)
206
+ if "".join(sents) != plaintext:
207
+ print("ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source")
208
+ i = 0
209
+ for sent in sents:
210
+ newfad = fad.envelope_marks(newfad, "sentence", i, i+len(sent) - 1)
211
+ self.values.put("document", newfad)
212
+ print(f"=== Sentence-split FAD : {newfad}")
213
+ # todo: pass this through LLM to verify and correct sentence splittings
214
+
215
+
216
+
217
 
218
+
219
+ def annotate_sentences_with_actors(self):
220
+ self.split_sentences()
221
+ markup = fad.get_plaintext_with_markup(self.values.get("document"), "sentence")
222
 
223
  print(f"=== Sentence markup: {markup}")
224
 
 
229
 
230
  {self.values.get("summary")}
231
 
232
+ A reader of this document has compiled a list of actors (parties that do something) mentioned or implied in the document. Here is their list:
233
 
234
  {newline.join(["- " + x for x in self.values.get("actors, generalized")])}
235
 
236
+ Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended actor(s) by picking one or more parties from this list. For each sentence, ask yourself: which of the parties from the list are the subjects? Who does the action described here? If there's no actor, leave the sentence unmarked.
237
 
238
+ Add the relevant partes, separated by commas, inside the "actors" attribute of each <sentence> tag. For example, suppose you have a document describing the game of Sudoku, and your parties are "Players" and "Creators". Here are example sentences with audience markup:
239
 
240
  <example>
241
+ <sentence>Sudoku is a popular logic puzzle with numbers with simple rules that even beginners can easily master.</sentence> <sentence actors="Creators">You should not create a puzzle with more than one possible solution, as that would usually be considered defective.</sentence> <sentence actors="Players">If you have completed finding all positions for a number, draw an X over that number in the list outside.</sentence>
242
  </example>
243
 
244
+ Output just the document adding the actors attributes for sentences, but otherwise unchanged:
245
 
246
  <document>{markup}</document>"""))
247
+ annotated0, annotated1 = self.extract_document_from_response(ret[0]), self.extract_document_from_response(ret[1])
248
  diff = newline.join([str(x) for x in zip(annotated0.split('</sentence>'), annotated1.split('</sentence>')) if x[0] != x[1]])
249
+ self.values.put("document with actors", annotated1)
250
+ self.values.put("document with actors, diff", diff)
251
+
252
+ #todo: reconcile two lists
253
+
254
+ sents = re.split('</sentence>', annotated0)
255
+ for i, sent in enumerate(sents):
256
+ match = re.search('<sentence[^>]+actors="([^"]+)"')
257
+
258
 
259
  def split_document(self):
260
  audience = self.values.get("actors, generalized")[0]
261
+ derivative_string = f"<document>{self.values.get('document with audience')}</document>"
 
262
 
263
  derivative = newline.join(etree.fromstring(derivative_string).xpath(f".//sentence[contains(@audience,'{audience}')]/text()"))
264
  ret = self.query(prompts.get_prompt_recursive(
 
277
 
278
  Here's the original document:
279
 
280
+ <document>{self.values.get("document")}</document>
281
 
282
  Here's the derivative document:
283
 
284
  <derivative-document for="{audience}">{derivative}</derivative-document>
285
 
286
  """))
287
+ self.values.put(f"""document for {self.values.get("actors, generalized")[0]}""", self.extract_document_from_response(ret))
288
 
289
 
290
  def chunk_document_for(self, audience):
 
299
 
300
  Output just your revision inside <revised-document>...</revised-document>.
301
 
302
+ <document>{self.values.get(f"document")}</document>
303
  """))
304
  titles = list(re.finditer("(CONCEPT:)|(PRINCIPLE:)", ret))
305
  for i, t in enumerate(titles):
 
318
 
319
  Below, there are a few excerpts from this document. Your goal is to go through the document and erase parts of it that is duplicated in these excertps. In other words, please do a subtraction: new document = document - excerpts. Note that the excertps may not literally match the text of the document; use your judgement to decide what parts of the document are redundant, given the excertps, and erase them.
320
 
321
+ <document>{self.values.get("document")}</document>
322
 
323
  {concepts}
324
  {principles}
325
  """))
326
+ self.values.put(f"""document minus concepts, principles""", self.extract_document_from_response(ret))
327
 
328
 
329
 
330
 
 
 
331
  def ingest(self, filename):
332
  soup = parse.parse(filename)
333
  self.values.put("document (soup)", soup)
334
+ self.values.put("document", soup.get_text())
335
 
336
  def understand(self):
337
  # fact finders:
338
+ if self.values.is_newer("document", "document type"):
339
+ self.query_document_type()
340
+ if self.values.is_newer("document", "summary"):
341
  self.query_summary()
342
+ if self.values.is_newer("document", "concepts"):
343
+ self.query_concepts_and_actors()
344
  self.generalize_actors()
345
+ if self.values.is_newer("document", "sentences"):
346
+ self.split_sentences()
347
+ self.annotate_sentences_with_actors()
348
+ self.split_document()
349
+ self.chunk_document_for(self.values.get('actors, generalized')[0])
350
 
351
  def get_understanding(self):
352
  self.understand()
 
357
  "=== Actors:\n"+newline.join(["- " + x for x in self.values.get("actors")]),
358
  self.values.get("actors jackard distance"),
359
  "=== Actors, generalized:\n"+newline.join(["- " + x for x in self.values.get("actors, generalized")]),
360
+ "=== Document annotated:\n"+self.values.get("document with audience"),
361
+ "=== Document annotated, diff of 2 attempts:\n"+self.values.get("document with audience, diff"),
362
+ f"=== Document rewritten for {self.values.get('actors, generalized')[0]}:\n"+self.values.get(f"""document for {self.values.get("actors, generalized")[0]}"""),
363
  f"=== Extracted concepts:\n"+"".join(self.values.get(f"""Concepts""")),
364
  f"=== Extracted principles:\n"+"".join(self.values.get(f"""Principles""")),
365
+ f"=== Document remainder:\n"+self.values.get(f"""document minus concepts, principles"""),
366
  )
367
 
368