Dmitry Kirsanov commited on
Commit
43e002c
·
1 Parent(s): 4d5bdd2

locally debuggable, lots of development

Browse files
Files changed (5) hide show
  1. fad.py +33 -16
  2. llm.py +55 -7
  3. llm_parsing.py +1 -1
  4. prompts.py +84 -9
  5. understand.py +216 -44
fad.py CHANGED
@@ -3,8 +3,10 @@ import bidict
3
  import inspect
4
  from note import Note
5
 
 
 
6
  notetypes = bidict.bidict({
7
- "document": 0,
8
 
9
  # generic
10
  "span": 1,
@@ -25,6 +27,8 @@ notetypes = bidict.bidict({
25
  "topic": 12,
26
  "section": 13,
27
 
 
 
28
  })
29
 
30
  chunktypes = bidict.bidict({
@@ -35,7 +39,9 @@ chunktypes = bidict.bidict({
35
  "reference": 5
36
  })
37
 
38
- supplementals = ["what-this-is","summary"]
 
 
39
 
40
  idlength = 8
41
 
@@ -91,15 +97,22 @@ class Fad:
91
  values.append(note.get())
92
  return values
93
 
94
- def get_value_by_type(self, valuetype): # if we know there's only one such note
95
  for id, note in self.notes.items():
96
  if note.typeid == notetypes[valuetype]:
97
  return note.get()
98
 
99
- def get(self, src="document"):
 
 
 
 
 
 
 
100
  return self.get_value_by_type(src)
101
 
102
- def get_note(self, src="document"):
103
  return self.get_note_by_type(src)
104
 
105
 
@@ -137,11 +150,11 @@ class Fad:
137
  break
138
  return "".join(texts)
139
 
140
- def get_plaintext(self, src="document"):
141
  fad = self.get_value_by_type(src)
142
  return self.__plaintext(fad)
143
 
144
- def get_plaintext_with_markup(self, notetype, src="document", show_id=False, show_properties=True, properties_to_show=[]): # [] means show all
145
  fad = self.get_value_by_type(src)
146
  i = 0
147
  plaintext = ''
@@ -160,7 +173,7 @@ class Fad:
160
  return plaintext
161
  i = newi
162
 
163
- def get_spans_by_note_type(self, notetype, src="document"):
164
  fad = self.get_value_by_type(src)
165
  i = 0
166
  spans = []
@@ -175,7 +188,7 @@ class Fad:
175
  i = newi
176
  return spans
177
 
178
- def get_spans_by_property_values(self, notetype, propertyname, propertyvalues, src="document"):
179
  fad = self.get_value_by_type(src)
180
  i = 0
181
  spans = []
@@ -190,7 +203,7 @@ class Fad:
190
  i = newi
191
  return spans
192
 
193
- def get_spans_by_property_value(self, notetype, proprtyname, propertyvalue, src="document"): # when we only need to check one value
194
  return self.get_spans_by_property_values(notetype, proprtyname, [propertyvalue], src)
195
 
196
  def is_newer(self, src, tgt):
@@ -227,7 +240,7 @@ class Fad:
227
  else:
228
  i = newi
229
 
230
- def add_span_mark(self, notetype, start, end, src="document") -> Note:
231
  fad = self.get_value_by_type(src)
232
  start_i = self.__goto_plaintext_i(fad, start)
233
  end_i = self.__goto_plaintext_i(fad, end)
@@ -238,7 +251,7 @@ class Fad:
238
  self.set(src, newfad)
239
  return newnote
240
 
241
- def add_point_mark(self, notetype, start, src="document") -> Note:
242
  fad = self.get_value_by_type(src)
243
  start_i = self.__goto_plaintext_i(fad, start)
244
  if start_i == None:
@@ -248,7 +261,7 @@ class Fad:
248
  self.set(src, newfad)
249
  return newnote
250
 
251
- def get_note_for_mark_n(self, notetype, mark_n, startfrom=0, src="document") -> Note:
252
  fad = self.get_value_by_type(src)
253
  i = startfrom
254
  n = 0
@@ -264,7 +277,7 @@ class Fad:
264
  i = newi
265
 
266
 
267
- def set_property_to_mark(self, notetype, mark_n, property, value, valuetype, startfrom=0, src="document"):
268
  note = self.get_note_for_mark_n(notetype, mark_n, startfrom, src)
269
  if valuetype != None: # we're setting value to ref to a note whose type is valuetype
270
  valuenote = self.get_by_value(value, valuetype)
@@ -272,7 +285,7 @@ class Fad:
272
  else: # we're setting leaf string value to property
273
  note.set_property(property, value)
274
 
275
- def add_property_to_mark(self, notetype, mark_n, property, value, valuetype, startfrom=0, src="document"):
276
  note = self.get_note_for_mark_n(notetype, mark_n, startfrom, src)
277
  if valuetype != None: # we're setting value to ref to a note whose type is valuetype
278
  valuenote = self.get_by_value(value, valuetype)
@@ -290,7 +303,7 @@ if __name__ == "__main__":
290
 
291
  f = Fad()
292
  orig = "A123456789B123456789C123456789"
293
- f.add("document", orig)
294
 
295
  f.add_point_mark("point", 1)
296
  f.add_point_mark("point", 1)
@@ -322,6 +335,10 @@ if __name__ == "__main__":
322
  f.add_from_list(actors_old, "actor")
323
  f.add_from_list(audiences, "audience")
324
 
 
 
 
 
325
  assert(actors_old == f.get_values_by_type("actor"))
326
  assert(audiences == f.get_values_by_type("audience"))
327
 
 
3
  import inspect
4
  from note import Note
5
 
6
+ defaultnote="document"
7
+
8
  notetypes = bidict.bidict({
9
+ defaultnote: 0,
10
 
11
  # generic
12
  "span": 1,
 
27
  "topic": 12,
28
  "section": 13,
29
 
30
+ "list": 14
31
+
32
  })
33
 
34
  chunktypes = bidict.bidict({
 
39
  "reference": 5
40
  })
41
 
42
+ summaries = ["what-this-is","summary"]
43
+
44
+ plurals = {'actor': 'actors', 'concept': 'concepts', 'audience': 'audiences'}
45
 
46
  idlength = 8
47
 
 
97
  values.append(note.get())
98
  return values
99
 
100
+ def get_value_by_type(self, valuetype): # if we know there's only one such note, or we only want to check if there's any
101
  for id, note in self.notes.items():
102
  if note.typeid == notetypes[valuetype]:
103
  return note.get()
104
 
105
+ def get_values_by_type_string(self, valuetype):
106
+ values = self.get_values_by_type(valuetype)
107
+ if len(values) > 1:
108
+ return "\n".join(['- '+v for v in values])
109
+ else:
110
+ return values[0]
111
+
112
+ def get(self, src=defaultnote):
113
  return self.get_value_by_type(src)
114
 
115
+ def get_note(self, src=defaultnote):
116
  return self.get_note_by_type(src)
117
 
118
 
 
150
  break
151
  return "".join(texts)
152
 
153
+ def get_plaintext(self, src=defaultnote):
154
  fad = self.get_value_by_type(src)
155
  return self.__plaintext(fad)
156
 
157
+ def get_plaintext_with_markup(self, notetype, src=defaultnote, show_id=False, show_properties=True, properties_to_show=[]): # [] means show all
158
  fad = self.get_value_by_type(src)
159
  i = 0
160
  plaintext = ''
 
173
  return plaintext
174
  i = newi
175
 
176
+ def get_spans_by_note_type(self, notetype, src=defaultnote):
177
  fad = self.get_value_by_type(src)
178
  i = 0
179
  spans = []
 
188
  i = newi
189
  return spans
190
 
191
+ def get_spans_by_property_values(self, notetype, propertyname, propertyvalues, src=defaultnote):
192
  fad = self.get_value_by_type(src)
193
  i = 0
194
  spans = []
 
203
  i = newi
204
  return spans
205
 
206
+ def get_spans_by_property_value(self, notetype, proprtyname, propertyvalue, src=defaultnote): # when we only need to check one value
207
  return self.get_spans_by_property_values(notetype, proprtyname, [propertyvalue], src)
208
 
209
  def is_newer(self, src, tgt):
 
240
  else:
241
  i = newi
242
 
243
+ def add_span_mark(self, notetype, start, end, src=defaultnote) -> Note:
244
  fad = self.get_value_by_type(src)
245
  start_i = self.__goto_plaintext_i(fad, start)
246
  end_i = self.__goto_plaintext_i(fad, end)
 
251
  self.set(src, newfad)
252
  return newnote
253
 
254
+ def add_point_mark(self, notetype, start, src=defaultnote) -> Note:
255
  fad = self.get_value_by_type(src)
256
  start_i = self.__goto_plaintext_i(fad, start)
257
  if start_i == None:
 
261
  self.set(src, newfad)
262
  return newnote
263
 
264
+ def get_note_for_mark_n(self, notetype, mark_n, startfrom=0, src=defaultnote) -> Note:
265
  fad = self.get_value_by_type(src)
266
  i = startfrom
267
  n = 0
 
277
  i = newi
278
 
279
 
280
+ def set_property_to_mark(self, notetype, mark_n, property, value, valuetype, startfrom=0, src=defaultnote):
281
  note = self.get_note_for_mark_n(notetype, mark_n, startfrom, src)
282
  if valuetype != None: # we're setting value to ref to a note whose type is valuetype
283
  valuenote = self.get_by_value(value, valuetype)
 
285
  else: # we're setting leaf string value to property
286
  note.set_property(property, value)
287
 
288
+ def add_property_to_mark(self, notetype, mark_n, property, value, valuetype, startfrom=0, src=defaultnote):
289
  note = self.get_note_for_mark_n(notetype, mark_n, startfrom, src)
290
  if valuetype != None: # we're setting value to ref to a note whose type is valuetype
291
  valuenote = self.get_by_value(value, valuetype)
 
303
 
304
  f = Fad()
305
  orig = "A123456789B123456789C123456789"
306
+ f.add(defaultnote, orig)
307
 
308
  f.add_point_mark("point", 1)
309
  f.add_point_mark("point", 1)
 
335
  f.add_from_list(actors_old, "actor")
336
  f.add_from_list(audiences, "audience")
337
 
338
+ assert(f.get_values_by_type_string("audience") == """- audience1
339
+ - audience2
340
+ - audience3""")
341
+
342
  assert(actors_old == f.get_values_by_type("actor"))
343
  assert(audiences == f.get_values_by_type("audience"))
344
 
llm.py CHANGED
@@ -1,10 +1,29 @@
1
- import os
2
  import requests
3
- import anthropic
4
- import google.generativeai as palm
5
- import openai
6
- import tiktoken
7
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  import logger
9
 
10
  class LLM:
@@ -21,7 +40,7 @@ class LLM:
21
 
22
  def get_models(self):
23
  raise NotImplementedError("Subclass must implement get_models method")
24
-
25
  class Anthropic(LLM):
26
  def __init__(self):
27
  super().__init__(context_length=200000, api_endpoint="https://api.anthropic.com", max_output_tokens=4096)
@@ -101,3 +120,32 @@ class OpenAI(LLM):
101
  def set_model(self, model):
102
  self.modelname = model
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, re
2
  import requests
3
+ try:
4
+ import anthropic
5
+ except ModuleNotFoundError:
6
+ # Error handling
7
+ print("No module anthropic")
8
+ pass
9
+ try:
10
+ import google.generativeai as palm
11
+ except ModuleNotFoundError:
12
+ # Error handling
13
+ print("No module google")
14
+ pass
15
+ try:
16
+ import openai
17
+ except ModuleNotFoundError:
18
+ # Error handling
19
+ print("No module openai")
20
+ pass
21
+ try:
22
+ import tiktoken
23
+ except ModuleNotFoundError:
24
+ # Error handling
25
+ print("No module tiktoken")
26
+ pass
27
  import logger
28
 
29
  class LLM:
 
40
 
41
  def get_models(self):
42
  raise NotImplementedError("Subclass must implement get_models method")
43
+
44
  class Anthropic(LLM):
45
  def __init__(self):
46
  super().__init__(context_length=200000, api_endpoint="https://api.anthropic.com", max_output_tokens=4096)
 
120
  def set_model(self, model):
121
  self.modelname = model
122
 
123
+
124
+
125
+
126
+ class Debug(LLM):
127
+ def __init__(self):
128
+ pass
129
+
130
+ def send_question(self, question):
131
+ list_regex = re.compile('Output the line "?([a-zA-Z]+):"? followed by a list')
132
+ match = re.search(list_regex, question)
133
+ if match:
134
+ title = match.group(1)
135
+ return f"""{title}:
136
+ - item1
137
+ - item2
138
+ - item3
139
+ """, False
140
+
141
+ if "annotate each sentence" in question and '"actors" attribute' in question:
142
+ doc = re.search(r'Here is the .*<document>(.+)</document>', question, flags=re.DOTALL).group(1)
143
+ doc = doc.replace('<sentence', '<sentence actors="item1, item2"')
144
+ return f'<document>{doc}</document>', False
145
+
146
+ if "break this document into sections" in question:
147
+ doc = re.search(r'Here is the .*<document>(.+)</document>', question, flags=re.DOTALL).group(1)
148
+ s = len(doc) // 2
149
+ return f'<document><section>{doc[:s]}</section><section>{doc[s:]}</section></document>', False
150
+
151
+ return("Mock answer from Debug LLM"), False
llm_parsing.py CHANGED
@@ -1,6 +1,6 @@
1
  import re
2
 
3
- def parse_lists(ret, what):
4
 
5
  lines = ret.splitlines()
6
  inlist = False
 
1
  import re
2
 
3
+ def parse_list(ret, what):
4
 
5
  lines = ret.splitlines()
6
  inlist = False
prompts.py CHANGED
@@ -1,6 +1,9 @@
1
  import re
2
 
3
  prompts = {
 
 
 
4
  "intro-structured-writing":
5
  "To write clearly, it is important to follow certain standards. You are a helpful assistant who can answer questions and make suggestions about the best ways to convey complex concepts in writing.",
6
 
@@ -142,6 +145,8 @@ Use task information to answer:
142
 
143
 
144
 
 
 
145
 
146
  "q-chunk-information-type":
147
  """Please determine the most likely content type of the chunk of content below. First, give one paragraph of your reasoning. Then, give a list of the information types applicable, one per line, in the order of most likely to least likely, so the first line contains the most likely information type.
@@ -160,38 +165,108 @@ CONTENT CHUNK:
160
 
161
 
162
 
163
- "q-document-type": """Give a short noun phrase (without article) describing what this document is. Examples: "Textbook about interpersonal psychology", "State standard on oil rig equipment", "User manual for a smartphone app".
164
 
165
- Write just one line with the description.""",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
166
 
 
167
 
168
- "fulldocument-intro": """Below is a document inside <document></document>. Please read this document carefully. Remember everything you know about the concepts it touches.""",
169
 
 
170
 
171
- "q-document-summary": """Give a one-paragraph summary of the document's content. Write just one paragraph with the summary.""",
 
 
 
 
 
 
 
 
 
 
 
 
172
 
173
 
174
- "q-concepts-actors": """Now, read the document again, and compile a list of the main concepts that this document talks about. Output the line "CONCEPTS:" followed by a list of concepts, one per line.
175
 
176
- Then, read it one more time, and think about who are the actors implied in the document. Who are the parties that this document is authored by, is governed by, is addressed to? Often, these parties are not explicitly stated in the document, so you will need to use your common sense. See if you can compile a minimal list of the main actors that the document is concerned with. Output the line "ACTORS:" followed by a list of actors, one per line.
177
 
178
- For example, for an instruction manual for a chainsaw, you might want to output:
179
  CONCEPTS:
180
  - Power tool
181
  - Safety
182
  - Work area
183
  - Saw chain
184
  - Wood
185
- ...
 
 
 
 
 
 
186
  ACTORS:
187
  - Chainsaw manufacturer
188
  - Chainsaw user
189
  - Children and bystanders
190
- ...
191
  """,
192
 
 
 
 
 
 
 
 
 
 
193
 
194
 
 
 
 
 
 
 
 
 
 
 
195
 
196
 
197
 
 
1
  import re
2
 
3
  prompts = {
4
+
5
+ #### INFORMATION TYPES
6
+
7
  "intro-structured-writing":
8
  "To write clearly, it is important to follow certain standards. You are a helpful assistant who can answer questions and make suggestions about the best ways to convey complex concepts in writing.",
9
 
 
145
 
146
 
147
 
148
+ #### CHUNKS
149
+
150
 
151
  "q-chunk-information-type":
152
  """Please determine the most likely content type of the chunk of content below. First, give one paragraph of your reasoning. Then, give a list of the information types applicable, one per line, in the order of most likely to least likely, so the first line contains the most likely information type.
 
165
 
166
 
167
 
 
168
 
169
+ #### GENERAL
170
+
171
+
172
+ "document-general": """You are a copyeditor's assistant. You will be helping me to check, improve, and rewrite a document.
173
+
174
+ """,
175
+
176
+ "section-general": """You are a copyeditor's assistant. You will be helping me to check, improve, and rewrite a section in a large document.
177
+
178
+ """,
179
+
180
+ #### SIDEINFO
181
+
182
+ "document-actor": """Someone has compiled the following list of actors that are relevant to a document:
183
+
184
+ {value}
185
+
186
+ """,
187
+
188
+ #### INTRO
189
+
190
+ "document-intro": """Below, the document goes inside <document></document>. Please read the whole document carefully. Remember everything you know about the concepts it touches.
191
+
192
+ """,
193
+
194
+
195
+ #### SUMMARIES
196
+
197
+
198
+ "document-what-this-is": """This {name} is a {value}.""",
199
+
200
+ "document-summary": """Here's a summary of the {name}:
201
+
202
+ {value}.""",
203
+
204
 
205
+ #### SELECT
206
 
 
207
 
208
+ "select-sentence-all": """In the document below, all sentences are marked up with <sentence></sentence>. """,
209
 
210
+
211
+
212
+ #### Q
213
+
214
+ "q-singlevalue-what-this-is": """Give a short noun phrase (without article) describing what this document is. Examples: "Textbook about interpersonal psychology", "State standard on oil rig equipment", "User manual for a smartphone app".
215
+
216
+ Write just one line with the description.
217
+
218
+ """,
219
+
220
+ "q-singlevalue-summary": """Give a one-paragraph summary of the document's content. Write just one paragraph with the summary.
221
+
222
+ """,
223
 
224
 
225
+ "q-list-concept": """Read the document. Compile a list of the main concepts that this document talks about. Output the line "CONCEPTS:" followed by a list of concepts, one per line.
226
 
227
+ For example, for an instruction manual for a chainsaw, you could output:
228
 
 
229
  CONCEPTS:
230
  - Power tool
231
  - Safety
232
  - Work area
233
  - Saw chain
234
  - Wood
235
+
236
+ """,
237
+
238
+ "q-list-actor": """Read the document. Think about who are the actors implied in the document. Who are the parties that this document is authored by, is governed by, is addressed to? Often, these parties are not explicitly stated in the document, so you will need to use your common sense. See if you can compile a minimal list of the main actors that the document is concerned with. Output the line "ACTORS:" followed by a list of actors, one per line.
239
+
240
+ For example, for an instruction manual for a chainsaw, you could output:
241
+
242
  ACTORS:
243
  - Chainsaw manufacturer
244
  - Chainsaw user
245
  - Children and bystanders
246
+
247
  """,
248
 
249
+ "q-sentences-actor": """Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended actor or actors by picking one or more parties from the list of actors. For each sentence, ask yourself: which of the parties from the list are the subjects? Who does the action described here? If there's no actor, leave the sentence unmarked.
250
+
251
+ Add the relevant partes, separated by commas, inside the "actors" attribute of each <sentence> tag. For example, suppose you have a document describing the game of Sudoku, and your parties are "Players" and "Creators". Here are example sentences with audience markup:
252
+
253
+ <sentence>Sudoku is a popular logic puzzle with numbers with simple rules that even beginners can easily master.</sentence> <sentence actors="Creators">You should not create a puzzle with more than one possible solution, as that would usually be considered defective.</sentence> <sentence actors="Players">If you have completed finding all positions for a number, draw an X over that number in the list outside.</sentence>
254
+
255
+ Output just the document adding the actors attributes for sentences, but otherwise unchanged.
256
+
257
+ """,
258
 
259
 
260
+ "q-section-section": """Your goal is to break this document into sections. Think of how to best locate section boundaries without changing the text of the document. Mark up the text with <section></section> tags enclosing each section, otherwise keep the text unchanged.
261
+
262
+ """,
263
+
264
+ #### MATERIAL
265
+
266
+ "material": """Here is the {name}:
267
+
268
+ {value}""",
269
+
270
 
271
 
272
 
understand.py CHANGED
@@ -18,7 +18,6 @@ class Document():
18
 
19
  def __init__(self, llm) -> None:
20
  self.LLM = llm
21
- self.values = VersionedDict()
22
 
23
  # TODO: move all understandings to this object instead of self.values
24
  # for this, need to implement timestamps and is_newer
@@ -33,7 +32,7 @@ class Document():
33
  print(f"===Truncated? {truncated}")
34
  return answer.strip()
35
  except Exception as e:
36
- print (e.message, e.args)
37
  return None
38
 
39
  def query_n(self, n, s):
@@ -44,7 +43,7 @@ class Document():
44
 
45
  def query_document_type(self, src="document", dst="document type"):
46
  ret = self.query(prompts.get_prompt_recursive(
47
- f"""#fulldocument-intro
48
 
49
  #q-document-type
50
 
@@ -56,7 +55,7 @@ class Document():
56
 
57
  def query_summary(self, src="document", dst="document summary"):
58
  ret = self.query(prompts.get_prompt_recursive(
59
- f"""#fulldocument-intro
60
 
61
  This document is a {self.fad.get_values_by_type("document type")[0]}.
62
 
@@ -74,7 +73,7 @@ This document is a {self.fad.get_values_by_type("document type")[0]}.
74
  two_attempts = [{actors: [], concepts: []}, {actors: [], concepts: []}]
75
 
76
  ret = self.query_n(2, prompts.get_prompt_recursive(
77
- f"""#fulldocument-intro
78
 
79
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
80
 
@@ -86,10 +85,10 @@ Here's the document:
86
 
87
  <document>{self.fad.get_plaintext(self.values.get(src))}</document>"""))
88
 
89
- two_attempts[0][concepts] = llm_parsing.parse_lists(ret[0], concepts)
90
- two_attempts[1][concepts] = llm_parsing.parse_lists(ret[1], concepts)
91
- two_attempts[0][actors] = llm_parsing.parse_lists(ret[0], actors)
92
- two_attempts[1][actors] = llm_parsing.parse_lists(ret[1], actors)
93
 
94
  print(f"===Reconcile: {two_attempts}")
95
 
@@ -107,7 +106,7 @@ Here's the document:
107
  if set1 != set2:
108
  print(f"===Two attempts of {what['plural']} are different, jaccard similarity: {jackard[what['plural']]}")
109
  ret = self.query(prompts.get_prompt_recursive(
110
- f"""#fulldocument-intro
111
 
112
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
113
 
@@ -142,10 +141,10 @@ Here's the document:
142
  self.values.put("concepts jackard distance", jackard[concepts])
143
  self.values.put("actors jackard distance", jackard[actors])
144
 
145
- # also store concepts in the fad marks
146
- self.fad.add_notes_from_list(reconciled[concepts], "concept")
147
- self.fad.add_notes_from_list(reconciled[actors], "actor")
148
- print("store concepts in the fad marks")
149
  print(reconciled[concepts])
150
  print(reconciled[actors])
151
 
@@ -154,7 +153,7 @@ Here's the document:
154
 
155
  def generalize_actors(self, src="actors", dst="actors, generalized"):
156
  ret = self.query(prompts.get_prompt_recursive(
157
- f"""#fulldocument-intro
158
 
159
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
160
 
@@ -173,38 +172,45 @@ Output the line "ACTORS:" followed by your list of actors, one per line.
173
  self.values.put("actors, generalized", generalized)
174
 
175
  # also store concepts in the fad marks
176
- self.fad.delete_notes_by_type("actor")
177
- self.fad.add_notes_from_list(generalized, "actor")
178
 
179
 
180
- def split_sentences(self, src="document", dst="document"):
181
- newfad = self.values.get(src)
182
- plaintext = self.fad.get_plaintext(newfad)
183
  sents = nltk.tokenize.sent_tokenize(plaintext)
184
  i = 0
185
  for sent in sents:
186
  start = plaintext.find(sent, i)
187
  if start == -1:
188
  raise(Exception(f"ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source: can't find '{sent}' in '{plaintext[i:]}'"))
189
- newfad = self.fad.add_span_mark(newfad, "sentence", start, start+len(sent))
190
  i = start+len(sent)
191
- self.values.put(dst, newfad)
192
- print(f"=== Sentence-split FAD : {newfad}")
193
  # todo: pass this through LLM to verify and correct sentence splittings
194
 
195
 
196
- def annotate_sentences_in_fad(self, llmtext, property, notetype):
197
- sents = re.split('</sentence>', llmtext)
 
198
  for i, sent in enumerate(sents):
199
  print(i, sent)
200
- if '<sentence' in sent:
201
- match = re.match(f'[^<]*<sentence[^>]+{property}="([^"]+)"', sent)
202
  if match:
203
  vals = match.group(1)
204
  for val in re.split(",\s+", vals):
205
  print(f"property {property} val {val} notetype {notetype}")
206
- self.fad.add_property_to_mark(self.values.get("document"), "sentence", i + 1, property, val, notetype)
207
 
 
 
 
 
 
 
 
 
208
 
209
  def annotate_sentences_with_actors(self, src="document", dst="document with actors"):
210
  markup = self.fad.get_plaintext_with_markup(self.values.get(src), "sentence")
@@ -212,7 +218,7 @@ Output the line "ACTORS:" followed by your list of actors, one per line.
212
  print(f"=== Sentence markup: {markup}")
213
 
214
  ret = self.query_n(2, prompts.get_prompt_recursive(
215
- f"""#fulldocument-intro
216
 
217
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
218
 
@@ -240,7 +246,7 @@ Output just the document adding the actors attributes for sentences, but otherw
240
 
241
  #todo: reconcile two lists
242
 
243
- self.annotate_sentences_in_fad(annotated0, "actors", "actor")
244
 
245
 
246
 
@@ -251,7 +257,7 @@ Output just the document adding the actors attributes for sentences, but otherw
251
  print(f"=== Sentence markup: {markup}")
252
 
253
  ret = self.query_n(2, prompts.get_prompt_recursive(
254
- f"""#fulldocument-intro
255
 
256
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
257
 
@@ -279,7 +285,7 @@ Output just the document adding the "audiences" attributes for sentences, but o
279
 
280
  #todo: reconcile two lists
281
 
282
- self.annotate_sentences_in_fad(annotated0, "audiences", "actor") # not an error, we use actors but tell llm it's audiences
283
 
284
 
285
 
@@ -291,7 +297,7 @@ Output just the document adding the "audiences" attributes for sentences, but o
291
  derivative = newline.join(self.fad.get_spans_by_property_value(self.values.get(src), 'sentence', name="audiences", values=[audience]))
292
 
293
  ret = self.query(prompts.get_prompt_recursive(
294
- f"""#fulldocument-intro
295
 
296
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
297
 
@@ -317,16 +323,180 @@ Here's the derivative document:
317
 
318
 
319
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
320
 
321
 
322
  def ingest(self, filename):
323
  soup = parse.parse(filename)
324
- self.values.put("document (soup)", soup)
325
- self.values.put("document", soup.get_text())
326
 
327
  def understand(self):
328
  # fact finders:
329
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
330
  if self.values.is_newer("document", "document type"):
331
  self.query_document_type(src="document", dst="document type")
332
 
@@ -342,20 +512,22 @@ Here's the derivative document:
342
  self.annotate_sentences_with_actors( src="document", dst="document with actors")
343
  self.annotate_sentences_with_audiences(src="document", dst="document with audiences")
344
  self.extract_and_cleanup_for_audience(src='document', audience=self.fad.get_values_by_type("actor")[0], dst="document for {audience}")
 
345
 
346
  def get_understanding(self):
347
  self.understand()
348
 
349
  return (
350
- self.fad.get_values_by_type("document type")[0],
351
- self.fad.get_values_by_type("document summary")[0],
352
- "=== Concepts:\n"+newline.join(["- " + x for x in self.fad.get_values_by_type("concept")]),
353
- self.values.get("concepts jackard distance"),
354
- "=== Actors:\n"+newline.join(["- " + x for x in self.fad.get_values_by_type("actor")]),
355
- self.values.get("actors jackard distance"),
356
- "=== Document annotated:\n"+self.values.get("document with audiences"),
357
- "=== Document annotated, diff of 2 attempts:\n"+self.values.get("document with audiences, diff"),
358
- f"=== Document rewritten for {self.values.get('actors, generalized')[0]}:\n"+self.values.get(f"""document for {self.fad.get_values_by_type("actor")[0]}""")
359
  )
360
 
361
 
 
 
 
 
 
18
 
19
  def __init__(self, llm) -> None:
20
  self.LLM = llm
 
21
 
22
  # TODO: move all understandings to this object instead of self.values
23
  # for this, need to implement timestamps and is_newer
 
32
  print(f"===Truncated? {truncated}")
33
  return answer.strip()
34
  except Exception as e:
35
+ print (e)
36
  return None
37
 
38
  def query_n(self, n, s):
 
43
 
44
  def query_document_type(self, src="document", dst="document type"):
45
  ret = self.query(prompts.get_prompt_recursive(
46
+ f"""#document-intro
47
 
48
  #q-document-type
49
 
 
55
 
56
  def query_summary(self, src="document", dst="document summary"):
57
  ret = self.query(prompts.get_prompt_recursive(
58
+ f"""#document-intro
59
 
60
  This document is a {self.fad.get_values_by_type("document type")[0]}.
61
 
 
73
  two_attempts = [{actors: [], concepts: []}, {actors: [], concepts: []}]
74
 
75
  ret = self.query_n(2, prompts.get_prompt_recursive(
76
+ f"""#document-intro
77
 
78
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
79
 
 
85
 
86
  <document>{self.fad.get_plaintext(self.values.get(src))}</document>"""))
87
 
88
+ two_attempts[0][concepts] = llm_parsing.parse_list(ret[0], concepts)
89
+ two_attempts[1][concepts] = llm_parsing.parse_list(ret[1], concepts)
90
+ two_attempts[0][actors] = llm_parsing.parse_list(ret[0], actors)
91
+ two_attempts[1][actors] = llm_parsing.parse_list(ret[1], actors)
92
 
93
  print(f"===Reconcile: {two_attempts}")
94
 
 
106
  if set1 != set2:
107
  print(f"===Two attempts of {what['plural']} are different, jaccard similarity: {jackard[what['plural']]}")
108
  ret = self.query(prompts.get_prompt_recursive(
109
+ f"""#document-intro
110
 
111
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
112
 
 
141
  self.values.put("concepts jackard distance", jackard[concepts])
142
  self.values.put("actors jackard distance", jackard[actors])
143
 
144
+ # also store concepts in the fad notes
145
+ self.fad.add_from_list(reconciled[concepts], "concept")
146
+ self.fad.add_from_list(reconciled[actors], "actor")
147
+ print("store concepts in the fad notes")
148
  print(reconciled[concepts])
149
  print(reconciled[actors])
150
 
 
153
 
154
  def generalize_actors(self, src="actors", dst="actors, generalized"):
155
  ret = self.query(prompts.get_prompt_recursive(
156
+ f"""#document-intro
157
 
158
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
159
 
 
172
  self.values.put("actors, generalized", generalized)
173
 
174
  # also store concepts in the fad marks
175
+ self.fad.delete_by_type("actor")
176
+ self.fad.add_from_list(generalized, "actor")
177
 
178
 
179
+ def split_sentences(self, src="document"):
180
+ plaintext = self.fad.get_plaintext(src)
 
181
  sents = nltk.tokenize.sent_tokenize(plaintext)
182
  i = 0
183
  for sent in sents:
184
  start = plaintext.find(sent, i)
185
  if start == -1:
186
  raise(Exception(f"ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source: can't find '{sent}' in '{plaintext[i:]}'"))
187
+ self.fad.add_span_mark("sentence", start, start+len(sent), src=src)
188
  i = start+len(sent)
189
+ print(f"=== Sentence-split FAD : {self.fad.get_plaintext_with_markup('sentence', src=src)}")
 
190
  # todo: pass this through LLM to verify and correct sentence splittings
191
 
192
 
193
+ def annotate_spans_in_fad(self, llmtext, span, property, notetype):
194
+ sents = re.split(f'</{span}>', llmtext)
195
+ print(property, notetype)
196
  for i, sent in enumerate(sents):
197
  print(i, sent)
198
+ if f'<{span}' in sent:
199
+ match = re.match(f'[^<]*<{span}[^>]+{property}="([^"]+)"', sent)
200
  if match:
201
  vals = match.group(1)
202
  for val in re.split(",\s+", vals):
203
  print(f"property {property} val {val} notetype {notetype}")
204
+ self.fad.add_property_to_mark(span, i + 1, property, val, notetype)
205
 
206
+ def create_spans_in_fad(self, llmtext, span):
207
+ chunks = re.split(f'</{span}>', llmtext)
208
+ t = 0
209
+ for i, chunk in enumerate(chunks):
210
+ if f'<{span}>' in chunk:
211
+ text = chunk.replace(f'<{span}>', '')
212
+ self.fad.add_span_mark(span, t, t + len(text))
213
+ t += len(text)
214
 
215
  def annotate_sentences_with_actors(self, src="document", dst="document with actors"):
216
  markup = self.fad.get_plaintext_with_markup(self.values.get(src), "sentence")
 
218
  print(f"=== Sentence markup: {markup}")
219
 
220
  ret = self.query_n(2, prompts.get_prompt_recursive(
221
+ f"""#document-intro
222
 
223
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
224
 
 
246
 
247
  #todo: reconcile two lists
248
 
249
+ self.annotate_spans_in_fad(annotated0, "sentence", "actors", "actor")
250
 
251
 
252
 
 
257
  print(f"=== Sentence markup: {markup}")
258
 
259
  ret = self.query_n(2, prompts.get_prompt_recursive(
260
+ f"""#document-intro
261
 
262
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
263
 
 
285
 
286
  #todo: reconcile two lists
287
 
288
+ self.annotate_spans_in_fad(annotated0, "sentence", "audiences", "actor") # not an error, we use actors but tell llm it's audiences
289
 
290
 
291
 
 
297
  derivative = newline.join(self.fad.get_spans_by_property_value(self.values.get(src), 'sentence', name="audiences", values=[audience]))
298
 
299
  ret = self.query(prompts.get_prompt_recursive(
300
+ f"""#document-intro
301
 
302
  This document is a {self.fad.get_values_by_type("document type")[0]}. Here's a summary of the document:
303
 
 
323
 
324
 
325
 
326
+ def formatinput(self, src, selectors=None):
327
+ if selectors:
328
+ for selector in selectors.split(","):
329
+ what = selector.split('-')[0]
330
+ which = selector.split('-')[1]
331
+ match which:
332
+ case 'all':
333
+ material = self.fad.get_plaintext_with_markup(what, src=src, show_properties=False) # todo: add a way to select properties
334
+ case _:
335
+ # todo
336
+ pass
337
+ else:
338
+ material = self.fad.get_plaintext(src)
339
+
340
+ match src:
341
+ case "document":
342
+ return f"<document>{material}</document>"
343
+ case _:
344
+ return material
345
+
346
+ def parseoutput(self, outputformat, val, title=None, type=None):
347
+ match outputformat:
348
+ case "singlevalue":
349
+ return val
350
+ case "document" | "sentence" | "section":
351
+ return llm_parsing.extract_document_from_response(val)
352
+ case "list":
353
+ list = llm_parsing.parse_list(val, title)
354
+ return list
355
+
356
+
357
+ def derive(self,
358
+ sideinfo = None,
359
+
360
+ src = "document",
361
+ selectors = None, # can be a comma-separated list of selectors applied in turn
362
+
363
+ target = "what-this-is",
364
+
365
+ outputformat="singlevalue"
366
+ ):
367
+
368
+ if self.fad.get(src) == None or not(self.fad.is_newer(src, target)):
369
+ print(f"No {src}, or {src} already has up-to-date (newer) {target}, skip query")
370
+ return # we already have an up-to-date target
371
+
372
+ ##### constructing prompt
373
+
374
+ ## GENERAL: what is the task we're working on here
375
+
376
+ general_prompt = prompts.get_prompt(f"#{src}-general")
377
+
378
+ ## SIDEINFO: information not in the document which is needed for the current task
379
+
380
+ sideinfo_prompt = ''
381
+ if sideinfo and self.fad.get(sideinfo):
382
+ sideinfo_prompt = prompts.get_prompt(f"#{src}-{sideinfo}").format(value=self.fad.get_values_by_type_string(sideinfo))
383
+
384
+ ## INTRO: introducing the document or part we're working on
385
+
386
+ intro_prompt = prompts.get_prompt(f"{src}-intro")
387
+
388
+ ## SUMMARIES: already-extracted summary-like information bits about the document or part
389
+
390
+ summaries = []
391
+ summaries_prompt = ''
392
+ for s in fad.summaries: # a set list for now
393
+ if self.fad.get_note(src).get(s): # if this src has this summary
394
+ summaries.append(prompts.get_prompt(f'#{src}-{s}').format(name=src, value=self.fad.get_note(src).get(s)))
395
+ if summaries:
396
+ summaries_prompt = " ".join(summaries) + "\n\n"
397
+
398
+ ## SELECT: explaining which part of the document we have here
399
+
400
+ select_prompt = ''
401
+ if selectors:
402
+ for selector in selectors.split(','):
403
+ select_prompt += prompts.get_prompt(f"#select-{selector})") + " "
404
+
405
+ ## Q: question or task, defined by our target
406
+
407
+ q = f"q-{outputformat}-{target}"
408
+ q_prompt = prompts.get_prompt(q)
409
+
410
+ ## MATERIAL: document or part that we're working on; comes in the end because it is often the largest part of the prompt
411
+
412
+ material_prompt = prompts.get_prompt("#material").format(name=src, value=self.formatinput(src, selectors))
413
+
414
+ ## run the query
415
+
416
+ ret = self.query(f"""{general_prompt}{sideinfo_prompt}{intro_prompt}{summaries_prompt}{q_prompt}{material_prompt}""")
417
+
418
+ what = fad.plurals[target] if outputformat == 'list' or outputformat == 'sentence' else target
419
+ val = self.parseoutput(outputformat, ret, title=what)
420
+
421
+ match outputformat:
422
+ case 'singlevalue':
423
+ # the whole output is a single return value
424
+ self.fad.get_note(src).set_property_newnote(name=target, value=val, valuetype=target)
425
+ case 'list':
426
+ # the output is a list
427
+ self.fad.add_from_list(val, target)
428
+ case 'sentence' | 'section':
429
+ if target == outputformat:
430
+ # the output is newly created spans
431
+ self.create_spans_in_fad(val, outputformat)
432
+ else:
433
+ # the output is added attributes in XML markup passed as input
434
+ self.annotate_spans_in_fad(val, outputformat, what, target)
435
+ pass
436
+ case 'spans':
437
+ # the output is XML markup (elements only) added by LLM
438
+ pass
439
+
440
+
441
+
442
+
443
 
444
 
445
  def ingest(self, filename):
446
  soup = parse.parse(filename)
447
+ self.fad.add("document", soup.get_text())
 
448
 
449
  def understand(self):
450
  # fact finders:
451
 
452
+ self.derive(
453
+ src="document",
454
+ target="what-this-is",
455
+ outputformat="singlevalue"
456
+ )
457
+
458
+ self.derive(
459
+ src="document",
460
+ target="summary",
461
+ outputformat="singlevalue"
462
+ )
463
+
464
+ self.derive(
465
+ src="document",
466
+ target="concept",
467
+ outputformat="list"
468
+ )
469
+
470
+ self.derive(
471
+ src="document",
472
+ target="actor",
473
+ outputformat="list"
474
+ )
475
+
476
+ self.split_sentences(src="document") # not LLM!
477
+
478
+ self.derive(
479
+ sideinfo="actor",
480
+ src="document",
481
+ selectors="sentence-all",
482
+ target="actor",
483
+ outputformat="sentences"
484
+ )
485
+
486
+ print(self.fad.get_plaintext_with_markup("sentence", show_properties=True, properties_to_show="actors"))
487
+
488
+ self.derive(
489
+ src="document",
490
+ target="section",
491
+ outputformat="section"
492
+ )
493
+
494
+ print(self.fad.get_plaintext_with_markup("section"))
495
+
496
+
497
+
498
+
499
+ """
500
  if self.values.is_newer("document", "document type"):
501
  self.query_document_type(src="document", dst="document type")
502
 
 
512
  self.annotate_sentences_with_actors( src="document", dst="document with actors")
513
  self.annotate_sentences_with_audiences(src="document", dst="document with audiences")
514
  self.extract_and_cleanup_for_audience(src='document', audience=self.fad.get_values_by_type("actor")[0], dst="document for {audience}")
515
+ """
516
 
517
  def get_understanding(self):
518
  self.understand()
519
 
520
  return (
521
+ self.fad.get_value_by_type("what-this-is"),
522
+ self.fad.get_value_by_type("summary"),
523
+ "=== Concepts:\n"+self.fad.get_values_by_type_string("concept"),
524
+ "=== Actors:\n"+self.fad.get_values_by_type_string("actor"),
525
+ "=== Document annotated:\n"+self.fad.get_plaintext_with_markup("sentence", show_properties=True, properties_to_show="actors"),
526
+ f"=== Document broken into sections {self.fad.get_plaintext_with_markup('section')}"
 
 
 
527
  )
528
 
529
 
530
+ if __name__ == "__main__":
531
+ d = Document(llm.Debug())
532
+ d.ingest("samples/submission.html")
533
+ print(d.get_understanding())