Spaces:
Runtime error
Runtime error
Dmitry Kirsanov commited on
Commit ·
7b67970
1
Parent(s): 215f84e
start of fad class, fails test at this time
Browse files- fad.py +224 -0
- understand.py +73 -42
fad.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
|
| 3 |
+
class Fad:
|
| 4 |
+
|
| 5 |
+
idlength = 8
|
| 6 |
+
|
| 7 |
+
notetypes = {
|
| 8 |
+
# generic
|
| 9 |
+
"span": 1,
|
| 10 |
+
"point": 2,
|
| 11 |
+
|
| 12 |
+
"sentence": 3,
|
| 13 |
+
|
| 14 |
+
"concept": 4,
|
| 15 |
+
"actor": 5,
|
| 16 |
+
"audience": 6,
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
notes = {}
|
| 20 |
+
|
| 21 |
+
pattern_marks = re.compile("[⌦⌫⌧]")
|
| 22 |
+
|
| 23 |
+
highest_note_id = 0
|
| 24 |
+
|
| 25 |
+
class Note:
|
| 26 |
+
def __init__(self, parentfad, id, type, value=None) -> None:
|
| 27 |
+
self.id = id
|
| 28 |
+
self.type = type
|
| 29 |
+
self.parentfad = parentfad
|
| 30 |
+
self.properties = {}
|
| 31 |
+
if value is not None:
|
| 32 |
+
self.properties["value"] = value
|
| 33 |
+
|
| 34 |
+
def get(self, name=None):
|
| 35 |
+
value = self.properties[name if name is not None else "value"]
|
| 36 |
+
if isinstance(value, int): # any int value is supposed to be a ref to id
|
| 37 |
+
return self.parentfad.notes[value].get() # so we pass it to the target mark, recursively
|
| 38 |
+
elif isinstance(value, list):
|
| 39 |
+
return ", ".join([(self.parentfad.notes[v].get() if isinstance(v, int) else v) for v in value]) # do the same for each element in list
|
| 40 |
+
else:
|
| 41 |
+
return value
|
| 42 |
+
|
| 43 |
+
def set(self, name, value):
|
| 44 |
+
if isinstance(value, int): # any int value is supposed to be a ref to id
|
| 45 |
+
assert(value in self.parentfad.notes.keys()) # so we ensure this id exists
|
| 46 |
+
self.properties[name] = value
|
| 47 |
+
|
| 48 |
+
def add(self, name, value):
|
| 49 |
+
if isinstance(value, int): # any int value is supposed to be a ref to id
|
| 50 |
+
assert(value in self.parentfad.notes.keys()) # so we ensure this id exists
|
| 51 |
+
if isinstance(self.properties[name], list):
|
| 52 |
+
self.properties[name].append(value)
|
| 53 |
+
elif self.properties[name] is None:
|
| 54 |
+
self.properties[name] = [value]
|
| 55 |
+
else:
|
| 56 |
+
raise("Note: add: trying to add to list (properties) but the existing value is not a list")
|
| 57 |
+
|
| 58 |
+
def add_notes_from_list(self, values, type):
|
| 59 |
+
for value in values:
|
| 60 |
+
self.newnote(type, value)
|
| 61 |
+
|
| 62 |
+
def get_note_by_value(self, value, valuetype):
|
| 63 |
+
for id, note in self.notes.items():
|
| 64 |
+
if note.type == self.notetypes[valuetype] and note.properties["value"] == value:
|
| 65 |
+
return id
|
| 66 |
+
return None
|
| 67 |
+
|
| 68 |
+
def get_notes_by_type_and_value(self, type, value):
|
| 69 |
+
ret = []
|
| 70 |
+
for id, m in self.notes.items():
|
| 71 |
+
if self.notetypes[type] == m.type:
|
| 72 |
+
if value == m.properties["value"]:
|
| 73 |
+
ret.append(id)
|
| 74 |
+
return ret
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def get_next_mark(self, fad, start=0):
|
| 80 |
+
tstart = start
|
| 81 |
+
new = self.pattern_marks.search(fad, start)
|
| 82 |
+
if new != None:
|
| 83 |
+
tend = new.start()
|
| 84 |
+
id = int(fad[tend + 1:tend + self.idlength + 1])
|
| 85 |
+
return fad[tstart:tend], fad[tend], id, tend + self.idlength + 1
|
| 86 |
+
return fad[tstart:], -1, -1, -1 # no marks till the end
|
| 87 |
+
|
| 88 |
+
def get_plaintext(self, fad):
|
| 89 |
+
texts = []
|
| 90 |
+
i = 0
|
| 91 |
+
while True:
|
| 92 |
+
text, markchar, id, i = self.get_next_mark(fad, i)
|
| 93 |
+
texts.append(text)
|
| 94 |
+
if i == -1:
|
| 95 |
+
break
|
| 96 |
+
return "".join(texts)
|
| 97 |
+
|
| 98 |
+
def get_plaintext_with_markup(self, fad, notetype, show_id=False, show_properties=True):
|
| 99 |
+
i = 0
|
| 100 |
+
newfad = fad
|
| 101 |
+
while True:
|
| 102 |
+
text, markchar, id, newi = self.get_next_mark(newfad, i)
|
| 103 |
+
if newi == -1:
|
| 104 |
+
break
|
| 105 |
+
if self.notes[id].type == self.notetypes[notetype]:
|
| 106 |
+
id_s = f' id="{id}"' if show_id else ''
|
| 107 |
+
properties_s = ''.join([(f' {name}="{str(self.notes[id].get(name))}"' if show_properties else '') for name in self.notes[id].properties.keys()])
|
| 108 |
+
if markchar == '⌧':
|
| 109 |
+
newfad = newfad[:newi - self.idlength - 1] + f'<{notetype}{id_s}{properties_s}/>' + newfad[newi:]
|
| 110 |
+
else:
|
| 111 |
+
newfad = newfad[:newi - self.idlength - 1] + f'<{"/" if markchar == "⌫" else ""}{notetype}{id_s}{properties_s if markchar == "⌦" else ""}>' + newfad[newi:]
|
| 112 |
+
i = newi
|
| 113 |
+
return self.get_plaintext(newfad)
|
| 114 |
+
|
| 115 |
+
def newnote(self, type, value=None):
|
| 116 |
+
id = self.highest_note_id + 1
|
| 117 |
+
type_code = self.notetypes[type]
|
| 118 |
+
self.highest_note_id += 1
|
| 119 |
+
self.notes[id] = self.Note(self, id, type_code, value)
|
| 120 |
+
return id
|
| 121 |
+
|
| 122 |
+
def goto_plaintext_i(self, fad, to):
|
| 123 |
+
# fixme: suboptimal, can skip faster
|
| 124 |
+
passed = 0
|
| 125 |
+
i = 0
|
| 126 |
+
while True:
|
| 127 |
+
text, markchar, id, newi = self.get_next_mark(fad, i)
|
| 128 |
+
passed += len(text)
|
| 129 |
+
if passed >= to:
|
| 130 |
+
return i + len(text) - (passed - to)
|
| 131 |
+
elif newi == -1:
|
| 132 |
+
return -1 # could not reach to
|
| 133 |
+
else:
|
| 134 |
+
i = newi
|
| 135 |
+
|
| 136 |
+
def add_span_mark(self, fad, notetype, start, end):
|
| 137 |
+
id = self.newnote(notetype)
|
| 138 |
+
start_i = self.goto_plaintext_i(fad, start)
|
| 139 |
+
end_i = self.goto_plaintext_i(fad, end)
|
| 140 |
+
if start_i == -1 or end_i == -1:
|
| 141 |
+
raise Exception("fad: add_span_mark: start or end outside range")
|
| 142 |
+
newfad = fad[:start_i] + "⌦" + '{:08d}'.format(id) + fad[start_i:end_i] + "⌫" + '{:08d}'.format(id) + fad[end_i:]
|
| 143 |
+
return newfad
|
| 144 |
+
|
| 145 |
+
def add_point_mark(self, fad, notetype, start):
|
| 146 |
+
id = self.newnote(notetype)
|
| 147 |
+
start_i = self.goto_plaintext_i(fad, start)
|
| 148 |
+
if start_i == -1:
|
| 149 |
+
raise Exception("fad: point_mark: start outside range")
|
| 150 |
+
newfad = fad[:start_i] + "⌧" + '{:08d}'.format(id) + fad[start_i:]
|
| 151 |
+
return newfad
|
| 152 |
+
|
| 153 |
+
def get_note_for_mark_n(self, fad, notetype, mark_n, startfrom=0):
|
| 154 |
+
i = startfrom
|
| 155 |
+
n = 0
|
| 156 |
+
while True:
|
| 157 |
+
text, markchar, id, newi = self.get_next_mark(fad, i)
|
| 158 |
+
if newi == -1:
|
| 159 |
+
return None
|
| 160 |
+
if self.notes[id].type == self.notetypes[notetype]:
|
| 161 |
+
n += 1
|
| 162 |
+
if n == mark_n:
|
| 163 |
+
return id
|
| 164 |
+
i = newi
|
| 165 |
+
|
| 166 |
+
def set_property_to_mark(self, fad, notetype, mark_n, property, value, valuetype, startfrom=0):
|
| 167 |
+
noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
|
| 168 |
+
if valuetype != None:
|
| 169 |
+
id = self.get_note_by_value(value, valuetype)
|
| 170 |
+
self.notes[noteid].set(property, id)
|
| 171 |
+
else:
|
| 172 |
+
self.notes[noteid].set(property, value)
|
| 173 |
+
|
| 174 |
+
def add_property_to_mark(self, fad, notetype, mark_n, property, value, valuetype, startfrom=0):
|
| 175 |
+
noteid = self.get_note_for_mark_n(fad, notetype, mark_n, startfrom)
|
| 176 |
+
if valuetype != None:
|
| 177 |
+
id = self.get_note_by_value(value, valuetype)
|
| 178 |
+
self.marks[noteid].set(property, id)
|
| 179 |
+
else:
|
| 180 |
+
self.marks[noteid].set(property, value)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
if __name__ == "__main__":
|
| 184 |
+
# tests
|
| 185 |
+
|
| 186 |
+
f = Fad()
|
| 187 |
+
orig = "A123456789B123456789C123456789"
|
| 188 |
+
fad = orig
|
| 189 |
+
fad = f.add_point_mark(fad, "point", 1)
|
| 190 |
+
fad = f.add_point_mark(fad, "point", 1)
|
| 191 |
+
fad = f.add_point_mark(fad, "point", 0)
|
| 192 |
+
|
| 193 |
+
#print(fad)
|
| 194 |
+
assert(fad == "⌧00000003A⌧00000002⌧00000001123456789B123456789C123456789")
|
| 195 |
+
#print(f.get_plaintext(fad))
|
| 196 |
+
assert(f.get_plaintext(fad) == orig)
|
| 197 |
+
#print(f.get_plaintext_with_markup(fad, "point", show_id=True))
|
| 198 |
+
assert(f.get_plaintext_with_markup(fad, "point", show_id=True) == '<point id="3"/>A<point id="2"/><point id="1"/>123456789B123456789C123456789')
|
| 199 |
+
|
| 200 |
+
fad = f.add_span_mark(fad, "span", 0, 10)
|
| 201 |
+
fad = f.add_span_mark(fad, "span", 20, 30)
|
| 202 |
+
fad = f.add_span_mark(fad, "span", 5, 25)
|
| 203 |
+
|
| 204 |
+
#print(fad)
|
| 205 |
+
assert(fad == "⌦00000004⌧00000003A⌧00000002⌧000000011234⌦0000000656789⌫00000004B123456789⌦00000005C1234⌫0000000656789⌫00000005")
|
| 206 |
+
#print(f.get_plaintext(fad))
|
| 207 |
+
assert(f.get_plaintext(fad) == orig)
|
| 208 |
+
#print(f.get_plaintext_with_markup(fad, "span"))
|
| 209 |
+
assert(f.get_plaintext_with_markup(fad, "span", show_id=True) == '<span id="4">A1234<span id="6">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
|
| 210 |
+
#print(f.get_plaintext_with_markup(fad, "point"))
|
| 211 |
+
assert(f.get_plaintext_with_markup(fad, "point", show_id=True) == '<point id="3"/>A<point id="2"/><point id="1"/>123456789B123456789C123456789')
|
| 212 |
+
|
| 213 |
+
f.add_notes_from_list(["actor1", "actor2"], "actor")
|
| 214 |
+
f.add_notes_from_list(["audience1", "audience2", "audience3"], "audience")
|
| 215 |
+
|
| 216 |
+
f.set_property_to_mark(fad, "span", 1, "actors", "actor1", "actor")
|
| 217 |
+
|
| 218 |
+
#print(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True))
|
| 219 |
+
assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6">56789</span id="4">B123456789<span id="5">C1234</span id="6">56789</span id="5">')
|
| 220 |
+
|
| 221 |
+
f.set_property_to_mark(fad, "span", 3, "actors", "actor2", "actor")
|
| 222 |
+
|
| 223 |
+
print(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True))
|
| 224 |
+
assert(f.get_plaintext_with_markup(fad, "span", show_id=True, show_properties=True) == '<span id="4" actors="actor1">A1234<span id="6">56789</span id="4">B123456789<span id="5" actors="actor2">C1234</span id="6">56789</span id="5">')
|
understand.py
CHANGED
|
@@ -1,12 +1,13 @@
|
|
| 1 |
import re
|
| 2 |
from lxml import etree
|
| 3 |
-
|
| 4 |
import nltk
|
| 5 |
nltk.download('punkt')
|
| 6 |
|
| 7 |
import prompts
|
| 8 |
import parse
|
| 9 |
import llm
|
|
|
|
| 10 |
from versioneddict import VersionedDict
|
| 11 |
|
| 12 |
newline = "\n"
|
|
@@ -26,6 +27,7 @@ class Document():
|
|
| 26 |
def __init__(self, llm) -> None:
|
| 27 |
self.LLM = llm
|
| 28 |
self.values = VersionedDict()
|
|
|
|
| 29 |
|
| 30 |
def query(self, s):
|
| 31 |
try:
|
|
@@ -43,13 +45,13 @@ class Document():
|
|
| 43 |
return [self.query(s) for i in range(n)]
|
| 44 |
|
| 45 |
|
| 46 |
-
def
|
| 47 |
self.values.put("document type", self.query(prompts.get_prompt_recursive(
|
| 48 |
f"""#fulldocument-intro
|
| 49 |
|
| 50 |
#q-document-type
|
| 51 |
|
| 52 |
-
<document>{self.values.get("
|
| 53 |
|
| 54 |
|
| 55 |
def query_summary(self):
|
|
@@ -60,7 +62,7 @@ This document is a {self.values.get("document type")}.
|
|
| 60 |
|
| 61 |
#q-document-summary
|
| 62 |
|
| 63 |
-
<document>{self.values.get("
|
| 64 |
|
| 65 |
|
| 66 |
def parse_lists(self, ret, what):
|
|
@@ -89,13 +91,13 @@ This document is a {self.values.get("document type")}.
|
|
| 89 |
return retlist
|
| 90 |
|
| 91 |
|
| 92 |
-
def
|
| 93 |
r = re.sub(r'^<derivative-document>(.*)</derivative-document>$', r'\1', ret, flags=re.DOTALL)
|
| 94 |
r = re.sub(r'^<revised-document>(.*)</revised-document>$', r'\1', r, flags=re.DOTALL)
|
| 95 |
r = re.sub(r'^<document>(.*)</document>$', r'\1', r, flags=re.DOTALL)
|
| 96 |
return r
|
| 97 |
|
| 98 |
-
def
|
| 99 |
two_attempts = [{"actors": [], "concepts": []}, {"actors": [], "concepts": []}]
|
| 100 |
|
| 101 |
ret = self.query_n(2, prompts.get_prompt_recursive(
|
|
@@ -109,7 +111,7 @@ This document is a {self.values.get("document type")}. Here's a summary of the d
|
|
| 109 |
|
| 110 |
Here's the document:
|
| 111 |
|
| 112 |
-
<document>{self.values.get("
|
| 113 |
|
| 114 |
two_attempts[0]["concepts"] = self.parse_lists(ret[0], "concepts")
|
| 115 |
two_attempts[1]["concepts"] = self.parse_lists(ret[1], "concepts")
|
|
@@ -155,7 +157,7 @@ Output the line "{what['plural'].upper()}:" followed by your combined list of {w
|
|
| 155 |
|
| 156 |
Here's the document:
|
| 157 |
|
| 158 |
-
<document>{self.values.get("
|
| 159 |
reconciled[what['plural']] = self.parse_lists(ret, what['plural'])
|
| 160 |
print(f"===Reconciled {what['plural']}: {reconciled[what['plural']]}")
|
| 161 |
else:
|
|
@@ -167,6 +169,9 @@ Here's the document:
|
|
| 167 |
self.values.put("concepts jackard distance", jackard["concepts"])
|
| 168 |
self.values.put("actors jackard distance", jackard["actors"])
|
| 169 |
|
|
|
|
|
|
|
|
|
|
| 170 |
print(f"===Reconciled: Concepts: {self.values.get('concepts')}, Actors: {self.values.get('actors')}")
|
| 171 |
|
| 172 |
|
|
@@ -186,14 +191,34 @@ Your goal is to generalize this list, replacing groups of actors playing the sam
|
|
| 186 |
|
| 187 |
Output the line "ACTORS:" followed by your list of actors, one per line.
|
| 188 |
|
| 189 |
-
<document>{self.values.get("
|
| 190 |
generalized = self.parse_lists(ret, "actors")
|
| 191 |
self.values.put("actors, generalized", generalized)
|
| 192 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
|
|
|
| 197 |
|
| 198 |
print(f"=== Sentence markup: {markup}")
|
| 199 |
|
|
@@ -204,30 +229,36 @@ This document is a {self.values.get("document type")}. Here's a summary of the d
|
|
| 204 |
|
| 205 |
{self.values.get("summary")}
|
| 206 |
|
| 207 |
-
A reader of this document has compiled a list of actors (parties) mentioned or implied in the document. Here is their list:
|
| 208 |
|
| 209 |
{newline.join(["- " + x for x in self.values.get("actors, generalized")])}
|
| 210 |
|
| 211 |
-
Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended
|
| 212 |
|
| 213 |
-
Add the relevant partes, separated by commas, inside the "
|
| 214 |
|
| 215 |
<example>
|
| 216 |
-
<sentence
|
| 217 |
</example>
|
| 218 |
|
| 219 |
-
Output just the document
|
| 220 |
|
| 221 |
<document>{markup}</document>"""))
|
| 222 |
-
annotated0, annotated1 = self.
|
| 223 |
diff = newline.join([str(x) for x in zip(annotated0.split('</sentence>'), annotated1.split('</sentence>')) if x[0] != x[1]])
|
| 224 |
-
self.values.put("
|
| 225 |
-
self.values.put("
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
|
| 227 |
def split_document(self):
|
| 228 |
audience = self.values.get("actors, generalized")[0]
|
| 229 |
-
derivative_string = f"<document>{self.values.get('
|
| 230 |
-
sentences = re.findall(f'.*?(?=\</sentence>|$)', self.values.get('FAD with audience'))
|
| 231 |
|
| 232 |
derivative = newline.join(etree.fromstring(derivative_string).xpath(f".//sentence[contains(@audience,'{audience}')]/text()"))
|
| 233 |
ret = self.query(prompts.get_prompt_recursive(
|
|
@@ -246,14 +277,14 @@ Output just your revision inside <derivative-document>...</derivative-document>.
|
|
| 246 |
|
| 247 |
Here's the original document:
|
| 248 |
|
| 249 |
-
<document>{self.values.get("
|
| 250 |
|
| 251 |
Here's the derivative document:
|
| 252 |
|
| 253 |
<derivative-document for="{audience}">{derivative}</derivative-document>
|
| 254 |
|
| 255 |
"""))
|
| 256 |
-
self.values.put(f"""
|
| 257 |
|
| 258 |
|
| 259 |
def chunk_document_for(self, audience):
|
|
@@ -268,7 +299,7 @@ Each concept or principle can be anything from a single sentence or several para
|
|
| 268 |
|
| 269 |
Output just your revision inside <revised-document>...</revised-document>.
|
| 270 |
|
| 271 |
-
<document>{self.values.get(f"
|
| 272 |
"""))
|
| 273 |
titles = list(re.finditer("(CONCEPT:)|(PRINCIPLE:)", ret))
|
| 274 |
for i, t in enumerate(titles):
|
|
@@ -287,35 +318,35 @@ Output just your revision inside <revised-document>...</revised-document>.
|
|
| 287 |
|
| 288 |
Below, there are a few excerpts from this document. Your goal is to go through the document and erase parts of it that is duplicated in these excertps. In other words, please do a subtraction: new document = document - excerpts. Note that the excertps may not literally match the text of the document; use your judgement to decide what parts of the document are redundant, given the excertps, and erase them.
|
| 289 |
|
| 290 |
-
<document>{self.values.get("
|
| 291 |
|
| 292 |
{concepts}
|
| 293 |
{principles}
|
| 294 |
"""))
|
| 295 |
-
self.values.put(f"""
|
| 296 |
|
| 297 |
|
| 298 |
|
| 299 |
|
| 300 |
-
|
| 301 |
-
|
| 302 |
def ingest(self, filename):
|
| 303 |
soup = parse.parse(filename)
|
| 304 |
self.values.put("document (soup)", soup)
|
| 305 |
-
self.values.put("
|
| 306 |
|
| 307 |
def understand(self):
|
| 308 |
# fact finders:
|
| 309 |
-
if self.values.is_newer("
|
| 310 |
-
self.
|
| 311 |
-
if self.values.is_newer("
|
| 312 |
self.query_summary()
|
| 313 |
-
if self.values.is_newer("
|
| 314 |
-
self.
|
| 315 |
self.generalize_actors()
|
| 316 |
-
|
| 317 |
-
self.
|
| 318 |
-
self.
|
|
|
|
|
|
|
| 319 |
|
| 320 |
def get_understanding(self):
|
| 321 |
self.understand()
|
|
@@ -326,12 +357,12 @@ Below, there are a few excerpts from this document. Your goal is to go through t
|
|
| 326 |
"=== Actors:\n"+newline.join(["- " + x for x in self.values.get("actors")]),
|
| 327 |
self.values.get("actors jackard distance"),
|
| 328 |
"=== Actors, generalized:\n"+newline.join(["- " + x for x in self.values.get("actors, generalized")]),
|
| 329 |
-
"=== Document annotated:\n"+self.values.get("
|
| 330 |
-
"=== Document annotated, diff of 2 attempts:\n"+self.values.get("
|
| 331 |
-
f"=== Document rewritten for {self.values.get('actors, generalized')[0]}:\n"+self.values.get(f"""
|
| 332 |
f"=== Extracted concepts:\n"+"".join(self.values.get(f"""Concepts""")),
|
| 333 |
f"=== Extracted principles:\n"+"".join(self.values.get(f"""Principles""")),
|
| 334 |
-
f"=== Document remainder:\n"+self.values.get(f"""
|
| 335 |
)
|
| 336 |
|
| 337 |
|
|
|
|
| 1 |
import re
|
| 2 |
from lxml import etree
|
| 3 |
+
import nltk.tokenize
|
| 4 |
import nltk
|
| 5 |
nltk.download('punkt')
|
| 6 |
|
| 7 |
import prompts
|
| 8 |
import parse
|
| 9 |
import llm
|
| 10 |
+
import fad
|
| 11 |
from versioneddict import VersionedDict
|
| 12 |
|
| 13 |
newline = "\n"
|
|
|
|
| 27 |
def __init__(self, llm) -> None:
|
| 28 |
self.LLM = llm
|
| 29 |
self.values = VersionedDict()
|
| 30 |
+
self.fad = fad.Fad()
|
| 31 |
|
| 32 |
def query(self, s):
|
| 33 |
try:
|
|
|
|
| 45 |
return [self.query(s) for i in range(n)]
|
| 46 |
|
| 47 |
|
| 48 |
+
def query_document_type(self):
|
| 49 |
self.values.put("document type", self.query(prompts.get_prompt_recursive(
|
| 50 |
f"""#fulldocument-intro
|
| 51 |
|
| 52 |
#q-document-type
|
| 53 |
|
| 54 |
+
<document>{self.values.get("document")}</document>""")))
|
| 55 |
|
| 56 |
|
| 57 |
def query_summary(self):
|
|
|
|
| 62 |
|
| 63 |
#q-document-summary
|
| 64 |
|
| 65 |
+
<document>{self.values.get("document")}</document>""")))
|
| 66 |
|
| 67 |
|
| 68 |
def parse_lists(self, ret, what):
|
|
|
|
| 91 |
return retlist
|
| 92 |
|
| 93 |
|
| 94 |
+
def extract_document_from_response(self, ret):
|
| 95 |
r = re.sub(r'^<derivative-document>(.*)</derivative-document>$', r'\1', ret, flags=re.DOTALL)
|
| 96 |
r = re.sub(r'^<revised-document>(.*)</revised-document>$', r'\1', r, flags=re.DOTALL)
|
| 97 |
r = re.sub(r'^<document>(.*)</document>$', r'\1', r, flags=re.DOTALL)
|
| 98 |
return r
|
| 99 |
|
| 100 |
+
def query_concepts_and_actors(self):
|
| 101 |
two_attempts = [{"actors": [], "concepts": []}, {"actors": [], "concepts": []}]
|
| 102 |
|
| 103 |
ret = self.query_n(2, prompts.get_prompt_recursive(
|
|
|
|
| 111 |
|
| 112 |
Here's the document:
|
| 113 |
|
| 114 |
+
<document>{self.values.get("document")}</document>"""))
|
| 115 |
|
| 116 |
two_attempts[0]["concepts"] = self.parse_lists(ret[0], "concepts")
|
| 117 |
two_attempts[1]["concepts"] = self.parse_lists(ret[1], "concepts")
|
|
|
|
| 157 |
|
| 158 |
Here's the document:
|
| 159 |
|
| 160 |
+
<document>{self.values.get("document")}</document>"""))
|
| 161 |
reconciled[what['plural']] = self.parse_lists(ret, what['plural'])
|
| 162 |
print(f"===Reconciled {what['plural']}: {reconciled[what['plural']]}")
|
| 163 |
else:
|
|
|
|
| 169 |
self.values.put("concepts jackard distance", jackard["concepts"])
|
| 170 |
self.values.put("actors jackard distance", jackard["actors"])
|
| 171 |
|
| 172 |
+
# also store concepts in the fad marks
|
| 173 |
+
self.fad.add_marks_from_list(reconciled["concepts"], "concept")
|
| 174 |
+
|
| 175 |
print(f"===Reconciled: Concepts: {self.values.get('concepts')}, Actors: {self.values.get('actors')}")
|
| 176 |
|
| 177 |
|
|
|
|
| 191 |
|
| 192 |
Output the line "ACTORS:" followed by your list of actors, one per line.
|
| 193 |
|
| 194 |
+
<document>{self.values.get("document")}</document>"""))
|
| 195 |
generalized = self.parse_lists(ret, "actors")
|
| 196 |
self.values.put("actors, generalized", generalized)
|
| 197 |
|
| 198 |
+
# also store concepts in the fad marks
|
| 199 |
+
self.fad.add_marks_from_list(generalized, "actor")
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def split_sentences(self):
|
| 203 |
+
newfad = self.values.get("document")
|
| 204 |
+
plaintext = fad.get_plaintext(newfad)
|
| 205 |
+
sents = nltk.tokenize.sent_tokenize(plaintext)
|
| 206 |
+
if "".join(sents) != plaintext:
|
| 207 |
+
print("ERROR:nltk.tokenize.sent_tokenize output does not cleanly match source")
|
| 208 |
+
i = 0
|
| 209 |
+
for sent in sents:
|
| 210 |
+
newfad = fad.envelope_marks(newfad, "sentence", i, i+len(sent) - 1)
|
| 211 |
+
self.values.put("document", newfad)
|
| 212 |
+
print(f"=== Sentence-split FAD : {newfad}")
|
| 213 |
+
# todo: pass this through LLM to verify and correct sentence splittings
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
|
| 217 |
|
| 218 |
+
|
| 219 |
+
def annotate_sentences_with_actors(self):
|
| 220 |
+
self.split_sentences()
|
| 221 |
+
markup = fad.get_plaintext_with_markup(self.values.get("document"), "sentence")
|
| 222 |
|
| 223 |
print(f"=== Sentence markup: {markup}")
|
| 224 |
|
|
|
|
| 229 |
|
| 230 |
{self.values.get("summary")}
|
| 231 |
|
| 232 |
+
A reader of this document has compiled a list of actors (parties that do something) mentioned or implied in the document. Here is their list:
|
| 233 |
|
| 234 |
{newline.join(["- " + x for x in self.values.get("actors, generalized")])}
|
| 235 |
|
| 236 |
+
Your goal is to annotate each sentence of this document, inside a <sentence></sentence>, with its intended actor(s) by picking one or more parties from this list. For each sentence, ask yourself: which of the parties from the list are the subjects? Who does the action described here? If there's no actor, leave the sentence unmarked.
|
| 237 |
|
| 238 |
+
Add the relevant partes, separated by commas, inside the "actors" attribute of each <sentence> tag. For example, suppose you have a document describing the game of Sudoku, and your parties are "Players" and "Creators". Here are example sentences with audience markup:
|
| 239 |
|
| 240 |
<example>
|
| 241 |
+
<sentence>Sudoku is a popular logic puzzle with numbers with simple rules that even beginners can easily master.</sentence> <sentence actors="Creators">You should not create a puzzle with more than one possible solution, as that would usually be considered defective.</sentence> <sentence actors="Players">If you have completed finding all positions for a number, draw an X over that number in the list outside.</sentence>
|
| 242 |
</example>
|
| 243 |
|
| 244 |
+
Output just the document adding the actors attributes for sentences, but otherwise unchanged:
|
| 245 |
|
| 246 |
<document>{markup}</document>"""))
|
| 247 |
+
annotated0, annotated1 = self.extract_document_from_response(ret[0]), self.extract_document_from_response(ret[1])
|
| 248 |
diff = newline.join([str(x) for x in zip(annotated0.split('</sentence>'), annotated1.split('</sentence>')) if x[0] != x[1]])
|
| 249 |
+
self.values.put("document with actors", annotated1)
|
| 250 |
+
self.values.put("document with actors, diff", diff)
|
| 251 |
+
|
| 252 |
+
#todo: reconcile two lists
|
| 253 |
+
|
| 254 |
+
sents = re.split('</sentence>', annotated0)
|
| 255 |
+
for i, sent in enumerate(sents):
|
| 256 |
+
match = re.search('<sentence[^>]+actors="([^"]+)"')
|
| 257 |
+
|
| 258 |
|
| 259 |
def split_document(self):
|
| 260 |
audience = self.values.get("actors, generalized")[0]
|
| 261 |
+
derivative_string = f"<document>{self.values.get('document with audience')}</document>"
|
|
|
|
| 262 |
|
| 263 |
derivative = newline.join(etree.fromstring(derivative_string).xpath(f".//sentence[contains(@audience,'{audience}')]/text()"))
|
| 264 |
ret = self.query(prompts.get_prompt_recursive(
|
|
|
|
| 277 |
|
| 278 |
Here's the original document:
|
| 279 |
|
| 280 |
+
<document>{self.values.get("document")}</document>
|
| 281 |
|
| 282 |
Here's the derivative document:
|
| 283 |
|
| 284 |
<derivative-document for="{audience}">{derivative}</derivative-document>
|
| 285 |
|
| 286 |
"""))
|
| 287 |
+
self.values.put(f"""document for {self.values.get("actors, generalized")[0]}""", self.extract_document_from_response(ret))
|
| 288 |
|
| 289 |
|
| 290 |
def chunk_document_for(self, audience):
|
|
|
|
| 299 |
|
| 300 |
Output just your revision inside <revised-document>...</revised-document>.
|
| 301 |
|
| 302 |
+
<document>{self.values.get(f"document")}</document>
|
| 303 |
"""))
|
| 304 |
titles = list(re.finditer("(CONCEPT:)|(PRINCIPLE:)", ret))
|
| 305 |
for i, t in enumerate(titles):
|
|
|
|
| 318 |
|
| 319 |
Below, there are a few excerpts from this document. Your goal is to go through the document and erase parts of it that is duplicated in these excertps. In other words, please do a subtraction: new document = document - excerpts. Note that the excertps may not literally match the text of the document; use your judgement to decide what parts of the document are redundant, given the excertps, and erase them.
|
| 320 |
|
| 321 |
+
<document>{self.values.get("document")}</document>
|
| 322 |
|
| 323 |
{concepts}
|
| 324 |
{principles}
|
| 325 |
"""))
|
| 326 |
+
self.values.put(f"""document minus concepts, principles""", self.extract_document_from_response(ret))
|
| 327 |
|
| 328 |
|
| 329 |
|
| 330 |
|
|
|
|
|
|
|
| 331 |
def ingest(self, filename):
|
| 332 |
soup = parse.parse(filename)
|
| 333 |
self.values.put("document (soup)", soup)
|
| 334 |
+
self.values.put("document", soup.get_text())
|
| 335 |
|
| 336 |
def understand(self):
|
| 337 |
# fact finders:
|
| 338 |
+
if self.values.is_newer("document", "document type"):
|
| 339 |
+
self.query_document_type()
|
| 340 |
+
if self.values.is_newer("document", "summary"):
|
| 341 |
self.query_summary()
|
| 342 |
+
if self.values.is_newer("document", "concepts"):
|
| 343 |
+
self.query_concepts_and_actors()
|
| 344 |
self.generalize_actors()
|
| 345 |
+
if self.values.is_newer("document", "sentences"):
|
| 346 |
+
self.split_sentences()
|
| 347 |
+
self.annotate_sentences_with_actors()
|
| 348 |
+
self.split_document()
|
| 349 |
+
self.chunk_document_for(self.values.get('actors, generalized')[0])
|
| 350 |
|
| 351 |
def get_understanding(self):
|
| 352 |
self.understand()
|
|
|
|
| 357 |
"=== Actors:\n"+newline.join(["- " + x for x in self.values.get("actors")]),
|
| 358 |
self.values.get("actors jackard distance"),
|
| 359 |
"=== Actors, generalized:\n"+newline.join(["- " + x for x in self.values.get("actors, generalized")]),
|
| 360 |
+
"=== Document annotated:\n"+self.values.get("document with audience"),
|
| 361 |
+
"=== Document annotated, diff of 2 attempts:\n"+self.values.get("document with audience, diff"),
|
| 362 |
+
f"=== Document rewritten for {self.values.get('actors, generalized')[0]}:\n"+self.values.get(f"""document for {self.values.get("actors, generalized")[0]}"""),
|
| 363 |
f"=== Extracted concepts:\n"+"".join(self.values.get(f"""Concepts""")),
|
| 364 |
f"=== Extracted principles:\n"+"".join(self.values.get(f"""Principles""")),
|
| 365 |
+
f"=== Document remainder:\n"+self.values.get(f"""document minus concepts, principles"""),
|
| 366 |
)
|
| 367 |
|
| 368 |
|