Spaces:
Runtime error
Runtime error
Dmitry Kirsanov commited on
Commit ·
46e2995
1
Parent(s): 7c7e6bb
versioned storage
Browse files- app.py +2 -1
- understand.py +56 -29
- versioneddict.py +28 -0
app.py
CHANGED
|
@@ -13,7 +13,8 @@ LLM = llm.OpenAI()
|
|
| 13 |
def process(filename, model):
|
| 14 |
LLM.set_model(model)
|
| 15 |
doc = understand.Document(LLM)
|
| 16 |
-
|
|
|
|
| 17 |
|
| 18 |
gradio_app = gr.Interface(
|
| 19 |
fn=process,
|
|
|
|
| 13 |
def process(filename, model):
|
| 14 |
LLM.set_model(model)
|
| 15 |
doc = understand.Document(LLM)
|
| 16 |
+
doc.ingest(filename)
|
| 17 |
+
return doc.get_understanding()
|
| 18 |
|
| 19 |
gradio_app = gr.Interface(
|
| 20 |
fn=process,
|
understand.py
CHANGED
|
@@ -1,14 +1,12 @@
|
|
| 1 |
import parse
|
| 2 |
import llm
|
|
|
|
| 3 |
|
| 4 |
class Document():
|
| 5 |
|
| 6 |
def __init__(self, llm) -> None:
|
| 7 |
self.LLM = llm
|
| 8 |
-
self.
|
| 9 |
-
self.fad = None # string
|
| 10 |
-
self.doctype = None
|
| 11 |
-
self.summary = None
|
| 12 |
|
| 13 |
def query(self, s):
|
| 14 |
try:
|
|
@@ -20,27 +18,27 @@ class Document():
|
|
| 20 |
|
| 21 |
|
| 22 |
def query_doctype(self):
|
| 23 |
-
|
| 24 |
f"""Please read this document carefully. Remember everything you know about the concepts it touches.
|
| 25 |
|
| 26 |
Give a brief description of what this document is. Examples: "Textbook about interpersonal psychology", "State standard on oil rig equipment", "User manual for a smartphone app".
|
| 27 |
|
| 28 |
Write just one line with the description.
|
| 29 |
|
| 30 |
-
<document>{self.fad}</document>""")
|
| 31 |
|
| 32 |
def query_summary(self):
|
| 33 |
-
|
| 34 |
f"""Below is a document inside <document></document>. It is a {self.doctype}.
|
| 35 |
|
| 36 |
Read it carefully. Remember everything you know about the concepts it touches.
|
| 37 |
|
| 38 |
Give a one-paragraph summary of the document's content. Write just one paragraph with the summary.
|
| 39 |
|
| 40 |
-
<document>{self.fad}</document>""")
|
| 41 |
|
| 42 |
def query_concepts(self):
|
| 43 |
-
|
| 44 |
f"""Below is a document inside <document></document>. It is a {self.doctype}.
|
| 45 |
|
| 46 |
Read it carefully. Remember everything you know about the concepts it touches.
|
|
@@ -51,31 +49,60 @@ Then, read it one more time, and think about who are the actors implied. Who are
|
|
| 51 |
|
| 52 |
For example, for an instruction manual for a chainsaw, you might want to output:
|
| 53 |
CONCEPTS:
|
| 54 |
-
Power tool
|
| 55 |
-
Safety
|
| 56 |
-
Work area
|
| 57 |
-
Saw chain
|
| 58 |
-
Wood
|
| 59 |
...
|
| 60 |
ACTORS:
|
| 61 |
-
Chainsaw manufacturer
|
| 62 |
-
Chainsaw user
|
| 63 |
-
Children and bystanders
|
| 64 |
...
|
| 65 |
|
| 66 |
<document>{self.fad}</document>""")
|
| 67 |
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
# fact finders:
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
self.
|
| 79 |
-
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
|
|
|
| 1 |
import parse
|
| 2 |
import llm
|
| 3 |
+
from versioneddict import VersionedDict
|
| 4 |
|
| 5 |
class Document():
|
| 6 |
|
| 7 |
def __init__(self, llm) -> None:
|
| 8 |
self.LLM = llm
|
| 9 |
+
self.values = VersionedDict()
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
def query(self, s):
|
| 12 |
try:
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
def query_doctype(self):
|
| 21 |
+
self.values.put("document type", self.query(
|
| 22 |
f"""Please read this document carefully. Remember everything you know about the concepts it touches.
|
| 23 |
|
| 24 |
Give a brief description of what this document is. Examples: "Textbook about interpersonal psychology", "State standard on oil rig equipment", "User manual for a smartphone app".
|
| 25 |
|
| 26 |
Write just one line with the description.
|
| 27 |
|
| 28 |
+
<document>{self.fad}</document>"""))
|
| 29 |
|
| 30 |
def query_summary(self):
|
| 31 |
+
self.values.put("summary", self.query(
|
| 32 |
f"""Below is a document inside <document></document>. It is a {self.doctype}.
|
| 33 |
|
| 34 |
Read it carefully. Remember everything you know about the concepts it touches.
|
| 35 |
|
| 36 |
Give a one-paragraph summary of the document's content. Write just one paragraph with the summary.
|
| 37 |
|
| 38 |
+
<document>{self.fad}</document>"""))
|
| 39 |
|
| 40 |
def query_concepts(self):
|
| 41 |
+
ret = self.query(
|
| 42 |
f"""Below is a document inside <document></document>. It is a {self.doctype}.
|
| 43 |
|
| 44 |
Read it carefully. Remember everything you know about the concepts it touches.
|
|
|
|
| 49 |
|
| 50 |
For example, for an instruction manual for a chainsaw, you might want to output:
|
| 51 |
CONCEPTS:
|
| 52 |
+
- Power tool
|
| 53 |
+
- Safety
|
| 54 |
+
- Work area
|
| 55 |
+
- Saw chain
|
| 56 |
+
- Wood
|
| 57 |
...
|
| 58 |
ACTORS:
|
| 59 |
+
- Chainsaw manufacturer
|
| 60 |
+
- Chainsaw user
|
| 61 |
+
- Children and bystanders
|
| 62 |
...
|
| 63 |
|
| 64 |
<document>{self.fad}</document>""")
|
| 65 |
|
| 66 |
+
self.values.put("concepts", [])
|
| 67 |
+
self.values.put("actors", [])
|
| 68 |
+
|
| 69 |
+
lines = ret.splitlines()
|
| 70 |
+
current_list = None
|
| 71 |
+
|
| 72 |
+
for line in lines:
|
| 73 |
+
line = line.strip()
|
| 74 |
+
if line.trim() == "CONCEPTS:":
|
| 75 |
+
current_list = "concepts"
|
| 76 |
+
elif line.trim() == "ACTORS:":
|
| 77 |
+
current_list = "actors"
|
| 78 |
+
elif current_list is not None:
|
| 79 |
+
if line.startswith("- "):
|
| 80 |
+
item = line[2:].strip()
|
| 81 |
+
else:
|
| 82 |
+
item = line.strip()
|
| 83 |
+
self.values.get(current_list).append(item)
|
| 84 |
+
self.values.touch(current_list)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def ingest(self, filename):
|
| 88 |
+
self.values.put("document (soup)", parse.parse(filename))
|
| 89 |
+
self.values.put("FAD", self.documentSoup.get_text())
|
| 90 |
+
|
| 91 |
+
def understand(self):
|
| 92 |
# fact finders:
|
| 93 |
+
if self.values.is_newer("FAD", "document type"):
|
| 94 |
+
self.query_doctype()
|
| 95 |
+
if self.values.is_newer("FAD", "summary"):
|
| 96 |
+
self.query_summary()
|
| 97 |
+
if self.values.is_newer("FAD", "concepts"):
|
| 98 |
+
self.query_concepts()
|
| 99 |
+
|
| 100 |
+
def get_understanding(self):
|
| 101 |
+
self.understand()
|
| 102 |
+
|
| 103 |
+
return (self.values.get("document type"), self.values.get("summary"),
|
| 104 |
+
"=== Concepts:\n"+"\n".join(["- " + x for x in self.values.get("concepts")]),
|
| 105 |
+
"=== Actors:\n"+"\n".join(["- " + x for x in self.values.get("actors")]),
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
|
versioneddict.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import time
|
| 2 |
+
|
| 3 |
+
class VersionedDict:
|
| 4 |
+
def __init__(self):
|
| 5 |
+
self.data = {}
|
| 6 |
+
|
| 7 |
+
def put(self, name, value):
|
| 8 |
+
timestamp = time.time()
|
| 9 |
+
self.data[name] = (value, timestamp)
|
| 10 |
+
|
| 11 |
+
def get(self, name):
|
| 12 |
+
return self.data.get(name, (None, None))[0]
|
| 13 |
+
|
| 14 |
+
def get_timestamp(self, name):
|
| 15 |
+
return self.data.get(name, (None, None))[1]
|
| 16 |
+
|
| 17 |
+
def touch(self, name):
|
| 18 |
+
timestamp = time.time()
|
| 19 |
+
self.data[name] = (self.get(name), timestamp)
|
| 20 |
+
|
| 21 |
+
def is_newer(self, name1, name2):
|
| 22 |
+
timestamp1 = self.get_timestamp(name1)
|
| 23 |
+
timestamp2 = self.get_timestamp(name2)
|
| 24 |
+
if timestamp1 is None:
|
| 25 |
+
return False
|
| 26 |
+
if timestamp2 is None:
|
| 27 |
+
return True
|
| 28 |
+
return timestamp1 > timestamp2
|