Dmitry Kirsanov commited on
Commit
46e2995
·
1 Parent(s): 7c7e6bb

versioned storage

Browse files
Files changed (3) hide show
  1. app.py +2 -1
  2. understand.py +56 -29
  3. versioneddict.py +28 -0
app.py CHANGED
@@ -13,7 +13,8 @@ LLM = llm.OpenAI()
13
  def process(filename, model):
14
  LLM.set_model(model)
15
  doc = understand.Document(LLM)
16
- return doc.understand(filename)
 
17
 
18
  gradio_app = gr.Interface(
19
  fn=process,
 
13
  def process(filename, model):
14
  LLM.set_model(model)
15
  doc = understand.Document(LLM)
16
+ doc.ingest(filename)
17
+ return doc.get_understanding()
18
 
19
  gradio_app = gr.Interface(
20
  fn=process,
understand.py CHANGED
@@ -1,14 +1,12 @@
1
  import parse
2
  import llm
 
3
 
4
  class Document():
5
 
6
  def __init__(self, llm) -> None:
7
  self.LLM = llm
8
- self.documentSoup = None # beautifulSoup
9
- self.fad = None # string
10
- self.doctype = None
11
- self.summary = None
12
 
13
  def query(self, s):
14
  try:
@@ -20,27 +18,27 @@ class Document():
20
 
21
 
22
  def query_doctype(self):
23
- return self.query(
24
  f"""Please read this document carefully. Remember everything you know about the concepts it touches.
25
 
26
  Give a brief description of what this document is. Examples: "Textbook about interpersonal psychology", "State standard on oil rig equipment", "User manual for a smartphone app".
27
 
28
  Write just one line with the description.
29
 
30
- <document>{self.fad}</document>""")
31
 
32
  def query_summary(self):
33
- return self.query(
34
  f"""Below is a document inside <document></document>. It is a {self.doctype}.
35
 
36
  Read it carefully. Remember everything you know about the concepts it touches.
37
 
38
  Give a one-paragraph summary of the document's content. Write just one paragraph with the summary.
39
 
40
- <document>{self.fad}</document>""")
41
 
42
  def query_concepts(self):
43
- return self.query(
44
  f"""Below is a document inside <document></document>. It is a {self.doctype}.
45
 
46
  Read it carefully. Remember everything you know about the concepts it touches.
@@ -51,31 +49,60 @@ Then, read it one more time, and think about who are the actors implied. Who are
51
 
52
  For example, for an instruction manual for a chainsaw, you might want to output:
53
  CONCEPTS:
54
- Power tool
55
- Safety
56
- Work area
57
- Saw chain
58
- Wood
59
  ...
60
  ACTORS:
61
- Chainsaw manufacturer
62
- Chainsaw user
63
- Children and bystanders
64
  ...
65
 
66
  <document>{self.fad}</document>""")
67
 
68
-
69
-
70
- def understand(self, filename):
71
- self.documentSoup = parse.parse(filename)
72
- self.fad = self.documentSoup.get_text()
73
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74
  # fact finders:
75
- self.doctype = self.query_doctype()
76
- if self.doctype != None:
77
- self.summary = self.query_summary()
78
- self.concepts = self.query_concepts()
79
-
80
- return self.doctype, self.summary, self.concepts
 
 
 
 
 
 
 
 
 
81
 
 
1
  import parse
2
  import llm
3
+ from versioneddict import VersionedDict
4
 
5
  class Document():
6
 
7
  def __init__(self, llm) -> None:
8
  self.LLM = llm
9
+ self.values = VersionedDict()
 
 
 
10
 
11
  def query(self, s):
12
  try:
 
18
 
19
 
20
  def query_doctype(self):
21
+ self.values.put("document type", self.query(
22
  f"""Please read this document carefully. Remember everything you know about the concepts it touches.
23
 
24
  Give a brief description of what this document is. Examples: "Textbook about interpersonal psychology", "State standard on oil rig equipment", "User manual for a smartphone app".
25
 
26
  Write just one line with the description.
27
 
28
+ <document>{self.fad}</document>"""))
29
 
30
  def query_summary(self):
31
+ self.values.put("summary", self.query(
32
  f"""Below is a document inside <document></document>. It is a {self.doctype}.
33
 
34
  Read it carefully. Remember everything you know about the concepts it touches.
35
 
36
  Give a one-paragraph summary of the document's content. Write just one paragraph with the summary.
37
 
38
+ <document>{self.fad}</document>"""))
39
 
40
  def query_concepts(self):
41
+ ret = self.query(
42
  f"""Below is a document inside <document></document>. It is a {self.doctype}.
43
 
44
  Read it carefully. Remember everything you know about the concepts it touches.
 
49
 
50
  For example, for an instruction manual for a chainsaw, you might want to output:
51
  CONCEPTS:
52
+ - Power tool
53
+ - Safety
54
+ - Work area
55
+ - Saw chain
56
+ - Wood
57
  ...
58
  ACTORS:
59
+ - Chainsaw manufacturer
60
+ - Chainsaw user
61
+ - Children and bystanders
62
  ...
63
 
64
  <document>{self.fad}</document>""")
65
 
66
+ self.values.put("concepts", [])
67
+ self.values.put("actors", [])
68
+
69
+ lines = ret.splitlines()
70
+ current_list = None
71
+
72
+ for line in lines:
73
+ line = line.strip()
74
+ if line.trim() == "CONCEPTS:":
75
+ current_list = "concepts"
76
+ elif line.trim() == "ACTORS:":
77
+ current_list = "actors"
78
+ elif current_list is not None:
79
+ if line.startswith("- "):
80
+ item = line[2:].strip()
81
+ else:
82
+ item = line.strip()
83
+ self.values.get(current_list).append(item)
84
+ self.values.touch(current_list)
85
+
86
+
87
+ def ingest(self, filename):
88
+ self.values.put("document (soup)", parse.parse(filename))
89
+ self.values.put("FAD", self.documentSoup.get_text())
90
+
91
+ def understand(self):
92
  # fact finders:
93
+ if self.values.is_newer("FAD", "document type"):
94
+ self.query_doctype()
95
+ if self.values.is_newer("FAD", "summary"):
96
+ self.query_summary()
97
+ if self.values.is_newer("FAD", "concepts"):
98
+ self.query_concepts()
99
+
100
+ def get_understanding(self):
101
+ self.understand()
102
+
103
+ return (self.values.get("document type"), self.values.get("summary"),
104
+ "=== Concepts:\n"+"\n".join(["- " + x for x in self.values.get("concepts")]),
105
+ "=== Actors:\n"+"\n".join(["- " + x for x in self.values.get("actors")]),
106
+ )
107
+
108
 
versioneddict.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import time
2
+
3
+ class VersionedDict:
4
+ def __init__(self):
5
+ self.data = {}
6
+
7
+ def put(self, name, value):
8
+ timestamp = time.time()
9
+ self.data[name] = (value, timestamp)
10
+
11
+ def get(self, name):
12
+ return self.data.get(name, (None, None))[0]
13
+
14
+ def get_timestamp(self, name):
15
+ return self.data.get(name, (None, None))[1]
16
+
17
+ def touch(self, name):
18
+ timestamp = time.time()
19
+ self.data[name] = (self.get(name), timestamp)
20
+
21
+ def is_newer(self, name1, name2):
22
+ timestamp1 = self.get_timestamp(name1)
23
+ timestamp2 = self.get_timestamp(name2)
24
+ if timestamp1 is None:
25
+ return False
26
+ if timestamp2 is None:
27
+ return True
28
+ return timestamp1 > timestamp2