import re
import bidict
import inspect
import jsonpickle
from note import Note
defaultnote="document"
notetypes = bidict.bidict({
defaultnote: 0,
# generic
"span": 1,
"point": 2,
"sentence": 3,
"paragraph": 4,
"line": 5,
"concept": 6,
"actor": 7,
"audience": 8,
"what-this-is": 9,
"summary": 10,
"chunk": 11,
"topic": 12,
"section": 13,
"list": 14
})
chunktypes = bidict.bidict({
"task": 1,
"process": 2,
"principle": 3,
"concept": 4,
"reference": 5
})
summaries = ["what-this-is","summary"]
plurals = {'actor': 'actors', 'concept': 'concepts', 'audience': 'audiences'}
idlength = 8
pattern_marks = re.compile("[⌦⌫⌧]")
class Fad:
def __init__(self):
self.notes = {}
self.highest_note_id = 0
def newnote(self, type, value=None):
id = self.highest_note_id + 1
type_code = notetypes[type]
self.highest_note_id += 1
self.notes[id] = Note(self, id, type_code, value)
return self.notes[id]
def add(self, type, value=None):
self.newnote(type, value)
def add_from_list(self, values, type):
for value in values:
self.newnote(type, value)
def set(self, type, value=None):
for note in self.get_notes_by_type(type):
note.set(value)
def get_notes_by_type(self, valuetype):
notes = []
for id, note in self.notes.items():
if note.typeid == notetypes[valuetype]:
notes.append(note)
return notes
def get_note_by_type(self, valuetype): # if we know there's only one such note
for id, note in self.notes.items():
if note.typeid == notetypes[valuetype]:
return note
def delete_by_type(self, valuetype):
for note in self.get_notes_by_type(valuetype):
note.delete()
def get_values_by_type(self, valuetype):
values = []
for id, note in self.notes.items():
if note.typeid == notetypes[valuetype]:
values.append(note.get())
return values
def get_value_by_type(self, valuetype): # if we know there's only one such note, or we only want to check if there's any
for id, note in self.notes.items():
if note.typeid == notetypes[valuetype]:
return note.get()
def get_values_by_type_string(self, valuetype):
values = self.get_values_by_type(valuetype)
if len(values) > 1:
return "\n".join(['- '+v for v in values])
else:
return values[0]
def get(self, src=defaultnote):
return self.get_value_by_type(src)
def get_note(self, src=defaultnote):
return self.get_note_by_type(src)
def get_by_value(self, value, valuetype) -> Note:
for id, note in self.notes.items():
if note.typeid == notetypes[valuetype] and note.get() == value:
return note
return None
def get_by_type_and_value(self, type, value) -> list[Note]:
ret = []
for id, note in self.notes.items():
if notetypes[type] == note.typeid:
if value == note.get():
ret.append(note)
return ret
def __next_mark(self, fad, start=0):
tstart = start
new = pattern_marks.search(fad, start)
if new != None:
tend = new.start()
id = int(fad[tend + 1:tend + idlength + 1])
return fad[tstart:tend], fad[tend], id, tend + idlength + 1
return fad[tstart:], None, None, None # no marks till the end
def __plaintext(self, fad):
texts = []
i = 0
while True:
text, markchar, id, i = self.__next_mark(fad, i)
texts.append(text)
if i == None:
break
return "".join(texts)
def get_plaintext(self, src=defaultnote):
fad = self.get_value_by_type(src)
return self.__plaintext(fad)
def get_plaintext_with_markup(self, notetype, src=defaultnote, show_id=False, show_properties=True, properties_to_show=[]): # [] means show all
fad = self.get_value_by_type(src)
i = 0
plaintext = ''
while True:
text, markchar, id, newi = self.__next_mark(fad, i)
if id and self.notes[id].typeid == notetypes[notetype]:
id_s = f' id="{id}"' if show_id else ''
properties_s = ''.join([(f' {name}="{str(self.notes[id].get_string(name))}"' if show_properties and self.notes[id].get(name) and (not(properties_to_show) or name in properties_to_show) else '') for name in self.notes[id].properties.keys()]) # show if a property's value is not [], None, or 0
if markchar == '⌧':
plaintext += text + f'<{notetype}{id_s}{properties_s}/>'
else:
plaintext += text + f'<{"/" if markchar == "⌫" else ""}{notetype}{id_s}{properties_s if markchar == "⌦" else ""}>'
else:
plaintext += text
if newi == None:
return plaintext
i = newi
def get_spans_by_note_type(self, notetype, src=defaultnote):
fad = self.get_value_by_type(src)
i = 0
spans = []
while True:
text, markchar, id, newi = self.__next_mark(fad, i)
if newi == None:
break
if self.notes[id].typeid == notetypes[notetype] and markchar == '⌦':
close_i = fad.find(self.closemark(id), newi)
if close_i != -1:
spans.append(self.__plaintext(fad[newi:close_i]))
i = newi
return spans
def get_spans_by_property_values(self, notetype, propertyname, propertyvalues, src=defaultnote):
fad = self.get_value_by_type(src)
i = 0
spans = []
while True:
text, markchar, id, newi = self.__next_mark(fad, i)
if newi == None:
break
if self.notes[id].typeid == notetypes[notetype] and markchar == '⌦' and propertyname in self.notes[id].properties.keys() and set(self.notes[id].get(propertyname)).intersection(set(propertyvalues)):
close_i = fad.find(self.closemark(id), newi)
if close_i != -1:
spans.append(self.__plaintext(fad[newi:close_i]))
i = newi
return spans
def get_spans_by_property_value(self, notetype, proprtyname, propertyvalue, src=defaultnote): # when we only need to check one value
return self.get_spans_by_property_values(notetype, proprtyname, [propertyvalue], src)
def is_newer(self, src, tgt):
s = self.get_note(src)
if s == None:
return False
t = s.get(tgt)
if t == None:
return True
return s.is_newer(tgt)
def openmark(self, id):
return "⌦" + '{:08d}'.format(id)
def closemark(self, id):
return "⌫" + '{:08d}'.format(id)
def pointmark(self, id):
return "⌧" + '{:08d}'.format(id)
def __goto_plaintext_i(self, fad, to, endskip=None):
# fixme: suboptimal, can skip faster
passed = 0
i = 0
while True:
text, markchar, id, newi = self.__next_mark(fad, i)
passed += len(text)
if passed >= to:
if endskip and passed == to and self.notes[id].typeid == notetypes[endskip]:
return i + len(text) + idlength + 1
return i + len(text) - (passed - to)
elif newi == None:
return None # could not reach to
else:
i = newi
def add_span_mark(self, notetype, start, end, src=defaultnote) -> Note:
fad = self.get_value_by_type(src)
start_i = self.__goto_plaintext_i(fad, start, endskip=notetype)
end_i = self.__goto_plaintext_i(fad, end)
if start_i == None or end_i == None:
raise Exception(f"Fad: {inspect.stack()[0][3]}: start {start} or end {end} outside range {len(self.__plaintext(fad))}")
newnote = self.newnote(notetype)
newfad = fad[:start_i] + self.openmark(newnote.id) + fad[start_i:end_i] + self.closemark(newnote.id) + fad[end_i:]
self.set(src, newfad)
return newnote
def add_point_mark(self, notetype, start, src=defaultnote) -> Note:
fad = self.get_value_by_type(src)
start_i = self.__goto_plaintext_i(fad, start)
if start_i == None:
raise Exception(f"Fad: {inspect.stack()[0][3]}: start outside range")
newnote = self.newnote(notetype)
newfad = fad[:start_i] + self.pointmark(newnote.id) + fad[start_i:]
self.set(src, newfad)
return newnote
def get_note_for_mark_n(self, notetype, mark_n, startfrom=0, src=defaultnote) -> Note:
fad = self.get_value_by_type(src)
i = startfrom
n = 0
while True:
text, markchar, id, newi = self.__next_mark(fad, i)
if markchar != '⌫':
if newi == None:
return None
if self.notes[id].typeid == notetypes[notetype]:
n += 1
if n == mark_n:
return self.notes[id]
i = newi
def set_property_to_mark(self, notetype, mark_n, property, value, valuetype, startfrom=0, src=defaultnote):
note = self.get_note_for_mark_n(notetype, mark_n, startfrom, src)
if valuetype != None: # we're setting value to ref to a note whose type is valuetype
valuenote = self.get_by_value(value, valuetype)
note.set_property(property, valuenote.id if valuenote else None)
else: # we're setting leaf string value to property
note.set_property(property, value)
def add_property_to_mark(self, notetype, mark_n, property, value, valuetype, startfrom=0, src=defaultnote):
note = self.get_note_for_mark_n(notetype, mark_n, startfrom, src)
if valuetype != None: # we're setting value to ref to a note whose type is valuetype
valuenote = self.get_by_value(value, valuetype)
if valuenote:
note.add_to_property(property, valuenote.id)
else: # we're setting leaf string value to property
note.add_to_property(property, value)
if __name__ == "__main__":
# tests
f = Fad()
orig = "A123456789B123456789C123456789"
f.add(defaultnote, orig)
f.add_point_mark("point", 1)
f.add_point_mark("point", 1)
f.add_point_mark("point", 0)
#print(f.get())
assert(f.get() == "⌧00000004A⌧00000003⌧00000002123456789B123456789C123456789")
#print(f.get_plaintext())
assert(f.get_plaintext() == orig)
#print(f.get_plaintext_with_markup("point", show_id=True))
assert(f.get_plaintext_with_markup("point", show_id=True) == 'A123456789B123456789C123456789')
f.add_span_mark("span", 0, 10)
f.add_span_mark("span", 20, 30)
f.add_span_mark("span", 5, 25)
#print(f.get())
assert(f.get() == "⌦00000005⌧00000004A⌧00000003⌧000000021234⌦0000000756789⌫00000005B123456789⌦00000006C1234⌫0000000756789⌫00000006")
#print(f.get_plaintext())
assert(f.get_plaintext() == orig)
#print(f.get_plaintext_with_markup("span", show_id=True))
assert(f.get_plaintext_with_markup("span", show_id=True) == 'A123456789B123456789C123456789')
#print(f.get_plaintext_with_markup("point", show_id=True))
assert(f.get_plaintext_with_markup("point", show_id=True) == 'A123456789B123456789C123456789')
actors_old = ["actorold1", "actorold2"]
actors = ["actor1", "actor2"]
audiences = ["audience1", "audience2", "audience3"]
f.add_from_list(actors_old, "actor")
f.add_from_list(audiences, "audience")
assert(f.get_values_by_type_string("audience") == """- audience1
- audience2
- audience3""")
assert(actors_old == f.get_values_by_type("actor"))
assert(audiences == f.get_values_by_type("audience"))
f.delete_by_type("actor")
f.add_from_list(actors, "actor")
assert(actors == f.get_values_by_type("actor"))
f.set_property_to_mark("span", 1, "actors", "actor1", "actor")
#print(f.get_plaintext_with_markup("span", show_id=True, show_properties=True))
assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789')
f.set_property_to_mark("span", 3, "actors", "actor2", "actor")
assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789')
f.set_property_to_mark("span", 3, "actors", "actor1", "actor")
f.set_property_to_mark("span", 2, "actors", "actor1", "actor")
assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789')
f.set_property_to_mark("span", 3, "actors", None, "actor")
assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789')
f.add_property_to_mark("span", 1, "audiences", None, "audience")
f.add_property_to_mark("span", 1, "audiences", "audience1", "audience")
f.add_property_to_mark("span", 1, "audiences", "audience2", "audience")
assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789')
f.add_property_to_mark("span", 1, "audiences", "audience3", "audience")
f.add_property_to_mark("span", 2, "audiences", "audience3", "audience")
#print(f.get_plaintext_with_markup("span", show_id=True, show_properties=True))
assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789')
f.set_property_to_mark("span", 2, "audiences", None, "audience")
assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789')
f.add_property_to_mark("span", 3, "audiences", "audience3", "audience")
assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True, properties_to_show=['actors']) == 'A123456789B123456789C123456789')
f.add_property_to_mark("span", 2, "audiences", "audience2", "audience")
assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True, properties_to_show=['audiences']) == 'A123456789B123456789C123456789')
assert(f.get_spans_by_note_type("span") == ['A123456789', '56789B123456789C1234', 'C123456789'])
assert(f.get_spans_by_property_values("span", propertyname="audiences", propertyvalues=['audience1', 'audience3']) == ['A123456789', 'C123456789'])
assert(f.get_spans_by_property_value("span", "audiences", 'audience2') == ['A123456789', '56789B123456789C1234'])
point2 = f.get_note_for_mark_n("point", 2)
point3 = f.get_note_for_mark_n("point", 3)
point2.set_property_newnote("concepts", "important concept 1", "concept")
#print(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']))
assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == 'A123456789B123456789C123456789')
point3.set_property_newnote("concepts", "important concept 2", "concept")
point2.add_to_property_newnote("concepts", "important concept 2", "concept")
assert(len(f.get_by_type_and_value("concept", "important concept 2")) == 1)
assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == 'A123456789B123456789C123456789')
point2.set_property("concepts", None)
#print(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']))
assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == 'A123456789B123456789C123456789')
assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=False) == 'A123456789B123456789C123456789')
for id, note in f.notes.items():
assert(note.needs_updating_from() == []) # in normal operation, there should be no contratemporal refs X -> something newer than X (it means X needs updating)
json =jsonpickle.encode(f, indent=4, keys=True)
with open("fad.json", "w") as fo:
fo.write(json)
with open("fad.json", "r") as fin:
fn = jsonpickle.decode(fin.read(), keys=True)
assert(fn.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == 'A123456789B123456789C123456789')