import re import bidict import inspect import jsonpickle from note import Note defaultnote="document" notetypes = bidict.bidict({ defaultnote: 0, # generic "span": 1, "point": 2, "sentence": 3, "paragraph": 4, "line": 5, "concept": 6, "actor": 7, "audience": 8, "what-this-is": 9, "summary": 10, "chunk": 11, "topic": 12, "section": 13, "list": 14 }) chunktypes = bidict.bidict({ "task": 1, "process": 2, "principle": 3, "concept": 4, "reference": 5 }) summaries = ["what-this-is","summary"] plurals = {'actor': 'actors', 'concept': 'concepts', 'audience': 'audiences'} idlength = 8 pattern_marks = re.compile("[⌦⌫⌧]") class Fad: def __init__(self): self.notes = {} self.highest_note_id = 0 def newnote(self, type, value=None): id = self.highest_note_id + 1 type_code = notetypes[type] self.highest_note_id += 1 self.notes[id] = Note(self, id, type_code, value) return self.notes[id] def add(self, type, value=None): self.newnote(type, value) def add_from_list(self, values, type): for value in values: self.newnote(type, value) def set(self, type, value=None): for note in self.get_notes_by_type(type): note.set(value) def get_notes_by_type(self, valuetype): notes = [] for id, note in self.notes.items(): if note.typeid == notetypes[valuetype]: notes.append(note) return notes def get_note_by_type(self, valuetype): # if we know there's only one such note for id, note in self.notes.items(): if note.typeid == notetypes[valuetype]: return note def delete_by_type(self, valuetype): for note in self.get_notes_by_type(valuetype): note.delete() def get_values_by_type(self, valuetype): values = [] for id, note in self.notes.items(): if note.typeid == notetypes[valuetype]: values.append(note.get()) return values def get_value_by_type(self, valuetype): # if we know there's only one such note, or we only want to check if there's any for id, note in self.notes.items(): if note.typeid == notetypes[valuetype]: return note.get() def get_values_by_type_string(self, valuetype): values = self.get_values_by_type(valuetype) if len(values) > 1: return "\n".join(['- '+v for v in values]) else: return values[0] def get(self, src=defaultnote): return self.get_value_by_type(src) def get_note(self, src=defaultnote): return self.get_note_by_type(src) def get_by_value(self, value, valuetype) -> Note: for id, note in self.notes.items(): if note.typeid == notetypes[valuetype] and note.get() == value: return note return None def get_by_type_and_value(self, type, value) -> list[Note]: ret = [] for id, note in self.notes.items(): if notetypes[type] == note.typeid: if value == note.get(): ret.append(note) return ret def __next_mark(self, fad, start=0): tstart = start new = pattern_marks.search(fad, start) if new != None: tend = new.start() id = int(fad[tend + 1:tend + idlength + 1]) return fad[tstart:tend], fad[tend], id, tend + idlength + 1 return fad[tstart:], None, None, None # no marks till the end def __plaintext(self, fad): texts = [] i = 0 while True: text, markchar, id, i = self.__next_mark(fad, i) texts.append(text) if i == None: break return "".join(texts) def get_plaintext(self, src=defaultnote): fad = self.get_value_by_type(src) return self.__plaintext(fad) def get_plaintext_with_markup(self, notetype, src=defaultnote, show_id=False, show_properties=True, properties_to_show=[]): # [] means show all fad = self.get_value_by_type(src) i = 0 plaintext = '' while True: text, markchar, id, newi = self.__next_mark(fad, i) if id and self.notes[id].typeid == notetypes[notetype]: id_s = f' id="{id}"' if show_id else '' properties_s = ''.join([(f' {name}="{str(self.notes[id].get_string(name))}"' if show_properties and self.notes[id].get(name) and (not(properties_to_show) or name in properties_to_show) else '') for name in self.notes[id].properties.keys()]) # show if a property's value is not [], None, or 0 if markchar == '⌧': plaintext += text + f'<{notetype}{id_s}{properties_s}/>' else: plaintext += text + f'<{"/" if markchar == "⌫" else ""}{notetype}{id_s}{properties_s if markchar == "⌦" else ""}>' else: plaintext += text if newi == None: return plaintext i = newi def get_spans_by_note_type(self, notetype, src=defaultnote): fad = self.get_value_by_type(src) i = 0 spans = [] while True: text, markchar, id, newi = self.__next_mark(fad, i) if newi == None: break if self.notes[id].typeid == notetypes[notetype] and markchar == '⌦': close_i = fad.find(self.closemark(id), newi) if close_i != -1: spans.append(self.__plaintext(fad[newi:close_i])) i = newi return spans def get_spans_by_property_values(self, notetype, propertyname, propertyvalues, src=defaultnote): fad = self.get_value_by_type(src) i = 0 spans = [] while True: text, markchar, id, newi = self.__next_mark(fad, i) if newi == None: break if self.notes[id].typeid == notetypes[notetype] and markchar == '⌦' and propertyname in self.notes[id].properties.keys() and set(self.notes[id].get(propertyname)).intersection(set(propertyvalues)): close_i = fad.find(self.closemark(id), newi) if close_i != -1: spans.append(self.__plaintext(fad[newi:close_i])) i = newi return spans def get_spans_by_property_value(self, notetype, proprtyname, propertyvalue, src=defaultnote): # when we only need to check one value return self.get_spans_by_property_values(notetype, proprtyname, [propertyvalue], src) def is_newer(self, src, tgt): s = self.get_note(src) if s == None: return False t = s.get(tgt) if t == None: return True return s.is_newer(tgt) def openmark(self, id): return "⌦" + '{:08d}'.format(id) def closemark(self, id): return "⌫" + '{:08d}'.format(id) def pointmark(self, id): return "⌧" + '{:08d}'.format(id) def __goto_plaintext_i(self, fad, to, endskip=None): # fixme: suboptimal, can skip faster passed = 0 i = 0 while True: text, markchar, id, newi = self.__next_mark(fad, i) passed += len(text) if passed >= to: if endskip and passed == to and self.notes[id].typeid == notetypes[endskip]: return i + len(text) + idlength + 1 return i + len(text) - (passed - to) elif newi == None: return None # could not reach to else: i = newi def add_span_mark(self, notetype, start, end, src=defaultnote) -> Note: fad = self.get_value_by_type(src) start_i = self.__goto_plaintext_i(fad, start, endskip=notetype) end_i = self.__goto_plaintext_i(fad, end) if start_i == None or end_i == None: raise Exception(f"Fad: {inspect.stack()[0][3]}: start {start} or end {end} outside range {len(self.__plaintext(fad))}") newnote = self.newnote(notetype) newfad = fad[:start_i] + self.openmark(newnote.id) + fad[start_i:end_i] + self.closemark(newnote.id) + fad[end_i:] self.set(src, newfad) return newnote def add_point_mark(self, notetype, start, src=defaultnote) -> Note: fad = self.get_value_by_type(src) start_i = self.__goto_plaintext_i(fad, start) if start_i == None: raise Exception(f"Fad: {inspect.stack()[0][3]}: start outside range") newnote = self.newnote(notetype) newfad = fad[:start_i] + self.pointmark(newnote.id) + fad[start_i:] self.set(src, newfad) return newnote def get_note_for_mark_n(self, notetype, mark_n, startfrom=0, src=defaultnote) -> Note: fad = self.get_value_by_type(src) i = startfrom n = 0 while True: text, markchar, id, newi = self.__next_mark(fad, i) if markchar != '⌫': if newi == None: return None if self.notes[id].typeid == notetypes[notetype]: n += 1 if n == mark_n: return self.notes[id] i = newi def set_property_to_mark(self, notetype, mark_n, property, value, valuetype, startfrom=0, src=defaultnote): note = self.get_note_for_mark_n(notetype, mark_n, startfrom, src) if valuetype != None: # we're setting value to ref to a note whose type is valuetype valuenote = self.get_by_value(value, valuetype) note.set_property(property, valuenote.id if valuenote else None) else: # we're setting leaf string value to property note.set_property(property, value) def add_property_to_mark(self, notetype, mark_n, property, value, valuetype, startfrom=0, src=defaultnote): note = self.get_note_for_mark_n(notetype, mark_n, startfrom, src) if valuetype != None: # we're setting value to ref to a note whose type is valuetype valuenote = self.get_by_value(value, valuetype) if valuenote: note.add_to_property(property, valuenote.id) else: # we're setting leaf string value to property note.add_to_property(property, value) if __name__ == "__main__": # tests f = Fad() orig = "A123456789B123456789C123456789" f.add(defaultnote, orig) f.add_point_mark("point", 1) f.add_point_mark("point", 1) f.add_point_mark("point", 0) #print(f.get()) assert(f.get() == "⌧00000004A⌧00000003⌧00000002123456789B123456789C123456789") #print(f.get_plaintext()) assert(f.get_plaintext() == orig) #print(f.get_plaintext_with_markup("point", show_id=True)) assert(f.get_plaintext_with_markup("point", show_id=True) == 'A123456789B123456789C123456789') f.add_span_mark("span", 0, 10) f.add_span_mark("span", 20, 30) f.add_span_mark("span", 5, 25) #print(f.get()) assert(f.get() == "⌦00000005⌧00000004A⌧00000003⌧000000021234⌦0000000756789⌫00000005B123456789⌦00000006C1234⌫0000000756789⌫00000006") #print(f.get_plaintext()) assert(f.get_plaintext() == orig) #print(f.get_plaintext_with_markup("span", show_id=True)) assert(f.get_plaintext_with_markup("span", show_id=True) == 'A123456789B123456789C123456789') #print(f.get_plaintext_with_markup("point", show_id=True)) assert(f.get_plaintext_with_markup("point", show_id=True) == 'A123456789B123456789C123456789') actors_old = ["actorold1", "actorold2"] actors = ["actor1", "actor2"] audiences = ["audience1", "audience2", "audience3"] f.add_from_list(actors_old, "actor") f.add_from_list(audiences, "audience") assert(f.get_values_by_type_string("audience") == """- audience1 - audience2 - audience3""") assert(actors_old == f.get_values_by_type("actor")) assert(audiences == f.get_values_by_type("audience")) f.delete_by_type("actor") f.add_from_list(actors, "actor") assert(actors == f.get_values_by_type("actor")) f.set_property_to_mark("span", 1, "actors", "actor1", "actor") #print(f.get_plaintext_with_markup("span", show_id=True, show_properties=True)) assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789') f.set_property_to_mark("span", 3, "actors", "actor2", "actor") assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789') f.set_property_to_mark("span", 3, "actors", "actor1", "actor") f.set_property_to_mark("span", 2, "actors", "actor1", "actor") assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789') f.set_property_to_mark("span", 3, "actors", None, "actor") assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789') f.add_property_to_mark("span", 1, "audiences", None, "audience") f.add_property_to_mark("span", 1, "audiences", "audience1", "audience") f.add_property_to_mark("span", 1, "audiences", "audience2", "audience") assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789') f.add_property_to_mark("span", 1, "audiences", "audience3", "audience") f.add_property_to_mark("span", 2, "audiences", "audience3", "audience") #print(f.get_plaintext_with_markup("span", show_id=True, show_properties=True)) assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789') f.set_property_to_mark("span", 2, "audiences", None, "audience") assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == 'A123456789B123456789C123456789') f.add_property_to_mark("span", 3, "audiences", "audience3", "audience") assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True, properties_to_show=['actors']) == 'A123456789B123456789C123456789') f.add_property_to_mark("span", 2, "audiences", "audience2", "audience") assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True, properties_to_show=['audiences']) == 'A123456789B123456789C123456789') assert(f.get_spans_by_note_type("span") == ['A123456789', '56789B123456789C1234', 'C123456789']) assert(f.get_spans_by_property_values("span", propertyname="audiences", propertyvalues=['audience1', 'audience3']) == ['A123456789', 'C123456789']) assert(f.get_spans_by_property_value("span", "audiences", 'audience2') == ['A123456789', '56789B123456789C1234']) point2 = f.get_note_for_mark_n("point", 2) point3 = f.get_note_for_mark_n("point", 3) point2.set_property_newnote("concepts", "important concept 1", "concept") #print(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts'])) assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == 'A123456789B123456789C123456789') point3.set_property_newnote("concepts", "important concept 2", "concept") point2.add_to_property_newnote("concepts", "important concept 2", "concept") assert(len(f.get_by_type_and_value("concept", "important concept 2")) == 1) assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == 'A123456789B123456789C123456789') point2.set_property("concepts", None) #print(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts'])) assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == 'A123456789B123456789C123456789') assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=False) == 'A123456789B123456789C123456789') for id, note in f.notes.items(): assert(note.needs_updating_from() == []) # in normal operation, there should be no contratemporal refs X -> something newer than X (it means X needs updating) json =jsonpickle.encode(f, indent=4, keys=True) with open("fad.json", "w") as fo: fo.write(json) with open("fad.json", "r") as fin: fn = jsonpickle.decode(fin.read(), keys=True) assert(fn.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == 'A123456789B123456789C123456789')