Spaces:
Runtime error
Runtime error
Download fad.py from PrecisionContent/DU: direct link, hf CLI and curl.
- Browser
- Download file 18.5 kB
-
https://huggingface.co/spaces/PrecisionContent/DU/resolve/main/fad.py
- Command line
-
hf download hf://spaces/PrecisionContent/DU/fad.py
-
curl -L -o fad.py https://huggingface.co/spaces/PrecisionContent/DU/resolve/main/fad.py
18.5 kB
| import re | |
| import bidict | |
| import inspect | |
| import jsonpickle | |
| from note import Note | |
| defaultnote="document" | |
| notetypes = bidict.bidict({ | |
| defaultnote: 0, | |
| # generic | |
| "span": 1, | |
| "point": 2, | |
| "sentence": 3, | |
| "paragraph": 4, | |
| "line": 5, | |
| "concept": 6, | |
| "actor": 7, | |
| "audience": 8, | |
| "what-this-is": 9, | |
| "summary": 10, | |
| "chunk": 11, | |
| "topic": 12, | |
| "section": 13, | |
| "list": 14 | |
| }) | |
| chunktypes = bidict.bidict({ | |
| "task": 1, | |
| "process": 2, | |
| "principle": 3, | |
| "concept": 4, | |
| "reference": 5 | |
| }) | |
| summaries = ["what-this-is","summary"] | |
| plurals = {'actor': 'actors', 'concept': 'concepts', 'audience': 'audiences'} | |
| idlength = 8 | |
| pattern_marks = re.compile("[⌦⌫⌧]") | |
| class Fad: | |
| def __init__(self): | |
| self.notes = {} | |
| self.highest_note_id = 0 | |
| def newnote(self, type, value=None): | |
| id = self.highest_note_id + 1 | |
| type_code = notetypes[type] | |
| self.highest_note_id += 1 | |
| self.notes[id] = Note(self, id, type_code, value) | |
| return self.notes[id] | |
| def add(self, type, value=None): | |
| self.newnote(type, value) | |
| def add_from_list(self, values, type): | |
| for value in values: | |
| self.newnote(type, value) | |
| def set(self, type, value=None): | |
| for note in self.get_notes_by_type(type): | |
| note.set(value) | |
| def get_notes_by_type(self, valuetype): | |
| notes = [] | |
| for id, note in self.notes.items(): | |
| if note.typeid == notetypes[valuetype]: | |
| notes.append(note) | |
| return notes | |
| def get_note_by_type(self, valuetype): # if we know there's only one such note | |
| for id, note in self.notes.items(): | |
| if note.typeid == notetypes[valuetype]: | |
| return note | |
| def delete_by_type(self, valuetype): | |
| for note in self.get_notes_by_type(valuetype): | |
| note.delete() | |
| def get_values_by_type(self, valuetype): | |
| values = [] | |
| for id, note in self.notes.items(): | |
| if note.typeid == notetypes[valuetype]: | |
| values.append(note.get()) | |
| return values | |
| def get_value_by_type(self, valuetype): # if we know there's only one such note, or we only want to check if there's any | |
| for id, note in self.notes.items(): | |
| if note.typeid == notetypes[valuetype]: | |
| return note.get() | |
| def get_values_by_type_string(self, valuetype): | |
| values = self.get_values_by_type(valuetype) | |
| if len(values) > 1: | |
| return "\n".join(['- '+v for v in values]) | |
| else: | |
| return values[0] | |
| def get(self, src=defaultnote): | |
| return self.get_value_by_type(src) | |
| def get_note(self, src=defaultnote): | |
| return self.get_note_by_type(src) | |
| def get_by_value(self, value, valuetype) -> Note: | |
| for id, note in self.notes.items(): | |
| if note.typeid == notetypes[valuetype] and note.get() == value: | |
| return note | |
| return None | |
| def get_by_type_and_value(self, type, value) -> list[Note]: | |
| ret = [] | |
| for id, note in self.notes.items(): | |
| if notetypes[type] == note.typeid: | |
| if value == note.get(): | |
| ret.append(note) | |
| return ret | |
| def __next_mark(self, fad, start=0): | |
| tstart = start | |
| new = pattern_marks.search(fad, start) | |
| if new != None: | |
| tend = new.start() | |
| id = int(fad[tend + 1:tend + idlength + 1]) | |
| return fad[tstart:tend], fad[tend], id, tend + idlength + 1 | |
| return fad[tstart:], None, None, None # no marks till the end | |
| def __plaintext(self, fad): | |
| texts = [] | |
| i = 0 | |
| while True: | |
| text, markchar, id, i = self.__next_mark(fad, i) | |
| texts.append(text) | |
| if i == None: | |
| break | |
| return "".join(texts) | |
| def get_plaintext(self, src=defaultnote): | |
| fad = self.get_value_by_type(src) | |
| return self.__plaintext(fad) | |
| def get_plaintext_with_markup(self, notetype, src=defaultnote, show_id=False, show_properties=True, properties_to_show=[]): # [] means show all | |
| fad = self.get_value_by_type(src) | |
| i = 0 | |
| plaintext = '' | |
| while True: | |
| text, markchar, id, newi = self.__next_mark(fad, i) | |
| if id and self.notes[id].typeid == notetypes[notetype]: | |
| id_s = f' id="{id}"' if show_id else '' | |
| properties_s = ''.join([(f' {name}="{str(self.notes[id].get_string(name))}"' if show_properties and self.notes[id].get(name) and (not(properties_to_show) or name in properties_to_show) else '') for name in self.notes[id].properties.keys()]) # show if a property's value is not [], None, or 0 | |
| if markchar == '⌧': | |
| plaintext += text + f'<{notetype}{id_s}{properties_s}/>' | |
| else: | |
| plaintext += text + f'<{"/" if markchar == "⌫" else ""}{notetype}{id_s}{properties_s if markchar == "⌦" else ""}>' | |
| else: | |
| plaintext += text | |
| if newi == None: | |
| return plaintext | |
| i = newi | |
| def get_spans_by_note_type(self, notetype, src=defaultnote): | |
| fad = self.get_value_by_type(src) | |
| i = 0 | |
| spans = [] | |
| while True: | |
| text, markchar, id, newi = self.__next_mark(fad, i) | |
| if newi == None: | |
| break | |
| if self.notes[id].typeid == notetypes[notetype] and markchar == '⌦': | |
| close_i = fad.find(self.closemark(id), newi) | |
| if close_i != -1: | |
| spans.append(self.__plaintext(fad[newi:close_i])) | |
| i = newi | |
| return spans | |
| def get_spans_by_property_values(self, notetype, propertyname, propertyvalues, src=defaultnote): | |
| fad = self.get_value_by_type(src) | |
| i = 0 | |
| spans = [] | |
| while True: | |
| text, markchar, id, newi = self.__next_mark(fad, i) | |
| if newi == None: | |
| break | |
| if self.notes[id].typeid == notetypes[notetype] and markchar == '⌦' and propertyname in self.notes[id].properties.keys() and set(self.notes[id].get(propertyname)).intersection(set(propertyvalues)): | |
| close_i = fad.find(self.closemark(id), newi) | |
| if close_i != -1: | |
| spans.append(self.__plaintext(fad[newi:close_i])) | |
| i = newi | |
| return spans | |
| def get_spans_by_property_value(self, notetype, proprtyname, propertyvalue, src=defaultnote): # when we only need to check one value | |
| return self.get_spans_by_property_values(notetype, proprtyname, [propertyvalue], src) | |
| def is_newer(self, src, tgt): | |
| s = self.get_note(src) | |
| if s == None: | |
| return False | |
| t = s.get(tgt) | |
| if t == None: | |
| return True | |
| return s.is_newer(tgt) | |
| def openmark(self, id): | |
| return "⌦" + '{:08d}'.format(id) | |
| def closemark(self, id): | |
| return "⌫" + '{:08d}'.format(id) | |
| def pointmark(self, id): | |
| return "⌧" + '{:08d}'.format(id) | |
| def __goto_plaintext_i(self, fad, to, endskip=None): | |
| # fixme: suboptimal, can skip faster | |
| passed = 0 | |
| i = 0 | |
| while True: | |
| text, markchar, id, newi = self.__next_mark(fad, i) | |
| passed += len(text) | |
| if passed >= to: | |
| if endskip and passed == to and self.notes[id].typeid == notetypes[endskip]: | |
| return i + len(text) + idlength + 1 | |
| return i + len(text) - (passed - to) | |
| elif newi == None: | |
| return None # could not reach to | |
| else: | |
| i = newi | |
| def add_span_mark(self, notetype, start, end, src=defaultnote) -> Note: | |
| fad = self.get_value_by_type(src) | |
| start_i = self.__goto_plaintext_i(fad, start, endskip=notetype) | |
| end_i = self.__goto_plaintext_i(fad, end) | |
| if start_i == None or end_i == None: | |
| raise Exception(f"Fad: {inspect.stack()[0][3]}: start {start} or end {end} outside range {len(self.__plaintext(fad))}") | |
| newnote = self.newnote(notetype) | |
| newfad = fad[:start_i] + self.openmark(newnote.id) + fad[start_i:end_i] + self.closemark(newnote.id) + fad[end_i:] | |
| self.set(src, newfad) | |
| return newnote | |
| def add_point_mark(self, notetype, start, src=defaultnote) -> Note: | |
| fad = self.get_value_by_type(src) | |
| start_i = self.__goto_plaintext_i(fad, start) | |
| if start_i == None: | |
| raise Exception(f"Fad: {inspect.stack()[0][3]}: start outside range") | |
| newnote = self.newnote(notetype) | |
| newfad = fad[:start_i] + self.pointmark(newnote.id) + fad[start_i:] | |
| self.set(src, newfad) | |
| return newnote | |
| def get_note_for_mark_n(self, notetype, mark_n, startfrom=0, src=defaultnote) -> Note: | |
| fad = self.get_value_by_type(src) | |
| i = startfrom | |
| n = 0 | |
| while True: | |
| text, markchar, id, newi = self.__next_mark(fad, i) | |
| if markchar != '⌫': | |
| if newi == None: | |
| return None | |
| if self.notes[id].typeid == notetypes[notetype]: | |
| n += 1 | |
| if n == mark_n: | |
| return self.notes[id] | |
| i = newi | |
| def set_property_to_mark(self, notetype, mark_n, property, value, valuetype, startfrom=0, src=defaultnote): | |
| note = self.get_note_for_mark_n(notetype, mark_n, startfrom, src) | |
| if valuetype != None: # we're setting value to ref to a note whose type is valuetype | |
| valuenote = self.get_by_value(value, valuetype) | |
| note.set_property(property, valuenote.id if valuenote else None) | |
| else: # we're setting leaf string value to property | |
| note.set_property(property, value) | |
| def add_property_to_mark(self, notetype, mark_n, property, value, valuetype, startfrom=0, src=defaultnote): | |
| note = self.get_note_for_mark_n(notetype, mark_n, startfrom, src) | |
| if valuetype != None: # we're setting value to ref to a note whose type is valuetype | |
| valuenote = self.get_by_value(value, valuetype) | |
| if valuenote: | |
| note.add_to_property(property, valuenote.id) | |
| else: # we're setting leaf string value to property | |
| note.add_to_property(property, value) | |
| if __name__ == "__main__": | |
| # tests | |
| f = Fad() | |
| orig = "A123456789B123456789C123456789" | |
| f.add(defaultnote, orig) | |
| f.add_point_mark("point", 1) | |
| f.add_point_mark("point", 1) | |
| f.add_point_mark("point", 0) | |
| #print(f.get()) | |
| assert(f.get() == "⌧00000004A⌧00000003⌧00000002123456789B123456789C123456789") | |
| #print(f.get_plaintext()) | |
| assert(f.get_plaintext() == orig) | |
| #print(f.get_plaintext_with_markup("point", show_id=True)) | |
| assert(f.get_plaintext_with_markup("point", show_id=True) == '<point id="4"/>A<point id="3"/><point id="2"/>123456789B123456789C123456789') | |
| f.add_span_mark("span", 0, 10) | |
| f.add_span_mark("span", 20, 30) | |
| f.add_span_mark("span", 5, 25) | |
| #print(f.get()) | |
| assert(f.get() == "⌦00000005⌧00000004A⌧00000003⌧000000021234⌦0000000756789⌫00000005B123456789⌦00000006C1234⌫0000000756789⌫00000006") | |
| #print(f.get_plaintext()) | |
| assert(f.get_plaintext() == orig) | |
| #print(f.get_plaintext_with_markup("span", show_id=True)) | |
| assert(f.get_plaintext_with_markup("span", show_id=True) == '<span id="5">A1234<span id="7">56789</span id="5">B123456789<span id="6">C1234</span id="7">56789</span id="6">') | |
| #print(f.get_plaintext_with_markup("point", show_id=True)) | |
| assert(f.get_plaintext_with_markup("point", show_id=True) == '<point id="4"/>A<point id="3"/><point id="2"/>123456789B123456789C123456789') | |
| actors_old = ["actorold1", "actorold2"] | |
| actors = ["actor1", "actor2"] | |
| audiences = ["audience1", "audience2", "audience3"] | |
| f.add_from_list(actors_old, "actor") | |
| f.add_from_list(audiences, "audience") | |
| assert(f.get_values_by_type_string("audience") == """- audience1 | |
| - audience2 | |
| - audience3""") | |
| assert(actors_old == f.get_values_by_type("actor")) | |
| assert(audiences == f.get_values_by_type("audience")) | |
| f.delete_by_type("actor") | |
| f.add_from_list(actors, "actor") | |
| assert(actors == f.get_values_by_type("actor")) | |
| f.set_property_to_mark("span", 1, "actors", "actor1", "actor") | |
| #print(f.get_plaintext_with_markup("span", show_id=True, show_properties=True)) | |
| assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == '<span id="5" actors="actor1">A1234<span id="7">56789</span id="5">B123456789<span id="6">C1234</span id="7">56789</span id="6">') | |
| f.set_property_to_mark("span", 3, "actors", "actor2", "actor") | |
| assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == '<span id="5" actors="actor1">A1234<span id="7">56789</span id="5">B123456789<span id="6" actors="actor2">C1234</span id="7">56789</span id="6">') | |
| f.set_property_to_mark("span", 3, "actors", "actor1", "actor") | |
| f.set_property_to_mark("span", 2, "actors", "actor1", "actor") | |
| assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == '<span id="5" actors="actor1">A1234<span id="7" actors="actor1">56789</span id="5">B123456789<span id="6" actors="actor1">C1234</span id="7">56789</span id="6">') | |
| f.set_property_to_mark("span", 3, "actors", None, "actor") | |
| assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == '<span id="5" actors="actor1">A1234<span id="7" actors="actor1">56789</span id="5">B123456789<span id="6">C1234</span id="7">56789</span id="6">') | |
| f.add_property_to_mark("span", 1, "audiences", None, "audience") | |
| f.add_property_to_mark("span", 1, "audiences", "audience1", "audience") | |
| f.add_property_to_mark("span", 1, "audiences", "audience2", "audience") | |
| assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == '<span id="5" actors="actor1" audiences="audience1, audience2">A1234<span id="7" actors="actor1">56789</span id="5">B123456789<span id="6">C1234</span id="7">56789</span id="6">') | |
| f.add_property_to_mark("span", 1, "audiences", "audience3", "audience") | |
| f.add_property_to_mark("span", 2, "audiences", "audience3", "audience") | |
| #print(f.get_plaintext_with_markup("span", show_id=True, show_properties=True)) | |
| assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == '<span id="5" actors="actor1" audiences="audience1, audience2, audience3">A1234<span id="7" actors="actor1" audiences="audience3">56789</span id="5">B123456789<span id="6">C1234</span id="7">56789</span id="6">') | |
| f.set_property_to_mark("span", 2, "audiences", None, "audience") | |
| assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True) == '<span id="5" actors="actor1" audiences="audience1, audience2, audience3">A1234<span id="7" actors="actor1">56789</span id="5">B123456789<span id="6">C1234</span id="7">56789</span id="6">') | |
| f.add_property_to_mark("span", 3, "audiences", "audience3", "audience") | |
| assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True, properties_to_show=['actors']) == '<span id="5" actors="actor1">A1234<span id="7" actors="actor1">56789</span id="5">B123456789<span id="6">C1234</span id="7">56789</span id="6">') | |
| f.add_property_to_mark("span", 2, "audiences", "audience2", "audience") | |
| assert(f.get_plaintext_with_markup("span", show_id=True, show_properties=True, properties_to_show=['audiences']) == '<span id="5" audiences="audience1, audience2, audience3">A1234<span id="7" audiences="audience2">56789</span id="5">B123456789<span id="6" audiences="audience3">C1234</span id="7">56789</span id="6">') | |
| assert(f.get_spans_by_note_type("span") == ['A123456789', '56789B123456789C1234', 'C123456789']) | |
| assert(f.get_spans_by_property_values("span", propertyname="audiences", propertyvalues=['audience1', 'audience3']) == ['A123456789', 'C123456789']) | |
| assert(f.get_spans_by_property_value("span", "audiences", 'audience2') == ['A123456789', '56789B123456789C1234']) | |
| point2 = f.get_note_for_mark_n("point", 2) | |
| point3 = f.get_note_for_mark_n("point", 3) | |
| point2.set_property_newnote("concepts", "important concept 1", "concept") | |
| #print(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts'])) | |
| assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == '<point/>A<point concepts="important concept 1"/><point/>123456789B123456789C123456789') | |
| point3.set_property_newnote("concepts", "important concept 2", "concept") | |
| point2.add_to_property_newnote("concepts", "important concept 2", "concept") | |
| assert(len(f.get_by_type_and_value("concept", "important concept 2")) == 1) | |
| assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == '<point/>A<point concepts="important concept 1, important concept 2"/><point concepts="important concept 2"/>123456789B123456789C123456789') | |
| point2.set_property("concepts", None) | |
| #print(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts'])) | |
| assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == '<point/>A<point/><point concepts="important concept 2"/>123456789B123456789C123456789') | |
| assert(f.get_plaintext_with_markup("point", show_id=False, show_properties=False) == '<point/>A<point/><point/>123456789B123456789C123456789') | |
| for id, note in f.notes.items(): | |
| assert(note.needs_updating_from() == []) # in normal operation, there should be no contratemporal refs X -> something newer than X (it means X needs updating) | |
| json =jsonpickle.encode(f, indent=4, keys=True) | |
| with open("fad.json", "w") as fo: | |
| fo.write(json) | |
| with open("fad.json", "r") as fin: | |
| fn = jsonpickle.decode(fin.read(), keys=True) | |
| assert(fn.get_plaintext_with_markup("point", show_id=False, show_properties=True, properties_to_show=['concepts']) == '<point/>A<point/><point concepts="important concept 2"/>123456789B123456789C123456789') | |