diff --git a/.gitattributes b/.gitattributes index ce225dc1a1249b70c23f7f265913b20d0d8ae3ac..c989d81b111366a12559261771d9fe9b88b3999a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -47,3 +47,29 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text .venv/lib/python3.11/site-packages/fontTools/misc/bezierTools.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text .venv/lib/python3.11/site-packages/fontTools/pens/momentsPen.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text .venv/lib/python3.11/site-packages/fontTools/qu2cu/qu2cu.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/fontTools/varLib/iup.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/fsspec/__pycache__/spec.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/_conv.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/_errors.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/_npystrings.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/_objects.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/_proxy.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/_selector.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/defs.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5a.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5ac.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5d.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5ds.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5f.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5fd.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5g.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5i.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5l.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5o.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5p.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5pl.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5r.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5s.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5t.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.11/site-packages/h5py/h5z.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/sfnt.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/sfnt.py new file mode 100644 index 0000000000000000000000000000000000000000..d0ed17ae2945fbf55dbbba9e897783d6e25b7cc8 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/sfnt.py @@ -0,0 +1,664 @@ +"""ttLib/sfnt.py -- low-level module to deal with the sfnt file format. + +Defines two public classes: + +- SFNTReader +- SFNTWriter + +(Normally you don't have to use these classes explicitly; they are +used automatically by ttLib.TTFont.) + +The reading and writing of sfnt files is separated in two distinct +classes, since whenever the number of tables changes or whenever +a table's length changes you need to rewrite the whole file anyway. +""" + +from __future__ import annotations + +from collections.abc import KeysView +from io import BytesIO +from types import SimpleNamespace +from fontTools.misc.textTools import Tag +from fontTools.misc import sstruct +from fontTools.ttLib import TTLibError, TTLibFileIsCollectionError +import struct +from collections import OrderedDict +import logging + + +log = logging.getLogger(__name__) + + +class SFNTReader(object): + def __new__(cls, *args, **kwargs): + """Return an instance of the SFNTReader sub-class which is compatible + with the input file type. + """ + if args and cls is SFNTReader: + infile = args[0] + infile.seek(0) + sfntVersion = Tag(infile.read(4)) + infile.seek(0) + if sfntVersion == "wOF2": + # return new WOFF2Reader object + from fontTools.ttLib.woff2 import WOFF2Reader + + return object.__new__(WOFF2Reader) + # return default object + return object.__new__(cls) + + def __init__(self, file, checkChecksums=0, fontNumber=-1): + self.file = file + self.checkChecksums = checkChecksums + + self.flavor = None + self.flavorData = None + self.DirectoryEntry = SFNTDirectoryEntry + self.file.seek(0) + self.sfntVersion = self.file.read(4) + self.file.seek(0) + if self.sfntVersion == b"ttcf": + header = readTTCHeader(self.file) + numFonts = header.numFonts + if not 0 <= fontNumber < numFonts: + raise TTLibFileIsCollectionError( + "specify a font number between 0 and %d (inclusive)" + % (numFonts - 1) + ) + self.numFonts = numFonts + self.file.seek(header.offsetTable[fontNumber]) + data = self.file.read(sfntDirectorySize) + if len(data) != sfntDirectorySize: + raise TTLibError("Not a Font Collection (not enough data)") + sstruct.unpack(sfntDirectoryFormat, data, self) + elif self.sfntVersion == b"wOFF": + self.flavor = "woff" + self.DirectoryEntry = WOFFDirectoryEntry + data = self.file.read(woffDirectorySize) + if len(data) != woffDirectorySize: + raise TTLibError("Not a WOFF font (not enough data)") + sstruct.unpack(woffDirectoryFormat, data, self) + else: + data = self.file.read(sfntDirectorySize) + if len(data) != sfntDirectorySize: + raise TTLibError("Not a TrueType or OpenType font (not enough data)") + sstruct.unpack(sfntDirectoryFormat, data, self) + self.sfntVersion = Tag(self.sfntVersion) + + if self.sfntVersion not in ("\x00\x01\x00\x00", "OTTO", "true"): + raise TTLibError("Not a TrueType or OpenType font (bad sfntVersion)") + tables: dict[Tag, DirectoryEntry] = {} + for i in range(self.numTables): + entry = self.DirectoryEntry() + entry.fromFile(self.file) + tag = Tag(entry.tag) + tables[tag] = entry + self.tables = OrderedDict(sorted(tables.items(), key=lambda i: i[1].offset)) + + # Load flavor data if any + if self.flavor == "woff": + self.flavorData = WOFFFlavorData(self) + + def has_key(self, tag: str | bytes) -> bool: + return tag in self.tables + + __contains__ = has_key + + def keys(self) -> KeysView[Tag]: + return self.tables.keys() + + def __getitem__(self, tag: str | bytes) -> bytes: + """Fetch the raw table data.""" + entry = self.tables[Tag(tag)] + data = entry.loadData(self.file) + if self.checkChecksums: + if tag == "head": + # Beh: we have to special-case the 'head' table. + checksum = calcChecksum(data[:8] + b"\0\0\0\0" + data[12:]) + else: + checksum = calcChecksum(data) + if self.checkChecksums > 1: + # Be obnoxious, and barf when it's wrong + assert checksum == entry.checkSum, "bad checksum for '%s' table" % tag + elif checksum != entry.checkSum: + # Be friendly, and just log a warning. + log.warning("bad checksum for '%s' table", tag) + return data + + def __delitem__(self, tag: str | bytes) -> None: + del self.tables[Tag(tag)] + + def close(self) -> None: + self.file.close() + + # We define custom __getstate__ and __setstate__ to make SFNTReader pickle-able + # and deepcopy-able. When a TTFont is loaded as lazy=True, SFNTReader holds a + # reference to an external file object which is not pickleable. So in __getstate__ + # we store the file name and current position, and in __setstate__ we reopen the + # same named file after unpickling. + + def __getstate__(self): + if isinstance(self.file, BytesIO): + # BytesIO is already pickleable, return the state unmodified + return self.__dict__ + + # remove unpickleable file attribute, and only store its name and pos + state = self.__dict__.copy() + del state["file"] + state["_filename"] = self.file.name + state["_filepos"] = self.file.tell() + return state + + def __setstate__(self, state): + if "file" not in state: + self.file = open(state.pop("_filename"), "rb") + self.file.seek(state.pop("_filepos")) + self.__dict__.update(state) + + +# default compression level for WOFF 1.0 tables and metadata +ZLIB_COMPRESSION_LEVEL = 6 + +# if set to True, use zopfli instead of zlib for compressing WOFF 1.0. +# The Python bindings are available at https://pypi.python.org/pypi/zopfli +USE_ZOPFLI = False + +# mapping between zlib's compression levels and zopfli's 'numiterations'. +# Use lower values for files over several MB in size or it will be too slow +ZOPFLI_LEVELS = { + # 0: 0, # can't do 0 iterations... + 1: 1, + 2: 3, + 3: 5, + 4: 8, + 5: 10, + 6: 15, + 7: 25, + 8: 50, + 9: 100, +} + + +def compress(data, level=ZLIB_COMPRESSION_LEVEL): + """Compress 'data' to Zlib format. If 'USE_ZOPFLI' variable is True, + zopfli is used instead of the zlib module. + The compression 'level' must be between 0 and 9. 1 gives best speed, + 9 gives best compression (0 gives no compression at all). + The default value is a compromise between speed and compression (6). + """ + if not (0 <= level <= 9): + raise ValueError("Bad compression level: %s" % level) + if not USE_ZOPFLI or level == 0: + from zlib import compress + + return compress(data, level) + else: + from zopfli.zlib import compress + + return compress(data, numiterations=ZOPFLI_LEVELS[level]) + + +class SFNTWriter(object): + def __new__(cls, *args, **kwargs): + """Return an instance of the SFNTWriter sub-class which is compatible + with the specified 'flavor'. + """ + flavor = None + if kwargs and "flavor" in kwargs: + flavor = kwargs["flavor"] + elif args and len(args) > 3: + flavor = args[3] + if cls is SFNTWriter: + if flavor == "woff2": + # return new WOFF2Writer object + from fontTools.ttLib.woff2 import WOFF2Writer + + return object.__new__(WOFF2Writer) + # return default object + return object.__new__(cls) + + def __init__( + self, + file, + numTables, + sfntVersion="\000\001\000\000", + flavor=None, + flavorData=None, + ): + self.file = file + self.numTables = numTables + self.sfntVersion = Tag(sfntVersion) + self.flavor = flavor + self.flavorData = flavorData + + if self.flavor == "woff": + self.directoryFormat = woffDirectoryFormat + self.directorySize = woffDirectorySize + self.DirectoryEntry = WOFFDirectoryEntry + + self.signature = "wOFF" + + # to calculate WOFF checksum adjustment, we also need the original SFNT offsets + self.origNextTableOffset = ( + sfntDirectorySize + numTables * sfntDirectoryEntrySize + ) + else: + assert not self.flavor, "Unknown flavor '%s'" % self.flavor + self.directoryFormat = sfntDirectoryFormat + self.directorySize = sfntDirectorySize + self.DirectoryEntry = SFNTDirectoryEntry + + from fontTools.ttLib import getSearchRange + + self.searchRange, self.entrySelector, self.rangeShift = getSearchRange( + numTables, 16 + ) + + self.directoryOffset = self.file.tell() + self.nextTableOffset = ( + self.directoryOffset + + self.directorySize + + numTables * self.DirectoryEntry.formatSize + ) + # clear out directory area + self.file.seek(self.nextTableOffset) + # make sure we're actually where we want to be. (old cStringIO bug) + self.file.write(b"\0" * (self.nextTableOffset - self.file.tell())) + self.tables = OrderedDict() + + def setEntry(self, tag, entry): + if tag in self.tables: + raise TTLibError("cannot rewrite '%s' table" % tag) + + self.tables[tag] = entry + + def __setitem__(self, tag, data): + """Write raw table data to disk.""" + if tag in self.tables: + raise TTLibError("cannot rewrite '%s' table" % tag) + + entry = self.DirectoryEntry() + entry.tag = tag + entry.offset = self.nextTableOffset + if tag == "head": + entry.checkSum = calcChecksum(data[:8] + b"\0\0\0\0" + data[12:]) + self.headTable = data + entry.uncompressed = True + else: + entry.checkSum = calcChecksum(data) + entry.saveData(self.file, data) + + if self.flavor == "woff": + entry.origOffset = self.origNextTableOffset + self.origNextTableOffset += (entry.origLength + 3) & ~3 + + self.nextTableOffset = self.nextTableOffset + ((entry.length + 3) & ~3) + # Add NUL bytes to pad the table data to a 4-byte boundary. + # Don't depend on f.seek() as we need to add the padding even if no + # subsequent write follows (seek is lazy), ie. after the final table + # in the font. + self.file.write(b"\0" * (self.nextTableOffset - self.file.tell())) + assert self.nextTableOffset == self.file.tell() + + self.setEntry(tag, entry) + + def __getitem__(self, tag): + return self.tables[tag] + + def close(self): + """All tables must have been written to disk. Now write the + directory. + """ + tables = sorted(self.tables.items()) + if len(tables) != self.numTables: + raise TTLibError( + "wrong number of tables; expected %d, found %d" + % (self.numTables, len(tables)) + ) + + if self.flavor == "woff": + self.signature = b"wOFF" + self.reserved = 0 + + self.totalSfntSize = 12 + self.totalSfntSize += 16 * len(tables) + for tag, entry in tables: + self.totalSfntSize += (entry.origLength + 3) & ~3 + + data = self.flavorData if self.flavorData else WOFFFlavorData() + if data.majorVersion is not None and data.minorVersion is not None: + self.majorVersion = data.majorVersion + self.minorVersion = data.minorVersion + else: + if hasattr(self, "headTable"): + self.majorVersion, self.minorVersion = struct.unpack( + ">HH", self.headTable[4:8] + ) + else: + self.majorVersion = self.minorVersion = 0 + if data.metaData: + self.metaOrigLength = len(data.metaData) + self.file.seek(0, 2) + self.metaOffset = self.file.tell() + compressedMetaData = compress(data.metaData) + self.metaLength = len(compressedMetaData) + self.file.write(compressedMetaData) + else: + self.metaOffset = self.metaLength = self.metaOrigLength = 0 + if data.privData: + self.file.seek(0, 2) + off = self.file.tell() + paddedOff = (off + 3) & ~3 + self.file.write(b"\0" * (paddedOff - off)) + self.privOffset = self.file.tell() + self.privLength = len(data.privData) + self.file.write(data.privData) + else: + self.privOffset = self.privLength = 0 + + self.file.seek(0, 2) + self.length = self.file.tell() + + else: + assert not self.flavor, "Unknown flavor '%s'" % self.flavor + pass + + directory = sstruct.pack(self.directoryFormat, self) + + self.file.seek(self.directoryOffset + self.directorySize) + seenHead = 0 + for tag, entry in tables: + if tag == "head": + seenHead = 1 + directory = directory + entry.toString() + if seenHead: + self.writeMasterChecksum(directory) + self.file.seek(self.directoryOffset) + self.file.write(directory) + + def _calcMasterChecksum(self, directory): + # calculate checkSumAdjustment + checksums = [] + for tag in self.tables.keys(): + checksums.append(self.tables[tag].checkSum) + + if self.DirectoryEntry != SFNTDirectoryEntry: + # Create a SFNT directory for checksum calculation purposes + from fontTools.ttLib import getSearchRange + + self.searchRange, self.entrySelector, self.rangeShift = getSearchRange( + self.numTables, 16 + ) + directory = sstruct.pack(sfntDirectoryFormat, self) + tables = sorted(self.tables.items()) + for tag, entry in tables: + sfntEntry = SFNTDirectoryEntry() + sfntEntry.tag = entry.tag + sfntEntry.checkSum = entry.checkSum + sfntEntry.offset = entry.origOffset + sfntEntry.length = entry.origLength + directory = directory + sfntEntry.toString() + + directory_end = sfntDirectorySize + len(self.tables) * sfntDirectoryEntrySize + assert directory_end == len(directory) + + checksums.append(calcChecksum(directory)) + checksum = sum(checksums) & 0xFFFFFFFF + # BiboAfba! + checksumadjustment = (0xB1B0AFBA - checksum) & 0xFFFFFFFF + return checksumadjustment + + def writeMasterChecksum(self, directory): + checksumadjustment = self._calcMasterChecksum(directory) + # write the checksum to the file + self.file.seek(self.tables["head"].offset + 8) + self.file.write(struct.pack(">L", checksumadjustment)) + + def reordersTables(self): + return False + + +# -- sfnt directory helpers and cruft + +ttcHeaderFormat = """ + > # big endian + TTCTag: 4s # "ttcf" + Version: L # 0x00010000 or 0x00020000 + numFonts: L # number of fonts + # OffsetTable[numFonts]: L # array with offsets from beginning of file + # ulDsigTag: L # version 2.0 only + # ulDsigLength: L # version 2.0 only + # ulDsigOffset: L # version 2.0 only +""" + +ttcHeaderSize = sstruct.calcsize(ttcHeaderFormat) + +sfntDirectoryFormat = """ + > # big endian + sfntVersion: 4s + numTables: H # number of tables + searchRange: H # (max2 <= numTables)*16 + entrySelector: H # log2(max2 <= numTables) + rangeShift: H # numTables*16-searchRange +""" + +sfntDirectorySize = sstruct.calcsize(sfntDirectoryFormat) + +sfntDirectoryEntryFormat = """ + > # big endian + tag: 4s + checkSum: L + offset: L + length: L +""" + +sfntDirectoryEntrySize = sstruct.calcsize(sfntDirectoryEntryFormat) + +woffDirectoryFormat = """ + > # big endian + signature: 4s # "wOFF" + sfntVersion: 4s + length: L # total woff file size + numTables: H # number of tables + reserved: H # set to 0 + totalSfntSize: L # uncompressed size + majorVersion: H # major version of WOFF file + minorVersion: H # minor version of WOFF file + metaOffset: L # offset to metadata block + metaLength: L # length of compressed metadata + metaOrigLength: L # length of uncompressed metadata + privOffset: L # offset to private data block + privLength: L # length of private data block +""" + +woffDirectorySize = sstruct.calcsize(woffDirectoryFormat) + +woffDirectoryEntryFormat = """ + > # big endian + tag: 4s + offset: L + length: L # compressed length + origLength: L # original length + checkSum: L # original checksum +""" + +woffDirectoryEntrySize = sstruct.calcsize(woffDirectoryEntryFormat) + + +class DirectoryEntry(object): + def __init__(self): + self.uncompressed = False # if True, always embed entry raw + + def fromFile(self, file): + sstruct.unpack(self.format, file.read(self.formatSize), self) + + def fromString(self, str): + sstruct.unpack(self.format, str, self) + + def toString(self): + return sstruct.pack(self.format, self) + + def __repr__(self): + if hasattr(self, "tag"): + return "<%s '%s' at %x>" % (self.__class__.__name__, self.tag, id(self)) + else: + return "<%s at %x>" % (self.__class__.__name__, id(self)) + + def loadData(self, file): + file.seek(self.offset) + data = file.read(self.length) + assert len(data) == self.length + if hasattr(self.__class__, "decodeData"): + data = self.decodeData(data) + return data + + def saveData(self, file, data): + if hasattr(self.__class__, "encodeData"): + data = self.encodeData(data) + self.length = len(data) + file.seek(self.offset) + file.write(data) + + def decodeData(self, rawData): + return rawData + + def encodeData(self, data): + return data + + +class SFNTDirectoryEntry(DirectoryEntry): + format = sfntDirectoryEntryFormat + formatSize = sfntDirectoryEntrySize + + +class WOFFDirectoryEntry(DirectoryEntry): + format = woffDirectoryEntryFormat + formatSize = woffDirectoryEntrySize + + def __init__(self): + super(WOFFDirectoryEntry, self).__init__() + # With fonttools<=3.1.2, the only way to set a different zlib + # compression level for WOFF directory entries was to set the class + # attribute 'zlibCompressionLevel'. This is now replaced by a globally + # defined `ZLIB_COMPRESSION_LEVEL`, which is also applied when + # compressing the metadata. For backward compatibility, we still + # use the class attribute if it was already set. + if not hasattr(WOFFDirectoryEntry, "zlibCompressionLevel"): + self.zlibCompressionLevel = ZLIB_COMPRESSION_LEVEL + + def decodeData(self, rawData): + import zlib + + if self.length == self.origLength: + data = rawData + else: + assert self.length < self.origLength + data = zlib.decompress(rawData) + assert len(data) == self.origLength + return data + + def encodeData(self, data): + self.origLength = len(data) + if not self.uncompressed: + compressedData = compress(data, self.zlibCompressionLevel) + if self.uncompressed or len(compressedData) >= self.origLength: + # Encode uncompressed + rawData = data + self.length = self.origLength + else: + rawData = compressedData + self.length = len(rawData) + return rawData + + +class WOFFFlavorData: + Flavor = "woff" + + def __init__(self, reader=None): + self.majorVersion = None + self.minorVersion = None + self.metaData = None + self.privData = None + if reader: + self.majorVersion = reader.majorVersion + self.minorVersion = reader.minorVersion + if reader.metaLength: + reader.file.seek(reader.metaOffset) + rawData = reader.file.read(reader.metaLength) + assert len(rawData) == reader.metaLength + data = self._decompress(rawData) + assert len(data) == reader.metaOrigLength + self.metaData = data + if reader.privLength: + reader.file.seek(reader.privOffset) + data = reader.file.read(reader.privLength) + assert len(data) == reader.privLength + self.privData = data + + def _decompress(self, rawData): + import zlib + + return zlib.decompress(rawData) + + +def calcChecksum(data): + """Calculate the checksum for an arbitrary block of data. + + If the data length is not a multiple of four, it assumes + it is to be padded with null byte. + + >>> print(calcChecksum(b"abcd")) + 1633837924 + >>> print(calcChecksum(b"abcdxyz")) + 3655064932 + """ + remainder = len(data) % 4 + if remainder: + data += b"\0" * (4 - remainder) + value = 0 + blockSize = 4096 + assert blockSize % 4 == 0 + for i in range(0, len(data), blockSize): + block = data[i : i + blockSize] + longs = struct.unpack(">%dL" % (len(block) // 4), block) + value = (value + sum(longs)) & 0xFFFFFFFF + return value + + +def readTTCHeader(file): + file.seek(0) + data = file.read(ttcHeaderSize) + if len(data) != ttcHeaderSize: + raise TTLibError("Not a Font Collection (not enough data)") + self = SimpleNamespace() + sstruct.unpack(ttcHeaderFormat, data, self) + if self.TTCTag != "ttcf": + raise TTLibError("Not a Font Collection") + assert self.Version == 0x00010000 or self.Version == 0x00020000, ( + "unrecognized TTC version 0x%08x" % self.Version + ) + self.offsetTable = struct.unpack( + ">%dL" % self.numFonts, file.read(self.numFonts * 4) + ) + if self.Version == 0x00020000: + pass # ignoring version 2.0 signatures + return self + + +def writeTTCHeader(file, numFonts): + self = SimpleNamespace() + self.TTCTag = "ttcf" + self.Version = 0x00010000 + self.numFonts = numFonts + file.seek(0) + file.write(sstruct.pack(ttcHeaderFormat, self)) + offset = file.tell() + file.write(struct.pack(">%dL" % self.numFonts, *([0] * self.numFonts))) + return offset + + +if __name__ == "__main__": + import sys + import doctest + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/standardGlyphOrder.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/standardGlyphOrder.py new file mode 100644 index 0000000000000000000000000000000000000000..4062385240096ac822814aebb8bf7c59cf003a8f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/standardGlyphOrder.py @@ -0,0 +1,271 @@ +# +# 'post' table formats 1.0 and 2.0 rely on this list of "standard" +# glyphs. +# +# My list is correct according to the Apple documentation for the 'post' table: +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6post.html +# (However, it seems that TTFdump (from MS) and FontLab disagree, at +# least with respect to the last glyph, which they list as 'dslash' +# instead of 'dcroat'.) +# + +standardGlyphOrder = [ + ".notdef", # 0 + ".null", # 1 + "nonmarkingreturn", # 2 + "space", # 3 + "exclam", # 4 + "quotedbl", # 5 + "numbersign", # 6 + "dollar", # 7 + "percent", # 8 + "ampersand", # 9 + "quotesingle", # 10 + "parenleft", # 11 + "parenright", # 12 + "asterisk", # 13 + "plus", # 14 + "comma", # 15 + "hyphen", # 16 + "period", # 17 + "slash", # 18 + "zero", # 19 + "one", # 20 + "two", # 21 + "three", # 22 + "four", # 23 + "five", # 24 + "six", # 25 + "seven", # 26 + "eight", # 27 + "nine", # 28 + "colon", # 29 + "semicolon", # 30 + "less", # 31 + "equal", # 32 + "greater", # 33 + "question", # 34 + "at", # 35 + "A", # 36 + "B", # 37 + "C", # 38 + "D", # 39 + "E", # 40 + "F", # 41 + "G", # 42 + "H", # 43 + "I", # 44 + "J", # 45 + "K", # 46 + "L", # 47 + "M", # 48 + "N", # 49 + "O", # 50 + "P", # 51 + "Q", # 52 + "R", # 53 + "S", # 54 + "T", # 55 + "U", # 56 + "V", # 57 + "W", # 58 + "X", # 59 + "Y", # 60 + "Z", # 61 + "bracketleft", # 62 + "backslash", # 63 + "bracketright", # 64 + "asciicircum", # 65 + "underscore", # 66 + "grave", # 67 + "a", # 68 + "b", # 69 + "c", # 70 + "d", # 71 + "e", # 72 + "f", # 73 + "g", # 74 + "h", # 75 + "i", # 76 + "j", # 77 + "k", # 78 + "l", # 79 + "m", # 80 + "n", # 81 + "o", # 82 + "p", # 83 + "q", # 84 + "r", # 85 + "s", # 86 + "t", # 87 + "u", # 88 + "v", # 89 + "w", # 90 + "x", # 91 + "y", # 92 + "z", # 93 + "braceleft", # 94 + "bar", # 95 + "braceright", # 96 + "asciitilde", # 97 + "Adieresis", # 98 + "Aring", # 99 + "Ccedilla", # 100 + "Eacute", # 101 + "Ntilde", # 102 + "Odieresis", # 103 + "Udieresis", # 104 + "aacute", # 105 + "agrave", # 106 + "acircumflex", # 107 + "adieresis", # 108 + "atilde", # 109 + "aring", # 110 + "ccedilla", # 111 + "eacute", # 112 + "egrave", # 113 + "ecircumflex", # 114 + "edieresis", # 115 + "iacute", # 116 + "igrave", # 117 + "icircumflex", # 118 + "idieresis", # 119 + "ntilde", # 120 + "oacute", # 121 + "ograve", # 122 + "ocircumflex", # 123 + "odieresis", # 124 + "otilde", # 125 + "uacute", # 126 + "ugrave", # 127 + "ucircumflex", # 128 + "udieresis", # 129 + "dagger", # 130 + "degree", # 131 + "cent", # 132 + "sterling", # 133 + "section", # 134 + "bullet", # 135 + "paragraph", # 136 + "germandbls", # 137 + "registered", # 138 + "copyright", # 139 + "trademark", # 140 + "acute", # 141 + "dieresis", # 142 + "notequal", # 143 + "AE", # 144 + "Oslash", # 145 + "infinity", # 146 + "plusminus", # 147 + "lessequal", # 148 + "greaterequal", # 149 + "yen", # 150 + "mu", # 151 + "partialdiff", # 152 + "summation", # 153 + "product", # 154 + "pi", # 155 + "integral", # 156 + "ordfeminine", # 157 + "ordmasculine", # 158 + "Omega", # 159 + "ae", # 160 + "oslash", # 161 + "questiondown", # 162 + "exclamdown", # 163 + "logicalnot", # 164 + "radical", # 165 + "florin", # 166 + "approxequal", # 167 + "Delta", # 168 + "guillemotleft", # 169 + "guillemotright", # 170 + "ellipsis", # 171 + "nonbreakingspace", # 172 + "Agrave", # 173 + "Atilde", # 174 + "Otilde", # 175 + "OE", # 176 + "oe", # 177 + "endash", # 178 + "emdash", # 179 + "quotedblleft", # 180 + "quotedblright", # 181 + "quoteleft", # 182 + "quoteright", # 183 + "divide", # 184 + "lozenge", # 185 + "ydieresis", # 186 + "Ydieresis", # 187 + "fraction", # 188 + "currency", # 189 + "guilsinglleft", # 190 + "guilsinglright", # 191 + "fi", # 192 + "fl", # 193 + "daggerdbl", # 194 + "periodcentered", # 195 + "quotesinglbase", # 196 + "quotedblbase", # 197 + "perthousand", # 198 + "Acircumflex", # 199 + "Ecircumflex", # 200 + "Aacute", # 201 + "Edieresis", # 202 + "Egrave", # 203 + "Iacute", # 204 + "Icircumflex", # 205 + "Idieresis", # 206 + "Igrave", # 207 + "Oacute", # 208 + "Ocircumflex", # 209 + "apple", # 210 + "Ograve", # 211 + "Uacute", # 212 + "Ucircumflex", # 213 + "Ugrave", # 214 + "dotlessi", # 215 + "circumflex", # 216 + "tilde", # 217 + "macron", # 218 + "breve", # 219 + "dotaccent", # 220 + "ring", # 221 + "cedilla", # 222 + "hungarumlaut", # 223 + "ogonek", # 224 + "caron", # 225 + "Lslash", # 226 + "lslash", # 227 + "Scaron", # 228 + "scaron", # 229 + "Zcaron", # 230 + "zcaron", # 231 + "brokenbar", # 232 + "Eth", # 233 + "eth", # 234 + "Yacute", # 235 + "yacute", # 236 + "Thorn", # 237 + "thorn", # 238 + "minus", # 239 + "multiply", # 240 + "onesuperior", # 241 + "twosuperior", # 242 + "threesuperior", # 243 + "onehalf", # 244 + "onequarter", # 245 + "threequarters", # 246 + "franc", # 247 + "Gbreve", # 248 + "gbreve", # 249 + "Idotaccent", # 250 + "Scedilla", # 251 + "scedilla", # 252 + "Cacute", # 253 + "cacute", # 254 + "Ccaron", # 255 + "ccaron", # 256 + "dcroat", # 257 +] diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/B_A_S_E_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/B_A_S_E_.py new file mode 100644 index 0000000000000000000000000000000000000000..0f4b1c3c5a7a3fe53de316dfd9b23fe5096eeb37 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/B_A_S_E_.py @@ -0,0 +1,14 @@ +from .otBase import BaseTTXConverter + + +class table_B_A_S_E_(BaseTTXConverter): + """Baseline table + + The ``BASE`` table contains information needed to align glyphs in + different scripts, from different fonts, or at different sizes + within the same line of text. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/base + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/BitmapGlyphMetrics.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/BitmapGlyphMetrics.py new file mode 100644 index 0000000000000000000000000000000000000000..10b4f828213b8320d54eefed3d5e66f2ba532101 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/BitmapGlyphMetrics.py @@ -0,0 +1,64 @@ +# Since bitmap glyph metrics are shared between EBLC and EBDT +# this class gets its own python file. +from fontTools.misc import sstruct +from fontTools.misc.textTools import safeEval +import logging + + +log = logging.getLogger(__name__) + +bigGlyphMetricsFormat = """ + > # big endian + height: B + width: B + horiBearingX: b + horiBearingY: b + horiAdvance: B + vertBearingX: b + vertBearingY: b + vertAdvance: B +""" + +smallGlyphMetricsFormat = """ + > # big endian + height: B + width: B + BearingX: b + BearingY: b + Advance: B +""" + + +class BitmapGlyphMetrics(object): + def toXML(self, writer, ttFont): + writer.begintag(self.__class__.__name__) + writer.newline() + for metricName in sstruct.getformat(self.__class__.binaryFormat)[1]: + writer.simpletag(metricName, value=getattr(self, metricName)) + writer.newline() + writer.endtag(self.__class__.__name__) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + metricNames = set(sstruct.getformat(self.__class__.binaryFormat)[1]) + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + # Make sure this is a metric that is needed by GlyphMetrics. + if name in metricNames: + vars(self)[name] = safeEval(attrs["value"]) + else: + log.warning( + "unknown name '%s' being ignored in %s.", + name, + self.__class__.__name__, + ) + + +class BigGlyphMetrics(BitmapGlyphMetrics): + binaryFormat = bigGlyphMetricsFormat + + +class SmallGlyphMetrics(BitmapGlyphMetrics): + binaryFormat = smallGlyphMetricsFormat diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_B_D_T_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_B_D_T_.py new file mode 100644 index 0000000000000000000000000000000000000000..0f9924745ebb4519be2d0cdcddadb86cc741cf8b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_B_D_T_.py @@ -0,0 +1,113 @@ +# Copyright 2013 Google, Inc. All Rights Reserved. +# +# Google Author(s): Matt Fontaine + + +from fontTools.misc.textTools import bytesjoin +from fontTools.misc import sstruct +from . import E_B_D_T_ +from .BitmapGlyphMetrics import ( + BigGlyphMetrics, + bigGlyphMetricsFormat, + SmallGlyphMetrics, + smallGlyphMetricsFormat, +) +from .E_B_D_T_ import ( + BitmapGlyph, + BitmapPlusSmallMetricsMixin, + BitmapPlusBigMetricsMixin, +) +import struct + + +class table_C_B_D_T_(E_B_D_T_.table_E_B_D_T_): + """Color Bitmap Data table + + The ``CBDT`` table contains color bitmap data for glyphs. It must + be used in concert with the ``CBLC`` table. + + It is backwards-compatible with the monochrome/grayscale ``EBDT`` table. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/cbdt + """ + + # Change the data locator table being referenced. + locatorName = "CBLC" + + # Modify the format class accessor for color bitmap use. + def getImageFormatClass(self, imageFormat): + try: + return E_B_D_T_.table_E_B_D_T_.getImageFormatClass(self, imageFormat) + except KeyError: + return cbdt_bitmap_classes[imageFormat] + + +# Helper method for removing export features not supported by color bitmaps. +# Write data in the parent class will default to raw if an option is unsupported. +def _removeUnsupportedForColor(dataFunctions): + dataFunctions = dict(dataFunctions) + del dataFunctions["row"] + return dataFunctions + + +class ColorBitmapGlyph(BitmapGlyph): + fileExtension = ".png" + xmlDataFunctions = _removeUnsupportedForColor(BitmapGlyph.xmlDataFunctions) + + +class cbdt_bitmap_format_17(BitmapPlusSmallMetricsMixin, ColorBitmapGlyph): + def decompile(self): + self.metrics = SmallGlyphMetrics() + dummy, data = sstruct.unpack2(smallGlyphMetricsFormat, self.data, self.metrics) + (dataLen,) = struct.unpack(">L", data[:4]) + data = data[4:] + + # For the image data cut it to the size specified by dataLen. + assert dataLen <= len(data), "Data overun in format 17" + self.imageData = data[:dataLen] + + def compile(self, ttFont): + dataList = [] + dataList.append(sstruct.pack(smallGlyphMetricsFormat, self.metrics)) + dataList.append(struct.pack(">L", len(self.imageData))) + dataList.append(self.imageData) + return bytesjoin(dataList) + + +class cbdt_bitmap_format_18(BitmapPlusBigMetricsMixin, ColorBitmapGlyph): + def decompile(self): + self.metrics = BigGlyphMetrics() + dummy, data = sstruct.unpack2(bigGlyphMetricsFormat, self.data, self.metrics) + (dataLen,) = struct.unpack(">L", data[:4]) + data = data[4:] + + # For the image data cut it to the size specified by dataLen. + assert dataLen <= len(data), "Data overun in format 18" + self.imageData = data[:dataLen] + + def compile(self, ttFont): + dataList = [] + dataList.append(sstruct.pack(bigGlyphMetricsFormat, self.metrics)) + dataList.append(struct.pack(">L", len(self.imageData))) + dataList.append(self.imageData) + return bytesjoin(dataList) + + +class cbdt_bitmap_format_19(ColorBitmapGlyph): + def decompile(self): + (dataLen,) = struct.unpack(">L", self.data[:4]) + data = self.data[4:] + + assert dataLen <= len(data), "Data overun in format 19" + self.imageData = data[:dataLen] + + def compile(self, ttFont): + return struct.pack(">L", len(self.imageData)) + self.imageData + + +# Dict for CBDT extended formats. +cbdt_bitmap_classes = { + 17: cbdt_bitmap_format_17, + 18: cbdt_bitmap_format_18, + 19: cbdt_bitmap_format_19, +} diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_B_L_C_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_B_L_C_.py new file mode 100644 index 0000000000000000000000000000000000000000..0673fb7ed2215f481f303c5b98c9941accd916bf --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_B_L_C_.py @@ -0,0 +1,19 @@ +# Copyright 2013 Google, Inc. All Rights Reserved. +# +# Google Author(s): Matt Fontaine + +from . import E_B_L_C_ + + +class table_C_B_L_C_(E_B_L_C_.table_E_B_L_C_): + """Color Bitmap Location table + + The ``CBLC`` table contains the locations of color bitmaps for glyphs. It must + be used in concert with the ``CBDT`` table. + + It is backwards-compatible with the monochrome/grayscale ``EBLC`` table. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/cblc + """ + + dependencies = ["CBDT"] diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_F_F_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_F_F_.py new file mode 100644 index 0000000000000000000000000000000000000000..3d974ced88e9d9a1e9c0494c4d0eac996ec315b8 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_F_F_.py @@ -0,0 +1,61 @@ +from io import BytesIO +from fontTools import cffLib +from . import DefaultTable + + +class table_C_F_F_(DefaultTable.DefaultTable): + """Compact Font Format table (version 1) + + The ``CFF`` table embeds a CFF-formatted font. The CFF font format + predates OpenType and could be used as a standalone font file, but the + ``CFF`` table is also used to package CFF fonts into an OpenType + container. + + .. note:: + ``CFF`` has been succeeded by ``CFF2``, which eliminates much of + the redundancy incurred by embedding CFF version 1 in an OpenType + font. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/cff + """ + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.cff = cffLib.CFFFontSet() + self._gaveGlyphOrder = False + + def decompile(self, data, otFont): + self.cff.decompile(BytesIO(data), otFont, isCFF2=False) + assert len(self.cff) == 1, "can't deal with multi-font CFF tables." + + def compile(self, otFont): + f = BytesIO() + self.cff.compile(f, otFont, isCFF2=False) + return f.getvalue() + + def haveGlyphNames(self): + if hasattr(self.cff[self.cff.fontNames[0]], "ROS"): + return False # CID-keyed font + else: + return True + + def getGlyphOrder(self): + if self._gaveGlyphOrder: + from fontTools import ttLib + + raise ttLib.TTLibError("illegal use of getGlyphOrder()") + self._gaveGlyphOrder = True + return self.cff[self.cff.fontNames[0]].getGlyphOrder() + + def setGlyphOrder(self, glyphOrder): + pass + # XXX + # self.cff[self.cff.fontNames[0]].setGlyphOrder(glyphOrder) + + def toXML(self, writer, otFont): + self.cff.toXML(writer) + + def fromXML(self, name, attrs, content, otFont): + if not hasattr(self, "cff"): + self.cff = cffLib.CFFFontSet() + self.cff.fromXML(name, attrs, content, otFont) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_F_F__2.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_F_F__2.py new file mode 100644 index 0000000000000000000000000000000000000000..ff07682d24f0399b6579f529ae70cf4fd87abf21 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_F_F__2.py @@ -0,0 +1,26 @@ +from io import BytesIO +from fontTools.ttLib.tables.C_F_F_ import table_C_F_F_ + + +class table_C_F_F__2(table_C_F_F_): + """Compact Font Format version 2 table + + The ``CFF2`` table contains glyph data for a CFF2-flavored OpenType + font. + + .. note:: + ``CFF2`` is the successor to ``CFF``, and eliminates much of + the redundancy incurred by embedding CFF version 1 in an OpenType + font. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/cff2 + """ + + def decompile(self, data, otFont): + self.cff.decompile(BytesIO(data), otFont, isCFF2=True) + assert len(self.cff) == 1, "can't deal with multi-font CFF tables." + + def compile(self, otFont): + f = BytesIO() + self.cff.compile(f, otFont, isCFF2=True) + return f.getvalue() diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_O_L_R_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_O_L_R_.py new file mode 100644 index 0000000000000000000000000000000000000000..266a11c41dffc5eed42ee446c0268373104cbf7a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_O_L_R_.py @@ -0,0 +1,165 @@ +# Copyright 2013 Google, Inc. All Rights Reserved. +# +# Google Author(s): Behdad Esfahbod + +from fontTools.misc.textTools import safeEval +from . import DefaultTable + + +class table_C_O_L_R_(DefaultTable.DefaultTable): + """Color table + + The ``COLR`` table defines color presentation of outline glyphs. It must + be used in concert with the ``CPAL`` table, which contains the color + descriptors used. + + This table is structured so that you can treat it like a dictionary keyed by glyph name. + + ``ttFont['COLR'][]`` will return the color layers for any glyph. + + ``ttFont['COLR'][] = `` will set the color layers for any glyph. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/colr + """ + + @staticmethod + def _decompileColorLayersV0(table): + if not table.LayerRecordArray: + return {} + colorLayerLists = {} + layerRecords = table.LayerRecordArray.LayerRecord + numLayerRecords = len(layerRecords) + for baseRec in table.BaseGlyphRecordArray.BaseGlyphRecord: + baseGlyph = baseRec.BaseGlyph + firstLayerIndex = baseRec.FirstLayerIndex + numLayers = baseRec.NumLayers + assert firstLayerIndex + numLayers <= numLayerRecords + layers = [] + for i in range(firstLayerIndex, firstLayerIndex + numLayers): + layerRec = layerRecords[i] + layers.append(LayerRecord(layerRec.LayerGlyph, layerRec.PaletteIndex)) + colorLayerLists[baseGlyph] = layers + return colorLayerLists + + def _toOTTable(self, ttFont): + from . import otTables + from fontTools.colorLib.builder import populateCOLRv0 + + tableClass = getattr(otTables, self.tableTag) + table = tableClass() + table.Version = self.version + + populateCOLRv0( + table, + { + baseGlyph: [(layer.name, layer.colorID) for layer in layers] + for baseGlyph, layers in self.ColorLayers.items() + }, + glyphMap=ttFont.getReverseGlyphMap(rebuild=True), + ) + return table + + def decompile(self, data, ttFont): + from .otBase import OTTableReader + from . import otTables + + # We use otData to decompile, but we adapt the decompiled otTables to the + # existing COLR v0 API for backward compatibility. + reader = OTTableReader(data, tableTag=self.tableTag) + tableClass = getattr(otTables, self.tableTag) + table = tableClass() + table.decompile(reader, ttFont) + + self.version = table.Version + if self.version == 0: + self.ColorLayers = self._decompileColorLayersV0(table) + else: + # for new versions, keep the raw otTables around + self.table = table + + def compile(self, ttFont): + from .otBase import OTTableWriter + + if hasattr(self, "table"): + table = self.table + else: + table = self._toOTTable(ttFont) + + writer = OTTableWriter(tableTag=self.tableTag) + table.compile(writer, ttFont) + return writer.getAllData() + + def toXML(self, writer, ttFont): + if hasattr(self, "table"): + self.table.toXML2(writer, ttFont) + else: + writer.simpletag("version", value=self.version) + writer.newline() + for baseGlyph in sorted(self.ColorLayers.keys(), key=ttFont.getGlyphID): + writer.begintag("ColorGlyph", name=baseGlyph) + writer.newline() + for layer in self.ColorLayers[baseGlyph]: + layer.toXML(writer, ttFont) + writer.endtag("ColorGlyph") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": # old COLR v0 API + setattr(self, name, safeEval(attrs["value"])) + elif name == "ColorGlyph": + if not hasattr(self, "ColorLayers"): + self.ColorLayers = {} + glyphName = attrs["name"] + for element in content: + if isinstance(element, str): + continue + layers = [] + for element in content: + if isinstance(element, str): + continue + layer = LayerRecord() + layer.fromXML(element[0], element[1], element[2], ttFont) + layers.append(layer) + self.ColorLayers[glyphName] = layers + else: # new COLR v1 API + from . import otTables + + if not hasattr(self, "table"): + tableClass = getattr(otTables, self.tableTag) + self.table = tableClass() + self.table.fromXML(name, attrs, content, ttFont) + self.table.populateDefaults() + self.version = self.table.Version + + def __getitem__(self, glyphName): + if not isinstance(glyphName, str): + raise TypeError(f"expected str, found {type(glyphName).__name__}") + return self.ColorLayers[glyphName] + + def __setitem__(self, glyphName, value): + if not isinstance(glyphName, str): + raise TypeError(f"expected str, found {type(glyphName).__name__}") + if value is not None: + self.ColorLayers[glyphName] = value + elif glyphName in self.ColorLayers: + del self.ColorLayers[glyphName] + + def __delitem__(self, glyphName): + del self.ColorLayers[glyphName] + + +class LayerRecord(object): + def __init__(self, name=None, colorID=None): + self.name = name + self.colorID = colorID + + def toXML(self, writer, ttFont): + writer.simpletag("layer", name=self.name, colorID=self.colorID) + writer.newline() + + def fromXML(self, eltname, attrs, content, ttFont): + for name, value in attrs.items(): + if name == "name": + setattr(self, name, value) + else: + setattr(self, name, safeEval(value)) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_P_A_L_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_P_A_L_.py new file mode 100644 index 0000000000000000000000000000000000000000..5ec8c843c516f2ac31bbbc9ff2dd0166a3a0e159 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/C_P_A_L_.py @@ -0,0 +1,305 @@ +# Copyright 2013 Google, Inc. All Rights Reserved. +# +# Google Author(s): Behdad Esfahbod + +from fontTools.misc.textTools import bytesjoin, safeEval +from . import DefaultTable +import array +from collections import namedtuple +import struct +import sys + + +class table_C_P_A_L_(DefaultTable.DefaultTable): + """Color Palette table + + The ``CPAL`` table contains a set of one or more color palettes. The color + records in each palette can be referenced by the ``COLR`` table to specify + the colors used in a color glyph. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/cpal + """ + + NO_NAME_ID = 0xFFFF + DEFAULT_PALETTE_TYPE = 0 + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.palettes = [] + self.paletteTypes = [] + self.paletteLabels = [] + self.paletteEntryLabels = [] + + def decompile(self, data, ttFont): + ( + self.version, + self.numPaletteEntries, + numPalettes, + numColorRecords, + goffsetFirstColorRecord, + ) = struct.unpack(">HHHHL", data[:12]) + assert ( + self.version <= 1 + ), "Version of CPAL table is higher than I know how to handle" + self.palettes = [] + pos = 12 + for i in range(numPalettes): + startIndex = struct.unpack(">H", data[pos : pos + 2])[0] + assert startIndex + self.numPaletteEntries <= numColorRecords + pos += 2 + palette = [] + ppos = goffsetFirstColorRecord + startIndex * 4 + for j in range(self.numPaletteEntries): + palette.append(Color(*struct.unpack(">BBBB", data[ppos : ppos + 4]))) + ppos += 4 + self.palettes.append(palette) + if self.version == 0: + offsetToPaletteTypeArray = 0 + offsetToPaletteLabelArray = 0 + offsetToPaletteEntryLabelArray = 0 + else: + pos = 12 + numPalettes * 2 + ( + offsetToPaletteTypeArray, + offsetToPaletteLabelArray, + offsetToPaletteEntryLabelArray, + ) = struct.unpack(">LLL", data[pos : pos + 12]) + self.paletteTypes = self._decompileUInt32Array( + data, + offsetToPaletteTypeArray, + numPalettes, + default=self.DEFAULT_PALETTE_TYPE, + ) + self.paletteLabels = self._decompileUInt16Array( + data, offsetToPaletteLabelArray, numPalettes, default=self.NO_NAME_ID + ) + self.paletteEntryLabels = self._decompileUInt16Array( + data, + offsetToPaletteEntryLabelArray, + self.numPaletteEntries, + default=self.NO_NAME_ID, + ) + + def _decompileUInt16Array(self, data, offset, numElements, default=0): + if offset == 0: + return [default] * numElements + result = array.array("H", data[offset : offset + 2 * numElements]) + if sys.byteorder != "big": + result.byteswap() + assert len(result) == numElements, result + return result.tolist() + + def _decompileUInt32Array(self, data, offset, numElements, default=0): + if offset == 0: + return [default] * numElements + result = array.array("I", data[offset : offset + 4 * numElements]) + if sys.byteorder != "big": + result.byteswap() + assert len(result) == numElements, result + return result.tolist() + + def compile(self, ttFont): + colorRecordIndices, colorRecords = self._compileColorRecords() + paletteTypes = self._compilePaletteTypes() + paletteLabels = self._compilePaletteLabels() + paletteEntryLabels = self._compilePaletteEntryLabels() + numColorRecords = len(colorRecords) // 4 + offsetToFirstColorRecord = 12 + len(colorRecordIndices) + if self.version >= 1: + offsetToFirstColorRecord += 12 + header = struct.pack( + ">HHHHL", + self.version, + self.numPaletteEntries, + len(self.palettes), + numColorRecords, + offsetToFirstColorRecord, + ) + if self.version == 0: + dataList = [header, colorRecordIndices, colorRecords] + else: + pos = offsetToFirstColorRecord + len(colorRecords) + if len(paletteTypes) == 0: + offsetToPaletteTypeArray = 0 + else: + offsetToPaletteTypeArray = pos + pos += len(paletteTypes) + if len(paletteLabels) == 0: + offsetToPaletteLabelArray = 0 + else: + offsetToPaletteLabelArray = pos + pos += len(paletteLabels) + if len(paletteEntryLabels) == 0: + offsetToPaletteEntryLabelArray = 0 + else: + offsetToPaletteEntryLabelArray = pos + pos += len(paletteLabels) + header1 = struct.pack( + ">LLL", + offsetToPaletteTypeArray, + offsetToPaletteLabelArray, + offsetToPaletteEntryLabelArray, + ) + dataList = [ + header, + colorRecordIndices, + header1, + colorRecords, + paletteTypes, + paletteLabels, + paletteEntryLabels, + ] + return bytesjoin(dataList) + + def _compilePalette(self, palette): + assert len(palette) == self.numPaletteEntries + pack = lambda c: struct.pack(">BBBB", c.blue, c.green, c.red, c.alpha) + return bytesjoin([pack(color) for color in palette]) + + def _compileColorRecords(self): + colorRecords, colorRecordIndices, pool = [], [], {} + for palette in self.palettes: + packedPalette = self._compilePalette(palette) + if packedPalette in pool: + index = pool[packedPalette] + else: + index = len(colorRecords) + colorRecords.append(packedPalette) + pool[packedPalette] = index + colorRecordIndices.append(struct.pack(">H", index * self.numPaletteEntries)) + return bytesjoin(colorRecordIndices), bytesjoin(colorRecords) + + def _compilePaletteTypes(self): + if self.version == 0 or not any(self.paletteTypes): + return b"" + assert len(self.paletteTypes) == len(self.palettes) + result = bytesjoin([struct.pack(">I", ptype) for ptype in self.paletteTypes]) + assert len(result) == 4 * len(self.palettes) + return result + + def _compilePaletteLabels(self): + if self.version == 0 or all(l == self.NO_NAME_ID for l in self.paletteLabels): + return b"" + assert len(self.paletteLabels) == len(self.palettes) + result = bytesjoin([struct.pack(">H", label) for label in self.paletteLabels]) + assert len(result) == 2 * len(self.palettes) + return result + + def _compilePaletteEntryLabels(self): + if self.version == 0 or all( + l == self.NO_NAME_ID for l in self.paletteEntryLabels + ): + return b"" + assert len(self.paletteEntryLabels) == self.numPaletteEntries + result = bytesjoin( + [struct.pack(">H", label) for label in self.paletteEntryLabels] + ) + assert len(result) == 2 * self.numPaletteEntries + return result + + def toXML(self, writer, ttFont): + numPalettes = len(self.palettes) + paletteLabels = {i: nameID for (i, nameID) in enumerate(self.paletteLabels)} + paletteTypes = {i: typ for (i, typ) in enumerate(self.paletteTypes)} + writer.simpletag("version", value=self.version) + writer.newline() + writer.simpletag("numPaletteEntries", value=self.numPaletteEntries) + writer.newline() + for index, palette in enumerate(self.palettes): + attrs = {"index": index} + paletteType = paletteTypes.get(index, self.DEFAULT_PALETTE_TYPE) + paletteLabel = paletteLabels.get(index, self.NO_NAME_ID) + if self.version > 0 and paletteLabel != self.NO_NAME_ID: + attrs["label"] = paletteLabel + if self.version > 0 and paletteType != self.DEFAULT_PALETTE_TYPE: + attrs["type"] = paletteType + writer.begintag("palette", **attrs) + writer.newline() + if ( + self.version > 0 + and paletteLabel != self.NO_NAME_ID + and ttFont + and "name" in ttFont + ): + name = ttFont["name"].getDebugName(paletteLabel) + if name is not None: + writer.comment(name) + writer.newline() + assert len(palette) == self.numPaletteEntries + for cindex, color in enumerate(palette): + color.toXML(writer, ttFont, cindex) + writer.endtag("palette") + writer.newline() + if self.version > 0 and not all( + l == self.NO_NAME_ID for l in self.paletteEntryLabels + ): + writer.begintag("paletteEntryLabels") + writer.newline() + for index, label in enumerate(self.paletteEntryLabels): + if label != self.NO_NAME_ID: + writer.simpletag("label", index=index, value=label) + if self.version > 0 and label and ttFont and "name" in ttFont: + name = ttFont["name"].getDebugName(label) + if name is not None: + writer.comment(name) + writer.newline() + writer.endtag("paletteEntryLabels") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "palette": + self.paletteLabels.append(int(attrs.get("label", self.NO_NAME_ID))) + self.paletteTypes.append(int(attrs.get("type", self.DEFAULT_PALETTE_TYPE))) + palette = [] + for element in content: + if isinstance(element, str): + continue + attrs = element[1] + color = Color.fromHex(attrs["value"]) + palette.append(color) + self.palettes.append(palette) + elif name == "paletteEntryLabels": + colorLabels = {} + for element in content: + if isinstance(element, str): + continue + elementName, elementAttr, _ = element + if elementName == "label": + labelIndex = safeEval(elementAttr["index"]) + nameID = safeEval(elementAttr["value"]) + colorLabels[labelIndex] = nameID + self.paletteEntryLabels = [ + colorLabels.get(i, self.NO_NAME_ID) + for i in range(self.numPaletteEntries) + ] + elif "value" in attrs: + value = safeEval(attrs["value"]) + setattr(self, name, value) + if name == "numPaletteEntries": + self.paletteEntryLabels = [self.NO_NAME_ID] * self.numPaletteEntries + + +class Color(namedtuple("Color", "blue green red alpha")): + def hex(self): + return "#%02X%02X%02X%02X" % (self.red, self.green, self.blue, self.alpha) + + def __repr__(self): + return self.hex() + + def toXML(self, writer, ttFont, index=None): + writer.simpletag("color", value=self.hex(), index=index) + writer.newline() + + @classmethod + def fromHex(cls, value): + if value[0] == "#": + value = value[1:] + red = int(value[0:2], 16) + green = int(value[2:4], 16) + blue = int(value[4:6], 16) + alpha = int(value[6:8], 16) if len(value) >= 8 else 0xFF + return cls(red=red, green=green, blue=blue, alpha=alpha) + + @classmethod + def fromRGBA(cls, red, green, blue, alpha): + return cls(red=red, green=green, blue=blue, alpha=alpha) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/D_S_I_G_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/D_S_I_G_.py new file mode 100644 index 0000000000000000000000000000000000000000..deb7d58d92e0e26664aad32fac6c48ea10a0afc6 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/D_S_I_G_.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import base64 +from typing import TYPE_CHECKING + +from fontTools.misc import sstruct +from fontTools.misc.textTools import bytesjoin, safeEval, strjoin, tobytes, tostr + +from . import DefaultTable + +if TYPE_CHECKING: + from fontTools.misc.xmlWriter import XMLWriter + from fontTools.ttLib import TTFont + +DSIG_HeaderFormat = """ + > # big endian + ulVersion: L + usNumSigs: H + usFlag: H +""" +# followed by an array of usNumSigs DSIG_Signature records +DSIG_SignatureFormat = """ + > # big endian + ulFormat: L + ulLength: L # length includes DSIG_SignatureBlock header + ulOffset: L +""" +# followed by an array of usNumSigs DSIG_SignatureBlock records, +# each followed immediately by the pkcs7 bytes +DSIG_SignatureBlockFormat = """ + > # big endian + usReserved1: H + usReserved2: H + cbSignature: l # length of following raw pkcs7 data +""" + +# +# NOTE +# the DSIG table format allows for SignatureBlocks residing +# anywhere in the table and possibly in a different order as +# listed in the array after the first table header +# +# this implementation does not keep track of any gaps and/or data +# before or after the actual signature blocks while decompiling, +# and puts them in the same physical order as listed in the header +# on compilation with no padding whatsoever. +# + + +class table_D_S_I_G_(DefaultTable.DefaultTable): + """Digital Signature table + + The ``DSIG`` table contains cryptographic signatures for the font. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/dsig + """ + + def decompile(self, data: bytes, ttFont: TTFont) -> None: + dummy, newData = sstruct.unpack2(DSIG_HeaderFormat, data, self) + assert self.ulVersion == 1, "DSIG ulVersion must be 1" + assert self.usFlag & ~1 == 0, "DSIG usFlag must be 0x1 or 0x0" + self.signatureRecords: list[SignatureRecord] = [] + sigrecs = self.signatureRecords + for n in range(self.usNumSigs): + sigrec, newData = sstruct.unpack2( + DSIG_SignatureFormat, newData, SignatureRecord() + ) + assert sigrec.ulFormat == 1, ( + "DSIG signature record #%d ulFormat must be 1" % n + ) + sigrecs.append(sigrec) + for sigrec in sigrecs: + dummy, newData = sstruct.unpack2( + DSIG_SignatureBlockFormat, data[sigrec.ulOffset :], sigrec + ) + assert sigrec.usReserved1 == 0, ( + "DSIG signature record #%d usReserverd1 must be 0" % n + ) + assert sigrec.usReserved2 == 0, ( + "DSIG signature record #%d usReserverd2 must be 0" % n + ) + sigrec.pkcs7 = newData[: sigrec.cbSignature] + + def compile(self, ttFont: TTFont) -> bytes: + packed = sstruct.pack(DSIG_HeaderFormat, self) + headers = [packed] + offset = len(packed) + self.usNumSigs * sstruct.calcsize(DSIG_SignatureFormat) + data = [] + for sigrec in self.signatureRecords: + # first pack signature block + sigrec.cbSignature = len(sigrec.pkcs7) + packed = sstruct.pack(DSIG_SignatureBlockFormat, sigrec) + sigrec.pkcs7 + data.append(packed) + # update redundant length field + sigrec.ulLength = len(packed) + # update running table offset + sigrec.ulOffset = offset + headers.append(sstruct.pack(DSIG_SignatureFormat, sigrec)) + offset += sigrec.ulLength + if offset % 2: + # Pad to even bytes + data.append(b"\0") + return bytesjoin(headers + data) + + def toXML(self, xmlWriter: XMLWriter, ttFont: TTFont) -> None: + xmlWriter.comment( + "note that the Digital Signature will be invalid after recompilation!" + ) + xmlWriter.newline() + xmlWriter.simpletag( + "tableHeader", + version=self.ulVersion, + numSigs=self.usNumSigs, + flag="0x%X" % self.usFlag, + ) + for sigrec in self.signatureRecords: + xmlWriter.newline() + sigrec.toXML(xmlWriter, ttFont) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "tableHeader": + self.signatureRecords = [] + self.ulVersion = safeEval(attrs["version"]) + self.usNumSigs = safeEval(attrs["numSigs"]) + self.usFlag = safeEval(attrs["flag"]) + return + if name == "SignatureRecord": + sigrec = SignatureRecord() + sigrec.fromXML(name, attrs, content, ttFont) + self.signatureRecords.append(sigrec) + + +pem_spam = lambda l, spam={ + "-----BEGIN PKCS7-----": True, + "-----END PKCS7-----": True, + "": True, +}: not spam.get(l.strip()) + + +def b64encode(b): + s = base64.b64encode(b) + # Line-break at 76 chars. + items = [] + while s: + items.append(tostr(s[:76])) + items.append("\n") + s = s[76:] + return strjoin(items) + + +class SignatureRecord: + def __repr__(self) -> str: + return "<%s: %s>" % (self.__class__.__name__, self.__dict__) + + def toXML(self, writer, ttFont: TTFont) -> None: + writer.begintag(self.__class__.__name__, format=self.ulFormat) + writer.newline() + writer.write_noindent("-----BEGIN PKCS7-----\n") + writer.write_noindent(b64encode(self.pkcs7)) + writer.write_noindent("-----END PKCS7-----\n") + writer.endtag(self.__class__.__name__) + + def fromXML( + self, name: str, attrs: dict[str, str], content: str, ttFont: TTFont + ) -> None: + self.ulFormat: int = safeEval(attrs["format"]) + self.usReserved1: int = safeEval(attrs.get("reserved1", "0")) + self.usReserved2: int = safeEval(attrs.get("reserved2", "0")) + self.pkcs7 = base64.b64decode(tobytes(strjoin(filter(pem_spam, content)))) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/D__e_b_g.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/D__e_b_g.py new file mode 100644 index 0000000000000000000000000000000000000000..cb1653d79d4f99ad729b25c96346459c40a4681e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/D__e_b_g.py @@ -0,0 +1,35 @@ +import json +from textwrap import indent + +from . import DefaultTable +from fontTools.misc.textTools import tostr + + +class table_D__e_b_g(DefaultTable.DefaultTable): + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.data = {} + + def decompile(self, data, ttFont): + self.data = json.loads(data) + + def compile(self, ttFont): + return json.dumps(self.data).encode("utf-8") + + def toXML(self, writer, ttFont): + # make sure json indentation inside CDATA block matches XMLWriter's + data = json.dumps(self.data, indent=len(writer.indentwhite)) + prefix = tostr(writer.indentwhite) * (writer.indentlevel + 1) + # but don't indent the first json line so it's adjacent to ` None: + if tag is None: + tag = getClassTag(self.__class__) + self.tableTag = Tag(tag) + + def decompile(self, data: bytes, ttFont: TTFont) -> None: + self.data = data + + def compile(self, ttFont: TTFont) -> bytes: + return self.data + + def toXML( + self, writer: XMLWriter, ttFont: TTFont, **kwargs: dict[str, Any] + ) -> None: + if hasattr(self, "ERROR"): + writer.comment("An error occurred during the decompilation of this table") + writer.newline() + writer.comment(self.ERROR) + writer.newline() + writer.begintag("hexdata") + writer.newline() + writer.dumphex(self.compile(ttFont)) + writer.endtag("hexdata") + writer.newline() + + def fromXML( + self, name: str, attrs: dict[str, str], content: str, ttFont: TTFont + ) -> None: + from fontTools import ttLib + from fontTools.misc.textTools import readHex + + if name != "hexdata": + raise ttLib.TTLibError("can't handle '%s' element" % name) + self.decompile(readHex(content), ttFont) + + def __repr__(self) -> str: + return "<'%s' table at %x>" % (self.tableTag, id(self)) + + def __eq__(self, other: Any) -> bool: + if type(self) != type(other): + return NotImplemented + return self.__dict__ == other.__dict__ + + def __ne__(self, other: Any) -> bool: + result = self.__eq__(other) + return result if result is NotImplemented else not result diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/E_B_D_T_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/E_B_D_T_.py new file mode 100644 index 0000000000000000000000000000000000000000..ea23e30f83d38f77587f29cf40b3eec41f715be7 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/E_B_D_T_.py @@ -0,0 +1,835 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import ( + bytechr, + byteord, + bytesjoin, + strjoin, + safeEval, + readHex, + hexStr, + deHexStr, +) +from .BitmapGlyphMetrics import ( + BigGlyphMetrics, + bigGlyphMetricsFormat, + SmallGlyphMetrics, + smallGlyphMetricsFormat, +) +from . import DefaultTable +import itertools +import os +import struct +import logging + + +log = logging.getLogger(__name__) + +ebdtTableVersionFormat = """ + > # big endian + version: 16.16F +""" + +ebdtComponentFormat = """ + > # big endian + glyphCode: H + xOffset: b + yOffset: b +""" + + +class table_E_B_D_T_(DefaultTable.DefaultTable): + """Embedded Bitmap Data table + + The ``EBDT`` table contains monochrome or grayscale bitmap data for + glyphs. It must be used in concert with the ``EBLC`` table. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/ebdt + """ + + # Keep a reference to the name of the data locator table. + locatorName = "EBLC" + + # This method can be overridden in subclasses to support new formats + # without changing the other implementation. Also can be used as a + # convenience method for coverting a font file to an alternative format. + def getImageFormatClass(self, imageFormat): + return ebdt_bitmap_classes[imageFormat] + + def decompile(self, data, ttFont): + # Get the version but don't advance the slice. + # Most of the lookup for this table is done relative + # to the begining so slice by the offsets provided + # in the EBLC table. + sstruct.unpack2(ebdtTableVersionFormat, data, self) + + # Keep a dict of glyphs that have been seen so they aren't remade. + # This dict maps intervals of data to the BitmapGlyph. + glyphDict = {} + + # Pull out the EBLC table and loop through glyphs. + # A strike is a concept that spans both tables. + # The actual bitmap data is stored in the EBDT. + locator = ttFont[self.__class__.locatorName] + self.strikeData = [] + for curStrike in locator.strikes: + bitmapGlyphDict = {} + self.strikeData.append(bitmapGlyphDict) + for indexSubTable in curStrike.indexSubTables: + dataIter = zip(indexSubTable.names, indexSubTable.locations) + for curName, curLoc in dataIter: + # Don't create duplicate data entries for the same glyphs. + # Instead just use the structures that already exist if they exist. + if curLoc in glyphDict: + curGlyph = glyphDict[curLoc] + else: + curGlyphData = data[slice(*curLoc)] + imageFormatClass = self.getImageFormatClass( + indexSubTable.imageFormat + ) + curGlyph = imageFormatClass(curGlyphData, ttFont) + glyphDict[curLoc] = curGlyph + bitmapGlyphDict[curName] = curGlyph + + def compile(self, ttFont): + dataList = [] + dataList.append(sstruct.pack(ebdtTableVersionFormat, self)) + dataSize = len(dataList[0]) + + # Keep a dict of glyphs that have been seen so they aren't remade. + # This dict maps the id of the BitmapGlyph to the interval + # in the data. + glyphDict = {} + + # Go through the bitmap glyph data. Just in case the data for a glyph + # changed the size metrics should be recalculated. There are a variety + # of formats and they get stored in the EBLC table. That is why + # recalculation is defered to the EblcIndexSubTable class and just + # pass what is known about bitmap glyphs from this particular table. + locator = ttFont[self.__class__.locatorName] + for curStrike, curGlyphDict in zip(locator.strikes, self.strikeData): + for curIndexSubTable in curStrike.indexSubTables: + dataLocations = [] + for curName in curIndexSubTable.names: + # Handle the data placement based on seeing the glyph or not. + # Just save a reference to the location if the glyph has already + # been saved in compile. This code assumes that glyphs will only + # be referenced multiple times from indexFormat5. By luck the + # code may still work when referencing poorly ordered fonts with + # duplicate references. If there is a font that is unlucky the + # respective compile methods for the indexSubTables will fail + # their assertions. All fonts seem to follow this assumption. + # More complicated packing may be needed if a counter-font exists. + glyph = curGlyphDict[curName] + objectId = id(glyph) + if objectId not in glyphDict: + data = glyph.compile(ttFont) + data = curIndexSubTable.padBitmapData(data) + startByte = dataSize + dataSize += len(data) + endByte = dataSize + dataList.append(data) + dataLoc = (startByte, endByte) + glyphDict[objectId] = dataLoc + else: + dataLoc = glyphDict[objectId] + dataLocations.append(dataLoc) + # Just use the new data locations in the indexSubTable. + # The respective compile implementations will take care + # of any of the problems in the convertion that may arise. + curIndexSubTable.locations = dataLocations + + return bytesjoin(dataList) + + def toXML(self, writer, ttFont): + # When exporting to XML if one of the data export formats + # requires metrics then those metrics may be in the locator. + # In this case populate the bitmaps with "export metrics". + if ttFont.bitmapGlyphDataFormat in ("row", "bitwise"): + locator = ttFont[self.__class__.locatorName] + for curStrike, curGlyphDict in zip(locator.strikes, self.strikeData): + for curIndexSubTable in curStrike.indexSubTables: + for curName in curIndexSubTable.names: + glyph = curGlyphDict[curName] + # I'm not sure which metrics have priority here. + # For now if both metrics exist go with glyph metrics. + if hasattr(glyph, "metrics"): + glyph.exportMetrics = glyph.metrics + else: + glyph.exportMetrics = curIndexSubTable.metrics + glyph.exportBitDepth = curStrike.bitmapSizeTable.bitDepth + + writer.simpletag("header", [("version", self.version)]) + writer.newline() + locator = ttFont[self.__class__.locatorName] + for strikeIndex, bitmapGlyphDict in enumerate(self.strikeData): + writer.begintag("strikedata", [("index", strikeIndex)]) + writer.newline() + for curName, curBitmap in bitmapGlyphDict.items(): + curBitmap.toXML(strikeIndex, curName, writer, ttFont) + writer.endtag("strikedata") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "header": + self.version = safeEval(attrs["version"]) + elif name == "strikedata": + if not hasattr(self, "strikeData"): + self.strikeData = [] + strikeIndex = safeEval(attrs["index"]) + + bitmapGlyphDict = {} + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name[4:].startswith(_bitmapGlyphSubclassPrefix[4:]): + imageFormat = safeEval(name[len(_bitmapGlyphSubclassPrefix) :]) + glyphName = attrs["name"] + imageFormatClass = self.getImageFormatClass(imageFormat) + curGlyph = imageFormatClass(None, None) + curGlyph.fromXML(name, attrs, content, ttFont) + assert glyphName not in bitmapGlyphDict, ( + "Duplicate glyphs with the same name '%s' in the same strike." + % glyphName + ) + bitmapGlyphDict[glyphName] = curGlyph + else: + log.warning("%s being ignored by %s", name, self.__class__.__name__) + + # Grow the strike data array to the appropriate size. The XML + # format allows the strike index value to be out of order. + if strikeIndex >= len(self.strikeData): + self.strikeData += [None] * (strikeIndex + 1 - len(self.strikeData)) + assert ( + self.strikeData[strikeIndex] is None + ), "Duplicate strike EBDT indices." + self.strikeData[strikeIndex] = bitmapGlyphDict + + +class EbdtComponent(object): + def toXML(self, writer, ttFont): + writer.begintag("ebdtComponent", [("name", self.name)]) + writer.newline() + for componentName in sstruct.getformat(ebdtComponentFormat)[1][1:]: + writer.simpletag(componentName, value=getattr(self, componentName)) + writer.newline() + writer.endtag("ebdtComponent") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.name = attrs["name"] + componentNames = set(sstruct.getformat(ebdtComponentFormat)[1][1:]) + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name in componentNames: + vars(self)[name] = safeEval(attrs["value"]) + else: + log.warning("unknown name '%s' being ignored by EbdtComponent.", name) + + +# Helper functions for dealing with binary. + + +def _data2binary(data, numBits): + binaryList = [] + for curByte in data: + value = byteord(curByte) + numBitsCut = min(8, numBits) + for i in range(numBitsCut): + if value & 0x1: + binaryList.append("1") + else: + binaryList.append("0") + value = value >> 1 + numBits -= numBitsCut + return strjoin(binaryList) + + +def _binary2data(binary): + byteList = [] + for bitLoc in range(0, len(binary), 8): + byteString = binary[bitLoc : bitLoc + 8] + curByte = 0 + for curBit in reversed(byteString): + curByte = curByte << 1 + if curBit == "1": + curByte |= 1 + byteList.append(bytechr(curByte)) + return bytesjoin(byteList) + + +def _memoize(f): + class memodict(dict): + def __missing__(self, key): + ret = f(key) + if isinstance(key, int) or len(key) == 1: + self[key] = ret + return ret + + return memodict().__getitem__ + + +# 00100111 -> 11100100 per byte, not to be confused with little/big endian. +# Bitmap data per byte is in the order that binary is written on the page +# with the least significant bit as far right as possible. This is the +# opposite of what makes sense algorithmically and hence this function. +@_memoize +def _reverseBytes(data): + r""" + >>> bin(ord(_reverseBytes(0b00100111))) + '0b11100100' + >>> _reverseBytes(b'\x00\xf0') + b'\x00\x0f' + """ + if isinstance(data, bytes) and len(data) != 1: + return bytesjoin(map(_reverseBytes, data)) + byte = byteord(data) + result = 0 + for i in range(8): + result = result << 1 + result |= byte & 1 + byte = byte >> 1 + return bytechr(result) + + +# This section of code is for reading and writing image data to/from XML. + + +def _writeRawImageData(strikeIndex, glyphName, bitmapObject, writer, ttFont): + writer.begintag("rawimagedata") + writer.newline() + writer.dumphex(bitmapObject.imageData) + writer.endtag("rawimagedata") + writer.newline() + + +def _readRawImageData(bitmapObject, name, attrs, content, ttFont): + bitmapObject.imageData = readHex(content) + + +def _writeRowImageData(strikeIndex, glyphName, bitmapObject, writer, ttFont): + metrics = bitmapObject.exportMetrics + del bitmapObject.exportMetrics + bitDepth = bitmapObject.exportBitDepth + del bitmapObject.exportBitDepth + + writer.begintag( + "rowimagedata", bitDepth=bitDepth, width=metrics.width, height=metrics.height + ) + writer.newline() + for curRow in range(metrics.height): + rowData = bitmapObject.getRow(curRow, bitDepth=bitDepth, metrics=metrics) + writer.simpletag("row", value=hexStr(rowData)) + writer.newline() + writer.endtag("rowimagedata") + writer.newline() + + +def _readRowImageData(bitmapObject, name, attrs, content, ttFont): + bitDepth = safeEval(attrs["bitDepth"]) + metrics = SmallGlyphMetrics() + metrics.width = safeEval(attrs["width"]) + metrics.height = safeEval(attrs["height"]) + + dataRows = [] + for element in content: + if not isinstance(element, tuple): + continue + name, attr, content = element + # Chop off 'imagedata' from the tag to get just the option. + if name == "row": + dataRows.append(deHexStr(attr["value"])) + bitmapObject.setRows(dataRows, bitDepth=bitDepth, metrics=metrics) + + +def _writeBitwiseImageData(strikeIndex, glyphName, bitmapObject, writer, ttFont): + metrics = bitmapObject.exportMetrics + del bitmapObject.exportMetrics + bitDepth = bitmapObject.exportBitDepth + del bitmapObject.exportBitDepth + + # A dict for mapping binary to more readable/artistic ASCII characters. + binaryConv = {"0": ".", "1": "@"} + + writer.begintag( + "bitwiseimagedata", + bitDepth=bitDepth, + width=metrics.width, + height=metrics.height, + ) + writer.newline() + for curRow in range(metrics.height): + rowData = bitmapObject.getRow( + curRow, bitDepth=1, metrics=metrics, reverseBytes=True + ) + rowData = _data2binary(rowData, metrics.width) + # Make the output a readable ASCII art form. + rowData = strjoin(map(binaryConv.get, rowData)) + writer.simpletag("row", value=rowData) + writer.newline() + writer.endtag("bitwiseimagedata") + writer.newline() + + +def _readBitwiseImageData(bitmapObject, name, attrs, content, ttFont): + bitDepth = safeEval(attrs["bitDepth"]) + metrics = SmallGlyphMetrics() + metrics.width = safeEval(attrs["width"]) + metrics.height = safeEval(attrs["height"]) + + # A dict for mapping from ASCII to binary. All characters are considered + # a '1' except space, period and '0' which maps to '0'. + binaryConv = {" ": "0", ".": "0", "0": "0"} + + dataRows = [] + for element in content: + if not isinstance(element, tuple): + continue + name, attr, content = element + if name == "row": + mapParams = zip(attr["value"], itertools.repeat("1")) + rowData = strjoin(itertools.starmap(binaryConv.get, mapParams)) + dataRows.append(_binary2data(rowData)) + + bitmapObject.setRows( + dataRows, bitDepth=bitDepth, metrics=metrics, reverseBytes=True + ) + + +def _writeExtFileImageData(strikeIndex, glyphName, bitmapObject, writer, ttFont): + try: + folder = os.path.dirname(writer.file.name) + except AttributeError: + # fall back to current directory if output file's directory isn't found + folder = "." + folder = os.path.join(folder, "bitmaps") + filename = glyphName + bitmapObject.fileExtension + if not os.path.isdir(folder): + os.makedirs(folder) + folder = os.path.join(folder, "strike%d" % strikeIndex) + if not os.path.isdir(folder): + os.makedirs(folder) + + fullPath = os.path.join(folder, filename) + writer.simpletag("extfileimagedata", value=fullPath) + writer.newline() + + with open(fullPath, "wb") as file: + file.write(bitmapObject.imageData) + + +def _readExtFileImageData(bitmapObject, name, attrs, content, ttFont): + fullPath = attrs["value"] + with open(fullPath, "rb") as file: + bitmapObject.imageData = file.read() + + +# End of XML writing code. + +# Important information about the naming scheme. Used for identifying formats +# in XML. +_bitmapGlyphSubclassPrefix = "ebdt_bitmap_format_" + + +class BitmapGlyph(object): + # For the external file format. This can be changed in subclasses. This way + # when the extfile option is turned on files have the form: glyphName.ext + # The default is just a flat binary file with no meaning. + fileExtension = ".bin" + + # Keep track of reading and writing of various forms. + xmlDataFunctions = { + "raw": (_writeRawImageData, _readRawImageData), + "row": (_writeRowImageData, _readRowImageData), + "bitwise": (_writeBitwiseImageData, _readBitwiseImageData), + "extfile": (_writeExtFileImageData, _readExtFileImageData), + } + + def __init__(self, data, ttFont): + self.data = data + self.ttFont = ttFont + # TODO Currently non-lazy decompilation is untested here... + # if not ttFont.lazy: + # self.decompile() + # del self.data + + def __getattr__(self, attr): + # Allow lazy decompile. + if attr[:2] == "__": + raise AttributeError(attr) + if attr == "data": + raise AttributeError(attr) + self.decompile() + del self.data + return getattr(self, attr) + + def ensureDecompiled(self, recurse=False): + if hasattr(self, "data"): + self.decompile() + del self.data + + # Not a fan of this but it is needed for safer safety checking. + def getFormat(self): + return safeEval(self.__class__.__name__[len(_bitmapGlyphSubclassPrefix) :]) + + def toXML(self, strikeIndex, glyphName, writer, ttFont): + writer.begintag(self.__class__.__name__, [("name", glyphName)]) + writer.newline() + + self.writeMetrics(writer, ttFont) + # Use the internal write method to write using the correct output format. + self.writeData(strikeIndex, glyphName, writer, ttFont) + + writer.endtag(self.__class__.__name__) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.readMetrics(name, attrs, content, ttFont) + for element in content: + if not isinstance(element, tuple): + continue + name, attr, content = element + if not name.endswith("imagedata"): + continue + # Chop off 'imagedata' from the tag to get just the option. + option = name[: -len("imagedata")] + assert option in self.__class__.xmlDataFunctions + self.readData(name, attr, content, ttFont) + + # Some of the glyphs have the metrics. This allows for metrics to be + # added if the glyph format has them. Default behavior is to do nothing. + def writeMetrics(self, writer, ttFont): + pass + + # The opposite of write metrics. + def readMetrics(self, name, attrs, content, ttFont): + pass + + def writeData(self, strikeIndex, glyphName, writer, ttFont): + try: + writeFunc, readFunc = self.__class__.xmlDataFunctions[ + ttFont.bitmapGlyphDataFormat + ] + except KeyError: + writeFunc = _writeRawImageData + writeFunc(strikeIndex, glyphName, self, writer, ttFont) + + def readData(self, name, attrs, content, ttFont): + # Chop off 'imagedata' from the tag to get just the option. + option = name[: -len("imagedata")] + writeFunc, readFunc = self.__class__.xmlDataFunctions[option] + readFunc(self, name, attrs, content, ttFont) + + +# A closure for creating a mixin for the two types of metrics handling. +# Most of the code is very similar so its easier to deal with here. +# Everything works just by passing the class that the mixin is for. +def _createBitmapPlusMetricsMixin(metricsClass): + # Both metrics names are listed here to make meaningful error messages. + metricStrings = [BigGlyphMetrics.__name__, SmallGlyphMetrics.__name__] + curMetricsName = metricsClass.__name__ + # Find which metrics this is for and determine the opposite name. + metricsId = metricStrings.index(curMetricsName) + oppositeMetricsName = metricStrings[1 - metricsId] + + class BitmapPlusMetricsMixin(object): + def writeMetrics(self, writer, ttFont): + self.metrics.toXML(writer, ttFont) + + def readMetrics(self, name, attrs, content, ttFont): + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name == curMetricsName: + self.metrics = metricsClass() + self.metrics.fromXML(name, attrs, content, ttFont) + elif name == oppositeMetricsName: + log.warning( + "Warning: %s being ignored in format %d.", + oppositeMetricsName, + self.getFormat(), + ) + + return BitmapPlusMetricsMixin + + +# Since there are only two types of mixin's just create them here. +BitmapPlusBigMetricsMixin = _createBitmapPlusMetricsMixin(BigGlyphMetrics) +BitmapPlusSmallMetricsMixin = _createBitmapPlusMetricsMixin(SmallGlyphMetrics) + + +# Data that is bit aligned can be tricky to deal with. These classes implement +# helper functionality for dealing with the data and getting a particular row +# of bitwise data. Also helps implement fancy data export/import in XML. +class BitAlignedBitmapMixin(object): + def _getBitRange(self, row, bitDepth, metrics): + rowBits = bitDepth * metrics.width + bitOffset = row * rowBits + return (bitOffset, bitOffset + rowBits) + + def getRow(self, row, bitDepth=1, metrics=None, reverseBytes=False): + if metrics is None: + metrics = self.metrics + assert 0 <= row and row < metrics.height, "Illegal row access in bitmap" + + # Loop through each byte. This can cover two bytes in the original data or + # a single byte if things happen to be aligned. The very last entry might + # not be aligned so take care to trim the binary data to size and pad with + # zeros in the row data. Bit aligned data is somewhat tricky. + # + # Example of data cut. Data cut represented in x's. + # '|' represents byte boundary. + # data = ...0XX|XXXXXX00|000... => XXXXXXXX + # or + # data = ...0XX|XXXX0000|000... => XXXXXX00 + # or + # data = ...000|XXXXXXXX|000... => XXXXXXXX + # or + # data = ...000|00XXXX00|000... => XXXX0000 + # + dataList = [] + bitRange = self._getBitRange(row, bitDepth, metrics) + stepRange = bitRange + (8,) + for curBit in range(*stepRange): + endBit = min(curBit + 8, bitRange[1]) + numBits = endBit - curBit + cutPoint = curBit % 8 + firstByteLoc = curBit // 8 + secondByteLoc = endBit // 8 + if firstByteLoc < secondByteLoc: + numBitsCut = 8 - cutPoint + else: + numBitsCut = endBit - curBit + curByte = _reverseBytes(self.imageData[firstByteLoc]) + firstHalf = byteord(curByte) >> cutPoint + firstHalf = ((1 << numBitsCut) - 1) & firstHalf + newByte = firstHalf + if firstByteLoc < secondByteLoc and secondByteLoc < len(self.imageData): + curByte = _reverseBytes(self.imageData[secondByteLoc]) + secondHalf = byteord(curByte) << numBitsCut + newByte = (firstHalf | secondHalf) & ((1 << numBits) - 1) + dataList.append(bytechr(newByte)) + + # The way the data is kept is opposite the algorithm used. + data = bytesjoin(dataList) + if not reverseBytes: + data = _reverseBytes(data) + return data + + def setRows(self, dataRows, bitDepth=1, metrics=None, reverseBytes=False): + if metrics is None: + metrics = self.metrics + if not reverseBytes: + dataRows = list(map(_reverseBytes, dataRows)) + + # Keep track of a list of ordinal values as they are easier to modify + # than a list of strings. Map to actual strings later. + numBytes = (self._getBitRange(len(dataRows), bitDepth, metrics)[0] + 7) // 8 + ordDataList = [0] * numBytes + for row, data in enumerate(dataRows): + bitRange = self._getBitRange(row, bitDepth, metrics) + stepRange = bitRange + (8,) + for curBit, curByte in zip(range(*stepRange), data): + endBit = min(curBit + 8, bitRange[1]) + cutPoint = curBit % 8 + firstByteLoc = curBit // 8 + secondByteLoc = endBit // 8 + if firstByteLoc < secondByteLoc: + numBitsCut = 8 - cutPoint + else: + numBitsCut = endBit - curBit + curByte = byteord(curByte) + firstByte = curByte & ((1 << numBitsCut) - 1) + ordDataList[firstByteLoc] |= firstByte << cutPoint + if firstByteLoc < secondByteLoc and secondByteLoc < numBytes: + secondByte = (curByte >> numBitsCut) & ((1 << 8 - numBitsCut) - 1) + ordDataList[secondByteLoc] |= secondByte + + # Save the image data with the bits going the correct way. + self.imageData = _reverseBytes(bytesjoin(map(bytechr, ordDataList))) + + +class ByteAlignedBitmapMixin(object): + def _getByteRange(self, row, bitDepth, metrics): + rowBytes = (bitDepth * metrics.width + 7) // 8 + byteOffset = row * rowBytes + return (byteOffset, byteOffset + rowBytes) + + def getRow(self, row, bitDepth=1, metrics=None, reverseBytes=False): + if metrics is None: + metrics = self.metrics + assert 0 <= row and row < metrics.height, "Illegal row access in bitmap" + byteRange = self._getByteRange(row, bitDepth, metrics) + data = self.imageData[slice(*byteRange)] + if reverseBytes: + data = _reverseBytes(data) + return data + + def setRows(self, dataRows, bitDepth=1, metrics=None, reverseBytes=False): + if metrics is None: + metrics = self.metrics + if reverseBytes: + dataRows = map(_reverseBytes, dataRows) + self.imageData = bytesjoin(dataRows) + + +class ebdt_bitmap_format_1( + ByteAlignedBitmapMixin, BitmapPlusSmallMetricsMixin, BitmapGlyph +): + def decompile(self): + self.metrics = SmallGlyphMetrics() + dummy, data = sstruct.unpack2(smallGlyphMetricsFormat, self.data, self.metrics) + self.imageData = data + + def compile(self, ttFont): + data = sstruct.pack(smallGlyphMetricsFormat, self.metrics) + return data + self.imageData + + +class ebdt_bitmap_format_2( + BitAlignedBitmapMixin, BitmapPlusSmallMetricsMixin, BitmapGlyph +): + def decompile(self): + self.metrics = SmallGlyphMetrics() + dummy, data = sstruct.unpack2(smallGlyphMetricsFormat, self.data, self.metrics) + self.imageData = data + + def compile(self, ttFont): + data = sstruct.pack(smallGlyphMetricsFormat, self.metrics) + return data + self.imageData + + +class ebdt_bitmap_format_5(BitAlignedBitmapMixin, BitmapGlyph): + def decompile(self): + self.imageData = self.data + + def compile(self, ttFont): + return self.imageData + + +class ebdt_bitmap_format_6( + ByteAlignedBitmapMixin, BitmapPlusBigMetricsMixin, BitmapGlyph +): + def decompile(self): + self.metrics = BigGlyphMetrics() + dummy, data = sstruct.unpack2(bigGlyphMetricsFormat, self.data, self.metrics) + self.imageData = data + + def compile(self, ttFont): + data = sstruct.pack(bigGlyphMetricsFormat, self.metrics) + return data + self.imageData + + +class ebdt_bitmap_format_7( + BitAlignedBitmapMixin, BitmapPlusBigMetricsMixin, BitmapGlyph +): + def decompile(self): + self.metrics = BigGlyphMetrics() + dummy, data = sstruct.unpack2(bigGlyphMetricsFormat, self.data, self.metrics) + self.imageData = data + + def compile(self, ttFont): + data = sstruct.pack(bigGlyphMetricsFormat, self.metrics) + return data + self.imageData + + +class ComponentBitmapGlyph(BitmapGlyph): + def toXML(self, strikeIndex, glyphName, writer, ttFont): + writer.begintag(self.__class__.__name__, [("name", glyphName)]) + writer.newline() + + self.writeMetrics(writer, ttFont) + + writer.begintag("components") + writer.newline() + for curComponent in self.componentArray: + curComponent.toXML(writer, ttFont) + writer.endtag("components") + writer.newline() + + writer.endtag(self.__class__.__name__) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.readMetrics(name, attrs, content, ttFont) + for element in content: + if not isinstance(element, tuple): + continue + name, attr, content = element + if name == "components": + self.componentArray = [] + for compElement in content: + if not isinstance(compElement, tuple): + continue + name, attrs, content = compElement + if name == "ebdtComponent": + curComponent = EbdtComponent() + curComponent.fromXML(name, attrs, content, ttFont) + self.componentArray.append(curComponent) + else: + log.warning("'%s' being ignored in component array.", name) + + +class ebdt_bitmap_format_8(BitmapPlusSmallMetricsMixin, ComponentBitmapGlyph): + def decompile(self): + self.metrics = SmallGlyphMetrics() + dummy, data = sstruct.unpack2(smallGlyphMetricsFormat, self.data, self.metrics) + data = data[1:] + + (numComponents,) = struct.unpack(">H", data[:2]) + data = data[2:] + self.componentArray = [] + for i in range(numComponents): + curComponent = EbdtComponent() + dummy, data = sstruct.unpack2(ebdtComponentFormat, data, curComponent) + curComponent.name = self.ttFont.getGlyphName(curComponent.glyphCode) + self.componentArray.append(curComponent) + + def compile(self, ttFont): + dataList = [] + dataList.append(sstruct.pack(smallGlyphMetricsFormat, self.metrics)) + dataList.append(b"\0") + dataList.append(struct.pack(">H", len(self.componentArray))) + for curComponent in self.componentArray: + curComponent.glyphCode = ttFont.getGlyphID(curComponent.name) + dataList.append(sstruct.pack(ebdtComponentFormat, curComponent)) + return bytesjoin(dataList) + + +class ebdt_bitmap_format_9(BitmapPlusBigMetricsMixin, ComponentBitmapGlyph): + def decompile(self): + self.metrics = BigGlyphMetrics() + dummy, data = sstruct.unpack2(bigGlyphMetricsFormat, self.data, self.metrics) + (numComponents,) = struct.unpack(">H", data[:2]) + data = data[2:] + self.componentArray = [] + for i in range(numComponents): + curComponent = EbdtComponent() + dummy, data = sstruct.unpack2(ebdtComponentFormat, data, curComponent) + curComponent.name = self.ttFont.getGlyphName(curComponent.glyphCode) + self.componentArray.append(curComponent) + + def compile(self, ttFont): + dataList = [] + dataList.append(sstruct.pack(bigGlyphMetricsFormat, self.metrics)) + dataList.append(struct.pack(">H", len(self.componentArray))) + for curComponent in self.componentArray: + curComponent.glyphCode = ttFont.getGlyphID(curComponent.name) + dataList.append(sstruct.pack(ebdtComponentFormat, curComponent)) + return bytesjoin(dataList) + + +# Dictionary of bitmap formats to the class representing that format +# currently only the ones listed in this map are the ones supported. +ebdt_bitmap_classes = { + 1: ebdt_bitmap_format_1, + 2: ebdt_bitmap_format_2, + 5: ebdt_bitmap_format_5, + 6: ebdt_bitmap_format_6, + 7: ebdt_bitmap_format_7, + 8: ebdt_bitmap_format_8, + 9: ebdt_bitmap_format_9, +} diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/E_B_L_C_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/E_B_L_C_.py new file mode 100644 index 0000000000000000000000000000000000000000..ff93aa8cf67de480419dbe66b7a4702114cddfd9 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/E_B_L_C_.py @@ -0,0 +1,718 @@ +from fontTools.misc import sstruct +from . import DefaultTable +from fontTools.misc.textTools import bytesjoin, safeEval +from .BitmapGlyphMetrics import ( + BigGlyphMetrics, + bigGlyphMetricsFormat, + SmallGlyphMetrics, + smallGlyphMetricsFormat, +) +import struct +import itertools +from collections import deque +import logging + + +log = logging.getLogger(__name__) + +eblcHeaderFormat = """ + > # big endian + version: 16.16F + numSizes: I +""" +# The table format string is split to handle sbitLineMetrics simply. +bitmapSizeTableFormatPart1 = """ + > # big endian + indexSubTableArrayOffset: I + indexTablesSize: I + numberOfIndexSubTables: I + colorRef: I +""" +# The compound type for hori and vert. +sbitLineMetricsFormat = """ + > # big endian + ascender: b + descender: b + widthMax: B + caretSlopeNumerator: b + caretSlopeDenominator: b + caretOffset: b + minOriginSB: b + minAdvanceSB: b + maxBeforeBL: b + minAfterBL: b + pad1: b + pad2: b +""" +# hori and vert go between the two parts. +bitmapSizeTableFormatPart2 = """ + > # big endian + startGlyphIndex: H + endGlyphIndex: H + ppemX: B + ppemY: B + bitDepth: B + flags: b +""" + +indexSubTableArrayFormat = ">HHL" +indexSubTableArraySize = struct.calcsize(indexSubTableArrayFormat) + +indexSubHeaderFormat = ">HHL" +indexSubHeaderSize = struct.calcsize(indexSubHeaderFormat) + +codeOffsetPairFormat = ">HH" +codeOffsetPairSize = struct.calcsize(codeOffsetPairFormat) + + +class table_E_B_L_C_(DefaultTable.DefaultTable): + """Embedded Bitmap Location table + + The ``EBLC`` table contains the locations of monochrome or grayscale + bitmaps for glyphs. It must be used in concert with the ``EBDT`` table. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/eblc + """ + + dependencies = ["EBDT"] + + # This method can be overridden in subclasses to support new formats + # without changing the other implementation. Also can be used as a + # convenience method for coverting a font file to an alternative format. + def getIndexFormatClass(self, indexFormat): + return eblc_sub_table_classes[indexFormat] + + def decompile(self, data, ttFont): + # Save the original data because offsets are from the start of the table. + origData = data + i = 0 + + dummy = sstruct.unpack(eblcHeaderFormat, data[:8], self) + i += 8 + + self.strikes = [] + for curStrikeIndex in range(self.numSizes): + curStrike = Strike() + self.strikes.append(curStrike) + curTable = curStrike.bitmapSizeTable + dummy = sstruct.unpack2( + bitmapSizeTableFormatPart1, data[i : i + 16], curTable + ) + i += 16 + for metric in ("hori", "vert"): + metricObj = SbitLineMetrics() + vars(curTable)[metric] = metricObj + dummy = sstruct.unpack2( + sbitLineMetricsFormat, data[i : i + 12], metricObj + ) + i += 12 + dummy = sstruct.unpack( + bitmapSizeTableFormatPart2, data[i : i + 8], curTable + ) + i += 8 + + for curStrike in self.strikes: + curTable = curStrike.bitmapSizeTable + for subtableIndex in range(curTable.numberOfIndexSubTables): + i = ( + curTable.indexSubTableArrayOffset + + subtableIndex * indexSubTableArraySize + ) + + tup = struct.unpack( + indexSubTableArrayFormat, data[i : i + indexSubTableArraySize] + ) + (firstGlyphIndex, lastGlyphIndex, additionalOffsetToIndexSubtable) = tup + i = curTable.indexSubTableArrayOffset + additionalOffsetToIndexSubtable + + tup = struct.unpack( + indexSubHeaderFormat, data[i : i + indexSubHeaderSize] + ) + (indexFormat, imageFormat, imageDataOffset) = tup + + indexFormatClass = self.getIndexFormatClass(indexFormat) + indexSubTable = indexFormatClass(data[i + indexSubHeaderSize :], ttFont) + indexSubTable.firstGlyphIndex = firstGlyphIndex + indexSubTable.lastGlyphIndex = lastGlyphIndex + indexSubTable.additionalOffsetToIndexSubtable = ( + additionalOffsetToIndexSubtable + ) + indexSubTable.indexFormat = indexFormat + indexSubTable.imageFormat = imageFormat + indexSubTable.imageDataOffset = imageDataOffset + indexSubTable.decompile() # https://github.com/fonttools/fonttools/issues/317 + curStrike.indexSubTables.append(indexSubTable) + + def compile(self, ttFont): + dataList = [] + self.numSizes = len(self.strikes) + dataList.append(sstruct.pack(eblcHeaderFormat, self)) + + # Data size of the header + bitmapSizeTable needs to be calculated + # in order to form offsets. This value will hold the size of the data + # in dataList after all the data is consolidated in dataList. + dataSize = len(dataList[0]) + + # The table will be structured in the following order: + # (0) header + # (1) Each bitmapSizeTable [1 ... self.numSizes] + # (2) Alternate between indexSubTableArray and indexSubTable + # for each bitmapSizeTable present. + # + # The issue is maintaining the proper offsets when table information + # gets moved around. All offsets and size information must be recalculated + # when building the table to allow editing within ttLib and also allow easy + # import/export to and from XML. All of this offset information is lost + # when exporting to XML so everything must be calculated fresh so importing + # from XML will work cleanly. Only byte offset and size information is + # calculated fresh. Count information like numberOfIndexSubTables is + # checked through assertions. If the information in this table was not + # touched or was changed properly then these types of values should match. + # + # The table will be rebuilt the following way: + # (0) Precompute the size of all the bitmapSizeTables. This is needed to + # compute the offsets properly. + # (1) For each bitmapSizeTable compute the indexSubTable and + # indexSubTableArray pair. The indexSubTable must be computed first + # so that the offset information in indexSubTableArray can be + # calculated. Update the data size after each pairing. + # (2) Build each bitmapSizeTable. + # (3) Consolidate all the data into the main dataList in the correct order. + + for _ in self.strikes: + dataSize += sstruct.calcsize(bitmapSizeTableFormatPart1) + dataSize += len(("hori", "vert")) * sstruct.calcsize(sbitLineMetricsFormat) + dataSize += sstruct.calcsize(bitmapSizeTableFormatPart2) + + indexSubTablePairDataList = [] + for curStrike in self.strikes: + curTable = curStrike.bitmapSizeTable + curTable.numberOfIndexSubTables = len(curStrike.indexSubTables) + curTable.indexSubTableArrayOffset = dataSize + + # Precompute the size of the indexSubTableArray. This information + # is important for correctly calculating the new value for + # additionalOffsetToIndexSubtable. + sizeOfSubTableArray = ( + curTable.numberOfIndexSubTables * indexSubTableArraySize + ) + lowerBound = dataSize + dataSize += sizeOfSubTableArray + upperBound = dataSize + + indexSubTableDataList = [] + for indexSubTable in curStrike.indexSubTables: + indexSubTable.additionalOffsetToIndexSubtable = ( + dataSize - curTable.indexSubTableArrayOffset + ) + glyphIds = list(map(ttFont.getGlyphID, indexSubTable.names)) + indexSubTable.firstGlyphIndex = min(glyphIds) + indexSubTable.lastGlyphIndex = max(glyphIds) + data = indexSubTable.compile(ttFont) + indexSubTableDataList.append(data) + dataSize += len(data) + curTable.startGlyphIndex = min( + ist.firstGlyphIndex for ist in curStrike.indexSubTables + ) + curTable.endGlyphIndex = max( + ist.lastGlyphIndex for ist in curStrike.indexSubTables + ) + + for i in curStrike.indexSubTables: + data = struct.pack( + indexSubHeaderFormat, + i.firstGlyphIndex, + i.lastGlyphIndex, + i.additionalOffsetToIndexSubtable, + ) + indexSubTablePairDataList.append(data) + indexSubTablePairDataList.extend(indexSubTableDataList) + curTable.indexTablesSize = dataSize - curTable.indexSubTableArrayOffset + + for curStrike in self.strikes: + curTable = curStrike.bitmapSizeTable + data = sstruct.pack(bitmapSizeTableFormatPart1, curTable) + dataList.append(data) + for metric in ("hori", "vert"): + metricObj = vars(curTable)[metric] + data = sstruct.pack(sbitLineMetricsFormat, metricObj) + dataList.append(data) + data = sstruct.pack(bitmapSizeTableFormatPart2, curTable) + dataList.append(data) + dataList.extend(indexSubTablePairDataList) + + return bytesjoin(dataList) + + def toXML(self, writer, ttFont): + writer.simpletag("header", [("version", self.version)]) + writer.newline() + for curIndex, curStrike in enumerate(self.strikes): + curStrike.toXML(curIndex, writer, ttFont) + + def fromXML(self, name, attrs, content, ttFont): + if name == "header": + self.version = safeEval(attrs["version"]) + elif name == "strike": + if not hasattr(self, "strikes"): + self.strikes = [] + strikeIndex = safeEval(attrs["index"]) + curStrike = Strike() + curStrike.fromXML(name, attrs, content, ttFont, self) + + # Grow the strike array to the appropriate size. The XML format + # allows for the strike index value to be out of order. + if strikeIndex >= len(self.strikes): + self.strikes += [None] * (strikeIndex + 1 - len(self.strikes)) + assert self.strikes[strikeIndex] is None, "Duplicate strike EBLC indices." + self.strikes[strikeIndex] = curStrike + + +class Strike(object): + def __init__(self): + self.bitmapSizeTable = BitmapSizeTable() + self.indexSubTables = [] + + def toXML(self, strikeIndex, writer, ttFont): + writer.begintag("strike", [("index", strikeIndex)]) + writer.newline() + self.bitmapSizeTable.toXML(writer, ttFont) + writer.comment( + "GlyphIds are written but not read. The firstGlyphIndex and\nlastGlyphIndex values will be recalculated by the compiler." + ) + writer.newline() + for indexSubTable in self.indexSubTables: + indexSubTable.toXML(writer, ttFont) + writer.endtag("strike") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont, locator): + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name == "bitmapSizeTable": + self.bitmapSizeTable.fromXML(name, attrs, content, ttFont) + elif name.startswith(_indexSubTableSubclassPrefix): + indexFormat = safeEval(name[len(_indexSubTableSubclassPrefix) :]) + indexFormatClass = locator.getIndexFormatClass(indexFormat) + indexSubTable = indexFormatClass(None, None) + indexSubTable.indexFormat = indexFormat + indexSubTable.fromXML(name, attrs, content, ttFont) + self.indexSubTables.append(indexSubTable) + + +class BitmapSizeTable(object): + # Returns all the simple metric names that bitmap size table + # cares about in terms of XML creation. + def _getXMLMetricNames(self): + dataNames = sstruct.getformat(bitmapSizeTableFormatPart1)[1] + dataNames = {**dataNames, **sstruct.getformat(bitmapSizeTableFormatPart2)[1]} + # Skip the first 3 data names because they are byte offsets and counts. + return list(dataNames.keys())[3:] + + def toXML(self, writer, ttFont): + writer.begintag("bitmapSizeTable") + writer.newline() + for metric in ("hori", "vert"): + getattr(self, metric).toXML(metric, writer, ttFont) + for metricName in self._getXMLMetricNames(): + writer.simpletag(metricName, value=getattr(self, metricName)) + writer.newline() + writer.endtag("bitmapSizeTable") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + # Create a lookup for all the simple names that make sense to + # bitmap size table. Only read the information from these names. + dataNames = set(self._getXMLMetricNames()) + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name == "sbitLineMetrics": + direction = attrs["direction"] + assert direction in ( + "hori", + "vert", + ), "SbitLineMetrics direction specified invalid." + metricObj = SbitLineMetrics() + metricObj.fromXML(name, attrs, content, ttFont) + vars(self)[direction] = metricObj + elif name in dataNames: + vars(self)[name] = safeEval(attrs["value"]) + else: + log.warning("unknown name '%s' being ignored in BitmapSizeTable.", name) + + +class SbitLineMetrics(object): + def toXML(self, name, writer, ttFont): + writer.begintag("sbitLineMetrics", [("direction", name)]) + writer.newline() + for metricName in sstruct.getformat(sbitLineMetricsFormat)[1]: + writer.simpletag(metricName, value=getattr(self, metricName)) + writer.newline() + writer.endtag("sbitLineMetrics") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + metricNames = set(sstruct.getformat(sbitLineMetricsFormat)[1]) + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name in metricNames: + vars(self)[name] = safeEval(attrs["value"]) + + +# Important information about the naming scheme. Used for identifying subtables. +_indexSubTableSubclassPrefix = "eblc_index_sub_table_" + + +class EblcIndexSubTable(object): + def __init__(self, data, ttFont): + self.data = data + self.ttFont = ttFont + # TODO Currently non-lazy decompiling doesn't work for this class... + # if not ttFont.lazy: + # self.decompile() + # del self.data, self.ttFont + + def __getattr__(self, attr): + # Allow lazy decompile. + if attr[:2] == "__": + raise AttributeError(attr) + if attr == "data": + raise AttributeError(attr) + self.decompile() + return getattr(self, attr) + + def ensureDecompiled(self, recurse=False): + if hasattr(self, "data"): + self.decompile() + + # This method just takes care of the indexSubHeader. Implementing subclasses + # should call it to compile the indexSubHeader and then continue compiling + # the remainder of their unique format. + def compile(self, ttFont): + return struct.pack( + indexSubHeaderFormat, + self.indexFormat, + self.imageFormat, + self.imageDataOffset, + ) + + # Creates the XML for bitmap glyphs. Each index sub table basically makes + # the same XML except for specific metric information that is written + # out via a method call that a subclass implements optionally. + def toXML(self, writer, ttFont): + writer.begintag( + self.__class__.__name__, + [ + ("imageFormat", self.imageFormat), + ("firstGlyphIndex", self.firstGlyphIndex), + ("lastGlyphIndex", self.lastGlyphIndex), + ], + ) + writer.newline() + self.writeMetrics(writer, ttFont) + # Write out the names as thats all thats needed to rebuild etc. + # For font debugging of consecutive formats the ids are also written. + # The ids are not read when moving from the XML format. + glyphIds = map(ttFont.getGlyphID, self.names) + for glyphName, glyphId in zip(self.names, glyphIds): + writer.simpletag("glyphLoc", name=glyphName, id=glyphId) + writer.newline() + writer.endtag(self.__class__.__name__) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + # Read all the attributes. Even though the glyph indices are + # recalculated, they are still read in case there needs to + # be an immediate export of the data. + self.imageFormat = safeEval(attrs["imageFormat"]) + self.firstGlyphIndex = safeEval(attrs["firstGlyphIndex"]) + self.lastGlyphIndex = safeEval(attrs["lastGlyphIndex"]) + + self.readMetrics(name, attrs, content, ttFont) + + self.names = [] + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name == "glyphLoc": + self.names.append(attrs["name"]) + + # A helper method that writes the metrics for the index sub table. It also + # is responsible for writing the image size for fixed size data since fixed + # size is not recalculated on compile. Default behavior is to do nothing. + def writeMetrics(self, writer, ttFont): + pass + + # A helper method that is the inverse of writeMetrics. + def readMetrics(self, name, attrs, content, ttFont): + pass + + # This method is for fixed glyph data sizes. There are formats where + # the glyph data is fixed but are actually composite glyphs. To handle + # this the font spec in indexSubTable makes the data the size of the + # fixed size by padding the component arrays. This function abstracts + # out this padding process. Input is data unpadded. Output is data + # padded only in fixed formats. Default behavior is to return the data. + def padBitmapData(self, data): + return data + + # Remove any of the glyph locations and names that are flagged as skipped. + # This only occurs in formats {1,3}. + def removeSkipGlyphs(self): + # Determines if a name, location pair is a valid data location. + # Skip glyphs are marked when the size is equal to zero. + def isValidLocation(args): + (name, (startByte, endByte)) = args + return startByte < endByte + + # Remove all skip glyphs. + dataPairs = list(filter(isValidLocation, zip(self.names, self.locations))) + self.names, self.locations = list(map(list, zip(*dataPairs))) + + +# A closure for creating a custom mixin. This is done because formats 1 and 3 +# are very similar. The only difference between them is the size per offset +# value. Code put in here should handle both cases generally. +def _createOffsetArrayIndexSubTableMixin(formatStringForDataType): + # Prep the data size for the offset array data format. + dataFormat = ">" + formatStringForDataType + offsetDataSize = struct.calcsize(dataFormat) + + class OffsetArrayIndexSubTableMixin(object): + def decompile(self): + numGlyphs = self.lastGlyphIndex - self.firstGlyphIndex + 1 + indexingOffsets = [ + glyphIndex * offsetDataSize for glyphIndex in range(numGlyphs + 2) + ] + indexingLocations = zip(indexingOffsets, indexingOffsets[1:]) + offsetArray = [ + struct.unpack(dataFormat, self.data[slice(*loc)])[0] + for loc in indexingLocations + ] + + glyphIds = list(range(self.firstGlyphIndex, self.lastGlyphIndex + 1)) + modifiedOffsets = [offset + self.imageDataOffset for offset in offsetArray] + self.locations = list(zip(modifiedOffsets, modifiedOffsets[1:])) + + self.names = list(map(self.ttFont.getGlyphName, glyphIds)) + self.removeSkipGlyphs() + del self.data, self.ttFont + + def compile(self, ttFont): + # First make sure that all the data lines up properly. Formats 1 and 3 + # must have all its data lined up consecutively. If not this will fail. + for curLoc, nxtLoc in zip(self.locations, self.locations[1:]): + assert ( + curLoc[1] == nxtLoc[0] + ), "Data must be consecutive in indexSubTable offset formats" + + glyphIds = list(map(ttFont.getGlyphID, self.names)) + # Make sure that all ids are sorted strictly increasing. + assert all(glyphIds[i] < glyphIds[i + 1] for i in range(len(glyphIds) - 1)) + + # Run a simple algorithm to add skip glyphs to the data locations at + # the places where an id is not present. + idQueue = deque(glyphIds) + locQueue = deque(self.locations) + allGlyphIds = list(range(self.firstGlyphIndex, self.lastGlyphIndex + 1)) + allLocations = [] + for curId in allGlyphIds: + if curId != idQueue[0]: + allLocations.append((locQueue[0][0], locQueue[0][0])) + else: + idQueue.popleft() + allLocations.append(locQueue.popleft()) + + # Now that all the locations are collected, pack them appropriately into + # offsets. This is the form where offset[i] is the location and + # offset[i+1]-offset[i] is the size of the data location. + offsets = list(allLocations[0]) + [loc[1] for loc in allLocations[1:]] + # Image data offset must be less than or equal to the minimum of locations. + # This offset may change the value for round tripping but is safer and + # allows imageDataOffset to not be required to be in the XML version. + self.imageDataOffset = min(offsets) + offsetArray = [offset - self.imageDataOffset for offset in offsets] + + dataList = [EblcIndexSubTable.compile(self, ttFont)] + dataList += [ + struct.pack(dataFormat, offsetValue) for offsetValue in offsetArray + ] + # Take care of any padding issues. Only occurs in format 3. + if offsetDataSize * len(offsetArray) % 4 != 0: + dataList.append(struct.pack(dataFormat, 0)) + return bytesjoin(dataList) + + return OffsetArrayIndexSubTableMixin + + +# A Mixin for functionality shared between the different kinds +# of fixed sized data handling. Both kinds have big metrics so +# that kind of special processing is also handled in this mixin. +class FixedSizeIndexSubTableMixin(object): + def writeMetrics(self, writer, ttFont): + writer.simpletag("imageSize", value=self.imageSize) + writer.newline() + self.metrics.toXML(writer, ttFont) + + def readMetrics(self, name, attrs, content, ttFont): + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name == "imageSize": + self.imageSize = safeEval(attrs["value"]) + elif name == BigGlyphMetrics.__name__: + self.metrics = BigGlyphMetrics() + self.metrics.fromXML(name, attrs, content, ttFont) + elif name == SmallGlyphMetrics.__name__: + log.warning( + "SmallGlyphMetrics being ignored in format %d.", self.indexFormat + ) + + def padBitmapData(self, data): + # Make sure that the data isn't bigger than the fixed size. + assert len(data) <= self.imageSize, ( + "Data in indexSubTable format %d must be less than the fixed size." + % self.indexFormat + ) + # Pad the data so that it matches the fixed size. + pad = (self.imageSize - len(data)) * b"\0" + return data + pad + + +class eblc_index_sub_table_1( + _createOffsetArrayIndexSubTableMixin("L"), EblcIndexSubTable +): + pass + + +class eblc_index_sub_table_2(FixedSizeIndexSubTableMixin, EblcIndexSubTable): + def decompile(self): + (self.imageSize,) = struct.unpack(">L", self.data[:4]) + self.metrics = BigGlyphMetrics() + sstruct.unpack2(bigGlyphMetricsFormat, self.data[4:], self.metrics) + glyphIds = list(range(self.firstGlyphIndex, self.lastGlyphIndex + 1)) + offsets = [ + self.imageSize * i + self.imageDataOffset for i in range(len(glyphIds) + 1) + ] + self.locations = list(zip(offsets, offsets[1:])) + self.names = list(map(self.ttFont.getGlyphName, glyphIds)) + del self.data, self.ttFont + + def compile(self, ttFont): + glyphIds = list(map(ttFont.getGlyphID, self.names)) + # Make sure all the ids are consecutive. This is required by Format 2. + assert glyphIds == list( + range(self.firstGlyphIndex, self.lastGlyphIndex + 1) + ), "Format 2 ids must be consecutive." + self.imageDataOffset = min(next(iter(zip(*self.locations)))) + + dataList = [EblcIndexSubTable.compile(self, ttFont)] + dataList.append(struct.pack(">L", self.imageSize)) + dataList.append(sstruct.pack(bigGlyphMetricsFormat, self.metrics)) + return bytesjoin(dataList) + + +class eblc_index_sub_table_3( + _createOffsetArrayIndexSubTableMixin("H"), EblcIndexSubTable +): + pass + + +class eblc_index_sub_table_4(EblcIndexSubTable): + def decompile(self): + (numGlyphs,) = struct.unpack(">L", self.data[:4]) + data = self.data[4:] + indexingOffsets = [ + glyphIndex * codeOffsetPairSize for glyphIndex in range(numGlyphs + 2) + ] + indexingLocations = zip(indexingOffsets, indexingOffsets[1:]) + glyphArray = [ + struct.unpack(codeOffsetPairFormat, data[slice(*loc)]) + for loc in indexingLocations + ] + glyphIds, offsets = list(map(list, zip(*glyphArray))) + # There are one too many glyph ids. Get rid of the last one. + glyphIds.pop() + + offsets = [offset + self.imageDataOffset for offset in offsets] + self.locations = list(zip(offsets, offsets[1:])) + self.names = list(map(self.ttFont.getGlyphName, glyphIds)) + del self.data, self.ttFont + + def compile(self, ttFont): + # First make sure that all the data lines up properly. Format 4 + # must have all its data lined up consecutively. If not this will fail. + for curLoc, nxtLoc in zip(self.locations, self.locations[1:]): + assert ( + curLoc[1] == nxtLoc[0] + ), "Data must be consecutive in indexSubTable format 4" + + offsets = list(self.locations[0]) + [loc[1] for loc in self.locations[1:]] + # Image data offset must be less than or equal to the minimum of locations. + # Resetting this offset may change the value for round tripping but is safer + # and allows imageDataOffset to not be required to be in the XML version. + self.imageDataOffset = min(offsets) + offsets = [offset - self.imageDataOffset for offset in offsets] + glyphIds = list(map(ttFont.getGlyphID, self.names)) + # Create an iterator over the ids plus a padding value. + idsPlusPad = list(itertools.chain(glyphIds, [0])) + + dataList = [EblcIndexSubTable.compile(self, ttFont)] + dataList.append(struct.pack(">L", len(glyphIds))) + tmp = [ + struct.pack(codeOffsetPairFormat, *cop) for cop in zip(idsPlusPad, offsets) + ] + dataList += tmp + data = bytesjoin(dataList) + return data + + +class eblc_index_sub_table_5(FixedSizeIndexSubTableMixin, EblcIndexSubTable): + def decompile(self): + self.origDataLen = 0 + (self.imageSize,) = struct.unpack(">L", self.data[:4]) + data = self.data[4:] + self.metrics, data = sstruct.unpack2( + bigGlyphMetricsFormat, data, BigGlyphMetrics() + ) + (numGlyphs,) = struct.unpack(">L", data[:4]) + data = data[4:] + glyphIds = [ + struct.unpack(">H", data[2 * i : 2 * (i + 1)])[0] for i in range(numGlyphs) + ] + + offsets = [ + self.imageSize * i + self.imageDataOffset for i in range(len(glyphIds) + 1) + ] + self.locations = list(zip(offsets, offsets[1:])) + self.names = list(map(self.ttFont.getGlyphName, glyphIds)) + del self.data, self.ttFont + + def compile(self, ttFont): + self.imageDataOffset = min(next(iter(zip(*self.locations)))) + dataList = [EblcIndexSubTable.compile(self, ttFont)] + dataList.append(struct.pack(">L", self.imageSize)) + dataList.append(sstruct.pack(bigGlyphMetricsFormat, self.metrics)) + glyphIds = list(map(ttFont.getGlyphID, self.names)) + dataList.append(struct.pack(">L", len(glyphIds))) + dataList += [struct.pack(">H", curId) for curId in glyphIds] + if len(glyphIds) % 2 == 1: + dataList.append(struct.pack(">H", 0)) + return bytesjoin(dataList) + + +# Dictionary of indexFormat to the class representing that format. +eblc_sub_table_classes = { + 1: eblc_index_sub_table_1, + 2: eblc_index_sub_table_2, + 3: eblc_index_sub_table_3, + 4: eblc_index_sub_table_4, + 5: eblc_index_sub_table_5, +} diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/F_F_T_M_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/F_F_T_M_.py new file mode 100644 index 0000000000000000000000000000000000000000..20b72346611911a1044ae0d52eb2a5c9efdf5540 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/F_F_T_M_.py @@ -0,0 +1,52 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import safeEval +from fontTools.misc.timeTools import timestampFromString, timestampToString +from . import DefaultTable + +FFTMFormat = """ + > # big endian + version: I + FFTimeStamp: Q + sourceCreated: Q + sourceModified: Q +""" + + +class table_F_F_T_M_(DefaultTable.DefaultTable): + """FontForge Time Stamp table + + The ``FFTM`` table is used by the free-software font editor + FontForge to record timestamps for the creation and modification + of font source (.sfd) files and a timestamp for FontForge's + own source code. + + See also https://fontforge.org/docs/techref/non-standard.html + """ + + def decompile(self, data, ttFont): + dummy, rest = sstruct.unpack2(FFTMFormat, data, self) + + def compile(self, ttFont): + data = sstruct.pack(FFTMFormat, self) + return data + + def toXML(self, writer, ttFont): + writer.comment( + "FontForge's timestamp, font source creation and modification dates" + ) + writer.newline() + formatstring, names, fixes = sstruct.getformat(FFTMFormat) + for name in names: + value = getattr(self, name) + if name in ("FFTimeStamp", "sourceCreated", "sourceModified"): + value = timestampToString(value) + writer.simpletag(name, value=value) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + value = attrs["value"] + if name in ("FFTimeStamp", "sourceCreated", "sourceModified"): + value = timestampFromString(value) + else: + value = safeEval(value) + setattr(self, name, value) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/F__e_a_t.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/F__e_a_t.py new file mode 100644 index 0000000000000000000000000000000000000000..58094d9d28df2526ecd5de54836e4c797d7484c8 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/F__e_a_t.py @@ -0,0 +1,159 @@ +from fontTools.misc import sstruct +from fontTools.misc.fixedTools import floatToFixedToStr +from fontTools.misc.textTools import safeEval +from . import DefaultTable +from . import grUtils +import struct + +Feat_hdr_format = """ + > + version: 16.16F +""" + + +class table_F__e_a_t(DefaultTable.DefaultTable): + """Feature table + + The ``Feat`` table is used exclusively by the Graphite shaping engine + to store features and possible settings specified in GDL. Graphite features + determine what rules are applied to transform a glyph stream. + + Not to be confused with ``feat``, or the OpenType Layout tables + ``GSUB``/``GPOS``. + + See also https://graphite.sil.org/graphite_techAbout#graphite-font-tables + """ + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.features = {} + + def decompile(self, data, ttFont): + (_, data) = sstruct.unpack2(Feat_hdr_format, data, self) + self.version = float(floatToFixedToStr(self.version, precisionBits=16)) + (numFeats,) = struct.unpack(">H", data[:2]) + data = data[8:] + allfeats = [] + maxsetting = 0 + for i in range(numFeats): + if self.version >= 2.0: + (fid, nums, _, offset, flags, lid) = struct.unpack( + ">LHHLHH", data[16 * i : 16 * (i + 1)] + ) + offset = int((offset - 12 - 16 * numFeats) / 4) + else: + (fid, nums, offset, flags, lid) = struct.unpack( + ">HHLHH", data[12 * i : 12 * (i + 1)] + ) + offset = int((offset - 12 - 12 * numFeats) / 4) + allfeats.append((fid, nums, offset, flags, lid)) + maxsetting = max(maxsetting, offset + nums) + data = data[16 * numFeats :] + allsettings = [] + for i in range(maxsetting): + if len(data) >= 4 * (i + 1): + (val, lid) = struct.unpack(">HH", data[4 * i : 4 * (i + 1)]) + allsettings.append((val, lid)) + for i, f in enumerate(allfeats): + (fid, nums, offset, flags, lid) = f + fobj = Feature() + fobj.flags = flags + fobj.label = lid + self.features[grUtils.num2tag(fid)] = fobj + fobj.settings = {} + fobj.default = None + fobj.index = i + for i in range(offset, offset + nums): + if i >= len(allsettings): + continue + (vid, vlid) = allsettings[i] + fobj.settings[vid] = vlid + if fobj.default is None: + fobj.default = vid + + def compile(self, ttFont): + fdat = b"" + vdat = b"" + offset = 0 + for f, v in sorted(self.features.items(), key=lambda x: x[1].index): + fnum = grUtils.tag2num(f) + if self.version >= 2.0: + fdat += struct.pack( + ">LHHLHH", + grUtils.tag2num(f), + len(v.settings), + 0, + offset * 4 + 12 + 16 * len(self.features), + v.flags, + v.label, + ) + elif fnum > 65535: # self healing for alphabetic ids + self.version = 2.0 + return self.compile(ttFont) + else: + fdat += struct.pack( + ">HHLHH", + grUtils.tag2num(f), + len(v.settings), + offset * 4 + 12 + 12 * len(self.features), + v.flags, + v.label, + ) + for s, l in sorted( + v.settings.items(), key=lambda x: (-1, x[1]) if x[0] == v.default else x + ): + vdat += struct.pack(">HH", s, l) + offset += len(v.settings) + hdr = sstruct.pack(Feat_hdr_format, self) + return hdr + struct.pack(">HHL", len(self.features), 0, 0) + fdat + vdat + + def toXML(self, writer, ttFont): + writer.simpletag("version", version=self.version) + writer.newline() + nameTable = ttFont["name"] if "name" in ttFont else None + for f, v in sorted(self.features.items(), key=lambda x: x[1].index): + name = nameTable.getDebugName(v.label) if nameTable else None + if name is not None: + writer.newline() + writer.comment(name) + writer.newline() + writer.begintag( + "feature", + fid=f, + label=v.label, + flags=v.flags, + default=(v.default if v.default else 0), + ) + writer.newline() + for s, l in sorted(v.settings.items()): + writer.simpletag("setting", value=s, label=l) + name = nameTable.getDebugName(l) if nameTable else None + if name is not None: + writer.write(" ") + writer.comment(name) + writer.newline() + writer.endtag("feature") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": + self.version = float(safeEval(attrs["version"])) + elif name == "feature": + fid = attrs["fid"] + fobj = Feature() + fobj.flags = int(safeEval(attrs["flags"])) + fobj.label = int(safeEval(attrs["label"])) + fobj.default = int(safeEval(attrs.get("default", "0"))) + fobj.index = len(self.features) + self.features[fid] = fobj + fobj.settings = {} + for element in content: + if not isinstance(element, tuple): + continue + tag, a, c = element + if tag == "setting": + fobj.settings[int(safeEval(a["value"]))] = int(safeEval(a["label"])) + + +class Feature(object): + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_D_E_F_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_D_E_F_.py new file mode 100644 index 0000000000000000000000000000000000000000..922a9cb2063777cae486a3369d3d6380f9f626cf --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_D_E_F_.py @@ -0,0 +1,13 @@ +from .otBase import BaseTTXConverter + + +class table_G_D_E_F_(BaseTTXConverter): + """Glyph Definition table + + The ``GDEF`` table stores various glyph properties that are used + by OpenType Layout. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/gdef + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_P_O_S_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_P_O_S_.py new file mode 100644 index 0000000000000000000000000000000000000000..03bdc612ead945133c97f4a4d0002e8ca31c60c4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_P_O_S_.py @@ -0,0 +1,14 @@ +from .otBase import BaseTTXConverter + + +class table_G_P_O_S_(BaseTTXConverter): + """Glyph Positioning table + + The ``GPOS`` table stores advanced glyph-positioning data + used in OpenType Layout features, such as mark attachment, + cursive attachment, kerning, and other position adjustments. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/gpos + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_S_U_B_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_S_U_B_.py new file mode 100644 index 0000000000000000000000000000000000000000..ca1aff8b0680bf22550c460819627b4d2b14a463 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_S_U_B_.py @@ -0,0 +1,13 @@ +from .otBase import BaseTTXConverter + + +class table_G_S_U_B_(BaseTTXConverter): + """Glyph Substitution table + + The ``GSUB`` table contains glyph-substitution rules used in + OpenType Layout. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/gsub + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_V_A_R_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_V_A_R_.py new file mode 100644 index 0000000000000000000000000000000000000000..889b1f2a3bd712b68140a6b9102e732ce794587c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G_V_A_R_.py @@ -0,0 +1,5 @@ +from ._g_v_a_r import table__g_v_a_r + + +class table_G_V_A_R_(table__g_v_a_r): + gid_size = 3 diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G__l_a_t.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G__l_a_t.py new file mode 100644 index 0000000000000000000000000000000000000000..fe1e0534fb75e372933add1e71799c272a01c3dd --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G__l_a_t.py @@ -0,0 +1,235 @@ +from fontTools.misc import sstruct +from fontTools.misc.fixedTools import floatToFixedToStr +from fontTools.misc.textTools import safeEval + +# from itertools import * +from functools import partial +from . import DefaultTable +from . import grUtils +import struct + + +Glat_format_0 = """ + > # big endian + version: 16.16F +""" + +Glat_format_3 = """ + > + version: 16.16F + compression:L # compression scheme or reserved +""" + +Glat_format_1_entry = """ + > + attNum: B # Attribute number of first attribute + num: B # Number of attributes in this run +""" +Glat_format_23_entry = """ + > + attNum: H # Attribute number of first attribute + num: H # Number of attributes in this run +""" + +Glat_format_3_octabox_metrics = """ + > + subboxBitmap: H # Which subboxes exist on 4x4 grid + diagNegMin: B # Defines minimum negatively-sloped diagonal (si) + diagNegMax: B # Defines maximum negatively-sloped diagonal (sa) + diagPosMin: B # Defines minimum positively-sloped diagonal (di) + diagPosMax: B # Defines maximum positively-sloped diagonal (da) +""" + +Glat_format_3_subbox_entry = """ + > + left: B # xi + right: B # xa + bottom: B # yi + top: B # ya + diagNegMin: B # Defines minimum negatively-sloped diagonal (si) + diagNegMax: B # Defines maximum negatively-sloped diagonal (sa) + diagPosMin: B # Defines minimum positively-sloped diagonal (di) + diagPosMax: B # Defines maximum positively-sloped diagonal (da) +""" + + +class _Object: + pass + + +class _Dict(dict): + pass + + +class table_G__l_a_t(DefaultTable.DefaultTable): + """Graphite Glyph Attributes table + + See also https://graphite.sil.org/graphite_techAbout#graphite-font-tables + """ + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.scheme = 0 + + def decompile(self, data, ttFont): + sstruct.unpack2(Glat_format_0, data, self) + self.version = float(floatToFixedToStr(self.version, precisionBits=16)) + if self.version <= 1.9: + decoder = partial(self.decompileAttributes12, fmt=Glat_format_1_entry) + elif self.version <= 2.9: + decoder = partial(self.decompileAttributes12, fmt=Glat_format_23_entry) + elif self.version >= 3.0: + (data, self.scheme) = grUtils.decompress(data) + sstruct.unpack2(Glat_format_3, data, self) + self.hasOctaboxes = (self.compression & 1) == 1 + decoder = self.decompileAttributes3 + + gloc = ttFont["Gloc"] + self.attributes = {} + count = 0 + for s, e in zip(gloc, gloc[1:]): + self.attributes[ttFont.getGlyphName(count)] = decoder(data[s:e]) + count += 1 + + def decompileAttributes12(self, data, fmt): + attributes = _Dict() + while len(data) > 3: + e, data = sstruct.unpack2(fmt, data, _Object()) + keys = range(e.attNum, e.attNum + e.num) + if len(data) >= 2 * e.num: + vals = struct.unpack_from((">%dh" % e.num), data) + attributes.update(zip(keys, vals)) + data = data[2 * e.num :] + return attributes + + def decompileAttributes3(self, data): + if self.hasOctaboxes: + o, data = sstruct.unpack2(Glat_format_3_octabox_metrics, data, _Object()) + numsub = bin(o.subboxBitmap).count("1") + o.subboxes = [] + for b in range(numsub): + if len(data) >= 8: + subbox, data = sstruct.unpack2( + Glat_format_3_subbox_entry, data, _Object() + ) + o.subboxes.append(subbox) + attrs = self.decompileAttributes12(data, Glat_format_23_entry) + if self.hasOctaboxes: + attrs.octabox = o + return attrs + + def compile(self, ttFont): + data = sstruct.pack(Glat_format_0, self) + if self.version <= 1.9: + encoder = partial(self.compileAttributes12, fmt=Glat_format_1_entry) + elif self.version <= 2.9: + encoder = partial(self.compileAttributes12, fmt=Glat_format_1_entry) + elif self.version >= 3.0: + self.compression = (self.scheme << 27) + (1 if self.hasOctaboxes else 0) + data = sstruct.pack(Glat_format_3, self) + encoder = self.compileAttributes3 + + glocs = [] + for n in range(len(self.attributes)): + glocs.append(len(data)) + data += encoder(self.attributes[ttFont.getGlyphName(n)]) + glocs.append(len(data)) + ttFont["Gloc"].set(glocs) + + if self.version >= 3.0: + data = grUtils.compress(self.scheme, data) + return data + + def compileAttributes12(self, attrs, fmt): + data = b"" + for e in grUtils.entries(attrs): + data += sstruct.pack(fmt, {"attNum": e[0], "num": e[1]}) + struct.pack( + (">%dh" % len(e[2])), *e[2] + ) + return data + + def compileAttributes3(self, attrs): + if self.hasOctaboxes: + o = attrs.octabox + data = sstruct.pack(Glat_format_3_octabox_metrics, o) + numsub = bin(o.subboxBitmap).count("1") + for b in range(numsub): + data += sstruct.pack(Glat_format_3_subbox_entry, o.subboxes[b]) + else: + data = "" + return data + self.compileAttributes12(attrs, Glat_format_23_entry) + + def toXML(self, writer, ttFont): + writer.simpletag("version", version=self.version, compressionScheme=self.scheme) + writer.newline() + for n, a in sorted( + self.attributes.items(), key=lambda x: ttFont.getGlyphID(x[0]) + ): + writer.begintag("glyph", name=n) + writer.newline() + if hasattr(a, "octabox"): + o = a.octabox + formatstring, names, fixes = sstruct.getformat( + Glat_format_3_octabox_metrics + ) + vals = {} + for k in names: + if k == "subboxBitmap": + continue + vals[k] = "{:.3f}%".format(getattr(o, k) * 100.0 / 255) + vals["bitmap"] = "{:0X}".format(o.subboxBitmap) + writer.begintag("octaboxes", **vals) + writer.newline() + formatstring, names, fixes = sstruct.getformat( + Glat_format_3_subbox_entry + ) + for s in o.subboxes: + vals = {} + for k in names: + vals[k] = "{:.3f}%".format(getattr(s, k) * 100.0 / 255) + writer.simpletag("octabox", **vals) + writer.newline() + writer.endtag("octaboxes") + writer.newline() + for k, v in sorted(a.items()): + writer.simpletag("attribute", index=k, value=v) + writer.newline() + writer.endtag("glyph") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": + self.version = float(safeEval(attrs["version"])) + self.scheme = int(safeEval(attrs["compressionScheme"])) + if name != "glyph": + return + if not hasattr(self, "attributes"): + self.attributes = {} + gname = attrs["name"] + attributes = _Dict() + for element in content: + if not isinstance(element, tuple): + continue + tag, attrs, subcontent = element + if tag == "attribute": + k = int(safeEval(attrs["index"])) + v = int(safeEval(attrs["value"])) + attributes[k] = v + elif tag == "octaboxes": + self.hasOctaboxes = True + o = _Object() + o.subboxBitmap = int(attrs["bitmap"], 16) + o.subboxes = [] + del attrs["bitmap"] + for k, v in attrs.items(): + setattr(o, k, int(float(v[:-1]) * 255.0 / 100.0 + 0.5)) + for element in subcontent: + if not isinstance(element, tuple): + continue + (tag, attrs, subcontent) = element + so = _Object() + for k, v in attrs.items(): + setattr(so, k, int(float(v[:-1]) * 255.0 / 100.0 + 0.5)) + o.subboxes.append(so) + attributes.octabox = o + self.attributes[gname] = attributes diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G__l_o_c.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G__l_o_c.py new file mode 100644 index 0000000000000000000000000000000000000000..4c3d78ab2e4b993708a83c13efe45c3c4a1eaf92 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/G__l_o_c.py @@ -0,0 +1,85 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import safeEval +from . import DefaultTable +import array +import sys + + +Gloc_header = """ + > # big endian + version: 16.16F # Table version + flags: H # bit 0: 1=long format, 0=short format + # bit 1: 1=attribute names, 0=no names + numAttribs: H # NUmber of attributes +""" + + +class table_G__l_o_c(DefaultTable.DefaultTable): + """Graphite Index to Glyph Atttributes table + + See also https://graphite.sil.org/graphite_techAbout#graphite-font-tables + """ + + dependencies = ["Glat"] + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.attribIds = None + self.numAttribs = 0 + + def decompile(self, data, ttFont): + _, data = sstruct.unpack2(Gloc_header, data, self) + flags = self.flags + del self.flags + self.locations = array.array("I" if flags & 1 else "H") + self.locations.frombytes(data[: len(data) - self.numAttribs * (flags & 2)]) + if sys.byteorder != "big": + self.locations.byteswap() + self.attribIds = array.array("H") + if flags & 2: + self.attribIds.frombytes(data[-self.numAttribs * 2 :]) + if sys.byteorder != "big": + self.attribIds.byteswap() + + def compile(self, ttFont): + data = sstruct.pack( + Gloc_header, + dict( + version=1.0, + flags=(bool(self.attribIds) << 1) + (self.locations.typecode == "I"), + numAttribs=self.numAttribs, + ), + ) + if sys.byteorder != "big": + self.locations.byteswap() + data += self.locations.tobytes() + if sys.byteorder != "big": + self.locations.byteswap() + if self.attribIds: + if sys.byteorder != "big": + self.attribIds.byteswap() + data += self.attribIds.tobytes() + if sys.byteorder != "big": + self.attribIds.byteswap() + return data + + def set(self, locations): + long_format = max(locations) >= 65536 + self.locations = array.array("I" if long_format else "H", locations) + + def toXML(self, writer, ttFont): + writer.simpletag("attributes", number=self.numAttribs) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "attributes": + self.numAttribs = int(safeEval(attrs["number"])) + + def __getitem__(self, index): + return self.locations[index] + + def __len__(self): + return len(self.locations) + + def __iter__(self): + return iter(self.locations) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/H_V_A_R_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/H_V_A_R_.py new file mode 100644 index 0000000000000000000000000000000000000000..48e7fd67cf0f89e5dc6057e5cbae7f07324a0402 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/H_V_A_R_.py @@ -0,0 +1,13 @@ +from .otBase import BaseTTXConverter + + +class table_H_V_A_R_(BaseTTXConverter): + """Horizontal Metrics Variations table + + The ``HVAR`` table contains variations in horizontal glyph metrics + in variable fonts. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/hvar + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/I_F_T_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/I_F_T_.py new file mode 100644 index 0000000000000000000000000000000000000000..3aa5f3f61ce694c10175c4d3a1d71b4bf96686f7 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/I_F_T_.py @@ -0,0 +1,12 @@ +"""IFT - Incremental Font Transfer table.""" + +# Reference: https://github.com/googlefonts/fontations/blob/main/read-fonts/src/tables/ift.rs + + +from .otBase import BaseTTXConverter + + +class table_I_F_T_(BaseTTXConverter): + """Incremental Font Transfer table.""" + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/I_F_T_X_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/I_F_T_X_.py new file mode 100644 index 0000000000000000000000000000000000000000..4ce3b9a50da8f04259988314bb46d0d7a411b4aa --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/I_F_T_X_.py @@ -0,0 +1,12 @@ +"""IFT - Incremental Font Transfer table.""" + +# Reference: https://github.com/googlefonts/fontations/blob/main/read-fonts/src/tables/ift.rs + + +from .otBase import BaseTTXConverter + + +class table_I_F_T_X_(BaseTTXConverter): + """Incremental Font Transfer table.""" + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/J_S_T_F_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/J_S_T_F_.py new file mode 100644 index 0000000000000000000000000000000000000000..b185f30f3a9a2978c1d4278b920f69969fe79b97 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/J_S_T_F_.py @@ -0,0 +1,13 @@ +from .otBase import BaseTTXConverter + + +class table_J_S_T_F_(BaseTTXConverter): + """Justification table + + The ``JSTF`` table contains glyph substitution and positioning + data used to perform text justification. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/jstf + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/L_T_S_H_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/L_T_S_H_.py new file mode 100644 index 0000000000000000000000000000000000000000..e691c06e9bfdc270127c2a5be33acf6505177d85 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/L_T_S_H_.py @@ -0,0 +1,58 @@ +from fontTools.misc.textTools import safeEval +from . import DefaultTable +import struct +import array + +# XXX I've lowered the strictness, to make sure Apple's own Chicago +# XXX gets through. They're looking into it, I hope to raise the standards +# XXX back to normal eventually. + + +class table_L_T_S_H_(DefaultTable.DefaultTable): + """Linear Threshold table + + The ``LTSH`` table contains per-glyph settings indicating the ppem sizes + at which the advance width metric should be scaled linearly, despite the + effects of any TrueType instructions that might otherwise alter the + advance width. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/ltsh + """ + + def decompile(self, data, ttFont): + version, numGlyphs = struct.unpack(">HH", data[:4]) + data = data[4:] + assert version == 0, "unknown version: %s" % version + assert (len(data) % numGlyphs) < 4, "numGlyphs doesn't match data length" + # ouch: the assertion is not true in Chicago! + # assert numGlyphs == ttFont['maxp'].numGlyphs + yPels = array.array("B") + yPels.frombytes(data) + self.yPels = {} + for i in range(numGlyphs): + self.yPels[ttFont.getGlyphName(i)] = yPels[i] + + def compile(self, ttFont): + version = 0 + names = list(self.yPels.keys()) + numGlyphs = len(names) + yPels = [0] * numGlyphs + # ouch: the assertion is not true in Chicago! + # assert len(self.yPels) == ttFont['maxp'].numGlyphs == numGlyphs + for name in names: + yPels[ttFont.getGlyphID(name)] = self.yPels[name] + yPels = array.array("B", yPels) + return struct.pack(">HH", version, numGlyphs) + yPels.tobytes() + + def toXML(self, writer, ttFont): + names = sorted(self.yPels.keys()) + for name in names: + writer.simpletag("yPel", name=name, value=self.yPels[name]) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if not hasattr(self, "yPels"): + self.yPels = {} + if name != "yPel": + return # ignore unknown tags + self.yPels[attrs["name"]] = safeEval(attrs["value"]) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/M_A_T_H_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/M_A_T_H_.py new file mode 100644 index 0000000000000000000000000000000000000000..35d29e9b13e9c8aeca0a95a3e965c52e7c96aed1 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/M_A_T_H_.py @@ -0,0 +1,13 @@ +from .otBase import BaseTTXConverter + + +class table_M_A_T_H_(BaseTTXConverter): + """Mathematical Typesetting table + + The ``MATH`` table contains a variety of information needed to + typeset glyphs in mathematical formulas and expressions. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/math + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/M_V_A_R_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/M_V_A_R_.py new file mode 100644 index 0000000000000000000000000000000000000000..c7e7e90fb49965b962f502a3ac9135a1c00b0374 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/M_V_A_R_.py @@ -0,0 +1,13 @@ +from .otBase import BaseTTXConverter + + +class table_M_V_A_R_(BaseTTXConverter): + """Metrics Variations table + + The ``MVAR`` table contains variation information for font-wide + metrics in a variable font. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/mvar + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/O_S_2f_2.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/O_S_2f_2.py new file mode 100644 index 0000000000000000000000000000000000000000..6480653a450221f26e643784bf16c4bbf3b287ee --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/O_S_2f_2.py @@ -0,0 +1,752 @@ +from fontTools.misc import sstruct +from fontTools.misc.roundTools import otRound +from fontTools.misc.textTools import safeEval, num2binary, binary2num +from fontTools.ttLib.tables import DefaultTable +import bisect +import logging + + +log = logging.getLogger(__name__) + +# panose classification + +panoseFormat = """ + bFamilyType: B + bSerifStyle: B + bWeight: B + bProportion: B + bContrast: B + bStrokeVariation: B + bArmStyle: B + bLetterForm: B + bMidline: B + bXHeight: B +""" + + +class Panose(object): + def __init__(self, **kwargs): + _, names, _ = sstruct.getformat(panoseFormat) + for name in names: + setattr(self, name, kwargs.pop(name, 0)) + for k in kwargs: + raise TypeError(f"Panose() got an unexpected keyword argument {k!r}") + + def toXML(self, writer, ttFont): + formatstring, names, fixes = sstruct.getformat(panoseFormat) + for name in names: + writer.simpletag(name, value=getattr(self, name)) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + setattr(self, name, safeEval(attrs["value"])) + + +# 'sfnt' OS/2 and Windows Metrics table - 'OS/2' + +OS2_format_0 = """ + > # big endian + version: H # version + xAvgCharWidth: h # average character width + usWeightClass: H # degree of thickness of strokes + usWidthClass: H # aspect ratio + fsType: H # type flags + ySubscriptXSize: h # subscript horizontal font size + ySubscriptYSize: h # subscript vertical font size + ySubscriptXOffset: h # subscript x offset + ySubscriptYOffset: h # subscript y offset + ySuperscriptXSize: h # superscript horizontal font size + ySuperscriptYSize: h # superscript vertical font size + ySuperscriptXOffset: h # superscript x offset + ySuperscriptYOffset: h # superscript y offset + yStrikeoutSize: h # strikeout size + yStrikeoutPosition: h # strikeout position + sFamilyClass: h # font family class and subclass + panose: 10s # panose classification number + ulUnicodeRange1: L # character range + ulUnicodeRange2: L # character range + ulUnicodeRange3: L # character range + ulUnicodeRange4: L # character range + achVendID: 4s # font vendor identification + fsSelection: H # font selection flags + usFirstCharIndex: H # first unicode character index + usLastCharIndex: H # last unicode character index + sTypoAscender: h # typographic ascender + sTypoDescender: h # typographic descender + sTypoLineGap: h # typographic line gap + usWinAscent: H # Windows ascender + usWinDescent: H # Windows descender +""" + +OS2_format_1_addition = """ + ulCodePageRange1: L + ulCodePageRange2: L +""" + +OS2_format_2_addition = ( + OS2_format_1_addition + + """ + sxHeight: h + sCapHeight: h + usDefaultChar: H + usBreakChar: H + usMaxContext: H +""" +) + +OS2_format_5_addition = ( + OS2_format_2_addition + + """ + usLowerOpticalPointSize: H + usUpperOpticalPointSize: H +""" +) + +bigendian = " > # big endian\n" + +OS2_format_1 = OS2_format_0 + OS2_format_1_addition +OS2_format_2 = OS2_format_0 + OS2_format_2_addition +OS2_format_5 = OS2_format_0 + OS2_format_5_addition +OS2_format_1_addition = bigendian + OS2_format_1_addition +OS2_format_2_addition = bigendian + OS2_format_2_addition +OS2_format_5_addition = bigendian + OS2_format_5_addition + + +class table_O_S_2f_2(DefaultTable.DefaultTable): + """OS/2 and Windows Metrics table + + The ``OS/2`` table contains a variety of font-wide metrics and + parameters that may be useful to an operating system or other + software for system-integration purposes. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/os2 + """ + + dependencies = ["head"] + + def decompile(self, data, ttFont): + dummy, data = sstruct.unpack2(OS2_format_0, data, self) + + if self.version == 1: + dummy, data = sstruct.unpack2(OS2_format_1_addition, data, self) + elif self.version in (2, 3, 4): + dummy, data = sstruct.unpack2(OS2_format_2_addition, data, self) + elif self.version == 5: + dummy, data = sstruct.unpack2(OS2_format_5_addition, data, self) + self.usLowerOpticalPointSize /= 20 + self.usUpperOpticalPointSize /= 20 + elif self.version != 0: + from fontTools import ttLib + + raise ttLib.TTLibError( + "unknown format for OS/2 table: version %s" % self.version + ) + if len(data): + log.warning("too much 'OS/2' table data") + + self.panose = sstruct.unpack(panoseFormat, self.panose, Panose()) + + def compile(self, ttFont): + self.updateFirstAndLastCharIndex(ttFont) + panose = self.panose + head = ttFont["head"] + if (self.fsSelection & 1) and not (head.macStyle & 1 << 1): + log.warning( + "fsSelection bit 0 (italic) and " + "head table macStyle bit 1 (italic) should match" + ) + if (self.fsSelection & 1 << 5) and not (head.macStyle & 1): + log.warning( + "fsSelection bit 5 (bold) and " + "head table macStyle bit 0 (bold) should match" + ) + if (self.fsSelection & 1 << 6) and (self.fsSelection & 1 + (1 << 5)): + log.warning( + "fsSelection bit 6 (regular) is set, " + "bits 0 (italic) and 5 (bold) must be clear" + ) + if self.version < 4 and self.fsSelection & 0b1110000000: + log.warning( + "fsSelection bits 7, 8 and 9 are only defined in " + "OS/2 table version 4 and up: version %s", + self.version, + ) + self.panose = sstruct.pack(panoseFormat, self.panose) + if self.version == 0: + data = sstruct.pack(OS2_format_0, self) + elif self.version == 1: + data = sstruct.pack(OS2_format_1, self) + elif self.version in (2, 3, 4): + data = sstruct.pack(OS2_format_2, self) + elif self.version == 5: + d = self.__dict__.copy() + d["usLowerOpticalPointSize"] = round(self.usLowerOpticalPointSize * 20) + d["usUpperOpticalPointSize"] = round(self.usUpperOpticalPointSize * 20) + data = sstruct.pack(OS2_format_5, d) + else: + from fontTools import ttLib + + raise ttLib.TTLibError( + "unknown format for OS/2 table: version %s" % self.version + ) + self.panose = panose + return data + + def toXML(self, writer, ttFont): + writer.comment( + "The fields 'usFirstCharIndex' and 'usLastCharIndex'\n" + "will be recalculated by the compiler" + ) + writer.newline() + if self.version == 1: + format = OS2_format_1 + elif self.version in (2, 3, 4): + format = OS2_format_2 + elif self.version == 5: + format = OS2_format_5 + else: + format = OS2_format_0 + formatstring, names, fixes = sstruct.getformat(format) + for name in names: + value = getattr(self, name) + if name == "panose": + writer.begintag("panose") + writer.newline() + value.toXML(writer, ttFont) + writer.endtag("panose") + elif name in ( + "ulUnicodeRange1", + "ulUnicodeRange2", + "ulUnicodeRange3", + "ulUnicodeRange4", + "ulCodePageRange1", + "ulCodePageRange2", + ): + writer.simpletag(name, value=num2binary(value)) + elif name in ("fsType", "fsSelection"): + writer.simpletag(name, value=num2binary(value, 16)) + elif name == "achVendID": + writer.simpletag(name, value=repr(value)[1:-1]) + else: + writer.simpletag(name, value=value) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "panose": + self.panose = panose = Panose() + for element in content: + if isinstance(element, tuple): + name, attrs, content = element + panose.fromXML(name, attrs, content, ttFont) + elif name in ( + "ulUnicodeRange1", + "ulUnicodeRange2", + "ulUnicodeRange3", + "ulUnicodeRange4", + "ulCodePageRange1", + "ulCodePageRange2", + "fsType", + "fsSelection", + ): + setattr(self, name, binary2num(attrs["value"])) + elif name == "achVendID": + setattr(self, name, safeEval("'''" + attrs["value"] + "'''")) + else: + setattr(self, name, safeEval(attrs["value"])) + + def updateFirstAndLastCharIndex(self, ttFont): + if "cmap" not in ttFont: + return + codes = set() + for table in getattr(ttFont["cmap"], "tables", []): + if table.isUnicode(): + codes.update(table.cmap.keys()) + if codes: + minCode = min(codes) + maxCode = max(codes) + # USHORT cannot hold codepoints greater than 0xFFFF + self.usFirstCharIndex = min(0xFFFF, minCode) + self.usLastCharIndex = min(0xFFFF, maxCode) + + # misspelled attributes kept for legacy reasons + + @property + def usMaxContex(self): + return self.usMaxContext + + @usMaxContex.setter + def usMaxContex(self, value): + self.usMaxContext = value + + @property + def fsFirstCharIndex(self): + return self.usFirstCharIndex + + @fsFirstCharIndex.setter + def fsFirstCharIndex(self, value): + self.usFirstCharIndex = value + + @property + def fsLastCharIndex(self): + return self.usLastCharIndex + + @fsLastCharIndex.setter + def fsLastCharIndex(self, value): + self.usLastCharIndex = value + + def getUnicodeRanges(self): + """Return the set of 'ulUnicodeRange*' bits currently enabled.""" + bits = set() + ul1, ul2 = self.ulUnicodeRange1, self.ulUnicodeRange2 + ul3, ul4 = self.ulUnicodeRange3, self.ulUnicodeRange4 + for i in range(32): + if ul1 & (1 << i): + bits.add(i) + if ul2 & (1 << i): + bits.add(i + 32) + if ul3 & (1 << i): + bits.add(i + 64) + if ul4 & (1 << i): + bits.add(i + 96) + return bits + + def setUnicodeRanges(self, bits): + """Set the 'ulUnicodeRange*' fields to the specified 'bits'.""" + ul1, ul2, ul3, ul4 = 0, 0, 0, 0 + for bit in bits: + if 0 <= bit < 32: + ul1 |= 1 << bit + elif 32 <= bit < 64: + ul2 |= 1 << (bit - 32) + elif 64 <= bit < 96: + ul3 |= 1 << (bit - 64) + elif 96 <= bit < 128: + ul4 |= 1 << (bit - 96) + else: + raise ValueError("expected 0 <= int <= 127, found: %r" % bit) + self.ulUnicodeRange1, self.ulUnicodeRange2 = ul1, ul2 + self.ulUnicodeRange3, self.ulUnicodeRange4 = ul3, ul4 + + def recalcUnicodeRanges(self, ttFont, pruneOnly=False): + """Intersect the codepoints in the font's Unicode cmap subtables with + the Unicode block ranges defined in the OpenType specification (v1.7), + and set the respective 'ulUnicodeRange*' bits if there is at least ONE + intersection. + If 'pruneOnly' is True, only clear unused bits with NO intersection. + """ + unicodes = set() + for table in ttFont["cmap"].tables: + if table.isUnicode(): + unicodes.update(table.cmap.keys()) + if pruneOnly: + empty = intersectUnicodeRanges(unicodes, inverse=True) + bits = self.getUnicodeRanges() - empty + else: + bits = intersectUnicodeRanges(unicodes) + self.setUnicodeRanges(bits) + return bits + + def getCodePageRanges(self): + """Return the set of 'ulCodePageRange*' bits currently enabled.""" + bits = set() + if self.version < 1: + return bits + ul1, ul2 = self.ulCodePageRange1, self.ulCodePageRange2 + for i in range(32): + if ul1 & (1 << i): + bits.add(i) + if ul2 & (1 << i): + bits.add(i + 32) + return bits + + def setCodePageRanges(self, bits): + """Set the 'ulCodePageRange*' fields to the specified 'bits'.""" + ul1, ul2 = 0, 0 + for bit in bits: + if 0 <= bit < 32: + ul1 |= 1 << bit + elif 32 <= bit < 64: + ul2 |= 1 << (bit - 32) + else: + raise ValueError(f"expected 0 <= int <= 63, found: {bit:r}") + if self.version < 1: + self.version = 1 + self.ulCodePageRange1, self.ulCodePageRange2 = ul1, ul2 + + def recalcCodePageRanges(self, ttFont, pruneOnly=False): + unicodes = set() + for table in ttFont["cmap"].tables: + if table.isUnicode(): + unicodes.update(table.cmap.keys()) + bits = calcCodePageRanges(unicodes) + if pruneOnly: + bits &= self.getCodePageRanges() + # when no codepage ranges can be enabled, fall back to enabling bit 0 + # (Latin 1) so that the font works in MS Word: + # https://github.com/googlei18n/fontmake/issues/468 + if not bits: + bits = {0} + self.setCodePageRanges(bits) + return bits + + def recalcAvgCharWidth(self, ttFont): + """Recalculate xAvgCharWidth using metrics from ttFont's 'hmtx' table. + + Set it to 0 if the unlikely event 'hmtx' table is not found. + """ + avg_width = 0 + hmtx = ttFont.get("hmtx") + if hmtx is not None: + widths = [width for width, _ in hmtx.metrics.values() if width > 0] + if widths: + avg_width = otRound(sum(widths) / len(widths)) + self.xAvgCharWidth = avg_width + return avg_width + + +# Unicode ranges data from the OpenType OS/2 table specification v1.7 + +OS2_UNICODE_RANGES = ( + (("Basic Latin", (0x0000, 0x007F)),), + (("Latin-1 Supplement", (0x0080, 0x00FF)),), + (("Latin Extended-A", (0x0100, 0x017F)),), + (("Latin Extended-B", (0x0180, 0x024F)),), + ( + ("IPA Extensions", (0x0250, 0x02AF)), + ("Phonetic Extensions", (0x1D00, 0x1D7F)), + ("Phonetic Extensions Supplement", (0x1D80, 0x1DBF)), + ), + ( + ("Spacing Modifier Letters", (0x02B0, 0x02FF)), + ("Modifier Tone Letters", (0xA700, 0xA71F)), + ), + ( + ("Combining Diacritical Marks", (0x0300, 0x036F)), + ("Combining Diacritical Marks Supplement", (0x1DC0, 0x1DFF)), + ), + (("Greek and Coptic", (0x0370, 0x03FF)),), + (("Coptic", (0x2C80, 0x2CFF)),), + ( + ("Cyrillic", (0x0400, 0x04FF)), + ("Cyrillic Supplement", (0x0500, 0x052F)), + ("Cyrillic Extended-A", (0x2DE0, 0x2DFF)), + ("Cyrillic Extended-B", (0xA640, 0xA69F)), + ), + (("Armenian", (0x0530, 0x058F)),), + (("Hebrew", (0x0590, 0x05FF)),), + (("Vai", (0xA500, 0xA63F)),), + (("Arabic", (0x0600, 0x06FF)), ("Arabic Supplement", (0x0750, 0x077F))), + (("NKo", (0x07C0, 0x07FF)),), + (("Devanagari", (0x0900, 0x097F)),), + (("Bengali", (0x0980, 0x09FF)),), + (("Gurmukhi", (0x0A00, 0x0A7F)),), + (("Gujarati", (0x0A80, 0x0AFF)),), + (("Oriya", (0x0B00, 0x0B7F)),), + (("Tamil", (0x0B80, 0x0BFF)),), + (("Telugu", (0x0C00, 0x0C7F)),), + (("Kannada", (0x0C80, 0x0CFF)),), + (("Malayalam", (0x0D00, 0x0D7F)),), + (("Thai", (0x0E00, 0x0E7F)),), + (("Lao", (0x0E80, 0x0EFF)),), + (("Georgian", (0x10A0, 0x10FF)), ("Georgian Supplement", (0x2D00, 0x2D2F))), + (("Balinese", (0x1B00, 0x1B7F)),), + (("Hangul Jamo", (0x1100, 0x11FF)),), + ( + ("Latin Extended Additional", (0x1E00, 0x1EFF)), + ("Latin Extended-C", (0x2C60, 0x2C7F)), + ("Latin Extended-D", (0xA720, 0xA7FF)), + ), + (("Greek Extended", (0x1F00, 0x1FFF)),), + ( + ("General Punctuation", (0x2000, 0x206F)), + ("Supplemental Punctuation", (0x2E00, 0x2E7F)), + ), + (("Superscripts And Subscripts", (0x2070, 0x209F)),), + (("Currency Symbols", (0x20A0, 0x20CF)),), + (("Combining Diacritical Marks For Symbols", (0x20D0, 0x20FF)),), + (("Letterlike Symbols", (0x2100, 0x214F)),), + (("Number Forms", (0x2150, 0x218F)),), + ( + ("Arrows", (0x2190, 0x21FF)), + ("Supplemental Arrows-A", (0x27F0, 0x27FF)), + ("Supplemental Arrows-B", (0x2900, 0x297F)), + ("Miscellaneous Symbols and Arrows", (0x2B00, 0x2BFF)), + ), + ( + ("Mathematical Operators", (0x2200, 0x22FF)), + ("Supplemental Mathematical Operators", (0x2A00, 0x2AFF)), + ("Miscellaneous Mathematical Symbols-A", (0x27C0, 0x27EF)), + ("Miscellaneous Mathematical Symbols-B", (0x2980, 0x29FF)), + ), + (("Miscellaneous Technical", (0x2300, 0x23FF)),), + (("Control Pictures", (0x2400, 0x243F)),), + (("Optical Character Recognition", (0x2440, 0x245F)),), + (("Enclosed Alphanumerics", (0x2460, 0x24FF)),), + (("Box Drawing", (0x2500, 0x257F)),), + (("Block Elements", (0x2580, 0x259F)),), + (("Geometric Shapes", (0x25A0, 0x25FF)),), + (("Miscellaneous Symbols", (0x2600, 0x26FF)),), + (("Dingbats", (0x2700, 0x27BF)),), + (("CJK Symbols And Punctuation", (0x3000, 0x303F)),), + (("Hiragana", (0x3040, 0x309F)),), + ( + ("Katakana", (0x30A0, 0x30FF)), + ("Katakana Phonetic Extensions", (0x31F0, 0x31FF)), + ), + (("Bopomofo", (0x3100, 0x312F)), ("Bopomofo Extended", (0x31A0, 0x31BF))), + (("Hangul Compatibility Jamo", (0x3130, 0x318F)),), + (("Phags-pa", (0xA840, 0xA87F)),), + (("Enclosed CJK Letters And Months", (0x3200, 0x32FF)),), + (("CJK Compatibility", (0x3300, 0x33FF)),), + (("Hangul Syllables", (0xAC00, 0xD7AF)),), + (("Non-Plane 0 *", (0xD800, 0xDFFF)),), + (("Phoenician", (0x10900, 0x1091F)),), + ( + ("CJK Unified Ideographs", (0x4E00, 0x9FFF)), + ("CJK Radicals Supplement", (0x2E80, 0x2EFF)), + ("Kangxi Radicals", (0x2F00, 0x2FDF)), + ("Ideographic Description Characters", (0x2FF0, 0x2FFF)), + ("CJK Unified Ideographs Extension A", (0x3400, 0x4DBF)), + ("CJK Unified Ideographs Extension B", (0x20000, 0x2A6DF)), + ("Kanbun", (0x3190, 0x319F)), + ), + (("Private Use Area (plane 0)", (0xE000, 0xF8FF)),), + ( + ("CJK Strokes", (0x31C0, 0x31EF)), + ("CJK Compatibility Ideographs", (0xF900, 0xFAFF)), + ("CJK Compatibility Ideographs Supplement", (0x2F800, 0x2FA1F)), + ), + (("Alphabetic Presentation Forms", (0xFB00, 0xFB4F)),), + (("Arabic Presentation Forms-A", (0xFB50, 0xFDFF)),), + (("Combining Half Marks", (0xFE20, 0xFE2F)),), + ( + ("Vertical Forms", (0xFE10, 0xFE1F)), + ("CJK Compatibility Forms", (0xFE30, 0xFE4F)), + ), + (("Small Form Variants", (0xFE50, 0xFE6F)),), + (("Arabic Presentation Forms-B", (0xFE70, 0xFEFF)),), + (("Halfwidth And Fullwidth Forms", (0xFF00, 0xFFEF)),), + (("Specials", (0xFFF0, 0xFFFF)),), + (("Tibetan", (0x0F00, 0x0FFF)),), + (("Syriac", (0x0700, 0x074F)),), + (("Thaana", (0x0780, 0x07BF)),), + (("Sinhala", (0x0D80, 0x0DFF)),), + (("Myanmar", (0x1000, 0x109F)),), + ( + ("Ethiopic", (0x1200, 0x137F)), + ("Ethiopic Supplement", (0x1380, 0x139F)), + ("Ethiopic Extended", (0x2D80, 0x2DDF)), + ), + (("Cherokee", (0x13A0, 0x13FF)),), + (("Unified Canadian Aboriginal Syllabics", (0x1400, 0x167F)),), + (("Ogham", (0x1680, 0x169F)),), + (("Runic", (0x16A0, 0x16FF)),), + (("Khmer", (0x1780, 0x17FF)), ("Khmer Symbols", (0x19E0, 0x19FF))), + (("Mongolian", (0x1800, 0x18AF)),), + (("Braille Patterns", (0x2800, 0x28FF)),), + (("Yi Syllables", (0xA000, 0xA48F)), ("Yi Radicals", (0xA490, 0xA4CF))), + ( + ("Tagalog", (0x1700, 0x171F)), + ("Hanunoo", (0x1720, 0x173F)), + ("Buhid", (0x1740, 0x175F)), + ("Tagbanwa", (0x1760, 0x177F)), + ), + (("Old Italic", (0x10300, 0x1032F)),), + (("Gothic", (0x10330, 0x1034F)),), + (("Deseret", (0x10400, 0x1044F)),), + ( + ("Byzantine Musical Symbols", (0x1D000, 0x1D0FF)), + ("Musical Symbols", (0x1D100, 0x1D1FF)), + ("Ancient Greek Musical Notation", (0x1D200, 0x1D24F)), + ), + (("Mathematical Alphanumeric Symbols", (0x1D400, 0x1D7FF)),), + ( + ("Private Use (plane 15)", (0xF0000, 0xFFFFD)), + ("Private Use (plane 16)", (0x100000, 0x10FFFD)), + ), + ( + ("Variation Selectors", (0xFE00, 0xFE0F)), + ("Variation Selectors Supplement", (0xE0100, 0xE01EF)), + ), + (("Tags", (0xE0000, 0xE007F)),), + (("Limbu", (0x1900, 0x194F)),), + (("Tai Le", (0x1950, 0x197F)),), + (("New Tai Lue", (0x1980, 0x19DF)),), + (("Buginese", (0x1A00, 0x1A1F)),), + (("Glagolitic", (0x2C00, 0x2C5F)),), + (("Tifinagh", (0x2D30, 0x2D7F)),), + (("Yijing Hexagram Symbols", (0x4DC0, 0x4DFF)),), + (("Syloti Nagri", (0xA800, 0xA82F)),), + ( + ("Linear B Syllabary", (0x10000, 0x1007F)), + ("Linear B Ideograms", (0x10080, 0x100FF)), + ("Aegean Numbers", (0x10100, 0x1013F)), + ), + (("Ancient Greek Numbers", (0x10140, 0x1018F)),), + (("Ugaritic", (0x10380, 0x1039F)),), + (("Old Persian", (0x103A0, 0x103DF)),), + (("Shavian", (0x10450, 0x1047F)),), + (("Osmanya", (0x10480, 0x104AF)),), + (("Cypriot Syllabary", (0x10800, 0x1083F)),), + (("Kharoshthi", (0x10A00, 0x10A5F)),), + (("Tai Xuan Jing Symbols", (0x1D300, 0x1D35F)),), + ( + ("Cuneiform", (0x12000, 0x123FF)), + ("Cuneiform Numbers and Punctuation", (0x12400, 0x1247F)), + ), + (("Counting Rod Numerals", (0x1D360, 0x1D37F)),), + (("Sundanese", (0x1B80, 0x1BBF)),), + (("Lepcha", (0x1C00, 0x1C4F)),), + (("Ol Chiki", (0x1C50, 0x1C7F)),), + (("Saurashtra", (0xA880, 0xA8DF)),), + (("Kayah Li", (0xA900, 0xA92F)),), + (("Rejang", (0xA930, 0xA95F)),), + (("Cham", (0xAA00, 0xAA5F)),), + (("Ancient Symbols", (0x10190, 0x101CF)),), + (("Phaistos Disc", (0x101D0, 0x101FF)),), + ( + ("Carian", (0x102A0, 0x102DF)), + ("Lycian", (0x10280, 0x1029F)), + ("Lydian", (0x10920, 0x1093F)), + ), + (("Domino Tiles", (0x1F030, 0x1F09F)), ("Mahjong Tiles", (0x1F000, 0x1F02F))), +) + + +_unicodeStarts = [] +_unicodeValues = [None] + + +def _getUnicodeRanges(): + # build the ranges of codepoints for each unicode range bit, and cache result + if not _unicodeStarts: + unicodeRanges = [ + (start, (stop, bit)) + for bit, blocks in enumerate(OS2_UNICODE_RANGES) + for _, (start, stop) in blocks + ] + for start, (stop, bit) in sorted(unicodeRanges): + _unicodeStarts.append(start) + _unicodeValues.append((stop, bit)) + return _unicodeStarts, _unicodeValues + + +def intersectUnicodeRanges(unicodes, inverse=False): + """Intersect a sequence of (int) Unicode codepoints with the Unicode block + ranges defined in the OpenType specification v1.7, and return the set of + 'ulUnicodeRanges' bits for which there is at least ONE intersection. + If 'inverse' is True, return the the bits for which there is NO intersection. + + >>> intersectUnicodeRanges([0x0410]) == {9} + True + >>> intersectUnicodeRanges([0x0410, 0x1F000]) == {9, 57, 122} + True + >>> intersectUnicodeRanges([0x0410, 0x1F000], inverse=True) == ( + ... set(range(len(OS2_UNICODE_RANGES))) - {9, 57, 122}) + True + """ + unicodes = set(unicodes) + unicodestarts, unicodevalues = _getUnicodeRanges() + bits = set() + for code in unicodes: + stop, bit = unicodevalues[bisect.bisect(unicodestarts, code)] + if code <= stop: + bits.add(bit) + # The spec says that bit 57 ("Non Plane 0") implies that there's + # at least one codepoint beyond the BMP; so I also include all + # the non-BMP codepoints here + if any(0x10000 <= code < 0x110000 for code in unicodes): + bits.add(57) + return set(range(len(OS2_UNICODE_RANGES))) - bits if inverse else bits + + +def calcCodePageRanges(unicodes): + """Given a set of Unicode codepoints (integers), calculate the + corresponding OS/2 CodePage range bits. + This is a direct translation of FontForge implementation: + https://github.com/fontforge/fontforge/blob/7b2c074/fontforge/tottf.c#L3158 + """ + bits = set() + hasAscii = set(range(0x20, 0x7E)).issubset(unicodes) + hasLineart = ord("┤") in unicodes + + for uni in unicodes: + if uni == ord("Þ") and hasAscii: + bits.add(0) # Latin 1 + elif uni == ord("Ľ") and hasAscii: + bits.add(1) # Latin 2: Eastern Europe + if hasLineart: + bits.add(58) # Latin 2 + elif uni == ord("Б"): + bits.add(2) # Cyrillic + if ord("Ѕ") in unicodes and hasLineart: + bits.add(57) # IBM Cyrillic + if ord("╜") in unicodes and hasLineart: + bits.add(49) # MS-DOS Russian + elif uni == ord("Ά"): + bits.add(3) # Greek + if hasLineart and ord("½") in unicodes: + bits.add(48) # IBM Greek + if hasLineart and ord("√") in unicodes: + bits.add(60) # Greek, former 437 G + elif uni == ord("İ") and hasAscii: + bits.add(4) # Turkish + if hasLineart: + bits.add(56) # IBM turkish + elif uni == ord("א"): + bits.add(5) # Hebrew + if hasLineart and ord("√") in unicodes: + bits.add(53) # Hebrew + elif uni == ord("ر"): + bits.add(6) # Arabic + if ord("√") in unicodes: + bits.add(51) # Arabic + if hasLineart: + bits.add(61) # Arabic; ASMO 708 + elif uni == ord("ŗ") and hasAscii: + bits.add(7) # Windows Baltic + if hasLineart: + bits.add(59) # MS-DOS Baltic + elif uni == ord("₫") and hasAscii: + bits.add(8) # Vietnamese + elif uni == ord("ๅ"): + bits.add(16) # Thai + elif uni == ord("エ"): + bits.add(17) # JIS/Japan + elif uni == ord("ㄅ"): + bits.add(18) # Chinese: Simplified + elif uni == ord("ㄱ"): + bits.add(19) # Korean wansung + elif uni == ord("央"): + bits.add(20) # Chinese: Traditional + elif uni == ord("곴"): + bits.add(21) # Korean Johab + elif uni == ord("♥") and hasAscii: + bits.add(30) # OEM Character Set + # TODO: Symbol bit has a special meaning (check the spec), we need + # to confirm if this is wanted by default. + # elif chr(0xF000) <= char <= chr(0xF0FF): + # codepageRanges.add(31) # Symbol Character Set + elif uni == ord("þ") and hasAscii and hasLineart: + bits.add(54) # MS-DOS Icelandic + elif uni == ord("╚") and hasAscii: + bits.add(62) # WE/Latin 1 + bits.add(63) # US + elif hasAscii and hasLineart and ord("√") in unicodes: + if uni == ord("Å"): + bits.add(50) # MS-DOS Nordic + elif uni == ord("é"): + bits.add(52) # MS-DOS Canadian French + elif uni == ord("õ"): + bits.add(55) # MS-DOS Portuguese + + if hasAscii and ord("‰") in unicodes and ord("∑") in unicodes: + bits.add(29) # Macintosh Character Set (US Roman) + + return bits + + +if __name__ == "__main__": + import doctest, sys + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S_T_A_T_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S_T_A_T_.py new file mode 100644 index 0000000000000000000000000000000000000000..86e1271a9817ecb71b663e60ffa685cbf9205bd5 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S_T_A_T_.py @@ -0,0 +1,15 @@ +from .otBase import BaseTTXConverter + + +class table_S_T_A_T_(BaseTTXConverter): + """Style Attributes table + + The ``STAT`` table records stylistic or typeface-design attributes that + differentiate the individual fonts within a font family from one another. + Those attributes can be used to assist users when navigating the style + variations of a variable font or a family of static fonts. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/stat + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S_V_G_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S_V_G_.py new file mode 100644 index 0000000000000000000000000000000000000000..76e860c5f627d7b6ed0a3d8fef29efde7b647a70 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S_V_G_.py @@ -0,0 +1,223 @@ +"""Compiles/decompiles SVG table. + +https://docs.microsoft.com/en-us/typography/opentype/spec/svg + +The XML format is: + +.. code-block:: xml + + + + <complete SVG doc> ]] + </svgDoc> + ... + <svgDoc endGlyphID="n" startGlyphID="m"> + <![CDATA[ <complete SVG doc> ]] + </svgDoc> + </SVG> +""" + +from fontTools.misc.textTools import bytesjoin, safeEval, strjoin, tobytes, tostr +from fontTools.misc import sstruct +from . import DefaultTable +from collections.abc import Sequence +from dataclasses import dataclass, astuple +from io import BytesIO +import struct +import logging + + +log = logging.getLogger(__name__) + + +SVG_format_0 = """ + > # big endian + version: H + offsetToSVGDocIndex: L + reserved: L +""" + +SVG_format_0Size = sstruct.calcsize(SVG_format_0) + +doc_index_entry_format_0 = """ + > # big endian + startGlyphID: H + endGlyphID: H + svgDocOffset: L + svgDocLength: L +""" + +doc_index_entry_format_0Size = sstruct.calcsize(doc_index_entry_format_0) + + +class table_S_V_G_(DefaultTable.DefaultTable): + """Scalable Vector Graphics table + + The ``SVG`` table contains representations for glyphs in the SVG + image format. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/stat + """ + + def decompile(self, data, ttFont): + self.docList = [] + # Version 0 is the standardized version of the table; and current. + # https://www.microsoft.com/typography/otspec/svg.htm + sstruct.unpack(SVG_format_0, data[:SVG_format_0Size], self) + if self.version != 0: + log.warning( + "Unknown SVG table version '%s'. Decompiling as version 0.", + self.version, + ) + # read in SVG Documents Index + # data starts with the first entry of the entry list. + pos = subTableStart = self.offsetToSVGDocIndex + self.numEntries = struct.unpack(">H", data[pos : pos + 2])[0] + pos += 2 + if self.numEntries > 0: + data2 = data[pos:] + entries = [] + for i in range(self.numEntries): + record_data = data2[ + i + * doc_index_entry_format_0Size : (i + 1) + * doc_index_entry_format_0Size + ] + docIndexEntry = sstruct.unpack( + doc_index_entry_format_0, record_data, DocumentIndexEntry() + ) + entries.append(docIndexEntry) + + for entry in entries: + start = entry.svgDocOffset + subTableStart + end = start + entry.svgDocLength + doc = data[start:end] + compressed = False + if doc.startswith(b"\x1f\x8b"): + import gzip + + bytesIO = BytesIO(doc) + with gzip.GzipFile(None, "r", fileobj=bytesIO) as gunzipper: + doc = gunzipper.read() + del bytesIO + compressed = True + doc = tostr(doc, "utf_8") + self.docList.append( + SVGDocument(doc, entry.startGlyphID, entry.endGlyphID, compressed) + ) + + def compile(self, ttFont): + version = 0 + offsetToSVGDocIndex = ( + SVG_format_0Size # I start the SVGDocIndex right after the header. + ) + # get SGVDoc info. + docList = [] + entryList = [] + numEntries = len(self.docList) + datum = struct.pack(">H", numEntries) + entryList.append(datum) + curOffset = len(datum) + doc_index_entry_format_0Size * numEntries + seenDocs = {} + allCompressed = getattr(self, "compressed", False) + for i, doc in enumerate(self.docList): + if isinstance(doc, (list, tuple)): + doc = SVGDocument(*doc) + self.docList[i] = doc + docBytes = tobytes(doc.data, encoding="utf_8") + if (allCompressed or doc.compressed) and not docBytes.startswith( + b"\x1f\x8b" + ): + import gzip + + bytesIO = BytesIO() + # mtime=0 strips the useless timestamp and makes gzip output reproducible; + # equivalent to `gzip -n` + with gzip.GzipFile(None, "w", fileobj=bytesIO, mtime=0) as gzipper: + gzipper.write(docBytes) + gzipped = bytesIO.getvalue() + if len(gzipped) < len(docBytes): + docBytes = gzipped + del gzipped, bytesIO + docLength = len(docBytes) + if docBytes in seenDocs: + docOffset = seenDocs[docBytes] + else: + docOffset = curOffset + curOffset += docLength + seenDocs[docBytes] = docOffset + docList.append(docBytes) + entry = struct.pack( + ">HHLL", doc.startGlyphID, doc.endGlyphID, docOffset, docLength + ) + entryList.append(entry) + entryList.extend(docList) + svgDocData = bytesjoin(entryList) + + reserved = 0 + header = struct.pack(">HLL", version, offsetToSVGDocIndex, reserved) + data = [header, svgDocData] + data = bytesjoin(data) + return data + + def toXML(self, writer, ttFont): + for i, doc in enumerate(self.docList): + if isinstance(doc, (list, tuple)): + doc = SVGDocument(*doc) + self.docList[i] = doc + attrs = {"startGlyphID": doc.startGlyphID, "endGlyphID": doc.endGlyphID} + if doc.compressed: + attrs["compressed"] = 1 + writer.begintag("svgDoc", **attrs) + writer.newline() + writer.writecdata(doc.data) + writer.newline() + writer.endtag("svgDoc") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "svgDoc": + if not hasattr(self, "docList"): + self.docList = [] + doc = strjoin(content) + doc = doc.strip() + startGID = int(attrs["startGlyphID"]) + endGID = int(attrs["endGlyphID"]) + compressed = bool(safeEval(attrs.get("compressed", "0"))) + self.docList.append(SVGDocument(doc, startGID, endGID, compressed)) + else: + log.warning("Unknown %s %s", name, content) + + +class DocumentIndexEntry(object): + def __init__(self): + self.startGlyphID = None # USHORT + self.endGlyphID = None # USHORT + self.svgDocOffset = None # ULONG + self.svgDocLength = None # ULONG + + def __repr__(self): + return ( + "startGlyphID: %s, endGlyphID: %s, svgDocOffset: %s, svgDocLength: %s" + % (self.startGlyphID, self.endGlyphID, self.svgDocOffset, self.svgDocLength) + ) + + +@dataclass +class SVGDocument(Sequence): + data: str + startGlyphID: int + endGlyphID: int + compressed: bool = False + + # Previously, the SVG table's docList attribute contained a lists of 3 items: + # [doc, startGlyphID, endGlyphID]; later, we added a `compressed` attribute. + # For backward compatibility with code that depends of them being sequences of + # fixed length=3, we subclass the Sequence abstract base class and pretend only + # the first three items are present. 'compressed' is only accessible via named + # attribute lookup like regular dataclasses: i.e. `doc.compressed`, not `doc[3]` + def __getitem__(self, index): + return astuple(self)[:3][index] + + def __len__(self): + return 3 diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S__i_l_f.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S__i_l_f.py new file mode 100644 index 0000000000000000000000000000000000000000..e8090af1e520737919e6041c259c112a15695035 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S__i_l_f.py @@ -0,0 +1,1040 @@ +from fontTools.misc import sstruct +from fontTools.misc.fixedTools import floatToFixedToStr +from fontTools.misc.textTools import byteord, safeEval + +# from itertools import * +from . import DefaultTable +from . import grUtils +from array import array +from functools import reduce +import struct, re, sys + +Silf_hdr_format = """ + > + version: 16.16F +""" + +Silf_hdr_format_3 = """ + > + version: 16.16F + compilerVersion: L + numSilf: H + x + x +""" + +Silf_part1_format_v3 = """ + > + ruleVersion: 16.16F + passOffset: H + pseudosOffset: H +""" + +Silf_part1_format = """ + > + maxGlyphID: H + extraAscent: h + extraDescent: h + numPasses: B + iSubst: B + iPos: B + iJust: B + iBidi: B + flags: B + maxPreContext: B + maxPostContext: B + attrPseudo: B + attrBreakWeight: B + attrDirectionality: B + attrMirroring: B + attrSkipPasses: B + numJLevels: B +""" + +Silf_justify_format = """ + > + attrStretch: B + attrShrink: B + attrStep: B + attrWeight: B + runto: B + x + x + x +""" + +Silf_part2_format = """ + > + numLigComp: H + numUserDefn: B + maxCompPerLig: B + direction: B + attCollisions: B + x + x + x + numCritFeatures: B +""" + +Silf_pseudomap_format = """ + > + unicode: L + nPseudo: H +""" + +Silf_pseudomap_format_h = """ + > + unicode: H + nPseudo: H +""" + +Silf_classmap_format = """ + > + numClass: H + numLinear: H +""" + +Silf_lookupclass_format = """ + > + numIDs: H + searchRange: H + entrySelector: H + rangeShift: H +""" + +Silf_lookuppair_format = """ + > + glyphId: H + index: H +""" + +Silf_pass_format = """ + > + flags: B + maxRuleLoop: B + maxRuleContext: B + maxBackup: B + numRules: H + fsmOffset: H + pcCode: L + rcCode: L + aCode: L + oDebug: L + numRows: H + numTransitional: H + numSuccess: H + numColumns: H +""" + +aCode_info = ( + ("NOP", 0), + ("PUSH_BYTE", "b"), + ("PUSH_BYTE_U", "B"), + ("PUSH_SHORT", ">h"), + ("PUSH_SHORT_U", ">H"), + ("PUSH_LONG", ">L"), + ("ADD", 0), + ("SUB", 0), + ("MUL", 0), + ("DIV", 0), + ("MIN", 0), + ("MAX", 0), + ("NEG", 0), + ("TRUNC8", 0), + ("TRUNC16", 0), + ("COND", 0), + ("AND", 0), # x10 + ("OR", 0), + ("NOT", 0), + ("EQUAL", 0), + ("NOT_EQ", 0), + ("LESS", 0), + ("GTR", 0), + ("LESS_EQ", 0), + ("GTR_EQ", 0), + ("NEXT", 0), + ("NEXT_N", "b"), + ("COPY_NEXT", 0), + ("PUT_GLYPH_8BIT_OBS", "B"), + ("PUT_SUBS_8BIT_OBS", "bBB"), + ("PUT_COPY", "b"), + ("INSERT", 0), + ("DELETE", 0), # x20 + ("ASSOC", -1), + ("CNTXT_ITEM", "bB"), + ("ATTR_SET", "B"), + ("ATTR_ADD", "B"), + ("ATTR_SUB", "B"), + ("ATTR_SET_SLOT", "B"), + ("IATTR_SET_SLOT", "BB"), + ("PUSH_SLOT_ATTR", "Bb"), + ("PUSH_GLYPH_ATTR_OBS", "Bb"), + ("PUSH_GLYPH_METRIC", "Bbb"), + ("PUSH_FEAT", "Bb"), + ("PUSH_ATT_TO_GATTR_OBS", "Bb"), + ("PUSH_ATT_TO_GLYPH_METRIC", "Bbb"), + ("PUSH_ISLOT_ATTR", "Bbb"), + ("PUSH_IGLYPH_ATTR", "Bbb"), + ("POP_RET", 0), # x30 + ("RET_ZERO", 0), + ("RET_TRUE", 0), + ("IATTR_SET", "BB"), + ("IATTR_ADD", "BB"), + ("IATTR_SUB", "BB"), + ("PUSH_PROC_STATE", "B"), + ("PUSH_VERSION", 0), + ("PUT_SUBS", ">bHH"), + ("PUT_SUBS2", 0), + ("PUT_SUBS3", 0), + ("PUT_GLYPH", ">H"), + ("PUSH_GLYPH_ATTR", ">Hb"), + ("PUSH_ATT_TO_GLYPH_ATTR", ">Hb"), + ("BITOR", 0), + ("BITAND", 0), + ("BITNOT", 0), # x40 + ("BITSET", ">HH"), + ("SET_FEAT", "Bb"), +) +aCode_map = dict([(x[0], (i, x[1])) for i, x in enumerate(aCode_info)]) + + +def disassemble(aCode): + codelen = len(aCode) + pc = 0 + res = [] + while pc < codelen: + opcode = byteord(aCode[pc : pc + 1]) + if opcode > len(aCode_info): + instr = aCode_info[0] + else: + instr = aCode_info[opcode] + pc += 1 + if instr[1] != 0 and pc >= codelen: + return res + if instr[1] == -1: + count = byteord(aCode[pc]) + fmt = "%dB" % count + pc += 1 + elif instr[1] == 0: + fmt = "" + else: + fmt = instr[1] + if fmt == "": + res.append(instr[0]) + continue + parms = struct.unpack_from(fmt, aCode[pc:]) + res.append(instr[0] + "(" + ", ".join(map(str, parms)) + ")") + pc += struct.calcsize(fmt) + return res + + +instre = re.compile(r"^\s*([^(]+)\s*(?:\(([^)]+)\))?") + + +def assemble(instrs): + res = b"" + for inst in instrs: + m = instre.match(inst) + if not m or not m.group(1) in aCode_map: + continue + opcode, parmfmt = aCode_map[m.group(1)] + res += struct.pack("B", opcode) + if m.group(2): + if parmfmt == 0: + continue + parms = [int(x) for x in re.split(r",\s*", m.group(2))] + if parmfmt == -1: + l = len(parms) + res += struct.pack(("%dB" % (l + 1)), l, *parms) + else: + res += struct.pack(parmfmt, *parms) + return res + + +def writecode(tag, writer, instrs): + writer.begintag(tag) + writer.newline() + for l in disassemble(instrs): + writer.write(l) + writer.newline() + writer.endtag(tag) + writer.newline() + + +def readcode(content): + res = [] + for e in content_string(content).split("\n"): + e = e.strip() + if not len(e): + continue + res.append(e) + return assemble(res) + + +attrs_info = ( + "flags", + "extraAscent", + "extraDescent", + "maxGlyphID", + "numLigComp", + "numUserDefn", + "maxCompPerLig", + "direction", + "lbGID", +) +attrs_passindexes = ("iSubst", "iPos", "iJust", "iBidi") +attrs_contexts = ("maxPreContext", "maxPostContext") +attrs_attributes = ( + "attrPseudo", + "attrBreakWeight", + "attrDirectionality", + "attrMirroring", + "attrSkipPasses", + "attCollisions", +) +pass_attrs_info = ( + "flags", + "maxRuleLoop", + "maxRuleContext", + "maxBackup", + "minRulePreContext", + "maxRulePreContext", + "collisionThreshold", +) +pass_attrs_fsm = ("numRows", "numTransitional", "numSuccess", "numColumns") + + +def writesimple(tag, self, writer, *attrkeys): + attrs = dict([(k, getattr(self, k)) for k in attrkeys]) + writer.simpletag(tag, **attrs) + writer.newline() + + +def getSimple(self, attrs, *attr_list): + for k in attr_list: + if k in attrs: + setattr(self, k, int(safeEval(attrs[k]))) + + +def content_string(contents): + res = "" + for element in contents: + if isinstance(element, tuple): + continue + res += element + return res.strip() + + +def wrapline(writer, dat, length=80): + currline = "" + for d in dat: + if len(currline) > length: + writer.write(currline[:-1]) + writer.newline() + currline = "" + currline += d + " " + if len(currline): + writer.write(currline[:-1]) + writer.newline() + + +class _Object: + pass + + +class table_S__i_l_f(DefaultTable.DefaultTable): + """Graphite Rules table + + See also https://graphite.sil.org/graphite_techAbout#graphite-font-tables + """ + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.silfs = [] + + def decompile(self, data, ttFont): + sstruct.unpack2(Silf_hdr_format, data, self) + self.version = float(floatToFixedToStr(self.version, precisionBits=16)) + if self.version >= 5.0: + (data, self.scheme) = grUtils.decompress(data) + sstruct.unpack2(Silf_hdr_format_3, data, self) + base = sstruct.calcsize(Silf_hdr_format_3) + elif self.version < 3.0: + self.numSilf = struct.unpack(">H", data[4:6]) + self.scheme = 0 + self.compilerVersion = 0 + base = 8 + else: + self.scheme = 0 + sstruct.unpack2(Silf_hdr_format_3, data, self) + base = sstruct.calcsize(Silf_hdr_format_3) + + silfoffsets = struct.unpack_from((">%dL" % self.numSilf), data[base:]) + for offset in silfoffsets: + s = Silf() + self.silfs.append(s) + s.decompile(data[offset:], ttFont, self.version) + + def compile(self, ttFont): + self.numSilf = len(self.silfs) + if self.version < 3.0: + hdr = sstruct.pack(Silf_hdr_format, self) + hdr += struct.pack(">HH", self.numSilf, 0) + else: + hdr = sstruct.pack(Silf_hdr_format_3, self) + offset = len(hdr) + 4 * self.numSilf + data = b"" + for s in self.silfs: + hdr += struct.pack(">L", offset) + subdata = s.compile(ttFont, self.version) + offset += len(subdata) + data += subdata + if self.version >= 5.0: + return grUtils.compress(self.scheme, hdr + data) + return hdr + data + + def toXML(self, writer, ttFont): + writer.comment("Attributes starting with _ are informative only") + writer.newline() + writer.simpletag( + "version", + version=self.version, + compilerVersion=self.compilerVersion, + compressionScheme=self.scheme, + ) + writer.newline() + for s in self.silfs: + writer.begintag("silf") + writer.newline() + s.toXML(writer, ttFont, self.version) + writer.endtag("silf") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": + self.scheme = int(safeEval(attrs["compressionScheme"])) + self.version = float(safeEval(attrs["version"])) + self.compilerVersion = int(safeEval(attrs["compilerVersion"])) + return + if name == "silf": + s = Silf() + self.silfs.append(s) + for element in content: + if not isinstance(element, tuple): + continue + tag, attrs, subcontent = element + s.fromXML(tag, attrs, subcontent, ttFont, self.version) + + +class Silf(object): + """A particular Silf subtable""" + + def __init__(self): + self.passes = [] + self.scriptTags = [] + self.critFeatures = [] + self.jLevels = [] + self.pMap = {} + + def decompile(self, data, ttFont, version=2.0): + if version >= 3.0: + _, data = sstruct.unpack2(Silf_part1_format_v3, data, self) + self.ruleVersion = float( + floatToFixedToStr(self.ruleVersion, precisionBits=16) + ) + _, data = sstruct.unpack2(Silf_part1_format, data, self) + for jlevel in range(self.numJLevels): + j, data = sstruct.unpack2(Silf_justify_format, data, _Object()) + self.jLevels.append(j) + _, data = sstruct.unpack2(Silf_part2_format, data, self) + if self.numCritFeatures: + self.critFeatures = struct.unpack_from( + (">%dH" % self.numCritFeatures), data + ) + data = data[self.numCritFeatures * 2 + 1 :] + (numScriptTag,) = struct.unpack_from("B", data) + if numScriptTag: + self.scriptTags = [ + struct.unpack("4s", data[x : x + 4])[0].decode("ascii") + for x in range(1, 1 + 4 * numScriptTag, 4) + ] + data = data[1 + 4 * numScriptTag :] + (self.lbGID,) = struct.unpack(">H", data[:2]) + if self.numPasses: + self.oPasses = struct.unpack( + (">%dL" % (self.numPasses + 1)), data[2 : 6 + 4 * self.numPasses] + ) + data = data[6 + 4 * self.numPasses :] + (numPseudo,) = struct.unpack(">H", data[:2]) + for i in range(numPseudo): + if version >= 3.0: + pseudo = sstruct.unpack( + Silf_pseudomap_format, data[8 + 6 * i : 14 + 6 * i], _Object() + ) + else: + pseudo = sstruct.unpack( + Silf_pseudomap_format_h, data[8 + 4 * i : 12 + 4 * i], _Object() + ) + self.pMap[pseudo.unicode] = ttFont.getGlyphName(pseudo.nPseudo) + data = data[8 + 6 * numPseudo :] + currpos = ( + sstruct.calcsize(Silf_part1_format) + + sstruct.calcsize(Silf_justify_format) * self.numJLevels + + sstruct.calcsize(Silf_part2_format) + + 2 * self.numCritFeatures + + 1 + + 1 + + 4 * numScriptTag + + 6 + + 4 * self.numPasses + + 8 + + 6 * numPseudo + ) + if version >= 3.0: + currpos += sstruct.calcsize(Silf_part1_format_v3) + self.classes = Classes() + self.classes.decompile(data, ttFont, version) + for i in range(self.numPasses): + p = Pass() + self.passes.append(p) + p.decompile( + data[self.oPasses[i] - currpos : self.oPasses[i + 1] - currpos], + ttFont, + version, + ) + + def compile(self, ttFont, version=2.0): + self.numPasses = len(self.passes) + self.numJLevels = len(self.jLevels) + self.numCritFeatures = len(self.critFeatures) + numPseudo = len(self.pMap) + data = b"" + if version >= 3.0: + hdroffset = sstruct.calcsize(Silf_part1_format_v3) + else: + hdroffset = 0 + data += sstruct.pack(Silf_part1_format, self) + for j in self.jLevels: + data += sstruct.pack(Silf_justify_format, j) + data += sstruct.pack(Silf_part2_format, self) + if self.numCritFeatures: + data += struct.pack((">%dH" % self.numCritFeaturs), *self.critFeatures) + data += struct.pack("BB", 0, len(self.scriptTags)) + if len(self.scriptTags): + tdata = [struct.pack("4s", x.encode("ascii")) for x in self.scriptTags] + data += b"".join(tdata) + data += struct.pack(">H", self.lbGID) + self.passOffset = len(data) + + data1 = grUtils.bininfo(numPseudo, 6) + currpos = hdroffset + len(data) + 4 * (self.numPasses + 1) + self.pseudosOffset = currpos + len(data1) + for u, p in sorted(self.pMap.items()): + data1 += struct.pack( + (">LH" if version >= 3.0 else ">HH"), u, ttFont.getGlyphID(p) + ) + data1 += self.classes.compile(ttFont, version) + currpos += len(data1) + data2 = b"" + datao = b"" + for i, p in enumerate(self.passes): + base = currpos + len(data2) + datao += struct.pack(">L", base) + data2 += p.compile(ttFont, base, version) + datao += struct.pack(">L", currpos + len(data2)) + + if version >= 3.0: + data3 = sstruct.pack(Silf_part1_format_v3, self) + else: + data3 = b"" + return data3 + data + datao + data1 + data2 + + def toXML(self, writer, ttFont, version=2.0): + if version >= 3.0: + writer.simpletag("version", ruleVersion=self.ruleVersion) + writer.newline() + writesimple("info", self, writer, *attrs_info) + writesimple("passindexes", self, writer, *attrs_passindexes) + writesimple("contexts", self, writer, *attrs_contexts) + writesimple("attributes", self, writer, *attrs_attributes) + if len(self.jLevels): + writer.begintag("justifications") + writer.newline() + jformat, jnames, jfixes = sstruct.getformat(Silf_justify_format) + for i, j in enumerate(self.jLevels): + attrs = dict([(k, getattr(j, k)) for k in jnames]) + writer.simpletag("justify", **attrs) + writer.newline() + writer.endtag("justifications") + writer.newline() + if len(self.critFeatures): + writer.begintag("critFeatures") + writer.newline() + writer.write(" ".join(map(str, self.critFeatures))) + writer.newline() + writer.endtag("critFeatures") + writer.newline() + if len(self.scriptTags): + writer.begintag("scriptTags") + writer.newline() + writer.write(" ".join(self.scriptTags)) + writer.newline() + writer.endtag("scriptTags") + writer.newline() + if self.pMap: + writer.begintag("pseudoMap") + writer.newline() + for k, v in sorted(self.pMap.items()): + writer.simpletag("pseudo", unicode=hex(k), pseudo=v) + writer.newline() + writer.endtag("pseudoMap") + writer.newline() + self.classes.toXML(writer, ttFont, version) + if len(self.passes): + writer.begintag("passes") + writer.newline() + for i, p in enumerate(self.passes): + writer.begintag("pass", _index=i) + writer.newline() + p.toXML(writer, ttFont, version) + writer.endtag("pass") + writer.newline() + writer.endtag("passes") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont, version=2.0): + if name == "version": + self.ruleVersion = float(safeEval(attrs.get("ruleVersion", "0"))) + if name == "info": + getSimple(self, attrs, *attrs_info) + elif name == "passindexes": + getSimple(self, attrs, *attrs_passindexes) + elif name == "contexts": + getSimple(self, attrs, *attrs_contexts) + elif name == "attributes": + getSimple(self, attrs, *attrs_attributes) + elif name == "justifications": + for element in content: + if not isinstance(element, tuple): + continue + (tag, attrs, subcontent) = element + if tag == "justify": + j = _Object() + for k, v in attrs.items(): + setattr(j, k, int(v)) + self.jLevels.append(j) + elif name == "critFeatures": + self.critFeatures = [] + element = content_string(content) + self.critFeatures.extend(map(int, element.split())) + elif name == "scriptTags": + self.scriptTags = [] + element = content_string(content) + for n in element.split(): + self.scriptTags.append(n) + elif name == "pseudoMap": + self.pMap = {} + for element in content: + if not isinstance(element, tuple): + continue + (tag, attrs, subcontent) = element + if tag == "pseudo": + k = int(attrs["unicode"], 16) + v = attrs["pseudo"] + self.pMap[k] = v + elif name == "classes": + self.classes = Classes() + for element in content: + if not isinstance(element, tuple): + continue + tag, attrs, subcontent = element + self.classes.fromXML(tag, attrs, subcontent, ttFont, version) + elif name == "passes": + for element in content: + if not isinstance(element, tuple): + continue + tag, attrs, subcontent = element + if tag == "pass": + p = Pass() + for e in subcontent: + if not isinstance(e, tuple): + continue + p.fromXML(e[0], e[1], e[2], ttFont, version) + self.passes.append(p) + + +class Classes(object): + def __init__(self): + self.linear = [] + self.nonLinear = [] + + def decompile(self, data, ttFont, version=2.0): + sstruct.unpack2(Silf_classmap_format, data, self) + if version >= 4.0: + oClasses = struct.unpack( + (">%dL" % (self.numClass + 1)), data[4 : 8 + 4 * self.numClass] + ) + else: + oClasses = struct.unpack( + (">%dH" % (self.numClass + 1)), data[4 : 6 + 2 * self.numClass] + ) + for s, e in zip(oClasses[: self.numLinear], oClasses[1 : self.numLinear + 1]): + self.linear.append( + ttFont.getGlyphName(x) + for x in struct.unpack((">%dH" % ((e - s) / 2)), data[s:e]) + ) + for s, e in zip( + oClasses[self.numLinear : self.numClass], + oClasses[self.numLinear + 1 : self.numClass + 1], + ): + nonLinids = [ + struct.unpack(">HH", data[x : x + 4]) for x in range(s + 8, e, 4) + ] + nonLin = dict([(ttFont.getGlyphName(x[0]), x[1]) for x in nonLinids]) + self.nonLinear.append(nonLin) + + def compile(self, ttFont, version=2.0): + data = b"" + oClasses = [] + if version >= 4.0: + offset = 8 + 4 * (len(self.linear) + len(self.nonLinear)) + else: + offset = 6 + 2 * (len(self.linear) + len(self.nonLinear)) + for l in self.linear: + oClasses.append(len(data) + offset) + gs = [ttFont.getGlyphID(x) for x in l] + data += struct.pack((">%dH" % len(l)), *gs) + for l in self.nonLinear: + oClasses.append(len(data) + offset) + gs = [(ttFont.getGlyphID(x[0]), x[1]) for x in l.items()] + data += grUtils.bininfo(len(gs)) + data += b"".join([struct.pack(">HH", *x) for x in sorted(gs)]) + oClasses.append(len(data) + offset) + self.numClass = len(oClasses) - 1 + self.numLinear = len(self.linear) + return ( + sstruct.pack(Silf_classmap_format, self) + + struct.pack( + ((">%dL" if version >= 4.0 else ">%dH") % len(oClasses)), *oClasses + ) + + data + ) + + def toXML(self, writer, ttFont, version=2.0): + writer.begintag("classes") + writer.newline() + writer.begintag("linearClasses") + writer.newline() + for i, l in enumerate(self.linear): + writer.begintag("linear", _index=i) + writer.newline() + wrapline(writer, l) + writer.endtag("linear") + writer.newline() + writer.endtag("linearClasses") + writer.newline() + writer.begintag("nonLinearClasses") + writer.newline() + for i, l in enumerate(self.nonLinear): + writer.begintag("nonLinear", _index=i + self.numLinear) + writer.newline() + for inp, ind in l.items(): + writer.simpletag("map", glyph=inp, index=ind) + writer.newline() + writer.endtag("nonLinear") + writer.newline() + writer.endtag("nonLinearClasses") + writer.newline() + writer.endtag("classes") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont, version=2.0): + if name == "linearClasses": + for element in content: + if not isinstance(element, tuple): + continue + tag, attrs, subcontent = element + if tag == "linear": + l = content_string(subcontent).split() + self.linear.append(l) + elif name == "nonLinearClasses": + for element in content: + if not isinstance(element, tuple): + continue + tag, attrs, subcontent = element + if tag == "nonLinear": + l = {} + for e in subcontent: + if not isinstance(e, tuple): + continue + tag, attrs, subsubcontent = e + if tag == "map": + l[attrs["glyph"]] = int(safeEval(attrs["index"])) + self.nonLinear.append(l) + + +class Pass(object): + def __init__(self): + self.colMap = {} + self.rules = [] + self.rulePreContexts = [] + self.ruleSortKeys = [] + self.ruleConstraints = [] + self.passConstraints = b"" + self.actions = [] + self.stateTrans = [] + self.startStates = [] + + def decompile(self, data, ttFont, version=2.0): + _, data = sstruct.unpack2(Silf_pass_format, data, self) + (numRange, _, _, _) = struct.unpack(">4H", data[:8]) + data = data[8:] + for i in range(numRange): + (first, last, col) = struct.unpack(">3H", data[6 * i : 6 * i + 6]) + for g in range(first, last + 1): + self.colMap[ttFont.getGlyphName(g)] = col + data = data[6 * numRange :] + oRuleMap = struct.unpack_from((">%dH" % (self.numSuccess + 1)), data) + data = data[2 + 2 * self.numSuccess :] + rules = struct.unpack_from((">%dH" % oRuleMap[-1]), data) + self.rules = [rules[s:e] for (s, e) in zip(oRuleMap, oRuleMap[1:])] + data = data[2 * oRuleMap[-1] :] + (self.minRulePreContext, self.maxRulePreContext) = struct.unpack("BB", data[:2]) + numStartStates = self.maxRulePreContext - self.minRulePreContext + 1 + self.startStates = struct.unpack( + (">%dH" % numStartStates), data[2 : 2 + numStartStates * 2] + ) + data = data[2 + numStartStates * 2 :] + self.ruleSortKeys = struct.unpack( + (">%dH" % self.numRules), data[: 2 * self.numRules] + ) + data = data[2 * self.numRules :] + self.rulePreContexts = struct.unpack( + ("%dB" % self.numRules), data[: self.numRules] + ) + data = data[self.numRules :] + (self.collisionThreshold, pConstraint) = struct.unpack(">BH", data[:3]) + oConstraints = list( + struct.unpack( + (">%dH" % (self.numRules + 1)), data[3 : 5 + self.numRules * 2] + ) + ) + data = data[5 + self.numRules * 2 :] + oActions = list( + struct.unpack((">%dH" % (self.numRules + 1)), data[: 2 + self.numRules * 2]) + ) + data = data[2 * self.numRules + 2 :] + for i in range(self.numTransitional): + a = array( + "H", data[i * self.numColumns * 2 : (i + 1) * self.numColumns * 2] + ) + if sys.byteorder != "big": + a.byteswap() + self.stateTrans.append(a) + data = data[self.numTransitional * self.numColumns * 2 + 1 :] + self.passConstraints = data[:pConstraint] + data = data[pConstraint:] + for i in range(len(oConstraints) - 2, -1, -1): + if oConstraints[i] == 0: + oConstraints[i] = oConstraints[i + 1] + self.ruleConstraints = [ + (data[s:e] if (e - s > 1) else b"") + for (s, e) in zip(oConstraints, oConstraints[1:]) + ] + data = data[oConstraints[-1] :] + self.actions = [ + (data[s:e] if (e - s > 1) else "") for (s, e) in zip(oActions, oActions[1:]) + ] + data = data[oActions[-1] :] + # not using debug + + def compile(self, ttFont, base, version=2.0): + # build it all up backwards + oActions = reduce( + lambda a, x: (a[0] + len(x), a[1] + [a[0]]), self.actions + [b""], (0, []) + )[1] + oConstraints = reduce( + lambda a, x: (a[0] + len(x), a[1] + [a[0]]), + self.ruleConstraints + [b""], + (1, []), + )[1] + constraintCode = b"\000" + b"".join(self.ruleConstraints) + transes = [] + for t in self.stateTrans: + if sys.byteorder != "big": + t.byteswap() + transes.append(t.tobytes()) + if sys.byteorder != "big": + t.byteswap() + if not len(transes): + self.startStates = [0] + oRuleMap = reduce( + lambda a, x: (a[0] + len(x), a[1] + [a[0]]), self.rules + [[]], (0, []) + )[1] + passRanges = [] + gidcolmap = dict([(ttFont.getGlyphID(x[0]), x[1]) for x in self.colMap.items()]) + for e in grUtils.entries(gidcolmap, sameval=True): + if e[1]: + passRanges.append((e[0], e[0] + e[1] - 1, e[2][0])) + self.numRules = len(self.actions) + self.fsmOffset = ( + sstruct.calcsize(Silf_pass_format) + + 8 + + len(passRanges) * 6 + + len(oRuleMap) * 2 + + 2 * oRuleMap[-1] + + 2 + + 2 * len(self.startStates) + + 3 * self.numRules + + 3 + + 4 * self.numRules + + 4 + ) + self.pcCode = ( + self.fsmOffset + 2 * self.numTransitional * self.numColumns + 1 + base + ) + self.rcCode = self.pcCode + len(self.passConstraints) + self.aCode = self.rcCode + len(constraintCode) + self.oDebug = 0 + # now generate output + data = sstruct.pack(Silf_pass_format, self) + data += grUtils.bininfo(len(passRanges), 6) + data += b"".join(struct.pack(">3H", *p) for p in passRanges) + data += struct.pack((">%dH" % len(oRuleMap)), *oRuleMap) + flatrules = reduce(lambda a, x: a + x, self.rules, []) + data += struct.pack((">%dH" % oRuleMap[-1]), *flatrules) + data += struct.pack("BB", self.minRulePreContext, self.maxRulePreContext) + data += struct.pack((">%dH" % len(self.startStates)), *self.startStates) + data += struct.pack((">%dH" % self.numRules), *self.ruleSortKeys) + data += struct.pack(("%dB" % self.numRules), *self.rulePreContexts) + data += struct.pack(">BH", self.collisionThreshold, len(self.passConstraints)) + data += struct.pack((">%dH" % (self.numRules + 1)), *oConstraints) + data += struct.pack((">%dH" % (self.numRules + 1)), *oActions) + return ( + data + + b"".join(transes) + + struct.pack("B", 0) + + self.passConstraints + + constraintCode + + b"".join(self.actions) + ) + + def toXML(self, writer, ttFont, version=2.0): + writesimple("info", self, writer, *pass_attrs_info) + writesimple("fsminfo", self, writer, *pass_attrs_fsm) + writer.begintag("colmap") + writer.newline() + wrapline( + writer, + [ + "{}={}".format(*x) + for x in sorted( + self.colMap.items(), key=lambda x: ttFont.getGlyphID(x[0]) + ) + ], + ) + writer.endtag("colmap") + writer.newline() + writer.begintag("staterulemap") + writer.newline() + for i, r in enumerate(self.rules): + writer.simpletag( + "state", + number=self.numRows - self.numSuccess + i, + rules=" ".join(map(str, r)), + ) + writer.newline() + writer.endtag("staterulemap") + writer.newline() + writer.begintag("rules") + writer.newline() + for i, action in enumerate(self.actions): + writer.begintag( + "rule", + index=i, + precontext=self.rulePreContexts[i], + sortkey=self.ruleSortKeys[i], + ) + writer.newline() + if len(self.ruleConstraints[i]): + writecode("constraint", writer, self.ruleConstraints[i]) + writecode("action", writer, action) + writer.endtag("rule") + writer.newline() + writer.endtag("rules") + writer.newline() + if len(self.passConstraints): + writecode("passConstraint", writer, self.passConstraints) + if len(self.stateTrans): + writer.begintag("fsm") + writer.newline() + writer.begintag("starts") + writer.write(" ".join(map(str, self.startStates))) + writer.endtag("starts") + writer.newline() + for i, s in enumerate(self.stateTrans): + writer.begintag("row", _i=i) + # no newlines here + writer.write(" ".join(map(str, s))) + writer.endtag("row") + writer.newline() + writer.endtag("fsm") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont, version=2.0): + if name == "info": + getSimple(self, attrs, *pass_attrs_info) + elif name == "fsminfo": + getSimple(self, attrs, *pass_attrs_fsm) + elif name == "colmap": + e = content_string(content) + for w in e.split(): + x = w.split("=") + if len(x) != 2 or x[0] == "" or x[1] == "": + continue + self.colMap[x[0]] = int(x[1]) + elif name == "staterulemap": + for e in content: + if not isinstance(e, tuple): + continue + tag, a, c = e + if tag == "state": + self.rules.append([int(x) for x in a["rules"].split(" ")]) + elif name == "rules": + for element in content: + if not isinstance(element, tuple): + continue + tag, a, c = element + if tag != "rule": + continue + self.rulePreContexts.append(int(a["precontext"])) + self.ruleSortKeys.append(int(a["sortkey"])) + con = b"" + act = b"" + for e in c: + if not isinstance(e, tuple): + continue + tag, a, subc = e + if tag == "constraint": + con = readcode(subc) + elif tag == "action": + act = readcode(subc) + self.actions.append(act) + self.ruleConstraints.append(con) + elif name == "passConstraint": + self.passConstraints = readcode(content) + elif name == "fsm": + for element in content: + if not isinstance(element, tuple): + continue + tag, a, c = element + if tag == "row": + s = array("H") + e = content_string(c) + s.extend(map(int, e.split())) + self.stateTrans.append(s) + elif tag == "starts": + s = [] + e = content_string(c) + s.extend(map(int, e.split())) + self.startStates = s diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S__i_l_l.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S__i_l_l.py new file mode 100644 index 0000000000000000000000000000000000000000..61106ea5284ac67219590d87544d44e2e1f970c2 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/S__i_l_l.py @@ -0,0 +1,92 @@ +from fontTools.misc import sstruct +from fontTools.misc.fixedTools import floatToFixedToStr +from fontTools.misc.textTools import safeEval +from . import DefaultTable +from . import grUtils +import struct + +Sill_hdr = """ + > + version: 16.16F +""" + + +class table_S__i_l_l(DefaultTable.DefaultTable): + """Graphite Languages table + + See also https://graphite.sil.org/graphite_techAbout#graphite-font-tables + """ + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.langs = {} + + def decompile(self, data, ttFont): + (_, data) = sstruct.unpack2(Sill_hdr, data, self) + self.version = float(floatToFixedToStr(self.version, precisionBits=16)) + (numLangs,) = struct.unpack(">H", data[:2]) + data = data[8:] + maxsetting = 0 + langinfo = [] + for i in range(numLangs): + (langcode, numsettings, offset) = struct.unpack( + ">4sHH", data[i * 8 : (i + 1) * 8] + ) + offset = int(offset / 8) - (numLangs + 1) + langcode = langcode.replace(b"\000", b"") + langinfo.append((langcode.decode("utf-8"), numsettings, offset)) + maxsetting = max(maxsetting, offset + numsettings) + data = data[numLangs * 8 :] + finfo = [] + for i in range(maxsetting): + (fid, val, _) = struct.unpack(">LHH", data[i * 8 : (i + 1) * 8]) + finfo.append((fid, val)) + self.langs = {} + for c, n, o in langinfo: + self.langs[c] = [] + for i in range(o, o + n): + self.langs[c].append(finfo[i]) + + def compile(self, ttFont): + ldat = b"" + fdat = b"" + offset = len(self.langs) + for c, inf in sorted(self.langs.items()): + ldat += struct.pack(">4sHH", c.encode("utf8"), len(inf), 8 * offset + 20) + for fid, val in inf: + fdat += struct.pack(">LHH", fid, val, 0) + offset += len(inf) + ldat += struct.pack(">LHH", 0x80808080, 0, 8 * offset + 20) + return ( + sstruct.pack(Sill_hdr, self) + + grUtils.bininfo(len(self.langs)) + + ldat + + fdat + ) + + def toXML(self, writer, ttFont): + writer.simpletag("version", version=self.version) + writer.newline() + for c, inf in sorted(self.langs.items()): + writer.begintag("lang", name=c) + writer.newline() + for fid, val in inf: + writer.simpletag("feature", fid=grUtils.num2tag(fid), val=val) + writer.newline() + writer.endtag("lang") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": + self.version = float(safeEval(attrs["version"])) + elif name == "lang": + c = attrs["name"] + self.langs[c] = [] + for element in content: + if not isinstance(element, tuple): + continue + tag, a, subcontent = element + if tag == "feature": + self.langs[c].append( + (grUtils.tag2num(a["fid"]), int(safeEval(a["val"]))) + ) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_B_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_B_.py new file mode 100644 index 0000000000000000000000000000000000000000..ea19fcee856e75fc8aa494e083b4beed303f831c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_B_.py @@ -0,0 +1,13 @@ +""" TSI{B,C,D,J,P,S,V} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its table source data. + +TSIB contains the source text for the ``BASE`` table. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +from .T_S_I_V_ import table_T_S_I_V_ + + +class table_T_S_I_B_(table_T_S_I_V_): + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_C_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_C_.py new file mode 100644 index 0000000000000000000000000000000000000000..14e353cde58b739660b10fe666e26492f5a0069f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_C_.py @@ -0,0 +1,14 @@ +""" TSI{B,C,D,J,P,S,V} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its table source data. + +TSIC contains the source text for the Variation CVT window and data for +the ``cvar`` table. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +from .otBase import BaseTTXConverter + + +class table_T_S_I_C_(BaseTTXConverter): + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_D_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_D_.py new file mode 100644 index 0000000000000000000000000000000000000000..2bb9455efaeda5baac2c1f425c35019723b592bd --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_D_.py @@ -0,0 +1,13 @@ +""" TSI{B,C,D,J,P,S,V} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its table source data. + +TSID contains the source text for the ``GDEF`` table. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +from .T_S_I_V_ import table_T_S_I_V_ + + +class table_T_S_I_D_(table_T_S_I_V_): + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_J_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_J_.py new file mode 100644 index 0000000000000000000000000000000000000000..379b949c6cd380268b139f6c3cfd61fffc43e223 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_J_.py @@ -0,0 +1,13 @@ +""" TSI{B,C,D,J,P,S,V} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its table source data. + +TSIJ contains the source text for the ``JSTF`` table. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +from .T_S_I_V_ import table_T_S_I_V_ + + +class table_T_S_I_J_(table_T_S_I_V_): + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_P_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_P_.py new file mode 100644 index 0000000000000000000000000000000000000000..8b1786970489f4511dcd156e5562eb2156f927a3 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_P_.py @@ -0,0 +1,13 @@ +""" TSI{B,C,D,J,P,S,V} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its table source data. + +TSIP contains the source text for the ``GPOS`` table. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +from .T_S_I_V_ import table_T_S_I_V_ + + +class table_T_S_I_P_(table_T_S_I_V_): + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_S_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_S_.py new file mode 100644 index 0000000000000000000000000000000000000000..91f36e7c019087f954a7fa779df045a9097fa289 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_S_.py @@ -0,0 +1,13 @@ +""" TSI{B,C,D,J,P,S,V} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its table source data. + +TSIS contains the source text for the ``GSUB`` table. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +from .T_S_I_V_ import table_T_S_I_V_ + + +class table_T_S_I_S_(table_T_S_I_V_): + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_V_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_V_.py new file mode 100644 index 0000000000000000000000000000000000000000..32af783f26b0f9224e39d3e243f72bfcd1da150b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I_V_.py @@ -0,0 +1,26 @@ +""" TSI{B,C,D,J,P,S,V} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its table source data. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +from fontTools.misc.textTools import strjoin, tobytes, tostr +from . import asciiTable + + +class table_T_S_I_V_(asciiTable.asciiTable): + def toXML(self, writer, ttFont): + data = tostr(self.data) + # removing null bytes. XXX needed?? + data = data.split("\0") + data = strjoin(data) + writer.begintag("source") + writer.newline() + writer.write_noindent(data.replace("\r", "\n")) + writer.newline() + writer.endtag("source") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + lines = strjoin(content).split("\n") + self.data = tobytes("\r".join(lines[1:-1])) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__0.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__0.py new file mode 100644 index 0000000000000000000000000000000000000000..d60e783c60873c85487463f151dc25f061cde97f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__0.py @@ -0,0 +1,70 @@ +"""TSI{0,1,2,3,5} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its hinting source data. + +TSI0 is the index table containing the lengths and offsets for the glyph +programs and 'extra' programs ('fpgm', 'prep', and 'cvt') that are contained +in the TSI1 table. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +import logging +import struct + +from . import DefaultTable + +log = logging.getLogger(__name__) + +tsi0Format = ">HHL" + + +def fixlongs(glyphID, textLength, textOffset): + return int(glyphID), int(textLength), textOffset + + +class table_T_S_I__0(DefaultTable.DefaultTable): + dependencies = ["TSI1"] + + def decompile(self, data, ttFont): + numGlyphs = ttFont["maxp"].numGlyphs + indices = [] + size = struct.calcsize(tsi0Format) + numEntries = len(data) // size + if numEntries != numGlyphs + 5: + diff = numEntries - numGlyphs - 5 + log.warning( + "Number of glyphPrograms differs from the number of glyphs in the font " + f"by {abs(diff)} ({numEntries - 5} programs vs. {numGlyphs} glyphs)." + ) + for _ in range(numEntries): + glyphID, textLength, textOffset = fixlongs( + *struct.unpack(tsi0Format, data[:size]) + ) + indices.append((glyphID, textLength, textOffset)) + data = data[size:] + assert len(data) == 0 + assert indices[-5] == (0xFFFE, 0, 0xABFC1F34), "bad magic number" + self.indices = indices[:-5] + self.extra_indices = indices[-4:] + + def compile(self, ttFont): + if not hasattr(self, "indices"): + # We have no corresponding table (TSI1 or TSI3); let's return + # no data, which effectively means "ignore us". + return b"" + data = b"" + for index, textLength, textOffset in self.indices: + data = data + struct.pack(tsi0Format, index, textLength, textOffset) + data = data + struct.pack(tsi0Format, 0xFFFE, 0, 0xABFC1F34) + for index, textLength, textOffset in self.extra_indices: + data = data + struct.pack(tsi0Format, index, textLength, textOffset) + return data + + def set(self, indices, extra_indices): + # gets called by 'TSI1' or 'TSI3' + self.indices = indices + self.extra_indices = extra_indices + + def toXML(self, writer, ttFont): + writer.comment("This table will be calculated by the compiler") + writer.newline() diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__1.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__1.py new file mode 100644 index 0000000000000000000000000000000000000000..19537dac9c512dc009a159cb25bb926eebca61ed --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__1.py @@ -0,0 +1,163 @@ +""" TSI{0,1,2,3,5} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its hinting source data. + +TSI1 contains the text of the glyph programs in the form of low-level assembly +code, as well as the 'extra' programs 'fpgm', 'ppgm' (i.e. 'prep'), and 'cvt'. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +from . import DefaultTable +from fontTools.misc.loggingTools import LogMixin +from fontTools.misc.textTools import strjoin, tobytes, tostr + + +class table_T_S_I__1(LogMixin, DefaultTable.DefaultTable): + extras = {0xFFFA: "ppgm", 0xFFFB: "cvt", 0xFFFC: "reserved", 0xFFFD: "fpgm"} + + indextable = "TSI0" + + def decompile(self, data, ttFont): + totalLength = len(data) + indextable = ttFont[self.indextable] + for indices, isExtra in zip( + (indextable.indices, indextable.extra_indices), (False, True) + ): + programs = {} + for i, (glyphID, textLength, textOffset) in enumerate(indices): + if isExtra: + name = self.extras[glyphID] + else: + name = ttFont.getGlyphName(glyphID) + if textOffset > totalLength: + self.log.warning("textOffset > totalLength; %r skipped" % name) + continue + if textLength < 0x8000: + # If the length stored in the record is less than 32768, then use + # that as the length of the record. + pass + elif textLength == 0x8000: + # If the length is 32768, compute the actual length as follows: + isLast = i == (len(indices) - 1) + if isLast: + if isExtra: + # For the last "extra" record (the very last record of the + # table), the length is the difference between the total + # length of the TSI1 table and the textOffset of the final + # record. + nextTextOffset = totalLength + else: + # For the last "normal" record (the last record just prior + # to the record containing the "magic number"), the length + # is the difference between the textOffset of the record + # following the "magic number" (0xFFFE) record (i.e. the + # first "extra" record), and the textOffset of the last + # "normal" record. + nextTextOffset = indextable.extra_indices[0][2] + else: + # For all other records with a length of 0x8000, the length is + # the difference between the textOffset of the record in + # question and the textOffset of the next record. + nextTextOffset = indices[i + 1][2] + assert nextTextOffset >= textOffset, "entries not sorted by offset" + if nextTextOffset > totalLength: + self.log.warning( + "nextTextOffset > totalLength; %r truncated" % name + ) + nextTextOffset = totalLength + textLength = nextTextOffset - textOffset + else: + from fontTools import ttLib + + raise ttLib.TTLibError( + "%r textLength (%d) must not be > 32768" % (name, textLength) + ) + text = data[textOffset : textOffset + textLength] + assert len(text) == textLength + text = tostr(text, encoding="utf-8") + if text: + programs[name] = text + if isExtra: + self.extraPrograms = programs + else: + self.glyphPrograms = programs + + def compile(self, ttFont): + if not hasattr(self, "glyphPrograms"): + self.glyphPrograms = {} + self.extraPrograms = {} + data = b"" + indextable = ttFont[self.indextable] + glyphNames = ttFont.getGlyphOrder() + + indices = [] + for i, name in enumerate(glyphNames): + if len(data) % 2: + data = ( + data + b"\015" + ) # align on 2-byte boundaries, fill with return chars. Yum. + if name in self.glyphPrograms: + text = tobytes(self.glyphPrograms[name], encoding="utf-8") + else: + text = b"" + textLength = len(text) + if textLength >= 0x8000: + textLength = 0x8000 + indices.append((i, textLength, len(data))) + data = data + text + + extra_indices = [] + for code, name in sorted(self.extras.items()): + if len(data) % 2: + data = ( + data + b"\015" + ) # align on 2-byte boundaries, fill with return chars. + if name in self.extraPrograms: + text = tobytes(self.extraPrograms[name], encoding="utf-8") + else: + text = b"" + textLength = len(text) + if textLength >= 0x8000: + textLength = 0x8000 + extra_indices.append((code, textLength, len(data))) + data = data + text + indextable.set(indices, extra_indices) + return data + + def toXML(self, writer, ttFont): + names = sorted(self.glyphPrograms.keys()) + writer.newline() + for name in names: + text = self.glyphPrograms[name] + if not text: + continue + writer.begintag("glyphProgram", name=name) + writer.newline() + writer.write_noindent(text.replace("\r", "\n")) + writer.newline() + writer.endtag("glyphProgram") + writer.newline() + writer.newline() + extra_names = sorted(self.extraPrograms.keys()) + for name in extra_names: + text = self.extraPrograms[name] + if not text: + continue + writer.begintag("extraProgram", name=name) + writer.newline() + writer.write_noindent(text.replace("\r", "\n")) + writer.newline() + writer.endtag("extraProgram") + writer.newline() + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if not hasattr(self, "glyphPrograms"): + self.glyphPrograms = {} + self.extraPrograms = {} + lines = strjoin(content).replace("\r", "\n").split("\n") + text = "\r".join(lines[1:-1]) + if name == "glyphProgram": + self.glyphPrograms[attrs["name"]] = text + elif name == "extraProgram": + self.extraPrograms[attrs["name"]] = text diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__2.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__2.py new file mode 100644 index 0000000000000000000000000000000000000000..63608c60414c8e6cfd67aa6ab46caf4f14a918e1 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__2.py @@ -0,0 +1,17 @@ +""" TSI{0,1,2,3,5} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its hinting source data. + +TSI2 is the index table containing the lengths and offsets for the glyph +programs that are contained in the TSI3 table. It uses the same format as +the TSI0 table. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +from fontTools import ttLib + +superclass = ttLib.getTableClass("TSI0") + + +class table_T_S_I__2(superclass): + dependencies = ["TSI3"] diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__3.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__3.py new file mode 100644 index 0000000000000000000000000000000000000000..e866826db488d178833836f203f5e634cbdfbd3f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__3.py @@ -0,0 +1,22 @@ +""" TSI{0,1,2,3,5} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its hinting source data. + +TSI3 contains the text of the glyph programs in the form of 'VTTTalk' code. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +from fontTools import ttLib + +superclass = ttLib.getTableClass("TSI1") + + +class table_T_S_I__3(superclass): + extras = { + 0xFFFA: "reserved0", + 0xFFFB: "reserved1", + 0xFFFC: "reserved2", + 0xFFFD: "reserved3", + } + + indextable = "TSI2" diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__5.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__5.py new file mode 100644 index 0000000000000000000000000000000000000000..8aa382e43ca0e4bb33994d2e093459a09ba12a9b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_S_I__5.py @@ -0,0 +1,60 @@ +"""TSI{0,1,2,3,5} are private tables used by Microsoft Visual TrueType (VTT) +tool to store its hinting source data. + +TSI5 contains the VTT character groups. + +See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables +""" + +import array +import logging +import sys + +from fontTools.misc.textTools import safeEval + +from . import DefaultTable + +log = logging.getLogger(__name__) + + +class table_T_S_I__5(DefaultTable.DefaultTable): + def decompile(self, data, ttFont): + numGlyphs = ttFont["maxp"].numGlyphs + a = array.array("H") + a.frombytes(data) + if sys.byteorder != "big": + a.byteswap() + self.glyphGrouping = {} + numEntries = len(data) // 2 + if numEntries != numGlyphs: + diff = numEntries - numGlyphs + log.warning( + "Number of entries differs from the number of glyphs in the font " + f"by {abs(diff)} ({numEntries} entries vs. {numGlyphs} glyphs)." + ) + for i in range(numEntries): + self.glyphGrouping[ttFont.getGlyphName(i)] = a[i] + + def compile(self, ttFont): + glyphNames = ttFont.getGlyphOrder() + a = array.array("H") + for glyphName in glyphNames: + a.append(self.glyphGrouping.get(glyphName, 0)) + if sys.byteorder != "big": + a.byteswap() + return a.tobytes() + + def toXML(self, writer, ttFont): + names = sorted(self.glyphGrouping.keys()) + for glyphName in names: + writer.simpletag( + "glyphgroup", name=glyphName, value=self.glyphGrouping[glyphName] + ) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if not hasattr(self, "glyphGrouping"): + self.glyphGrouping = {} + if name != "glyphgroup": + return + self.glyphGrouping[attrs["name"]] = safeEval(attrs["value"]) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_T_F_A_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_T_F_A_.py new file mode 100644 index 0000000000000000000000000000000000000000..4b6b06871a91d9401bcf9fbd99aeb3c30ca0705f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/T_T_F_A_.py @@ -0,0 +1,14 @@ +from . import asciiTable + + +class table_T_T_F_A_(asciiTable.asciiTable): + """ttfautohint parameters table + + The ``TTFA`` table is used by the free-software `ttfautohint` program + to record the parameters that `ttfautohint` was called with when it + was used to auto-hint the font. + + See also http://freetype.org/ttfautohint/doc/ttfautohint.html#miscellaneous-1 + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/TupleVariation.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/TupleVariation.py new file mode 100644 index 0000000000000000000000000000000000000000..4916f91d993a64d3a9b32f8f5e14c53aabe4df7c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/TupleVariation.py @@ -0,0 +1,886 @@ +from fontTools.misc.fixedTools import ( + fixedToFloat as fi2fl, + floatToFixed as fl2fi, + floatToFixedToStr as fl2str, + strToFixedToFloat as str2fl, + otRound, +) +from fontTools.misc.textTools import safeEval +import array +from collections import Counter, defaultdict +import logging +import struct +import sys +import warnings + + +# https://www.microsoft.com/typography/otspec/otvarcommonformats.htm + +EMBEDDED_PEAK_TUPLE = 0x8000 +INTERMEDIATE_REGION = 0x4000 +PRIVATE_POINT_NUMBERS = 0x2000 + +DELTAS_ARE_ZERO = 0x80 +DELTAS_ARE_WORDS = 0x40 +DELTAS_ARE_LONGS = 0xC0 +DELTAS_SIZE_MASK = 0xC0 +DELTA_RUN_COUNT_MASK = 0x3F + +POINTS_ARE_WORDS = 0x80 +POINT_RUN_COUNT_MASK = 0x7F + +TUPLES_SHARE_POINT_NUMBERS = 0x8000 +TUPLE_COUNT_MASK = 0x0FFF +TUPLE_INDEX_MASK = 0x0FFF + +log = logging.getLogger(__name__) + + +class TupleVariation(object): + def __init__(self, axes, coordinates): + self.axes = axes.copy() + self.coordinates = list(coordinates) + + def __repr__(self): + axes = ",".join( + sorted(["%s=%s" % (name, value) for (name, value) in self.axes.items()]) + ) + return "<TupleVariation %s %s>" % (axes, self.coordinates) + + def __eq__(self, other): + return self.coordinates == other.coordinates and self.axes == other.axes + + def getUsedPoints(self): + # Empty set means "all points used". + if None not in self.coordinates: + return frozenset() + used = frozenset([i for i, p in enumerate(self.coordinates) if p is not None]) + # Return None if no points used. + return used if used else None + + def hasImpact(self): + """Returns True if this TupleVariation has any visible impact. + + If the result is False, the TupleVariation can be omitted from the font + without making any visible difference. + """ + return any(c is not None for c in self.coordinates) + + def toXML(self, writer, axisTags): + writer.begintag("tuple") + writer.newline() + for axis in axisTags: + value = self.axes.get(axis) + if value is not None: + minValue, value, maxValue = value + defaultMinValue = min(value, 0.0) # -0.3 --> -0.3; 0.7 --> 0.0 + defaultMaxValue = max(value, 0.0) # -0.3 --> 0.0; 0.7 --> 0.7 + if minValue == defaultMinValue and maxValue == defaultMaxValue: + writer.simpletag("coord", axis=axis, value=fl2str(value, 14)) + else: + attrs = [ + ("axis", axis), + ("min", fl2str(minValue, 14)), + ("value", fl2str(value, 14)), + ("max", fl2str(maxValue, 14)), + ] + writer.simpletag("coord", attrs) + writer.newline() + wrote_any_deltas = False + for i, delta in enumerate(self.coordinates): + if type(delta) == tuple and len(delta) == 2: + writer.simpletag("delta", pt=i, x=delta[0], y=delta[1]) + writer.newline() + wrote_any_deltas = True + elif type(delta) == int: + writer.simpletag("delta", cvt=i, value=delta) + writer.newline() + wrote_any_deltas = True + elif delta is not None: + log.error("bad delta format") + writer.comment("bad delta #%d" % i) + writer.newline() + wrote_any_deltas = True + if not wrote_any_deltas: + writer.comment("no deltas") + writer.newline() + writer.endtag("tuple") + writer.newline() + + def fromXML(self, name, attrs, _content): + if name == "coord": + axis = attrs["axis"] + value = str2fl(attrs["value"], 14) + defaultMinValue = min(value, 0.0) # -0.3 --> -0.3; 0.7 --> 0.0 + defaultMaxValue = max(value, 0.0) # -0.3 --> 0.0; 0.7 --> 0.7 + minValue = str2fl(attrs.get("min", defaultMinValue), 14) + maxValue = str2fl(attrs.get("max", defaultMaxValue), 14) + self.axes[axis] = (minValue, value, maxValue) + elif name == "delta": + if "pt" in attrs: + point = safeEval(attrs["pt"]) + x = safeEval(attrs["x"]) + y = safeEval(attrs["y"]) + self.coordinates[point] = (x, y) + elif "cvt" in attrs: + cvt = safeEval(attrs["cvt"]) + value = safeEval(attrs["value"]) + self.coordinates[cvt] = value + else: + log.warning("bad delta format: %s" % ", ".join(sorted(attrs.keys()))) + + def compile( + self, axisTags, sharedCoordIndices={}, pointData=None, *, optimizeSize=True + ): + assert set(self.axes.keys()) <= set(axisTags), ( + "Unknown axis tag found.", + self.axes.keys(), + axisTags, + ) + + tupleData = [] + auxData = [] + + if pointData is None: + usedPoints = self.getUsedPoints() + if usedPoints is None: # Nothing to encode + return b"", b"" + pointData = self.compilePoints(usedPoints) + + coord = self.compileCoord(axisTags) + flags = sharedCoordIndices.get(coord) + if flags is None: + flags = EMBEDDED_PEAK_TUPLE + tupleData.append(coord) + + intermediateCoord = self.compileIntermediateCoord(axisTags) + if intermediateCoord is not None: + flags |= INTERMEDIATE_REGION + tupleData.append(intermediateCoord) + + # pointData of b'' implies "use shared points". + if pointData: + flags |= PRIVATE_POINT_NUMBERS + auxData.append(pointData) + + auxData.append(self.compileDeltas(optimizeSize=optimizeSize)) + auxData = b"".join(auxData) + + tupleData.insert(0, struct.pack(">HH", len(auxData), flags)) + return b"".join(tupleData), auxData + + def compileCoord(self, axisTags): + result = [] + axes = self.axes + for axis in axisTags: + triple = axes.get(axis) + if triple is None: + result.append(b"\0\0") + else: + result.append(struct.pack(">h", fl2fi(triple[1], 14))) + return b"".join(result) + + def compileIntermediateCoord(self, axisTags): + needed = False + for axis in axisTags: + minValue, value, maxValue = self.axes.get(axis, (0.0, 0.0, 0.0)) + defaultMinValue = min(value, 0.0) # -0.3 --> -0.3; 0.7 --> 0.0 + defaultMaxValue = max(value, 0.0) # -0.3 --> 0.0; 0.7 --> 0.7 + if (minValue != defaultMinValue) or (maxValue != defaultMaxValue): + needed = True + break + if not needed: + return None + minCoords = [] + maxCoords = [] + for axis in axisTags: + minValue, value, maxValue = self.axes.get(axis, (0.0, 0.0, 0.0)) + minCoords.append(struct.pack(">h", fl2fi(minValue, 14))) + maxCoords.append(struct.pack(">h", fl2fi(maxValue, 14))) + return b"".join(minCoords + maxCoords) + + @staticmethod + def decompileCoord_(axisTags, data, offset): + coord = {} + pos = offset + for axis in axisTags: + coord[axis] = fi2fl(struct.unpack(">h", data[pos : pos + 2])[0], 14) + pos += 2 + return coord, pos + + @staticmethod + def compilePoints(points): + # If the set consists of all points in the glyph, it gets encoded with + # a special encoding: a single zero byte. + # + # To use this optimization, points passed in must be empty set. + # The following two lines are not strictly necessary as the main code + # below would emit the same. But this is most common and faster. + if not points: + return b"\0" + + # In the 'gvar' table, the packing of point numbers is a little surprising. + # It consists of multiple runs, each being a delta-encoded list of integers. + # For example, the point set {17, 18, 19, 20, 21, 22, 23} gets encoded as + # [6, 17, 1, 1, 1, 1, 1, 1]. The first value (6) is the run length minus 1. + # There are two types of runs, with values being either 8 or 16 bit unsigned + # integers. + points = list(points) + points.sort() + numPoints = len(points) + + result = bytearray() + # The binary representation starts with the total number of points in the set, + # encoded into one or two bytes depending on the value. + if numPoints < 0x80: + result.append(numPoints) + else: + result.append((numPoints >> 8) | 0x80) + result.append(numPoints & 0xFF) + + MAX_RUN_LENGTH = 127 + pos = 0 + lastValue = 0 + while pos < numPoints: + runLength = 0 + + headerPos = len(result) + result.append(0) + + useByteEncoding = None + while pos < numPoints and runLength <= MAX_RUN_LENGTH: + curValue = points[pos] + delta = curValue - lastValue + if useByteEncoding is None: + useByteEncoding = 0 <= delta <= 0xFF + if useByteEncoding and (delta > 0xFF or delta < 0): + # we need to start a new run (which will not use byte encoding) + break + # TODO This never switches back to a byte-encoding from a short-encoding. + # That's suboptimal. + if useByteEncoding: + result.append(delta) + else: + result.append(delta >> 8) + result.append(delta & 0xFF) + lastValue = curValue + pos += 1 + runLength += 1 + if useByteEncoding: + result[headerPos] = runLength - 1 + else: + result[headerPos] = (runLength - 1) | POINTS_ARE_WORDS + + return result + + @staticmethod + def decompilePoints_(numPoints, data, offset, tableTag): + """(numPoints, data, offset, tableTag) --> ([point1, point2, ...], newOffset)""" + assert tableTag in ("cvar", "gvar") + pos = offset + numPointsInData = data[pos] + pos += 1 + if (numPointsInData & POINTS_ARE_WORDS) != 0: + numPointsInData = (numPointsInData & POINT_RUN_COUNT_MASK) << 8 | data[pos] + pos += 1 + if numPointsInData == 0: + return (range(numPoints), pos) + + result = [] + while len(result) < numPointsInData: + runHeader = data[pos] + pos += 1 + numPointsInRun = (runHeader & POINT_RUN_COUNT_MASK) + 1 + point = 0 + if (runHeader & POINTS_ARE_WORDS) != 0: + points = array.array("H") + pointsSize = numPointsInRun * 2 + else: + points = array.array("B") + pointsSize = numPointsInRun + points.frombytes(data[pos : pos + pointsSize]) + if sys.byteorder != "big": + points.byteswap() + + assert len(points) == numPointsInRun + pos += pointsSize + + result.extend(points) + + # Convert relative to absolute + absolute = [] + current = 0 + for delta in result: + current += delta + absolute.append(current) + result = absolute + del absolute + + badPoints = {str(p) for p in result if p < 0 or p >= numPoints} + if badPoints: + log.warning( + "point %s out of range in '%s' table" + % (",".join(sorted(badPoints)), tableTag) + ) + return (result, pos) + + def compileDeltas(self, optimizeSize=True): + deltaX = [] + deltaY = [] + if self.getCoordWidth() == 2: + for c in self.coordinates: + if c is None: + continue + deltaX.append(c[0]) + deltaY.append(c[1]) + else: + for c in self.coordinates: + if c is None: + continue + deltaX.append(c) + bytearr = bytearray() + self.compileDeltaValues_(deltaX, bytearr, optimizeSize=optimizeSize) + self.compileDeltaValues_(deltaY, bytearr, optimizeSize=optimizeSize) + return bytearr + + @staticmethod + def compileDeltaValues_(deltas, bytearr=None, *, optimizeSize=True): + """[value1, value2, value3, ...] --> bytearray + + Emits a sequence of runs. Each run starts with a + byte-sized header whose 6 least significant bits + (header & 0x3F) indicate how many values are encoded + in this run. The stored length is the actual length + minus one; run lengths are thus in the range [1..64]. + If the header byte has its most significant bit (0x80) + set, all values in this run are zero, and no data + follows. Otherwise, the header byte is followed by + ((header & 0x3F) + 1) signed values. If (header & + 0x40) is clear, the delta values are stored as signed + bytes; if (header & 0x40) is set, the delta values are + signed 16-bit integers. + """ # Explaining the format because the 'gvar' spec is hard to understand. + if bytearr is None: + bytearr = bytearray() + + pos = 0 + numDeltas = len(deltas) + + if optimizeSize: + while pos < numDeltas: + value = deltas[pos] + if value == 0: + pos = TupleVariation.encodeDeltaRunAsZeroes_(deltas, pos, bytearr) + elif -128 <= value <= 127: + pos = TupleVariation.encodeDeltaRunAsBytes_(deltas, pos, bytearr) + elif -32768 <= value <= 32767: + pos = TupleVariation.encodeDeltaRunAsWords_(deltas, pos, bytearr) + else: + pos = TupleVariation.encodeDeltaRunAsLongs_(deltas, pos, bytearr) + else: + minVal, maxVal = min(deltas), max(deltas) + if minVal == 0 == maxVal: + pos = TupleVariation.encodeDeltaRunAsZeroes_(deltas, pos, bytearr) + elif -128 <= minVal <= maxVal <= 127: + pos = TupleVariation.encodeDeltaRunAsBytes_( + deltas, pos, bytearr, optimizeSize=False + ) + elif -32768 <= minVal <= maxVal <= 32767: + pos = TupleVariation.encodeDeltaRunAsWords_( + deltas, pos, bytearr, optimizeSize=False + ) + else: + pos = TupleVariation.encodeDeltaRunAsLongs_( + deltas, pos, bytearr, optimizeSize=False + ) + + assert pos == numDeltas, (pos, numDeltas) + + return bytearr + + @staticmethod + def encodeDeltaRunAsZeroes_(deltas, offset, bytearr): + pos = offset + numDeltas = len(deltas) + while pos < numDeltas and deltas[pos] == 0: + pos += 1 + runLength = pos - offset + while runLength >= 64: + bytearr.append(DELTAS_ARE_ZERO | 63) + runLength -= 64 + if runLength: + bytearr.append(DELTAS_ARE_ZERO | (runLength - 1)) + return pos + + @staticmethod + def encodeDeltaRunAsBytes_(deltas, offset, bytearr, optimizeSize=True): + pos = offset + numDeltas = len(deltas) + while pos < numDeltas: + value = deltas[pos] + if not (-128 <= value <= 127): + break + # Within a byte-encoded run of deltas, a single zero + # is best stored literally as 0x00 value. However, + # if are two or more zeroes in a sequence, it is + # better to start a new run. For example, the sequence + # of deltas [15, 15, 0, 15, 15] becomes 6 bytes + # (04 0F 0F 00 0F 0F) when storing the zero value + # literally, but 7 bytes (01 0F 0F 80 01 0F 0F) + # when starting a new run. + if ( + optimizeSize + and value == 0 + and pos + 1 < numDeltas + and deltas[pos + 1] == 0 + ): + break + pos += 1 + runLength = pos - offset + while runLength >= 64: + bytearr.append(63) + bytearr.extend(array.array("b", deltas[offset : offset + 64])) + offset += 64 + runLength -= 64 + if runLength: + bytearr.append(runLength - 1) + bytearr.extend(array.array("b", deltas[offset:pos])) + return pos + + @staticmethod + def encodeDeltaRunAsWords_(deltas, offset, bytearr, optimizeSize=True): + pos = offset + numDeltas = len(deltas) + while pos < numDeltas: + value = deltas[pos] + + # Within a word-encoded run of deltas, it is easiest + # to start a new run (with a different encoding) + # whenever we encounter a zero value. For example, + # the sequence [0x6666, 0, 0x7777] needs 7 bytes when + # storing the zero literally (42 66 66 00 00 77 77), + # and equally 7 bytes when starting a new run + # (40 66 66 80 40 77 77). + if optimizeSize and value == 0: + break + + # Within a word-encoded run of deltas, a single value + # in the range (-128..127) should be encoded literally + # because it is more compact. For example, the sequence + # [0x6666, 2, 0x7777] becomes 7 bytes when storing + # the value literally (42 66 66 00 02 77 77), but 8 bytes + # when starting a new run (40 66 66 00 02 40 77 77). + if ( + optimizeSize + and (-128 <= value <= 127) + and pos + 1 < numDeltas + and (-128 <= deltas[pos + 1] <= 127) + ): + break + + if not (-32768 <= value <= 32767): + break + + pos += 1 + runLength = pos - offset + while runLength >= 64: + bytearr.append(DELTAS_ARE_WORDS | 63) + a = array.array("h", deltas[offset : offset + 64]) + if sys.byteorder != "big": + a.byteswap() + bytearr.extend(a) + offset += 64 + runLength -= 64 + if runLength: + bytearr.append(DELTAS_ARE_WORDS | (runLength - 1)) + a = array.array("h", deltas[offset:pos]) + if sys.byteorder != "big": + a.byteswap() + bytearr.extend(a) + return pos + + @staticmethod + def encodeDeltaRunAsLongs_(deltas, offset, bytearr, optimizeSize=True): + pos = offset + numDeltas = len(deltas) + while pos < numDeltas: + value = deltas[pos] + if optimizeSize and -32768 <= value <= 32767: + break + pos += 1 + runLength = pos - offset + while runLength >= 64: + bytearr.append(DELTAS_ARE_LONGS | 63) + a = array.array("i", deltas[offset : offset + 64]) + if sys.byteorder != "big": + a.byteswap() + bytearr.extend(a) + offset += 64 + runLength -= 64 + if runLength: + bytearr.append(DELTAS_ARE_LONGS | (runLength - 1)) + a = array.array("i", deltas[offset:pos]) + if sys.byteorder != "big": + a.byteswap() + bytearr.extend(a) + return pos + + @staticmethod + def decompileDeltas_(numDeltas, data, offset=0): + """(numDeltas, data, offset) --> ([delta, delta, ...], newOffset)""" + result = [] + pos = offset + while len(result) < numDeltas if numDeltas is not None else pos < len(data): + runHeader = data[pos] + pos += 1 + numDeltasInRun = (runHeader & DELTA_RUN_COUNT_MASK) + 1 + if (runHeader & DELTAS_SIZE_MASK) == DELTAS_ARE_ZERO: + result.extend([0] * numDeltasInRun) + else: + if (runHeader & DELTAS_SIZE_MASK) == DELTAS_ARE_LONGS: + deltas = array.array("i") + deltasSize = numDeltasInRun * 4 + elif (runHeader & DELTAS_SIZE_MASK) == DELTAS_ARE_WORDS: + deltas = array.array("h") + deltasSize = numDeltasInRun * 2 + else: + deltas = array.array("b") + deltasSize = numDeltasInRun + deltas.frombytes(data[pos : pos + deltasSize]) + if sys.byteorder != "big": + deltas.byteswap() + assert len(deltas) == numDeltasInRun, (len(deltas), numDeltasInRun) + pos += deltasSize + result.extend(deltas) + assert numDeltas is None or len(result) == numDeltas + return (result, pos) + + @staticmethod + def getTupleSize_(flags, axisCount): + size = 4 + if (flags & EMBEDDED_PEAK_TUPLE) != 0: + size += axisCount * 2 + if (flags & INTERMEDIATE_REGION) != 0: + size += axisCount * 4 + return size + + def getCoordWidth(self): + """Return 2 if coordinates are (x, y) as in gvar, 1 if single values + as in cvar, or 0 if empty. + """ + firstDelta = next((c for c in self.coordinates if c is not None), None) + if firstDelta is None: + return 0 # empty or has no impact + if type(firstDelta) in (int, float): + return 1 + if type(firstDelta) is tuple and len(firstDelta) == 2: + return 2 + raise TypeError( + "invalid type of delta; expected (int or float) number, or " + "Tuple[number, number]: %r" % firstDelta + ) + + def scaleDeltas(self, scalar): + if scalar == 1.0: + return # no change + coordWidth = self.getCoordWidth() + self.coordinates = [ + ( + None + if d is None + else d * scalar if coordWidth == 1 else (d[0] * scalar, d[1] * scalar) + ) + for d in self.coordinates + ] + + def roundDeltas(self): + coordWidth = self.getCoordWidth() + self.coordinates = [ + ( + None + if d is None + else otRound(d) if coordWidth == 1 else (otRound(d[0]), otRound(d[1])) + ) + for d in self.coordinates + ] + + def calcInferredDeltas(self, origCoords, endPts): + from fontTools.varLib.iup import iup_delta + + if self.getCoordWidth() == 1: + raise TypeError("Only 'gvar' TupleVariation can have inferred deltas") + if None in self.coordinates: + if len(self.coordinates) != len(origCoords): + raise ValueError( + "Expected len(origCoords) == %d; found %d" + % (len(self.coordinates), len(origCoords)) + ) + self.coordinates = iup_delta(self.coordinates, origCoords, endPts) + + def optimize(self, origCoords, endPts, tolerance=0.5, isComposite=False): + from fontTools.varLib.iup import iup_delta_optimize + + if isComposite: + warnings.warn( + "The isComposite argument is deprecated and may be removed in a future version", + DeprecationWarning, + ) + + if None in self.coordinates: + return # already optimized + + deltaOpt = iup_delta_optimize( + self.coordinates, origCoords, endPts, tolerance=tolerance + ) + if None in deltaOpt: + # Use "optimized" version only if smaller... + varOpt = TupleVariation(self.axes, deltaOpt) + + # Shouldn't matter that this is different from fvar...? + axisTags = sorted(self.axes.keys()) + tupleData, auxData = self.compile(axisTags) + unoptimizedLength = len(tupleData) + len(auxData) + tupleData, auxData = varOpt.compile(axisTags) + optimizedLength = len(tupleData) + len(auxData) + + if optimizedLength < unoptimizedLength: + self.coordinates = varOpt.coordinates + + def __imul__(self, scalar): + self.scaleDeltas(scalar) + return self + + def __iadd__(self, other): + if not isinstance(other, TupleVariation): + return NotImplemented + deltas1 = self.coordinates + length = len(deltas1) + deltas2 = other.coordinates + if len(deltas2) != length: + raise ValueError("cannot sum TupleVariation deltas with different lengths") + # 'None' values have different meanings in gvar vs cvar TupleVariations: + # within the gvar, when deltas are not provided explicitly for some points, + # they need to be inferred; whereas for the 'cvar' table, if deltas are not + # provided for some CVT values, then no adjustments are made (i.e. None == 0). + # Thus, we cannot sum deltas for gvar TupleVariations if they contain + # inferred inferred deltas (the latter need to be computed first using + # 'calcInferredDeltas' method), but we can treat 'None' values in cvar + # deltas as if they are zeros. + if self.getCoordWidth() == 2: + for i, d2 in zip(range(length), deltas2): + d1 = deltas1[i] + try: + deltas1[i] = (d1[0] + d2[0], d1[1] + d2[1]) + except TypeError: + raise ValueError("cannot sum gvar deltas with inferred points") + else: + for i, d2 in zip(range(length), deltas2): + d1 = deltas1[i] + if d1 is not None and d2 is not None: + deltas1[i] = d1 + d2 + elif d1 is None and d2 is not None: + deltas1[i] = d2 + # elif d2 is None do nothing + return self + + +def decompileSharedTuples(axisTags, sharedTupleCount, data, offset): + result = [] + for _ in range(sharedTupleCount): + t, offset = TupleVariation.decompileCoord_(axisTags, data, offset) + result.append(t) + return result + + +def compileSharedTuples( + axisTags, variations, MAX_NUM_SHARED_COORDS=TUPLE_INDEX_MASK + 1 +): + coordCount = Counter() + for var in variations: + coord = var.compileCoord(axisTags) + coordCount[coord] += 1 + # In python < 3.7, most_common() ordering is non-deterministic + # so apply a sort to make sure the ordering is consistent. + sharedCoords = sorted( + coordCount.most_common(MAX_NUM_SHARED_COORDS), + key=lambda item: (-item[1], item[0]), + ) + return [c[0] for c in sharedCoords if c[1] > 1] + + +def compileTupleVariationStore( + variations, + pointCount, + axisTags, + sharedTupleIndices, + useSharedPoints=True, + *, + optimizeSize=True, +): + # pointCount is actually unused. Keeping for API compat. + del pointCount + newVariations = [] + pointDatas = [] + # Compile all points and figure out sharing if desired + sharedPoints = None + + # Collect, count, and compile point-sets for all variation sets + pointSetCount = defaultdict(int) + for v in variations: + points = v.getUsedPoints() + if points is None: # Empty variations + continue + pointSetCount[points] += 1 + newVariations.append(v) + pointDatas.append(points) + variations = newVariations + del newVariations + + if not variations: + return (0, b"", b"") + + n = len(variations[0].coordinates) + assert all( + len(v.coordinates) == n for v in variations + ), "Variation sets have different sizes" + + compiledPoints = { + pointSet: TupleVariation.compilePoints(pointSet) for pointSet in pointSetCount + } + + tupleVariationCount = len(variations) + tuples = [] + data = [] + + if useSharedPoints: + # Find point-set which saves most bytes. + def key(pn): + pointSet = pn[0] + count = pn[1] + return len(compiledPoints[pointSet]) * (count - 1) + + sharedPoints = max(pointSetCount.items(), key=key)[0] + + data.append(compiledPoints[sharedPoints]) + tupleVariationCount |= TUPLES_SHARE_POINT_NUMBERS + + # b'' implies "use shared points" + pointDatas = [ + compiledPoints[points] if points != sharedPoints else b"" + for points in pointDatas + ] + + for v, p in zip(variations, pointDatas): + thisTuple, thisData = v.compile( + axisTags, sharedTupleIndices, pointData=p, optimizeSize=optimizeSize + ) + + tuples.append(thisTuple) + data.append(thisData) + + tuples = b"".join(tuples) + data = b"".join(data) + return tupleVariationCount, tuples, data + + +def decompileTupleVariationStore( + tableTag, + axisTags, + tupleVariationCount, + pointCount, + sharedTuples, + data, + pos, + dataPos, +): + numAxes = len(axisTags) + result = [] + if (tupleVariationCount & TUPLES_SHARE_POINT_NUMBERS) != 0: + sharedPoints, dataPos = TupleVariation.decompilePoints_( + pointCount, data, dataPos, tableTag + ) + else: + sharedPoints = [] + for _ in range(tupleVariationCount & TUPLE_COUNT_MASK): + dataSize, flags = struct.unpack(">HH", data[pos : pos + 4]) + tupleSize = TupleVariation.getTupleSize_(flags, numAxes) + tupleData = data[pos : pos + tupleSize] + pointDeltaData = data[dataPos : dataPos + dataSize] + result.append( + decompileTupleVariation_( + pointCount, + sharedTuples, + sharedPoints, + tableTag, + axisTags, + tupleData, + pointDeltaData, + ) + ) + pos += tupleSize + dataPos += dataSize + return result + + +def decompileTupleVariation_( + pointCount, sharedTuples, sharedPoints, tableTag, axisTags, data, tupleData +): + assert tableTag in ("cvar", "gvar"), tableTag + flags = struct.unpack(">H", data[2:4])[0] + pos = 4 + if (flags & EMBEDDED_PEAK_TUPLE) == 0: + peak = sharedTuples[flags & TUPLE_INDEX_MASK] + else: + peak, pos = TupleVariation.decompileCoord_(axisTags, data, pos) + if (flags & INTERMEDIATE_REGION) != 0: + start, pos = TupleVariation.decompileCoord_(axisTags, data, pos) + end, pos = TupleVariation.decompileCoord_(axisTags, data, pos) + else: + start, end = inferRegion_(peak) + axes = {} + for axis in axisTags: + region = start[axis], peak[axis], end[axis] + if region != (0.0, 0.0, 0.0): + axes[axis] = region + pos = 0 + if (flags & PRIVATE_POINT_NUMBERS) != 0: + points, pos = TupleVariation.decompilePoints_( + pointCount, tupleData, pos, tableTag + ) + else: + points = sharedPoints + + deltas = [None] * pointCount + + if tableTag == "cvar": + deltas_cvt, pos = TupleVariation.decompileDeltas_(len(points), tupleData, pos) + for p, delta in zip(points, deltas_cvt): + if 0 <= p < pointCount: + deltas[p] = delta + + elif tableTag == "gvar": + deltas_x, pos = TupleVariation.decompileDeltas_(len(points), tupleData, pos) + deltas_y, pos = TupleVariation.decompileDeltas_(len(points), tupleData, pos) + for p, x, y in zip(points, deltas_x, deltas_y): + if 0 <= p < pointCount: + deltas[p] = (x, y) + + return TupleVariation(axes, deltas) + + +def inferRegion_(peak): + """Infer start and end for a (non-intermediate) region + + This helper function computes the applicability region for + variation tuples whose INTERMEDIATE_REGION flag is not set in the + TupleVariationHeader structure. Variation tuples apply only to + certain regions of the variation space; outside that region, the + tuple has no effect. To make the binary encoding more compact, + TupleVariationHeaders can omit the intermediateStartTuple and + intermediateEndTuple fields. + """ + start, end = {}, {} + for axis, value in peak.items(): + start[axis] = min(value, 0.0) # -0.3 --> -0.3; 0.7 --> 0.0 + end[axis] = max(value, 0.0) # -0.3 --> 0.0; 0.7 --> 0.7 + return (start, end) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_A_R_C_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_A_R_C_.py new file mode 100644 index 0000000000000000000000000000000000000000..afc1497d14ffd4aed6668af4cb0587135e5562f9 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_A_R_C_.py @@ -0,0 +1,12 @@ +from .otBase import BaseTTXConverter + + +class table_V_A_R_C_(BaseTTXConverter): + """Variable Components table + + The ``VARC`` table contains variation information for composite glyphs. + + See also https://github.com/harfbuzz/boring-expansion-spec/blob/main/VARC.md + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_D_M_X_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_D_M_X_.py new file mode 100644 index 0000000000000000000000000000000000000000..6f8abc46de903fbc300fc4ee4b803cb47bf03ba0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_D_M_X_.py @@ -0,0 +1,249 @@ +from . import DefaultTable +from fontTools.misc import sstruct +from fontTools.misc.textTools import safeEval +import struct + +VDMX_HeaderFmt = """ + > # big endian + version: H # Version number (0 or 1) + numRecs: H # Number of VDMX groups present + numRatios: H # Number of aspect ratio groupings +""" +# the VMDX header is followed by an array of RatRange[numRatios] (i.e. aspect +# ratio ranges); +VDMX_RatRangeFmt = """ + > # big endian + bCharSet: B # Character set + xRatio: B # Value to use for x-Ratio + yStartRatio: B # Starting y-Ratio value + yEndRatio: B # Ending y-Ratio value +""" +# followed by an array of offset[numRatios] from start of VDMX table to the +# VDMX Group for this ratio range (offsets will be re-calculated on compile); +# followed by an array of Group[numRecs] records; +VDMX_GroupFmt = """ + > # big endian + recs: H # Number of height records in this group + startsz: B # Starting yPelHeight + endsz: B # Ending yPelHeight +""" +# followed by an array of vTable[recs] records. +VDMX_vTableFmt = """ + > # big endian + yPelHeight: H # yPelHeight to which values apply + yMax: h # Maximum value (in pels) for this yPelHeight + yMin: h # Minimum value (in pels) for this yPelHeight +""" + + +class table_V_D_M_X_(DefaultTable.DefaultTable): + """Vertical Device Metrics table + + The ``VDMX`` table records changes to the vertical glyph minima + and maxima that result from Truetype instructions. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/vdmx + """ + + def decompile(self, data, ttFont): + pos = 0 # track current position from to start of VDMX table + dummy, data = sstruct.unpack2(VDMX_HeaderFmt, data, self) + pos += sstruct.calcsize(VDMX_HeaderFmt) + self.ratRanges = [] + for i in range(self.numRatios): + ratio, data = sstruct.unpack2(VDMX_RatRangeFmt, data) + pos += sstruct.calcsize(VDMX_RatRangeFmt) + # the mapping between a ratio and a group is defined further below + ratio["groupIndex"] = None + self.ratRanges.append(ratio) + lenOffset = struct.calcsize(">H") + _offsets = [] # temporarily store offsets to groups + for i in range(self.numRatios): + offset = struct.unpack(">H", data[0:lenOffset])[0] + data = data[lenOffset:] + pos += lenOffset + _offsets.append(offset) + self.groups = [] + for groupIndex in range(self.numRecs): + # the offset to this group from beginning of the VDMX table + currOffset = pos + group, data = sstruct.unpack2(VDMX_GroupFmt, data) + # the group lenght and bounding sizes are re-calculated on compile + recs = group.pop("recs") + startsz = group.pop("startsz") + endsz = group.pop("endsz") + pos += sstruct.calcsize(VDMX_GroupFmt) + for j in range(recs): + vTable, data = sstruct.unpack2(VDMX_vTableFmt, data) + vTableLength = sstruct.calcsize(VDMX_vTableFmt) + pos += vTableLength + # group is a dict of (yMax, yMin) tuples keyed by yPelHeight + group[vTable["yPelHeight"]] = (vTable["yMax"], vTable["yMin"]) + # make sure startsz and endsz match the calculated values + minSize = min(group.keys()) + maxSize = max(group.keys()) + assert ( + startsz == minSize + ), "startsz (%s) must equal min yPelHeight (%s): group %d" % ( + group.startsz, + minSize, + groupIndex, + ) + assert ( + endsz == maxSize + ), "endsz (%s) must equal max yPelHeight (%s): group %d" % ( + group.endsz, + maxSize, + groupIndex, + ) + self.groups.append(group) + # match the defined offsets with the current group's offset + for offsetIndex, offsetValue in enumerate(_offsets): + # when numRecs < numRatios there can more than one ratio range + # sharing the same VDMX group + if currOffset == offsetValue: + # map the group with the ratio range thas has the same + # index as the offset to that group (it took me a while..) + self.ratRanges[offsetIndex]["groupIndex"] = groupIndex + # check that all ratio ranges have a group + for i in range(self.numRatios): + ratio = self.ratRanges[i] + if ratio["groupIndex"] is None: + from fontTools import ttLib + + raise ttLib.TTLibError("no group defined for ratRange %d" % i) + + def _getOffsets(self): + """ + Calculate offsets to VDMX_Group records. + For each ratRange return a list of offset values from the beginning of + the VDMX table to a VDMX_Group. + """ + lenHeader = sstruct.calcsize(VDMX_HeaderFmt) + lenRatRange = sstruct.calcsize(VDMX_RatRangeFmt) + lenOffset = struct.calcsize(">H") + lenGroupHeader = sstruct.calcsize(VDMX_GroupFmt) + lenVTable = sstruct.calcsize(VDMX_vTableFmt) + # offset to the first group + pos = lenHeader + self.numRatios * lenRatRange + self.numRatios * lenOffset + groupOffsets = [] + for group in self.groups: + groupOffsets.append(pos) + lenGroup = lenGroupHeader + len(group) * lenVTable + pos += lenGroup # offset to next group + offsets = [] + for ratio in self.ratRanges: + groupIndex = ratio["groupIndex"] + offsets.append(groupOffsets[groupIndex]) + return offsets + + def compile(self, ttFont): + if not (self.version == 0 or self.version == 1): + from fontTools import ttLib + + raise ttLib.TTLibError( + "unknown format for VDMX table: version %s" % self.version + ) + data = sstruct.pack(VDMX_HeaderFmt, self) + for ratio in self.ratRanges: + data += sstruct.pack(VDMX_RatRangeFmt, ratio) + # recalculate offsets to VDMX groups + for offset in self._getOffsets(): + data += struct.pack(">H", offset) + for group in self.groups: + recs = len(group) + startsz = min(group.keys()) + endsz = max(group.keys()) + gHeader = {"recs": recs, "startsz": startsz, "endsz": endsz} + data += sstruct.pack(VDMX_GroupFmt, gHeader) + for yPelHeight, (yMax, yMin) in sorted(group.items()): + vTable = {"yPelHeight": yPelHeight, "yMax": yMax, "yMin": yMin} + data += sstruct.pack(VDMX_vTableFmt, vTable) + return data + + def toXML(self, writer, ttFont): + writer.simpletag("version", value=self.version) + writer.newline() + writer.begintag("ratRanges") + writer.newline() + for ratio in self.ratRanges: + groupIndex = ratio["groupIndex"] + writer.simpletag( + "ratRange", + bCharSet=ratio["bCharSet"], + xRatio=ratio["xRatio"], + yStartRatio=ratio["yStartRatio"], + yEndRatio=ratio["yEndRatio"], + groupIndex=groupIndex, + ) + writer.newline() + writer.endtag("ratRanges") + writer.newline() + writer.begintag("groups") + writer.newline() + for groupIndex in range(self.numRecs): + group = self.groups[groupIndex] + recs = len(group) + startsz = min(group.keys()) + endsz = max(group.keys()) + writer.begintag("group", index=groupIndex) + writer.newline() + writer.comment("recs=%d, startsz=%d, endsz=%d" % (recs, startsz, endsz)) + writer.newline() + for yPelHeight, (yMax, yMin) in sorted(group.items()): + writer.simpletag( + "record", + [("yPelHeight", yPelHeight), ("yMax", yMax), ("yMin", yMin)], + ) + writer.newline() + writer.endtag("group") + writer.newline() + writer.endtag("groups") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": + self.version = safeEval(attrs["value"]) + elif name == "ratRanges": + if not hasattr(self, "ratRanges"): + self.ratRanges = [] + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name == "ratRange": + if not hasattr(self, "numRatios"): + self.numRatios = 1 + else: + self.numRatios += 1 + ratio = { + "bCharSet": safeEval(attrs["bCharSet"]), + "xRatio": safeEval(attrs["xRatio"]), + "yStartRatio": safeEval(attrs["yStartRatio"]), + "yEndRatio": safeEval(attrs["yEndRatio"]), + "groupIndex": safeEval(attrs["groupIndex"]), + } + self.ratRanges.append(ratio) + elif name == "groups": + if not hasattr(self, "groups"): + self.groups = [] + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name == "group": + if not hasattr(self, "numRecs"): + self.numRecs = 1 + else: + self.numRecs += 1 + group = {} + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name == "record": + yPelHeight = safeEval(attrs["yPelHeight"]) + yMax = safeEval(attrs["yMax"]) + yMin = safeEval(attrs["yMin"]) + group[yPelHeight] = (yMax, yMin) + self.groups.append(group) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_O_R_G_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_O_R_G_.py new file mode 100644 index 0000000000000000000000000000000000000000..2c53acba431059f58f86dbb1b89755a03f699f94 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_O_R_G_.py @@ -0,0 +1,165 @@ +from fontTools.misc.textTools import bytesjoin, safeEval +from . import DefaultTable +import struct + + +class table_V_O_R_G_(DefaultTable.DefaultTable): + """Vertical Origin table + + The ``VORG`` table contains the vertical origin of each glyph + in a `CFF` or `CFF2` font. + + This table is structured so that you can treat it like a dictionary keyed by glyph name. + + ``ttFont['VORG'][<glyphName>]`` will return the vertical origin for any glyph. + + ``ttFont['VORG'][<glyphName>] = <value>`` will set the vertical origin for any glyph. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/vorg + """ + + def decompile(self, data, ttFont): + self.getGlyphName = ( + ttFont.getGlyphName + ) # for use in get/set item functions, for access by GID + ( + self.majorVersion, + self.minorVersion, + self.defaultVertOriginY, + self.numVertOriginYMetrics, + ) = struct.unpack(">HHhH", data[:8]) + assert ( + self.majorVersion <= 1 + ), "Major version of VORG table is higher than I know how to handle" + data = data[8:] + vids = [] + gids = [] + pos = 0 + for i in range(self.numVertOriginYMetrics): + gid, vOrigin = struct.unpack(">Hh", data[pos : pos + 4]) + pos += 4 + gids.append(gid) + vids.append(vOrigin) + + self.VOriginRecords = vOrig = {} + glyphOrder = ttFont.getGlyphOrder() + try: + names = [glyphOrder[gid] for gid in gids] + except IndexError: + getGlyphName = self.getGlyphName + names = map(getGlyphName, gids) + + for name, vid in zip(names, vids): + vOrig[name] = vid + + def compile(self, ttFont): + vorgs = list(self.VOriginRecords.values()) + names = list(self.VOriginRecords.keys()) + nameMap = ttFont.getReverseGlyphMap() + try: + gids = [nameMap[name] for name in names] + except KeyError: + nameMap = ttFont.getReverseGlyphMap(rebuild=True) + gids = [nameMap[name] for name in names] + vOriginTable = list(zip(gids, vorgs)) + self.numVertOriginYMetrics = len(vorgs) + vOriginTable.sort() # must be in ascending GID order + dataList = [struct.pack(">Hh", rec[0], rec[1]) for rec in vOriginTable] + header = struct.pack( + ">HHhH", + self.majorVersion, + self.minorVersion, + self.defaultVertOriginY, + self.numVertOriginYMetrics, + ) + dataList.insert(0, header) + data = bytesjoin(dataList) + return data + + def toXML(self, writer, ttFont): + writer.simpletag("majorVersion", value=self.majorVersion) + writer.newline() + writer.simpletag("minorVersion", value=self.minorVersion) + writer.newline() + writer.simpletag("defaultVertOriginY", value=self.defaultVertOriginY) + writer.newline() + writer.simpletag("numVertOriginYMetrics", value=self.numVertOriginYMetrics) + writer.newline() + vOriginTable = [] + glyphNames = self.VOriginRecords.keys() + for glyphName in glyphNames: + try: + gid = ttFont.getGlyphID(glyphName) + except: + assert 0, ( + "VORG table contains a glyph name not in ttFont.getGlyphNames(): " + + str(glyphName) + ) + vOriginTable.append([gid, glyphName, self.VOriginRecords[glyphName]]) + vOriginTable.sort() + for entry in vOriginTable: + vOriginRec = VOriginRecord(entry[1], entry[2]) + vOriginRec.toXML(writer, ttFont) + + def fromXML(self, name, attrs, content, ttFont): + if not hasattr(self, "VOriginRecords"): + self.VOriginRecords = {} + self.getGlyphName = ( + ttFont.getGlyphName + ) # for use in get/set item functions, for access by GID + if name == "VOriginRecord": + vOriginRec = VOriginRecord() + for element in content: + if isinstance(element, str): + continue + name, attrs, content = element + vOriginRec.fromXML(name, attrs, content, ttFont) + self.VOriginRecords[vOriginRec.glyphName] = vOriginRec.vOrigin + elif "value" in attrs: + setattr(self, name, safeEval(attrs["value"])) + + def __getitem__(self, glyphSelector): + if isinstance(glyphSelector, int): + # its a gid, convert to glyph name + glyphSelector = self.getGlyphName(glyphSelector) + + if glyphSelector not in self.VOriginRecords: + return self.defaultVertOriginY + + return self.VOriginRecords[glyphSelector] + + def __setitem__(self, glyphSelector, value): + if isinstance(glyphSelector, int): + # its a gid, convert to glyph name + glyphSelector = self.getGlyphName(glyphSelector) + + if value != self.defaultVertOriginY: + self.VOriginRecords[glyphSelector] = value + elif glyphSelector in self.VOriginRecords: + del self.VOriginRecords[glyphSelector] + + def __delitem__(self, glyphSelector): + del self.VOriginRecords[glyphSelector] + + +class VOriginRecord(object): + def __init__(self, name=None, vOrigin=None): + self.glyphName = name + self.vOrigin = vOrigin + + def toXML(self, writer, ttFont): + writer.begintag("VOriginRecord") + writer.newline() + writer.simpletag("glyphName", value=self.glyphName) + writer.newline() + writer.simpletag("vOrigin", value=self.vOrigin) + writer.newline() + writer.endtag("VOriginRecord") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + value = attrs["value"] + if name == "glyphName": + setattr(self, name, value) + else: + setattr(self, name, safeEval(value)) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_V_A_R_.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_V_A_R_.py new file mode 100644 index 0000000000000000000000000000000000000000..c0c94e348b2c6cdfc856b030477214604c95d7c4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/V_V_A_R_.py @@ -0,0 +1,13 @@ +from .otBase import BaseTTXConverter + + +class table_V_V_A_R_(BaseTTXConverter): + """Vertical Metrics Variations table + + The ``VVAR`` table contains variation data for per-glyph vertical metrics + in a variable font. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/vvar + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/__init__.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4a970ba9c02f14e081a277121aca81556e323edb --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/__init__.py @@ -0,0 +1,97 @@ +# DON'T EDIT! This file is generated by MetaTools/buildTableList.py. +def _moduleFinderHint(): + """Dummy function to let modulefinder know what tables may be + dynamically imported. Generated by MetaTools/buildTableList.py. + + >>> _moduleFinderHint() + """ + from . import B_A_S_E_ + from . import C_B_D_T_ + from . import C_B_L_C_ + from . import C_F_F_ + from . import C_F_F__2 + from . import C_O_L_R_ + from . import C_P_A_L_ + from . import D_S_I_G_ + from . import D__e_b_g + from . import E_B_D_T_ + from . import E_B_L_C_ + from . import F_F_T_M_ + from . import F__e_a_t + from . import G_D_E_F_ + from . import G_P_O_S_ + from . import G_S_U_B_ + from . import G_V_A_R_ + from . import G__l_a_t + from . import G__l_o_c + from . import H_V_A_R_ + from . import I_F_T_ + from . import I_F_T_X_ + from . import J_S_T_F_ + from . import L_T_S_H_ + from . import M_A_T_H_ + from . import M_V_A_R_ + from . import O_S_2f_2 + from . import S_T_A_T_ + from . import S_V_G_ + from . import S__i_l_f + from . import S__i_l_l + from . import T_S_I_B_ + from . import T_S_I_C_ + from . import T_S_I_D_ + from . import T_S_I_J_ + from . import T_S_I_P_ + from . import T_S_I_S_ + from . import T_S_I_V_ + from . import T_S_I__0 + from . import T_S_I__1 + from . import T_S_I__2 + from . import T_S_I__3 + from . import T_S_I__5 + from . import T_T_F_A_ + from . import V_A_R_C_ + from . import V_D_M_X_ + from . import V_O_R_G_ + from . import V_V_A_R_ + from . import _a_n_k_r + from . import _a_v_a_r + from . import _b_g_c_l + from . import _b_s_l_n + from . import _c_i_d_g + from . import _c_m_a_p + from . import _c_v_a_r + from . import _c_v_t + from . import _f_e_a_t + from . import _f_p_g_m + from . import _f_v_a_r + from . import _g_a_s_p + from . import _g_c_i_d + from . import _g_l_y_f + from . import _g_v_a_r + from . import _h_d_m_x + from . import _h_e_a_d + from . import _h_h_e_a + from . import _h_m_t_x + from . import _k_e_r_n + from . import _l_c_a_r + from . import _l_o_c_a + from . import _l_t_a_g + from . import _m_a_x_p + from . import _m_e_t_a + from . import _m_o_r_t + from . import _m_o_r_x + from . import _n_a_m_e + from . import _o_p_b_d + from . import _p_o_s_t + from . import _p_r_e_p + from . import _p_r_o_p + from . import _s_b_i_x + from . import _t_r_a_k + from . import _v_h_e_a + from . import _v_m_t_x + + +if __name__ == "__main__": + import doctest, sys + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_a_n_k_r.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_a_n_k_r.py new file mode 100644 index 0000000000000000000000000000000000000000..5466d42d4117f9935b54def71e5351fa1a0ee6ab --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_a_n_k_r.py @@ -0,0 +1,15 @@ +from .otBase import BaseTTXConverter + + +class table__a_n_k_r(BaseTTXConverter): + """Anchor Point table + + The anchor point table provides a way to define anchor points. + These are points within the coordinate space of a given glyph, + independent of the control points used to render the glyph. + Anchor points are used in conjunction with the ``kerx`` table. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6ankr.html + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_a_v_a_r.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_a_v_a_r.py new file mode 100644 index 0000000000000000000000000000000000000000..cb4593712c91388e085e371650e8bff8dfb24740 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_a_v_a_r.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +from typing import Mapping, TYPE_CHECKING +from fontTools.misc import sstruct +from fontTools.misc.fixedTools import ( + fixedToFloat as fi2fl, + floatToFixed as fl2fi, + floatToFixedToStr as fl2str, + strToFixedToFloat as str2fl, +) +from fontTools.misc.textTools import bytesjoin, safeEval +from fontTools.misc.roundTools import otRound +from fontTools.varLib.models import piecewiseLinearMap +from fontTools.varLib.varStore import VarStoreInstancer, NO_VARIATION_INDEX +from fontTools.ttLib import TTLibError +from . import DefaultTable +from . import otTables +import struct +import logging + + +log = logging.getLogger(__name__) + +from .otBase import BaseTTXConverter + +if TYPE_CHECKING: + from ..ttFont import TTFont + + +class table__a_v_a_r(BaseTTXConverter): + """Axis Variations table + + This class represents the ``avar`` table of a variable font. The object has one + substantive attribute, ``segments``, which maps axis tags to a segments dictionary:: + + >>> font["avar"].segments # doctest: +SKIP + {'wght': {-1.0: -1.0, + 0.0: 0.0, + 0.125: 0.11444091796875, + 0.25: 0.23492431640625, + 0.5: 0.35540771484375, + 0.625: 0.5, + 0.75: 0.6566162109375, + 0.875: 0.81927490234375, + 1.0: 1.0}, + 'ital': {-1.0: -1.0, 0.0: 0.0, 1.0: 1.0}} + + Notice that the segments dictionary is made up of normalized values. A valid + ``avar`` segment mapping must contain the entries ``-1.0: -1.0, 0.0: 0.0, 1.0: 1.0``. + fontTools does not enforce this, so it is your responsibility to ensure that + mappings are valid. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/avar + """ + + dependencies = ["fvar"] + + def __init__(self, tag=None): + super().__init__(tag) + self.segments = {} + + def compile(self, ttFont): + axisTags = [axis.axisTag for axis in ttFont["fvar"].axes] + if not hasattr(self, "table"): + self.table = otTables.avar() + if not hasattr(self.table, "Reserved"): + self.table.Reserved = 0 + self.table.Version = (getattr(self, "majorVersion", 1) << 16) | getattr( + self, "minorVersion", 0 + ) + self.table.AxisCount = len(axisTags) + self.table.AxisSegmentMap = [] + for axis in axisTags: + mappings = self.segments[axis] + segmentMap = otTables.AxisSegmentMap() + segmentMap.PositionMapCount = len(mappings) + segmentMap.AxisValueMap = [] + for key, value in sorted(mappings.items()): + valueMap = otTables.AxisValueMap() + valueMap.FromCoordinate = key + valueMap.ToCoordinate = value + segmentMap.AxisValueMap.append(valueMap) + self.table.AxisSegmentMap.append(segmentMap) + return super().compile(ttFont) + + def decompile(self, data, ttFont): + super().decompile(data, ttFont) + self.majorVersion = self.table.Version >> 16 + self.minorVersion = self.table.Version & 0xFFFF + if self.majorVersion not in (1, 2): + raise NotImplementedError("Unknown avar table version") + axisTags = [axis.axisTag for axis in ttFont["fvar"].axes] + for axis in axisTags: + self.segments[axis] = {} + for axis, segmentMap in zip(axisTags, self.table.AxisSegmentMap): + segments = self.segments[axis] = {} + for segment in segmentMap.AxisValueMap: + segments[segment.FromCoordinate] = segment.ToCoordinate + + def toXML(self, writer, ttFont): + writer.simpletag( + "version", + major=getattr(self, "majorVersion", 1), + minor=getattr(self, "minorVersion", 0), + ) + writer.newline() + axisTags = [axis.axisTag for axis in ttFont["fvar"].axes] + for axis in axisTags: + writer.begintag("segment", axis=axis) + writer.newline() + for key, value in sorted(self.segments[axis].items()): + key = fl2str(key, 14) + value = fl2str(value, 14) + writer.simpletag("mapping", **{"from": key, "to": value}) + writer.newline() + writer.endtag("segment") + writer.newline() + if getattr(self, "majorVersion", 1) >= 2: + if self.table.VarIdxMap: + self.table.VarIdxMap.toXML(writer, ttFont, name="VarIdxMap") + if self.table.VarStore: + self.table.VarStore.toXML(writer, ttFont) + + def fromXML(self, name, attrs, content, ttFont): + if not hasattr(self, "table"): + self.table = otTables.avar() + if not hasattr(self.table, "Reserved"): + self.table.Reserved = 0 + if name == "version": + self.majorVersion = safeEval(attrs["major"]) + self.minorVersion = safeEval(attrs["minor"]) + self.table.Version = (getattr(self, "majorVersion", 1) << 16) | getattr( + self, "minorVersion", 0 + ) + elif name == "segment": + axis = attrs["axis"] + segment = self.segments[axis] = {} + for element in content: + if isinstance(element, tuple): + elementName, elementAttrs, _ = element + if elementName == "mapping": + fromValue = str2fl(elementAttrs["from"], 14) + toValue = str2fl(elementAttrs["to"], 14) + if fromValue in segment: + log.warning( + "duplicate entry for %s in axis '%s'", fromValue, axis + ) + segment[fromValue] = toValue + else: + super().fromXML(name, attrs, content, ttFont) + + def renormalizeLocation( + self, location: Mapping[str, float], font: TTFont, dropZeroes: bool = True + ) -> dict[str, float]: + majorVersion = getattr(self, "majorVersion", 1) + + if majorVersion not in (1, 2): + raise NotImplementedError("Unknown avar table version") + + avarSegments = self.segments + mappedLocation = {} + for axisTag, value in location.items(): + avarMapping = avarSegments.get(axisTag, None) + if avarMapping is not None: + value = piecewiseLinearMap(value, avarMapping) + mappedLocation[axisTag] = value + + if majorVersion < 2: + return mappedLocation + + # Version 2 + + varIdxMap = self.table.VarIdxMap + varStore = self.table.VarStore + axes = font["fvar"].axes + if varStore is not None: + instancer = VarStoreInstancer(varStore, axes, mappedLocation) + + coords = list(fl2fi(mappedLocation.get(axis.axisTag, 0), 14) for axis in axes) + + out = [] + for varIdx, v in enumerate(coords): + + if varIdxMap is not None: + varIdx = varIdxMap[varIdx] + + if varStore is not None: + delta = instancer[varIdx] + v += otRound(delta) + v = min(max(v, -(1 << 14)), +(1 << 14)) + + out.append(v) + + mappedLocation = { + axis.axisTag: fi2fl(v, 14) + for v, axis in zip(out, axes) + if v != 0 or not dropZeroes + } + + return mappedLocation diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_b_g_c_l.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_b_g_c_l.py new file mode 100644 index 0000000000000000000000000000000000000000..7d4a2ffbfb292bbddf5ee65dd42bbc4f03eb7fe6 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_b_g_c_l.py @@ -0,0 +1,136 @@ +"""Support for the `bgcl` (Apple Color Emoji background) table. + +This table stores a JSON blob. We decode it to a Python object on +decompile and emit human readable JSON in the TTX via a <json> element. +On compile we re-encode the JSON to UTF-8 bytes which is what Apple code +reads via CTFontCopyTable then JSONDecoder. + +On iOS 16 and later the `bgcl` payload is used as the wallpaper +background when the user selects an emoji wallpaper. + +Fields: + +- ``colors``: list of palette entries. Each entry is an array of four + integers ``[R, G, B, A]``. R/G/B are 0-255. A is 0-1. + +- ``emojicolors``: list of per-emoji palettes. Each item is an array of + three sublists: primary/dominant, accent, contextual + (names inferred). Each sublist contains integer indexes referencing + entries in ``colors``; the runtime uses these to assemble layered + background tints for an emoji. + +- ``indexmap``: mapping (glyph identifier → palette index). The map + maps a glyph identity to an integer index selecting an entry in + ``emojicolors``. The font/UI uses this to pick the correct palette + for a glyph when rendering wallpaper backgrounds. + +- ``version``: integer table version used for parsing/compatibility. + +Runtime usage summary: + +- The system fetches the table bytes with ``CTFontCopyTable('bgcl')``, + decodes the bytes as UTF‑8 JSON and runs the JSON through the app's + decoder into an internal ``BgclTable`` structure. The app looks up a + glyph's entry in ``indexmap``, retrieves the corresponding + ``emojicolors`` palette, then converts the referenced ``colors`` + entries into color objects (normalizing channels/alpha as needed). + This resulting color(s) drive the wallpaper background appearance for + emoji wallpapers on supported iOS versions. + +This implementation preserves the JSON payload and exposes convenience +attributes ``colors``, ``emojicolors``, ``indexmap`` and ``version`` +when available. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from fontTools.misc.textTools import tostr, strjoin +from . import DefaultTable +import json + +if TYPE_CHECKING: + from fontTools.misc.xmlWriter import XMLWriter + from fontTools.ttLib import TTFont + + +class table__b_g_c_l(DefaultTable.DefaultTable): + """bgcl table: stores a JSON blob describing background palettes. + + The JSON structure typically contains the top-level keys: + - colors: [[R,G,B,A], ...] + - emojicolors: [[[dominant...],[accent...],[contextual...]], ...] + - indexmap: {"glyphIndex": emojicolors_index, ...} + - version: int + """ + + def decompile(self, data: bytes, ttFont: TTFont) -> None: + """Store raw bytes and attempt to parse JSON. + + The JSON commonly includes palette/lookup data used at runtime to + construct wallpaper/background colors for emoji wallpapers on + recent iOS releases. + """ + self.data = data + try: + text = tostr(data, "utf_8") + self.json = json.loads(text) + except Exception as e: # keep table decompilation robust + self.json = None + self.ERROR = f"bgcl JSON parse error: {e!r}" + return + # convenient attributes + self.colors = self.json.get("colors") + self.emojicolors = self.json.get("emojicolors") + self.indexmap = self.json.get("indexmap") + self.version = self.json.get("version") + + def compile(self, ttFont: TTFont) -> bytes: + """Encode the JSON object to UTF-8 bytes for font binary storage.""" + if getattr(self, "json", None) is None: + # fallback to raw bytes if parsing failed earlier + return getattr(self, "data", b"") + # use compact representation for binary table + return json.dumps(self.json, separators=(",", ":"), ensure_ascii=False).encode( + "utf_8" + ) + + def toXML(self, writer: XMLWriter, ttFont: TTFont) -> None: + """Emit pretty-printed JSON inside a <json> element for human inspection.""" + if getattr(self, "json", None) is None: + # fallback to default hex output + DefaultTable.DefaultTable.toXML(self, writer, ttFont) + return + writer.begintag("json") + writer.newline() + pretty = json.dumps(self.json, indent=2, ensure_ascii=False) + writer.writecdata(pretty) + writer.newline() + writer.endtag("json") + writer.newline() + + def fromXML( + self, name: str, attrs: dict[str, str], content, ttFont: TTFont + ) -> None: + """Read JSON from the <json> element. `content` may be a list. + + This mirrors SVG/other table `fromXML` handlers which accept a + list of content chunks. + """ + if name != "json": + # fall back to DefaultTable behavior for unknown elements + return DefaultTable.DefaultTable.fromXML(self, name, attrs, content, ttFont) + text = strjoin(content).strip() + try: + self.json = json.loads(text) + # keep raw bytes in sync + self.data = text.encode("utf_8") + self.colors = self.json.get("colors") + self.emojicolors = self.json.get("emojicolors") + self.indexmap = self.json.get("indexmap") + self.version = self.json.get("version") + except Exception as e: + # store error and fall back to raw + self.json = None + self.ERROR = f"bgcl JSON parse error in fromXML: {e!r}" diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_b_s_l_n.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_b_s_l_n.py new file mode 100644 index 0000000000000000000000000000000000000000..85796b0a0479d26104a3fb63d9fe6085be29853a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_b_s_l_n.py @@ -0,0 +1,15 @@ +from .otBase import BaseTTXConverter + + +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6bsln.html +class table__b_s_l_n(BaseTTXConverter): + """Baseline table + + The AAT ``bsln`` table is similar in purpose to the OpenType ``BASE`` + table; it stores per-script baselines to support automatic alignment + of lines of text. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6bsln.html + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_i_d_g.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_i_d_g.py new file mode 100644 index 0000000000000000000000000000000000000000..c283e5a4dd1619a621a78b20171d38b14752c836 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_i_d_g.py @@ -0,0 +1,24 @@ +# coding: utf-8 +from .otBase import BaseTTXConverter + + +class table__c_i_d_g(BaseTTXConverter): + """CID to Glyph ID table + + The AAT ``cidg`` table has almost the same structure as ``gidc``, + just mapping CIDs to GlyphIDs instead of the reverse direction. + + It is useful for fonts that may be used by a PDF renderer in lieu of + a font reference with a known glyph collection but no subsetted + glyphs. For instance, a PDF can say “please use a font conforming + to Adobe-Japan-1”; the ``cidg`` mapping is necessary if the font is, + say, a TrueType font. ``gidc`` is lossy for this purpose and is + obsoleted by ``cidg``. + + For example, the first font in ``/System/Library/Fonts/PingFang.ttc`` + (which Apple ships pre-installed on MacOS 10.12.6) has a ``cidg`` table. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6gcid.html + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_m_a_p.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_m_a_p.py new file mode 100644 index 0000000000000000000000000000000000000000..18d4bc64187405df7a54b57b26219016a9405f98 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_m_a_p.py @@ -0,0 +1,1597 @@ +from fontTools.misc.textTools import bytesjoin, safeEval, readHex +from fontTools.misc.encodingTools import getEncoding +from fontTools.ttLib import getSearchRange +from fontTools.unicode import Unicode +from . import DefaultTable +import sys +import struct +import array +import logging + + +log = logging.getLogger(__name__) + + +def _make_map(font, chars, gids): + assert len(chars) == len(gids) + glyphNames = font.getGlyphNameMany(gids) + cmap = {} + for char, gid, name in zip(chars, gids, glyphNames): + if gid == 0: + continue + cmap[char] = name + return cmap + + +class table__c_m_a_p(DefaultTable.DefaultTable): + """Character to Glyph Index Mapping Table + + This class represents the `cmap <https://docs.microsoft.com/en-us/typography/opentype/spec/cmap>`_ + table, which maps between input characters (in Unicode or other system encodings) + and glyphs within the font. The ``cmap`` table contains one or more subtables + which determine the mapping of of characters to glyphs across different platforms + and encoding systems. + + ``table__c_m_a_p`` objects expose an accessor ``.tables`` which provides access + to the subtables, although it is normally easier to retrieve individual subtables + through the utility methods described below. To add new subtables to a font, + first determine the subtable format (if in doubt use format 4 for glyphs within + the BMP, format 12 for glyphs outside the BMP, and format 14 for Unicode Variation + Sequences) construct subtable objects with ``CmapSubtable.newSubtable(format)``, + and append them to the ``.tables`` list. + + Within a subtable, the mapping of characters to glyphs is provided by the ``.cmap`` + attribute. + + Example:: + + cmap4_0_3 = CmapSubtable.newSubtable(4) + cmap4_0_3.platformID = 0 + cmap4_0_3.platEncID = 3 + cmap4_0_3.language = 0 + cmap4_0_3.cmap = { 0xC1: "Aacute" } + + cmap = newTable("cmap") + cmap.tableVersion = 0 + cmap.tables = [cmap4_0_3] + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/cmap + """ + + def getcmap(self, platformID, platEncID): + """Returns the first subtable which matches the given platform and encoding. + + Args: + platformID (int): The platform ID. Use 0 for Unicode, 1 for Macintosh + (deprecated for new fonts), 2 for ISO (deprecated) and 3 for Windows. + encodingID (int): Encoding ID. Interpretation depends on the platform ID. + See the OpenType specification for details. + + Returns: + An object which is a subclass of :py:class:`CmapSubtable` if a matching + subtable is found within the font, or ``None`` otherwise. + """ + + for subtable in self.tables: + if subtable.platformID == platformID and subtable.platEncID == platEncID: + return subtable + return None # not found + + def getBestCmap( + self, + cmapPreferences=( + (3, 10), + (0, 6), + (0, 4), + (3, 1), + (0, 3), + (0, 2), + (0, 1), + (0, 0), + ), + ): + """Returns the 'best' Unicode cmap dictionary available in the font + or ``None``, if no Unicode cmap subtable is available. + + By default it will search for the following (platformID, platEncID) + pairs in order:: + + (3, 10), # Windows Unicode full repertoire + (0, 6), # Unicode full repertoire (format 13 subtable) + (0, 4), # Unicode 2.0 full repertoire + (3, 1), # Windows Unicode BMP + (0, 3), # Unicode 2.0 BMP + (0, 2), # Unicode ISO/IEC 10646 + (0, 1), # Unicode 1.1 + (0, 0) # Unicode 1.0 + + This particular order matches what HarfBuzz uses to choose what + subtable to use by default. This order prefers the largest-repertoire + subtable, and among those, prefers the Windows-platform over the + Unicode-platform as the former has wider support. + + This order can be customized via the ``cmapPreferences`` argument. + """ + for platformID, platEncID in cmapPreferences: + cmapSubtable = self.getcmap(platformID, platEncID) + if cmapSubtable is not None: + return cmapSubtable.cmap + return None # None of the requested cmap subtables were found + + def buildReversed(self): + """Builds a reverse mapping dictionary + + Iterates over all Unicode cmap tables and returns a dictionary mapping + glyphs to sets of codepoints, such as:: + + { + 'one': {0x31} + 'A': {0x41,0x391} + } + + The values are sets of Unicode codepoints because + some fonts map different codepoints to the same glyph. + For example, ``U+0041 LATIN CAPITAL LETTER A`` and ``U+0391 + GREEK CAPITAL LETTER ALPHA`` are sometimes the same glyph. + """ + result = {} + for subtable in self.tables: + if subtable.isUnicode(): + for codepoint, name in subtable.cmap.items(): + result.setdefault(name, set()).add(codepoint) + return result + + def buildReversedMin(self): + result = {} + for subtable in self.tables: + if subtable.isUnicode(): + for codepoint, name in subtable.cmap.items(): + if name in result: + result[name] = min(result[name], codepoint) + else: + result[name] = codepoint + return result + + def decompile(self, data, ttFont): + tableVersion, numSubTables = struct.unpack(">HH", data[:4]) + self.tableVersion = int(tableVersion) + self.tables = tables = [] + seenOffsets = {} + for i in range(numSubTables): + platformID, platEncID, offset = struct.unpack( + ">HHl", data[4 + i * 8 : 4 + (i + 1) * 8] + ) + platformID, platEncID = int(platformID), int(platEncID) + format, length = struct.unpack(">HH", data[offset : offset + 4]) + if format in [8, 10, 12, 13]: + format, reserved, length = struct.unpack( + ">HHL", data[offset : offset + 8] + ) + elif format in [14]: + format, length = struct.unpack(">HL", data[offset : offset + 6]) + + if not length: + log.error( + "cmap subtable is reported as having zero length: platformID %s, " + "platEncID %s, format %s offset %s. Skipping table.", + platformID, + platEncID, + format, + offset, + ) + continue + table = CmapSubtable.newSubtable(format) + table.platformID = platformID + table.platEncID = platEncID + # Note that by default we decompile only the subtable header info; + # any other data gets decompiled only when an attribute of the + # subtable is referenced. + table.decompileHeader(data[offset : offset + int(length)], ttFont) + if offset in seenOffsets: + table.data = None # Mark as decompiled + table.cmap = tables[seenOffsets[offset]].cmap + else: + seenOffsets[offset] = i + tables.append(table) + if ttFont.lazy is False: # Be lazy for None and True + self.ensureDecompiled() + + def ensureDecompiled(self, recurse=False): + # The recurse argument is unused, but part of the signature of + # ensureDecompiled across the library. + for st in self.tables: + st.ensureDecompiled() + + def compile(self, ttFont): + self.tables.sort() # sort according to the spec; see CmapSubtable.__lt__() + keys = [(t.platformID, t.platEncID, t.language) for t in self.tables] + seen = set() + duplicates = {k for k in keys if k in seen or seen.add(k)} + if duplicates: + raise ValueError( + "cmap subtables have duplicate (platformID, platEncID, language) " + f"entries, which the OpenType spec does not allow: {duplicates}" + ) + numSubTables = len(self.tables) + totalOffset = 4 + 8 * numSubTables + data = struct.pack(">HH", self.tableVersion, numSubTables) + tableData = b"" + seen = ( + {} + ) # Some tables are the same object reference. Don't compile them twice. + done = ( + {} + ) # Some tables are different objects, but compile to the same data chunk + for table in self.tables: + offset = seen.get(id(table.cmap)) + if offset is None: + chunk = table.compile(ttFont) + offset = done.get(chunk) + if offset is None: + offset = seen[id(table.cmap)] = done[chunk] = totalOffset + len( + tableData + ) + tableData = tableData + chunk + data = data + struct.pack(">HHl", table.platformID, table.platEncID, offset) + return data + tableData + + def toXML(self, writer, ttFont): + writer.simpletag("tableVersion", version=self.tableVersion) + writer.newline() + for table in self.tables: + table.toXML(writer, ttFont) + + def fromXML(self, name, attrs, content, ttFont): + if name == "tableVersion": + self.tableVersion = safeEval(attrs["version"]) + return + if name[:12] != "cmap_format_": + return + if not hasattr(self, "tables"): + self.tables = [] + format = safeEval(name[12:]) + table = CmapSubtable.newSubtable(format) + table.platformID = safeEval(attrs["platformID"]) + table.platEncID = safeEval(attrs["platEncID"]) + table.fromXML(name, attrs, content, ttFont) + self.tables.append(table) + + +class CmapSubtable(object): + """Base class for all cmap subtable formats. + + Subclasses which handle the individual subtable formats are named + ``cmap_format_0``, ``cmap_format_2`` etc. Use :py:meth:`getSubtableClass` + to retrieve the concrete subclass, or :py:meth:`newSubtable` to get a + new subtable object for a given format. + + The object exposes a ``.cmap`` attribute, which contains a dictionary mapping + character codepoints to glyph names. + """ + + @staticmethod + def getSubtableClass(format): + """Return the subtable class for a format.""" + return cmap_classes.get(format, cmap_format_unknown) + + @staticmethod + def newSubtable(format): + """Return a new instance of a subtable for the given format + .""" + subtableClass = CmapSubtable.getSubtableClass(format) + return subtableClass(format) + + def __init__(self, format): + self.format = format + self.data = None + self.ttFont = None + self.platformID = None #: The platform ID of this subtable + self.platEncID = None #: The encoding ID of this subtable (interpretation depends on ``platformID``) + self.language = ( + None #: The language ID of this subtable (Macintosh platform only) + ) + + def ensureDecompiled(self, recurse=False): + # The recurse argument is unused, but part of the signature of + # ensureDecompiled across the library. + if self.data is None: + return + self.decompile(None, None) # use saved data. + self.data = None # Once this table has been decompiled, make sure we don't + # just return the original data. Also avoids recursion when + # called with an attribute that the cmap subtable doesn't have. + + def __getattr__(self, attr): + # allow lazy decompilation of subtables. + if attr[:2] == "__": # don't handle requests for member functions like '__lt__' + raise AttributeError(attr) + if self.data is None: + raise AttributeError(attr) + self.ensureDecompiled() + return getattr(self, attr) + + def decompileHeader(self, data, ttFont): + format, length, language = struct.unpack(">HHH", data[:6]) + assert ( + len(data) == length + ), "corrupt cmap table format %d (data length: %d, header length: %d)" % ( + format, + len(data), + length, + ) + self.format = int(format) + self.length = int(length) + self.language = int(language) + self.data = data[6:] + self.ttFont = ttFont + + def toXML(self, writer, ttFont): + writer.begintag( + self.__class__.__name__, + [ + ("platformID", self.platformID), + ("platEncID", self.platEncID), + ("language", self.language), + ], + ) + writer.newline() + codes = sorted(self.cmap.items()) + self._writeCodes(codes, writer) + writer.endtag(self.__class__.__name__) + writer.newline() + + def getEncoding(self, default=None): + """Returns the Python encoding name for this cmap subtable based on its platformID, + platEncID, and language. If encoding for these values is not known, by default + ``None`` is returned. That can be overridden by passing a value to the ``default`` + argument. + + Note that if you want to choose a "preferred" cmap subtable, most of the time + ``self.isUnicode()`` is what you want as that one only returns true for the modern, + commonly used, Unicode-compatible triplets, not the legacy ones. + """ + return getEncoding(self.platformID, self.platEncID, self.language, default) + + def isUnicode(self): + """Returns true if the characters are interpreted as Unicode codepoints.""" + return self.platformID == 0 or ( + self.platformID == 3 and self.platEncID in [0, 1, 10] + ) + + def isSymbol(self): + """Returns true if the subtable is for the Symbol encoding (3,0)""" + return self.platformID == 3 and self.platEncID == 0 + + def _writeCodes(self, codes, writer): + isUnicode = self.isUnicode() + for code, name in codes: + writer.simpletag("map", code=hex(code), name=name) + if isUnicode: + writer.comment(Unicode[code]) + writer.newline() + + def __lt__(self, other): + if not isinstance(other, CmapSubtable): + return NotImplemented + + # implemented so that list.sort() sorts according to the spec. + selfTuple = ( + getattr(self, "platformID", None), + getattr(self, "platEncID", None), + getattr(self, "language", None), + ) + otherTuple = ( + getattr(other, "platformID", None), + getattr(other, "platEncID", None), + getattr(other, "language", None), + ) + return selfTuple < otherTuple + + +class cmap_format_0(CmapSubtable): + def decompile(self, data, ttFont): + # we usually get here indirectly from the subtable __getattr__ function, in which case both args must be None. + # If not, someone is calling the subtable decompile() directly, and must provide both args. + if data is not None and ttFont is not None: + self.decompileHeader(data, ttFont) + else: + assert ( + data is None and ttFont is None + ), "Need both data and ttFont arguments" + data = ( + self.data + ) # decompileHeader assigns the data after the header to self.data + assert 262 == self.length, "Format 0 cmap subtable not 262 bytes" + gids = array.array("B") + gids.frombytes(self.data) + charCodes = range(len(gids)) + self.cmap = _make_map(self.ttFont, charCodes, gids) + + def compile(self, ttFont): + if self.data: + return struct.pack(">HHH", 0, 262, self.language) + self.data + + cmap = self.cmap + assert set(cmap.keys()).issubset(range(256)) + getGlyphID = ttFont.getGlyphID + valueList = [getGlyphID(cmap[i]) if i in cmap else 0 for i in range(256)] + + gids = array.array("B", valueList) + data = struct.pack(">HHH", 0, 262, self.language) + gids.tobytes() + assert len(data) == 262 + return data + + def fromXML(self, name, attrs, content, ttFont): + self.language = safeEval(attrs["language"]) + if not hasattr(self, "cmap"): + self.cmap = {} + cmap = self.cmap + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name != "map": + continue + cmap[safeEval(attrs["code"])] = attrs["name"] + + +subHeaderFormat = ">HHhH" + + +class SubHeader(object): + def __init__(self): + self.firstCode = None + self.entryCount = None + self.idDelta = None + self.idRangeOffset = None + self.glyphIndexArray = [] + + +class cmap_format_2(CmapSubtable): + def setIDDelta(self, subHeader): + subHeader.idDelta = 0 + # find the minGI which is not zero. + minGI = subHeader.glyphIndexArray[0] + for gid in subHeader.glyphIndexArray: + if (gid != 0) and (gid < minGI): + minGI = gid + # The lowest gid in glyphIndexArray, after subtracting idDelta, must be 1. + # idDelta is a short, and must be between -32K and 32K. minGI can be between 1 and 64K. + # We would like to pick an idDelta such that the first glyphArray GID is 1, + # so that we are more likely to be able to combine glypharray GID subranges. + # This means that we have a problem when minGI is > 32K + # Since the final gi is reconstructed from the glyphArray GID by: + # (short)finalGID = (gid + idDelta) % 0x10000), + # we can get from a glypharray GID of 1 to a final GID of 65K by subtracting 2, and casting the + # negative number to an unsigned short. + + if minGI > 1: + if minGI > 0x7FFF: + subHeader.idDelta = -(0x10000 - minGI) - 1 + else: + subHeader.idDelta = minGI - 1 + idDelta = subHeader.idDelta + for i in range(subHeader.entryCount): + gid = subHeader.glyphIndexArray[i] + if gid > 0: + subHeader.glyphIndexArray[i] = gid - idDelta + + def decompile(self, data, ttFont): + # we usually get here indirectly from the subtable __getattr__ function, in which case both args must be None. + # If not, someone is calling the subtable decompile() directly, and must provide both args. + if data is not None and ttFont is not None: + self.decompileHeader(data, ttFont) + else: + assert ( + data is None and ttFont is None + ), "Need both data and ttFont arguments" + + data = ( + self.data + ) # decompileHeader assigns the data after the header to self.data + subHeaderKeys = [] + maxSubHeaderindex = 0 + # get the key array, and determine the number of subHeaders. + allKeys = array.array("H") + allKeys.frombytes(data[:512]) + data = data[512:] + if sys.byteorder != "big": + allKeys.byteswap() + subHeaderKeys = [key // 8 for key in allKeys] + maxSubHeaderindex = max(subHeaderKeys) + + # Load subHeaders + subHeaderList = [] + pos = 0 + for i in range(maxSubHeaderindex + 1): + subHeader = SubHeader() + ( + subHeader.firstCode, + subHeader.entryCount, + subHeader.idDelta, + subHeader.idRangeOffset, + ) = struct.unpack(subHeaderFormat, data[pos : pos + 8]) + pos += 8 + giDataPos = pos + subHeader.idRangeOffset - 2 + giList = array.array("H") + giList.frombytes(data[giDataPos : giDataPos + subHeader.entryCount * 2]) + if sys.byteorder != "big": + giList.byteswap() + subHeader.glyphIndexArray = giList + subHeaderList.append(subHeader) + # How this gets processed. + # Charcodes may be one or two bytes. + # The first byte of a charcode is mapped through the subHeaderKeys, to select + # a subHeader. For any subheader but 0, the next byte is then mapped through the + # selected subheader. If subheader Index 0 is selected, then the byte itself is + # mapped through the subheader, and there is no second byte. + # Then assume that the subsequent byte is the first byte of the next charcode,and repeat. + # + # Each subheader references a range in the glyphIndexArray whose length is entryCount. + # The range in glyphIndexArray referenced by a sunheader may overlap with the range in glyphIndexArray + # referenced by another subheader. + # The only subheader that will be referenced by more than one first-byte value is the subheader + # that maps the entire range of glyphID values to glyphIndex 0, e.g notdef: + # {firstChar 0, EntryCount 0,idDelta 0,idRangeOffset xx} + # A byte being mapped though a subheader is treated as in index into a mapping of array index to font glyphIndex. + # A subheader specifies a subrange within (0...256) by the + # firstChar and EntryCount values. If the byte value is outside the subrange, then the glyphIndex is zero + # (e.g. glyph not in font). + # If the byte index is in the subrange, then an offset index is calculated as (byteIndex - firstChar). + # The index to glyphIndex mapping is a subrange of the glyphIndexArray. You find the start of the subrange by + # counting idRangeOffset bytes from the idRangeOffset word. The first value in this subrange is the + # glyphIndex for the index firstChar. The offset index should then be used in this array to get the glyphIndex. + # Example for Logocut-Medium + # first byte of charcode = 129; selects subheader 1. + # subheader 1 = {firstChar 64, EntryCount 108,idDelta 42,idRangeOffset 0252} + # second byte of charCode = 66 + # the index offset = 66-64 = 2. + # The subrange of the glyphIndexArray starting at 0x0252 bytes from the idRangeOffset word is: + # [glyphIndexArray index], [subrange array index] = glyphIndex + # [256], [0]=1 from charcode [129, 64] + # [257], [1]=2 from charcode [129, 65] + # [258], [2]=3 from charcode [129, 66] + # [259], [3]=4 from charcode [129, 67] + # So, the glyphIndex = 3 from the array. Then if idDelta is not zero and the glyph ID is not zero, + # add it to the glyphID to get the final glyphIndex + # value. In this case the final glyph index = 3+ 42 -> 45 for the final glyphIndex. Whew! + + self.data = b"" + cmap = {} + notdefGI = 0 + for firstByte in range(256): + subHeadindex = subHeaderKeys[firstByte] + subHeader = subHeaderList[subHeadindex] + if subHeadindex == 0: + if (firstByte < subHeader.firstCode) or ( + firstByte >= subHeader.firstCode + subHeader.entryCount + ): + continue # gi is notdef. + else: + charCode = firstByte + offsetIndex = firstByte - subHeader.firstCode + gi = subHeader.glyphIndexArray[offsetIndex] + if gi != 0: + gi = (gi + subHeader.idDelta) % 0x10000 + else: + continue # gi is notdef. + cmap[charCode] = gi + else: + if subHeader.entryCount: + charCodeOffset = firstByte * 256 + subHeader.firstCode + for offsetIndex in range(subHeader.entryCount): + charCode = charCodeOffset + offsetIndex + gi = subHeader.glyphIndexArray[offsetIndex] + if gi != 0: + gi = (gi + subHeader.idDelta) % 0x10000 + else: + continue + cmap[charCode] = gi + # If not subHeader.entryCount, then all char codes with this first byte are + # mapped to .notdef. We can skip this subtable, and leave the glyphs un-encoded, which is the + # same as mapping it to .notdef. + + gids = list(cmap.values()) + charCodes = list(cmap.keys()) + self.cmap = _make_map(self.ttFont, charCodes, gids) + + def compile(self, ttFont): + if self.data: + return ( + struct.pack(">HHH", self.format, self.length, self.language) + self.data + ) + kEmptyTwoCharCodeRange = -1 + notdefGI = 0 + + items = sorted(self.cmap.items()) + charCodes = [item[0] for item in items] + names = [item[1] for item in items] + nameMap = ttFont.getReverseGlyphMap() + try: + gids = [nameMap[name] for name in names] + except KeyError: + nameMap = ttFont.getReverseGlyphMap(rebuild=True) + try: + gids = [nameMap[name] for name in names] + except KeyError: + # allow virtual GIDs in format 2 tables + gids = [] + for name in names: + try: + gid = nameMap[name] + except KeyError: + try: + if name[:3] == "gid": + gid = int(name[3:]) + else: + gid = ttFont.getGlyphID(name) + except: + raise KeyError(name) + + gids.append(gid) + + # Process the (char code to gid) item list in char code order. + # By definition, all one byte char codes map to subheader 0. + # For all the two byte char codes, we assume that the first byte maps maps to the empty subhead (with an entry count of 0, + # which defines all char codes in its range to map to notdef) unless proven otherwise. + # Note that since the char code items are processed in char code order, all the char codes with the + # same first byte are in sequential order. + + subHeaderKeys = [ + kEmptyTwoCharCodeRange for x in range(256) + ] # list of indices into subHeaderList. + subHeaderList = [] + + # We force this subheader entry 0 to exist in the subHeaderList in the case where some one comes up + # with a cmap where all the one byte char codes map to notdef, + # with the result that the subhead 0 would not get created just by processing the item list. + charCode = charCodes[0] + if charCode > 255: + subHeader = SubHeader() + subHeader.firstCode = 0 + subHeader.entryCount = 0 + subHeader.idDelta = 0 + subHeader.idRangeOffset = 0 + subHeaderList.append(subHeader) + + lastFirstByte = -1 + items = zip(charCodes, gids) + for charCode, gid in items: + if gid == 0: + continue + firstbyte = charCode >> 8 + secondByte = charCode & 0x00FF + + if ( + firstbyte != lastFirstByte + ): # Need to update the current subhead, and start a new one. + if lastFirstByte > -1: + # fix GI's and iDelta of current subheader. + self.setIDDelta(subHeader) + + # If it was sunheader 0 for one-byte charCodes, then we need to set the subHeaderKeys value to zero + # for the indices matching the char codes. + if lastFirstByte == 0: + for index in range(subHeader.entryCount): + charCode = subHeader.firstCode + index + subHeaderKeys[charCode] = 0 + + assert subHeader.entryCount == len( + subHeader.glyphIndexArray + ), "Error - subhead entry count does not match len of glyphID subrange." + # init new subheader + subHeader = SubHeader() + subHeader.firstCode = secondByte + subHeader.entryCount = 1 + subHeader.glyphIndexArray.append(gid) + subHeaderList.append(subHeader) + subHeaderKeys[firstbyte] = len(subHeaderList) - 1 + lastFirstByte = firstbyte + else: + # need to fill in with notdefs all the code points between the last charCode and the current charCode. + codeDiff = secondByte - (subHeader.firstCode + subHeader.entryCount) + for i in range(codeDiff): + subHeader.glyphIndexArray.append(notdefGI) + subHeader.glyphIndexArray.append(gid) + subHeader.entryCount = subHeader.entryCount + codeDiff + 1 + + # fix GI's and iDelta of last subheader that we we added to the subheader array. + self.setIDDelta(subHeader) + + # Now we add a final subheader for the subHeaderKeys which maps to empty two byte charcode ranges. + subHeader = SubHeader() + subHeader.firstCode = 0 + subHeader.entryCount = 0 + subHeader.idDelta = 0 + subHeader.idRangeOffset = 2 + subHeaderList.append(subHeader) + emptySubheadIndex = len(subHeaderList) - 1 + for index in range(256): + if subHeaderKeys[index] == kEmptyTwoCharCodeRange: + subHeaderKeys[index] = emptySubheadIndex + # Since this is the last subheader, the GlyphIndex Array starts two bytes after the start of the + # idRangeOffset word of this subHeader. We can safely point to the first entry in the GlyphIndexArray, + # since the first subrange of the GlyphIndexArray is for subHeader 0, which always starts with + # charcode 0 and GID 0. + + idRangeOffset = ( + len(subHeaderList) - 1 + ) * 8 + 2 # offset to beginning of glyphIDArray from first subheader idRangeOffset. + subheadRangeLen = ( + len(subHeaderList) - 1 + ) # skip last special empty-set subheader; we've already hardocodes its idRangeOffset to 2. + for index in range(subheadRangeLen): + subHeader = subHeaderList[index] + subHeader.idRangeOffset = 0 + for j in range(index): + prevSubhead = subHeaderList[j] + if ( + prevSubhead.glyphIndexArray == subHeader.glyphIndexArray + ): # use the glyphIndexArray subarray + subHeader.idRangeOffset = ( + prevSubhead.idRangeOffset - (index - j) * 8 + ) + subHeader.glyphIndexArray = [] + break + if subHeader.idRangeOffset == 0: # didn't find one. + subHeader.idRangeOffset = idRangeOffset + idRangeOffset = ( + idRangeOffset - 8 + ) + subHeader.entryCount * 2 # one less subheader, one more subArray. + else: + idRangeOffset = idRangeOffset - 8 # one less subheader + + # Now we can write out the data! + length = ( + 6 + 512 + 8 * len(subHeaderList) + ) # header, 256 subHeaderKeys, and subheader array. + for subhead in subHeaderList[:-1]: + length = ( + length + len(subhead.glyphIndexArray) * 2 + ) # We can't use subhead.entryCount, as some of the subhead may share subArrays. + dataList = [struct.pack(">HHH", 2, length, self.language)] + for index in subHeaderKeys: + dataList.append(struct.pack(">H", index * 8)) + for subhead in subHeaderList: + dataList.append( + struct.pack( + subHeaderFormat, + subhead.firstCode, + subhead.entryCount, + subhead.idDelta, + subhead.idRangeOffset, + ) + ) + for subhead in subHeaderList[:-1]: + for gi in subhead.glyphIndexArray: + dataList.append(struct.pack(">H", gi)) + data = bytesjoin(dataList) + assert len(data) == length, ( + "Error: cmap format 2 is not same length as calculated! actual: " + + str(len(data)) + + " calc : " + + str(length) + ) + return data + + def fromXML(self, name, attrs, content, ttFont): + self.language = safeEval(attrs["language"]) + if not hasattr(self, "cmap"): + self.cmap = {} + cmap = self.cmap + + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name != "map": + continue + cmap[safeEval(attrs["code"])] = attrs["name"] + + +cmap_format_4_format = ">7H" + +# uint16 endCode[segCount] # Ending character code for each segment, last = 0xFFFF. +# uint16 reservedPad # This value should be zero +# uint16 startCode[segCount] # Starting character code for each segment +# uint16 idDelta[segCount] # Delta for all character codes in segment +# uint16 idRangeOffset[segCount] # Offset in bytes to glyph indexArray, or 0 +# uint16 glyphIndexArray[variable] # Glyph index array + + +def splitRange(startCode, endCode, cmap): + # Try to split a range of character codes into subranges with consecutive + # glyph IDs in such a way that the cmap4 subtable can be stored "most" + # efficiently. I can't prove I've got the optimal solution, but it seems + # to do well with the fonts I tested: none became bigger, many became smaller. + if startCode == endCode: + return [], [endCode] + + lastID = cmap[startCode] + lastCode = startCode + inOrder = None + orderedBegin = None + subRanges = [] + + # Gather subranges in which the glyph IDs are consecutive. + for code in range(startCode + 1, endCode + 1): + glyphID = cmap[code] + + if glyphID - 1 == lastID: + if inOrder is None or not inOrder: + inOrder = 1 + orderedBegin = lastCode + else: + if inOrder: + inOrder = 0 + subRanges.append((orderedBegin, lastCode)) + orderedBegin = None + + lastID = glyphID + lastCode = code + + if inOrder: + subRanges.append((orderedBegin, lastCode)) + assert lastCode == endCode + + # Now filter out those new subranges that would only make the data bigger. + # A new segment cost 8 bytes, not using a new segment costs 2 bytes per + # character. + newRanges = [] + for b, e in subRanges: + if b == startCode and e == endCode: + break # the whole range, we're fine + if b == startCode or e == endCode: + threshold = 4 # split costs one more segment + else: + threshold = 8 # split costs two more segments + if (e - b + 1) > threshold: + newRanges.append((b, e)) + subRanges = newRanges + + if not subRanges: + return [], [endCode] + + if subRanges[0][0] != startCode: + subRanges.insert(0, (startCode, subRanges[0][0] - 1)) + if subRanges[-1][1] != endCode: + subRanges.append((subRanges[-1][1] + 1, endCode)) + + # Fill the "holes" in the segments list -- those are the segments in which + # the glyph IDs are _not_ consecutive. + i = 1 + while i < len(subRanges): + if subRanges[i - 1][1] + 1 != subRanges[i][0]: + subRanges.insert(i, (subRanges[i - 1][1] + 1, subRanges[i][0] - 1)) + i = i + 1 + i = i + 1 + + # Transform the ranges into startCode/endCode lists. + start = [] + end = [] + for b, e in subRanges: + start.append(b) + end.append(e) + start.pop(0) + + assert len(start) + 1 == len(end) + return start, end + + +class cmap_format_4(CmapSubtable): + def decompile(self, data, ttFont): + # we usually get here indirectly from the subtable __getattr__ function, in which case both args must be None. + # If not, someone is calling the subtable decompile() directly, and must provide both args. + if data is not None and ttFont is not None: + self.decompileHeader(data, ttFont) + else: + assert ( + data is None and ttFont is None + ), "Need both data and ttFont arguments" + + data = ( + self.data + ) # decompileHeader assigns the data after the header to self.data + (segCountX2, searchRange, entrySelector, rangeShift) = struct.unpack( + ">4H", data[:8] + ) + data = data[8:] + segCount = segCountX2 // 2 + + allCodes = array.array("H") + allCodes.frombytes(data) + self.data = data = None + + if sys.byteorder != "big": + allCodes.byteswap() + + # divide the data + endCode = allCodes[:segCount] + allCodes = allCodes[segCount + 1 :] # the +1 is skipping the reservedPad field + startCode = allCodes[:segCount] + allCodes = allCodes[segCount:] + idDelta = allCodes[:segCount] + allCodes = allCodes[segCount:] + idRangeOffset = allCodes[:segCount] + glyphIndexArray = allCodes[segCount:] + lenGIArray = len(glyphIndexArray) + + # build 2-byte character mapping + charCodes = [] + gids = [] + for i in range(len(startCode) - 1): # don't do 0xffff! + start = startCode[i] + delta = idDelta[i] + rangeOffset = idRangeOffset[i] + partial = rangeOffset // 2 - start + i - len(idRangeOffset) + + rangeCharCodes = list(range(startCode[i], endCode[i] + 1)) + charCodes.extend(rangeCharCodes) + if rangeOffset == 0: + gids.extend( + [(charCode + delta) & 0xFFFF for charCode in rangeCharCodes] + ) + else: + for charCode in rangeCharCodes: + index = charCode + partial + assert index < lenGIArray, ( + "In format 4 cmap, range (%d), the calculated index (%d) into the glyph index array is not less than the length of the array (%d) !" + % (i, index, lenGIArray) + ) + if glyphIndexArray[index] != 0: # if not missing glyph + glyphID = glyphIndexArray[index] + delta + else: + glyphID = 0 # missing glyph + gids.append(glyphID & 0xFFFF) + + self.cmap = _make_map(self.ttFont, charCodes, gids) + + def compile(self, ttFont): + if self.data: + return ( + struct.pack(">HHH", self.format, self.length, self.language) + self.data + ) + + charCodes = list(self.cmap.keys()) + if not charCodes: + startCode = [0xFFFF] + endCode = [0xFFFF] + else: + charCodes.sort() + names = [self.cmap[code] for code in charCodes] + nameMap = ttFont.getReverseGlyphMap() + try: + gids = [nameMap[name] for name in names] + except KeyError: + nameMap = ttFont.getReverseGlyphMap(rebuild=True) + try: + gids = [nameMap[name] for name in names] + except KeyError: + # allow virtual GIDs in format 4 tables + gids = [] + for name in names: + try: + gid = nameMap[name] + except KeyError: + try: + if name[:3] == "gid": + gid = int(name[3:]) + else: + gid = ttFont.getGlyphID(name) + except: + raise KeyError(name) + + gids.append(gid) + cmap = {} # code:glyphID mapping + for code, gid in zip(charCodes, gids): + cmap[code] = gid + + # Build startCode and endCode lists. + # Split the char codes in ranges of consecutive char codes, then split + # each range in more ranges of consecutive/not consecutive glyph IDs. + # See splitRange(). + lastCode = charCodes[0] + endCode = [] + startCode = [lastCode] + for charCode in charCodes[ + 1: + ]: # skip the first code, it's the first start code + if charCode == lastCode + 1: + lastCode = charCode + continue + start, end = splitRange(startCode[-1], lastCode, cmap) + startCode.extend(start) + endCode.extend(end) + startCode.append(charCode) + lastCode = charCode + start, end = splitRange(startCode[-1], lastCode, cmap) + startCode.extend(start) + endCode.extend(end) + startCode.append(0xFFFF) + endCode.append(0xFFFF) + + # build up rest of cruft + idDelta = [] + idRangeOffset = [] + glyphIndexArray = [] + for i in range(len(endCode) - 1): # skip the closing codes (0xffff) + indices = [] + for charCode in range(startCode[i], endCode[i] + 1): + indices.append(cmap[charCode]) + if indices == list(range(indices[0], indices[0] + len(indices))): + idDelta.append((indices[0] - startCode[i]) % 0x10000) + idRangeOffset.append(0) + else: + idDelta.append(0) + idRangeOffset.append(2 * (len(endCode) + len(glyphIndexArray) - i)) + glyphIndexArray.extend(indices) + idDelta.append(1) # 0xffff + 1 == (tadaa!) 0. So this end code maps to .notdef + idRangeOffset.append(0) + + # Insane. + segCount = len(endCode) + segCountX2 = segCount * 2 + searchRange, entrySelector, rangeShift = getSearchRange(segCount, 2) + + charCodeArray = array.array("H", endCode + [0] + startCode) + idDeltaArray = array.array("H", idDelta) + restArray = array.array("H", idRangeOffset + glyphIndexArray) + if sys.byteorder != "big": + charCodeArray.byteswap() + if sys.byteorder != "big": + idDeltaArray.byteswap() + if sys.byteorder != "big": + restArray.byteswap() + data = charCodeArray.tobytes() + idDeltaArray.tobytes() + restArray.tobytes() + + length = struct.calcsize(cmap_format_4_format) + len(data) + header = struct.pack( + cmap_format_4_format, + self.format, + length, + self.language, + segCountX2, + searchRange, + entrySelector, + rangeShift, + ) + return header + data + + def fromXML(self, name, attrs, content, ttFont): + self.language = safeEval(attrs["language"]) + if not hasattr(self, "cmap"): + self.cmap = {} + cmap = self.cmap + + for element in content: + if not isinstance(element, tuple): + continue + nameMap, attrsMap, dummyContent = element + if nameMap != "map": + assert 0, "Unrecognized keyword in cmap subtable" + cmap[safeEval(attrsMap["code"])] = attrsMap["name"] + + +class cmap_format_6(CmapSubtable): + def decompile(self, data, ttFont): + # we usually get here indirectly from the subtable __getattr__ function, in which case both args must be None. + # If not, someone is calling the subtable decompile() directly, and must provide both args. + if data is not None and ttFont is not None: + self.decompileHeader(data, ttFont) + else: + assert ( + data is None and ttFont is None + ), "Need both data and ttFont arguments" + + data = ( + self.data + ) # decompileHeader assigns the data after the header to self.data + firstCode, entryCount = struct.unpack(">HH", data[:4]) + firstCode = int(firstCode) + data = data[4:] + # assert len(data) == 2 * entryCount # XXX not true in Apple's Helvetica!!! + gids = array.array("H") + gids.frombytes(data[: 2 * int(entryCount)]) + if sys.byteorder != "big": + gids.byteswap() + self.data = data = None + + charCodes = list(range(firstCode, firstCode + len(gids))) + self.cmap = _make_map(self.ttFont, charCodes, gids) + + def compile(self, ttFont): + if self.data: + return ( + struct.pack(">HHH", self.format, self.length, self.language) + self.data + ) + cmap = self.cmap + codes = sorted(cmap.keys()) + if codes: # yes, there are empty cmap tables. + codes = list(range(codes[0], codes[-1] + 1)) + firstCode = codes[0] + valueList = [ + ttFont.getGlyphID(cmap[code]) if code in cmap else 0 for code in codes + ] + gids = array.array("H", valueList) + if sys.byteorder != "big": + gids.byteswap() + data = gids.tobytes() + else: + data = b"" + firstCode = 0 + header = struct.pack( + ">HHHHH", 6, len(data) + 10, self.language, firstCode, len(codes) + ) + return header + data + + def fromXML(self, name, attrs, content, ttFont): + self.language = safeEval(attrs["language"]) + if not hasattr(self, "cmap"): + self.cmap = {} + cmap = self.cmap + + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name != "map": + continue + cmap[safeEval(attrs["code"])] = attrs["name"] + + +class cmap_format_12_or_13(CmapSubtable): + def __init__(self, format): + self.format = format + self.reserved = 0 + self.data = None + self.ttFont = None + + def decompileHeader(self, data, ttFont): + format, reserved, length, language, nGroups = struct.unpack(">HHLLL", data[:16]) + assert ( + len(data) == (16 + nGroups * 12) == (length) + ), "corrupt cmap table format %d (data length: %d, header length: %d)" % ( + self.format, + len(data), + length, + ) + self.format = format + self.reserved = reserved + self.length = length + self.language = language + self.nGroups = nGroups + self.data = data[16:] + self.ttFont = ttFont + + def decompile(self, data, ttFont): + # we usually get here indirectly from the subtable __getattr__ function, in which case both args must be None. + # If not, someone is calling the subtable decompile() directly, and must provide both args. + if data is not None and ttFont is not None: + self.decompileHeader(data, ttFont) + else: + assert ( + data is None and ttFont is None + ), "Need both data and ttFont arguments" + + data = ( + self.data + ) # decompileHeader assigns the data after the header to self.data + charCodes = [] + gids = [] + pos = 0 + groups = array.array("I", data[: self.nGroups * 12]) + if sys.byteorder != "big": + groups.byteswap() + for i in range(self.nGroups): + startCharCode = groups[i * 3] + endCharCode = groups[i * 3 + 1] + glyphID = groups[i * 3 + 2] + lenGroup = 1 + endCharCode - startCharCode + charCodes.extend(range(startCharCode, endCharCode + 1)) + gids.extend(self._computeGIDs(glyphID, lenGroup)) + self.data = data = None + self.cmap = _make_map(self.ttFont, charCodes, gids) + + def compile(self, ttFont): + if self.data: + return ( + struct.pack( + ">HHLLL", + self.format, + self.reserved, + self.length, + self.language, + self.nGroups, + ) + + self.data + ) + charCodes = list(self.cmap.keys()) + names = list(self.cmap.values()) + nameMap = ttFont.getReverseGlyphMap() + try: + gids = [nameMap[name] for name in names] + except KeyError: + nameMap = ttFont.getReverseGlyphMap(rebuild=True) + try: + gids = [nameMap[name] for name in names] + except KeyError: + # allow virtual GIDs in format 12 tables + gids = [] + for name in names: + try: + gid = nameMap[name] + except KeyError: + try: + if name[:3] == "gid": + gid = int(name[3:]) + else: + gid = ttFont.getGlyphID(name) + except: + raise KeyError(name) + + gids.append(gid) + + cmap = {} # code:glyphID mapping + for code, gid in zip(charCodes, gids): + cmap[code] = gid + + charCodes.sort() + index = 0 + startCharCode = charCodes[0] + startGlyphID = cmap[startCharCode] + lastGlyphID = startGlyphID - self._format_step + lastCharCode = startCharCode - 1 + nGroups = 0 + dataList = [] + maxIndex = len(charCodes) + for index in range(maxIndex): + charCode = charCodes[index] + glyphID = cmap[charCode] + if not self._IsInSameRun(glyphID, lastGlyphID, charCode, lastCharCode): + dataList.append( + struct.pack(">LLL", startCharCode, lastCharCode, startGlyphID) + ) + startCharCode = charCode + startGlyphID = glyphID + nGroups = nGroups + 1 + lastGlyphID = glyphID + lastCharCode = charCode + dataList.append(struct.pack(">LLL", startCharCode, lastCharCode, startGlyphID)) + nGroups = nGroups + 1 + data = bytesjoin(dataList) + lengthSubtable = len(data) + 16 + assert len(data) == (nGroups * 12) == (lengthSubtable - 16) + return ( + struct.pack( + ">HHLLL", + self.format, + self.reserved, + lengthSubtable, + self.language, + nGroups, + ) + + data + ) + + def toXML(self, writer, ttFont): + writer.begintag( + self.__class__.__name__, + [ + ("platformID", self.platformID), + ("platEncID", self.platEncID), + ("format", self.format), + ("reserved", self.reserved), + ("length", self.length), + ("language", self.language), + ("nGroups", self.nGroups), + ], + ) + writer.newline() + codes = sorted(self.cmap.items()) + self._writeCodes(codes, writer) + writer.endtag(self.__class__.__name__) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.format = safeEval(attrs["format"]) + self.reserved = safeEval(attrs["reserved"]) + self.length = safeEval(attrs["length"]) + self.language = safeEval(attrs["language"]) + self.nGroups = safeEval(attrs["nGroups"]) + if not hasattr(self, "cmap"): + self.cmap = {} + cmap = self.cmap + + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name != "map": + continue + cmap[safeEval(attrs["code"])] = attrs["name"] + + +class cmap_format_12(cmap_format_12_or_13): + _format_step = 1 + + def __init__(self, format=12): + cmap_format_12_or_13.__init__(self, format) + + def _computeGIDs(self, startingGlyph, numberOfGlyphs): + return range(startingGlyph, startingGlyph + numberOfGlyphs) + + def _IsInSameRun(self, glyphID, lastGlyphID, charCode, lastCharCode): + return (glyphID == 1 + lastGlyphID) and (charCode == 1 + lastCharCode) + + +class cmap_format_13(cmap_format_12_or_13): + _format_step = 0 + + def __init__(self, format=13): + cmap_format_12_or_13.__init__(self, format) + + def _computeGIDs(self, startingGlyph, numberOfGlyphs): + return [startingGlyph] * numberOfGlyphs + + def _IsInSameRun(self, glyphID, lastGlyphID, charCode, lastCharCode): + return (glyphID == lastGlyphID) and (charCode == 1 + lastCharCode) + + +def cvtToUVS(threeByteString): + data = b"\0" + threeByteString + (val,) = struct.unpack(">L", data) + return val + + +def cvtFromUVS(val): + assert 0 <= val < 0x1000000 + fourByteString = struct.pack(">L", val) + return fourByteString[1:] + + +class cmap_format_14(CmapSubtable): + def decompileHeader(self, data, ttFont): + format, length, numVarSelectorRecords = struct.unpack(">HLL", data[:10]) + self.data = data[10:] + self.length = length + self.numVarSelectorRecords = numVarSelectorRecords + self.ttFont = ttFont + self.language = 0xFF # has no language. + + def decompile(self, data, ttFont): + if data is not None and ttFont is not None: + self.decompileHeader(data, ttFont) + else: + assert ( + data is None and ttFont is None + ), "Need both data and ttFont arguments" + data = self.data + + self.cmap = ( + {} + ) # so that clients that expect this to exist in a cmap table won't fail. + uvsDict = {} + recOffset = 0 + for n in range(self.numVarSelectorRecords): + uvs, defOVSOffset, nonDefUVSOffset = struct.unpack( + ">3sLL", data[recOffset : recOffset + 11] + ) + recOffset += 11 + varUVS = cvtToUVS(uvs) + if defOVSOffset: + startOffset = defOVSOffset - 10 + (numValues,) = struct.unpack(">L", data[startOffset : startOffset + 4]) + startOffset += 4 + for r in range(numValues): + uv, addtlCnt = struct.unpack( + ">3sB", data[startOffset : startOffset + 4] + ) + startOffset += 4 + firstBaseUV = cvtToUVS(uv) + cnt = addtlCnt + 1 + baseUVList = list(range(firstBaseUV, firstBaseUV + cnt)) + glyphList = [None] * cnt + localUVList = zip(baseUVList, glyphList) + try: + uvsDict[varUVS].extend(localUVList) + except KeyError: + uvsDict[varUVS] = list(localUVList) + + if nonDefUVSOffset: + startOffset = nonDefUVSOffset - 10 + (numRecs,) = struct.unpack(">L", data[startOffset : startOffset + 4]) + startOffset += 4 + localUVList = [] + for r in range(numRecs): + uv, gid = struct.unpack(">3sH", data[startOffset : startOffset + 5]) + startOffset += 5 + uv = cvtToUVS(uv) + glyphName = self.ttFont.getGlyphName(gid) + localUVList.append((uv, glyphName)) + try: + uvsDict[varUVS].extend(localUVList) + except KeyError: + uvsDict[varUVS] = localUVList + + self.uvsDict = uvsDict + + def toXML(self, writer, ttFont): + writer.begintag( + self.__class__.__name__, + [ + ("platformID", self.platformID), + ("platEncID", self.platEncID), + ], + ) + writer.newline() + uvsDict = self.uvsDict + uvsList = sorted(uvsDict.keys()) + for uvs in uvsList: + uvList = uvsDict[uvs] + uvList.sort(key=lambda item: (item[1] is not None, item[0], item[1])) + for uv, gname in uvList: + attrs = [("uv", hex(uv)), ("uvs", hex(uvs))] + if gname is not None: + attrs.append(("name", gname)) + writer.simpletag("map", attrs) + writer.newline() + writer.endtag(self.__class__.__name__) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.language = 0xFF # provide a value so that CmapSubtable.__lt__() won't fail + if not hasattr(self, "cmap"): + self.cmap = ( + {} + ) # so that clients that expect this to exist in a cmap table won't fail. + if not hasattr(self, "uvsDict"): + self.uvsDict = {} + uvsDict = self.uvsDict + + # For backwards compatibility reasons we accept "None" as an indicator + # for "default mapping", unless the font actually has a glyph named + # "None". + _hasGlyphNamedNone = None + + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name != "map": + continue + uvs = safeEval(attrs["uvs"]) + uv = safeEval(attrs["uv"]) + gname = attrs.get("name") + if gname == "None": + if _hasGlyphNamedNone is None: + _hasGlyphNamedNone = "None" in ttFont.getGlyphOrder() + if not _hasGlyphNamedNone: + gname = None + try: + uvsDict[uvs].append((uv, gname)) + except KeyError: + uvsDict[uvs] = [(uv, gname)] + + def compile(self, ttFont): + if self.data: + return ( + struct.pack( + ">HLL", self.format, self.length, self.numVarSelectorRecords + ) + + self.data + ) + + uvsDict = self.uvsDict + uvsList = sorted(uvsDict.keys()) + self.numVarSelectorRecords = len(uvsList) + offset = ( + 10 + self.numVarSelectorRecords * 11 + ) # current value is end of VarSelectorRecords block. + data = [] + varSelectorRecords = [] + for uvs in uvsList: + entryList = uvsDict[uvs] + + defList = [entry for entry in entryList if entry[1] is None] + if defList: + defList = [entry[0] for entry in defList] + defOVSOffset = offset + defList.sort() + + lastUV = defList[0] + cnt = -1 + defRecs = [] + for defEntry in defList: + cnt += 1 + if (lastUV + cnt) != defEntry: + rec = struct.pack(">3sB", cvtFromUVS(lastUV), cnt - 1) + lastUV = defEntry + defRecs.append(rec) + cnt = 0 + + rec = struct.pack(">3sB", cvtFromUVS(lastUV), cnt) + defRecs.append(rec) + + numDefRecs = len(defRecs) + data.append(struct.pack(">L", numDefRecs)) + data.extend(defRecs) + offset += 4 + numDefRecs * 4 + else: + defOVSOffset = 0 + + ndefList = [entry for entry in entryList if entry[1] is not None] + if ndefList: + nonDefUVSOffset = offset + ndefList.sort() + numNonDefRecs = len(ndefList) + data.append(struct.pack(">L", numNonDefRecs)) + offset += 4 + numNonDefRecs * 5 + + for uv, gname in ndefList: + gid = ttFont.getGlyphID(gname) + ndrec = struct.pack(">3sH", cvtFromUVS(uv), gid) + data.append(ndrec) + else: + nonDefUVSOffset = 0 + + vrec = struct.pack(">3sLL", cvtFromUVS(uvs), defOVSOffset, nonDefUVSOffset) + varSelectorRecords.append(vrec) + + data = bytesjoin(varSelectorRecords) + bytesjoin(data) + self.length = 10 + len(data) + headerdata = struct.pack( + ">HLL", self.format, self.length, self.numVarSelectorRecords + ) + + return headerdata + data + + +class cmap_format_unknown(CmapSubtable): + def toXML(self, writer, ttFont): + cmapName = self.__class__.__name__[:12] + str(self.format) + writer.begintag( + cmapName, + [ + ("platformID", self.platformID), + ("platEncID", self.platEncID), + ], + ) + writer.newline() + writer.dumphex(self.data) + writer.endtag(cmapName) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.data = readHex(content) + self.cmap = {} + + def decompileHeader(self, data, ttFont): + self.language = 0 # dummy value + self.data = data + + def decompile(self, data, ttFont): + # we usually get here indirectly from the subtable __getattr__ function, in which case both args must be None. + # If not, someone is calling the subtable decompile() directly, and must provide both args. + if data is not None and ttFont is not None: + self.decompileHeader(data, ttFont) + else: + assert ( + data is None and ttFont is None + ), "Need both data and ttFont arguments" + + def compile(self, ttFont): + if self.data: + return self.data + else: + return None + + +cmap_classes = { + 0: cmap_format_0, + 2: cmap_format_2, + 4: cmap_format_4, + 6: cmap_format_6, + 12: cmap_format_12, + 13: cmap_format_13, + 14: cmap_format_14, +} diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_v_a_r.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_v_a_r.py new file mode 100644 index 0000000000000000000000000000000000000000..872710c8f0c93ade86e8be8640205a91b7b397af --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_v_a_r.py @@ -0,0 +1,94 @@ +from . import DefaultTable +from fontTools.misc import sstruct +from fontTools.misc.textTools import bytesjoin +from fontTools.ttLib.tables.TupleVariation import ( + compileTupleVariationStore, + decompileTupleVariationStore, + TupleVariation, +) + + +# https://www.microsoft.com/typography/otspec/cvar.htm +# https://www.microsoft.com/typography/otspec/otvarcommonformats.htm +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6cvar.html + +CVAR_HEADER_FORMAT = """ + > # big endian + majorVersion: H + minorVersion: H + tupleVariationCount: H + offsetToData: H +""" + +CVAR_HEADER_SIZE = sstruct.calcsize(CVAR_HEADER_FORMAT) + + +class table__c_v_a_r(DefaultTable.DefaultTable): + """Control Value Table (CVT) variations table + + The ``cvar`` table contains variations for the values in a ``cvt`` + table. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/cvar + """ + + dependencies = ["cvt ", "fvar"] + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.majorVersion, self.minorVersion = 1, 0 + self.variations = [] + + def compile(self, ttFont, useSharedPoints=False): + tupleVariationCount, tuples, data = compileTupleVariationStore( + variations=[v for v in self.variations if v.hasImpact()], + pointCount=len(ttFont["cvt "].values), + axisTags=[axis.axisTag for axis in ttFont["fvar"].axes], + sharedTupleIndices={}, + useSharedPoints=useSharedPoints, + ) + header = { + "majorVersion": self.majorVersion, + "minorVersion": self.minorVersion, + "tupleVariationCount": tupleVariationCount, + "offsetToData": CVAR_HEADER_SIZE + len(tuples), + } + return b"".join([sstruct.pack(CVAR_HEADER_FORMAT, header), tuples, data]) + + def decompile(self, data, ttFont): + axisTags = [axis.axisTag for axis in ttFont["fvar"].axes] + header = {} + sstruct.unpack(CVAR_HEADER_FORMAT, data[0:CVAR_HEADER_SIZE], header) + self.majorVersion = header["majorVersion"] + self.minorVersion = header["minorVersion"] + assert self.majorVersion == 1, self.majorVersion + self.variations = decompileTupleVariationStore( + tableTag=self.tableTag, + axisTags=axisTags, + tupleVariationCount=header["tupleVariationCount"], + pointCount=len(ttFont["cvt "].values), + sharedTuples=None, + data=data, + pos=CVAR_HEADER_SIZE, + dataPos=header["offsetToData"], + ) + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": + self.majorVersion = int(attrs.get("major", "1")) + self.minorVersion = int(attrs.get("minor", "0")) + elif name == "tuple": + valueCount = len(ttFont["cvt "].values) + var = TupleVariation({}, [None] * valueCount) + self.variations.append(var) + for tupleElement in content: + if isinstance(tupleElement, tuple): + tupleName, tupleAttrs, tupleContent = tupleElement + var.fromXML(tupleName, tupleAttrs, tupleContent) + + def toXML(self, writer, ttFont): + axisTags = [axis.axisTag for axis in ttFont["fvar"].axes] + writer.simpletag("version", major=self.majorVersion, minor=self.minorVersion) + writer.newline() + for var in self.variations: + var.toXML(writer, axisTags) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_v_t.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_v_t.py new file mode 100644 index 0000000000000000000000000000000000000000..c89fe2c239a0ab72fe763a1c0f700540d6b4126e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_c_v_t.py @@ -0,0 +1,56 @@ +from fontTools.misc.textTools import safeEval +from . import DefaultTable +import sys +import array + + +class table__c_v_t(DefaultTable.DefaultTable): + """Control Value Table + + The Control Value Table holds a list of values that can be referenced + by TrueType font instructions. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/cvt + """ + + def decompile(self, data, ttFont): + values = array.array("h") + values.frombytes(data) + if sys.byteorder != "big": + values.byteswap() + self.values = values + + def compile(self, ttFont): + if not hasattr(self, "values"): + return b"" + values = self.values[:] + if sys.byteorder != "big": + values.byteswap() + return values.tobytes() + + def toXML(self, writer, ttFont): + for i, value in enumerate(self.values): + writer.simpletag("cv", value=value, index=i) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if not hasattr(self, "values"): + self.values = array.array("h") + if name == "cv": + index = safeEval(attrs["index"]) + value = safeEval(attrs["value"]) + for i in range(1 + index - len(self.values)): + self.values.append(0) + self.values[index] = value + + def __len__(self): + return len(self.values) + + def __getitem__(self, index): + return self.values[index] + + def __setitem__(self, index, value): + self.values[index] = value + + def __delitem__(self, index): + del self.values[index] diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_f_e_a_t.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_f_e_a_t.py new file mode 100644 index 0000000000000000000000000000000000000000..8e279db0059613d5c0e0ac061dc9b2b62aefe73b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_f_e_a_t.py @@ -0,0 +1,15 @@ +from .otBase import BaseTTXConverter + + +class table__f_e_a_t(BaseTTXConverter): + """Feature name table + + The feature name table is an AAT (Apple Advanced Typography) table for + storing font features, settings, and their human-readable names. It should + not be confused with the ``Feat`` table or the OpenType Layout ``GSUB``/``GPOS`` + tables. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6feat.html + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_f_p_g_m.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_f_p_g_m.py new file mode 100644 index 0000000000000000000000000000000000000000..c21a9d4b6817d33bc68aabb8d866708657d1e068 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_f_p_g_m.py @@ -0,0 +1,62 @@ +from . import DefaultTable +from . import ttProgram + + +class table__f_p_g_m(DefaultTable.DefaultTable): + """Font Program table + + The ``fpgm`` table typically contains function defintions that are + used by font instructions. This Font Program is similar to the Control + Value Program that is stored in the ``prep`` table, but + the ``fpgm`` table is only executed one time, when the font is first + used. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/fpgm + """ + + def decompile(self, data, ttFont): + program = ttProgram.Program() + program.fromBytecode(data) + self.program = program + + def compile(self, ttFont): + if hasattr(self, "program"): + return self.program.getBytecode() + return b"" + + def toXML(self, writer, ttFont): + self.program.toXML(writer, ttFont) + + def fromXML(self, name, attrs, content, ttFont): + program = ttProgram.Program() + program.fromXML(name, attrs, content, ttFont) + self.program = program + + def __bool__(self): + """ + >>> fpgm = table__f_p_g_m() + >>> bool(fpgm) + False + >>> p = ttProgram.Program() + >>> fpgm.program = p + >>> bool(fpgm) + False + >>> bc = bytearray([0]) + >>> p.fromBytecode(bc) + >>> bool(fpgm) + True + >>> p.bytecode.pop() + 0 + >>> bool(fpgm) + False + """ + return hasattr(self, "program") and bool(self.program) + + __nonzero__ = __bool__ + + +if __name__ == "__main__": + import sys + import doctest + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_f_v_a_r.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_f_v_a_r.py new file mode 100644 index 0000000000000000000000000000000000000000..a7e61d01347da4ee5634343200a693e6b4d0d32d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_f_v_a_r.py @@ -0,0 +1,259 @@ +from fontTools.misc import sstruct +from fontTools.misc.fixedTools import ( + fixedToFloat as fi2fl, + floatToFixed as fl2fi, + floatToFixedToStr as fl2str, + strToFixedToFloat as str2fl, +) +from fontTools.misc.textTools import Tag, bytesjoin, safeEval +from fontTools.ttLib import TTLibError +from . import DefaultTable +import struct + + +# Apple's documentation of 'fvar': +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6fvar.html + +FVAR_HEADER_FORMAT = """ + > # big endian + version: L + offsetToData: H + countSizePairs: H + axisCount: H + axisSize: H + instanceCount: H + instanceSize: H +""" + +FVAR_AXIS_FORMAT = """ + > # big endian + axisTag: 4s + minValue: 16.16F + defaultValue: 16.16F + maxValue: 16.16F + flags: H + axisNameID: H +""" + +FVAR_INSTANCE_FORMAT = """ + > # big endian + subfamilyNameID: H + flags: H +""" + + +class table__f_v_a_r(DefaultTable.DefaultTable): + """FonT Variations table + + The ``fvar`` table contains records of the variation axes and of the + named instances in a variable font. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6fvar.html + """ + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.axes = [] + self.instances = [] + + def compile(self, ttFont): + instanceSize = sstruct.calcsize(FVAR_INSTANCE_FORMAT) + (len(self.axes) * 4) + includePostScriptNames = any( + instance.postscriptNameID != 0xFFFF for instance in self.instances + ) + if includePostScriptNames: + instanceSize += 2 + header = { + "version": 0x00010000, + "offsetToData": sstruct.calcsize(FVAR_HEADER_FORMAT), + "countSizePairs": 2, + "axisCount": len(self.axes), + "axisSize": sstruct.calcsize(FVAR_AXIS_FORMAT), + "instanceCount": len(self.instances), + "instanceSize": instanceSize, + } + result = [sstruct.pack(FVAR_HEADER_FORMAT, header)] + result.extend([axis.compile() for axis in self.axes]) + axisTags = [axis.axisTag for axis in self.axes] + for instance in self.instances: + result.append(instance.compile(axisTags, includePostScriptNames)) + return bytesjoin(result) + + def decompile(self, data, ttFont): + header = {} + headerSize = sstruct.calcsize(FVAR_HEADER_FORMAT) + header = sstruct.unpack(FVAR_HEADER_FORMAT, data[0:headerSize]) + if header["version"] != 0x00010000: + raise TTLibError("unsupported 'fvar' version %04x" % header["version"]) + pos = header["offsetToData"] + axisSize = header["axisSize"] + for _ in range(header["axisCount"]): + axis = Axis() + axis.decompile(data[pos : pos + axisSize]) + self.axes.append(axis) + pos += axisSize + instanceSize = header["instanceSize"] + axisTags = [axis.axisTag for axis in self.axes] + for _ in range(header["instanceCount"]): + instance = NamedInstance() + instance.decompile(data[pos : pos + instanceSize], axisTags) + self.instances.append(instance) + pos += instanceSize + + def toXML(self, writer, ttFont): + for axis in self.axes: + axis.toXML(writer, ttFont) + for instance in self.instances: + instance.toXML(writer, ttFont) + + def fromXML(self, name, attrs, content, ttFont): + if name == "Axis": + axis = Axis() + axis.fromXML(name, attrs, content, ttFont) + self.axes.append(axis) + elif name == "NamedInstance": + instance = NamedInstance() + instance.fromXML(name, attrs, content, ttFont) + self.instances.append(instance) + + def getAxes(self): + return {a.axisTag: (a.minValue, a.defaultValue, a.maxValue) for a in self.axes} + + +class Axis(object): + def __init__(self): + self.axisTag = None + self.axisNameID = 0 + self.flags = 0 + self.minValue = -1.0 + self.defaultValue = 0.0 + self.maxValue = 1.0 + + def compile(self): + return sstruct.pack(FVAR_AXIS_FORMAT, self) + + def decompile(self, data): + sstruct.unpack2(FVAR_AXIS_FORMAT, data, self) + + def toXML(self, writer, ttFont): + name = ( + ttFont["name"].getDebugName(self.axisNameID) if "name" in ttFont else None + ) + if name is not None: + writer.newline() + writer.comment(name) + writer.newline() + writer.begintag("Axis") + writer.newline() + for tag, value in [ + ("AxisTag", self.axisTag), + ("Flags", "0x%X" % self.flags), + ("MinValue", fl2str(self.minValue, 16)), + ("DefaultValue", fl2str(self.defaultValue, 16)), + ("MaxValue", fl2str(self.maxValue, 16)), + ("AxisNameID", str(self.axisNameID)), + ]: + writer.begintag(tag) + writer.write(value) + writer.endtag(tag) + writer.newline() + writer.endtag("Axis") + writer.newline() + + def fromXML(self, name, _attrs, content, ttFont): + assert name == "Axis" + for tag, _, value in filter(lambda t: type(t) is tuple, content): + value = "".join(value) + if tag == "AxisTag": + self.axisTag = Tag(value) + elif tag in {"Flags", "MinValue", "DefaultValue", "MaxValue", "AxisNameID"}: + setattr( + self, + tag[0].lower() + tag[1:], + str2fl(value, 16) if tag.endswith("Value") else safeEval(value), + ) + + +class NamedInstance(object): + def __init__(self): + self.subfamilyNameID = 0 + self.postscriptNameID = 0xFFFF + self.flags = 0 + self.coordinates = {} + + def compile(self, axisTags, includePostScriptName): + result = [sstruct.pack(FVAR_INSTANCE_FORMAT, self)] + for axis in axisTags: + fixedCoord = fl2fi(self.coordinates[axis], 16) + result.append(struct.pack(">l", fixedCoord)) + if includePostScriptName: + result.append(struct.pack(">H", self.postscriptNameID)) + return bytesjoin(result) + + def decompile(self, data, axisTags): + sstruct.unpack2(FVAR_INSTANCE_FORMAT, data, self) + pos = sstruct.calcsize(FVAR_INSTANCE_FORMAT) + for axis in axisTags: + value = struct.unpack(">l", data[pos : pos + 4])[0] + self.coordinates[axis] = fi2fl(value, 16) + pos += 4 + if pos + 2 <= len(data): + self.postscriptNameID = struct.unpack(">H", data[pos : pos + 2])[0] + else: + self.postscriptNameID = 0xFFFF + + def toXML(self, writer, ttFont): + name = ( + ttFont["name"].getDebugName(self.subfamilyNameID) + if "name" in ttFont + else None + ) + if name is not None: + writer.newline() + writer.comment(name) + writer.newline() + psname = ( + ttFont["name"].getDebugName(self.postscriptNameID) + if "name" in ttFont + else None + ) + if psname is not None: + writer.comment("PostScript: " + psname) + writer.newline() + if self.postscriptNameID == 0xFFFF: + writer.begintag( + "NamedInstance", + flags=("0x%X" % self.flags), + subfamilyNameID=self.subfamilyNameID, + ) + else: + writer.begintag( + "NamedInstance", + flags=("0x%X" % self.flags), + subfamilyNameID=self.subfamilyNameID, + postscriptNameID=self.postscriptNameID, + ) + writer.newline() + for axis in ttFont["fvar"].axes: + writer.simpletag( + "coord", + axis=axis.axisTag, + value=fl2str(self.coordinates[axis.axisTag], 16), + ) + writer.newline() + writer.endtag("NamedInstance") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + assert name == "NamedInstance" + self.subfamilyNameID = safeEval(attrs["subfamilyNameID"]) + self.flags = safeEval(attrs.get("flags", "0")) + if "postscriptNameID" in attrs: + self.postscriptNameID = safeEval(attrs["postscriptNameID"]) + else: + self.postscriptNameID = 0xFFFF + + for tag, elementAttrs, _ in filter(lambda t: type(t) is tuple, content): + if tag == "coord": + value = str2fl(elementAttrs["value"], 16) + self.coordinates[elementAttrs["axis"]] = value diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_a_s_p.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_a_s_p.py new file mode 100644 index 0000000000000000000000000000000000000000..1f605771e9b7d98da4384e0d6a5f692646f6c07d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_a_s_p.py @@ -0,0 +1,63 @@ +from fontTools.misc.textTools import safeEval +from . import DefaultTable +import struct + + +GASP_SYMMETRIC_GRIDFIT = 0x0004 +GASP_SYMMETRIC_SMOOTHING = 0x0008 +GASP_DOGRAY = 0x0002 +GASP_GRIDFIT = 0x0001 + + +class table__g_a_s_p(DefaultTable.DefaultTable): + """Grid-fitting and Scan-conversion Procedure table + + The ``gasp`` table defines the preferred rasterization settings for + the font when rendered on monochrome and greyscale output devices. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/gasp + """ + + def decompile(self, data, ttFont): + self.version, numRanges = struct.unpack(">HH", data[:4]) + assert 0 <= self.version <= 1, "unknown 'gasp' format: %s" % self.version + data = data[4:] + self.gaspRange = {} + for i in range(numRanges): + rangeMaxPPEM, rangeGaspBehavior = struct.unpack(">HH", data[:4]) + self.gaspRange[int(rangeMaxPPEM)] = int(rangeGaspBehavior) + data = data[4:] + assert not data, "too much data" + + def compile(self, ttFont): + version = 0 # ignore self.version + numRanges = len(self.gaspRange) + data = b"" + items = sorted(self.gaspRange.items()) + for rangeMaxPPEM, rangeGaspBehavior in items: + data = data + struct.pack(">HH", rangeMaxPPEM, rangeGaspBehavior) + if rangeGaspBehavior & ~(GASP_GRIDFIT | GASP_DOGRAY): + version = 1 + data = struct.pack(">HH", version, numRanges) + data + return data + + def toXML(self, writer, ttFont): + items = sorted(self.gaspRange.items()) + for rangeMaxPPEM, rangeGaspBehavior in items: + writer.simpletag( + "gaspRange", + [ + ("rangeMaxPPEM", rangeMaxPPEM), + ("rangeGaspBehavior", rangeGaspBehavior), + ], + ) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name != "gaspRange": + return + if not hasattr(self, "gaspRange"): + self.gaspRange = {} + self.gaspRange[safeEval(attrs["rangeMaxPPEM"])] = safeEval( + attrs["rangeGaspBehavior"] + ) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_c_i_d.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_c_i_d.py new file mode 100644 index 0000000000000000000000000000000000000000..9d6d6ae425384b47613c146d567847c00660a5cc --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_c_i_d.py @@ -0,0 +1,13 @@ +from .otBase import BaseTTXConverter + + +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6gcid.html +class table__g_c_i_d(BaseTTXConverter): + """Glyph ID to CID table + + The AAT ``gcid`` table stores glyphID-to-CID mappings. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6gcid.html + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_l_y_f.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_l_y_f.py new file mode 100644 index 0000000000000000000000000000000000000000..ffc7eaadbc7e8724dad1842037b9fe9bd6bdd22c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_l_y_f.py @@ -0,0 +1,2302 @@ +"""_g_l_y_f.py -- Converter classes for the 'glyf' table.""" + +from collections import namedtuple +from fontTools.misc import sstruct +from fontTools import ttLib +from fontTools import version +from fontTools.misc.transform import DecomposedTransform +from fontTools.misc.textTools import tostr, safeEval, pad +from fontTools.misc.arrayTools import updateBounds, pointInRect +from fontTools.misc.bezierTools import calcQuadraticBounds +from fontTools.misc.fixedTools import ( + fixedToFloat as fi2fl, + floatToFixed as fl2fi, + floatToFixedToStr as fl2str, + strToFixedToFloat as str2fl, +) +from fontTools.misc.roundTools import noRound, otRound +from fontTools.misc.vector import Vector +from numbers import Number +from . import DefaultTable +from . import ttProgram +import sys +import struct +import array +import logging +import math +import os +from fontTools.misc import xmlWriter +from fontTools.misc.filenames import userNameToFileName +from fontTools.misc.loggingTools import deprecateFunction +from enum import IntFlag +from functools import partial +from types import SimpleNamespace +from typing import Set + +log = logging.getLogger(__name__) + +# We compute the version the same as is computed in ttlib/__init__ +# so that we can write 'ttLibVersion' attribute of the glyf TTX files +# when glyf is written to separate files. +version = ".".join(version.split(".")[:2]) + +# +# The Apple and MS rasterizers behave differently for +# scaled composite components: one does scale first and then translate +# and the other does it vice versa. MS defined some flags to indicate +# the difference, but it seems nobody actually _sets_ those flags. +# +# Funny thing: Apple seems to _only_ do their thing in the +# WE_HAVE_A_SCALE (eg. Chicago) case, and not when it's WE_HAVE_AN_X_AND_Y_SCALE +# (eg. Charcoal)... +# +SCALE_COMPONENT_OFFSET_DEFAULT = 0 # 0 == MS, 1 == Apple + + +class table__g_l_y_f(DefaultTable.DefaultTable): + """Glyph Data table + + This class represents the `glyf <https://docs.microsoft.com/en-us/typography/opentype/spec/glyf>`_ + table, which contains outlines for glyphs in TrueType format. In many cases, + it is easier to access and manipulate glyph outlines through the ``GlyphSet`` + object returned from :py:meth:`fontTools.ttLib.ttFont.getGlyphSet`:: + + >> from fontTools.pens.boundsPen import BoundsPen + >> glyphset = font.getGlyphSet() + >> bp = BoundsPen(glyphset) + >> glyphset["A"].draw(bp) + >> bp.bounds + (19, 0, 633, 716) + + However, this class can be used for low-level access to the ``glyf`` table data. + Objects of this class support dictionary-like access, mapping glyph names to + :py:class:`Glyph` objects:: + + >> glyf = font["glyf"] + >> len(glyf["Aacute"].components) + 2 + + Note that when adding glyphs to the font via low-level access to the ``glyf`` + table, the new glyphs must also be added to the ``hmtx``/``vmtx`` table:: + + >> font["glyf"]["divisionslash"] = Glyph() + >> font["hmtx"]["divisionslash"] = (640, 0) + + """ + + # this attribute controls the amount of padding applied to glyph data upon compile. + # Glyph lenghts are aligned to multiples of the specified value. + # Allowed values are (0, 1, 2, 4). '0' means no padding; '1' (default) also means + # no padding, except for when padding would allow to use short loca offsets. + padding = 1 + + def decompile(self, data, ttFont): + loca = ttFont["loca"] + pos = int(loca[0]) + nextPos = 0 + noname = 0 + self.glyphs = {} + self.glyphOrder = glyphOrder = ttFont.getGlyphOrder() + self._reverseGlyphOrder = {} + for i in range(0, len(loca) - 1): + try: + glyphName = glyphOrder[i] + except IndexError: + noname = noname + 1 + glyphName = "ttxautoglyph%s" % i + nextPos = int(loca[i + 1]) + glyphdata = data[pos:nextPos] + if len(glyphdata) != (nextPos - pos): + raise ttLib.TTLibError("not enough 'glyf' table data") + glyph = Glyph(glyphdata) + self.glyphs[glyphName] = glyph + pos = nextPos + if len(data) - nextPos >= 4: + log.warning( + "too much 'glyf' table data: expected %d, received %d bytes", + nextPos, + len(data), + ) + if noname: + log.warning("%s glyphs have no name", noname) + if ttFont.lazy is False: # Be lazy for None and True + self.ensureDecompiled() + + def ensureDecompiled(self, recurse=False): + # The recurse argument is unused, but part of the signature of + # ensureDecompiled across the library. + for glyph in self.glyphs.values(): + glyph.expand(self) + + def compile(self, ttFont): + optimizeSpeed = ttFont.cfg[ttLib.OPTIMIZE_FONT_SPEED] + if not hasattr(self, "glyphOrder"): + self.glyphOrder = ttFont.getGlyphOrder() + padding = self.padding + assert padding in (0, 1, 2, 4) + locations = [] + currentLocation = 0 + dataList = [] + recalcBBoxes = ttFont.recalcBBoxes + boundsDone = set() + for glyphName in self.glyphOrder: + glyph = self.glyphs[glyphName] + glyphData = glyph.compile( + self, + recalcBBoxes, + boundsDone=boundsDone, + optimizeSize=not optimizeSpeed, + ) + if padding > 1: + glyphData = pad(glyphData, size=padding) + locations.append(currentLocation) + currentLocation = currentLocation + len(glyphData) + dataList.append(glyphData) + locations.append(currentLocation) + + if padding == 1 and currentLocation < 0x20000: + # See if we can pad any odd-lengthed glyphs to allow loca + # table to use the short offsets. + indices = [ + i for i, glyphData in enumerate(dataList) if len(glyphData) % 2 == 1 + ] + if indices and currentLocation + len(indices) < 0x20000: + # It fits. Do it. + for i in indices: + dataList[i] += b"\0" + currentLocation = 0 + for i, glyphData in enumerate(dataList): + locations[i] = currentLocation + currentLocation += len(glyphData) + locations[len(dataList)] = currentLocation + + data = b"".join(dataList) + if "loca" in ttFont: + ttFont["loca"].set(locations) + if "maxp" in ttFont: + ttFont["maxp"].numGlyphs = len(self.glyphs) + if not data: + # As a special case when all glyph in the font are empty, add a zero byte + # to the table, so that OTS doesn’t reject it, and to make the table work + # on Windows as well. + # See https://github.com/khaledhosny/ots/issues/52 + data = b"\0" + return data + + def toXML(self, writer, ttFont, splitGlyphs=False): + notice = ( + "The xMin, yMin, xMax and yMax values\n" + "will be recalculated by the compiler." + ) + glyphNames = ttFont.getGlyphNames() + if not splitGlyphs: + writer.newline() + writer.comment(notice) + writer.newline() + writer.newline() + numGlyphs = len(glyphNames) + if splitGlyphs: + path, ext = os.path.splitext(writer.file.name) + existingGlyphFiles = set() + for glyphName in glyphNames: + glyph = self.get(glyphName) + if glyph is None: + log.warning("glyph '%s' does not exist in glyf table", glyphName) + continue + if glyph.numberOfContours: + if splitGlyphs: + glyphPath = userNameToFileName( + tostr(glyphName, "utf-8"), + existingGlyphFiles, + prefix=path + ".", + suffix=ext, + ) + existingGlyphFiles.add(glyphPath.lower()) + glyphWriter = xmlWriter.XMLWriter( + glyphPath, + idlefunc=writer.idlefunc, + newlinestr=writer.newlinestr, + ) + glyphWriter.begintag("ttFont", ttLibVersion=version) + glyphWriter.newline() + glyphWriter.begintag("glyf") + glyphWriter.newline() + glyphWriter.comment(notice) + glyphWriter.newline() + writer.simpletag("TTGlyph", src=os.path.basename(glyphPath)) + else: + glyphWriter = writer + glyphWriter.begintag( + "TTGlyph", + [ + ("name", glyphName), + ("xMin", glyph.xMin), + ("yMin", glyph.yMin), + ("xMax", glyph.xMax), + ("yMax", glyph.yMax), + ], + ) + glyphWriter.newline() + glyph.toXML(glyphWriter, ttFont) + glyphWriter.endtag("TTGlyph") + glyphWriter.newline() + if splitGlyphs: + glyphWriter.endtag("glyf") + glyphWriter.newline() + glyphWriter.endtag("ttFont") + glyphWriter.newline() + glyphWriter.close() + else: + writer.simpletag("TTGlyph", name=glyphName) + writer.comment("contains no outline data") + if not splitGlyphs: + writer.newline() + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name != "TTGlyph": + return + if not hasattr(self, "glyphs"): + self.glyphs = {} + if not hasattr(self, "glyphOrder"): + self.glyphOrder = ttFont.getGlyphOrder() + glyphName = attrs["name"] + log.debug("unpacking glyph '%s'", glyphName) + glyph = Glyph() + for attr in ["xMin", "yMin", "xMax", "yMax"]: + setattr(glyph, attr, safeEval(attrs.get(attr, "0"))) + self.glyphs[glyphName] = glyph + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + glyph.fromXML(name, attrs, content, ttFont) + if not ttFont.recalcBBoxes: + glyph.compact(self, 0) + + def setGlyphOrder(self, glyphOrder): + """Sets the glyph order + + Args: + glyphOrder ([str]): List of glyph names in order. + """ + self.glyphOrder = glyphOrder + self._reverseGlyphOrder = {} + + def getGlyphName(self, glyphID): + """Returns the name for the glyph with the given ID. + + Raises a ``KeyError`` if the glyph name is not found in the font. + """ + return self.glyphOrder[glyphID] + + def _buildReverseGlyphOrderDict(self): + self._reverseGlyphOrder = d = {} + for glyphID, glyphName in enumerate(self.glyphOrder): + d[glyphName] = glyphID + + def getGlyphID(self, glyphName): + """Returns the ID of the glyph with the given name. + + Raises a ``ValueError`` if the glyph is not found in the font. + """ + glyphOrder = self.glyphOrder + id = getattr(self, "_reverseGlyphOrder", {}).get(glyphName) + if id is None or id >= len(glyphOrder) or glyphOrder[id] != glyphName: + self._buildReverseGlyphOrderDict() + id = self._reverseGlyphOrder.get(glyphName) + if id is None: + raise ValueError(glyphName) + return id + + def removeHinting(self): + """Removes TrueType hints from all glyphs in the glyphset. + + See :py:meth:`Glyph.removeHinting`. + """ + for glyph in self.glyphs.values(): + glyph.removeHinting() + + def keys(self): + return self.glyphs.keys() + + def has_key(self, glyphName): + return glyphName in self.glyphs + + __contains__ = has_key + + def get(self, glyphName, default=None): + glyph = self.glyphs.get(glyphName, default) + if glyph is not None: + glyph.expand(self) + return glyph + + def __getitem__(self, glyphName): + glyph = self.glyphs[glyphName] + glyph.expand(self) + return glyph + + def __setitem__(self, glyphName, glyph): + self.glyphs[glyphName] = glyph + if glyphName not in self.glyphOrder: + self.glyphOrder.append(glyphName) + + def __delitem__(self, glyphName): + del self.glyphs[glyphName] + self.glyphOrder.remove(glyphName) + + def __len__(self): + assert len(self.glyphOrder) == len(self.glyphs) + return len(self.glyphs) + + def _getPhantomPoints(self, glyphName, hMetrics, vMetrics=None): + """Compute the four "phantom points" for the given glyph from its bounding box + and the horizontal and vertical advance widths and sidebearings stored in the + ttFont's "hmtx" and "vmtx" tables. + + 'hMetrics' should be ttFont['hmtx'].metrics. + + 'vMetrics' should be ttFont['vmtx'].metrics if there is "vmtx" or None otherwise. + If there is no vMetrics passed in, vertical phantom points are set to the zero coordinate. + + https://docs.microsoft.com/en-us/typography/opentype/spec/tt_instructing_glyphs#phantoms + """ + glyph = self[glyphName] + if not hasattr(glyph, "xMin"): + glyph.recalcBounds(self) + + horizontalAdvanceWidth, leftSideBearing = hMetrics[glyphName] + leftSideX = glyph.xMin - leftSideBearing + rightSideX = leftSideX + horizontalAdvanceWidth + + if vMetrics: + verticalAdvanceWidth, topSideBearing = vMetrics[glyphName] + topSideY = topSideBearing + glyph.yMax + bottomSideY = topSideY - verticalAdvanceWidth + else: + bottomSideY = topSideY = 0 + + return [ + (leftSideX, 0), + (rightSideX, 0), + (0, topSideY), + (0, bottomSideY), + ] + + def _getCoordinatesAndControls( + self, glyphName, hMetrics, vMetrics=None, *, round=otRound + ): + """Return glyph coordinates and controls as expected by "gvar" table. + + The coordinates includes four "phantom points" for the glyph metrics, + as mandated by the "gvar" spec. + + The glyph controls is a namedtuple with the following attributes: + - numberOfContours: -1 for composite glyphs. + - endPts: list of indices of end points for each contour in simple + glyphs, or component indices in composite glyphs (used for IUP + optimization). + - flags: array of contour point flags for simple glyphs (None for + composite glyphs). + - components: list of base glyph names (str) for each component in + composite glyphs (None for simple glyphs). + + The "hMetrics" and vMetrics are used to compute the "phantom points" (see + the "_getPhantomPoints" method). + + Return None if the requested glyphName is not present. + """ + glyph = self.get(glyphName) + if glyph is None: + return None + if glyph.isComposite(): + coords = GlyphCoordinates( + [(getattr(c, "x", 0), getattr(c, "y", 0)) for c in glyph.components] + ) + controls = _GlyphControls( + numberOfContours=glyph.numberOfContours, + endPts=list(range(len(glyph.components))), + flags=None, + components=[ + (c.glyphName, getattr(c, "transform", None)) + for c in glyph.components + ], + ) + else: + coords, endPts, flags = glyph.getCoordinates(self) + coords = coords.copy() + controls = _GlyphControls( + numberOfContours=glyph.numberOfContours, + endPts=endPts, + flags=flags, + components=None, + ) + # Add phantom points for (left, right, top, bottom) positions. + phantomPoints = self._getPhantomPoints(glyphName, hMetrics, vMetrics) + coords.extend(phantomPoints) + coords.toInt(round=round) + return coords, controls + + def _setCoordinates(self, glyphName, coord, hMetrics, vMetrics=None): + """Set coordinates and metrics for the given glyph. + + "coord" is an array of GlyphCoordinates which must include the "phantom + points" as the last four coordinates. + + Both the horizontal/vertical advances and left/top sidebearings in "hmtx" + and "vmtx" tables (if any) are updated from four phantom points and + the glyph's bounding boxes. + + The "hMetrics" and vMetrics are used to propagate "phantom points" + into "hmtx" and "vmtx" tables if desired. (see the "_getPhantomPoints" + method). + """ + glyph = self[glyphName] + + # Handle phantom points for (left, right, top, bottom) positions. + assert len(coord) >= 4 + leftSideX = coord[-4][0] + rightSideX = coord[-3][0] + topSideY = coord[-2][1] + bottomSideY = coord[-1][1] + + coord = coord[:-4] + + if glyph.isComposite(): + assert len(coord) == len(glyph.components) + for p, comp in zip(coord, glyph.components): + if hasattr(comp, "x"): + comp.x, comp.y = p + elif glyph.numberOfContours == 0: + assert len(coord) == 0 + else: + assert len(coord) == len(glyph.coordinates) + glyph.coordinates = GlyphCoordinates(coord) + + glyph.recalcBounds(self, boundsDone=set()) + + horizontalAdvanceWidth = otRound(rightSideX - leftSideX) + if horizontalAdvanceWidth < 0: + # unlikely, but it can happen, see: + # https://github.com/fonttools/fonttools/pull/1198 + horizontalAdvanceWidth = 0 + leftSideBearing = otRound(glyph.xMin - leftSideX) + hMetrics[glyphName] = horizontalAdvanceWidth, leftSideBearing + + if vMetrics is not None: + verticalAdvanceWidth = otRound(topSideY - bottomSideY) + if verticalAdvanceWidth < 0: # unlikely but do the same as horizontal + verticalAdvanceWidth = 0 + topSideBearing = otRound(topSideY - glyph.yMax) + vMetrics[glyphName] = verticalAdvanceWidth, topSideBearing + + # Deprecated + + def _synthesizeVMetrics(self, glyphName, ttFont, defaultVerticalOrigin): + """This method is wrong and deprecated. + For rationale see: + https://github.com/fonttools/fonttools/pull/2266/files#r613569473 + """ + vMetrics = getattr(ttFont.get("vmtx"), "metrics", None) + if vMetrics is None: + verticalAdvanceWidth = ttFont["head"].unitsPerEm + topSideY = getattr(ttFont.get("hhea"), "ascent", None) + if topSideY is None: + if defaultVerticalOrigin is not None: + topSideY = defaultVerticalOrigin + else: + topSideY = verticalAdvanceWidth + glyph = self[glyphName] + glyph.recalcBounds(self) + topSideBearing = otRound(topSideY - glyph.yMax) + vMetrics = {glyphName: (verticalAdvanceWidth, topSideBearing)} + return vMetrics + + @deprecateFunction("use '_getPhantomPoints' instead", category=DeprecationWarning) + def getPhantomPoints(self, glyphName, ttFont, defaultVerticalOrigin=None): + """Old public name for self._getPhantomPoints(). + See: https://github.com/fonttools/fonttools/pull/2266""" + hMetrics = ttFont["hmtx"].metrics + vMetrics = self._synthesizeVMetrics(glyphName, ttFont, defaultVerticalOrigin) + return self._getPhantomPoints(glyphName, hMetrics, vMetrics) + + @deprecateFunction( + "use '_getCoordinatesAndControls' instead", category=DeprecationWarning + ) + def getCoordinatesAndControls(self, glyphName, ttFont, defaultVerticalOrigin=None): + """Old public name for self._getCoordinatesAndControls(). + See: https://github.com/fonttools/fonttools/pull/2266""" + hMetrics = ttFont["hmtx"].metrics + vMetrics = self._synthesizeVMetrics(glyphName, ttFont, defaultVerticalOrigin) + return self._getCoordinatesAndControls(glyphName, hMetrics, vMetrics) + + @deprecateFunction("use '_setCoordinates' instead", category=DeprecationWarning) + def setCoordinates(self, glyphName, ttFont): + """Old public name for self._setCoordinates(). + See: https://github.com/fonttools/fonttools/pull/2266""" + hMetrics = ttFont["hmtx"].metrics + vMetrics = getattr(ttFont.get("vmtx"), "metrics", None) + self._setCoordinates(glyphName, hMetrics, vMetrics) + + +_GlyphControls = namedtuple( + "_GlyphControls", "numberOfContours endPts flags components" +) + + +glyphHeaderFormat = """ + > # big endian + numberOfContours: h + xMin: h + yMin: h + xMax: h + yMax: h +""" + +# flags +flagOnCurve = 0x01 +flagXShort = 0x02 +flagYShort = 0x04 +flagRepeat = 0x08 +flagXsame = 0x10 +flagYsame = 0x20 +flagOverlapSimple = 0x40 +flagCubic = 0x80 + +# These flags are kept for XML output after decompiling the coordinates +keepFlags = flagOnCurve + flagOverlapSimple + flagCubic + +_flagSignBytes = { + 0: 2, + flagXsame: 0, + flagXShort | flagXsame: +1, + flagXShort: -1, + flagYsame: 0, + flagYShort | flagYsame: +1, + flagYShort: -1, +} + + +def flagBest(x, y, onCurve): + """For a given x,y delta pair, returns the flag that packs this pair + most efficiently, as well as the number of byte cost of such flag.""" + + flag = flagOnCurve if onCurve else 0 + cost = 0 + # do x + if x == 0: + flag = flag | flagXsame + elif -255 <= x <= 255: + flag = flag | flagXShort + if x > 0: + flag = flag | flagXsame + cost += 1 + else: + cost += 2 + # do y + if y == 0: + flag = flag | flagYsame + elif -255 <= y <= 255: + flag = flag | flagYShort + if y > 0: + flag = flag | flagYsame + cost += 1 + else: + cost += 2 + return flag, cost + + +def flagFits(newFlag, oldFlag, mask): + newBytes = _flagSignBytes[newFlag & mask] + oldBytes = _flagSignBytes[oldFlag & mask] + return newBytes == oldBytes or abs(newBytes) > abs(oldBytes) + + +def flagSupports(newFlag, oldFlag): + return ( + (oldFlag & flagOnCurve) == (newFlag & flagOnCurve) + and flagFits(newFlag, oldFlag, flagXsame | flagXShort) + and flagFits(newFlag, oldFlag, flagYsame | flagYShort) + ) + + +def flagEncodeCoord(flag, mask, coord, coordBytes): + byteCount = _flagSignBytes[flag & mask] + if byteCount == 1: + coordBytes.append(coord) + elif byteCount == -1: + coordBytes.append(-coord) + elif byteCount == 2: + coordBytes.extend(struct.pack(">h", coord)) + + +def flagEncodeCoords(flag, x, y, xBytes, yBytes): + flagEncodeCoord(flag, flagXsame | flagXShort, x, xBytes) + flagEncodeCoord(flag, flagYsame | flagYShort, y, yBytes) + + +ARG_1_AND_2_ARE_WORDS = 0x0001 # if set args are words otherwise they are bytes +ARGS_ARE_XY_VALUES = 0x0002 # if set args are xy values, otherwise they are points +ROUND_XY_TO_GRID = 0x0004 # for the xy values if above is true +WE_HAVE_A_SCALE = 0x0008 # Sx = Sy, otherwise scale == 1.0 +NON_OVERLAPPING = 0x0010 # set to same value for all components (obsolete!) +MORE_COMPONENTS = 0x0020 # indicates at least one more glyph after this one +WE_HAVE_AN_X_AND_Y_SCALE = 0x0040 # Sx, Sy +WE_HAVE_A_TWO_BY_TWO = 0x0080 # t00, t01, t10, t11 +WE_HAVE_INSTRUCTIONS = 0x0100 # instructions follow +USE_MY_METRICS = 0x0200 # apply these metrics to parent glyph +OVERLAP_COMPOUND = 0x0400 # used by Apple in GX fonts +SCALED_COMPONENT_OFFSET = 0x0800 # composite designed to have the component offset scaled (designed for Apple) +UNSCALED_COMPONENT_OFFSET = 0x1000 # composite designed not to have the component offset scaled (designed for MS) + + +CompositeMaxpValues = namedtuple( + "CompositeMaxpValues", ["nPoints", "nContours", "maxComponentDepth"] +) + + +class Glyph(object): + """This class represents an individual TrueType glyph. + + TrueType glyph objects come in two flavours: simple and composite. Simple + glyph objects contain contours, represented via the ``.coordinates``, + ``.flags``, ``.numberOfContours``, and ``.endPtsOfContours`` attributes; + composite glyphs contain components, available through the ``.components`` + attributes. + + Because the ``.coordinates`` attribute (and other simple glyph attributes mentioned + above) is only set on simple glyphs and the ``.components`` attribute is only + set on composite glyphs, it is necessary to use the :py:meth:`isComposite` + method to test whether a glyph is simple or composite before attempting to + access its data. + + For a composite glyph, the components can also be accessed via array-like access:: + + >> assert(font["glyf"]["Aacute"].isComposite()) + >> font["glyf"]["Aacute"][0] + <fontTools.ttLib.tables._g_l_y_f.GlyphComponent at 0x1027b2ee0> + + """ + + def __init__(self, data=b""): + if not data: + # empty char + self.numberOfContours = 0 + return + self.data = data + + def compact(self, glyfTable, recalcBBoxes=True): + data = self.compile(glyfTable, recalcBBoxes) + self.__dict__.clear() + self.data = data + + def expand(self, glyfTable): + if not hasattr(self, "data"): + # already unpacked + return + if not self.data: + # empty char + del self.data + self.numberOfContours = 0 + return + dummy, data = sstruct.unpack2(glyphHeaderFormat, self.data, self) + del self.data + # Some fonts (eg. Neirizi.ttf) have a 0 for numberOfContours in + # some glyphs; decompileCoordinates assumes that there's at least + # one, so short-circuit here. + if self.numberOfContours == 0: + return + if self.isComposite(): + self.decompileComponents(data, glyfTable) + else: + self.decompileCoordinates(data) + + def compile( + self, glyfTable, recalcBBoxes=True, *, boundsDone=None, optimizeSize=True + ): + if hasattr(self, "data"): + if recalcBBoxes: + # must unpack glyph in order to recalculate bounding box + self.expand(glyfTable) + else: + return self.data + if self.numberOfContours == 0: + return b"" + + if recalcBBoxes: + self.recalcBounds(glyfTable, boundsDone=boundsDone) + + data = sstruct.pack(glyphHeaderFormat, self) + if self.isComposite(): + data = data + self.compileComponents(glyfTable) + else: + data = data + self.compileCoordinates(optimizeSize=optimizeSize) + return data + + def toXML(self, writer, ttFont): + if self.isComposite(): + for compo in self.components: + compo.toXML(writer, ttFont) + haveInstructions = hasattr(self, "program") + else: + last = 0 + for i in range(self.numberOfContours): + writer.begintag("contour") + writer.newline() + for j in range(last, self.endPtsOfContours[i] + 1): + attrs = [ + ("x", self.coordinates[j][0]), + ("y", self.coordinates[j][1]), + ("on", self.flags[j] & flagOnCurve), + ] + if self.flags[j] & flagOverlapSimple: + # Apple's rasterizer uses flagOverlapSimple in the first contour/first pt to flag glyphs that contain overlapping contours + attrs.append(("overlap", 1)) + if self.flags[j] & flagCubic: + attrs.append(("cubic", 1)) + writer.simpletag("pt", attrs) + writer.newline() + last = self.endPtsOfContours[i] + 1 + writer.endtag("contour") + writer.newline() + haveInstructions = self.numberOfContours > 0 + if haveInstructions: + if self.program: + writer.begintag("instructions") + writer.newline() + self.program.toXML(writer, ttFont) + writer.endtag("instructions") + else: + writer.simpletag("instructions") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "contour": + if self.numberOfContours < 0: + raise ttLib.TTLibError("can't mix composites and contours in glyph") + self.numberOfContours = self.numberOfContours + 1 + coordinates = GlyphCoordinates() + flags = bytearray() + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name != "pt": + continue # ignore anything but "pt" + coordinates.append((safeEval(attrs["x"]), safeEval(attrs["y"]))) + flag = bool(safeEval(attrs["on"])) + if "overlap" in attrs and bool(safeEval(attrs["overlap"])): + flag |= flagOverlapSimple + if "cubic" in attrs and bool(safeEval(attrs["cubic"])): + flag |= flagCubic + flags.append(flag) + if not hasattr(self, "coordinates"): + self.coordinates = coordinates + self.flags = flags + self.endPtsOfContours = [len(coordinates) - 1] + else: + self.coordinates.extend(coordinates) + self.flags.extend(flags) + self.endPtsOfContours.append(len(self.coordinates) - 1) + elif name == "component": + if self.numberOfContours > 0: + raise ttLib.TTLibError("can't mix composites and contours in glyph") + self.numberOfContours = -1 + if not hasattr(self, "components"): + self.components = [] + component = GlyphComponent() + self.components.append(component) + component.fromXML(name, attrs, content, ttFont) + elif name == "instructions": + self.program = ttProgram.Program() + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + self.program.fromXML(name, attrs, content, ttFont) + + def getCompositeMaxpValues(self, glyfTable, maxComponentDepth=1): + assert self.isComposite() + nContours = 0 + nPoints = 0 + initialMaxComponentDepth = maxComponentDepth + for compo in self.components: + baseGlyph = glyfTable[compo.glyphName] + if baseGlyph.numberOfContours == 0: + continue + elif baseGlyph.numberOfContours > 0: + nP, nC = baseGlyph.getMaxpValues() + else: + nP, nC, componentDepth = baseGlyph.getCompositeMaxpValues( + glyfTable, initialMaxComponentDepth + 1 + ) + maxComponentDepth = max(maxComponentDepth, componentDepth) + nPoints = nPoints + nP + nContours = nContours + nC + return CompositeMaxpValues(nPoints, nContours, maxComponentDepth) + + def getMaxpValues(self): + assert self.numberOfContours > 0 + return len(self.coordinates), len(self.endPtsOfContours) + + def decompileComponents(self, data, glyfTable): + self.components = [] + more = 1 + haveInstructions = 0 + while more: + component = GlyphComponent() + more, haveInstr, data = component.decompile(data, glyfTable) + haveInstructions = haveInstructions | haveInstr + self.components.append(component) + if haveInstructions: + (numInstructions,) = struct.unpack(">h", data[:2]) + data = data[2:] + self.program = ttProgram.Program() + self.program.fromBytecode(data[:numInstructions]) + data = data[numInstructions:] + if len(data) >= 4: + log.warning( + "too much glyph data at the end of composite glyph: %d excess bytes", + len(data), + ) + + def decompileCoordinates(self, data): + endPtsOfContours = array.array("H") + endPtsOfContours.frombytes(data[: 2 * self.numberOfContours]) + if sys.byteorder != "big": + endPtsOfContours.byteswap() + self.endPtsOfContours = endPtsOfContours.tolist() + + pos = 2 * self.numberOfContours + (instructionLength,) = struct.unpack(">h", data[pos : pos + 2]) + self.program = ttProgram.Program() + self.program.fromBytecode(data[pos + 2 : pos + 2 + instructionLength]) + pos += 2 + instructionLength + nCoordinates = self.endPtsOfContours[-1] + 1 + flags, xCoordinates, yCoordinates = self.decompileCoordinatesRaw( + nCoordinates, data, pos + ) + + # fill in repetitions and apply signs + self.coordinates = coordinates = GlyphCoordinates.zeros(nCoordinates) + xIndex = 0 + yIndex = 0 + for i in range(nCoordinates): + flag = flags[i] + # x coordinate + if flag & flagXShort: + if flag & flagXsame: + x = xCoordinates[xIndex] + else: + x = -xCoordinates[xIndex] + xIndex = xIndex + 1 + elif flag & flagXsame: + x = 0 + else: + x = xCoordinates[xIndex] + xIndex = xIndex + 1 + # y coordinate + if flag & flagYShort: + if flag & flagYsame: + y = yCoordinates[yIndex] + else: + y = -yCoordinates[yIndex] + yIndex = yIndex + 1 + elif flag & flagYsame: + y = 0 + else: + y = yCoordinates[yIndex] + yIndex = yIndex + 1 + coordinates[i] = (x, y) + assert xIndex == len(xCoordinates) + assert yIndex == len(yCoordinates) + coordinates.relativeToAbsolute() + # discard all flags except "keepFlags" + for i in range(len(flags)): + flags[i] &= keepFlags + self.flags = flags + + def decompileCoordinatesRaw(self, nCoordinates, data, pos=0): + # unpack flags and prepare unpacking of coordinates + flags = bytearray(nCoordinates) + # Warning: deep Python trickery going on. We use the struct module to unpack + # the coordinates. We build a format string based on the flags, so we can + # unpack the coordinates in one struct.unpack() call. + xFormat = ">" # big endian + yFormat = ">" # big endian + j = 0 + while True: + flag = data[pos] + pos += 1 + repeat = 1 + if flag & flagRepeat: + repeat = data[pos] + 1 + pos += 1 + for k in range(repeat): + if flag & flagXShort: + xFormat = xFormat + "B" + elif not (flag & flagXsame): + xFormat = xFormat + "h" + if flag & flagYShort: + yFormat = yFormat + "B" + elif not (flag & flagYsame): + yFormat = yFormat + "h" + flags[j] = flag + j = j + 1 + if j >= nCoordinates: + break + assert j == nCoordinates, "bad glyph flags" + # unpack raw coordinates, krrrrrr-tching! + xDataLen = struct.calcsize(xFormat) + yDataLen = struct.calcsize(yFormat) + if len(data) - pos - (xDataLen + yDataLen) >= 4: + log.warning( + "too much glyph data: %d excess bytes", + len(data) - pos - (xDataLen + yDataLen), + ) + xCoordinates = struct.unpack(xFormat, data[pos : pos + xDataLen]) + yCoordinates = struct.unpack( + yFormat, data[pos + xDataLen : pos + xDataLen + yDataLen] + ) + return flags, xCoordinates, yCoordinates + + def compileComponents(self, glyfTable): + data = b"" + lastcomponent = len(self.components) - 1 + more = 1 + haveInstructions = 0 + for i, compo in enumerate(self.components): + if i == lastcomponent: + haveInstructions = hasattr(self, "program") + more = 0 + data = data + compo.compile(more, haveInstructions, glyfTable) + if haveInstructions: + instructions = self.program.getBytecode() + data = data + struct.pack(">h", len(instructions)) + instructions + return data + + def compileCoordinates(self, *, optimizeSize=True): + assert len(self.coordinates) == len(self.flags) + data = [] + endPtsOfContours = array.array("H", self.endPtsOfContours) + if sys.byteorder != "big": + endPtsOfContours.byteswap() + data.append(endPtsOfContours.tobytes()) + instructions = self.program.getBytecode() + data.append(struct.pack(">h", len(instructions))) + data.append(instructions) + + deltas = self.coordinates.copy() + deltas.toInt() + deltas.absoluteToRelative() + + if optimizeSize: + # TODO(behdad): Add a configuration option for this? + deltas = self.compileDeltasGreedy(self.flags, deltas) + # deltas = self.compileDeltasOptimal(self.flags, deltas) + else: + deltas = self.compileDeltasForSpeed(self.flags, deltas) + + data.extend(deltas) + return b"".join(data) + + def compileDeltasGreedy(self, flags, deltas): + # Implements greedy algorithm for packing coordinate deltas: + # uses shortest representation one coordinate at a time. + compressedFlags = bytearray() + compressedXs = bytearray() + compressedYs = bytearray() + lastflag = None + repeat = 0 + for flag, (x, y) in zip(flags, deltas): + # Oh, the horrors of TrueType + # do x + if x == 0: + flag = flag | flagXsame + elif -255 <= x <= 255: + flag = flag | flagXShort + if x > 0: + flag = flag | flagXsame + else: + x = -x + compressedXs.append(x) + else: + compressedXs.extend(struct.pack(">h", x)) + # do y + if y == 0: + flag = flag | flagYsame + elif -255 <= y <= 255: + flag = flag | flagYShort + if y > 0: + flag = flag | flagYsame + else: + y = -y + compressedYs.append(y) + else: + compressedYs.extend(struct.pack(">h", y)) + # handle repeating flags + if flag == lastflag and repeat != 255: + repeat = repeat + 1 + if repeat == 1: + compressedFlags.append(flag) + else: + compressedFlags[-2] = flag | flagRepeat + compressedFlags[-1] = repeat + else: + repeat = 0 + compressedFlags.append(flag) + lastflag = flag + return (compressedFlags, compressedXs, compressedYs) + + def compileDeltasOptimal(self, flags, deltas): + # Implements optimal, dynaic-programming, algorithm for packing coordinate + # deltas. The savings are negligible :(. + candidates = [] + bestTuple = None + bestCost = 0 + repeat = 0 + for flag, (x, y) in zip(flags, deltas): + # Oh, the horrors of TrueType + flag, coordBytes = flagBest(x, y, flag) + bestCost += 1 + coordBytes + newCandidates = [ + (bestCost, bestTuple, flag, coordBytes), + (bestCost + 1, bestTuple, (flag | flagRepeat), coordBytes), + ] + for lastCost, lastTuple, lastFlag, coordBytes in candidates: + if ( + lastCost + coordBytes <= bestCost + 1 + and (lastFlag & flagRepeat) + and (lastFlag < 0xFF00) + and flagSupports(lastFlag, flag) + ): + if (lastFlag & 0xFF) == ( + flag | flagRepeat + ) and lastCost == bestCost + 1: + continue + newCandidates.append( + (lastCost + coordBytes, lastTuple, lastFlag + 256, coordBytes) + ) + candidates = newCandidates + bestTuple = min(candidates, key=lambda t: t[0]) + bestCost = bestTuple[0] + + flags = [] + while bestTuple: + cost, bestTuple, flag, coordBytes = bestTuple + flags.append(flag) + flags.reverse() + + compressedFlags = bytearray() + compressedXs = bytearray() + compressedYs = bytearray() + coords = iter(deltas) + ff = [] + for flag in flags: + repeatCount, flag = flag >> 8, flag & 0xFF + compressedFlags.append(flag) + if flag & flagRepeat: + assert repeatCount > 0 + compressedFlags.append(repeatCount) + else: + assert repeatCount == 0 + for i in range(1 + repeatCount): + x, y = next(coords) + flagEncodeCoords(flag, x, y, compressedXs, compressedYs) + ff.append(flag) + try: + next(coords) + raise Exception("internal error") + except StopIteration: + pass + + return (compressedFlags, compressedXs, compressedYs) + + def compileDeltasForSpeed(self, flags, deltas): + # uses widest representation needed, for all deltas. + compressedFlags = bytearray() + compressedXs = bytearray() + compressedYs = bytearray() + + # Compute the necessary width for each axis + xs = [d[0] for d in deltas] + ys = [d[1] for d in deltas] + minX, minY, maxX, maxY = min(xs), min(ys), max(xs), max(ys) + xZero = minX == 0 and maxX == 0 + yZero = minY == 0 and maxY == 0 + xShort = -255 <= minX <= maxX <= 255 + yShort = -255 <= minY <= maxY <= 255 + + lastflag = None + repeat = 0 + for flag, (x, y) in zip(flags, deltas): + # Oh, the horrors of TrueType + # do x + if xZero: + flag = flag | flagXsame + elif xShort: + flag = flag | flagXShort + if x > 0: + flag = flag | flagXsame + else: + x = -x + compressedXs.append(x) + else: + compressedXs.extend(struct.pack(">h", x)) + # do y + if yZero: + flag = flag | flagYsame + elif yShort: + flag = flag | flagYShort + if y > 0: + flag = flag | flagYsame + else: + y = -y + compressedYs.append(y) + else: + compressedYs.extend(struct.pack(">h", y)) + # handle repeating flags + if flag == lastflag and repeat != 255: + repeat = repeat + 1 + if repeat == 1: + compressedFlags.append(flag) + else: + compressedFlags[-2] = flag | flagRepeat + compressedFlags[-1] = repeat + else: + repeat = 0 + compressedFlags.append(flag) + lastflag = flag + return (compressedFlags, compressedXs, compressedYs) + + def recalcBounds(self, glyfTable, *, boundsDone=None): + """Recalculates the bounds of the glyph. + + Each glyph object stores its bounding box in the + ``xMin``/``yMin``/``xMax``/``yMax`` attributes. These bounds must be + recomputed when the ``coordinates`` change. The ``table__g_l_y_f`` bounds + must be provided to resolve component bounds. + """ + if self.isComposite() and self.tryRecalcBoundsComposite( + glyfTable, boundsDone=boundsDone + ): + return + try: + coords, endPts, flags = self.getCoordinates(glyfTable, round=otRound) + self.xMin, self.yMin, self.xMax, self.yMax = coords.calcIntBounds() + except NotImplementedError: + pass + + def tryRecalcBoundsComposite(self, glyfTable, *, boundsDone=None): + """Try recalculating the bounds of a composite glyph that has + certain constrained properties. Namely, none of the components + have a transform other than an integer translate, and none + uses the anchor points. + + Each glyph object stores its bounding box in the + ``xMin``/``yMin``/``xMax``/``yMax`` attributes. These bounds must be + recomputed when the ``coordinates`` change. The ``table__g_l_y_f`` bounds + must be provided to resolve component bounds. + + Return True if bounds were calculated, False otherwise. + """ + for compo in self.components: + if not compo._hasOnlyIntegerTranslate(): + return False + + # All components are untransformed and have an integer x/y translate + bounds = None + for compo in self.components: + glyphName = compo.glyphName + g = glyfTable[glyphName] + + if boundsDone is None or glyphName not in boundsDone: + g.recalcBounds(glyfTable, boundsDone=boundsDone) + if boundsDone is not None: + boundsDone.add(glyphName) + # empty components shouldn't update the bounds of the parent glyph + if g.yMin == g.yMax and g.xMin == g.xMax: + continue + + x, y = compo.x, compo.y + bounds = updateBounds(bounds, (g.xMin + x, g.yMin + y)) + bounds = updateBounds(bounds, (g.xMax + x, g.yMax + y)) + + if bounds is None: + bounds = (0, 0, 0, 0) + self.xMin, self.yMin, self.xMax, self.yMax = bounds + return True + + def isComposite(self): + """Test whether a glyph has components""" + if hasattr(self, "data"): + return struct.unpack(">h", self.data[:2])[0] == -1 if self.data else False + else: + return self.numberOfContours == -1 + + def getCoordinates(self, glyfTable, *, round=noRound): + """Return the coordinates, end points and flags + + This method returns three values: A :py:class:`GlyphCoordinates` object, + a list of the indexes of the final points of each contour (allowing you + to split up the coordinates list into contours) and a list of flags. + + On simple glyphs, this method returns information from the glyph's own + contours; on composite glyphs, it "flattens" all components recursively + to return a list of coordinates representing all the components involved + in the glyph. + + To interpret the flags for each point, see the "Simple Glyph Flags" + section of the `glyf table specification <https://docs.microsoft.com/en-us/typography/opentype/spec/glyf#simple-glyph-description>`. + """ + + if self.numberOfContours > 0: + return self.coordinates, self.endPtsOfContours, self.flags + elif self.isComposite(): + # it's a composite + allCoords = GlyphCoordinates() + allFlags = bytearray() + allEndPts = [] + for compo in self.components: + g = glyfTable[compo.glyphName] + try: + coordinates, endPts, flags = g.getCoordinates( + glyfTable, round=round + ) + except RecursionError: + raise ttLib.TTLibError( + "glyph '%s' contains a recursive component reference" + % compo.glyphName + ) + coordinates = GlyphCoordinates(coordinates) + # if asked to round e.g. while computing bboxes, it's important we + # do it immediately before a component transform is applied to a + # simple glyph's coordinates in case these might still contain floats; + # however, if the referenced component glyph is another composite, we + # must not round here but only at the end, after all the nested + # transforms have been applied, or else rounding errors will compound. + if round is not noRound and g.numberOfContours > 0: + coordinates.toInt(round=round) + if hasattr(compo, "firstPt"): + # component uses two reference points: we apply the transform _before_ + # computing the offset between the points + if hasattr(compo, "transform"): + coordinates.transform(compo.transform) + x1, y1 = allCoords[compo.firstPt] + x2, y2 = coordinates[compo.secondPt] + move = x1 - x2, y1 - y2 + coordinates.translate(move) + else: + # component uses XY offsets + move = compo.x, compo.y + if not hasattr(compo, "transform"): + coordinates.translate(move) + else: + apple_way = compo.flags & SCALED_COMPONENT_OFFSET + ms_way = compo.flags & UNSCALED_COMPONENT_OFFSET + assert not (apple_way and ms_way) + if not (apple_way or ms_way): + scale_component_offset = ( + SCALE_COMPONENT_OFFSET_DEFAULT # see top of this file + ) + else: + scale_component_offset = apple_way + if scale_component_offset: + # the Apple way: first move, then scale (ie. scale the component offset) + coordinates.translate(move) + coordinates.transform(compo.transform) + else: + # the MS way: first scale, then move + coordinates.transform(compo.transform) + coordinates.translate(move) + offset = len(allCoords) + allEndPts.extend(e + offset for e in endPts) + allCoords.extend(coordinates) + allFlags.extend(flags) + return allCoords, allEndPts, allFlags + else: + return GlyphCoordinates(), [], bytearray() + + def getComponentNames(self, glyfTable): + """Returns a list of names of component glyphs used in this glyph + + This method can be used on simple glyphs (in which case it returns an + empty list) or composite glyphs. + """ + if not hasattr(self, "data"): + if self.isComposite(): + return [c.glyphName for c in self.components] + else: + return [] + + # Extract components without expanding glyph + + if not self.data or struct.unpack(">h", self.data[:2])[0] >= 0: + return [] # Not composite + + data = self.data + i = 10 + components = [] + more = 1 + while more: + flags, glyphID = struct.unpack(">HH", data[i : i + 4]) + i += 4 + flags = int(flags) + components.append(glyfTable.getGlyphName(int(glyphID))) + + if flags & ARG_1_AND_2_ARE_WORDS: + i += 4 + else: + i += 2 + if flags & WE_HAVE_A_SCALE: + i += 2 + elif flags & WE_HAVE_AN_X_AND_Y_SCALE: + i += 4 + elif flags & WE_HAVE_A_TWO_BY_TWO: + i += 8 + more = flags & MORE_COMPONENTS + + return components + + def trim(self, remove_hinting=False): + """Remove padding and, if requested, hinting, from a glyph. + This works on both expanded and compacted glyphs, without + expanding it.""" + if not hasattr(self, "data"): + if remove_hinting: + if self.isComposite(): + if hasattr(self, "program"): + del self.program + else: + self.program = ttProgram.Program() + self.program.fromBytecode([]) + # No padding to trim. + return + if not self.data: + return + numContours = struct.unpack(">h", self.data[:2])[0] + data = bytearray(self.data) + i = 10 + if numContours >= 0: + i += 2 * numContours # endPtsOfContours + nCoordinates = ((data[i - 2] << 8) | data[i - 1]) + 1 + instructionLen = (data[i] << 8) | data[i + 1] + if remove_hinting: + # Zero instruction length + data[i] = data[i + 1] = 0 + i += 2 + if instructionLen: + # Splice it out + data = data[:i] + data[i + instructionLen :] + instructionLen = 0 + else: + i += 2 + instructionLen + + coordBytes = 0 + j = 0 + while True: + flag = data[i] + i = i + 1 + repeat = 1 + if flag & flagRepeat: + repeat = data[i] + 1 + i = i + 1 + xBytes = yBytes = 0 + if flag & flagXShort: + xBytes = 1 + elif not (flag & flagXsame): + xBytes = 2 + if flag & flagYShort: + yBytes = 1 + elif not (flag & flagYsame): + yBytes = 2 + coordBytes += (xBytes + yBytes) * repeat + j += repeat + if j >= nCoordinates: + break + assert j == nCoordinates, "bad glyph flags" + i += coordBytes + # Remove padding + data = data[:i] + elif self.isComposite(): + more = 1 + we_have_instructions = False + while more: + flags = (data[i] << 8) | data[i + 1] + if remove_hinting: + flags &= ~WE_HAVE_INSTRUCTIONS + if flags & WE_HAVE_INSTRUCTIONS: + we_have_instructions = True + data[i + 0] = flags >> 8 + data[i + 1] = flags & 0xFF + i += 4 + flags = int(flags) + + if flags & ARG_1_AND_2_ARE_WORDS: + i += 4 + else: + i += 2 + if flags & WE_HAVE_A_SCALE: + i += 2 + elif flags & WE_HAVE_AN_X_AND_Y_SCALE: + i += 4 + elif flags & WE_HAVE_A_TWO_BY_TWO: + i += 8 + more = flags & MORE_COMPONENTS + if we_have_instructions: + instructionLen = (data[i] << 8) | data[i + 1] + i += 2 + instructionLen + # Remove padding + data = data[:i] + + self.data = data + + def removeHinting(self): + """Removes TrueType hinting instructions from the glyph.""" + self.trim(remove_hinting=True) + + def draw(self, pen, glyfTable, offset=0): + """Draws the glyph using the supplied pen object. + + Arguments: + pen: An object conforming to the pen protocol. + glyfTable: A :py:class:`table__g_l_y_f` object, to resolve components. + offset (int): A horizontal offset. If provided, all coordinates are + translated by this offset. + """ + + if self.isComposite(): + for component in self.components: + glyphName, transform = component.getComponentInfo() + pen.addComponent(glyphName, transform) + return + + self.expand(glyfTable) + coordinates, endPts, flags = self.getCoordinates(glyfTable) + if offset: + coordinates = coordinates.copy() + coordinates.translate((offset, 0)) + start = 0 + maybeInt = lambda v: int(v) if v == int(v) else v + for end in endPts: + end = end + 1 + contour = coordinates[start:end] + cFlags = [flagOnCurve & f for f in flags[start:end]] + cuFlags = [flagCubic & f for f in flags[start:end]] + start = end + if 1 not in cFlags: + assert all(cuFlags) or not any(cuFlags) + cubic = all(cuFlags) + if cubic: + count = len(contour) + assert count % 2 == 0, "Odd number of cubic off-curves undefined" + l = contour[-1] + f = contour[0] + p0 = (maybeInt((l[0] + f[0]) * 0.5), maybeInt((l[1] + f[1]) * 0.5)) + pen.moveTo(p0) + for i in range(0, count, 2): + p1 = contour[i] + p2 = contour[i + 1] + p4 = contour[i + 2 if i + 2 < count else 0] + p3 = ( + maybeInt((p2[0] + p4[0]) * 0.5), + maybeInt((p2[1] + p4[1]) * 0.5), + ) + pen.curveTo(p1, p2, p3) + else: + # There is not a single on-curve point on the curve, + # use pen.qCurveTo's special case by specifying None + # as the on-curve point. + contour.append(None) + pen.qCurveTo(*contour) + else: + # Shuffle the points so that the contour is guaranteed + # to *end* in an on-curve point, which we'll use for + # the moveTo. + firstOnCurve = cFlags.index(1) + 1 + contour = contour[firstOnCurve:] + contour[:firstOnCurve] + cFlags = cFlags[firstOnCurve:] + cFlags[:firstOnCurve] + cuFlags = cuFlags[firstOnCurve:] + cuFlags[:firstOnCurve] + pen.moveTo(contour[-1]) + while contour: + nextOnCurve = cFlags.index(1) + 1 + if nextOnCurve == 1: + # Skip a final lineTo(), as it is implied by + # pen.closePath() + if len(contour) > 1: + pen.lineTo(contour[0]) + else: + cubicFlags = [f for f in cuFlags[: nextOnCurve - 1]] + assert all(cubicFlags) or not any(cubicFlags) + cubic = any(cubicFlags) + if cubic: + assert all( + cubicFlags + ), "Mixed cubic and quadratic segment undefined" + + count = nextOnCurve + assert ( + count >= 3 + ), "At least two cubic off-curve points required" + assert ( + count - 1 + ) % 2 == 0, "Odd number of cubic off-curves undefined" + for i in range(0, count - 3, 2): + p1 = contour[i] + p2 = contour[i + 1] + p4 = contour[i + 2] + p3 = ( + maybeInt((p2[0] + p4[0]) * 0.5), + maybeInt((p2[1] + p4[1]) * 0.5), + ) + lastOnCurve = p3 + pen.curveTo(p1, p2, p3) + pen.curveTo(*contour[count - 3 : count]) + else: + pen.qCurveTo(*contour[:nextOnCurve]) + contour = contour[nextOnCurve:] + cFlags = cFlags[nextOnCurve:] + cuFlags = cuFlags[nextOnCurve:] + pen.closePath() + + def drawPoints(self, pen, glyfTable, offset=0): + """Draw the glyph using the supplied pointPen. As opposed to Glyph.draw(), + this will not change the point indices. + """ + + if self.isComposite(): + for component in self.components: + glyphName, transform = component.getComponentInfo() + pen.addComponent(glyphName, transform) + return + + coordinates, endPts, flags = self.getCoordinates(glyfTable) + if offset: + coordinates = coordinates.copy() + coordinates.translate((offset, 0)) + start = 0 + for end in endPts: + end = end + 1 + contour = coordinates[start:end] + cFlags = flags[start:end] + start = end + pen.beginPath() + # Start with the appropriate segment type based on the final segment + + if cFlags[-1] & flagOnCurve: + segmentType = "line" + elif cFlags[-1] & flagCubic: + segmentType = "curve" + else: + segmentType = "qcurve" + for i, pt in enumerate(contour): + if cFlags[i] & flagOnCurve: + pen.addPoint(pt, segmentType=segmentType) + segmentType = "line" + else: + pen.addPoint(pt) + segmentType = "curve" if cFlags[i] & flagCubic else "qcurve" + pen.endPath() + + def __eq__(self, other): + if type(self) != type(other): + return NotImplemented + return self.__dict__ == other.__dict__ + + def __ne__(self, other): + result = self.__eq__(other) + return result if result is NotImplemented else not result + + +# Vector.__round__ uses the built-in (Banker's) `round` but we want +# to use otRound below +_roundv = partial(Vector.__round__, round=otRound) + + +def _is_mid_point(p0: tuple, p1: tuple, p2: tuple) -> bool: + # True if p1 is in the middle of p0 and p2, either before or after rounding + p0 = Vector(p0) + p1 = Vector(p1) + p2 = Vector(p2) + return ((p0 + p2) * 0.5).isclose(p1) or _roundv(p0) + _roundv(p2) == _roundv(p1) * 2 + + +def dropImpliedOnCurvePoints(*interpolatable_glyphs: Glyph) -> Set[int]: + """Drop impliable on-curve points from the (simple) glyph or glyphs. + + In TrueType glyf outlines, on-curve points can be implied when they are located at + the midpoint of the line connecting two consecutive off-curve points. + + If more than one glyphs are passed, these are assumed to be interpolatable masters + of the same glyph impliable, and thus only the on-curve points that are impliable + for all of them will actually be implied. + Composite glyphs or empty glyphs are skipped, only simple glyphs with 1 or more + contours are considered. + The input glyph(s) is/are modified in-place. + + Args: + interpolatable_glyphs: The glyph or glyphs to modify in-place. + + Returns: + The set of point indices that were dropped if any. + + Raises: + ValueError if simple glyphs are not in fact interpolatable because they have + different point flags or number of contours. + + Reference: + https://developer.apple.com/fonts/TrueType-Reference-Manual/RM01/Chap1.html + """ + staticAttributes = SimpleNamespace( + numberOfContours=None, flags=None, endPtsOfContours=None + ) + drop = None + simple_glyphs = [] + for i, glyph in enumerate(interpolatable_glyphs): + if glyph.numberOfContours < 1: + # ignore composite or empty glyphs + continue + + for attr in staticAttributes.__dict__: + expected = getattr(staticAttributes, attr) + found = getattr(glyph, attr) + if expected is None: + setattr(staticAttributes, attr, found) + elif expected != found: + raise ValueError( + f"Incompatible {attr} for glyph at master index {i}: " + f"expected {expected}, found {found}" + ) + + may_drop = set() + start = 0 + coords = glyph.coordinates + flags = staticAttributes.flags + endPtsOfContours = staticAttributes.endPtsOfContours + for last in endPtsOfContours: + for i in range(start, last + 1): + if not (flags[i] & flagOnCurve): + continue + prv = i - 1 if i > start else last + nxt = i + 1 if i < last else start + if (flags[prv] & flagOnCurve) or flags[prv] != flags[nxt]: + continue + # we may drop the ith on-curve if halfway between previous/next off-curves + if not _is_mid_point(coords[prv], coords[i], coords[nxt]): + continue + + may_drop.add(i) + start = last + 1 + # we only want to drop if ALL interpolatable glyphs have the same implied oncurves + if drop is None: + drop = may_drop + else: + drop.intersection_update(may_drop) + + simple_glyphs.append(glyph) + + if drop: + # Do the actual dropping + flags = staticAttributes.flags + assert flags is not None + newFlags = array.array( + "B", (flags[i] for i in range(len(flags)) if i not in drop) + ) + + endPts = staticAttributes.endPtsOfContours + assert endPts is not None + newEndPts = [] + i = 0 + delta = 0 + for d in sorted(drop): + while d > endPts[i]: + newEndPts.append(endPts[i] - delta) + i += 1 + delta += 1 + while i < len(endPts): + newEndPts.append(endPts[i] - delta) + i += 1 + + for glyph in simple_glyphs: + coords = glyph.coordinates + glyph.coordinates = GlyphCoordinates( + coords[i] for i in range(len(coords)) if i not in drop + ) + glyph.flags = newFlags + glyph.endPtsOfContours = newEndPts + + return drop if drop is not None else set() + + +class GlyphComponent(object): + """Represents a component within a composite glyph. + + The component is represented internally with four attributes: ``glyphName``, + ``x``, ``y`` and ``transform``. If there is no "two-by-two" matrix (i.e + no scaling, reflection, or rotation; only translation), the ``transform`` + attribute is not present. + """ + + # The above documentation is not *completely* true, but is *true enough* because + # the rare firstPt/lastPt attributes are not totally supported and nobody seems to + # mind - see below. + + def __init__(self): + pass + + def getComponentInfo(self): + """Return information about the component + + This method returns a tuple of two values: the glyph name of the component's + base glyph, and a transformation matrix. As opposed to accessing the attributes + directly, ``getComponentInfo`` always returns a six-element tuple of the + component's transformation matrix, even when the two-by-two ``.transform`` + matrix is not present. + """ + # XXX Ignoring self.firstPt & self.lastpt for now: I need to implement + # something equivalent in fontTools.objects.glyph (I'd rather not + # convert it to an absolute offset, since it is valuable information). + # This method will now raise "AttributeError: x" on glyphs that use + # this TT feature. + if hasattr(self, "transform"): + [[xx, xy], [yx, yy]] = self.transform + trans = (xx, xy, yx, yy, self.x, self.y) + else: + trans = (1, 0, 0, 1, self.x, self.y) + return self.glyphName, trans + + def decompile(self, data, glyfTable): + flags, glyphID = struct.unpack(">HH", data[:4]) + self.flags = int(flags) + glyphID = int(glyphID) + self.glyphName = glyfTable.getGlyphName(int(glyphID)) + data = data[4:] + + if self.flags & ARG_1_AND_2_ARE_WORDS: + if self.flags & ARGS_ARE_XY_VALUES: + self.x, self.y = struct.unpack(">hh", data[:4]) + else: + x, y = struct.unpack(">HH", data[:4]) + self.firstPt, self.secondPt = int(x), int(y) + data = data[4:] + else: + if self.flags & ARGS_ARE_XY_VALUES: + self.x, self.y = struct.unpack(">bb", data[:2]) + else: + x, y = struct.unpack(">BB", data[:2]) + self.firstPt, self.secondPt = int(x), int(y) + data = data[2:] + + if self.flags & WE_HAVE_A_SCALE: + (scale,) = struct.unpack(">h", data[:2]) + self.transform = [ + [fi2fl(scale, 14), 0], + [0, fi2fl(scale, 14)], + ] # fixed 2.14 + data = data[2:] + elif self.flags & WE_HAVE_AN_X_AND_Y_SCALE: + xscale, yscale = struct.unpack(">hh", data[:4]) + self.transform = [ + [fi2fl(xscale, 14), 0], + [0, fi2fl(yscale, 14)], + ] # fixed 2.14 + data = data[4:] + elif self.flags & WE_HAVE_A_TWO_BY_TWO: + (xscale, scale01, scale10, yscale) = struct.unpack(">hhhh", data[:8]) + self.transform = [ + [fi2fl(xscale, 14), fi2fl(scale01, 14)], + [fi2fl(scale10, 14), fi2fl(yscale, 14)], + ] # fixed 2.14 + data = data[8:] + more = self.flags & MORE_COMPONENTS + haveInstructions = self.flags & WE_HAVE_INSTRUCTIONS + self.flags = self.flags & ( + ROUND_XY_TO_GRID + | USE_MY_METRICS + | SCALED_COMPONENT_OFFSET + | UNSCALED_COMPONENT_OFFSET + | NON_OVERLAPPING + | OVERLAP_COMPOUND + ) + return more, haveInstructions, data + + def compile(self, more, haveInstructions, glyfTable): + data = b"" + + # reset all flags we will calculate ourselves + flags = self.flags & ( + ROUND_XY_TO_GRID + | USE_MY_METRICS + | SCALED_COMPONENT_OFFSET + | UNSCALED_COMPONENT_OFFSET + | NON_OVERLAPPING + | OVERLAP_COMPOUND + ) + if more: + flags = flags | MORE_COMPONENTS + if haveInstructions: + flags = flags | WE_HAVE_INSTRUCTIONS + + if hasattr(self, "firstPt"): + if (0 <= self.firstPt <= 255) and (0 <= self.secondPt <= 255): + data = data + struct.pack(">BB", self.firstPt, self.secondPt) + else: + data = data + struct.pack(">HH", self.firstPt, self.secondPt) + flags = flags | ARG_1_AND_2_ARE_WORDS + else: + x = otRound(self.x) + y = otRound(self.y) + flags = flags | ARGS_ARE_XY_VALUES + if (-128 <= x <= 127) and (-128 <= y <= 127): + data = data + struct.pack(">bb", x, y) + else: + data = data + struct.pack(">hh", x, y) + flags = flags | ARG_1_AND_2_ARE_WORDS + + if hasattr(self, "transform"): + transform = [[fl2fi(x, 14) for x in row] for row in self.transform] + if transform[0][1] or transform[1][0]: + flags = flags | WE_HAVE_A_TWO_BY_TWO + data = data + struct.pack( + ">hhhh", + transform[0][0], + transform[0][1], + transform[1][0], + transform[1][1], + ) + elif transform[0][0] != transform[1][1]: + flags = flags | WE_HAVE_AN_X_AND_Y_SCALE + data = data + struct.pack(">hh", transform[0][0], transform[1][1]) + else: + flags = flags | WE_HAVE_A_SCALE + data = data + struct.pack(">h", transform[0][0]) + + glyphID = glyfTable.getGlyphID(self.glyphName) + return struct.pack(">HH", flags, glyphID) + data + + def toXML(self, writer, ttFont): + attrs = [("glyphName", self.glyphName)] + if not hasattr(self, "firstPt"): + attrs = attrs + [("x", self.x), ("y", self.y)] + else: + attrs = attrs + [("firstPt", self.firstPt), ("secondPt", self.secondPt)] + + if hasattr(self, "transform"): + transform = self.transform + if transform[0][1] or transform[1][0]: + attrs = attrs + [ + ("scalex", fl2str(transform[0][0], 14)), + ("scale01", fl2str(transform[0][1], 14)), + ("scale10", fl2str(transform[1][0], 14)), + ("scaley", fl2str(transform[1][1], 14)), + ] + elif transform[0][0] != transform[1][1]: + attrs = attrs + [ + ("scalex", fl2str(transform[0][0], 14)), + ("scaley", fl2str(transform[1][1], 14)), + ] + else: + attrs = attrs + [("scale", fl2str(transform[0][0], 14))] + attrs = attrs + [("flags", hex(self.flags))] + writer.simpletag("component", attrs) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.glyphName = attrs["glyphName"] + if "firstPt" in attrs: + self.firstPt = safeEval(attrs["firstPt"]) + self.secondPt = safeEval(attrs["secondPt"]) + else: + self.x = safeEval(attrs["x"]) + self.y = safeEval(attrs["y"]) + if "scale01" in attrs: + scalex = str2fl(attrs["scalex"], 14) + scale01 = str2fl(attrs["scale01"], 14) + scale10 = str2fl(attrs["scale10"], 14) + scaley = str2fl(attrs["scaley"], 14) + self.transform = [[scalex, scale01], [scale10, scaley]] + elif "scalex" in attrs: + scalex = str2fl(attrs["scalex"], 14) + scaley = str2fl(attrs["scaley"], 14) + self.transform = [[scalex, 0], [0, scaley]] + elif "scale" in attrs: + scale = str2fl(attrs["scale"], 14) + self.transform = [[scale, 0], [0, scale]] + self.flags = safeEval(attrs["flags"]) + + def __eq__(self, other): + if type(self) != type(other): + return NotImplemented + return self.__dict__ == other.__dict__ + + def __ne__(self, other): + result = self.__eq__(other) + return result if result is NotImplemented else not result + + def _hasOnlyIntegerTranslate(self): + """Return True if it's a 'simple' component. + + That is, it has no anchor points and no transform other than integer translate. + """ + return ( + not hasattr(self, "firstPt") + and not hasattr(self, "transform") + and float(self.x).is_integer() + and float(self.y).is_integer() + ) + + +class GlyphCoordinates(object): + """A list of glyph coordinates. + + Unlike an ordinary list, this is a numpy-like matrix object which supports + matrix addition, scalar multiplication and other operations described below. + """ + + def __init__(self, iterable=[]): + self._a = array.array("d") + self.extend(iterable) + + @property + def array(self): + """Returns the underlying array of coordinates""" + return self._a + + @staticmethod + def zeros(count): + """Creates a new ``GlyphCoordinates`` object with all coordinates set to (0,0)""" + g = GlyphCoordinates() + g._a.frombytes(bytes(count * 2 * g._a.itemsize)) + return g + + def copy(self): + """Creates a new ``GlyphCoordinates`` object which is a copy of the current one.""" + c = GlyphCoordinates() + c._a.extend(self._a) + return c + + def __len__(self): + """Returns the number of coordinates in the array.""" + return len(self._a) // 2 + + def __getitem__(self, k): + """Returns a two element tuple (x,y)""" + a = self._a + if isinstance(k, slice): + indices = range(*k.indices(len(self))) + # Instead of calling ourselves recursively, duplicate code; faster + ret = [] + for k in indices: + x = a[2 * k] + y = a[2 * k + 1] + ret.append( + (int(x) if x.is_integer() else x, int(y) if y.is_integer() else y) + ) + return ret + x = a[2 * k] + y = a[2 * k + 1] + return (int(x) if x.is_integer() else x, int(y) if y.is_integer() else y) + + def __setitem__(self, k, v): + """Sets a point's coordinates to a two element tuple (x,y)""" + if isinstance(k, slice): + indices = range(*k.indices(len(self))) + # XXX This only works if len(v) == len(indices) + for j, i in enumerate(indices): + self[i] = v[j] + return + self._a[2 * k], self._a[2 * k + 1] = v + + def __delitem__(self, i): + """Removes a point from the list""" + i = (2 * i) % len(self._a) + del self._a[i] + del self._a[i] + + def __repr__(self): + return "GlyphCoordinates([" + ",".join(str(c) for c in self) + "])" + + def append(self, p): + self._a.extend(tuple(p)) + + def extend(self, iterable): + for p in iterable: + self._a.extend(p) + + def toInt(self, *, round=otRound): + if round is noRound: + return + a = self._a + for i, value in enumerate(a): + a[i] = round(value) + + def calcBounds(self): + a = self._a + if not a: + return 0, 0, 0, 0 + xs = a[0::2] + ys = a[1::2] + return min(xs), min(ys), max(xs), max(ys) + + def calcIntBounds(self, round=otRound): + return tuple(round(v) for v in self.calcBounds()) + + def relativeToAbsolute(self): + a = self._a + x, y = 0, 0 + for i in range(0, len(a), 2): + a[i] = x = a[i] + x + a[i + 1] = y = a[i + 1] + y + + def absoluteToRelative(self): + a = self._a + x, y = 0, 0 + for i in range(0, len(a), 2): + nx = a[i] + ny = a[i + 1] + a[i] = nx - x + a[i + 1] = ny - y + x = nx + y = ny + + def translate(self, p): + """ + >>> GlyphCoordinates([(1,2)]).translate((.5,0)) + """ + x, y = p + if x == 0 and y == 0: + return + a = self._a + for i in range(0, len(a), 2): + a[i] += x + a[i + 1] += y + + def scale(self, p): + """ + >>> GlyphCoordinates([(1,2)]).scale((.5,0)) + """ + x, y = p + if x == 1 and y == 1: + return + a = self._a + for i in range(0, len(a), 2): + a[i] *= x + a[i + 1] *= y + + def transform(self, t): + """ + >>> GlyphCoordinates([(1,2)]).transform(((.5,0),(.2,.5))) + """ + a = self._a + for i in range(0, len(a), 2): + x = a[i] + y = a[i + 1] + px = x * t[0][0] + y * t[1][0] + py = x * t[0][1] + y * t[1][1] + a[i] = px + a[i + 1] = py + + def __eq__(self, other): + """ + >>> g = GlyphCoordinates([(1,2)]) + >>> g2 = GlyphCoordinates([(1.0,2)]) + >>> g3 = GlyphCoordinates([(1.5,2)]) + >>> g == g2 + True + >>> g == g3 + False + >>> g2 == g3 + False + """ + if type(self) != type(other): + return NotImplemented + return self._a == other._a + + def __ne__(self, other): + """ + >>> g = GlyphCoordinates([(1,2)]) + >>> g2 = GlyphCoordinates([(1.0,2)]) + >>> g3 = GlyphCoordinates([(1.5,2)]) + >>> g != g2 + False + >>> g != g3 + True + >>> g2 != g3 + True + """ + result = self.__eq__(other) + return result if result is NotImplemented else not result + + # Math operations + + def __pos__(self): + """ + >>> g = GlyphCoordinates([(1,2)]) + >>> g + GlyphCoordinates([(1, 2)]) + >>> g2 = +g + >>> g2 + GlyphCoordinates([(1, 2)]) + >>> g2.translate((1,0)) + >>> g2 + GlyphCoordinates([(2, 2)]) + >>> g + GlyphCoordinates([(1, 2)]) + """ + return self.copy() + + def __neg__(self): + """ + >>> g = GlyphCoordinates([(1,2)]) + >>> g + GlyphCoordinates([(1, 2)]) + >>> g2 = -g + >>> g2 + GlyphCoordinates([(-1, -2)]) + >>> g + GlyphCoordinates([(1, 2)]) + """ + r = self.copy() + a = r._a + for i, value in enumerate(a): + a[i] = -value + return r + + def __round__(self, *, round=otRound): + r = self.copy() + r.toInt(round=round) + return r + + def __add__(self, other): + return self.copy().__iadd__(other) + + def __sub__(self, other): + return self.copy().__isub__(other) + + def __mul__(self, other): + return self.copy().__imul__(other) + + def __truediv__(self, other): + return self.copy().__itruediv__(other) + + __radd__ = __add__ + __rmul__ = __mul__ + + def __rsub__(self, other): + return other + (-self) + + def __iadd__(self, other): + """ + >>> g = GlyphCoordinates([(1,2)]) + >>> g += (.5,0) + >>> g + GlyphCoordinates([(1.5, 2)]) + >>> g2 = GlyphCoordinates([(3,4)]) + >>> g += g2 + >>> g + GlyphCoordinates([(4.5, 6)]) + """ + if isinstance(other, tuple): + assert len(other) == 2 + self.translate(other) + return self + if isinstance(other, GlyphCoordinates): + other = other._a + a = self._a + assert len(a) == len(other) + for i, value in enumerate(other): + a[i] += value + return self + return NotImplemented + + def __isub__(self, other): + """ + >>> g = GlyphCoordinates([(1,2)]) + >>> g -= (.5,0) + >>> g + GlyphCoordinates([(0.5, 2)]) + >>> g2 = GlyphCoordinates([(3,4)]) + >>> g -= g2 + >>> g + GlyphCoordinates([(-2.5, -2)]) + """ + if isinstance(other, tuple): + assert len(other) == 2 + self.translate((-other[0], -other[1])) + return self + if isinstance(other, GlyphCoordinates): + other = other._a + a = self._a + assert len(a) == len(other) + for i, value in enumerate(other): + a[i] -= value + return self + return NotImplemented + + def __imul__(self, other): + """ + >>> g = GlyphCoordinates([(1,2)]) + >>> g *= (2,.5) + >>> g *= 2 + >>> g + GlyphCoordinates([(4, 2)]) + >>> g = GlyphCoordinates([(1,2)]) + >>> g *= 2 + >>> g + GlyphCoordinates([(2, 4)]) + """ + if isinstance(other, tuple): + assert len(other) == 2 + self.scale(other) + return self + if isinstance(other, Number): + if other == 1: + return self + a = self._a + for i in range(len(a)): + a[i] *= other + return self + return NotImplemented + + def __itruediv__(self, other): + """ + >>> g = GlyphCoordinates([(1,3)]) + >>> g /= (.5,1.5) + >>> g /= 2 + >>> g + GlyphCoordinates([(1, 1)]) + """ + if isinstance(other, Number): + other = (other, other) + if isinstance(other, tuple): + if other == (1, 1): + return self + assert len(other) == 2 + self.scale((1.0 / other[0], 1.0 / other[1])) + return self + return NotImplemented + + def __bool__(self): + """ + >>> g = GlyphCoordinates([]) + >>> bool(g) + False + >>> g = GlyphCoordinates([(0,0), (0.,0)]) + >>> bool(g) + True + >>> g = GlyphCoordinates([(0,0), (1,0)]) + >>> bool(g) + True + >>> g = GlyphCoordinates([(0,.5), (0,0)]) + >>> bool(g) + True + """ + return bool(self._a) + + __nonzero__ = __bool__ + + +if __name__ == "__main__": + import doctest, sys + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_v_a_r.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_v_a_r.py new file mode 100644 index 0000000000000000000000000000000000000000..15cc6fab5bd1b92fa0f778eb6d9952b195843786 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_g_v_a_r.py @@ -0,0 +1,340 @@ +from collections import deque +from functools import partial +from fontTools.misc import sstruct +from fontTools.misc.textTools import safeEval +from fontTools.misc.lazyTools import LazyDict +from fontTools.ttLib import OPTIMIZE_FONT_SPEED +from fontTools.ttLib.tables.TupleVariation import TupleVariation +from . import DefaultTable +import array +import itertools +import logging +import struct +import sys +import fontTools.ttLib.tables.TupleVariation as tv + +log = logging.getLogger(__name__) + +# https://www.microsoft.com/typography/otspec/gvar.htm +# https://www.microsoft.com/typography/otspec/otvarcommonformats.htm +# +# Apple's documentation of 'gvar': +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6gvar.html +# +# FreeType2 source code for parsing 'gvar': +# http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/src/truetype/ttgxvar.c + +GVAR_HEADER_FORMAT_HEAD = """ + > # big endian + version: H + reserved: H + axisCount: H + sharedTupleCount: H + offsetToSharedTuples: I +""" +# In between the HEAD and TAIL lies the glyphCount, which is +# of different size: 2 bytes for gvar, and 3 bytes for GVAR. +GVAR_HEADER_FORMAT_TAIL = """ + > # big endian + flags: H + offsetToGlyphVariationData: I +""" + +GVAR_HEADER_SIZE_HEAD = sstruct.calcsize(GVAR_HEADER_FORMAT_HEAD) +GVAR_HEADER_SIZE_TAIL = sstruct.calcsize(GVAR_HEADER_FORMAT_TAIL) + + +class table__g_v_a_r(DefaultTable.DefaultTable): + """Glyph Variations table + + The ``gvar`` table provides the per-glyph variation data that + describe how glyph outlines in the ``glyf`` table change across + the variation space that is defined for the font in the ``fvar`` + table. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/gvar + """ + + dependencies = ["fvar", "glyf"] + gid_size = 2 + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.version, self.reserved = 1, 0 + self.variations = {} + + def compile(self, ttFont): + axisTags = [axis.axisTag for axis in ttFont["fvar"].axes] + sharedTuples = tv.compileSharedTuples( + axisTags, itertools.chain(*self.variations.values()) + ) + sharedTupleIndices = {coord: i for i, coord in enumerate(sharedTuples)} + sharedTupleSize = sum([len(c) for c in sharedTuples]) + compiledGlyphs = self.compileGlyphs_(ttFont, axisTags, sharedTupleIndices) + offset = 0 + offsets = [] + for glyph in compiledGlyphs: + offsets.append(offset) + offset += len(glyph) + offsets.append(offset) + compiledOffsets, tableFormat = self.compileOffsets_(offsets) + + GVAR_HEADER_SIZE = GVAR_HEADER_SIZE_HEAD + self.gid_size + GVAR_HEADER_SIZE_TAIL + header = {} + header["version"] = self.version + header["reserved"] = self.reserved + header["axisCount"] = len(axisTags) + header["sharedTupleCount"] = len(sharedTuples) + header["offsetToSharedTuples"] = GVAR_HEADER_SIZE + len(compiledOffsets) + header["flags"] = tableFormat + header["offsetToGlyphVariationData"] = ( + header["offsetToSharedTuples"] + sharedTupleSize + ) + + result = [ + sstruct.pack(GVAR_HEADER_FORMAT_HEAD, header), + len(compiledGlyphs).to_bytes(self.gid_size, "big"), + sstruct.pack(GVAR_HEADER_FORMAT_TAIL, header), + ] + + result.append(compiledOffsets) + result.extend(sharedTuples) + result.extend(compiledGlyphs) + return b"".join(result) + + def compileGlyphs_(self, ttFont, axisTags, sharedCoordIndices): + optimizeSpeed = ttFont.cfg[OPTIMIZE_FONT_SPEED] + result = [] + glyf = ttFont["glyf"] + for glyphName in ttFont.getGlyphOrder(): + variations = self.variations.get(glyphName, []) + if not variations: + result.append(b"") + continue + pointCountUnused = 0 # pointCount is actually unused by compileGlyph + result.append( + compileGlyph_( + self.gid_size, + variations, + pointCountUnused, + axisTags, + sharedCoordIndices, + optimizeSize=not optimizeSpeed, + ) + ) + return result + + def decompile(self, data, ttFont): + axisTags = [axis.axisTag for axis in ttFont["fvar"].axes] + glyphs = ttFont.getGlyphOrder() + + # Parse the header + GVAR_HEADER_SIZE = GVAR_HEADER_SIZE_HEAD + self.gid_size + GVAR_HEADER_SIZE_TAIL + sstruct.unpack(GVAR_HEADER_FORMAT_HEAD, data[:GVAR_HEADER_SIZE_HEAD], self) + self.glyphCount = int.from_bytes( + data[GVAR_HEADER_SIZE_HEAD : GVAR_HEADER_SIZE_HEAD + self.gid_size], "big" + ) + sstruct.unpack( + GVAR_HEADER_FORMAT_TAIL, + data[GVAR_HEADER_SIZE_HEAD + self.gid_size : GVAR_HEADER_SIZE], + self, + ) + + assert len(glyphs) == self.glyphCount, (len(glyphs), self.glyphCount) + assert len(axisTags) == self.axisCount, ( + len(axisTags), + self.axisCount, + axisTags, + ) + sharedCoords = tv.decompileSharedTuples( + axisTags, self.sharedTupleCount, data, self.offsetToSharedTuples + ) + variations = {} + offsetToData = self.offsetToGlyphVariationData + glyf = ttFont["glyf"] + + def get_read_item(): + reverseGlyphMap = ttFont.getReverseGlyphMap() + tableFormat = self.flags & 1 + + def read_item(glyphName): + gid = reverseGlyphMap[glyphName] + offsetSize = 2 if tableFormat == 0 else 4 + startOffset = GVAR_HEADER_SIZE + offsetSize * gid + endOffset = startOffset + offsetSize * 2 + offsets = table__g_v_a_r.decompileOffsets_( + data[startOffset:endOffset], + tableFormat=tableFormat, + glyphCount=1, + ) + gvarData = data[offsetToData + offsets[0] : offsetToData + offsets[1]] + if not gvarData: + return [] + glyph = glyf[glyphName] + numPointsInGlyph = self.getNumPoints_(glyph) + return decompileGlyph_( + self.gid_size, numPointsInGlyph, sharedCoords, axisTags, gvarData + ) + + return read_item + + read_item = get_read_item() + l = LazyDict({glyphs[gid]: read_item for gid in range(self.glyphCount)}) + + self.variations = l + + if ttFont.lazy is False: # Be lazy for None and True + self.ensureDecompiled() + + def ensureDecompiled(self, recurse=False): + # The recurse argument is unused, but part of the signature of + # ensureDecompiled across the library. + # Use a zero-length deque to consume the lazy dict + deque(self.variations.values(), maxlen=0) + + @staticmethod + def decompileOffsets_(data, tableFormat, glyphCount): + if tableFormat == 0: + # Short format: array of UInt16 + offsets = array.array("H") + offsetsSize = (glyphCount + 1) * 2 + else: + # Long format: array of UInt32 + offsets = array.array("I") + offsetsSize = (glyphCount + 1) * 4 + offsets.frombytes(data[0:offsetsSize]) + if sys.byteorder != "big": + offsets.byteswap() + + # In the short format, offsets need to be multiplied by 2. + # This is not documented in Apple's TrueType specification, + # but can be inferred from the FreeType implementation, and + # we could verify it with two sample GX fonts. + if tableFormat == 0: + offsets = [off * 2 for off in offsets] + + return offsets + + @staticmethod + def compileOffsets_(offsets): + """Packs a list of offsets into a 'gvar' offset table. + + Returns a pair (bytestring, tableFormat). Bytestring is the + packed offset table. Format indicates whether the table + uses short (tableFormat=0) or long (tableFormat=1) integers. + The returned tableFormat should get packed into the flags field + of the 'gvar' header. + """ + assert len(offsets) >= 2 + for i in range(1, len(offsets)): + assert offsets[i - 1] <= offsets[i] + if max(offsets) <= 0xFFFF * 2: + packed = array.array("H", [n >> 1 for n in offsets]) + tableFormat = 0 + else: + packed = array.array("I", offsets) + tableFormat = 1 + if sys.byteorder != "big": + packed.byteswap() + return (packed.tobytes(), tableFormat) + + def toXML(self, writer, ttFont): + writer.simpletag("version", value=self.version) + writer.newline() + writer.simpletag("reserved", value=self.reserved) + writer.newline() + axisTags = [axis.axisTag for axis in ttFont["fvar"].axes] + for glyphName in ttFont.getGlyphNames(): + variations = self.variations.get(glyphName) + if not variations: + continue + writer.begintag("glyphVariations", glyph=glyphName) + writer.newline() + for gvar in variations: + gvar.toXML(writer, axisTags) + writer.endtag("glyphVariations") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": + self.version = safeEval(attrs["value"]) + elif name == "reserved": + self.reserved = safeEval(attrs["value"]) + elif name == "glyphVariations": + if not hasattr(self, "variations"): + self.variations = {} + glyphName = attrs["glyph"] + glyph = ttFont["glyf"][glyphName] + numPointsInGlyph = self.getNumPoints_(glyph) + glyphVariations = [] + for element in content: + if isinstance(element, tuple): + name, attrs, content = element + if name == "tuple": + gvar = TupleVariation({}, [None] * numPointsInGlyph) + glyphVariations.append(gvar) + for tupleElement in content: + if isinstance(tupleElement, tuple): + tupleName, tupleAttrs, tupleContent = tupleElement + gvar.fromXML(tupleName, tupleAttrs, tupleContent) + self.variations[glyphName] = glyphVariations + + @staticmethod + def getNumPoints_(glyph): + NUM_PHANTOM_POINTS = 4 + + if glyph.isComposite(): + return len(glyph.components) + NUM_PHANTOM_POINTS + else: + # Empty glyphs (eg. space, nonmarkingreturn) have no "coordinates" attribute. + return len(getattr(glyph, "coordinates", [])) + NUM_PHANTOM_POINTS + + +def compileGlyph_( + dataOffsetSize, + variations, + pointCount, + axisTags, + sharedCoordIndices, + *, + optimizeSize=True, +): + assert dataOffsetSize in (2, 3) + tupleVariationCount, tuples, data = tv.compileTupleVariationStore( + variations, pointCount, axisTags, sharedCoordIndices, optimizeSize=optimizeSize + ) + if tupleVariationCount == 0: + return b"" + + offsetToData = 2 + dataOffsetSize + len(tuples) + + result = [ + tupleVariationCount.to_bytes(2, "big"), + offsetToData.to_bytes(dataOffsetSize, "big"), + tuples, + data, + ] + if (offsetToData + len(data)) % 2 != 0: + result.append(b"\0") # padding + return b"".join(result) + + +def decompileGlyph_(dataOffsetSize, pointCount, sharedTuples, axisTags, data): + assert dataOffsetSize in (2, 3) + if len(data) < 2 + dataOffsetSize: + return [] + + tupleVariationCount = int.from_bytes(data[:2], "big") + offsetToData = int.from_bytes(data[2 : 2 + dataOffsetSize], "big") + + dataPos = offsetToData + return tv.decompileTupleVariationStore( + "gvar", + axisTags, + tupleVariationCount, + pointCount, + sharedTuples, + data, + 2 + dataOffsetSize, + offsetToData, + ) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_d_m_x.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_d_m_x.py new file mode 100644 index 0000000000000000000000000000000000000000..07c4566a98dcd36e9b3e2cdc9513793f902d1dd9 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_d_m_x.py @@ -0,0 +1,127 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import bytechr, byteord, strjoin +from . import DefaultTable +import array +from collections.abc import Mapping + +hdmxHeaderFormat = """ + > # big endian! + version: H + numRecords: H + recordSize: l +""" + + +class _GlyphnamedList(Mapping): + def __init__(self, reverseGlyphOrder, data): + self._array = data + self._map = dict(reverseGlyphOrder) + + def __getitem__(self, k): + return self._array[self._map[k]] + + def __len__(self): + return len(self._map) + + def __iter__(self): + return iter(self._map) + + def keys(self): + return self._map.keys() + + +class table__h_d_m_x(DefaultTable.DefaultTable): + """Horizontal Device Metrics table + + The ``hdmx`` table is an optional table that stores advance widths for + glyph outlines at specified pixel sizes. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/hdmx + """ + + def decompile(self, data, ttFont): + numGlyphs = ttFont["maxp"].numGlyphs + glyphOrder = ttFont.getGlyphOrder() + dummy, data = sstruct.unpack2(hdmxHeaderFormat, data, self) + self.hdmx = {} + for i in range(self.numRecords): + ppem = byteord(data[0]) + maxSize = byteord(data[1]) + widths = _GlyphnamedList( + ttFont.getReverseGlyphMap(), array.array("B", data[2 : 2 + numGlyphs]) + ) + self.hdmx[ppem] = widths + data = data[self.recordSize :] + assert len(data) == 0, "too much hdmx data" + + def compile(self, ttFont): + self.version = 0 + numGlyphs = ttFont["maxp"].numGlyphs + glyphOrder = ttFont.getGlyphOrder() + self.recordSize = 4 * ((2 + numGlyphs + 3) // 4) + pad = (self.recordSize - 2 - numGlyphs) * b"\0" + self.numRecords = len(self.hdmx) + data = sstruct.pack(hdmxHeaderFormat, self) + items = sorted(self.hdmx.items()) + for ppem, widths in items: + data = data + bytechr(ppem) + bytechr(max(widths.values())) + for glyphName in glyphOrder: + width = widths[glyphName] + data = data + bytechr(width) + data = data + pad + return data + + def toXML(self, writer, ttFont): + writer.begintag("hdmxData") + writer.newline() + ppems = sorted(self.hdmx.keys()) + records = [] + format = "" + for ppem in ppems: + widths = self.hdmx[ppem] + records.append(widths) + format = format + "%4d" + glyphNames = ttFont.getGlyphOrder()[:] + glyphNames.sort() + maxNameLen = max(map(len, glyphNames)) + format = "%" + repr(maxNameLen) + "s:" + format + " ;" + writer.write(format % (("ppem",) + tuple(ppems))) + writer.newline() + writer.newline() + for glyphName in glyphNames: + row = [] + for ppem in ppems: + widths = self.hdmx[ppem] + row.append(widths[glyphName]) + if ";" in glyphName: + glyphName = "\\x3b".join(glyphName.split(";")) + writer.write(format % ((glyphName,) + tuple(row))) + writer.newline() + writer.endtag("hdmxData") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name != "hdmxData": + return + content = strjoin(content) + lines = content.split(";") + topRow = lines[0].split() + assert topRow[0] == "ppem:", "illegal hdmx format" + ppems = list(map(int, topRow[1:])) + self.hdmx = hdmx = {} + for ppem in ppems: + hdmx[ppem] = {} + lines = (line.split() for line in lines[1:]) + for line in lines: + if not line: + continue + assert line[0][-1] == ":", "illegal hdmx format" + glyphName = line[0][:-1] + if "\\" in glyphName: + from fontTools.misc.textTools import safeEval + + glyphName = safeEval('"""' + glyphName + '"""') + line = list(map(int, line[1:])) + assert len(line) == len(ppems), "illegal hdmx format" + for i, ppem in enumerate(ppems): + hdmx[ppem][glyphName] = line[i] diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_e_a_d.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_e_a_d.py new file mode 100644 index 0000000000000000000000000000000000000000..dcadf68a275ce25748a50a3d257d50d849cb24fc --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_e_a_d.py @@ -0,0 +1,130 @@ +from fontTools.misc import sstruct +from fontTools.misc.fixedTools import floatToFixedToStr, strToFixedToFloat +from fontTools.misc.textTools import safeEval, num2binary, binary2num +from fontTools.misc.timeTools import ( + timestampFromString, + timestampToString, + timestampNow, +) +from fontTools.misc.timeTools import epoch_diff as mac_epoch_diff # For backward compat +from fontTools.misc.arrayTools import intRect, unionRect +from . import DefaultTable +import logging + + +log = logging.getLogger(__name__) + +headFormat = """ + > # big endian + tableVersion: 16.16F + fontRevision: 16.16F + checkSumAdjustment: I + magicNumber: I + flags: H + unitsPerEm: H + created: Q + modified: Q + xMin: h + yMin: h + xMax: h + yMax: h + macStyle: H + lowestRecPPEM: H + fontDirectionHint: h + indexToLocFormat: h + glyphDataFormat: h +""" + + +class table__h_e_a_d(DefaultTable.DefaultTable): + """Font Header table + + The ``head`` table contains a variety of font-wide information. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/head + """ + + dependencies = ["maxp", "loca", "CFF ", "CFF2"] + + def decompile(self, data, ttFont): + dummy, rest = sstruct.unpack2(headFormat, data, self) + if rest: + # this is quite illegal, but there seem to be fonts out there that do this + log.warning("extra bytes at the end of 'head' table") + assert rest == b"\0\0" + + # For timestamp fields, ignore the top four bytes. Some fonts have + # bogus values there. Since till 2038 those bytes only can be zero, + # ignore them. + # + # https://github.com/fonttools/fonttools/issues/99#issuecomment-66776810 + for stamp in "created", "modified": + value = getattr(self, stamp) + if value > 0xFFFFFFFF: + log.warning("'%s' timestamp out of range; ignoring top bytes", stamp) + value &= 0xFFFFFFFF + setattr(self, stamp, value) + if value < 0x7C259DC0: # January 1, 1970 00:00:00 + log.warning( + "'%s' timestamp seems very low; regarding as unix timestamp", stamp + ) + value += 0x7C259DC0 + setattr(self, stamp, value) + + def compile(self, ttFont): + if ttFont.recalcBBoxes: + # For TT-flavored fonts, xMin, yMin, xMax and yMax are set in table__m_a_x_p.recalc(). + if "CFF " in ttFont: + topDict = ttFont["CFF "].cff.topDictIndex[0] + self.xMin, self.yMin, self.xMax, self.yMax = intRect(topDict.FontBBox) + elif "CFF2" in ttFont: + topDict = ttFont["CFF2"].cff.topDictIndex[0] + charStrings = topDict.CharStrings + fontBBox = None + for charString in charStrings.values(): + bounds = charString.calcBounds(charStrings) + if bounds is not None: + if fontBBox is not None: + fontBBox = unionRect(fontBBox, bounds) + else: + fontBBox = bounds + if fontBBox is not None: + self.xMin, self.yMin, self.xMax, self.yMax = intRect(fontBBox) + if ttFont.recalcTimestamp: + self.modified = timestampNow() + data = sstruct.pack(headFormat, self) + return data + + def toXML(self, writer, ttFont): + writer.comment("Most of this table will be recalculated by the compiler") + writer.newline() + _, names, fixes = sstruct.getformat(headFormat) + for name in names: + value = getattr(self, name) + if name in fixes: + value = floatToFixedToStr(value, precisionBits=fixes[name]) + elif name in ("created", "modified"): + value = timestampToString(value) + elif name in ("magicNumber", "checkSumAdjustment"): + if value < 0: + value = value + 0x100000000 + value = hex(value) + if value[-1:] == "L": + value = value[:-1] + elif name in ("macStyle", "flags"): + value = num2binary(value, 16) + writer.simpletag(name, value=value) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + value = attrs["value"] + fixes = sstruct.getformat(headFormat)[2] + if name in fixes: + value = strToFixedToFloat(value, precisionBits=fixes[name]) + elif name in ("created", "modified"): + value = timestampFromString(value) + elif name in ("macStyle", "flags"): + value = binary2num(value) + else: + value = safeEval(value) + setattr(self, name, value) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_h_e_a.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_h_e_a.py new file mode 100644 index 0000000000000000000000000000000000000000..1f051e499ceb223e9260da5b7bb76ad47ae174dd --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_h_e_a.py @@ -0,0 +1,147 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import safeEval +from fontTools.misc.fixedTools import ( + ensureVersionIsLong as fi2ve, + versionToFixed as ve2fi, +) +from . import DefaultTable +import math + + +hheaFormat = """ + > # big endian + tableVersion: L + ascent: h + descent: h + lineGap: h + advanceWidthMax: H + minLeftSideBearing: h + minRightSideBearing: h + xMaxExtent: h + caretSlopeRise: h + caretSlopeRun: h + caretOffset: h + reserved0: h + reserved1: h + reserved2: h + reserved3: h + metricDataFormat: h + numberOfHMetrics: H +""" + + +class table__h_h_e_a(DefaultTable.DefaultTable): + """Horizontal Header table + + The ``hhea`` table contains information needed during horizontal + text layout. + + .. note:: + This converter class is kept in sync with the :class:`._v_h_e_a.table__v_h_e_a` + table constructor. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/hhea + """ + + # Note: Keep in sync with table__v_h_e_a + + dependencies = ["hmtx", "glyf", "CFF ", "CFF2"] + + # OpenType spec renamed these, add aliases for compatibility + @property + def ascender(self): + return self.ascent + + @ascender.setter + def ascender(self, value): + self.ascent = value + + @property + def descender(self): + return self.descent + + @descender.setter + def descender(self, value): + self.descent = value + + def decompile(self, data, ttFont): + sstruct.unpack(hheaFormat, data, self) + + def compile(self, ttFont): + if ttFont.recalcBBoxes and ( + ttFont.isLoaded("glyf") + or ttFont.isLoaded("CFF ") + or ttFont.isLoaded("CFF2") + ): + self.recalc(ttFont) + self.tableVersion = fi2ve(self.tableVersion) + return sstruct.pack(hheaFormat, self) + + def recalc(self, ttFont): + if "hmtx" not in ttFont: + return + + hmtxTable = ttFont["hmtx"] + self.advanceWidthMax = max(adv for adv, _ in hmtxTable.metrics.values()) + + boundsWidthDict = {} + if "glyf" in ttFont: + glyfTable = ttFont["glyf"] + for name in ttFont.getGlyphOrder(): + g = glyfTable[name] + if g.numberOfContours == 0: + continue + if g.numberOfContours < 0 and not hasattr(g, "xMax"): + # Composite glyph without extents set. + # Calculate those. + g.recalcBounds(glyfTable) + boundsWidthDict[name] = g.xMax - g.xMin + elif "CFF " in ttFont or "CFF2" in ttFont: + if "CFF " in ttFont: + topDict = ttFont["CFF "].cff.topDictIndex[0] + else: + topDict = ttFont["CFF2"].cff.topDictIndex[0] + charStrings = topDict.CharStrings + for name in ttFont.getGlyphOrder(): + cs = charStrings[name] + bounds = cs.calcBounds(charStrings) + if bounds is not None: + boundsWidthDict[name] = int( + math.ceil(bounds[2]) - math.floor(bounds[0]) + ) + + if boundsWidthDict: + minLeftSideBearing = float("inf") + minRightSideBearing = float("inf") + xMaxExtent = -float("inf") + for name, boundsWidth in boundsWidthDict.items(): + advanceWidth, lsb = hmtxTable[name] + rsb = advanceWidth - lsb - boundsWidth + extent = lsb + boundsWidth + minLeftSideBearing = min(minLeftSideBearing, lsb) + minRightSideBearing = min(minRightSideBearing, rsb) + xMaxExtent = max(xMaxExtent, extent) + self.minLeftSideBearing = minLeftSideBearing + self.minRightSideBearing = minRightSideBearing + self.xMaxExtent = xMaxExtent + + else: # No glyph has outlines. + self.minLeftSideBearing = 0 + self.minRightSideBearing = 0 + self.xMaxExtent = 0 + + def toXML(self, writer, ttFont): + formatstring, names, fixes = sstruct.getformat(hheaFormat) + for name in names: + value = getattr(self, name) + if name == "tableVersion": + value = fi2ve(value) + value = "0x%08x" % value + writer.simpletag(name, value=value) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "tableVersion": + setattr(self, name, ve2fi(attrs["value"])) + return + setattr(self, name, safeEval(attrs["value"])) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_m_t_x.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_m_t_x.py new file mode 100644 index 0000000000000000000000000000000000000000..0dc5077588b2bfd8892832256801af7cc83d83c0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_h_m_t_x.py @@ -0,0 +1,164 @@ +from fontTools.misc.roundTools import otRound +from fontTools import ttLib +from fontTools.misc.textTools import safeEval +from . import DefaultTable +import sys +import struct +import array +import logging + + +log = logging.getLogger(__name__) + + +class table__h_m_t_x(DefaultTable.DefaultTable): + """Horizontal Metrics table + + The ``hmtx`` table contains per-glyph metrics for the glyphs in a + ``glyf``, ``CFF ``, or ``CFF2`` table, as needed for horizontal text + layout. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/hmtx + """ + + headerTag = "hhea" + advanceName = "width" + sideBearingName = "lsb" + numberOfMetricsName = "numberOfHMetrics" + longMetricFormat = "Hh" + + def decompile(self, data, ttFont): + numGlyphs = ttFont["maxp"].numGlyphs + headerTable = ttFont.get(self.headerTag) + if headerTable is not None: + numberOfMetrics = int(getattr(headerTable, self.numberOfMetricsName)) + else: + numberOfMetrics = numGlyphs + if numberOfMetrics > numGlyphs: + log.warning( + "The %s.%s exceeds the maxp.numGlyphs" + % (self.headerTag, self.numberOfMetricsName) + ) + numberOfMetrics = numGlyphs + numberOfSideBearings = numGlyphs - numberOfMetrics + tableSize = 4 * numberOfMetrics + 2 * numberOfSideBearings + if len(data) < tableSize: + raise ttLib.TTLibError( + f"not enough '{self.tableTag}' table data: " + f"expected {tableSize} bytes, got {len(data)}" + ) + # Note: advanceWidth is unsigned, but some font editors might + # read/write as signed. We can't be sure whether it was a mistake + # or not, so we read as unsigned but also issue a warning... + metricsFmt = ">" + self.longMetricFormat * numberOfMetrics + metrics = struct.unpack(metricsFmt, data[: 4 * numberOfMetrics]) + data = data[4 * numberOfMetrics :] + sideBearings = array.array("h", data[: 2 * numberOfSideBearings]) + data = data[2 * numberOfSideBearings :] + + if sys.byteorder != "big": + sideBearings.byteswap() + if data: + log.warning("too much '%s' table data" % self.tableTag) + self.metrics = {} + glyphOrder = ttFont.getGlyphOrder() + for i in range(numberOfMetrics): + glyphName = glyphOrder[i] + advanceWidth, lsb = metrics[i * 2 : i * 2 + 2] + if advanceWidth > 32767: + log.warning( + "Glyph %r has a huge advance %s (%d); is it intentional or " + "an (invalid) negative value?", + glyphName, + self.advanceName, + advanceWidth, + ) + self.metrics[glyphName] = (advanceWidth, lsb) + lastAdvance = metrics[-2] + for i in range(numberOfSideBearings): + glyphName = glyphOrder[i + numberOfMetrics] + self.metrics[glyphName] = (lastAdvance, sideBearings[i]) + + def compile(self, ttFont): + metrics = [] + hasNegativeAdvances = False + for glyphName in ttFont.getGlyphOrder(): + advanceWidth, sideBearing = self.metrics[glyphName] + if advanceWidth < 0: + log.error( + "Glyph %r has negative advance %s" % (glyphName, self.advanceName) + ) + hasNegativeAdvances = True + metrics.append([advanceWidth, sideBearing]) + + headerTable = ttFont.get(self.headerTag) + if headerTable is not None: + lastAdvance = metrics[-1][0] + lastIndex = len(metrics) + while metrics[lastIndex - 2][0] == lastAdvance: + lastIndex -= 1 + if lastIndex <= 1: + # all advances are equal + lastIndex = 1 + break + additionalMetrics = metrics[lastIndex:] + additionalMetrics = [otRound(sb) for _, sb in additionalMetrics] + metrics = metrics[:lastIndex] + numberOfMetrics = len(metrics) + setattr(headerTable, self.numberOfMetricsName, numberOfMetrics) + else: + # no hhea/vhea, can't store numberOfMetrics; assume == numGlyphs + numberOfMetrics = ttFont["maxp"].numGlyphs + additionalMetrics = [] + + allMetrics = [] + for advance, sb in metrics: + allMetrics.extend([otRound(advance), otRound(sb)]) + metricsFmt = ">" + self.longMetricFormat * numberOfMetrics + try: + data = struct.pack(metricsFmt, *allMetrics) + except struct.error as e: + if "out of range" in str(e) and hasNegativeAdvances: + raise ttLib.TTLibError( + "'%s' table can't contain negative advance %ss" + % (self.tableTag, self.advanceName) + ) + else: + raise + additionalMetrics = array.array("h", additionalMetrics) + if sys.byteorder != "big": + additionalMetrics.byteswap() + data = data + additionalMetrics.tobytes() + return data + + def toXML(self, writer, ttFont): + names = sorted(self.metrics.keys()) + for glyphName in names: + advance, sb = self.metrics[glyphName] + writer.simpletag( + "mtx", + [ + ("name", glyphName), + (self.advanceName, advance), + (self.sideBearingName, sb), + ], + ) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if not hasattr(self, "metrics"): + self.metrics = {} + if name == "mtx": + self.metrics[attrs["name"]] = ( + safeEval(attrs[self.advanceName]), + safeEval(attrs[self.sideBearingName]), + ) + + def __delitem__(self, glyphName): + del self.metrics[glyphName] + + def __getitem__(self, glyphName): + return self.metrics[glyphName] + + def __setitem__(self, glyphName, advance_sb_pair): + self.metrics[glyphName] = tuple(advance_sb_pair) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_k_e_r_n.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_k_e_r_n.py new file mode 100644 index 0000000000000000000000000000000000000000..cdbaebedc63db51ecfb06601c9673f08f74a445b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_k_e_r_n.py @@ -0,0 +1,289 @@ +from fontTools.ttLib import getSearchRange +from fontTools.misc.textTools import safeEval, readHex +from fontTools.misc.fixedTools import fixedToFloat as fi2fl, floatToFixed as fl2fi +from . import DefaultTable +import struct +import sys +import array +import logging + + +log = logging.getLogger(__name__) + + +class table__k_e_r_n(DefaultTable.DefaultTable): + """Kerning table + + The ``kern`` table contains values that contextually adjust the inter-glyph + spacing for the glyphs in a ``glyf`` table. + + Note that similar contextual spacing adjustments can also be stored + in the "kern" feature of a ``GPOS`` table. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/kern + """ + + def getkern(self, format): + for subtable in self.kernTables: + if subtable.format == format: + return subtable + return None # not found + + def decompile(self, data, ttFont): + version, nTables = struct.unpack(">HH", data[:4]) + apple = False + if (len(data) >= 8) and (version == 1): + # AAT Apple's "new" format. Hm. + version, nTables = struct.unpack(">LL", data[:8]) + self.version = fi2fl(version, 16) + data = data[8:] + apple = True + else: + self.version = version + data = data[4:] + self.kernTables = [] + for i in range(nTables): + if self.version == 1.0: + # Apple + length, coverage, subtableFormat = struct.unpack(">LBB", data[:6]) + else: + # in OpenType spec the "version" field refers to the common + # subtable header; the actual subtable format is stored in + # the 8-15 mask bits of "coverage" field. + # This "version" is always 0 so we ignore it here + _, length, subtableFormat, coverage = struct.unpack(">HHBB", data[:6]) + if nTables == 1 and subtableFormat == 0: + # The "length" value is ignored since some fonts + # (like OpenSans and Calibri) have a subtable larger than + # its value. + (nPairs,) = struct.unpack(">H", data[6:8]) + calculated_length = (nPairs * 6) + 14 + if length != calculated_length: + log.warning( + "'kern' subtable longer than defined: " + "%d bytes instead of %d bytes" % (calculated_length, length) + ) + length = calculated_length + if subtableFormat not in kern_classes: + subtable = KernTable_format_unkown(subtableFormat) + else: + subtable = kern_classes[subtableFormat](apple) + subtable.decompile(data[:length], ttFont) + self.kernTables.append(subtable) + data = data[length:] + + def compile(self, ttFont): + if hasattr(self, "kernTables"): + nTables = len(self.kernTables) + else: + nTables = 0 + if self.version == 1.0: + # AAT Apple's "new" format. + data = struct.pack(">LL", fl2fi(self.version, 16), nTables) + else: + data = struct.pack(">HH", self.version, nTables) + if hasattr(self, "kernTables"): + for subtable in self.kernTables: + data = data + subtable.compile(ttFont) + return data + + def toXML(self, writer, ttFont): + writer.simpletag("version", value=self.version) + writer.newline() + for subtable in self.kernTables: + subtable.toXML(writer, ttFont) + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": + self.version = safeEval(attrs["value"]) + return + if name != "kernsubtable": + return + if not hasattr(self, "kernTables"): + self.kernTables = [] + format = safeEval(attrs["format"]) + if format not in kern_classes: + subtable = KernTable_format_unkown(format) + else: + apple = self.version == 1.0 + subtable = kern_classes[format](apple) + self.kernTables.append(subtable) + subtable.fromXML(name, attrs, content, ttFont) + + +class KernTable_format_0(object): + # 'version' is kept for backward compatibility + version = format = 0 + + def __init__(self, apple=False): + self.apple = apple + + def decompile(self, data, ttFont): + if not self.apple: + version, length, subtableFormat, coverage = struct.unpack(">HHBB", data[:6]) + if version != 0: + from fontTools.ttLib import TTLibError + + raise TTLibError("unsupported kern subtable version: %d" % version) + tupleIndex = None + # Should we also assert length == len(data)? + data = data[6:] + else: + length, coverage, subtableFormat, tupleIndex = struct.unpack( + ">LBBH", data[:8] + ) + data = data[8:] + assert self.format == subtableFormat, "unsupported format" + self.coverage = coverage + self.tupleIndex = tupleIndex + + self.kernTable = kernTable = {} + + nPairs, searchRange, entrySelector, rangeShift = struct.unpack( + ">HHHH", data[:8] + ) + data = data[8:] + + datas = array.array("H", data[: 6 * nPairs]) + if sys.byteorder != "big": + datas.byteswap() + it = iter(datas) + glyphOrder = ttFont.getGlyphOrder() + for k in range(nPairs): + left, right, value = next(it), next(it), next(it) + if value >= 32768: + value -= 65536 + try: + kernTable[(glyphOrder[left], glyphOrder[right])] = value + except IndexError: + # Slower, but will not throw an IndexError on an invalid + # glyph id. + kernTable[(ttFont.getGlyphName(left), ttFont.getGlyphName(right))] = ( + value + ) + if len(data) > 6 * nPairs + 4: # Ignore up to 4 bytes excess + log.warning( + "excess data in 'kern' subtable: %d bytes", len(data) - 6 * nPairs + ) + + def compile(self, ttFont): + nPairs = min(len(self.kernTable), 0xFFFF) + searchRange, entrySelector, rangeShift = getSearchRange(nPairs, 6) + searchRange &= 0xFFFF + entrySelector = min(entrySelector, 0xFFFF) + rangeShift = min(rangeShift, 0xFFFF) + data = struct.pack(">HHHH", nPairs, searchRange, entrySelector, rangeShift) + + # yeehee! (I mean, turn names into indices) + try: + reverseOrder = ttFont.getReverseGlyphMap() + kernTable = sorted( + (reverseOrder[left], reverseOrder[right], value) + for ((left, right), value) in self.kernTable.items() + ) + except KeyError: + # Slower, but will not throw KeyError on invalid glyph id. + getGlyphID = ttFont.getGlyphID + kernTable = sorted( + (getGlyphID(left), getGlyphID(right), value) + for ((left, right), value) in self.kernTable.items() + ) + + for left, right, value in kernTable: + data = data + struct.pack(">HHh", left, right, value) + + if not self.apple: + version = 0 + length = len(data) + 6 + if length >= 0x10000: + log.warning( + '"kern" subtable overflow, ' + "truncating length value while preserving pairs." + ) + length &= 0xFFFF + header = struct.pack(">HHBB", version, length, self.format, self.coverage) + else: + if self.tupleIndex is None: + # sensible default when compiling a TTX from an old fonttools + # or when inserting a Windows-style format 0 subtable into an + # Apple version=1.0 kern table + log.warning("'tupleIndex' is None; default to 0") + self.tupleIndex = 0 + length = len(data) + 8 + header = struct.pack( + ">LBBH", length, self.coverage, self.format, self.tupleIndex + ) + return header + data + + def toXML(self, writer, ttFont): + attrs = dict(coverage=self.coverage, format=self.format) + if self.apple: + if self.tupleIndex is None: + log.warning("'tupleIndex' is None; default to 0") + attrs["tupleIndex"] = 0 + else: + attrs["tupleIndex"] = self.tupleIndex + writer.begintag("kernsubtable", **attrs) + writer.newline() + items = sorted(self.kernTable.items()) + for (left, right), value in items: + writer.simpletag("pair", [("l", left), ("r", right), ("v", value)]) + writer.newline() + writer.endtag("kernsubtable") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.coverage = safeEval(attrs["coverage"]) + subtableFormat = safeEval(attrs["format"]) + if self.apple: + if "tupleIndex" in attrs: + self.tupleIndex = safeEval(attrs["tupleIndex"]) + else: + # previous fontTools versions didn't export tupleIndex + log.warning("Apple kern subtable is missing 'tupleIndex' attribute") + self.tupleIndex = None + else: + self.tupleIndex = None + assert subtableFormat == self.format, "unsupported format" + if not hasattr(self, "kernTable"): + self.kernTable = {} + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + self.kernTable[(attrs["l"], attrs["r"])] = safeEval(attrs["v"]) + + def __getitem__(self, pair): + return self.kernTable[pair] + + def __setitem__(self, pair, value): + self.kernTable[pair] = value + + def __delitem__(self, pair): + del self.kernTable[pair] + + +class KernTable_format_unkown(object): + def __init__(self, format): + self.format = format + + def decompile(self, data, ttFont): + self.data = data + + def compile(self, ttFont): + return self.data + + def toXML(self, writer, ttFont): + writer.begintag("kernsubtable", format=self.format) + writer.newline() + writer.comment("unknown 'kern' subtable format") + writer.newline() + writer.dumphex(self.data) + writer.endtag("kernsubtable") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.decompile(readHex(content), ttFont) + + +kern_classes = {0: KernTable_format_0} diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_l_c_a_r.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_l_c_a_r.py new file mode 100644 index 0000000000000000000000000000000000000000..bbb32d23d9d636361117e068fb7846d319f1b52c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_l_c_a_r.py @@ -0,0 +1,13 @@ +from .otBase import BaseTTXConverter + + +class table__l_c_a_r(BaseTTXConverter): + """Ligature Caret table + + The AAT ``lcar`` table stores division points within ligatures, which applications + can use to position carets properly between the logical parts of the ligature. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6lcar.html + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_l_o_c_a.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_l_o_c_a.py new file mode 100644 index 0000000000000000000000000000000000000000..f0b12fe57ee7ba2aafa1ea4d5aa503e14134b9af --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_l_o_c_a.py @@ -0,0 +1,70 @@ +from . import DefaultTable +import sys +import array +import logging + + +log = logging.getLogger(__name__) + + +class table__l_o_c_a(DefaultTable.DefaultTable): + """Index to Location table + + The ``loca`` table stores the offsets in the ``glyf`` table that correspond + to the descriptions of each glyph. The glyphs are references by Glyph ID. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/loca + """ + + dependencies = ["glyf"] + + def decompile(self, data, ttFont): + longFormat = ttFont["head"].indexToLocFormat + if longFormat: + format = "I" + else: + format = "H" + locations = array.array(format) + locations.frombytes(data) + if sys.byteorder != "big": + locations.byteswap() + if not longFormat: + locations = array.array("I", (2 * l for l in locations)) + if len(locations) < (ttFont["maxp"].numGlyphs + 1): + log.warning( + "corrupt 'loca' table, or wrong numGlyphs in 'maxp': %d %d", + len(locations) - 1, + ttFont["maxp"].numGlyphs, + ) + self.locations = locations + + def compile(self, ttFont): + try: + max_location = max(self.locations) + except AttributeError: + self.set([]) + max_location = 0 + if max_location < 0x20000 and all(l % 2 == 0 for l in self.locations): + locations = array.array("H") + for location in self.locations: + locations.append(location // 2) + ttFont["head"].indexToLocFormat = 0 + else: + locations = array.array("I", self.locations) + ttFont["head"].indexToLocFormat = 1 + if sys.byteorder != "big": + locations.byteswap() + return locations.tobytes() + + def set(self, locations): + self.locations = array.array("I", locations) + + def toXML(self, writer, ttFont): + writer.comment("The 'loca' table will be calculated by the compiler") + writer.newline() + + def __getitem__(self, index): + return self.locations[index] + + def __len__(self): + return len(self.locations) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_l_t_a_g.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_l_t_a_g.py new file mode 100644 index 0000000000000000000000000000000000000000..fc47cd648bc7b6be6b4ca3011c5125cc2e1d897e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_l_t_a_g.py @@ -0,0 +1,72 @@ +from fontTools.misc.textTools import bytesjoin, tobytes, safeEval +from . import DefaultTable +import struct + +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6ltag.html + + +class table__l_t_a_g(DefaultTable.DefaultTable): + """Language Tag table + + The AAT ``ltag`` table contains mappings between the numeric codes used + in the language field of the ``name`` table and IETF language tags. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6ltag.html + """ + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.version, self.flags = 1, 0 + self.tags = [] + + def addTag(self, tag): + """Add 'tag' to the list of langauge tags if not already there. + + Returns the integer index of 'tag' in the list of all tags. + """ + try: + return self.tags.index(tag) + except ValueError: + self.tags.append(tag) + return len(self.tags) - 1 + + def decompile(self, data, ttFont): + self.version, self.flags, numTags = struct.unpack(">LLL", data[:12]) + assert self.version == 1 + self.tags = [] + for i in range(numTags): + pos = 12 + i * 4 + offset, length = struct.unpack(">HH", data[pos : pos + 4]) + tag = data[offset : offset + length].decode("ascii") + self.tags.append(tag) + + def compile(self, ttFont): + dataList = [struct.pack(">LLL", self.version, self.flags, len(self.tags))] + stringPool = "" + for tag in self.tags: + offset = stringPool.find(tag) + if offset < 0: + offset = len(stringPool) + stringPool = stringPool + tag + offset = offset + 12 + len(self.tags) * 4 + dataList.append(struct.pack(">HH", offset, len(tag))) + dataList.append(tobytes(stringPool)) + return bytesjoin(dataList) + + def toXML(self, writer, ttFont): + writer.simpletag("version", value=self.version) + writer.newline() + writer.simpletag("flags", value=self.flags) + writer.newline() + for tag in self.tags: + writer.simpletag("LanguageTag", tag=tag) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if not hasattr(self, "tags"): + self.tags = [] + if name == "LanguageTag": + self.tags.append(attrs["tag"]) + elif "value" in attrs: + value = safeEval(attrs["value"]) + setattr(self, name, value) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_a_x_p.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_a_x_p.py new file mode 100644 index 0000000000000000000000000000000000000000..c1576a578f22c375ad4c5c4534b38162329aadde --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_a_x_p.py @@ -0,0 +1,147 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import safeEval +from . import DefaultTable + +maxpFormat_0_5 = """ + > # big endian + tableVersion: i + numGlyphs: H +""" + +maxpFormat_1_0_add = """ + > # big endian + maxPoints: H + maxContours: H + maxCompositePoints: H + maxCompositeContours: H + maxZones: H + maxTwilightPoints: H + maxStorage: H + maxFunctionDefs: H + maxInstructionDefs: H + maxStackElements: H + maxSizeOfInstructions: H + maxComponentElements: H + maxComponentDepth: H +""" + + +class table__m_a_x_p(DefaultTable.DefaultTable): + """Maximum Profile table + + The ``maxp`` table contains the memory requirements for the data in + the font. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/maxp + """ + + dependencies = ["glyf"] + + def decompile(self, data, ttFont): + dummy, data = sstruct.unpack2(maxpFormat_0_5, data, self) + self.numGlyphs = int(self.numGlyphs) + if self.tableVersion != 0x00005000: + dummy, data = sstruct.unpack2(maxpFormat_1_0_add, data, self) + assert len(data) == 0 + + def compile(self, ttFont): + if "glyf" in ttFont: + if ttFont.isLoaded("glyf") and ttFont.recalcBBoxes: + self.recalc(ttFont) + else: + pass # CFF + self.numGlyphs = len(ttFont.getGlyphOrder()) + if self.tableVersion != 0x00005000: + self.tableVersion = 0x00010000 + data = sstruct.pack(maxpFormat_0_5, self) + if self.tableVersion == 0x00010000: + data = data + sstruct.pack(maxpFormat_1_0_add, self) + return data + + def recalc(self, ttFont): + """Recalculate the font bounding box, and most other maxp values except + for the TT instructions values. Also recalculate the value of bit 1 + of the flags field and the font bounding box of the 'head' table. + """ + glyfTable = ttFont["glyf"] + hmtxTable = ttFont["hmtx"] + headTable = ttFont["head"] + self.numGlyphs = len(glyfTable) + INFINITY = 100000 + xMin = +INFINITY + yMin = +INFINITY + xMax = -INFINITY + yMax = -INFINITY + maxPoints = 0 + maxContours = 0 + maxCompositePoints = 0 + maxCompositeContours = 0 + maxComponentElements = 0 + maxComponentDepth = 0 + allXMinIsLsb = 1 + for glyphName in ttFont.getGlyphOrder(): + g = glyfTable[glyphName] + if g.numberOfContours: + if hmtxTable[glyphName][1] != g.xMin: + allXMinIsLsb = 0 + xMin = min(xMin, g.xMin) + yMin = min(yMin, g.yMin) + xMax = max(xMax, g.xMax) + yMax = max(yMax, g.yMax) + if g.numberOfContours > 0: + nPoints, nContours = g.getMaxpValues() + maxPoints = max(maxPoints, nPoints) + maxContours = max(maxContours, nContours) + elif g.isComposite(): + nPoints, nContours, componentDepth = g.getCompositeMaxpValues( + glyfTable + ) + maxCompositePoints = max(maxCompositePoints, nPoints) + maxCompositeContours = max(maxCompositeContours, nContours) + maxComponentElements = max(maxComponentElements, len(g.components)) + maxComponentDepth = max(maxComponentDepth, componentDepth) + if xMin == +INFINITY: + headTable.xMin = 0 + headTable.yMin = 0 + headTable.xMax = 0 + headTable.yMax = 0 + else: + headTable.xMin = xMin + headTable.yMin = yMin + headTable.xMax = xMax + headTable.yMax = yMax + self.maxPoints = maxPoints + self.maxContours = maxContours + self.maxCompositePoints = maxCompositePoints + self.maxCompositeContours = maxCompositeContours + self.maxComponentElements = maxComponentElements + self.maxComponentDepth = maxComponentDepth + if allXMinIsLsb: + headTable.flags = headTable.flags | 0x2 + else: + headTable.flags = headTable.flags & ~0x2 + + def testrepr(self): + items = sorted(self.__dict__.items()) + print(". . . . . . . . .") + for combo in items: + print(" %s: %s" % combo) + print(". . . . . . . . .") + + def toXML(self, writer, ttFont): + if self.tableVersion != 0x00005000: + writer.comment("Most of this table will be recalculated by the compiler") + writer.newline() + formatstring, names, fixes = sstruct.getformat(maxpFormat_0_5) + if self.tableVersion != 0x00005000: + formatstring, names_1_0, fixes = sstruct.getformat(maxpFormat_1_0_add) + names = {**names, **names_1_0} + for name in names: + value = getattr(self, name) + if name == "tableVersion": + value = hex(value) + writer.simpletag(name, value=value) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + setattr(self, name, safeEval(attrs["value"])) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_e_t_a.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_e_t_a.py new file mode 100644 index 0000000000000000000000000000000000000000..c5cea1b993e8e15968065f9a50b27784fcf4eb8c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_e_t_a.py @@ -0,0 +1,112 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import bytesjoin, strjoin, readHex +from fontTools.ttLib import TTLibError +from . import DefaultTable + +# Apple's documentation of 'meta': +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6meta.html + +META_HEADER_FORMAT = """ + > # big endian + version: L + flags: L + dataOffset: L + numDataMaps: L +""" + + +DATA_MAP_FORMAT = """ + > # big endian + tag: 4s + dataOffset: L + dataLength: L +""" + + +class table__m_e_t_a(DefaultTable.DefaultTable): + """Metadata table + + The ``meta`` table contains various metadata values for the font. Each + category of metadata in the table is identified by a four-character tag. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/meta + """ + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.data = {} + + def decompile(self, data, ttFont): + headerSize = sstruct.calcsize(META_HEADER_FORMAT) + header = sstruct.unpack(META_HEADER_FORMAT, data[0:headerSize]) + if header["version"] != 1: + raise TTLibError("unsupported 'meta' version %d" % header["version"]) + dataMapSize = sstruct.calcsize(DATA_MAP_FORMAT) + for i in range(header["numDataMaps"]): + dataMapOffset = headerSize + i * dataMapSize + dataMap = sstruct.unpack( + DATA_MAP_FORMAT, data[dataMapOffset : dataMapOffset + dataMapSize] + ) + tag = dataMap["tag"] + offset = dataMap["dataOffset"] + self.data[tag] = data[offset : offset + dataMap["dataLength"]] + if tag in ["dlng", "slng"]: + self.data[tag] = self.data[tag].decode("utf-8") + + def compile(self, ttFont): + keys = sorted(self.data.keys()) + headerSize = sstruct.calcsize(META_HEADER_FORMAT) + dataOffset = headerSize + len(keys) * sstruct.calcsize(DATA_MAP_FORMAT) + header = sstruct.pack( + META_HEADER_FORMAT, + { + "version": 1, + "flags": 0, + "dataOffset": dataOffset, + "numDataMaps": len(keys), + }, + ) + dataMaps = [] + dataBlocks = [] + for tag in keys: + if tag in ["dlng", "slng"]: + data = self.data[tag].encode("utf-8") + else: + data = self.data[tag] + dataMaps.append( + sstruct.pack( + DATA_MAP_FORMAT, + {"tag": tag, "dataOffset": dataOffset, "dataLength": len(data)}, + ) + ) + dataBlocks.append(data) + dataOffset += len(data) + return bytesjoin([header] + dataMaps + dataBlocks) + + def toXML(self, writer, ttFont): + for tag in sorted(self.data.keys()): + if tag in ["dlng", "slng"]: + writer.begintag("text", tag=tag) + writer.newline() + writer.write(self.data[tag]) + writer.newline() + writer.endtag("text") + writer.newline() + else: + writer.begintag("hexdata", tag=tag) + writer.newline() + data = self.data[tag] + if min(data) >= 0x20 and max(data) <= 0x7E: + writer.comment("ascii: " + data.decode("ascii")) + writer.newline() + writer.dumphex(data) + writer.endtag("hexdata") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "hexdata": + self.data[attrs["tag"]] = readHex(content) + elif name == "text" and attrs["tag"] in ["dlng", "slng"]: + self.data[attrs["tag"]] = strjoin(content).strip() + else: + raise TTLibError("can't handle '%s' element" % name) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_o_r_t.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_o_r_t.py new file mode 100644 index 0000000000000000000000000000000000000000..5e5fa77ce73fff4906d92610afd9914eb2b61a3c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_o_r_t.py @@ -0,0 +1,14 @@ +from .otBase import BaseTTXConverter + + +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6mort.html +class table__m_o_r_t(BaseTTXConverter): + """The AAT ``mort`` table contains glyph transformations used for script shaping and + for various other optional smart features. + + Note: ``mort`` has been deprecated in favor of the newer ``morx`` table. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6mort.html + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_o_r_x.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_o_r_x.py new file mode 100644 index 0000000000000000000000000000000000000000..dfb3abcc5e35e426bbd670417c429dd4b4ce0962 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_m_o_r_x.py @@ -0,0 +1,15 @@ +from .otBase import BaseTTXConverter + + +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6morx.html +class table__m_o_r_x(BaseTTXConverter): + """The AAT ``morx`` table contains glyph transformations used for script shaping and + for various other optional smart features, akin to ``GSUB`` and ``GPOS`` features + in OpenType Layout. + + Note: ``morx`` is a replacement for the now deprecated ``mort`` table. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6morx.html + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_n_a_m_e.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_n_a_m_e.py new file mode 100644 index 0000000000000000000000000000000000000000..b5913639bcf1823bbde584d1f2bdbe56a56b88d9 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_n_a_m_e.py @@ -0,0 +1,1242 @@ +# -*- coding: utf-8 -*- +from __future__ import annotations + +from fontTools.misc import sstruct +from fontTools.misc.textTools import ( + bytechr, + byteord, + bytesjoin, + strjoin, + tobytes, + tostr, + safeEval, +) +from fontTools.misc.encodingTools import getEncoding +from fontTools.ttLib import newTable +from fontTools.ttLib.ttVisitor import TTVisitor +from fontTools import ttLib +import fontTools.ttLib.tables.otTables as otTables +from fontTools.ttLib.tables import C_P_A_L_ +from . import DefaultTable +import struct +import logging + + +log = logging.getLogger(__name__) + +nameRecordFormat = """ + > # big endian + platformID: H + platEncID: H + langID: H + nameID: H + length: H + offset: H +""" + +nameRecordSize = sstruct.calcsize(nameRecordFormat) + + +class table__n_a_m_e(DefaultTable.DefaultTable): + """Naming table + + The ``name`` table is used to store a variety of strings that can be + associated with user-facing font information. Records in the ``name`` + table can be tagged with language tags to support multilingual naming + and can support platform-specific character-encoding variants. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/name + """ + + dependencies = ["ltag"] + + def __init__(self, tag=None): + super().__init__(tag) + self.names = [] + + def decompile(self, data, ttFont): + format, n, stringOffset = struct.unpack(b">HHH", data[:6]) + expectedStringOffset = 6 + n * nameRecordSize + if stringOffset != expectedStringOffset: + log.error( + "'name' table stringOffset incorrect. Expected: %s; Actual: %s", + expectedStringOffset, + stringOffset, + ) + stringData = data[stringOffset:] + data = data[6:] + self.names: list[NameRecord] = [] + for i in range(n): + if len(data) < 12: + log.error("skipping malformed name record #%d", i) + continue + name, data = sstruct.unpack2(nameRecordFormat, data, NameRecord()) + name.string = stringData[name.offset : name.offset + name.length] + if name.offset + name.length > len(stringData): + log.error("skipping malformed name record #%d", i) + continue + assert len(name.string) == name.length + # if (name.platEncID, name.platformID) in ((0, 0), (1, 3)): + # if len(name.string) % 2: + # print "2-byte string doesn't have even length!" + # print name.__dict__ + del name.offset, name.length + self.names.append(name) + + def compile(self, ttFont): + names = self.names + names.sort() # sort according to the spec; see NameRecord.__lt__() + stringData = b"" + format = 0 + n = len(names) + stringOffset = 6 + n * sstruct.calcsize(nameRecordFormat) + data = struct.pack(b">HHH", format, n, stringOffset) + lastoffset = 0 + done = {} # remember the data so we can reuse the "pointers" + for name in names: + string = name.toBytes() + if string in done: + name.offset, name.length = done[string] + else: + name.offset, name.length = done[string] = len(stringData), len(string) + stringData = bytesjoin([stringData, string]) + data = data + sstruct.pack(nameRecordFormat, name) + return data + stringData + + def toXML(self, writer, ttFont): + for name in self.names: + name.toXML(writer, ttFont) + + def fromXML(self, name, attrs, content, ttFont): + if name != "namerecord": + return # ignore unknown tags + name = NameRecord() + self.names.append(name) + name.fromXML(name, attrs, content, ttFont) + + def getName( + self, nameID: int, platformID: int, platEncID: int, langID: int | None = None + ) -> "NameRecord | None": + for namerecord in self.names: + if ( + namerecord.nameID == nameID + and namerecord.platformID == platformID + and namerecord.platEncID == platEncID + ): + if langID is None or namerecord.langID == langID: + return namerecord + return None # not found + + def getDebugName(self, nameID: int) -> str | None: + englishName: str | None = None + someName: str | None = None + for name in self.names: + if name.nameID != nameID: + continue + try: + unistr = name.toUnicode() + except UnicodeDecodeError: + continue + + someName = unistr + if (name.platformID, name.langID) in ((1, 0), (3, 0x409)): + englishName = unistr + break + if englishName: + return englishName + elif someName: + return someName + else: + return None + + def getFirstDebugName(self, nameIDs): + for nameID in nameIDs: + name = self.getDebugName(nameID) + if name is not None: + return name + return None + + def getBestFamilyName(self): + # 21 = WWS Family Name + # 16 = Typographic Family Name + # 1 = Family Name + return self.getFirstDebugName((21, 16, 1)) + + def getBestSubFamilyName(self): + # 22 = WWS SubFamily Name + # 17 = Typographic SubFamily Name + # 2 = SubFamily Name + return self.getFirstDebugName((22, 17, 2)) + + def getBestFullName(self): + # 4 = Full Name + # 6 = PostScript Name + for nameIDs in ((21, 22), (16, 17), (1, 2), (4,), (6,)): + if len(nameIDs) == 2: + name_fam = self.getDebugName(nameIDs[0]) + name_subfam = self.getDebugName(nameIDs[1]) + if None in [name_fam, name_subfam]: + continue # if any is None, skip + name = f"{name_fam} {name_subfam}" + if name_subfam.lower() == "regular": + name = f"{name_fam}" + return name + else: + name = self.getDebugName(nameIDs[0]) + if name is not None: + return name + return None + + def setName(self, string, nameID, platformID, platEncID, langID): + """Set the 'string' for the name record identified by 'nameID', 'platformID', + 'platEncID' and 'langID'. If a record with that nameID doesn't exist, create it + and append to the name table. + + 'string' can be of type `str` (`unicode` in PY2) or `bytes`. In the latter case, + it is assumed to be already encoded with the correct plaform-specific encoding + identified by the (platformID, platEncID, langID) triplet. A warning is issued + to prevent unexpected results. + """ + if not isinstance(string, str): + if isinstance(string, bytes): + log.warning( + "name string is bytes, ensure it's correctly encoded: %r", string + ) + else: + raise TypeError( + "expected unicode or bytes, found %s: %r" + % (type(string).__name__, string) + ) + namerecord = self.getName(nameID, platformID, platEncID, langID) + if namerecord: + namerecord.string = string + else: + self.names.append(makeName(string, nameID, platformID, platEncID, langID)) + + def removeNames(self, nameID=None, platformID=None, platEncID=None, langID=None): + """Remove any name records identified by the given combination of 'nameID', + 'platformID', 'platEncID' and 'langID'. + """ + args = { + argName: argValue + for argName, argValue in ( + ("nameID", nameID), + ("platformID", platformID), + ("platEncID", platEncID), + ("langID", langID), + ) + if argValue is not None + } + if not args: + # no arguments, nothing to do + return + self.names = [ + rec + for rec in self.names + if any( + argValue != getattr(rec, argName) for argName, argValue in args.items() + ) + ] + + @staticmethod + def removeUnusedNames(ttFont): + """Remove any name records which are not in NameID range 0-255 and not utilized + within the font itself.""" + visitor = NameRecordVisitor() + visitor.visit(ttFont) + toDelete = set() + for record in ttFont["name"].names: + # Name IDs 26 to 255, inclusive, are reserved for future standard names. + # https://learn.microsoft.com/en-us/typography/opentype/spec/name#name-ids + if record.nameID < 256: + continue + if record.nameID not in visitor.seen: + toDelete.add(record.nameID) + + for nameID in toDelete: + ttFont["name"].removeNames(nameID) + return toDelete + + def _findUnusedNameID(self, minNameID=256): + """Finds an unused name id. + + The nameID is assigned in the range between 'minNameID' and 32767 (inclusive), + following the last nameID in the name table. + """ + names = self.names + nameID = 1 + max([n.nameID for n in names] + [minNameID - 1]) + if nameID > 32767: + raise ValueError("nameID must be less than 32768") + return nameID + + def findMultilingualName( + self, names, windows=True, mac=True, minNameID=0, ttFont=None + ): + """Return the name ID of an existing multilingual name that + matches the 'names' dictionary, or None if not found. + + 'names' is a dictionary with the name in multiple languages, + such as {'en': 'Pale', 'de': 'Blaß', 'de-CH': 'Blass'}. + The keys can be arbitrary IETF BCP 47 language codes; + the values are Unicode strings. + + If 'windows' is True, the returned name ID is guaranteed + exist for all requested languages for platformID=3 and + platEncID=1. + If 'mac' is True, the returned name ID is guaranteed to exist + for all requested languages for platformID=1 and platEncID=0. + + The returned name ID will not be less than the 'minNameID' + argument. + """ + # Gather the set of requested + # (string, platformID, platEncID, langID) + # tuples + reqNameSet = set() + for lang, name in sorted(names.items()): + if windows: + windowsName = _makeWindowsName(name, None, lang) + if windowsName is not None: + reqNameSet.add( + ( + windowsName.string, + windowsName.platformID, + windowsName.platEncID, + windowsName.langID, + ) + ) + if mac: + macName = _makeMacName(name, None, lang, ttFont) + if macName is not None: + reqNameSet.add( + ( + macName.string, + macName.platformID, + macName.platEncID, + macName.langID, + ) + ) + + # Collect matching name IDs + matchingNames = dict() + for name in self.names: + try: + key = (name.toUnicode(), name.platformID, name.platEncID, name.langID) + except UnicodeDecodeError: + continue + if key in reqNameSet and name.nameID >= minNameID: + nameSet = matchingNames.setdefault(name.nameID, set()) + nameSet.add(key) + + # Return the first name ID that defines all requested strings + for nameID, nameSet in sorted(matchingNames.items()): + if nameSet == reqNameSet: + return nameID + + return None # not found + + def addMultilingualName( + self, names, ttFont=None, nameID=None, windows=True, mac=True, minNameID=0 + ): + """Add a multilingual name, returning its name ID + + 'names' is a dictionary with the name in multiple languages, + such as {'en': 'Pale', 'de': 'Blaß', 'de-CH': 'Blass'}. + The keys can be arbitrary IETF BCP 47 language codes; + the values are Unicode strings. + + 'ttFont' is the TTFont to which the names are added, or None. + If present, the font's 'ltag' table can get populated + to store exotic language codes, which allows encoding + names that otherwise cannot get encoded at all. + + 'nameID' is the name ID to be used, or None to let the library + find an existing set of name records that match, or pick an + unused name ID. + + If 'windows' is True, a platformID=3 name record will be added. + If 'mac' is True, a platformID=1 name record will be added. + + If the 'nameID' argument is None, the created nameID will not + be less than the 'minNameID' argument. + """ + if nameID is None: + # Reuse nameID if possible + nameID = self.findMultilingualName( + names, windows=windows, mac=mac, minNameID=minNameID, ttFont=ttFont + ) + if nameID is not None: + return nameID + nameID = self._findUnusedNameID() + # TODO: Should minimize BCP 47 language codes. + # https://github.com/fonttools/fonttools/issues/930 + for lang, name in sorted(names.items()): + if windows: + windowsName = _makeWindowsName(name, nameID, lang) + if windowsName is not None: + self.names.append(windowsName) + else: + # We cannot not make a Windows name: make sure we add a + # Mac name as a fallback. This can happen for exotic + # BCP47 language tags that have no Windows language code. + mac = True + if mac: + macName = _makeMacName(name, nameID, lang, ttFont) + if macName is not None: + self.names.append(macName) + return nameID + + def addName(self, string, platforms=((1, 0, 0), (3, 1, 0x409)), minNameID=255): + """Add a new name record containing 'string' for each (platformID, platEncID, + langID) tuple specified in the 'platforms' list. + + The nameID is assigned in the range between 'minNameID'+1 and 32767 (inclusive), + following the last nameID in the name table. + If no 'platforms' are specified, two English name records are added, one for the + Macintosh (platformID=0), and one for the Windows platform (3). + + The 'string' must be a Unicode string, so it can be encoded with different, + platform-specific encodings. + + Return the new nameID. + """ + assert ( + len(platforms) > 0 + ), "'platforms' must contain at least one (platformID, platEncID, langID) tuple" + if not isinstance(string, str): + raise TypeError( + "expected str, found %s: %r" % (type(string).__name__, string) + ) + nameID = self._findUnusedNameID(minNameID + 1) + for platformID, platEncID, langID in platforms: + self.names.append(makeName(string, nameID, platformID, platEncID, langID)) + return nameID + + +def makeName(string, nameID, platformID, platEncID, langID): + name = NameRecord() + name.string, name.nameID, name.platformID, name.platEncID, name.langID = ( + string, + nameID, + platformID, + platEncID, + langID, + ) + return name + + +def _makeWindowsName(name, nameID, language): + """Create a NameRecord for the Microsoft Windows platform + + 'language' is an arbitrary IETF BCP 47 language identifier such + as 'en', 'de-CH', 'de-AT-1901', or 'fa-Latn'. If Microsoft Windows + does not support the desired language, the result will be None. + Future versions of fonttools might return a NameRecord for the + OpenType 'name' table format 1, but this is not implemented yet. + """ + langID = _WINDOWS_LANGUAGE_CODES.get(language.lower()) + if langID is not None: + return makeName(name, nameID, 3, 1, langID) + else: + log.warning( + "cannot add Windows name in language %s " + "because fonttools does not yet support " + "name table format 1" % language + ) + return None + + +def _makeMacName(name, nameID, language, font=None): + """Create a NameRecord for Apple platforms + + 'language' is an arbitrary IETF BCP 47 language identifier such + as 'en', 'de-CH', 'de-AT-1901', or 'fa-Latn'. When possible, we + create a Macintosh NameRecord that is understood by old applications + (platform ID 1 and an old-style Macintosh language enum). If this + is not possible, we create a Unicode NameRecord (platform ID 0) + whose language points to the font’s 'ltag' table. The latter + can encode any string in any language, but legacy applications + might not recognize the format (in which case they will ignore + those names). + + 'font' should be the TTFont for which you want to create a name. + If 'font' is None, we only return NameRecords for legacy Macintosh; + in that case, the result will be None for names that need to + be encoded with an 'ltag' table. + + See the section “The language identifier” in Apple’s specification: + https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6name.html + """ + macLang = _MAC_LANGUAGE_CODES.get(language.lower()) + macScript = _MAC_LANGUAGE_TO_SCRIPT.get(macLang) + if macLang is not None and macScript is not None: + encoding = getEncoding(1, macScript, macLang, default="ascii") + # Check if we can actually encode this name. If we can't, + # for example because we have no support for the legacy + # encoding, or because the name string contains Unicode + # characters that the legacy encoding cannot represent, + # we fall back to encoding the name in Unicode and put + # the language tag into the ltag table. + try: + _ = tobytes(name, encoding, errors="strict") + return makeName(name, nameID, 1, macScript, macLang) + except UnicodeEncodeError: + pass + if font is not None: + ltag = font.tables.get("ltag") + if ltag is None: + ltag = font["ltag"] = newTable("ltag") + # 0 = Unicode; 4 = “Unicode 2.0 or later semantics (non-BMP characters allowed)” + # “The preferred platform-specific code for Unicode would be 3 or 4.” + # https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6name.html + return makeName(name, nameID, 0, 4, ltag.addTag(language)) + else: + log.warning( + "cannot store language %s into 'ltag' table " + "without having access to the TTFont object" % language + ) + return None + + +class NameRecord(object): + def getEncoding(self, default="ascii"): + """Returns the Python encoding name for this name entry based on its platformID, + platEncID, and langID. If encoding for these values is not known, by default + 'ascii' is returned. That can be overriden by passing a value to the default + argument. + """ + return getEncoding(self.platformID, self.platEncID, self.langID, default) + + def encodingIsUnicodeCompatible(self): + return self.getEncoding(None) in ["utf_16_be", "ucs2be", "ascii", "latin1"] + + def __str__(self): + return self.toStr(errors="backslashreplace") + + def isUnicode(self): + return self.platformID == 0 or ( + self.platformID == 3 and self.platEncID in [0, 1, 10] + ) + + def toUnicode(self, errors: str = "strict") -> str: + """ + If self.string is a Unicode string, return it; otherwise try decoding the + bytes in self.string to a Unicode string using the encoding of this + entry as returned by self.getEncoding(); Note that self.getEncoding() + returns 'ascii' if the encoding is unknown to the library. + + Certain heuristics are performed to recover data from bytes that are + ill-formed in the chosen encoding, or that otherwise look misencoded + (mostly around bad UTF-16BE encoded bytes, or bytes that look like UTF-16BE + but marked otherwise). If the bytes are ill-formed and the heuristics fail, + the error is handled according to the errors parameter to this function, which is + passed to the underlying decode() function; by default it throws a + UnicodeDecodeError exception. + + Note: The mentioned heuristics mean that roundtripping a font to XML and back + to binary might recover some misencoded data whereas just loading the font + and saving it back will not change them. + """ + + def isascii(b: int) -> bool: + return (b >= 0x20 and b <= 0x7E) or b in [0x09, 0x0A, 0x0D] + + encoding = self.getEncoding() + string = self.string + + if ( + isinstance(string, bytes) + and encoding == "utf_16_be" + and len(string) % 2 == 1 + ): + # Recover badly encoded UTF-16 strings that have an odd number of bytes: + # - If the last byte is zero, drop it. Otherwise, + # - If all the odd bytes are zero and all the even bytes are ASCII, + # prepend one zero byte. Otherwise, + # - If first byte is zero and all other bytes are ASCII, insert zero + # bytes between consecutive ASCII bytes. + # + # (Yes, I've seen all of these in the wild... sigh) + if byteord(string[-1]) == 0: + string = string[:-1] + elif all( + byteord(b) == 0 if i % 2 else isascii(byteord(b)) + for i, b in enumerate(string) + ): + string = b"\0" + string + elif byteord(string[0]) == 0 and all( + isascii(byteord(b)) for b in string[1:] + ): + string = bytesjoin(b"\0" + bytechr(byteord(b)) for b in string[1:]) + + string = tostr(string, encoding=encoding, errors=errors) + + # If decoded strings still looks like UTF-16BE, it suggests a double-encoding. + # Fix it up. + if all( + ord(c) == 0 if i % 2 == 0 else isascii(ord(c)) for i, c in enumerate(string) + ): + # If string claims to be Mac encoding, but looks like UTF-16BE with ASCII text, + # narrow it down. + string = "".join(c for c in string[1::2]) + + return string + + def toBytes(self, errors="strict"): + """If self.string is a bytes object, return it; otherwise try encoding + the Unicode string in self.string to bytes using the encoding of this + entry as returned by self.getEncoding(); Note that self.getEncoding() + returns 'ascii' if the encoding is unknown to the library. + + If the Unicode string cannot be encoded to bytes in the chosen encoding, + the error is handled according to the errors parameter to this function, + which is passed to the underlying encode() function; by default it throws a + UnicodeEncodeError exception. + """ + return tobytes(self.string, encoding=self.getEncoding(), errors=errors) + + toStr = toUnicode + + def toXML(self, writer, ttFont): + try: + unistr = self.toUnicode() + except UnicodeDecodeError: + unistr = None + attrs = [ + ("nameID", self.nameID), + ("platformID", self.platformID), + ("platEncID", self.platEncID), + ("langID", hex(self.langID)), + ] + + if unistr is None or not self.encodingIsUnicodeCompatible(): + attrs.append(("unicode", unistr is not None)) + + writer.begintag("namerecord", attrs) + writer.newline() + if unistr is not None: + writer.write(unistr) + else: + writer.write8bit(self.string) + writer.newline() + writer.endtag("namerecord") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.nameID = safeEval(attrs["nameID"]) + self.platformID = safeEval(attrs["platformID"]) + self.platEncID = safeEval(attrs["platEncID"]) + self.langID = safeEval(attrs["langID"]) + s = strjoin(content).strip() + encoding = self.getEncoding() + if self.encodingIsUnicodeCompatible() or safeEval( + attrs.get("unicode", "False") + ): + self.string = s.encode(encoding) + else: + # This is the inverse of write8bit... + self.string = s.encode("latin1") + + def __lt__(self, other): + if type(self) != type(other): + return NotImplemented + + try: + selfTuple = ( + self.platformID, + self.platEncID, + self.langID, + self.nameID, + ) + otherTuple = ( + other.platformID, + other.platEncID, + other.langID, + other.nameID, + ) + except AttributeError: + # This can only happen for + # 1) an object that is not a NameRecord, or + # 2) an unlikely incomplete NameRecord object which has not been + # fully populated + return NotImplemented + + try: + # Include the actual NameRecord string in the comparison tuples + selfTuple = selfTuple + (self.toBytes(),) + otherTuple = otherTuple + (other.toBytes(),) + except UnicodeEncodeError as e: + # toBytes caused an encoding error in either of the two, so content + # to sorting based on IDs only + log.error("NameRecord sorting failed to encode: %s" % e) + + # Implemented so that list.sort() sorts according to the spec by using + # the order of the tuple items and their comparison + return selfTuple < otherTuple + + def __repr__(self): + return "<NameRecord NameID=%d; PlatformID=%d; LanguageID=%d>" % ( + self.nameID, + self.platformID, + self.langID, + ) + + +# Windows language ID → IETF BCP-47 language tag +# +# While Microsoft indicates a region/country for all its language +# IDs, we follow Unicode practice by omitting “most likely subtags” +# as per Unicode CLDR. For example, English is simply “en” and not +# “en-Latn” because according to Unicode, the default script +# for English is Latin. +# +# http://www.unicode.org/cldr/charts/latest/supplemental/likely_subtags.html +# http://www.iana.org/assignments/language-subtag-registry/language-subtag-registry +_WINDOWS_LANGUAGES = { + 0x0436: "af", + 0x041C: "sq", + 0x0484: "gsw", + 0x045E: "am", + 0x1401: "ar-DZ", + 0x3C01: "ar-BH", + 0x0C01: "ar", + 0x0801: "ar-IQ", + 0x2C01: "ar-JO", + 0x3401: "ar-KW", + 0x3001: "ar-LB", + 0x1001: "ar-LY", + 0x1801: "ary", + 0x2001: "ar-OM", + 0x4001: "ar-QA", + 0x0401: "ar-SA", + 0x2801: "ar-SY", + 0x1C01: "aeb", + 0x3801: "ar-AE", + 0x2401: "ar-YE", + 0x042B: "hy", + 0x044D: "as", + 0x082C: "az-Cyrl", + 0x042C: "az", + 0x046D: "ba", + 0x042D: "eu", + 0x0423: "be", + 0x0845: "bn", + 0x0445: "bn-IN", + 0x201A: "bs-Cyrl", + 0x141A: "bs", + 0x047E: "br", + 0x0402: "bg", + 0x0403: "ca", + 0x0C04: "zh-HK", + 0x1404: "zh-MO", + 0x0804: "zh", + 0x1004: "zh-SG", + 0x0404: "zh-TW", + 0x0483: "co", + 0x041A: "hr", + 0x101A: "hr-BA", + 0x0405: "cs", + 0x0406: "da", + 0x048C: "prs", + 0x0465: "dv", + 0x0813: "nl-BE", + 0x0413: "nl", + 0x0C09: "en-AU", + 0x2809: "en-BZ", + 0x1009: "en-CA", + 0x2409: "en-029", + 0x4009: "en-IN", + 0x1809: "en-IE", + 0x2009: "en-JM", + 0x4409: "en-MY", + 0x1409: "en-NZ", + 0x3409: "en-PH", + 0x4809: "en-SG", + 0x1C09: "en-ZA", + 0x2C09: "en-TT", + 0x0809: "en-GB", + 0x0409: "en", + 0x3009: "en-ZW", + 0x0425: "et", + 0x0438: "fo", + 0x0464: "fil", + 0x040B: "fi", + 0x080C: "fr-BE", + 0x0C0C: "fr-CA", + 0x040C: "fr", + 0x140C: "fr-LU", + 0x180C: "fr-MC", + 0x100C: "fr-CH", + 0x0462: "fy", + 0x0456: "gl", + 0x0437: "ka", + 0x0C07: "de-AT", + 0x0407: "de", + 0x1407: "de-LI", + 0x1007: "de-LU", + 0x0807: "de-CH", + 0x0408: "el", + 0x046F: "kl", + 0x0447: "gu", + 0x0468: "ha", + 0x040D: "he", + 0x0439: "hi", + 0x040E: "hu", + 0x040F: "is", + 0x0470: "ig", + 0x0421: "id", + 0x045D: "iu", + 0x085D: "iu-Latn", + 0x083C: "ga", + 0x0434: "xh", + 0x0435: "zu", + 0x0410: "it", + 0x0810: "it-CH", + 0x0411: "ja", + 0x044B: "kn", + 0x043F: "kk", + 0x0453: "km", + 0x0486: "quc", + 0x0487: "rw", + 0x0441: "sw", + 0x0457: "kok", + 0x0412: "ko", + 0x0440: "ky", + 0x0454: "lo", + 0x0426: "lv", + 0x0427: "lt", + 0x082E: "dsb", + 0x046E: "lb", + 0x042F: "mk", + 0x083E: "ms-BN", + 0x043E: "ms", + 0x044C: "ml", + 0x043A: "mt", + 0x0481: "mi", + 0x047A: "arn", + 0x044E: "mr", + 0x047C: "moh", + 0x0450: "mn", + 0x0850: "mn-CN", + 0x0461: "ne", + 0x0414: "nb", + 0x0814: "nn", + 0x0482: "oc", + 0x0448: "or", + 0x0463: "ps", + 0x0415: "pl", + 0x0416: "pt", + 0x0816: "pt-PT", + 0x0446: "pa", + 0x046B: "qu-BO", + 0x086B: "qu-EC", + 0x0C6B: "qu", + 0x0418: "ro", + 0x0417: "rm", + 0x0419: "ru", + 0x243B: "smn", + 0x103B: "smj-NO", + 0x143B: "smj", + 0x0C3B: "se-FI", + 0x043B: "se", + 0x083B: "se-SE", + 0x203B: "sms", + 0x183B: "sma-NO", + 0x1C3B: "sms", + 0x044F: "sa", + 0x1C1A: "sr-Cyrl-BA", + 0x0C1A: "sr", + 0x181A: "sr-Latn-BA", + 0x081A: "sr-Latn", + 0x046C: "nso", + 0x0432: "tn", + 0x045B: "si", + 0x041B: "sk", + 0x0424: "sl", + 0x2C0A: "es-AR", + 0x400A: "es-BO", + 0x340A: "es-CL", + 0x240A: "es-CO", + 0x140A: "es-CR", + 0x1C0A: "es-DO", + 0x300A: "es-EC", + 0x440A: "es-SV", + 0x100A: "es-GT", + 0x480A: "es-HN", + 0x080A: "es-MX", + 0x4C0A: "es-NI", + 0x180A: "es-PA", + 0x3C0A: "es-PY", + 0x280A: "es-PE", + 0x500A: "es-PR", + # Microsoft has defined two different language codes for + # “Spanish with modern sorting” and “Spanish with traditional + # sorting”. This makes sense for collation APIs, and it would be + # possible to express this in BCP 47 language tags via Unicode + # extensions (eg., “es-u-co-trad” is “Spanish with traditional + # sorting”). However, for storing names in fonts, this distinction + # does not make sense, so we use “es” in both cases. + 0x0C0A: "es", + 0x040A: "es", + 0x540A: "es-US", + 0x380A: "es-UY", + 0x200A: "es-VE", + 0x081D: "sv-FI", + 0x041D: "sv", + 0x045A: "syr", + 0x0428: "tg", + 0x085F: "tzm", + 0x0449: "ta", + 0x0444: "tt", + 0x044A: "te", + 0x041E: "th", + 0x0451: "bo", + 0x041F: "tr", + 0x0442: "tk", + 0x0480: "ug", + 0x0422: "uk", + 0x042E: "hsb", + 0x0420: "ur", + 0x0843: "uz-Cyrl", + 0x0443: "uz", + 0x042A: "vi", + 0x0452: "cy", + 0x0488: "wo", + 0x0485: "sah", + 0x0478: "ii", + 0x046A: "yo", +} + + +_MAC_LANGUAGES = { + 0: "en", + 1: "fr", + 2: "de", + 3: "it", + 4: "nl", + 5: "sv", + 6: "es", + 7: "da", + 8: "pt", + 9: "no", + 10: "he", + 11: "ja", + 12: "ar", + 13: "fi", + 14: "el", + 15: "is", + 16: "mt", + 17: "tr", + 18: "hr", + 19: "zh-Hant", + 20: "ur", + 21: "hi", + 22: "th", + 23: "ko", + 24: "lt", + 25: "pl", + 26: "hu", + 27: "es", + 28: "lv", + 29: "se", + 30: "fo", + 31: "fa", + 32: "ru", + 33: "zh", + 34: "nl-BE", + 35: "ga", + 36: "sq", + 37: "ro", + 38: "cz", + 39: "sk", + 40: "sl", + 41: "yi", + 42: "sr", + 43: "mk", + 44: "bg", + 45: "uk", + 46: "be", + 47: "uz", + 48: "kk", + 49: "az-Cyrl", + 50: "az-Arab", + 51: "hy", + 52: "ka", + 53: "mo", + 54: "ky", + 55: "tg", + 56: "tk", + 57: "mn-CN", + 58: "mn", + 59: "ps", + 60: "ks", + 61: "ku", + 62: "sd", + 63: "bo", + 64: "ne", + 65: "sa", + 66: "mr", + 67: "bn", + 68: "as", + 69: "gu", + 70: "pa", + 71: "or", + 72: "ml", + 73: "kn", + 74: "ta", + 75: "te", + 76: "si", + 77: "my", + 78: "km", + 79: "lo", + 80: "vi", + 81: "id", + 82: "tl", + 83: "ms", + 84: "ms-Arab", + 85: "am", + 86: "ti", + 87: "om", + 88: "so", + 89: "sw", + 90: "rw", + 91: "rn", + 92: "ny", + 93: "mg", + 94: "eo", + 128: "cy", + 129: "eu", + 130: "ca", + 131: "la", + 132: "qu", + 133: "gn", + 134: "ay", + 135: "tt", + 136: "ug", + 137: "dz", + 138: "jv", + 139: "su", + 140: "gl", + 141: "af", + 142: "br", + 143: "iu", + 144: "gd", + 145: "gv", + 146: "ga", + 147: "to", + 148: "el-polyton", + 149: "kl", + 150: "az", + 151: "nn", +} + + +_WINDOWS_LANGUAGE_CODES = { + lang.lower(): code for code, lang in _WINDOWS_LANGUAGES.items() +} +_MAC_LANGUAGE_CODES = {lang.lower(): code for code, lang in _MAC_LANGUAGES.items()} + + +# MacOS language ID → MacOS script ID +# +# Note that the script ID is not sufficient to determine what encoding +# to use in TrueType files. For some languages, MacOS used a modification +# of a mainstream script. For example, an Icelandic name would be stored +# with smRoman in the TrueType naming table, but the actual encoding +# is a special Icelandic version of the normal Macintosh Roman encoding. +# As another example, Inuktitut uses an 8-bit encoding for Canadian Aboriginal +# Syllables but MacOS had run out of available script codes, so this was +# done as a (pretty radical) “modification” of Ethiopic. +# +# http://unicode.org/Public/MAPPINGS/VENDORS/APPLE/Readme.txt +_MAC_LANGUAGE_TO_SCRIPT = { + 0: 0, # langEnglish → smRoman + 1: 0, # langFrench → smRoman + 2: 0, # langGerman → smRoman + 3: 0, # langItalian → smRoman + 4: 0, # langDutch → smRoman + 5: 0, # langSwedish → smRoman + 6: 0, # langSpanish → smRoman + 7: 0, # langDanish → smRoman + 8: 0, # langPortuguese → smRoman + 9: 0, # langNorwegian → smRoman + 10: 5, # langHebrew → smHebrew + 11: 1, # langJapanese → smJapanese + 12: 4, # langArabic → smArabic + 13: 0, # langFinnish → smRoman + 14: 6, # langGreek → smGreek + 15: 0, # langIcelandic → smRoman (modified) + 16: 0, # langMaltese → smRoman + 17: 0, # langTurkish → smRoman (modified) + 18: 0, # langCroatian → smRoman (modified) + 19: 2, # langTradChinese → smTradChinese + 20: 4, # langUrdu → smArabic + 21: 9, # langHindi → smDevanagari + 22: 21, # langThai → smThai + 23: 3, # langKorean → smKorean + 24: 29, # langLithuanian → smCentralEuroRoman + 25: 29, # langPolish → smCentralEuroRoman + 26: 29, # langHungarian → smCentralEuroRoman + 27: 29, # langEstonian → smCentralEuroRoman + 28: 29, # langLatvian → smCentralEuroRoman + 29: 0, # langSami → smRoman + 30: 0, # langFaroese → smRoman (modified) + 31: 4, # langFarsi → smArabic (modified) + 32: 7, # langRussian → smCyrillic + 33: 25, # langSimpChinese → smSimpChinese + 34: 0, # langFlemish → smRoman + 35: 0, # langIrishGaelic → smRoman (modified) + 36: 0, # langAlbanian → smRoman + 37: 0, # langRomanian → smRoman (modified) + 38: 29, # langCzech → smCentralEuroRoman + 39: 29, # langSlovak → smCentralEuroRoman + 40: 0, # langSlovenian → smRoman (modified) + 41: 5, # langYiddish → smHebrew + 42: 7, # langSerbian → smCyrillic + 43: 7, # langMacedonian → smCyrillic + 44: 7, # langBulgarian → smCyrillic + 45: 7, # langUkrainian → smCyrillic (modified) + 46: 7, # langByelorussian → smCyrillic + 47: 7, # langUzbek → smCyrillic + 48: 7, # langKazakh → smCyrillic + 49: 7, # langAzerbaijani → smCyrillic + 50: 4, # langAzerbaijanAr → smArabic + 51: 24, # langArmenian → smArmenian + 52: 23, # langGeorgian → smGeorgian + 53: 7, # langMoldavian → smCyrillic + 54: 7, # langKirghiz → smCyrillic + 55: 7, # langTajiki → smCyrillic + 56: 7, # langTurkmen → smCyrillic + 57: 27, # langMongolian → smMongolian + 58: 7, # langMongolianCyr → smCyrillic + 59: 4, # langPashto → smArabic + 60: 4, # langKurdish → smArabic + 61: 4, # langKashmiri → smArabic + 62: 4, # langSindhi → smArabic + 63: 26, # langTibetan → smTibetan + 64: 9, # langNepali → smDevanagari + 65: 9, # langSanskrit → smDevanagari + 66: 9, # langMarathi → smDevanagari + 67: 13, # langBengali → smBengali + 68: 13, # langAssamese → smBengali + 69: 11, # langGujarati → smGujarati + 70: 10, # langPunjabi → smGurmukhi + 71: 12, # langOriya → smOriya + 72: 17, # langMalayalam → smMalayalam + 73: 16, # langKannada → smKannada + 74: 14, # langTamil → smTamil + 75: 15, # langTelugu → smTelugu + 76: 18, # langSinhalese → smSinhalese + 77: 19, # langBurmese → smBurmese + 78: 20, # langKhmer → smKhmer + 79: 22, # langLao → smLao + 80: 30, # langVietnamese → smVietnamese + 81: 0, # langIndonesian → smRoman + 82: 0, # langTagalog → smRoman + 83: 0, # langMalayRoman → smRoman + 84: 4, # langMalayArabic → smArabic + 85: 28, # langAmharic → smEthiopic + 86: 28, # langTigrinya → smEthiopic + 87: 28, # langOromo → smEthiopic + 88: 0, # langSomali → smRoman + 89: 0, # langSwahili → smRoman + 90: 0, # langKinyarwanda → smRoman + 91: 0, # langRundi → smRoman + 92: 0, # langNyanja → smRoman + 93: 0, # langMalagasy → smRoman + 94: 0, # langEsperanto → smRoman + 128: 0, # langWelsh → smRoman (modified) + 129: 0, # langBasque → smRoman + 130: 0, # langCatalan → smRoman + 131: 0, # langLatin → smRoman + 132: 0, # langQuechua → smRoman + 133: 0, # langGuarani → smRoman + 134: 0, # langAymara → smRoman + 135: 7, # langTatar → smCyrillic + 136: 4, # langUighur → smArabic + 137: 26, # langDzongkha → smTibetan + 138: 0, # langJavaneseRom → smRoman + 139: 0, # langSundaneseRom → smRoman + 140: 0, # langGalician → smRoman + 141: 0, # langAfrikaans → smRoman + 142: 0, # langBreton → smRoman (modified) + 143: 28, # langInuktitut → smEthiopic (modified) + 144: 0, # langScottishGaelic → smRoman (modified) + 145: 0, # langManxGaelic → smRoman (modified) + 146: 0, # langIrishGaelicScript → smRoman (modified) + 147: 0, # langTongan → smRoman + 148: 6, # langGreekAncient → smRoman + 149: 0, # langGreenlandic → smRoman + 150: 0, # langAzerbaijanRoman → smRoman + 151: 0, # langNynorsk → smRoman +} + + +class NameRecordVisitor(TTVisitor): + # Font tables that have NameIDs we need to collect. + TABLES = ("GSUB", "GPOS", "fvar", "CPAL", "STAT") + + def __init__(self): + self.seen = set() + + +@NameRecordVisitor.register_attrs( + ( + (otTables.FeatureParamsSize, ("SubfamilyNameID",)), + (otTables.FeatureParamsStylisticSet, ("UINameID",)), + (otTables.STAT, ("ElidedFallbackNameID",)), + (otTables.AxisRecord, ("AxisNameID",)), + (otTables.AxisValue, ("ValueNameID",)), + (otTables.FeatureName, ("FeatureNameID",)), + (otTables.Setting, ("SettingNameID",)), + ) +) +def visit(visitor, obj, attr, value): + visitor.seen.add(value) + + +@NameRecordVisitor.register(otTables.FeatureParamsCharacterVariants) +def visit(visitor, obj): + for attr in ("FeatUILabelNameID", "FeatUITooltipTextNameID", "SampleTextNameID"): + value = getattr(obj, attr) + visitor.seen.add(value) + # also include the sequence of UI strings for individual variants, if any + if obj.FirstParamUILabelNameID == 0 or obj.NumNamedParameters == 0: + return + visitor.seen.update( + range( + obj.FirstParamUILabelNameID, + obj.FirstParamUILabelNameID + obj.NumNamedParameters, + ) + ) + + +@NameRecordVisitor.register(ttLib.getTableClass("fvar")) +def visit(visitor, obj): + for inst in obj.instances: + if inst.postscriptNameID != 0xFFFF: + visitor.seen.add(inst.postscriptNameID) + visitor.seen.add(inst.subfamilyNameID) + + for axis in obj.axes: + visitor.seen.add(axis.axisNameID) + + +@NameRecordVisitor.register(ttLib.getTableClass("CPAL")) +def visit(visitor, obj): + if obj.version == 1: + visitor.seen.update(obj.paletteLabels) + visitor.seen.update(obj.paletteEntryLabels) + + +@NameRecordVisitor.register(ttLib.TTFont) +def visit(visitor, font, *args, **kwargs): + if hasattr(visitor, "font"): + return False + + visitor.font = font + for tag in visitor.TABLES: + if tag in font: + visitor.visit(font[tag], *args, **kwargs) + del visitor.font + return False diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_o_p_b_d.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_o_p_b_d.py new file mode 100644 index 0000000000000000000000000000000000000000..b4c5f568290e20563992de91c7a0f5e6928652bf --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_o_p_b_d.py @@ -0,0 +1,14 @@ +from .otBase import BaseTTXConverter + + +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6opbd.html +class table__o_p_b_d(BaseTTXConverter): + """Optical Bounds table + + The AAT ``opbd`` table contains optical boundary points for glyphs, which + applications can use for the visual alignment of lines of text. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6opbd.html + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_p_o_s_t.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_p_o_s_t.py new file mode 100644 index 0000000000000000000000000000000000000000..1564c89347b4b7934b978c8f4ae383851b6d18d5 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_p_o_s_t.py @@ -0,0 +1,319 @@ +from fontTools import ttLib +from fontTools.ttLib.standardGlyphOrder import standardGlyphOrder +from fontTools.misc import sstruct +from fontTools.misc.textTools import bytechr, byteord, tobytes, tostr, safeEval, readHex +from . import DefaultTable +import sys +import struct +import array +import logging + +log = logging.getLogger(__name__) + +postFormat = """ + > + formatType: 16.16F + italicAngle: 16.16F # italic angle in degrees + underlinePosition: h + underlineThickness: h + isFixedPitch: L + minMemType42: L # minimum memory if TrueType font is downloaded + maxMemType42: L # maximum memory if TrueType font is downloaded + minMemType1: L # minimum memory if Type1 font is downloaded + maxMemType1: L # maximum memory if Type1 font is downloaded +""" + +postFormatSize = sstruct.calcsize(postFormat) + + +class table__p_o_s_t(DefaultTable.DefaultTable): + """PostScript table + + The ``post`` table contains information needed to use the font on + PostScript printers, including the PostScript names of glyphs and + data that was stored in the ``FontInfo`` dictionary for Type 1 fonts. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/post + """ + + def decompile(self, data, ttFont): + sstruct.unpack(postFormat, data[:postFormatSize], self) + data = data[postFormatSize:] + if self.formatType == 1.0: + self.decode_format_1_0(data, ttFont) + elif self.formatType == 2.0: + self.decode_format_2_0(data, ttFont) + elif self.formatType == 3.0: + self.decode_format_3_0(data, ttFont) + elif self.formatType == 4.0: + self.decode_format_4_0(data, ttFont) + else: + # supported format + raise ttLib.TTLibError( + "'post' table format %f not supported" % self.formatType + ) + + def compile(self, ttFont): + data = sstruct.pack(postFormat, self) + if self.formatType == 1.0: + pass # we're done + elif self.formatType == 2.0: + data = data + self.encode_format_2_0(ttFont) + elif self.formatType == 3.0: + pass # we're done + elif self.formatType == 4.0: + data = data + self.encode_format_4_0(ttFont) + else: + # supported format + raise ttLib.TTLibError( + "'post' table format %f not supported" % self.formatType + ) + return data + + def getGlyphOrder(self): + """This function will get called by a ttLib.TTFont instance. + Do not call this function yourself, use TTFont().getGlyphOrder() + or its relatives instead! + """ + if not hasattr(self, "glyphOrder"): + raise ttLib.TTLibError("illegal use of getGlyphOrder()") + glyphOrder = self.glyphOrder + del self.glyphOrder + return glyphOrder + + def decode_format_1_0(self, data, ttFont): + self.glyphOrder = standardGlyphOrder[: ttFont["maxp"].numGlyphs] + + def decode_format_2_0(self, data, ttFont): + (numGlyphs,) = struct.unpack(">H", data[:2]) + numGlyphs = int(numGlyphs) + if numGlyphs > ttFont["maxp"].numGlyphs: + # Assume the numGlyphs field is bogus, so sync with maxp. + # I've seen this in one font, and if the assumption is + # wrong elsewhere, well, so be it: it's hard enough to + # work around _one_ non-conforming post format... + numGlyphs = ttFont["maxp"].numGlyphs + data = data[2:] + indices = array.array("H") + indices.frombytes(data[: 2 * numGlyphs]) + if sys.byteorder != "big": + indices.byteswap() + data = data[2 * numGlyphs :] + maxIndex = max(indices) + self.extraNames = extraNames = unpackPStrings(data, maxIndex - 257) + self.glyphOrder = glyphOrder = [""] * int(ttFont["maxp"].numGlyphs) + for glyphID in range(numGlyphs): + index = indices[glyphID] + if index > 257: + try: + name = extraNames[index - 258] + except IndexError: + name = "" + else: + # fetch names from standard list + name = standardGlyphOrder[index] + glyphOrder[glyphID] = name + self.build_psNameMapping(ttFont) + + def build_psNameMapping(self, ttFont): + mapping = {} + allNames = {} + glyphOrderNames = set(self.glyphOrder) + for i in range(ttFont["maxp"].numGlyphs): + glyphName = psName = self.glyphOrder[i] + if glyphName == "": + glyphName = "glyph%.5d" % i + + if glyphName in allNames: + # make up a new glyphName that's unique + n = allNames[glyphName] + # check if the glyph name exists in the glyph order + while f"{glyphName}.{n}" in glyphOrderNames: + n += 1 + allNames[glyphName] = n + 1 + glyphName = f"{glyphName}.{n}" + + allNames[glyphName] = 1 + if glyphName != psName: + self.glyphOrder[i] = glyphName + mapping[glyphName] = psName + + self.mapping = mapping + + def decode_format_3_0(self, data, ttFont): + # Setting self.glyphOrder to None will cause the TTFont object + # try and construct glyph names from a Unicode cmap table. + self.glyphOrder = None + + def decode_format_4_0(self, data, ttFont): + from fontTools import agl + + numGlyphs = ttFont["maxp"].numGlyphs + indices = array.array("H") + indices.frombytes(data) + if sys.byteorder != "big": + indices.byteswap() + # In some older fonts, the size of the post table doesn't match + # the number of glyphs. Sometimes it's bigger, sometimes smaller. + self.glyphOrder = glyphOrder = [""] * int(numGlyphs) + for i in range(min(len(indices), numGlyphs)): + if indices[i] == 0xFFFF: + self.glyphOrder[i] = "" + elif indices[i] in agl.UV2AGL: + self.glyphOrder[i] = agl.UV2AGL[indices[i]] + else: + self.glyphOrder[i] = "uni%04X" % indices[i] + self.build_psNameMapping(ttFont) + + def encode_format_2_0(self, ttFont): + numGlyphs = ttFont["maxp"].numGlyphs + glyphOrder = ttFont.getGlyphOrder() + assert len(glyphOrder) == numGlyphs + indices = array.array("H") + extraDict = {} + extraNames = self.extraNames = [ + n for n in self.extraNames if n not in standardGlyphOrder + ] + for i, name in enumerate(extraNames): + extraDict[name] = i + for glyphName in glyphOrder: + if glyphName in self.mapping: + psName = self.mapping[glyphName] + else: + psName = glyphName + if psName in extraDict: + index = 258 + extraDict[psName] + elif psName in standardGlyphOrder: + index = standardGlyphOrder.index(psName) + else: + index = 258 + len(extraNames) + extraDict[psName] = len(extraNames) + extraNames.append(psName) + indices.append(index) + if sys.byteorder != "big": + indices.byteswap() + return ( + struct.pack(">H", numGlyphs) + indices.tobytes() + packPStrings(extraNames) + ) + + def encode_format_4_0(self, ttFont): + from fontTools import agl + + numGlyphs = ttFont["maxp"].numGlyphs + glyphOrder = ttFont.getGlyphOrder() + assert len(glyphOrder) == numGlyphs + indices = array.array("H") + for glyphID in glyphOrder: + glyphID = glyphID.split("#")[0] + if glyphID in agl.AGL2UV: + indices.append(agl.AGL2UV[glyphID]) + elif len(glyphID) == 7 and glyphID[:3] == "uni": + indices.append(int(glyphID[3:], 16)) + else: + indices.append(0xFFFF) + if sys.byteorder != "big": + indices.byteswap() + return indices.tobytes() + + def toXML(self, writer, ttFont): + formatstring, names, fixes = sstruct.getformat(postFormat) + for name in names: + value = getattr(self, name) + writer.simpletag(name, value=value) + writer.newline() + if hasattr(self, "mapping"): + writer.begintag("psNames") + writer.newline() + writer.comment( + "This file uses unique glyph names based on the information\n" + "found in the 'post' table. Since these names might not be unique,\n" + "we have to invent artificial names in case of clashes. In order to\n" + "be able to retain the original information, we need a name to\n" + "ps name mapping for those cases where they differ. That's what\n" + "you see below.\n" + ) + writer.newline() + items = sorted(self.mapping.items()) + for name, psName in items: + writer.simpletag("psName", name=name, psName=psName) + writer.newline() + writer.endtag("psNames") + writer.newline() + if hasattr(self, "extraNames"): + writer.begintag("extraNames") + writer.newline() + writer.comment( + "following are the name that are not taken from the standard Mac glyph order" + ) + writer.newline() + for name in self.extraNames: + writer.simpletag("psName", name=name) + writer.newline() + writer.endtag("extraNames") + writer.newline() + if hasattr(self, "data"): + writer.begintag("hexdata") + writer.newline() + writer.dumphex(self.data) + writer.endtag("hexdata") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name not in ("psNames", "extraNames", "hexdata"): + setattr(self, name, safeEval(attrs["value"])) + elif name == "psNames": + self.mapping = {} + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name == "psName": + self.mapping[attrs["name"]] = attrs["psName"] + elif name == "extraNames": + self.extraNames = [] + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + if name == "psName": + self.extraNames.append(attrs["name"]) + else: + self.data = readHex(content) + + +def unpackPStrings(data, n): + # extract n Pascal strings from data. + # if there is not enough data, use "" + + strings = [] + index = 0 + dataLen = len(data) + + for _ in range(n): + if dataLen <= index: + length = 0 + else: + length = byteord(data[index]) + index += 1 + + if dataLen <= index + length - 1: + name = "" + else: + name = tostr(data[index : index + length], encoding="latin1") + strings.append(name) + index += length + + if index < dataLen: + log.warning("%d extra bytes in post.stringData array", dataLen - index) + + elif dataLen < index: + log.warning("not enough data in post.stringData array") + + return strings + + +def packPStrings(strings): + data = b"" + for s in strings: + data = data + bytechr(len(s)) + tobytes(s, encoding="latin1") + return data diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_p_r_e_p.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_p_r_e_p.py new file mode 100644 index 0000000000000000000000000000000000000000..cf4a5dd24f27ab08082cf9976bc2f5a9e1a967a5 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_p_r_e_p.py @@ -0,0 +1,16 @@ +from fontTools import ttLib + +superclass = ttLib.getTableClass("fpgm") + + +class table__p_r_e_p(superclass): + """Control Value Program table + + The ``prep`` table contains TrueType instructions that can makee font-wide + alterations to the Control Value Table. It may potentially be executed + before any glyph is processed. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/prep + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_p_r_o_p.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_p_r_o_p.py new file mode 100644 index 0000000000000000000000000000000000000000..1d7de8098de797268cd14607939e78e0c5c7bf5a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_p_r_o_p.py @@ -0,0 +1,12 @@ +from .otBase import BaseTTXConverter + + +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6prop.html +class table__p_r_o_p(BaseTTXConverter): + """The AAT ``prop`` table can store a variety of per-glyph properties, such as + Unicode directionality or whether glyphs are non-spacing marks. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6prop.html + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_s_b_i_x.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_s_b_i_x.py new file mode 100644 index 0000000000000000000000000000000000000000..a282ea9a60d1a1e5f58747db6079d7c493504295 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_s_b_i_x.py @@ -0,0 +1,129 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import safeEval, num2binary, binary2num +from . import DefaultTable +from .sbixStrike import Strike + + +sbixHeaderFormat = """ + > + version: H # Version number (set to 1) + flags: H # The only two bits used in the flags field are bits 0 + # and 1. For historical reasons, bit 0 must always be 1. + # Bit 1 is a sbixDrawOutlines flag and is interpreted as + # follows: + # 0: Draw only 'sbix' bitmaps + # 1: Draw both 'sbix' bitmaps and outlines, in that + # order + numStrikes: L # Number of bitmap strikes to follow +""" +sbixHeaderFormatSize = sstruct.calcsize(sbixHeaderFormat) + + +sbixStrikeOffsetFormat = """ + > + strikeOffset: L # Offset from begining of table to data for the + # individual strike +""" +sbixStrikeOffsetFormatSize = sstruct.calcsize(sbixStrikeOffsetFormat) + + +class table__s_b_i_x(DefaultTable.DefaultTable): + """Standard Bitmap Graphics table + + The ``sbix`` table stores bitmap image data in standard graphics formats + like JPEG, PNG, or TIFF. The glyphs for which the ``sbix`` table provides + data are indexed by Glyph ID. For each such glyph, the ``sbix`` table can + hold different data for different sizes, called "strikes." + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/sbix + """ + + def __init__(self, tag=None): + DefaultTable.DefaultTable.__init__(self, tag) + self.version = 1 + self.flags = 1 + self.numStrikes = 0 + self.strikes = {} + self.strikeOffsets = [] + + def decompile(self, data, ttFont): + # read table header + sstruct.unpack(sbixHeaderFormat, data[:sbixHeaderFormatSize], self) + # collect offsets to individual strikes in self.strikeOffsets + for i in range(self.numStrikes): + current_offset = sbixHeaderFormatSize + i * sbixStrikeOffsetFormatSize + offset_entry = sbixStrikeOffset() + sstruct.unpack( + sbixStrikeOffsetFormat, + data[current_offset : current_offset + sbixStrikeOffsetFormatSize], + offset_entry, + ) + self.strikeOffsets.append(offset_entry.strikeOffset) + + # decompile Strikes + for i in range(self.numStrikes - 1, -1, -1): + current_strike = Strike(rawdata=data[self.strikeOffsets[i] :]) + data = data[: self.strikeOffsets[i]] + current_strike.decompile(ttFont) + # print " Strike length: %xh" % len(bitmapSetData) + # print "Number of Glyph entries:", len(current_strike.glyphs) + if current_strike.ppem in self.strikes: + from fontTools import ttLib + + raise ttLib.TTLibError("Pixel 'ppem' must be unique for each Strike") + self.strikes[current_strike.ppem] = current_strike + + # after the glyph data records have been extracted, we don't need the offsets anymore + del self.strikeOffsets + del self.numStrikes + + def compile(self, ttFont): + sbixData = b"" + self.numStrikes = len(self.strikes) + sbixHeader = sstruct.pack(sbixHeaderFormat, self) + + # calculate offset to start of first strike + setOffset = sbixHeaderFormatSize + sbixStrikeOffsetFormatSize * self.numStrikes + + for si in sorted(self.strikes.keys()): + current_strike = self.strikes[si] + current_strike.compile(ttFont) + # append offset to this strike to table header + current_strike.strikeOffset = setOffset + sbixHeader += sstruct.pack(sbixStrikeOffsetFormat, current_strike) + setOffset += len(current_strike.data) + sbixData += current_strike.data + + return sbixHeader + sbixData + + def toXML(self, xmlWriter, ttFont): + xmlWriter.simpletag("version", value=self.version) + xmlWriter.newline() + xmlWriter.simpletag("flags", value=num2binary(self.flags, 16)) + xmlWriter.newline() + for i in sorted(self.strikes.keys()): + self.strikes[i].toXML(xmlWriter, ttFont) + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": + setattr(self, name, safeEval(attrs["value"])) + elif name == "flags": + setattr(self, name, binary2num(attrs["value"])) + elif name == "strike": + current_strike = Strike() + for element in content: + if isinstance(element, tuple): + name, attrs, content = element + current_strike.fromXML(name, attrs, content, ttFont) + self.strikes[current_strike.ppem] = current_strike + else: + from fontTools import ttLib + + raise ttLib.TTLibError("can't handle '%s' element" % name) + + +# Helper classes + + +class sbixStrikeOffset(object): + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_t_r_a_k.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_t_r_a_k.py new file mode 100644 index 0000000000000000000000000000000000000000..c49509e6fd12b025c626bf1738fae64c1c905a20 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_t_r_a_k.py @@ -0,0 +1,330 @@ +from fontTools.misc import sstruct +from fontTools.misc.fixedTools import ( + fixedToFloat as fi2fl, + floatToFixed as fl2fi, + floatToFixedToStr as fl2str, + strToFixedToFloat as str2fl, +) +from fontTools.misc.textTools import bytesjoin, safeEval +from fontTools.ttLib import TTLibError +from . import DefaultTable +import struct +from collections.abc import MutableMapping + + +# Apple's documentation of 'trak': +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6trak.html + +TRAK_HEADER_FORMAT = """ + > # big endian + version: 16.16F + format: H + horizOffset: H + vertOffset: H + reserved: H +""" + +TRAK_HEADER_FORMAT_SIZE = sstruct.calcsize(TRAK_HEADER_FORMAT) + + +TRACK_DATA_FORMAT = """ + > # big endian + nTracks: H + nSizes: H + sizeTableOffset: L +""" + +TRACK_DATA_FORMAT_SIZE = sstruct.calcsize(TRACK_DATA_FORMAT) + + +TRACK_TABLE_ENTRY_FORMAT = """ + > # big endian + track: 16.16F + nameIndex: H + offset: H +""" + +TRACK_TABLE_ENTRY_FORMAT_SIZE = sstruct.calcsize(TRACK_TABLE_ENTRY_FORMAT) + + +# size values are actually '16.16F' fixed-point values, but here I do the +# fixedToFloat conversion manually instead of relying on sstruct +SIZE_VALUE_FORMAT = ">l" +SIZE_VALUE_FORMAT_SIZE = struct.calcsize(SIZE_VALUE_FORMAT) + +# per-Size values are in 'FUnits', i.e. 16-bit signed integers +PER_SIZE_VALUE_FORMAT = ">h" +PER_SIZE_VALUE_FORMAT_SIZE = struct.calcsize(PER_SIZE_VALUE_FORMAT) + + +class table__t_r_a_k(DefaultTable.DefaultTable): + """The AAT ``trak`` table can store per-size adjustments to each glyph's + sidebearings to make when tracking is enabled, which applications can + use to provide more visually balanced line spacing. + + See also https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6trak.html + """ + + def compile(self, ttFont): + dataList = [] + offset = TRAK_HEADER_FORMAT_SIZE + for direction in ("horiz", "vert"): + trackData = getattr(self, direction + "Data", TrackData()) + offsetName = direction + "Offset" + # set offset to 0 if None or empty + if not trackData: + setattr(self, offsetName, 0) + continue + # TrackData table format must be longword aligned + alignedOffset = (offset + 3) & ~3 + padding, offset = b"\x00" * (alignedOffset - offset), alignedOffset + setattr(self, offsetName, offset) + + data = trackData.compile(offset) + offset += len(data) + dataList.append(padding + data) + + self.reserved = 0 + tableData = bytesjoin([sstruct.pack(TRAK_HEADER_FORMAT, self)] + dataList) + return tableData + + def decompile(self, data, ttFont): + sstruct.unpack(TRAK_HEADER_FORMAT, data[:TRAK_HEADER_FORMAT_SIZE], self) + for direction in ("horiz", "vert"): + trackData = TrackData() + offset = getattr(self, direction + "Offset") + if offset != 0: + trackData.decompile(data, offset) + setattr(self, direction + "Data", trackData) + + def toXML(self, writer, ttFont): + writer.simpletag("version", value=self.version) + writer.newline() + writer.simpletag("format", value=self.format) + writer.newline() + for direction in ("horiz", "vert"): + dataName = direction + "Data" + writer.begintag(dataName) + writer.newline() + trackData = getattr(self, dataName, TrackData()) + trackData.toXML(writer, ttFont) + writer.endtag(dataName) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "version": + self.version = safeEval(attrs["value"]) + elif name == "format": + self.format = safeEval(attrs["value"]) + elif name in ("horizData", "vertData"): + trackData = TrackData() + setattr(self, name, trackData) + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content_ = element + trackData.fromXML(name, attrs, content_, ttFont) + + +class TrackData(MutableMapping): + def __init__(self, initialdata={}): + self._map = dict(initialdata) + + def compile(self, offset): + nTracks = len(self) + sizes = self.sizes() + nSizes = len(sizes) + + # offset to the start of the size subtable + offset += TRACK_DATA_FORMAT_SIZE + TRACK_TABLE_ENTRY_FORMAT_SIZE * nTracks + trackDataHeader = sstruct.pack( + TRACK_DATA_FORMAT, + {"nTracks": nTracks, "nSizes": nSizes, "sizeTableOffset": offset}, + ) + + entryDataList = [] + perSizeDataList = [] + # offset to per-size tracking values + offset += SIZE_VALUE_FORMAT_SIZE * nSizes + # sort track table entries by track value + for track, entry in sorted(self.items()): + assert entry.nameIndex is not None + entry.track = track + entry.offset = offset + entryDataList += [sstruct.pack(TRACK_TABLE_ENTRY_FORMAT, entry)] + # sort per-size values by size + for size, value in sorted(entry.items()): + perSizeDataList += [struct.pack(PER_SIZE_VALUE_FORMAT, value)] + offset += PER_SIZE_VALUE_FORMAT_SIZE * nSizes + # sort size values + sizeDataList = [ + struct.pack(SIZE_VALUE_FORMAT, fl2fi(sv, 16)) for sv in sorted(sizes) + ] + + data = bytesjoin( + [trackDataHeader] + entryDataList + sizeDataList + perSizeDataList + ) + return data + + def decompile(self, data, offset): + # initial offset is from the start of trak table to the current TrackData + trackDataHeader = data[offset : offset + TRACK_DATA_FORMAT_SIZE] + if len(trackDataHeader) != TRACK_DATA_FORMAT_SIZE: + raise TTLibError("not enough data to decompile TrackData header") + sstruct.unpack(TRACK_DATA_FORMAT, trackDataHeader, self) + offset += TRACK_DATA_FORMAT_SIZE + + nSizes = self.nSizes + sizeTableOffset = self.sizeTableOffset + sizeTable = [] + for i in range(nSizes): + sizeValueData = data[ + sizeTableOffset : sizeTableOffset + SIZE_VALUE_FORMAT_SIZE + ] + if len(sizeValueData) < SIZE_VALUE_FORMAT_SIZE: + raise TTLibError("not enough data to decompile TrackData size subtable") + (sizeValue,) = struct.unpack(SIZE_VALUE_FORMAT, sizeValueData) + sizeTable.append(fi2fl(sizeValue, 16)) + sizeTableOffset += SIZE_VALUE_FORMAT_SIZE + + for i in range(self.nTracks): + entry = TrackTableEntry() + entryData = data[offset : offset + TRACK_TABLE_ENTRY_FORMAT_SIZE] + if len(entryData) < TRACK_TABLE_ENTRY_FORMAT_SIZE: + raise TTLibError("not enough data to decompile TrackTableEntry record") + sstruct.unpack(TRACK_TABLE_ENTRY_FORMAT, entryData, entry) + perSizeOffset = entry.offset + for j in range(nSizes): + size = sizeTable[j] + perSizeValueData = data[ + perSizeOffset : perSizeOffset + PER_SIZE_VALUE_FORMAT_SIZE + ] + if len(perSizeValueData) < PER_SIZE_VALUE_FORMAT_SIZE: + raise TTLibError( + "not enough data to decompile per-size track values" + ) + (perSizeValue,) = struct.unpack(PER_SIZE_VALUE_FORMAT, perSizeValueData) + entry[size] = perSizeValue + perSizeOffset += PER_SIZE_VALUE_FORMAT_SIZE + self[entry.track] = entry + offset += TRACK_TABLE_ENTRY_FORMAT_SIZE + + def toXML(self, writer, ttFont): + nTracks = len(self) + nSizes = len(self.sizes()) + writer.comment("nTracks=%d, nSizes=%d" % (nTracks, nSizes)) + writer.newline() + for track, entry in sorted(self.items()): + assert entry.nameIndex is not None + entry.track = track + entry.toXML(writer, ttFont) + + def fromXML(self, name, attrs, content, ttFont): + if name != "trackEntry": + return + entry = TrackTableEntry() + entry.fromXML(name, attrs, content, ttFont) + self[entry.track] = entry + + def sizes(self): + if not self: + return frozenset() + tracks = list(self.tracks()) + sizes = self[tracks.pop(0)].sizes() + for track in tracks: + entrySizes = self[track].sizes() + if sizes != entrySizes: + raise TTLibError( + "'trak' table entries must specify the same sizes: " + "%s != %s" % (sorted(sizes), sorted(entrySizes)) + ) + return frozenset(sizes) + + def __getitem__(self, track): + return self._map[track] + + def __delitem__(self, track): + del self._map[track] + + def __setitem__(self, track, entry): + self._map[track] = entry + + def __len__(self): + return len(self._map) + + def __iter__(self): + return iter(self._map) + + def keys(self): + return self._map.keys() + + tracks = keys + + def __repr__(self): + return "TrackData({})".format(self._map if self else "") + + +class TrackTableEntry(MutableMapping): + def __init__(self, values={}, nameIndex=None): + self.nameIndex = nameIndex + self._map = dict(values) + + def toXML(self, writer, ttFont): + name = ttFont["name"].getDebugName(self.nameIndex) + writer.begintag( + "trackEntry", + (("value", fl2str(self.track, 16)), ("nameIndex", self.nameIndex)), + ) + writer.newline() + if name: + writer.comment(name) + writer.newline() + for size, perSizeValue in sorted(self.items()): + writer.simpletag("track", size=fl2str(size, 16), value=perSizeValue) + writer.newline() + writer.endtag("trackEntry") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + self.track = str2fl(attrs["value"], 16) + self.nameIndex = safeEval(attrs["nameIndex"]) + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, _ = element + if name != "track": + continue + size = str2fl(attrs["size"], 16) + self[size] = safeEval(attrs["value"]) + + def __getitem__(self, size): + return self._map[size] + + def __delitem__(self, size): + del self._map[size] + + def __setitem__(self, size, value): + self._map[size] = value + + def __len__(self): + return len(self._map) + + def __iter__(self): + return iter(self._map) + + def keys(self): + return self._map.keys() + + sizes = keys + + def __repr__(self): + return "TrackTableEntry({}, nameIndex={})".format(self._map, self.nameIndex) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.nameIndex == other.nameIndex and dict(self) == dict(other) + + def __ne__(self, other): + result = self.__eq__(other) + return result if result is NotImplemented else not result diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_v_h_e_a.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_v_h_e_a.py new file mode 100644 index 0000000000000000000000000000000000000000..9c6fa3aefe2e91c042948b4d21a7df3e8087a17c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_v_h_e_a.py @@ -0,0 +1,139 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import safeEval +from fontTools.misc.fixedTools import ( + ensureVersionIsLong as fi2ve, + versionToFixed as ve2fi, +) +from . import DefaultTable +import math + + +vheaFormat = """ + > # big endian + tableVersion: L + ascent: h + descent: h + lineGap: h + advanceHeightMax: H + minTopSideBearing: h + minBottomSideBearing: h + yMaxExtent: h + caretSlopeRise: h + caretSlopeRun: h + caretOffset: h + reserved1: h + reserved2: h + reserved3: h + reserved4: h + metricDataFormat: h + numberOfVMetrics: H +""" + + +class table__v_h_e_a(DefaultTable.DefaultTable): + """Vertical Header table + + The ``vhea`` table contains information needed during vertical + text layout. + + .. note:: + This converter class is kept in sync with the :class:`._h_h_e_a.table__h_h_e_a` + table constructor. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/vhea + """ + + # Note: Keep in sync with table__h_h_e_a + + dependencies = ["vmtx", "glyf", "CFF ", "CFF2"] + + def decompile(self, data, ttFont): + sstruct.unpack(vheaFormat, data, self) + + def compile(self, ttFont): + if ttFont.recalcBBoxes and ( + ttFont.isLoaded("glyf") + or ttFont.isLoaded("CFF ") + or ttFont.isLoaded("CFF2") + ): + self.recalc(ttFont) + self.tableVersion = fi2ve(self.tableVersion) + return sstruct.pack(vheaFormat, self) + + def recalc(self, ttFont): + if "vmtx" not in ttFont: + return + + vmtxTable = ttFont["vmtx"] + self.advanceHeightMax = max(adv for adv, _ in vmtxTable.metrics.values()) + + boundsHeightDict = {} + if "glyf" in ttFont: + glyfTable = ttFont["glyf"] + for name in ttFont.getGlyphOrder(): + g = glyfTable[name] + if g.numberOfContours == 0: + continue + if g.numberOfContours < 0 and not hasattr(g, "yMax"): + # Composite glyph without extents set. + # Calculate those. + g.recalcBounds(glyfTable) + boundsHeightDict[name] = g.yMax - g.yMin + elif "CFF " in ttFont or "CFF2" in ttFont: + if "CFF " in ttFont: + topDict = ttFont["CFF "].cff.topDictIndex[0] + else: + topDict = ttFont["CFF2"].cff.topDictIndex[0] + charStrings = topDict.CharStrings + for name in ttFont.getGlyphOrder(): + cs = charStrings[name] + bounds = cs.calcBounds(charStrings) + if bounds is not None: + boundsHeightDict[name] = int( + math.ceil(bounds[3]) - math.floor(bounds[1]) + ) + + if boundsHeightDict: + minTopSideBearing = float("inf") + minBottomSideBearing = float("inf") + yMaxExtent = -float("inf") + for name, boundsHeight in boundsHeightDict.items(): + advanceHeight, tsb = vmtxTable[name] + bsb = advanceHeight - tsb - boundsHeight + extent = tsb + boundsHeight + minTopSideBearing = min(minTopSideBearing, tsb) + minBottomSideBearing = min(minBottomSideBearing, bsb) + yMaxExtent = max(yMaxExtent, extent) + self.minTopSideBearing = minTopSideBearing + self.minBottomSideBearing = minBottomSideBearing + self.yMaxExtent = yMaxExtent + + else: # No glyph has outlines. + self.minTopSideBearing = 0 + self.minBottomSideBearing = 0 + self.yMaxExtent = 0 + + def toXML(self, writer, ttFont): + formatstring, names, fixes = sstruct.getformat(vheaFormat) + for name in names: + value = getattr(self, name) + if name == "tableVersion": + value = fi2ve(value) + value = "0x%08x" % value + writer.simpletag(name, value=value) + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "tableVersion": + setattr(self, name, ve2fi(attrs["value"])) + return + setattr(self, name, safeEval(attrs["value"])) + + # reserved0 is caretOffset for legacy reasons + @property + def reserved0(self): + return self.caretOffset + + @reserved0.setter + def reserved0(self, value): + self.caretOffset = value diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_v_m_t_x.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_v_m_t_x.py new file mode 100644 index 0000000000000000000000000000000000000000..32662fc6a257e9bb3da60b259756d8f9f7fa7919 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/_v_m_t_x.py @@ -0,0 +1,19 @@ +from fontTools import ttLib + +superclass = ttLib.getTableClass("hmtx") + + +class table__v_m_t_x(superclass): + """Vertical Metrics table + + The ``vmtx`` table contains per-glyph metrics for the glyphs in a + ``glyf``, ``CFF ``, or ``CFF2`` table, as needed for vertical text + layout. + + See also https://learn.microsoft.com/en-us/typography/opentype/spec/vmtx + """ + + headerTag = "vhea" + advanceName = "height" + sideBearingName = "tsb" + numberOfMetricsName = "numberOfVMetrics" diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/asciiTable.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/asciiTable.py new file mode 100644 index 0000000000000000000000000000000000000000..6f81c526b372b268b253da47c337715e316ee4d4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/asciiTable.py @@ -0,0 +1,20 @@ +from fontTools.misc.textTools import strjoin, tobytes, tostr +from . import DefaultTable + + +class asciiTable(DefaultTable.DefaultTable): + def toXML(self, writer, ttFont): + data = tostr(self.data) + # removing null bytes. XXX needed?? + data = data.split("\0") + data = strjoin(data) + writer.begintag("source") + writer.newline() + writer.write_noindent(data) + writer.newline() + writer.endtag("source") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + lines = strjoin(content).split("\n") + self.data = tobytes("\n".join(lines[1:-1])) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/grUtils.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/grUtils.py new file mode 100644 index 0000000000000000000000000000000000000000..785684b1eb30a76ae598bfe46416d4556fc422a0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/grUtils.py @@ -0,0 +1,92 @@ +import struct, warnings + +try: + import lz4 +except ImportError: + lz4 = None +else: + import lz4.block + +# old scheme for VERSION < 0.9 otherwise use lz4.block + + +def decompress(data): + (compression,) = struct.unpack(">L", data[4:8]) + scheme = compression >> 27 + size = compression & 0x07FFFFFF + if scheme == 0: + pass + elif scheme == 1 and lz4: + res = lz4.block.decompress(struct.pack("<L", size) + data[8:]) + if len(res) != size: + warnings.warn("Table decompression failed.") + else: + data = res + else: + warnings.warn("Table is compressed with an unsupported compression scheme") + return (data, scheme) + + +def compress(scheme, data): + hdr = data[:4] + struct.pack(">L", (scheme << 27) + (len(data) & 0x07FFFFFF)) + if scheme == 0: + return data + elif scheme == 1 and lz4: + res = lz4.block.compress( + data, mode="high_compression", compression=16, store_size=False + ) + return hdr + res + else: + warnings.warn("Table failed to compress by unsupported compression scheme") + return data + + +def _entries(attrs, sameval): + ak = 0 + vals = [] + lastv = 0 + for k, v in attrs: + if len(vals) and (k != ak + 1 or (sameval and v != lastv)): + yield (ak - len(vals) + 1, len(vals), vals) + vals = [] + ak = k + vals.append(v) + lastv = v + yield (ak - len(vals) + 1, len(vals), vals) + + +def entries(attributes, sameval=False): + g = _entries(sorted(attributes.items(), key=lambda x: int(x[0])), sameval) + return g + + +def bininfo(num, size=1): + if num == 0: + return struct.pack(">4H", 0, 0, 0, 0) + srange = 1 + select = 0 + while srange <= num: + srange *= 2 + select += 1 + select -= 1 + srange //= 2 + srange *= size + shift = num * size - srange + return struct.pack(">4H", num, srange, select, shift) + + +def num2tag(n): + if n < 0x200000: + return str(n) + else: + return ( + struct.unpack("4s", struct.pack(">L", n))[0].replace(b"\000", b"").decode() + ) + + +def tag2num(n): + try: + return int(n) + except ValueError: + n = (n + " ")[:4] + return struct.unpack(">L", n.encode("ascii"))[0] diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otBase.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otBase.py new file mode 100644 index 0000000000000000000000000000000000000000..1b252db193fecd339383d4838c366e55cd2ee948 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otBase.py @@ -0,0 +1,1467 @@ +from fontTools.config import OPTIONS +from fontTools.misc.textTools import Tag, bytesjoin +from .DefaultTable import DefaultTable +from enum import IntEnum +import sys +import array +import struct +import logging +from functools import lru_cache +from typing import Iterator, NamedTuple, Optional, Tuple + +log = logging.getLogger(__name__) + +have_uharfbuzz = False +try: + import uharfbuzz as hb + + have_uharfbuzz = True +except ImportError: + pass + +USE_HARFBUZZ_REPACKER = OPTIONS[f"{__name__}:USE_HARFBUZZ_REPACKER"] + + +class OverflowErrorRecord(object): + def __init__(self, overflowTuple): + self.tableType = overflowTuple[0] + self.LookupListIndex = overflowTuple[1] + self.SubTableIndex = overflowTuple[2] + self.itemName = overflowTuple[3] + self.itemIndex = overflowTuple[4] + + def __repr__(self): + return str( + ( + self.tableType, + "LookupIndex:", + self.LookupListIndex, + "SubTableIndex:", + self.SubTableIndex, + "ItemName:", + self.itemName, + "ItemIndex:", + self.itemIndex, + ) + ) + + +class OTLOffsetOverflowError(Exception): + def __init__(self, overflowErrorRecord): + self.value = overflowErrorRecord + + def __str__(self): + return repr(self.value) + + +class RepackerState(IntEnum): + # Repacking control flow is implemnted using a state machine. The state machine table: + # + # State | Packing Success | Packing Failed | Exception Raised | + # ------------+-----------------+----------------+------------------+ + # PURE_FT | Return result | PURE_FT | Return failure | + # HB_FT | Return result | HB_FT | FT_FALLBACK | + # FT_FALLBACK | HB_FT | FT_FALLBACK | Return failure | + + # Pack only with fontTools, don't allow sharing between extensions. + PURE_FT = 1 + + # Attempt to pack with harfbuzz (allowing sharing between extensions) + # use fontTools to attempt overflow resolution. + HB_FT = 2 + + # Fallback if HB/FT packing gets stuck. Pack only with fontTools, don't allow sharing between + # extensions. + FT_FALLBACK = 3 + + +class BaseTTXConverter(DefaultTable): + """Generic base class for TTX table converters. It functions as an + adapter between the TTX (ttLib actually) table model and the model + we use for OpenType tables, which is necessarily subtly different. + """ + + def decompile(self, data, font): + """Create an object from the binary data. Called automatically on access.""" + from . import otTables + + reader = OTTableReader(data, tableTag=self.tableTag) + tableClass = getattr(otTables, self.tableTag) + self.table = tableClass() + self.table.decompile(reader, font) + + def compile(self, font): + """Compiles the table into binary. Called automatically on save.""" + + # General outline: + # Create a top-level OTTableWriter for the GPOS/GSUB table. + # Call the compile method for the the table + # for each 'converter' record in the table converter list + # call converter's write method for each item in the value. + # - For simple items, the write method adds a string to the + # writer's self.items list. + # - For Struct/Table/Subtable items, it add first adds new writer to the + # to the writer's self.items, then calls the item's compile method. + # This creates a tree of writers, rooted at the GUSB/GPOS writer, with + # each writer representing a table, and the writer.items list containing + # the child data strings and writers. + # call the getAllData method + # call _doneWriting, which removes duplicates + # call _gatherTables. This traverses the tables, adding unique occurences to a flat list of tables + # Traverse the flat list of tables, calling getDataLength on each to update their position + # Traverse the flat list of tables again, calling getData each get the data in the table, now that + # pos's and offset are known. + + # If a lookup subtable overflows an offset, we have to start all over. + overflowRecord = None + # this is 3-state option: default (None) means automatically use hb.repack or + # silently fall back if it fails; True, use it and raise error if not possible + # or it errors out; False, don't use it, even if you can. + use_hb_repack = font.cfg[USE_HARFBUZZ_REPACKER] + if self.tableTag in ("GSUB", "GPOS"): + if use_hb_repack is False: + log.debug( + "hb.repack disabled, compiling '%s' with pure-python serializer", + self.tableTag, + ) + elif not have_uharfbuzz: + if use_hb_repack is True: + raise ImportError("No module named 'uharfbuzz'") + else: + assert use_hb_repack is None + log.debug( + "uharfbuzz not found, compiling '%s' with pure-python serializer", + self.tableTag, + ) + + if ( + use_hb_repack in (None, True) + and have_uharfbuzz + and self.tableTag in ("GSUB", "GPOS") + ): + state = RepackerState.HB_FT + else: + state = RepackerState.PURE_FT + + hb_first_error_logged = False + lastOverflowRecord = None + while True: + try: + writer = OTTableWriter(tableTag=self.tableTag) + self.table.compile(writer, font) + if state == RepackerState.HB_FT: + return self.tryPackingHarfbuzz(writer, hb_first_error_logged) + elif state == RepackerState.PURE_FT: + return self.tryPackingFontTools(writer) + elif state == RepackerState.FT_FALLBACK: + # Run packing with FontTools only, but don't return the result as it will + # not be optimally packed. Once a successful packing has been found, state is + # changed back to harfbuzz packing to produce the final, optimal, packing. + self.tryPackingFontTools(writer) + log.debug( + "Re-enabling sharing between extensions and switching back to " + "harfbuzz+fontTools packing." + ) + state = RepackerState.HB_FT + + except OTLOffsetOverflowError as e: + hb_first_error_logged = True + ok = self.tryResolveOverflow(font, e, lastOverflowRecord) + lastOverflowRecord = e.value + + if ok: + continue + + if state is RepackerState.HB_FT: + log.debug( + "Harfbuzz packing out of resolutions, disabling sharing between extensions and " + "switching to fontTools only packing." + ) + state = RepackerState.FT_FALLBACK + else: + raise + + def tryPackingHarfbuzz(self, writer, hb_first_error_logged): + try: + log.debug("serializing '%s' with hb.repack", self.tableTag) + return writer.getAllDataUsingHarfbuzz(self.tableTag) + except (ValueError, MemoryError, hb.RepackerError) as e: + # Only log hb repacker errors the first time they occur in + # the offset-overflow resolution loop, they are just noisy. + # Maybe we can revisit this if/when uharfbuzz actually gives + # us more info as to why hb.repack failed... + if not hb_first_error_logged: + error_msg = f"{type(e).__name__}" + if str(e) != "": + error_msg += f": {e}" + log.warning( + "hb.repack failed to serialize '%s', attempting fonttools resolutions " + "; the error message was: %s", + self.tableTag, + error_msg, + ) + hb_first_error_logged = True + return writer.getAllData(remove_duplicate=False) + + def tryPackingFontTools(self, writer): + return writer.getAllData() + + def tryResolveOverflow(self, font, e, lastOverflowRecord): + ok = 0 + if lastOverflowRecord == e.value: + # Oh well... + return ok + + overflowRecord = e.value + log.info("Attempting to fix OTLOffsetOverflowError %s", e) + + if overflowRecord.itemName is None: + from .otTables import fixLookupOverFlows + + ok = fixLookupOverFlows(font, overflowRecord) + else: + from .otTables import fixSubTableOverFlows + + ok = fixSubTableOverFlows(font, overflowRecord) + + if ok: + return ok + + # Try upgrading lookup to Extension and hope + # that cross-lookup sharing not happening would + # fix overflow... + from .otTables import fixLookupOverFlows + + return fixLookupOverFlows(font, overflowRecord) + + def toXML(self, writer, font): + self.table.toXML2(writer, font) + + def fromXML(self, name, attrs, content, font): + from . import otTables + + if not hasattr(self, "table"): + tableClass = getattr(otTables, self.tableTag) + self.table = tableClass() + self.table.fromXML(name, attrs, content, font) + self.table.populateDefaults() + + def ensureDecompiled(self, recurse=True): + self.table.ensureDecompiled(recurse=recurse) + + +# https://github.com/fonttools/fonttools/pull/2285#issuecomment-834652928 +assert len(struct.pack("i", 0)) == 4 +assert array.array("i").itemsize == 4, "Oops, file a bug against fonttools." + + +class OTTableReader(object): + """Helper class to retrieve data from an OpenType table.""" + + __slots__ = ("data", "offset", "pos", "localState", "tableTag") + + def __init__(self, data, localState=None, offset=0, tableTag=None): + self.data = data + self.offset = offset + self.pos = offset + self.localState = localState + self.tableTag = tableTag + + def advance(self, count): + self.pos += count + + def seek(self, pos): + self.pos = pos + + def copy(self): + other = self.__class__(self.data, self.localState, self.offset, self.tableTag) + other.pos = self.pos + return other + + def getSubReader(self, offset): + offset = self.offset + offset + return self.__class__(self.data, self.localState, offset, self.tableTag) + + def readValue(self, typecode, staticSize): + pos = self.pos + newpos = pos + staticSize + (value,) = struct.unpack(f">{typecode}", self.data[pos:newpos]) + self.pos = newpos + return value + + def readArray(self, typecode, staticSize, count): + pos = self.pos + newpos = pos + count * staticSize + value = array.array(typecode, self.data[pos:newpos]) + if sys.byteorder != "big": + value.byteswap() + self.pos = newpos + return value.tolist() + + def readInt8(self): + return self.readValue("b", staticSize=1) + + def readInt8Array(self, count): + return self.readArray("b", staticSize=1, count=count) + + def readShort(self): + return self.readValue("h", staticSize=2) + + def readShortArray(self, count): + return self.readArray("h", staticSize=2, count=count) + + def readLong(self): + return self.readValue("i", staticSize=4) + + def readLongArray(self, count): + return self.readArray("i", staticSize=4, count=count) + + def readUInt8(self): + return self.readValue("B", staticSize=1) + + def readUInt8Array(self, count): + return self.readArray("B", staticSize=1, count=count) + + def readUShort(self): + return self.readValue("H", staticSize=2) + + def readUShortArray(self, count): + return self.readArray("H", staticSize=2, count=count) + + def readULong(self): + return self.readValue("I", staticSize=4) + + def readULongArray(self, count): + return self.readArray("I", staticSize=4, count=count) + + def readUInt24(self): + pos = self.pos + newpos = pos + 3 + (value,) = struct.unpack(">l", b"\0" + self.data[pos:newpos]) + self.pos = newpos + return value + + def readInt24(self): + pos = self.pos + newpos = pos + 3 + b = self.data[pos:newpos] + pad = b"\xff" if b[0] & 0x80 else b"\0" + (value,) = struct.unpack(">l", pad + b) + self.pos = newpos + return value + + def readUInt24Array(self, count): + return [self.readUInt24() for _ in range(count)] + + def readTag(self): + pos = self.pos + newpos = pos + 4 + value = Tag(self.data[pos:newpos]) + assert len(value) == 4, value + self.pos = newpos + return value + + def readData(self, count): + pos = self.pos + newpos = pos + count + value = self.data[pos:newpos] + self.pos = newpos + return value + + def __setitem__(self, name, value): + state = self.localState.copy() if self.localState else dict() + state[name] = value + self.localState = state + + def __getitem__(self, name): + return self.localState and self.localState[name] + + def __contains__(self, name): + return self.localState and name in self.localState + + +class OffsetToWriter(object): + def __init__(self, subWriter, offsetSize): + self.subWriter = subWriter + self.offsetSize = offsetSize + + def __eq__(self, other): + if type(self) != type(other): + return NotImplemented + return self.subWriter == other.subWriter and self.offsetSize == other.offsetSize + + def __hash__(self): + # only works after self._doneWriting() has been called + return hash((self.subWriter, self.offsetSize)) + + +class OTTableWriter(object): + """Helper class to gather and assemble data for OpenType tables.""" + + def __init__(self, localState=None, tableTag=None): + self.items = [] + self.pos = None + self.localState = localState + self.tableTag = tableTag + self.parent = None + self.name = "<none>" + + def __setitem__(self, name, value): + state = self.localState.copy() if self.localState else dict() + state[name] = value + self.localState = state + + def __getitem__(self, name): + return self.localState[name] + + def __delitem__(self, name): + del self.localState[name] + + # assembler interface + + def getDataLength(self): + """Return the length of this table in bytes, without subtables.""" + l = 0 + for item in self.items: + if hasattr(item, "getCountData"): + l += item.size + elif hasattr(item, "subWriter"): + l += item.offsetSize + else: + l = l + len(item) + return l + + def getData(self): + """Assemble the data for this writer/table, without subtables.""" + items = list(self.items) # make a shallow copy + pos = self.pos + numItems = len(items) + for i in range(numItems): + item = items[i] + + if hasattr(item, "subWriter"): + if item.offsetSize == 4: + items[i] = packULong(item.subWriter.pos - pos) + elif item.offsetSize == 2: + try: + items[i] = packUShort(item.subWriter.pos - pos) + except struct.error: + # provide data to fix overflow problem. + overflowErrorRecord = self.getOverflowErrorRecord( + item.subWriter + ) + + raise OTLOffsetOverflowError(overflowErrorRecord) + elif item.offsetSize == 3: + items[i] = packUInt24(item.subWriter.pos - pos) + else: + raise ValueError(item.offsetSize) + + return bytesjoin(items) + + def getDataForHarfbuzz(self): + """Assemble the data for this writer/table with all offset field set to 0""" + items = list(self.items) + packFuncs = {2: packUShort, 3: packUInt24, 4: packULong} + for i, item in enumerate(items): + if hasattr(item, "subWriter"): + # Offset value is not needed in harfbuzz repacker, so setting offset to 0 to avoid overflow here + if item.offsetSize in packFuncs: + items[i] = packFuncs[item.offsetSize](0) + else: + raise ValueError(item.offsetSize) + + return bytesjoin(items) + + def __hash__(self): + # only works after self._doneWriting() has been called + return hash(self.items) + + def __ne__(self, other): + result = self.__eq__(other) + return result if result is NotImplemented else not result + + def __eq__(self, other): + if type(self) != type(other): + return NotImplemented + return self.items == other.items + + def _doneWriting(self, internedTables, shareExtension=False): + # Convert CountData references to data string items + # collapse duplicate table references to a unique entry + # "tables" are OTTableWriter objects. + + # For Extension Lookup types, we can + # eliminate duplicates only within the tree under the Extension Lookup, + # as offsets may exceed 64K even between Extension LookupTable subtables. + isExtension = hasattr(self, "Extension") + + # Certain versions of Uniscribe reject the font if the GSUB/GPOS top-level + # arrays (ScriptList, FeatureList, LookupList) point to the same, possibly + # empty, array. So, we don't share those. + # See: https://github.com/fonttools/fonttools/issues/518 + dontShare = hasattr(self, "DontShare") + + if isExtension and not shareExtension: + internedTables = {} + + items = self.items + for i, item in enumerate(items): + if hasattr(item, "getCountData"): + items[i] = item.getCountData() + elif hasattr(item, "subWriter"): + item.subWriter._doneWriting( + internedTables, shareExtension=shareExtension + ) + # At this point, all subwriters are hashable based on their items. + # (See hash and comparison magic methods above.) So the ``setdefault`` + # call here will return the first writer object we've seen with + # equal content, or store it in the dictionary if it's not been + # seen yet. We therefore replace the subwriter object with an equivalent + # object, which deduplicates the tree. + if not dontShare: + items[i].subWriter = internedTables.setdefault( + item.subWriter, item.subWriter + ) + self.items = tuple(items) + + def _gatherTables(self, tables, extTables, done): + # Convert table references in self.items tree to a flat + # list of tables in depth-first traversal order. + # "tables" are OTTableWriter objects. + # We do the traversal in reverse order at each level, in order to + # resolve duplicate references to be the last reference in the list of tables. + # For extension lookups, duplicate references can be merged only within the + # writer tree under the extension lookup. + + done[id(self)] = True + + numItems = len(self.items) + iRange = list(range(numItems)) + iRange.reverse() + + isExtension = hasattr(self, "Extension") + + selfTables = tables + + if isExtension: + assert ( + extTables is not None + ), "Program or XML editing error. Extension subtables cannot contain extensions subtables" + tables, extTables, done = extTables, None, {} + + # add Coverage table if it is sorted last. + sortCoverageLast = False + if hasattr(self, "sortCoverageLast"): + # Find coverage table + for i in range(numItems): + item = self.items[i] + if ( + hasattr(item, "subWriter") + and getattr(item.subWriter, "name", None) == "Coverage" + ): + sortCoverageLast = True + break + if id(item.subWriter) not in done: + item.subWriter._gatherTables(tables, extTables, done) + else: + # We're a new parent of item + pass + + for i in iRange: + item = self.items[i] + if not hasattr(item, "subWriter"): + continue + + if ( + sortCoverageLast + and (i == 1) + and getattr(item.subWriter, "name", None) == "Coverage" + ): + # we've already 'gathered' it above + continue + + if id(item.subWriter) not in done: + item.subWriter._gatherTables(tables, extTables, done) + else: + # Item is already written out by other parent + pass + + selfTables.append(self) + + def _gatherGraphForHarfbuzz(self, tables, obj_list, done, objidx, virtual_edges): + real_links = [] + virtual_links = [] + item_idx = objidx + + # Merge virtual_links from parent + for idx in virtual_edges: + virtual_links.append((0, 0, idx)) + + sortCoverageLast = False + coverage_idx = 0 + if hasattr(self, "sortCoverageLast"): + # Find coverage table + for i, item in enumerate(self.items): + if getattr(item, "name", None) == "Coverage": + sortCoverageLast = True + if id(item) not in done: + coverage_idx = item_idx = item._gatherGraphForHarfbuzz( + tables, obj_list, done, item_idx, virtual_edges + ) + else: + coverage_idx = done[id(item)] + virtual_edges.append(coverage_idx) + break + + child_idx = 0 + offset_pos = 0 + for i, item in enumerate(self.items): + if hasattr(item, "subWriter"): + pos = offset_pos + elif hasattr(item, "getCountData"): + offset_pos += item.size + continue + else: + offset_pos = offset_pos + len(item) + continue + + if id(item.subWriter) not in done: + child_idx = item_idx = item.subWriter._gatherGraphForHarfbuzz( + tables, obj_list, done, item_idx, virtual_edges + ) + else: + child_idx = done[id(item.subWriter)] + + real_edge = (pos, item.offsetSize, child_idx) + real_links.append(real_edge) + offset_pos += item.offsetSize + + tables.append(self) + obj_list.append((real_links, virtual_links)) + item_idx += 1 + done[id(self)] = item_idx + if sortCoverageLast: + virtual_edges.pop() + + return item_idx + + def getAllDataUsingHarfbuzz(self, tableTag): + """The Whole table is represented as a Graph. + Assemble graph data and call Harfbuzz repacker to pack the table. + Harfbuzz repacker is faster and retain as much sub-table sharing as possible, see also: + https://github.com/harfbuzz/harfbuzz/blob/main/docs/repacker.md + The input format for hb.repack() method is explained here: + https://github.com/harfbuzz/uharfbuzz/blob/main/src/uharfbuzz/_harfbuzz.pyx#L1149 + """ + internedTables = {} + self._doneWriting(internedTables, shareExtension=True) + tables = [] + obj_list = [] + done = {} + objidx = 0 + virtual_edges = [] + self._gatherGraphForHarfbuzz(tables, obj_list, done, objidx, virtual_edges) + # Gather all data in two passes: the absolute positions of all + # subtable are needed before the actual data can be assembled. + pos = 0 + for table in tables: + table.pos = pos + pos = pos + table.getDataLength() + + data = [] + for table in tables: + tableData = table.getDataForHarfbuzz() + data.append(tableData) + + return hb.serialize_with_tag(str(tableTag), data, obj_list) + + def getAllData(self, remove_duplicate=True): + """Assemble all data, including all subtables.""" + if remove_duplicate: + internedTables = {} + self._doneWriting(internedTables) + tables = [] + extTables = [] + done = {} + self._gatherTables(tables, extTables, done) + tables.reverse() + extTables.reverse() + # Gather all data in two passes: the absolute positions of all + # subtable are needed before the actual data can be assembled. + pos = 0 + for table in tables: + table.pos = pos + pos = pos + table.getDataLength() + + for table in extTables: + table.pos = pos + pos = pos + table.getDataLength() + + data = [] + for table in tables: + tableData = table.getData() + data.append(tableData) + + for table in extTables: + tableData = table.getData() + data.append(tableData) + + return bytesjoin(data) + + # interface for gathering data, as used by table.compile() + + def getSubWriter(self): + subwriter = self.__class__(self.localState, self.tableTag) + subwriter.parent = ( + self # because some subtables have idential values, we discard + ) + # the duplicates under the getAllData method. Hence some + # subtable writers can have more than one parent writer. + # But we just care about first one right now. + return subwriter + + def writeValue(self, typecode, value): + self.items.append(struct.pack(f">{typecode}", value)) + + def writeArray(self, typecode, values): + a = array.array(typecode, values) + if sys.byteorder != "big": + a.byteswap() + self.items.append(a.tobytes()) + + def writeInt8(self, value): + assert -128 <= value < 128, value + self.items.append(struct.pack(">b", value)) + + def writeInt8Array(self, values): + self.writeArray("b", values) + + def writeShort(self, value): + assert -32768 <= value < 32768, value + self.items.append(struct.pack(">h", value)) + + def writeShortArray(self, values): + self.writeArray("h", values) + + def writeLong(self, value): + self.items.append(struct.pack(">i", value)) + + def writeLongArray(self, values): + self.writeArray("i", values) + + def writeUInt8(self, value): + assert 0 <= value < 256, value + self.items.append(struct.pack(">B", value)) + + def writeUInt8Array(self, values): + self.writeArray("B", values) + + def writeUShort(self, value): + assert 0 <= value < 0x10000, value + self.items.append(struct.pack(">H", value)) + + def writeUShortArray(self, values): + self.writeArray("H", values) + + def writeULong(self, value): + self.items.append(struct.pack(">I", value)) + + def writeULongArray(self, values): + self.writeArray("I", values) + + def writeUInt24(self, value): + assert 0 <= value < 0x1000000, value + b = struct.pack(">L", value) + self.items.append(b[1:]) + + def writeUInt24Array(self, values): + for value in values: + self.writeUInt24(value) + + def writeTag(self, tag): + tag = Tag(tag).tobytes() + assert len(tag) == 4, tag + self.items.append(tag) + + def writeSubTable(self, subWriter, offsetSize): + self.items.append(OffsetToWriter(subWriter, offsetSize)) + + def writeCountReference(self, table, name, size=2, value=None): + ref = CountReference(table, name, size=size, value=value) + self.items.append(ref) + return ref + + def writeStruct(self, format, values): + data = struct.pack(*(format,) + values) + self.items.append(data) + + def writeData(self, data): + self.items.append(data) + + def getOverflowErrorRecord(self, item): + LookupListIndex = SubTableIndex = itemName = itemIndex = None + if self.name == "LookupList": + LookupListIndex = item.repeatIndex + elif self.name == "Lookup": + LookupListIndex = self.repeatIndex + SubTableIndex = item.repeatIndex + else: + itemName = getattr(item, "name", "<none>") + if hasattr(item, "repeatIndex"): + itemIndex = item.repeatIndex + if self.name == "SubTable": + LookupListIndex = self.parent.repeatIndex + SubTableIndex = self.repeatIndex + elif self.name == "ExtSubTable": + LookupListIndex = self.parent.parent.repeatIndex + SubTableIndex = self.parent.repeatIndex + else: # who knows how far below the SubTable level we are! Climb back up to the nearest subtable. + itemName = ".".join([self.name, itemName]) + p1 = self.parent + while p1 and p1.name not in ["ExtSubTable", "SubTable"]: + itemName = ".".join([p1.name, itemName]) + p1 = p1.parent + if p1: + if p1.name == "ExtSubTable": + LookupListIndex = p1.parent.parent.repeatIndex + SubTableIndex = p1.parent.repeatIndex + else: + LookupListIndex = p1.parent.repeatIndex + SubTableIndex = p1.repeatIndex + + return OverflowErrorRecord( + (self.tableTag, LookupListIndex, SubTableIndex, itemName, itemIndex) + ) + + +class CountReference(object): + """A reference to a Count value, not a count of references.""" + + def __init__(self, table, name, size=None, value=None): + self.table = table + self.name = name + self.size = size + if value is not None: + self.setValue(value) + + def setValue(self, value): + table = self.table + name = self.name + if table[name] is None: + table[name] = value + else: + assert table[name] == value, (name, table[name], value) + + def getValue(self): + return self.table[self.name] + + def getCountData(self): + v = self.table[self.name] + if v is None: + v = 0 + return {1: packUInt8, 2: packUShort, 4: packULong}[self.size](v) + + +def packUInt8(value): + return struct.pack(">B", value) + + +def packUShort(value): + return struct.pack(">H", value) + + +def packULong(value): + assert 0 <= value < 0x100000000, value + return struct.pack(">I", value) + + +def packUInt24(value): + assert 0 <= value < 0x1000000, value + return struct.pack(">I", value)[1:] + + +class BaseTable(object): + """Generic base class for all OpenType (sub)tables.""" + + def __getattr__(self, attr): + reader = self.__dict__.get("reader") + if reader: + del self.reader + font = self.font + del self.font + self.decompile(reader, font) + return getattr(self, attr) + + raise AttributeError(attr) + + def ensureDecompiled(self, recurse=False): + reader = self.__dict__.get("reader") + if reader: + del self.reader + font = self.font + del self.font + self.decompile(reader, font) + if recurse: + for subtable in self.iterSubTables(): + subtable.value.ensureDecompiled(recurse) + + def __getstate__(self): + # before copying/pickling 'lazy' objects, make a shallow copy of OTTableReader + # https://github.com/fonttools/fonttools/issues/2965 + if "reader" in self.__dict__: + state = self.__dict__.copy() + state["reader"] = self.__dict__["reader"].copy() + return state + return self.__dict__ + + @classmethod + def getRecordSize(cls, reader): + totalSize = 0 + for conv in cls.converters: + size = conv.getRecordSize(reader) + if size is NotImplemented: + return NotImplemented + countValue = 1 + if conv.repeat: + if conv.repeat in reader: + countValue = reader[conv.repeat] + conv.aux + else: + return NotImplemented + totalSize += size * countValue + return totalSize + + def getConverters(self): + return self.converters + + def getConverterByName(self, name): + return self.convertersByName[name] + + def populateDefaults(self, propagator=None): + for conv in self.getConverters(): + if conv.repeat: + if not hasattr(self, conv.name): + setattr(self, conv.name, []) + countValue = len(getattr(self, conv.name)) - conv.aux + try: + count_conv = self.getConverterByName(conv.repeat) + setattr(self, conv.repeat, countValue) + except KeyError: + # conv.repeat is a propagated count + if propagator and conv.repeat in propagator: + propagator[conv.repeat].setValue(countValue) + else: + if conv.aux and not eval(conv.aux, None, self.__dict__): + continue + if hasattr(self, conv.name): + continue # Warn if it should NOT be present?! + if hasattr(conv, "writeNullOffset"): + setattr(self, conv.name, None) # Warn? + # elif not conv.isCount: + # # Warn? + # pass + if hasattr(conv, "DEFAULT"): + # OptionalValue converters (e.g. VarIndex) + setattr(self, conv.name, conv.DEFAULT) + + def decompile(self, reader, font): + self.readFormat(reader) + table = {} + self.__rawTable = table # for debugging + for conv in self.getConverters(): + if conv.name == "SubTable": + conv = conv.getConverter(reader.tableTag, table["LookupType"]) + if conv.name == "ExtSubTable": + conv = conv.getConverter(reader.tableTag, table["ExtensionLookupType"]) + if conv.name == "FeatureParams": + conv = conv.getConverter(reader["FeatureTag"]) + if conv.name == "SubStruct": + conv = conv.getConverter(reader.tableTag, table["MorphType"]) + try: + if conv.repeat: + if isinstance(conv.repeat, int): + countValue = conv.repeat + elif conv.repeat in table: + countValue = table[conv.repeat] + else: + # conv.repeat is a propagated count + countValue = reader[conv.repeat] + countValue += conv.aux + table[conv.name] = conv.readArray(reader, font, table, countValue) + else: + if conv.aux and not eval(conv.aux, None, table): + continue + table[conv.name] = conv.read(reader, font, table) + if conv.isPropagated: + reader[conv.name] = table[conv.name] + except Exception as e: + name = conv.name + e.args = e.args + (name,) + raise + + if hasattr(self, "postRead"): + self.postRead(table, font) + else: + self.__dict__.update(table) + + del self.__rawTable # succeeded, get rid of debugging info + + def compile(self, writer, font): + self.ensureDecompiled() + # TODO Following hack to be removed by rewriting how FormatSwitching tables + # are handled. + # https://github.com/fonttools/fonttools/pull/2238#issuecomment-805192631 + if hasattr(self, "preWrite"): + deleteFormat = not hasattr(self, "Format") + table = self.preWrite(font) + deleteFormat = deleteFormat and hasattr(self, "Format") + else: + deleteFormat = False + table = self.__dict__.copy() + + # some count references may have been initialized in a custom preWrite; we set + # these in the writer's state beforehand (instead of sequentially) so they will + # be propagated to all nested subtables even if the count appears in the current + # table only *after* the offset to the subtable that it is counting. + for conv in self.getConverters(): + if conv.isCount and conv.isPropagated: + value = table.get(conv.name) + if isinstance(value, CountReference): + writer[conv.name] = value + + if hasattr(self, "sortCoverageLast"): + writer.sortCoverageLast = 1 + + if hasattr(self, "DontShare"): + writer.DontShare = True + + if hasattr(self.__class__, "LookupType"): + writer["LookupType"].setValue(self.__class__.LookupType) + + self.writeFormat(writer) + for conv in self.getConverters(): + value = table.get( + conv.name + ) # TODO Handle defaults instead of defaulting to None! + if conv.repeat: + if value is None: + value = [] + countValue = len(value) - conv.aux + if isinstance(conv.repeat, int): + assert len(value) == conv.repeat, "expected %d values, got %d" % ( + conv.repeat, + len(value), + ) + elif conv.repeat in table: + CountReference(table, conv.repeat, value=countValue) + else: + # conv.repeat is a propagated count + writer[conv.repeat].setValue(countValue) + try: + conv.writeArray(writer, font, table, value) + except Exception as e: + e.args = e.args + (conv.name + "[]",) + raise + elif conv.isCount: + # Special-case Count values. + # Assumption: a Count field will *always* precede + # the actual array(s). + # We need a default value, as it may be set later by a nested + # table. We will later store it here. + # We add a reference: by the time the data is assembled + # the Count value will be filled in. + # We ignore the current count value since it will be recomputed, + # unless it's a CountReference that was already initialized in a custom preWrite. + if isinstance(value, CountReference): + ref = value + ref.size = conv.staticSize + writer.writeData(ref) + table[conv.name] = ref.getValue() + else: + ref = writer.writeCountReference(table, conv.name, conv.staticSize) + table[conv.name] = None + if conv.isPropagated: + writer[conv.name] = ref + elif conv.isLookupType: + # We make sure that subtables have the same lookup type, + # and that the type is the same as the one set on the + # Lookup object, if any is set. + if conv.name not in table: + table[conv.name] = None + ref = writer.writeCountReference( + table, conv.name, conv.staticSize, table[conv.name] + ) + writer["LookupType"] = ref + else: + if conv.aux and not eval(conv.aux, None, table): + continue + try: + conv.write(writer, font, table, value) + except Exception as e: + name = value.__class__.__name__ if value is not None else conv.name + e.args = e.args + (name,) + raise + if conv.isPropagated: + writer[conv.name] = value + + if deleteFormat: + del self.Format + + def readFormat(self, reader): + pass + + def writeFormat(self, writer): + pass + + def toXML(self, xmlWriter, font, attrs=None, name=None): + tableName = name if name else self.__class__.__name__ + if attrs is None: + attrs = [] + if hasattr(self, "Format"): + attrs = attrs + [("Format", self.Format)] + xmlWriter.begintag(tableName, attrs) + xmlWriter.newline() + self.toXML2(xmlWriter, font) + xmlWriter.endtag(tableName) + xmlWriter.newline() + + def toXML2(self, xmlWriter, font): + # Simpler variant of toXML, *only* for the top level tables (like GPOS, GSUB). + # This is because in TTX our parent writes our main tag, and in otBase.py we + # do it ourselves. I think I'm getting schizophrenic... + for conv in self.getConverters(): + if conv.repeat: + value = getattr(self, conv.name, []) + for i, item in enumerate(value): + conv.xmlWrite(xmlWriter, font, item, conv.name, [("index", i)]) + else: + if conv.aux and not eval(conv.aux, None, vars(self)): + continue + value = getattr( + self, conv.name, None + ) # TODO Handle defaults instead of defaulting to None! + conv.xmlWrite(xmlWriter, font, value, conv.name, []) + + def fromXML(self, name, attrs, content, font): + try: + conv = self.getConverterByName(name) + except KeyError: + raise # XXX on KeyError, raise nice error + value = conv.xmlRead(attrs, content, font) + # Some manually-written tables have a conv.repeat of "" + # to represent lists. Hence comparing to None here to + # allow those lists to be read correctly from XML. + if conv.repeat is not None: + seq = getattr(self, conv.name, None) + if seq is None: + seq = [] + setattr(self, conv.name, seq) + seq.append(value) + else: + setattr(self, conv.name, value) + + def __ne__(self, other): + result = self.__eq__(other) + return result if result is NotImplemented else not result + + def __eq__(self, other): + if type(self) != type(other): + return NotImplemented + + self.ensureDecompiled() + other.ensureDecompiled() + + return self.__dict__ == other.__dict__ + + class SubTableEntry(NamedTuple): + """See BaseTable.iterSubTables()""" + + name: str + value: "BaseTable" + index: Optional[int] = None # index into given array, None for single values + + def iterSubTables(self) -> Iterator[SubTableEntry]: + """Yield (name, value, index) namedtuples for all subtables of current table. + + A sub-table is an instance of BaseTable (or subclass thereof) that is a child + of self, the current parent table. + The tuples also contain the attribute name (str) of the of parent table to get + a subtable, and optionally, for lists of subtables (i.e. attributes associated + with a converter that has a 'repeat'), an index into the list containing the + given subtable value. + This method can be useful to traverse trees of otTables. + """ + for conv in self.getConverters(): + name = conv.name + value = getattr(self, name, None) + if value is None: + continue + if isinstance(value, BaseTable): + yield self.SubTableEntry(name, value) + elif isinstance(value, list): + yield from ( + self.SubTableEntry(name, v, index=i) + for i, v in enumerate(value) + if isinstance(v, BaseTable) + ) + + # instance (not @class)method for consistency with FormatSwitchingBaseTable + def getVariableAttrs(self): + return getVariableAttrs(self.__class__) + + +class FormatSwitchingBaseTable(BaseTable): + """Minor specialization of BaseTable, for tables that have multiple + formats, eg. CoverageFormat1 vs. CoverageFormat2.""" + + @classmethod + def getRecordSize(cls, reader): + return NotImplemented + + def getConverters(self): + try: + fmt = self.Format + except AttributeError: + # some FormatSwitchingBaseTables (e.g. Coverage) no longer have 'Format' + # attribute after fully decompiled, only gain one in preWrite before being + # recompiled. In the decompiled state, these hand-coded classes defined in + # otTables.py lose their format-specific nature and gain more high-level + # attributes that are not tied to converters. + return [] + return self.converters.get(self.Format, []) + + def getConverterByName(self, name): + return self.convertersByName[self.Format][name] + + def readFormat(self, reader): + self.Format = reader.readUShort() + + def writeFormat(self, writer): + writer.writeUShort(self.Format) + + def toXML(self, xmlWriter, font, attrs=None, name=None): + BaseTable.toXML(self, xmlWriter, font, attrs, name) + + def getVariableAttrs(self): + return getVariableAttrs(self.__class__, self.Format) + + +class UInt8FormatSwitchingBaseTable(FormatSwitchingBaseTable): + def readFormat(self, reader): + self.Format = reader.readUInt8() + + def writeFormat(self, writer): + writer.writeUInt8(self.Format) + + +formatSwitchingBaseTables = { + "uint16": FormatSwitchingBaseTable, + "uint8": UInt8FormatSwitchingBaseTable, +} + + +def getFormatSwitchingBaseTableClass(formatType): + try: + return formatSwitchingBaseTables[formatType] + except KeyError: + raise TypeError(f"Unsupported format type: {formatType!r}") + + +# memoize since these are parsed from otData.py, thus stay constant +@lru_cache() +def getVariableAttrs(cls: BaseTable, fmt: Optional[int] = None) -> Tuple[str]: + """Return sequence of variable table field names (can be empty). + + Attributes are deemed "variable" when their otData.py's description contain + 'VarIndexBase + {offset}', e.g. COLRv1 PaintVar* tables. + """ + if not issubclass(cls, BaseTable): + raise TypeError(cls) + if issubclass(cls, FormatSwitchingBaseTable): + if fmt is None: + raise TypeError(f"'fmt' is required for format-switching {cls.__name__}") + converters = cls.convertersByName[fmt] + else: + converters = cls.convertersByName + # assume if no 'VarIndexBase' field is present, table has no variable fields + if "VarIndexBase" not in converters: + return () + varAttrs = {} + for name, conv in converters.items(): + offset = conv.getVarIndexOffset() + if offset is not None: + varAttrs[name] = offset + return tuple(sorted(varAttrs, key=varAttrs.__getitem__)) + + +# +# Support for ValueRecords +# +# This data type is so different from all other OpenType data types that +# it requires quite a bit of code for itself. It even has special support +# in OTTableReader and OTTableWriter... +# + +valueRecordFormat = [ + # Mask Name isDevice signed + (0x0001, "XPlacement", 0, 1), + (0x0002, "YPlacement", 0, 1), + (0x0004, "XAdvance", 0, 1), + (0x0008, "YAdvance", 0, 1), + (0x0010, "XPlaDevice", 1, 0), + (0x0020, "YPlaDevice", 1, 0), + (0x0040, "XAdvDevice", 1, 0), + (0x0080, "YAdvDevice", 1, 0), + # reserved: + (0x0100, "Reserved1", 0, 0), + (0x0200, "Reserved2", 0, 0), + (0x0400, "Reserved3", 0, 0), + (0x0800, "Reserved4", 0, 0), + (0x1000, "Reserved5", 0, 0), + (0x2000, "Reserved6", 0, 0), + (0x4000, "Reserved7", 0, 0), + (0x8000, "Reserved8", 0, 0), +] + + +def _buildDict(): + d = {} + for mask, name, isDevice, signed in valueRecordFormat: + d[name] = mask, isDevice, signed + return d + + +valueRecordFormatDict = _buildDict() + + +class ValueRecordFactory(object): + """Given a format code, this object convert ValueRecords.""" + + def __init__(self, valueFormat): + format = [] + for mask, name, isDevice, signed in valueRecordFormat: + if valueFormat & mask: + format.append((name, isDevice, signed)) + self.format = format + + def __len__(self): + return len(self.format) + + def readValueRecord(self, reader, font): + format = self.format + if not format: + return None + valueRecord = ValueRecord() + for name, isDevice, signed in format: + if signed: + value = reader.readShort() + else: + value = reader.readUShort() + if isDevice: + if value: + from . import otTables + + subReader = reader.getSubReader(value) + value = getattr(otTables, name)() + value.decompile(subReader, font) + else: + value = None + setattr(valueRecord, name, value) + return valueRecord + + def writeValueRecord(self, writer, font, valueRecord): + for name, isDevice, signed in self.format: + value = getattr(valueRecord, name, 0) + if isDevice: + if value: + subWriter = writer.getSubWriter() + writer.writeSubTable(subWriter, offsetSize=2) + value.compile(subWriter, font) + else: + writer.writeUShort(0) + elif signed: + writer.writeShort(value) + else: + writer.writeUShort(value) + + +class ValueRecord(object): + # see ValueRecordFactory + + def __init__(self, valueFormat=None, src=None): + if valueFormat is not None: + for mask, name, isDevice, signed in valueRecordFormat: + if valueFormat & mask: + setattr(self, name, None if isDevice else 0) + if src is not None: + for key, val in src.__dict__.items(): + if not hasattr(self, key): + continue + setattr(self, key, val) + elif src is not None: + self.__dict__ = src.__dict__.copy() + + def getFormat(self): + format = 0 + for name in self.__dict__.keys(): + format = format | valueRecordFormatDict[name][0] + return format + + def getEffectiveFormat(self): + format = 0 + for name, value in self.__dict__.items(): + if value: + format = format | valueRecordFormatDict[name][0] + return format + + def toXML(self, xmlWriter, font, valueName, attrs=None): + if attrs is None: + simpleItems = [] + else: + simpleItems = list(attrs) + for mask, name, isDevice, format in valueRecordFormat[:4]: # "simple" values + if hasattr(self, name): + simpleItems.append((name, getattr(self, name))) + deviceItems = [] + for mask, name, isDevice, format in valueRecordFormat[4:8]: # device records + if hasattr(self, name): + device = getattr(self, name) + if device is not None: + deviceItems.append((name, device)) + if deviceItems: + xmlWriter.begintag(valueName, simpleItems) + xmlWriter.newline() + for name, deviceRecord in deviceItems: + if deviceRecord is not None: + deviceRecord.toXML(xmlWriter, font, name=name) + xmlWriter.endtag(valueName) + xmlWriter.newline() + else: + xmlWriter.simpletag(valueName, simpleItems) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + from . import otTables + + for k, v in attrs.items(): + setattr(self, k, int(v)) + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + value = getattr(otTables, name)() + for elem2 in content: + if not isinstance(elem2, tuple): + continue + name2, attrs2, content2 = elem2 + value.fromXML(name2, attrs2, content2, font) + setattr(self, name, value) + + def __ne__(self, other): + result = self.__eq__(other) + return result if result is NotImplemented else not result + + def __eq__(self, other): + if type(self) != type(other): + return NotImplemented + return self.__dict__ == other.__dict__ diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otConverters.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otConverters.py new file mode 100644 index 0000000000000000000000000000000000000000..732daadf64284d253049a946f071aee8d7e7114b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otConverters.py @@ -0,0 +1,2334 @@ +from fontTools.misc.fixedTools import ( + fixedToFloat as fi2fl, + floatToFixed as fl2fi, + floatToFixedToStr as fl2str, + strToFixedToFloat as str2fl, + ensureVersionIsLong as fi2ve, + versionToFixed as ve2fi, +) +from fontTools.ttLib.tables.TupleVariation import TupleVariation +from fontTools.misc.roundTools import nearestMultipleShortestRepr, otRound +from fontTools.misc.textTools import bytesjoin, tobytes, tostr, pad, safeEval +from fontTools.misc.lazyTools import LazyList +from fontTools.ttLib import OPTIMIZE_FONT_SPEED, getSearchRange +from .otBase import ( + CountReference, + FormatSwitchingBaseTable, + OTTableReader, + OTTableWriter, + ValueRecordFactory, +) +from .otDataSchema import FieldSpec +from .otTables import ( + lookupTypes, + VarCompositeGlyph, + AATStateTable, + AATState, + AATAction, + ContextualMorphAction, + LigatureMorphAction, + InsertionMorphAction, + MorxSubtable, + ExtendMode as _ExtendMode, + CompositeMode as _CompositeMode, + NO_VARIATION_INDEX, +) +from itertools import zip_longest, accumulate +from functools import partial +from types import SimpleNamespace +import re +import struct +from typing import Optional +import logging +from enum import IntFlag + +log = logging.getLogger(__name__) +istuple = lambda t: isinstance(t, tuple) + + +def buildConverters(tableSpec: list[FieldSpec], tableNamespace): + """Given a table spec from otData.py, build a converter object for each + field of the table. This is called for each table in otData.py, and + the results are assigned to the corresponding class in otTables.py.""" + converters = [] + convertersByName = {} + for spec in tableSpec: + tableName = spec.name + if spec.name.startswith("ValueFormat"): + assert spec.type == "uint16" + converterClass = ValueFormat + elif spec.name.endswith("Count") or spec.name in ("StructLength", "MorphType"): + converterClass = { + "uint8": ComputedUInt8, + "uint16": ComputedUShort, + "uint32": ComputedULong, + }[spec.type] + elif spec.name == "SubTable": + converterClass = SubTable + elif spec.name == "ExtSubTable": + converterClass = ExtSubTable + elif spec.name == "SubStruct": + converterClass = SubStruct + elif spec.name == "FeatureParams": + converterClass = FeatureParams + elif spec.name in ("CIDGlyphMapping", "GlyphCIDMapping"): + converterClass = StructWithLength + else: + if not spec.type in converterMapping and "(" not in spec.type: + tableName = spec.type + converterClass = Struct + else: + converterClass = eval(spec.type, tableNamespace, converterMapping) + + conv = converterClass( + spec.name, spec.repeat, spec.aux, description=spec.description + ) + + if conv.tableClass: + # A "template" such as OffsetTo(AType) knows the table class already + tableClass = conv.tableClass + elif spec.type in ("MortChain", "MortSubtable", "MorxChain"): + tableClass = tableNamespace.get(spec.type) + else: + tableClass = tableNamespace.get(tableName) + + if not conv.tableClass: + conv.tableClass = tableClass + + if spec.name in ["SubTable", "ExtSubTable", "SubStruct"]: + conv.lookupTypes = tableNamespace["lookupTypes"] + # also create reverse mapping + for t in conv.lookupTypes.values(): + for cls in t.values(): + convertersByName[cls.__name__] = Table( + spec.name, spec.repeat, spec.aux, cls + ) + if spec.name == "FeatureParams": + conv.featureParamTypes = tableNamespace["featureParamTypes"] + conv.defaultFeatureParams = tableNamespace["FeatureParams"] + for cls in conv.featureParamTypes.values(): + convertersByName[cls.__name__] = Table( + spec.name, spec.repeat, spec.aux, cls + ) + converters.append(conv) + assert spec.name not in convertersByName, spec.name + convertersByName[spec.name] = conv + return converters, convertersByName + + +class BaseConverter(object): + """Base class for converter objects. Apart from the constructor, this + is an abstract class.""" + + def __init__(self, name, repeat, aux, tableClass=None, *, description=""): + self.name = name + self.repeat = repeat + self.aux = aux + if self.aux and not self.repeat: + self.aux = compile(self.aux, "<string>", "eval") + self.tableClass = tableClass + self.isCount = name.endswith("Count") or name in [ + "DesignAxisRecordSize", + "ValueRecordSize", + ] + self.isLookupType = name.endswith("LookupType") or name == "MorphType" + self.isPropagated = name in [ + "ClassCount", + "Class2Count", + "FeatureTag", + "SettingsCount", + "VarRegionCount", + "MappingCount", + "RegionAxisCount", + "DesignAxisCount", + "DesignAxisRecordSize", + "AxisValueCount", + "ValueRecordSize", + "AxisCount", + "BaseGlyphRecordCount", + "LayerRecordCount", + "AxisIndicesList", + # IFT - propagated to GlyphMap/FeatureMap/MappingEntries subtables + "NumGlyphs", + "MaxGlyphMapEntryIndex", + "MaxEntryIndex", + "NumEntries", + ] + self.description = description + + def readArray(self, reader, font, tableDict, count): + """Read an array of values from the reader.""" + lazy = font.lazy and count > 8 + if lazy: + recordSize = self.getRecordSize(reader) + if recordSize is NotImplemented: + lazy = False + if not lazy: + l = [] + for i in range(count): + l.append(self.read(reader, font, tableDict)) + return l + else: + + def get_read_item(): + reader_copy = reader.copy() + pos = reader.pos + + def read_item(i): + reader_copy.seek(pos + i * recordSize) + return self.read(reader_copy, font, {}) + + return read_item + + read_item = get_read_item() + l = LazyList(read_item for i in range(count)) + reader.advance(count * recordSize) + + return l + + def getRecordSize(self, reader): + if hasattr(self, "staticSize"): + return self.staticSize + return NotImplemented + + def read(self, reader, font, tableDict): + """Read a value from the reader.""" + raise NotImplementedError(self) + + def writeArray(self, writer, font, tableDict, values): + try: + for i, value in enumerate(values): + self.write(writer, font, tableDict, value, i) + except Exception as e: + e.args = e.args + (i,) + raise + + def write(self, writer, font, tableDict, value, repeatIndex=None): + """Write a value to the writer.""" + raise NotImplementedError(self) + + def xmlRead(self, attrs, content, font): + """Read a value from XML.""" + raise NotImplementedError(self) + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + """Write a value to XML.""" + raise NotImplementedError(self) + + varIndexBasePlusOffsetRE = re.compile(r"VarIndexBase\s*\+\s*(\d+)") + + def getVarIndexOffset(self) -> Optional[int]: + """If description has `VarIndexBase + {offset}`, return the offset else None.""" + m = self.varIndexBasePlusOffsetRE.search(self.description) + if not m: + return None + return int(m.group(1)) + + +class SimpleValue(BaseConverter): + @staticmethod + def toString(value): + return value + + @staticmethod + def fromString(value): + return value + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.simpletag(name, attrs + [("value", self.toString(value))]) + xmlWriter.newline() + + def xmlRead(self, attrs, content, font): + return self.fromString(attrs["value"]) + + +class OptionalValue(SimpleValue): + DEFAULT = None + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + if value != self.DEFAULT: + attrs.append(("value", self.toString(value))) + xmlWriter.simpletag(name, attrs) + xmlWriter.newline() + + def xmlRead(self, attrs, content, font): + if "value" in attrs: + return self.fromString(attrs["value"]) + return self.DEFAULT + + +class IntValue(SimpleValue): + @staticmethod + def fromString(value): + return int(value, 0) + + +class Long(IntValue): + staticSize = 4 + + def read(self, reader, font, tableDict): + return reader.readLong() + + def readArray(self, reader, font, tableDict, count): + return reader.readLongArray(count) + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.writeLong(value) + + def writeArray(self, writer, font, tableDict, values): + writer.writeLongArray(values) + + +class ULong(IntValue): + staticSize = 4 + + def read(self, reader, font, tableDict): + return reader.readULong() + + def readArray(self, reader, font, tableDict, count): + return reader.readULongArray(count) + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.writeULong(value) + + def writeArray(self, writer, font, tableDict, values): + writer.writeULongArray(values) + + +class Flags32(ULong): + @staticmethod + def toString(value): + return "0x%08X" % value + + +class VarIndex(OptionalValue, ULong): + DEFAULT = NO_VARIATION_INDEX + + +class Short(IntValue): + staticSize = 2 + + def read(self, reader, font, tableDict): + return reader.readShort() + + def readArray(self, reader, font, tableDict, count): + return reader.readShortArray(count) + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.writeShort(value) + + def writeArray(self, writer, font, tableDict, values): + writer.writeShortArray(values) + + +class UShort(IntValue): + staticSize = 2 + + def read(self, reader, font, tableDict): + return reader.readUShort() + + def readArray(self, reader, font, tableDict, count): + return reader.readUShortArray(count) + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.writeUShort(value) + + def writeArray(self, writer, font, tableDict, values): + writer.writeUShortArray(values) + + +class Int8(IntValue): + staticSize = 1 + + def read(self, reader, font, tableDict): + return reader.readInt8() + + def readArray(self, reader, font, tableDict, count): + return reader.readInt8Array(count) + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.writeInt8(value) + + def writeArray(self, writer, font, tableDict, values): + writer.writeInt8Array(values) + + +class UInt8(IntValue): + staticSize = 1 + + def read(self, reader, font, tableDict): + return reader.readUInt8() + + def readArray(self, reader, font, tableDict, count): + return reader.readUInt8Array(count) + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.writeUInt8(value) + + def writeArray(self, writer, font, tableDict, values): + writer.writeUInt8Array(values) + + +class UInt24(IntValue): + staticSize = 3 + + def read(self, reader, font, tableDict): + return reader.readUInt24() + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.writeUInt24(value) + + +class ComputedInt(IntValue): + def xmlWrite(self, xmlWriter, font, value, name, attrs): + if value is not None: + xmlWriter.comment("%s=%s" % (name, value)) + xmlWriter.newline() + + +class ComputedUInt8(ComputedInt, UInt8): + pass + + +class ComputedUShort(ComputedInt, UShort): + pass + + +class ComputedULong(ComputedInt, ULong): + pass + + +class Tag(SimpleValue): + staticSize = 4 + + def read(self, reader, font, tableDict): + return reader.readTag() + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.writeTag(value) + + +class GlyphID(SimpleValue): + staticSize = 2 + typecode = "H" + + def readArray(self, reader, font, tableDict, count): + return font.getGlyphNameMany( + reader.readArray(self.typecode, self.staticSize, count) + ) + + def read(self, reader, font, tableDict): + return font.getGlyphName(reader.readValue(self.typecode, self.staticSize)) + + def writeArray(self, writer, font, tableDict, values): + writer.writeArray(self.typecode, font.getGlyphIDMany(values)) + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.writeValue(self.typecode, font.getGlyphID(value)) + + +class GlyphID32(GlyphID): + staticSize = 4 + typecode = "L" + + +class NameID(UShort): + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.simpletag(name, attrs + [("value", value)]) + if font and value: + nameTable = font.get("name") + if nameTable: + name = nameTable.getDebugName(value) + xmlWriter.write(" ") + if name: + xmlWriter.comment(name) + elif value == 0xFFFF: + xmlWriter.comment("None") + else: + xmlWriter.comment("missing from name table") + log.warning("name id %d missing from name table" % value) + xmlWriter.newline() + + +class STATFlags(UShort): + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.simpletag(name, attrs + [("value", value)]) + flags = [] + if value & 0x01: + flags.append("OlderSiblingFontAttribute") + if value & 0x02: + flags.append("ElidableAxisValueName") + if flags: + xmlWriter.write(" ") + xmlWriter.comment(" ".join(flags)) + xmlWriter.newline() + + +class FloatValue(SimpleValue): + @staticmethod + def fromString(value): + return float(value) + + +class DeciPoints(FloatValue): + staticSize = 2 + + def read(self, reader, font, tableDict): + return reader.readUShort() / 10 + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.writeUShort(round(value * 10)) + + +class BaseFixedValue(FloatValue): + staticSize = NotImplemented + precisionBits = NotImplemented + readerMethod = NotImplemented + writerMethod = NotImplemented + + def read(self, reader, font, tableDict): + return self.fromInt(getattr(reader, self.readerMethod)()) + + def write(self, writer, font, tableDict, value, repeatIndex=None): + getattr(writer, self.writerMethod)(self.toInt(value)) + + @classmethod + def fromInt(cls, value): + return fi2fl(value, cls.precisionBits) + + @classmethod + def toInt(cls, value): + return fl2fi(value, cls.precisionBits) + + @classmethod + def fromString(cls, value): + return str2fl(value, cls.precisionBits) + + @classmethod + def toString(cls, value): + return fl2str(value, cls.precisionBits) + + +class Fixed(BaseFixedValue): + staticSize = 4 + precisionBits = 16 + readerMethod = "readLong" + writerMethod = "writeLong" + + +class F2Dot14(BaseFixedValue): + staticSize = 2 + precisionBits = 14 + readerMethod = "readShort" + writerMethod = "writeShort" + + +class Angle(F2Dot14): + # angles are specified in degrees, and encoded as F2Dot14 fractions of half + # circle: e.g. 1.0 => 180, -0.5 => -90, -2.0 => -360, etc. + bias = 0.0 + factor = 1.0 / (1 << 14) * 180 # 0.010986328125 + + @classmethod + def fromInt(cls, value): + return (super().fromInt(value) + cls.bias) * 180 + + @classmethod + def toInt(cls, value): + return super().toInt((value / 180) - cls.bias) + + @classmethod + def fromString(cls, value): + # quantize to nearest multiples of minimum fixed-precision angle + return otRound(float(value) / cls.factor) * cls.factor + + @classmethod + def toString(cls, value): + return nearestMultipleShortestRepr(value, cls.factor) + + +class BiasedAngle(Angle): + # A bias of 1.0 is used in the representation of start and end angles + # of COLRv1 PaintSweepGradients to allow for encoding +360deg + bias = 1.0 + + +class Version(SimpleValue): + staticSize = 4 + + def read(self, reader, font, tableDict): + value = reader.readLong() + return value + + def write(self, writer, font, tableDict, value, repeatIndex=None): + value = fi2ve(value) + writer.writeLong(value) + + @staticmethod + def fromString(value): + return ve2fi(value) + + @staticmethod + def toString(value): + return "0x%08x" % value + + @staticmethod + def fromFloat(v): + return fl2fi(v, 16) + + +class Char64(SimpleValue): + """An ASCII string with up to 64 characters. + + Unused character positions are filled with 0x00 bytes. + Used in Apple AAT fonts in the `gcid` table. + """ + + staticSize = 64 + + def read(self, reader, font, tableDict): + data = reader.readData(self.staticSize) + zeroPos = data.find(b"\0") + if zeroPos >= 0: + data = data[:zeroPos] + s = tostr(data, encoding="ascii", errors="replace") + if s != tostr(data, encoding="ascii", errors="ignore"): + log.warning('replaced non-ASCII characters in "%s"' % s) + return s + + def write(self, writer, font, tableDict, value, repeatIndex=None): + data = tobytes(value, encoding="ascii", errors="replace") + if data != tobytes(value, encoding="ascii", errors="ignore"): + log.warning('replacing non-ASCII characters in "%s"' % value) + if len(data) > self.staticSize: + log.warning( + 'truncating overlong "%s" to %d bytes' % (value, self.staticSize) + ) + data = (data + b"\0" * self.staticSize)[: self.staticSize] + writer.writeData(data) + + +class Struct(BaseConverter): + def getRecordSize(self, reader): + return self.tableClass and self.tableClass.getRecordSize(reader) + + def read(self, reader, font, tableDict): + table = self.tableClass() + table.decompile(reader, font) + return table + + def write(self, writer, font, tableDict, value, repeatIndex=None): + value.compile(writer, font) + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + if value is None: + if attrs: + # If there are attributes (probably index), then + # don't drop this even if it's NULL. It will mess + # up the array indices of the containing element. + xmlWriter.simpletag(name, attrs + [("empty", 1)]) + xmlWriter.newline() + else: + pass # NULL table, ignore + else: + value.toXML(xmlWriter, font, attrs, name=name) + + def xmlRead(self, attrs, content, font): + if "empty" in attrs and safeEval(attrs["empty"]): + return None + table = self.tableClass() + Format = attrs.get("Format") + if Format is not None: + table.Format = int(Format) + + noPostRead = not hasattr(table, "postRead") + if noPostRead: + # TODO Cache table.hasPropagated. + cleanPropagation = False + for conv in table.getConverters(): + if conv.isPropagated: + cleanPropagation = True + if not hasattr(font, "_propagator"): + font._propagator = {} + propagator = font._propagator + assert conv.name not in propagator, (conv.name, propagator) + setattr(table, conv.name, None) + propagator[conv.name] = CountReference(table.__dict__, conv.name) + + for element in content: + if isinstance(element, tuple): + name, attrs, content = element + table.fromXML(name, attrs, content, font) + else: + pass + + table.populateDefaults(propagator=getattr(font, "_propagator", None)) + + if noPostRead: + if cleanPropagation: + for conv in table.getConverters(): + if conv.isPropagated: + propagator = font._propagator + del propagator[conv.name] + if not propagator: + del font._propagator + + return table + + def __repr__(self): + return "Struct of " + repr(self.tableClass) + + +class StructWithLength(Struct): + def read(self, reader, font, tableDict): + pos = reader.pos + table = self.tableClass() + table.decompile(reader, font) + reader.seek(pos + table.StructLength) + return table + + def write(self, writer, font, tableDict, value, repeatIndex=None): + for convIndex, conv in enumerate(value.getConverters()): + if conv.name == "StructLength": + break + lengthIndex = len(writer.items) + convIndex + if isinstance(value, FormatSwitchingBaseTable): + lengthIndex += 1 # implicit Format field + deadbeef = {1: 0xDE, 2: 0xDEAD, 4: 0xDEADBEEF}[conv.staticSize] + + before = writer.getDataLength() + value.StructLength = deadbeef + value.compile(writer, font) + length = writer.getDataLength() - before + lengthWriter = writer.getSubWriter() + conv.write(lengthWriter, font, tableDict, length) + assert writer.items[lengthIndex] == b"\xde\xad\xbe\xef"[: conv.staticSize] + writer.items[lengthIndex] = lengthWriter.getAllData() + + +class Table(Struct): + staticSize = 2 + + def readOffset(self, reader): + return reader.readUShort() + + def writeNullOffset(self, writer): + writer.writeUShort(0) + + def read(self, reader, font, tableDict): + offset = self.readOffset(reader) + if offset == 0: + return None + table = self.tableClass() + reader = reader.getSubReader(offset) + if font.lazy: + table.reader = reader + table.font = font + else: + table.decompile(reader, font) + return table + + def write(self, writer, font, tableDict, value, repeatIndex=None): + if value is None: + self.writeNullOffset(writer) + else: + subWriter = writer.getSubWriter() + subWriter.name = self.name + if repeatIndex is not None: + subWriter.repeatIndex = repeatIndex + writer.writeSubTable(subWriter, offsetSize=self.staticSize) + value.compile(subWriter, font) + + +class LTable(Table): + staticSize = 4 + + def readOffset(self, reader): + return reader.readULong() + + def writeNullOffset(self, writer): + writer.writeULong(0) + + +# Table pointed to by a 24-bit, 3-byte long offset +class Table24(Table): + staticSize = 3 + + def readOffset(self, reader): + return reader.readUInt24() + + def writeNullOffset(self, writer): + writer.writeUInt24(0) + + +# TODO Clean / merge the SubTable and SubStruct + + +class SubStruct(Struct): + def getConverter(self, tableType, lookupType): + tableClass = self.lookupTypes[tableType][lookupType] + return self.__class__(self.name, self.repeat, self.aux, tableClass) + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + super(SubStruct, self).xmlWrite(xmlWriter, font, value, None, attrs) + + +class SubTable(Table): + def getConverter(self, tableType, lookupType): + tableClass = self.lookupTypes[tableType][lookupType] + return self.__class__(self.name, self.repeat, self.aux, tableClass) + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + super(SubTable, self).xmlWrite(xmlWriter, font, value, None, attrs) + + +class ExtSubTable(LTable, SubTable): + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer.Extension = True # actually, mere presence of the field flags it as an Ext Subtable writer. + Table.write(self, writer, font, tableDict, value, repeatIndex) + + +class FeatureParams(Table): + def getConverter(self, featureTag): + tableClass = self.featureParamTypes.get(featureTag, self.defaultFeatureParams) + return self.__class__(self.name, self.repeat, self.aux, tableClass) + + +class ValueFormat(IntValue): + staticSize = 2 + + def __init__(self, name, repeat, aux, tableClass=None, *, description=""): + BaseConverter.__init__( + self, name, repeat, aux, tableClass, description=description + ) + self.which = "ValueFormat" + ("2" if name[-1] == "2" else "1") + + def read(self, reader, font, tableDict): + format = reader.readUShort() + reader[self.which] = ValueRecordFactory(format) + return format + + def write(self, writer, font, tableDict, format, repeatIndex=None): + writer.writeUShort(format) + writer[self.which] = ValueRecordFactory(format) + + +class ValueRecord(ValueFormat): + def getRecordSize(self, reader): + return 2 * len(reader[self.which]) + + def read(self, reader, font, tableDict): + return reader[self.which].readValueRecord(reader, font) + + def write(self, writer, font, tableDict, value, repeatIndex=None): + writer[self.which].writeValueRecord(writer, font, value) + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + if value is None: + pass # NULL table, ignore + else: + value.toXML(xmlWriter, font, self.name, attrs) + + def xmlRead(self, attrs, content, font): + from .otBase import ValueRecord + + value = ValueRecord() + value.fromXML(None, attrs, content, font) + return value + + +class AATLookup(BaseConverter): + BIN_SEARCH_HEADER_SIZE = 10 + + def __init__(self, name, repeat, aux, tableClass, *, description=""): + BaseConverter.__init__( + self, name, repeat, aux, tableClass, description=description + ) + if issubclass(self.tableClass, SimpleValue): + self.converter = self.tableClass(name="Value", repeat=None, aux=None) + else: + self.converter = Table( + name="Value", repeat=None, aux=None, tableClass=self.tableClass + ) + + def read(self, reader, font, tableDict): + format = reader.readUShort() + if format == 0: + return self.readFormat0(reader, font) + elif format == 2: + return self.readFormat2(reader, font) + elif format == 4: + return self.readFormat4(reader, font) + elif format == 6: + return self.readFormat6(reader, font) + elif format == 8: + return self.readFormat8(reader, font) + else: + assert False, "unsupported lookup format: %d" % format + + def write(self, writer, font, tableDict, value, repeatIndex=None): + values = list( + sorted([(font.getGlyphID(glyph), val) for glyph, val in value.items()]) + ) + # TODO: Also implement format 4. + formats = list( + sorted( + filter( + None, + [ + self.buildFormat0(writer, font, values), + self.buildFormat2(writer, font, values), + self.buildFormat6(writer, font, values), + self.buildFormat8(writer, font, values), + ], + ) + ) + ) + # We use the format ID as secondary sort key to make the output + # deterministic when multiple formats have same encoded size. + dataSize, lookupFormat, writeMethod = formats[0] + pos = writer.getDataLength() + writeMethod() + actualSize = writer.getDataLength() - pos + assert ( + actualSize == dataSize + ), "AATLookup format %d claimed to write %d bytes, but wrote %d" % ( + lookupFormat, + dataSize, + actualSize, + ) + + @staticmethod + def writeBinSearchHeader(writer, numUnits, unitSize): + writer.writeUShort(unitSize) + writer.writeUShort(numUnits) + searchRange, entrySelector, rangeShift = getSearchRange( + n=numUnits, itemSize=unitSize + ) + writer.writeUShort(searchRange) + writer.writeUShort(entrySelector) + writer.writeUShort(rangeShift) + + def buildFormat0(self, writer, font, values): + numGlyphs = len(font.getGlyphOrder()) + if len(values) != numGlyphs: + return None + valueSize = self.converter.staticSize + return ( + 2 + numGlyphs * valueSize, + 0, + lambda: self.writeFormat0(writer, font, values), + ) + + def writeFormat0(self, writer, font, values): + writer.writeUShort(0) + for glyphID_, value in values: + self.converter.write( + writer, font, tableDict=None, value=value, repeatIndex=None + ) + + def buildFormat2(self, writer, font, values): + segStart, segValue = values[0] + segEnd = segStart + segments = [] + for glyphID, curValue in values[1:]: + if glyphID != segEnd + 1 or curValue != segValue: + segments.append((segStart, segEnd, segValue)) + segStart = segEnd = glyphID + segValue = curValue + else: + segEnd = glyphID + segments.append((segStart, segEnd, segValue)) + valueSize = self.converter.staticSize + numUnits, unitSize = len(segments) + 1, valueSize + 4 + return ( + 2 + self.BIN_SEARCH_HEADER_SIZE + numUnits * unitSize, + 2, + lambda: self.writeFormat2(writer, font, segments), + ) + + def writeFormat2(self, writer, font, segments): + writer.writeUShort(2) + valueSize = self.converter.staticSize + numUnits, unitSize = len(segments), valueSize + 4 + self.writeBinSearchHeader(writer, numUnits, unitSize) + for firstGlyph, lastGlyph, value in segments: + writer.writeUShort(lastGlyph) + writer.writeUShort(firstGlyph) + self.converter.write( + writer, font, tableDict=None, value=value, repeatIndex=None + ) + writer.writeUShort(0xFFFF) + writer.writeUShort(0xFFFF) + writer.writeData(b"\x00" * valueSize) + + def buildFormat6(self, writer, font, values): + valueSize = self.converter.staticSize + numUnits, unitSize = len(values), valueSize + 2 + return ( + 2 + self.BIN_SEARCH_HEADER_SIZE + (numUnits + 1) * unitSize, + 6, + lambda: self.writeFormat6(writer, font, values), + ) + + def writeFormat6(self, writer, font, values): + writer.writeUShort(6) + valueSize = self.converter.staticSize + numUnits, unitSize = len(values), valueSize + 2 + self.writeBinSearchHeader(writer, numUnits, unitSize) + for glyphID, value in values: + writer.writeUShort(glyphID) + self.converter.write( + writer, font, tableDict=None, value=value, repeatIndex=None + ) + writer.writeUShort(0xFFFF) + writer.writeData(b"\x00" * valueSize) + + def buildFormat8(self, writer, font, values): + minGlyphID, maxGlyphID = values[0][0], values[-1][0] + if len(values) != maxGlyphID - minGlyphID + 1: + return None + valueSize = self.converter.staticSize + return ( + 6 + len(values) * valueSize, + 8, + lambda: self.writeFormat8(writer, font, values), + ) + + def writeFormat8(self, writer, font, values): + firstGlyphID = values[0][0] + writer.writeUShort(8) + writer.writeUShort(firstGlyphID) + writer.writeUShort(len(values)) + for _, value in values: + self.converter.write( + writer, font, tableDict=None, value=value, repeatIndex=None + ) + + def readFormat0(self, reader, font): + numGlyphs = len(font.getGlyphOrder()) + data = self.converter.readArray(reader, font, tableDict=None, count=numGlyphs) + return {font.getGlyphName(k): value for k, value in enumerate(data)} + + def readFormat2(self, reader, font): + mapping = {} + pos = reader.pos - 2 # start of table is at UShort for format + unitSize, numUnits = reader.readUShort(), reader.readUShort() + assert unitSize >= 4 + self.converter.staticSize, unitSize + for i in range(numUnits): + reader.seek(pos + i * unitSize + 12) + last = reader.readUShort() + first = reader.readUShort() + value = self.converter.read(reader, font, tableDict=None) + if last != 0xFFFF: + for k in range(first, last + 1): + mapping[font.getGlyphName(k)] = value + return mapping + + def readFormat4(self, reader, font): + mapping = {} + pos = reader.pos - 2 # start of table is at UShort for format + unitSize = reader.readUShort() + assert unitSize >= 6, unitSize + for i in range(reader.readUShort()): + reader.seek(pos + i * unitSize + 12) + last = reader.readUShort() + first = reader.readUShort() + offset = reader.readUShort() + if last != 0xFFFF: + dataReader = reader.getSubReader(0) # relative to current position + dataReader.seek(pos + offset) # relative to start of table + data = self.converter.readArray( + dataReader, font, tableDict=None, count=last - first + 1 + ) + for k, v in enumerate(data): + mapping[font.getGlyphName(first + k)] = v + return mapping + + def readFormat6(self, reader, font): + mapping = {} + pos = reader.pos - 2 # start of table is at UShort for format + unitSize = reader.readUShort() + assert unitSize >= 2 + self.converter.staticSize, unitSize + for i in range(reader.readUShort()): + reader.seek(pos + i * unitSize + 12) + glyphID = reader.readUShort() + value = self.converter.read(reader, font, tableDict=None) + if glyphID != 0xFFFF: + mapping[font.getGlyphName(glyphID)] = value + return mapping + + def readFormat8(self, reader, font): + first = reader.readUShort() + count = reader.readUShort() + data = self.converter.readArray(reader, font, tableDict=None, count=count) + return {font.getGlyphName(first + k): value for (k, value) in enumerate(data)} + + def xmlRead(self, attrs, content, font): + value = {} + for element in content: + if isinstance(element, tuple): + name, a, eltContent = element + if name == "Lookup": + value[a["glyph"]] = self.converter.xmlRead(a, eltContent, font) + return value + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.begintag(name, attrs) + xmlWriter.newline() + for glyph, value in sorted(value.items()): + self.converter.xmlWrite( + xmlWriter, font, value=value, name="Lookup", attrs=[("glyph", glyph)] + ) + xmlWriter.endtag(name) + xmlWriter.newline() + + +# The AAT 'ankr' table has an unusual structure: An offset to an AATLookup +# followed by an offset to a glyph data table. Other than usual, the +# offsets in the AATLookup are not relative to the beginning of +# the beginning of the 'ankr' table, but relative to the glyph data table. +# So, to find the anchor data for a glyph, one needs to add the offset +# to the data table to the offset found in the AATLookup, and then use +# the sum of these two offsets to find the actual data. +class AATLookupWithDataOffset(BaseConverter): + def read(self, reader, font, tableDict): + lookupOffset = reader.readULong() + dataOffset = reader.readULong() + lookupReader = reader.getSubReader(lookupOffset) + lookup = AATLookup("DataOffsets", None, None, UShort) + offsets = lookup.read(lookupReader, font, tableDict) + result = {} + for glyph, offset in offsets.items(): + dataReader = reader.getSubReader(offset + dataOffset) + item = self.tableClass() + item.decompile(dataReader, font) + result[glyph] = item + return result + + def write(self, writer, font, tableDict, value, repeatIndex=None): + # We do not work with OTTableWriter sub-writers because + # the offsets in our AATLookup are relative to our data + # table, for which we need to provide an offset value itself. + # It might have been possible to somehow make a kludge for + # performing this indirect offset computation directly inside + # OTTableWriter. But this would have made the internal logic + # of OTTableWriter even more complex than it already is, + # so we decided to roll our own offset computation for the + # contents of the AATLookup and associated data table. + offsetByGlyph, offsetByData, dataLen = {}, {}, 0 + compiledData = [] + for glyph in sorted(value, key=font.getGlyphID): + subWriter = OTTableWriter() + value[glyph].compile(subWriter, font) + data = subWriter.getAllData() + offset = offsetByData.get(data, None) + if offset == None: + offset = dataLen + dataLen = dataLen + len(data) + offsetByData[data] = offset + compiledData.append(data) + offsetByGlyph[glyph] = offset + # For calculating the offsets to our AATLookup and data table, + # we can use the regular OTTableWriter infrastructure. + lookupWriter = writer.getSubWriter() + lookup = AATLookup("DataOffsets", None, None, UShort) + lookup.write(lookupWriter, font, tableDict, offsetByGlyph, None) + + dataWriter = writer.getSubWriter() + writer.writeSubTable(lookupWriter, offsetSize=4) + writer.writeSubTable(dataWriter, offsetSize=4) + for d in compiledData: + dataWriter.writeData(d) + + def xmlRead(self, attrs, content, font): + lookup = AATLookup("DataOffsets", None, None, self.tableClass) + return lookup.xmlRead(attrs, content, font) + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + lookup = AATLookup("DataOffsets", None, None, self.tableClass) + lookup.xmlWrite(xmlWriter, font, value, name, attrs) + + +class MorxSubtableConverter(BaseConverter): + _PROCESSING_ORDERS = { + # bits 30 and 28 of morx.CoverageFlags; see morx spec + (False, False): "LayoutOrder", + (True, False): "ReversedLayoutOrder", + (False, True): "LogicalOrder", + (True, True): "ReversedLogicalOrder", + } + + _PROCESSING_ORDERS_REVERSED = {val: key for key, val in _PROCESSING_ORDERS.items()} + + def __init__(self, name, repeat, aux, tableClass=None, *, description=""): + BaseConverter.__init__( + self, name, repeat, aux, tableClass, description=description + ) + + def _setTextDirectionFromCoverageFlags(self, flags, subtable): + if (flags & 0x20) != 0: + subtable.TextDirection = "Any" + elif (flags & 0x80) != 0: + subtable.TextDirection = "Vertical" + else: + subtable.TextDirection = "Horizontal" + + def read(self, reader, font, tableDict): + pos = reader.pos + m = MorxSubtable() + m.StructLength = reader.readULong() + flags = reader.readUInt8() + orderKey = ((flags & 0x40) != 0, (flags & 0x10) != 0) + m.ProcessingOrder = self._PROCESSING_ORDERS[orderKey] + self._setTextDirectionFromCoverageFlags(flags, m) + m.Reserved = reader.readUShort() + m.Reserved |= (flags & 0xF) << 16 + m.MorphType = reader.readUInt8() + m.SubFeatureFlags = reader.readULong() + tableClass = lookupTypes["morx"].get(m.MorphType) + if tableClass is None: + assert False, "unsupported 'morx' lookup type %s" % m.MorphType + # To decode AAT ligatures, we need to know the subtable size. + # The easiest way to pass this along is to create a new reader + # that works on just the subtable as its data. + headerLength = reader.pos - pos + data = reader.data[reader.pos : reader.pos + m.StructLength - headerLength] + assert len(data) == m.StructLength - headerLength + subReader = OTTableReader(data=data, tableTag=reader.tableTag) + m.SubStruct = tableClass() + m.SubStruct.decompile(subReader, font) + reader.seek(pos + m.StructLength) + return m + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.begintag(name, attrs) + xmlWriter.newline() + xmlWriter.comment("StructLength=%d" % value.StructLength) + xmlWriter.newline() + xmlWriter.simpletag("TextDirection", value=value.TextDirection) + xmlWriter.newline() + xmlWriter.simpletag("ProcessingOrder", value=value.ProcessingOrder) + xmlWriter.newline() + if value.Reserved != 0: + xmlWriter.simpletag("Reserved", value="0x%04x" % value.Reserved) + xmlWriter.newline() + xmlWriter.comment("MorphType=%d" % value.MorphType) + xmlWriter.newline() + xmlWriter.simpletag("SubFeatureFlags", value="0x%08x" % value.SubFeatureFlags) + xmlWriter.newline() + value.SubStruct.toXML(xmlWriter, font) + xmlWriter.endtag(name) + xmlWriter.newline() + + def xmlRead(self, attrs, content, font): + m = MorxSubtable() + covFlags = 0 + m.Reserved = 0 + for eltName, eltAttrs, eltContent in filter(istuple, content): + if eltName == "CoverageFlags": + # Only in XML from old versions of fonttools. + covFlags = safeEval(eltAttrs["value"]) + orderKey = ((covFlags & 0x40) != 0, (covFlags & 0x10) != 0) + m.ProcessingOrder = self._PROCESSING_ORDERS[orderKey] + self._setTextDirectionFromCoverageFlags(covFlags, m) + elif eltName == "ProcessingOrder": + m.ProcessingOrder = eltAttrs["value"] + assert m.ProcessingOrder in self._PROCESSING_ORDERS_REVERSED, ( + "unknown ProcessingOrder: %s" % m.ProcessingOrder + ) + elif eltName == "TextDirection": + m.TextDirection = eltAttrs["value"] + assert m.TextDirection in {"Horizontal", "Vertical", "Any"}, ( + "unknown TextDirection %s" % m.TextDirection + ) + elif eltName == "Reserved": + m.Reserved = safeEval(eltAttrs["value"]) + elif eltName == "SubFeatureFlags": + m.SubFeatureFlags = safeEval(eltAttrs["value"]) + elif eltName.endswith("Morph"): + m.fromXML(eltName, eltAttrs, eltContent, font) + else: + assert False, eltName + m.Reserved = (covFlags & 0xF) << 16 | m.Reserved + return m + + def write(self, writer, font, tableDict, value, repeatIndex=None): + covFlags = (value.Reserved & 0x000F0000) >> 16 + reverseOrder, logicalOrder = self._PROCESSING_ORDERS_REVERSED[ + value.ProcessingOrder + ] + covFlags |= 0x80 if value.TextDirection == "Vertical" else 0 + covFlags |= 0x40 if reverseOrder else 0 + covFlags |= 0x20 if value.TextDirection == "Any" else 0 + covFlags |= 0x10 if logicalOrder else 0 + value.CoverageFlags = covFlags + lengthIndex = len(writer.items) + before = writer.getDataLength() + value.StructLength = 0xDEADBEEF + # The high nibble of value.Reserved is actuallly encoded + # into coverageFlags, so we need to clear it here. + origReserved = value.Reserved # including high nibble + value.Reserved = value.Reserved & 0xFFFF # without high nibble + value.compile(writer, font) + value.Reserved = origReserved # restore original value + assert writer.items[lengthIndex] == b"\xde\xad\xbe\xef" + length = writer.getDataLength() - before + writer.items[lengthIndex] = struct.pack(">L", length) + + +# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6Tables.html#ExtendedStateHeader +# TODO: Untangle the implementation of the various lookup-specific formats. +class STXHeader(BaseConverter): + def __init__(self, name, repeat, aux, tableClass, *, description=""): + BaseConverter.__init__( + self, name, repeat, aux, tableClass, description=description + ) + assert issubclass(self.tableClass, AATAction) + self.classLookup = AATLookup("GlyphClasses", None, None, UShort) + if issubclass(self.tableClass, ContextualMorphAction): + self.perGlyphLookup = AATLookup("PerGlyphLookup", None, None, GlyphID) + else: + self.perGlyphLookup = None + + def read(self, reader, font, tableDict): + table = AATStateTable() + pos = reader.pos + classTableReader = reader.getSubReader(0) + stateArrayReader = reader.getSubReader(0) + entryTableReader = reader.getSubReader(0) + actionReader = None + ligaturesReader = None + table.GlyphClassCount = reader.readULong() + classTableReader.seek(pos + reader.readULong()) + stateArrayReader.seek(pos + reader.readULong()) + entryTableReader.seek(pos + reader.readULong()) + if self.perGlyphLookup is not None: + perGlyphTableReader = reader.getSubReader(0) + perGlyphTableReader.seek(pos + reader.readULong()) + if issubclass(self.tableClass, LigatureMorphAction): + actionReader = reader.getSubReader(0) + actionReader.seek(pos + reader.readULong()) + ligComponentReader = reader.getSubReader(0) + ligComponentReader.seek(pos + reader.readULong()) + ligaturesReader = reader.getSubReader(0) + ligaturesReader.seek(pos + reader.readULong()) + numLigComponents = (ligaturesReader.pos - ligComponentReader.pos) // 2 + assert numLigComponents >= 0 + table.LigComponents = ligComponentReader.readUShortArray(numLigComponents) + table.Ligatures = self._readLigatures(ligaturesReader, font) + elif issubclass(self.tableClass, InsertionMorphAction): + actionReader = reader.getSubReader(0) + actionReader.seek(pos + reader.readULong()) + table.GlyphClasses = self.classLookup.read(classTableReader, font, tableDict) + numStates = int( + (entryTableReader.pos - stateArrayReader.pos) / (table.GlyphClassCount * 2) + ) + for stateIndex in range(numStates): + state = AATState() + table.States.append(state) + for glyphClass in range(table.GlyphClassCount): + entryIndex = stateArrayReader.readUShort() + state.Transitions[glyphClass] = self._readTransition( + entryTableReader, entryIndex, font, actionReader + ) + if self.perGlyphLookup is not None: + table.PerGlyphLookups = self._readPerGlyphLookups( + table, perGlyphTableReader, font + ) + return table + + def _readTransition(self, reader, entryIndex, font, actionReader): + transition = self.tableClass() + entryReader = reader.getSubReader( + reader.pos + entryIndex * transition.staticSize + ) + transition.decompile(entryReader, font, actionReader) + return transition + + def _readLigatures(self, reader, font): + limit = len(reader.data) + numLigatureGlyphs = (limit - reader.pos) // 2 + return font.getGlyphNameMany(reader.readUShortArray(numLigatureGlyphs)) + + def _countPerGlyphLookups(self, table): + # Somewhat annoyingly, the morx table does not encode + # the size of the per-glyph table. So we need to find + # the maximum value that MorphActions use as index + # into this table. + numLookups = 0 + for state in table.States: + for t in state.Transitions.values(): + if isinstance(t, ContextualMorphAction): + if t.MarkIndex != 0xFFFF: + numLookups = max(numLookups, t.MarkIndex + 1) + if t.CurrentIndex != 0xFFFF: + numLookups = max(numLookups, t.CurrentIndex + 1) + return numLookups + + def _readPerGlyphLookups(self, table, reader, font): + pos = reader.pos + lookups = [] + for _ in range(self._countPerGlyphLookups(table)): + lookupReader = reader.getSubReader(0) + lookupReader.seek(pos + reader.readULong()) + lookups.append(self.perGlyphLookup.read(lookupReader, font, {})) + return lookups + + def write(self, writer, font, tableDict, value, repeatIndex=None): + glyphClassWriter = OTTableWriter() + self.classLookup.write( + glyphClassWriter, font, tableDict, value.GlyphClasses, repeatIndex=None + ) + glyphClassData = pad(glyphClassWriter.getAllData(), 2) + glyphClassCount = max(value.GlyphClasses.values()) + 1 + glyphClassTableOffset = 16 # size of STXHeader + if self.perGlyphLookup is not None: + glyphClassTableOffset += 4 + + glyphClassTableOffset += self.tableClass.actionHeaderSize + actionData, actionIndex = self.tableClass.compileActions(font, value.States) + stateArrayData, entryTableData = self._compileStates( + font, value.States, glyphClassCount, actionIndex + ) + stateArrayOffset = glyphClassTableOffset + len(glyphClassData) + entryTableOffset = stateArrayOffset + len(stateArrayData) + perGlyphOffset = entryTableOffset + len(entryTableData) + perGlyphData = pad(self._compilePerGlyphLookups(value, font), 4) + if actionData is not None: + actionOffset = entryTableOffset + len(entryTableData) + else: + actionOffset = None + + ligaturesOffset, ligComponentsOffset = None, None + ligComponentsData = self._compileLigComponents(value, font) + ligaturesData = self._compileLigatures(value, font) + if ligComponentsData is not None: + assert len(perGlyphData) == 0 + ligComponentsOffset = actionOffset + len(actionData) + ligaturesOffset = ligComponentsOffset + len(ligComponentsData) + + writer.writeULong(glyphClassCount) + writer.writeULong(glyphClassTableOffset) + writer.writeULong(stateArrayOffset) + writer.writeULong(entryTableOffset) + if self.perGlyphLookup is not None: + writer.writeULong(perGlyphOffset) + if actionOffset is not None: + writer.writeULong(actionOffset) + if ligComponentsOffset is not None: + writer.writeULong(ligComponentsOffset) + writer.writeULong(ligaturesOffset) + writer.writeData(glyphClassData) + writer.writeData(stateArrayData) + writer.writeData(entryTableData) + writer.writeData(perGlyphData) + if actionData is not None: + writer.writeData(actionData) + if ligComponentsData is not None: + writer.writeData(ligComponentsData) + if ligaturesData is not None: + writer.writeData(ligaturesData) + + def _compileStates(self, font, states, glyphClassCount, actionIndex): + stateArrayWriter = OTTableWriter() + entries, entryIDs = [], {} + for state in states: + for glyphClass in range(glyphClassCount): + transition = state.Transitions[glyphClass] + entryWriter = OTTableWriter() + transition.compile(entryWriter, font, actionIndex) + entryData = entryWriter.getAllData() + assert ( + len(entryData) == transition.staticSize + ), "%s has staticSize %d, " "but actually wrote %d bytes" % ( + repr(transition), + transition.staticSize, + len(entryData), + ) + entryIndex = entryIDs.get(entryData) + if entryIndex is None: + entryIndex = len(entries) + entryIDs[entryData] = entryIndex + entries.append(entryData) + stateArrayWriter.writeUShort(entryIndex) + stateArrayData = pad(stateArrayWriter.getAllData(), 4) + entryTableData = pad(bytesjoin(entries), 4) + return stateArrayData, entryTableData + + def _compilePerGlyphLookups(self, table, font): + if self.perGlyphLookup is None: + return b"" + numLookups = self._countPerGlyphLookups(table) + assert len(table.PerGlyphLookups) == numLookups, ( + "len(AATStateTable.PerGlyphLookups) is %d, " + "but the actions inside the table refer to %d" + % (len(table.PerGlyphLookups), numLookups) + ) + writer = OTTableWriter() + for lookup in table.PerGlyphLookups: + lookupWriter = writer.getSubWriter() + self.perGlyphLookup.write(lookupWriter, font, {}, lookup, None) + writer.writeSubTable(lookupWriter, offsetSize=4) + return writer.getAllData() + + def _compileLigComponents(self, table, font): + if not hasattr(table, "LigComponents"): + return None + writer = OTTableWriter() + for component in table.LigComponents: + writer.writeUShort(component) + return writer.getAllData() + + def _compileLigatures(self, table, font): + if not hasattr(table, "Ligatures"): + return None + writer = OTTableWriter() + for glyphName in table.Ligatures: + writer.writeUShort(font.getGlyphID(glyphName)) + return writer.getAllData() + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.begintag(name, attrs) + xmlWriter.newline() + xmlWriter.comment("GlyphClassCount=%s" % value.GlyphClassCount) + xmlWriter.newline() + for g, klass in sorted(value.GlyphClasses.items()): + xmlWriter.simpletag("GlyphClass", glyph=g, value=klass) + xmlWriter.newline() + for stateIndex, state in enumerate(value.States): + xmlWriter.begintag("State", index=stateIndex) + xmlWriter.newline() + for glyphClass, trans in sorted(state.Transitions.items()): + trans.toXML( + xmlWriter, + font=font, + attrs={"onGlyphClass": glyphClass}, + name="Transition", + ) + xmlWriter.endtag("State") + xmlWriter.newline() + for i, lookup in enumerate(value.PerGlyphLookups): + xmlWriter.begintag("PerGlyphLookup", index=i) + xmlWriter.newline() + for glyph, val in sorted(lookup.items()): + xmlWriter.simpletag("Lookup", glyph=glyph, value=val) + xmlWriter.newline() + xmlWriter.endtag("PerGlyphLookup") + xmlWriter.newline() + if hasattr(value, "LigComponents"): + xmlWriter.begintag("LigComponents") + xmlWriter.newline() + for i, val in enumerate(getattr(value, "LigComponents")): + xmlWriter.simpletag("LigComponent", index=i, value=val) + xmlWriter.newline() + xmlWriter.endtag("LigComponents") + xmlWriter.newline() + self._xmlWriteLigatures(xmlWriter, font, value, name, attrs) + xmlWriter.endtag(name) + xmlWriter.newline() + + def _xmlWriteLigatures(self, xmlWriter, font, value, name, attrs): + if not hasattr(value, "Ligatures"): + return + xmlWriter.begintag("Ligatures") + xmlWriter.newline() + for i, g in enumerate(getattr(value, "Ligatures")): + xmlWriter.simpletag("Ligature", index=i, glyph=g) + xmlWriter.newline() + xmlWriter.endtag("Ligatures") + xmlWriter.newline() + + def xmlRead(self, attrs, content, font): + table = AATStateTable() + for eltName, eltAttrs, eltContent in filter(istuple, content): + if eltName == "GlyphClass": + glyph = eltAttrs["glyph"] + value = eltAttrs["value"] + table.GlyphClasses[glyph] = safeEval(value) + elif eltName == "State": + state = self._xmlReadState(eltAttrs, eltContent, font) + table.States.append(state) + elif eltName == "PerGlyphLookup": + lookup = self.perGlyphLookup.xmlRead(eltAttrs, eltContent, font) + table.PerGlyphLookups.append(lookup) + elif eltName == "LigComponents": + table.LigComponents = self._xmlReadLigComponents( + eltAttrs, eltContent, font + ) + elif eltName == "Ligatures": + table.Ligatures = self._xmlReadLigatures(eltAttrs, eltContent, font) + table.GlyphClassCount = max(table.GlyphClasses.values()) + 1 + return table + + def _xmlReadState(self, attrs, content, font): + state = AATState() + for eltName, eltAttrs, eltContent in filter(istuple, content): + if eltName == "Transition": + glyphClass = safeEval(eltAttrs["onGlyphClass"]) + transition = self.tableClass() + transition.fromXML(eltName, eltAttrs, eltContent, font) + state.Transitions[glyphClass] = transition + return state + + def _xmlReadLigComponents(self, attrs, content, font): + ligComponents = [] + for eltName, eltAttrs, _eltContent in filter(istuple, content): + if eltName == "LigComponent": + ligComponents.append(safeEval(eltAttrs["value"])) + return ligComponents + + def _xmlReadLigatures(self, attrs, content, font): + ligs = [] + for eltName, eltAttrs, _eltContent in filter(istuple, content): + if eltName == "Ligature": + ligs.append(eltAttrs["glyph"]) + return ligs + + +class CIDGlyphMap(BaseConverter): + def read(self, reader, font, tableDict): + numCIDs = reader.readUShort() + result = {} + for cid, glyphID in enumerate(reader.readUShortArray(numCIDs)): + if glyphID != 0xFFFF: + result[cid] = font.getGlyphName(glyphID) + return result + + def write(self, writer, font, tableDict, value, repeatIndex=None): + items = {cid: font.getGlyphID(glyph) for cid, glyph in value.items()} + count = max(items) + 1 if items else 0 + writer.writeUShort(count) + for cid in range(count): + writer.writeUShort(items.get(cid, 0xFFFF)) + + def xmlRead(self, attrs, content, font): + result = {} + for eName, eAttrs, _eContent in filter(istuple, content): + if eName == "CID": + result[safeEval(eAttrs["cid"])] = eAttrs["glyph"].strip() + return result + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.begintag(name, attrs) + xmlWriter.newline() + for cid, glyph in sorted(value.items()): + if glyph is not None and glyph != 0xFFFF: + xmlWriter.simpletag("CID", cid=cid, glyph=glyph) + xmlWriter.newline() + xmlWriter.endtag(name) + xmlWriter.newline() + + +class GlyphCIDMap(BaseConverter): + def read(self, reader, font, tableDict): + glyphOrder = font.getGlyphOrder() + count = reader.readUShort() + cids = reader.readUShortArray(count) + if count > len(glyphOrder): + log.warning( + "GlyphCIDMap has %d elements, " + "but the font has only %d glyphs; " + "ignoring the rest" % (count, len(glyphOrder)) + ) + result = {} + for glyphID in range(min(len(cids), len(glyphOrder))): + cid = cids[glyphID] + if cid != 0xFFFF: + result[glyphOrder[glyphID]] = cid + return result + + def write(self, writer, font, tableDict, value, repeatIndex=None): + items = { + font.getGlyphID(g): cid + for g, cid in value.items() + if cid is not None and cid != 0xFFFF + } + count = max(items) + 1 if items else 0 + writer.writeUShort(count) + for glyphID in range(count): + writer.writeUShort(items.get(glyphID, 0xFFFF)) + + def xmlRead(self, attrs, content, font): + result = {} + for eName, eAttrs, _eContent in filter(istuple, content): + if eName == "CID": + result[eAttrs["glyph"]] = safeEval(eAttrs["value"]) + return result + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.begintag(name, attrs) + xmlWriter.newline() + for glyph, cid in sorted(value.items()): + if cid is not None and cid != 0xFFFF: + xmlWriter.simpletag("CID", glyph=glyph, value=cid) + xmlWriter.newline() + xmlWriter.endtag(name) + xmlWriter.newline() + + +class DeltaValue(BaseConverter): + def read(self, reader, font, tableDict): + StartSize = tableDict["StartSize"] + EndSize = tableDict["EndSize"] + DeltaFormat = tableDict["DeltaFormat"] + assert DeltaFormat in (1, 2, 3), "illegal DeltaFormat" + nItems = EndSize - StartSize + 1 + nBits = 1 << DeltaFormat + minusOffset = 1 << nBits + mask = (1 << nBits) - 1 + signMask = 1 << (nBits - 1) + + DeltaValue = [] + tmp, shift = 0, 0 + for i in range(nItems): + if shift == 0: + tmp, shift = reader.readUShort(), 16 + shift = shift - nBits + value = (tmp >> shift) & mask + if value & signMask: + value = value - minusOffset + DeltaValue.append(value) + return DeltaValue + + def write(self, writer, font, tableDict, value, repeatIndex=None): + StartSize = tableDict["StartSize"] + EndSize = tableDict["EndSize"] + DeltaFormat = tableDict["DeltaFormat"] + DeltaValue = value + assert DeltaFormat in (1, 2, 3), "illegal DeltaFormat" + nItems = EndSize - StartSize + 1 + nBits = 1 << DeltaFormat + assert len(DeltaValue) == nItems + mask = (1 << nBits) - 1 + + tmp, shift = 0, 16 + for value in DeltaValue: + shift = shift - nBits + tmp = tmp | ((value & mask) << shift) + if shift == 0: + writer.writeUShort(tmp) + tmp, shift = 0, 16 + if shift != 16: + writer.writeUShort(tmp) + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.simpletag(name, attrs + [("value", value)]) + xmlWriter.newline() + + def xmlRead(self, attrs, content, font): + return safeEval(attrs["value"]) + + +class VarIdxMapValue(BaseConverter): + def read(self, reader, font, tableDict): + fmt = tableDict["EntryFormat"] + nItems = tableDict["MappingCount"] + + innerBits = 1 + (fmt & 0x000F) + innerMask = (1 << innerBits) - 1 + outerMask = 0xFFFFFFFF - innerMask + outerShift = 16 - innerBits + + entrySize = 1 + ((fmt & 0x0030) >> 4) + readArray = { + 1: reader.readUInt8Array, + 2: reader.readUShortArray, + 3: reader.readUInt24Array, + 4: reader.readULongArray, + }[entrySize] + + return [ + (((raw & outerMask) << outerShift) | (raw & innerMask)) + for raw in readArray(nItems) + ] + + def write(self, writer, font, tableDict, value, repeatIndex=None): + fmt = tableDict["EntryFormat"] + mapping = value + writer["MappingCount"].setValue(len(mapping)) + + innerBits = 1 + (fmt & 0x000F) + innerMask = (1 << innerBits) - 1 + outerShift = 16 - innerBits + + entrySize = 1 + ((fmt & 0x0030) >> 4) + writeArray = { + 1: writer.writeUInt8Array, + 2: writer.writeUShortArray, + 3: writer.writeUInt24Array, + 4: writer.writeULongArray, + }[entrySize] + + writeArray( + [ + (((idx & 0xFFFF0000) >> outerShift) | (idx & innerMask)) + for idx in mapping + ] + ) + + +class VarDataValue(BaseConverter): + def read(self, reader, font, tableDict): + values = [] + + regionCount = tableDict["VarRegionCount"] + wordCount = tableDict["NumShorts"] + + # https://github.com/fonttools/fonttools/issues/2279 + longWords = bool(wordCount & 0x8000) + wordCount = wordCount & 0x7FFF + + if longWords: + readBigArray, readSmallArray = reader.readLongArray, reader.readShortArray + else: + readBigArray, readSmallArray = reader.readShortArray, reader.readInt8Array + + n1, n2 = min(regionCount, wordCount), max(regionCount, wordCount) + values.extend(readBigArray(n1)) + values.extend(readSmallArray(n2 - n1)) + if n2 > regionCount: # Padding + del values[regionCount:] + + return values + + def write(self, writer, font, tableDict, values, repeatIndex=None): + regionCount = tableDict["VarRegionCount"] + wordCount = tableDict["NumShorts"] + + # https://github.com/fonttools/fonttools/issues/2279 + longWords = bool(wordCount & 0x8000) + wordCount = wordCount & 0x7FFF + + writeBigArray, writeSmallArray = { + False: (writer.writeShortArray, writer.writeInt8Array), + True: (writer.writeLongArray, writer.writeShortArray), + }[longWords] + + n1, n2 = min(regionCount, wordCount), max(regionCount, wordCount) + writeBigArray(values[:n1]) + writeSmallArray(values[n1:regionCount]) + if n2 > regionCount: # Padding + writer.writeSmallArray([0] * (n2 - regionCount)) + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.simpletag(name, attrs + [("value", value)]) + xmlWriter.newline() + + def xmlRead(self, attrs, content, font): + return safeEval(attrs["value"]) + + +class TupleValues: + def read(self, data, font): + return TupleVariation.decompileDeltas_(None, data)[0] + + def write(self, writer, font, tableDict, values, repeatIndex=None): + optimizeSpeed = font.cfg[OPTIMIZE_FONT_SPEED] + return bytes( + TupleVariation.compileDeltaValues_(values, optimizeSize=not optimizeSpeed) + ) + + def xmlRead(self, attrs, content, font): + return safeEval(attrs["value"]) + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.simpletag(name, attrs + [("value", value)]) + xmlWriter.newline() + + +class CFF2Index(BaseConverter): + def __init__( + self, + name, + repeat, + aux, + tableClass=None, + *, + itemClass=None, + itemConverterClass=None, + description="", + ): + BaseConverter.__init__( + self, name, repeat, aux, tableClass, description=description + ) + self._itemClass = itemClass + self._converter = ( + itemConverterClass() if itemConverterClass is not None else None + ) + + def read(self, reader, font, tableDict): + count = reader.readULong() + if count == 0: + return [] + offSize = reader.readUInt8() + + def getReadArray(reader, offSize): + return { + 1: reader.readUInt8Array, + 2: reader.readUShortArray, + 3: reader.readUInt24Array, + 4: reader.readULongArray, + }[offSize] + + readArray = getReadArray(reader, offSize) + + lazy = font.lazy is not False and count > 8 + if not lazy: + offsets = readArray(count + 1) + items = [] + lastOffset = offsets.pop(0) + reader.readData(lastOffset - 1) # In case first offset is not 1 + + for offset in offsets: + assert lastOffset <= offset + item = reader.readData(offset - lastOffset) + + if self._itemClass is not None: + obj = self._itemClass() + obj.decompile(item, font, reader.localState) + item = obj + elif self._converter is not None: + item = self._converter.read(item, font) + + items.append(item) + lastOffset = offset + return items + else: + + def get_read_item(): + reader_copy = reader.copy() + offset_pos = reader.pos + data_pos = offset_pos + (count + 1) * offSize - 1 + readArray = getReadArray(reader_copy, offSize) + + def read_item(i): + reader_copy.seek(offset_pos + i * offSize) + offsets = readArray(2) + reader_copy.seek(data_pos + offsets[0]) + item = reader_copy.readData(offsets[1] - offsets[0]) + + if self._itemClass is not None: + obj = self._itemClass() + obj.decompile(item, font, reader_copy.localState) + item = obj + elif self._converter is not None: + item = self._converter.read(item, font) + return item + + return read_item + + read_item = get_read_item() + l = LazyList([read_item] * count) + + # TODO: Advance reader + + return l + + def write(self, writer, font, tableDict, values, repeatIndex=None): + items = values + + writer.writeULong(len(items)) + if not len(items): + return + + if self._itemClass is not None: + items = [item.compile(font) for item in items] + elif self._converter is not None: + items = [ + self._converter.write(writer, font, tableDict, item, i) + for i, item in enumerate(items) + ] + + offsets = [len(item) for item in items] + offsets = list(accumulate(offsets, initial=1)) + + lastOffset = offsets[-1] + offSize = ( + 1 + if lastOffset < 0x100 + else 2 if lastOffset < 0x10000 else 3 if lastOffset < 0x1000000 else 4 + ) + writer.writeUInt8(offSize) + + writeArray = { + 1: writer.writeUInt8Array, + 2: writer.writeUShortArray, + 3: writer.writeUInt24Array, + 4: writer.writeULongArray, + }[offSize] + + writeArray(offsets) + for item in items: + writer.writeData(item) + + def xmlRead(self, attrs, content, font): + if self._itemClass is not None: + obj = self._itemClass() + obj.fromXML(None, attrs, content, font) + return obj + elif self._converter is not None: + return self._converter.xmlRead(attrs, content, font) + else: + raise NotImplementedError() + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + if self._itemClass is not None: + for i, item in enumerate(value): + item.toXML(xmlWriter, font, [("index", i)], name) + elif self._converter is not None: + for i, item in enumerate(value): + self._converter.xmlWrite( + xmlWriter, font, item, name, attrs + [("index", i)] + ) + else: + raise NotImplementedError() + + +class LookupFlag(UShort): + def xmlWrite(self, xmlWriter, font, value, name, attrs): + xmlWriter.simpletag(name, attrs + [("value", value)]) + flags = [] + if value & 0x01: + flags.append("rightToLeft") + if value & 0x02: + flags.append("ignoreBaseGlyphs") + if value & 0x04: + flags.append("ignoreLigatures") + if value & 0x08: + flags.append("ignoreMarks") + if value & 0x10: + flags.append("useMarkFilteringSet") + if value & 0xFF00: + flags.append("markAttachmentType[%i]" % (value >> 8)) + if flags: + xmlWriter.comment(" ".join(flags)) + xmlWriter.newline() + + +class _UInt8Enum(UInt8): + enumClass = NotImplemented + + def read(self, reader, font, tableDict): + return self.enumClass(super().read(reader, font, tableDict)) + + @classmethod + def fromString(cls, value): + return getattr(cls.enumClass, value.upper()) + + @classmethod + def toString(cls, value): + return cls.enumClass(value).name.lower() + + +class ExtendMode(_UInt8Enum): + enumClass = _ExtendMode + + +class CompositeMode(_UInt8Enum): + enumClass = _CompositeMode + + +class MappingEntryFormat(IntFlag): + HAS_SUBSET_DEF = 0x01 + HAS_CHILD_ENTRIES = 0x02 + HAS_ENTRY_ID = 0x04 + HAS_PATCH_FORMAT = 0x08 + # The code point bias is essentially a 2bit flag. + CODE_POINT_BIAS_BITS = 0x30 + + @property + def bias_present_u16(self): + return (self & MappingEntryFormat.CODE_POINT_BIAS_BITS) == 0x20 + + @property + def bias_present_u24(self): + return (self & MappingEntryFormat.CODE_POINT_BIAS_BITS) == 0x30 + + +# Reference implementation: +# https://github.com/googlefonts/fontations/blob/main/read-fonts/src/tables/ift.rs +class MappingEntriesConverter(BaseConverter): + def read(self, reader, font, tableDict): + from fontTools.misc.fixedTools import fixedToFloat as fi2fl + from fontTools.misc.iftSparseBitSet import decode as sbsDecode + + entryCount = reader["NumEntries"] + entries = [] + lastEntryId = -1 + + for _ in range(entryCount): + entry = {} + formatFlags = MappingEntryFormat(reader.readUInt8()) + entry["formatFlags"] = formatFlags + + if formatFlags & MappingEntryFormat.HAS_SUBSET_DEF: + featureCount = reader.readUInt8() + features = [] + for _ in range(featureCount): + tag = reader.readTag() + features.append(tag) + entry["featureTags"] = features + designSpaceCount = reader.readUShort() + segments = [] + for _ in range(designSpaceCount): + segTag = reader.readTag() + start = fi2fl(reader.readLong(), 16) + end = fi2fl(reader.readLong(), 16) + segments.append({"tag": segTag, "start": start, "end": end}) + entry["designSpaceSegments"] = segments + + if formatFlags & MappingEntryFormat.HAS_CHILD_ENTRIES: + modeAndCount = reader.readUInt8() + conjunctive = bool(modeAndCount & 0x80) + childCount = modeAndCount & 0x7F + entry["childEntryConjunctive"] = conjunctive + childIndices = [] + for _ in range(childCount): + idx = reader.readUInt24() + childIndices.append(idx) + entry["childEntryIndices"] = childIndices + + if formatFlags & MappingEntryFormat.HAS_ENTRY_ID: + ids = [] + while True: + # Per IFT spec, Entry IDs are delta-encoded. The value is a + # signed 24-bit integer. The least significant bit is a + # continuation flag (1 if more IDs follow). The remaining + # upper bits are a signed integer, representing the delta + # from the previous ID. + encodedVal = reader.readInt24() + delta = encodedVal >> 1 + hasMore = encodedVal & 0x01 + currentEntryId = lastEntryId + 1 + delta + ids.append(currentEntryId) + lastEntryId = currentEntryId + + if not hasMore: + break + entry["entryIds"] = ids + else: + entry["entryIds"] = [lastEntryId + 1] + lastEntryId = lastEntryId + 1 + + if formatFlags & MappingEntryFormat.HAS_PATCH_FORMAT: + entry["patchFormat"] = reader.readUInt8() + + if formatFlags & MappingEntryFormat.CODE_POINT_BIAS_BITS: + bias = 0 + if formatFlags.bias_present_u24: + bias = reader.readUInt24() + elif formatFlags.bias_present_u16: + bias = reader.readUShort() + entry["codePointsBias"] = bias + + codepoints, consumed = sbsDecode( + reader.data[reader.pos :], bias=bias, maxValue=0x10FFFF + ) + reader.advance(consumed) + entry["codePoints"] = sorted(codepoints) + + entries.append(entry) + return entries + + def write(self, writer, font, tableDict, value, repeatIndex=None): + from fontTools.misc.iftSparseBitSet import encode as sbsEncode + from fontTools.misc.fixedTools import floatToFixed as fl2fi + + entries = value + lastId = -1 + + for entry in entries: + # TODO: Automatically recompute formatFlags based on the contents of + # entry. + formatFlags = entry.get("formatFlags", MappingEntryFormat(0)) + writer.writeUInt8(formatFlags.value) + + if formatFlags & MappingEntryFormat.HAS_SUBSET_DEF: + features = entry.get("featureTags", []) + writer.writeUInt8(len(features)) + for tag in features: + writer.writeTag(tag) + segments = entry.get("designSpaceSegments", []) + writer.writeUShort(len(segments)) + for seg in segments: + writer.writeTag(seg["tag"]) + writer.writeLong(fl2fi(seg["start"], 16)) + writer.writeLong(fl2fi(seg["end"], 16)) + + if formatFlags & MappingEntryFormat.HAS_CHILD_ENTRIES: + childIndices = entry.get("childEntryIndices", []) + conjunctive = entry.get("childEntryConjunctive", False) + modeAndCount = (len(childIndices) & 0x7F) | (0x80 if conjunctive else 0) + writer.writeUInt8(modeAndCount) + for idx in childIndices: + writer.writeUInt24(idx) + + if formatFlags & MappingEntryFormat.HAS_ENTRY_ID: + ids = entry.get("entryIds", []) + for i, eid in enumerate(ids): + # Recreate the delta-encoded Entry ID. The delta is shifted + # left by one bit, and the least significant bit is set to 1 + # if more IDs follow in the list. + delta = eid - lastId - 1 + has_more = 1 if i < len(ids) - 1 else 0 + encoded_value = (delta << 1) | has_more + writer.writeUInt24(encoded_value & 0xFFFFFF) + lastId = eid + else: + lastId += 1 + + if formatFlags & MappingEntryFormat.HAS_PATCH_FORMAT: + writer.writeUInt8(entry.get("patchFormat", 0)) + + if formatFlags & MappingEntryFormat.CODE_POINT_BIAS_BITS: + bias = entry.get("codePointsBias", 0) + if formatFlags.bias_present_u24: + writer.writeUInt24(bias) + elif formatFlags.bias_present_u16: + writer.writeUShort(bias) + codepoints = entry.get("codePoints", []) + writer.writeData(sbsEncode([c - bias for c in codepoints if c >= bias])) + + def xmlWrite(self, xmlWriter, font, value, name, attrs): + for entry in value: + format_flags = entry.get("formatFlags", MappingEntryFormat(0)) + xmlWriter.begintag("entry", formatFlags=format_flags.value) + xmlWriter.newline() + if "entryIds" in entry: + for eid in entry["entryIds"]: + xmlWriter.simpletag("entryId", value=eid) + xmlWriter.newline() + if "featureTags" in entry: + for tag in entry["featureTags"]: + xmlWriter.simpletag("featureTag", value=tag) + xmlWriter.newline() + if "designSpaceSegments" in entry: + for seg in entry["designSpaceSegments"]: + xmlWriter.simpletag( + "designSpaceSegment", + tag=seg["tag"], + start=seg["start"], + end=seg["end"], + ) + xmlWriter.newline() + if "childEntryIndices" in entry: + xmlWriter.simpletag( + "childEntryConjunctive", + value=int(entry.get("childEntryConjunctive", False)), + ) + xmlWriter.newline() + for idx in entry["childEntryIndices"]: + xmlWriter.simpletag("childEntry", index=idx) + xmlWriter.newline() + if "patchFormat" in entry: + xmlWriter.simpletag("patchFormat", value=entry["patchFormat"]) + xmlWriter.newline() + if "codePoints" in entry: + xmlWriter.simpletag( + "codePointsBias", value=entry.get("codePointsBias", 0) + ) + xmlWriter.newline() + codePointsStr = " ".join(str(c) for c in entry["codePoints"]) + xmlWriter.simpletag("codePoints", value=codePointsStr) + xmlWriter.newline() + xmlWriter.endtag("entry") + xmlWriter.newline() + + def xmlRead(self, attrs, content, font): + entries = [] + for elem in content: + if not isinstance(elem, tuple): + continue + name, attrs, content = elem + if name == "entry": + format_flags_int = safeEval(attrs.get("formatFlags", "0")) + entry = {"formatFlags": MappingEntryFormat(format_flags_int)} + for e_elem in content: + if not isinstance(e_elem, tuple): + continue + ename, eattrs, econtent = e_elem + if ename == "entryId": + entry.setdefault("entryIds", []).append( + safeEval(eattrs["value"]) + ) + elif ename == "featureTag": + entry.setdefault("featureTags", []).append(eattrs["value"]) + elif ename == "designSpaceSegment": + entry.setdefault("designSpaceSegments", []).append( + { + "tag": eattrs["tag"], + "start": float(eattrs["start"]), + "end": float(eattrs["end"]), + } + ) + elif ename == "childEntryConjunctive": + entry["childEntryConjunctive"] = bool(safeEval(eattrs["value"])) + elif ename == "childEntry": + entry.setdefault("childEntryIndices", []).append( + safeEval(eattrs["index"]) + ) + elif ename == "patchFormat": + entry["patchFormat"] = safeEval(eattrs["value"]) + elif ename == "codePointsBias": + entry["codePointsBias"] = safeEval(eattrs["value"]) + elif ename == "codePoints": + entry["codePoints"] = [ + int(c) for c in eattrs["value"].split() if c + ] + entries.append(entry) + return entries + + +converterMapping = { + # type class + "int8": Int8, + "int16": Short, + "int32": Long, + "uint8": UInt8, + "uint16": UShort, + "uint24": UInt24, + "uint32": ULong, + "char64": Char64, + "Flags32": Flags32, + "VarIndex": VarIndex, + "Version": Version, + "Tag": Tag, + "GlyphID": GlyphID, + "GlyphID32": GlyphID32, + "NameID": NameID, + "DeciPoints": DeciPoints, + "Fixed": Fixed, + "F2Dot14": F2Dot14, + "Angle": Angle, + "BiasedAngle": BiasedAngle, + "struct": Struct, + "MappingEntries": MappingEntriesConverter, + "Offset": Table, + "LOffset": LTable, + "Offset24": Table24, + "ValueRecord": ValueRecord, + "DeltaValue": DeltaValue, + "VarIdxMapValue": VarIdxMapValue, + "VarDataValue": VarDataValue, + "LookupFlag": LookupFlag, + "ExtendMode": ExtendMode, + "CompositeMode": CompositeMode, + "STATFlags": STATFlags, + "TupleList": partial(CFF2Index, itemConverterClass=TupleValues), + "VarCompositeGlyphList": partial(CFF2Index, itemClass=VarCompositeGlyph), + # AAT + "CIDGlyphMap": CIDGlyphMap, + "GlyphCIDMap": GlyphCIDMap, + "MortChain": StructWithLength, + "MortSubtable": StructWithLength, + "MorxChain": StructWithLength, + "MorxSubtable": MorxSubtableConverter, + # "Template" types + "AATLookup": lambda C: partial(AATLookup, tableClass=C), + "AATLookupWithDataOffset": lambda C: partial(AATLookupWithDataOffset, tableClass=C), + "STXHeader": lambda C: partial(STXHeader, tableClass=C), + "OffsetTo": lambda C: partial(Table, tableClass=C), + "LOffsetTo": lambda C: partial(LTable, tableClass=C), + "LOffset24To": lambda C: partial(Table24, tableClass=C), +} diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otData.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otData.py new file mode 100644 index 0000000000000000000000000000000000000000..06934390eb48aac341df8d32f94f447d573b133c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otData.py @@ -0,0 +1,5983 @@ +from .otDataSchema import FieldSpec + +otData = [ + # + # common + # + ("LookupOrder", []), + ( + "ScriptList", + [ + FieldSpec("uint16", "ScriptCount", description="Number of ScriptRecords"), + FieldSpec( + "struct", + "ScriptRecord", + repeat="ScriptCount", + aux=0, + description="Array of ScriptRecords -listed alphabetically by ScriptTag", + ), + ], + ), + ( + "ScriptRecord", + [ + FieldSpec("Tag", "ScriptTag", description="4-byte ScriptTag identifier"), + FieldSpec( + "Offset", + "Script", + description="Offset to Script table-from beginning of ScriptList", + ), + ], + ), + ( + "Script", + [ + FieldSpec( + "Offset", + "DefaultLangSys", + description="Offset to DefaultLangSys table-from beginning of Script table-may be NULL", + ), + FieldSpec( + "uint16", + "LangSysCount", + description="Number of LangSysRecords for this script-excluding the DefaultLangSys", + ), + FieldSpec( + "struct", + "LangSysRecord", + repeat="LangSysCount", + aux=0, + description="Array of LangSysRecords-listed alphabetically by LangSysTag", + ), + ], + ), + ( + "LangSysRecord", + [ + FieldSpec("Tag", "LangSysTag", description="4-byte LangSysTag identifier"), + FieldSpec( + "Offset", + "LangSys", + description="Offset to LangSys table-from beginning of Script table", + ), + ], + ), + ( + "LangSys", + [ + FieldSpec( + "Offset", + "LookupOrder", + description="= NULL (reserved for an offset to a reordering table)", + ), + FieldSpec( + "uint16", + "ReqFeatureIndex", + description="Index of a feature required for this language system- if no required features = 0xFFFF", + ), + FieldSpec( + "uint16", + "FeatureCount", + description="Number of FeatureIndex values for this language system-excludes the required feature", + ), + FieldSpec( + "uint16", + "FeatureIndex", + repeat="FeatureCount", + aux=0, + description="Array of indices into the FeatureList-in arbitrary order", + ), + ], + ), + ( + "FeatureList", + [ + FieldSpec( + "uint16", + "FeatureCount", + description="Number of FeatureRecords in this table", + ), + FieldSpec( + "struct", + "FeatureRecord", + repeat="FeatureCount", + aux=0, + description="Array of FeatureRecords-zero-based (first feature has FeatureIndex = 0)-listed alphabetically by FeatureTag", + ), + ], + ), + ( + "FeatureRecord", + [ + FieldSpec( + "Tag", "FeatureTag", description="4-byte feature identification tag" + ), + FieldSpec( + "Offset", + "Feature", + description="Offset to Feature table-from beginning of FeatureList", + ), + ], + ), + ( + "Feature", + [ + FieldSpec( + "Offset", + "FeatureParams", + description="= NULL (reserved for offset to FeatureParams)", + ), + FieldSpec( + "uint16", + "LookupCount", + description="Number of LookupList indices for this feature", + ), + FieldSpec( + "uint16", + "LookupListIndex", + repeat="LookupCount", + aux=0, + description="Array of LookupList indices for this feature -zero-based (first lookup is LookupListIndex = 0)", + ), + ], + ), + ("FeatureParams", []), + ( + "FeatureParamsSize", + [ + FieldSpec( + "DeciPoints", + "DesignSize", + description="The design size in 720/inch units (decipoints).", + ), + FieldSpec( + "uint16", + "SubfamilyID", + description="Serves as an identifier that associates fonts in a subfamily.", + ), + FieldSpec("NameID", "SubfamilyNameID", description="Subfamily NameID."), + FieldSpec( + "DeciPoints", + "RangeStart", + description="Small end of recommended usage range (exclusive) in 720/inch units.", + ), + FieldSpec( + "DeciPoints", + "RangeEnd", + description="Large end of recommended usage range (inclusive) in 720/inch units.", + ), + ], + ), + ( + "FeatureParamsStylisticSet", + [ + FieldSpec("uint16", "Version", description="Set to 0."), + FieldSpec("NameID", "UINameID", description="UI NameID."), + ], + ), + ( + "FeatureParamsCharacterVariants", + [ + FieldSpec("uint16", "Format", description="Set to 0."), + FieldSpec( + "NameID", "FeatUILabelNameID", description="Feature UI label NameID." + ), + FieldSpec( + "NameID", + "FeatUITooltipTextNameID", + description="Feature UI tooltip text NameID.", + ), + FieldSpec("NameID", "SampleTextNameID", description="Sample text NameID."), + FieldSpec( + "uint16", + "NumNamedParameters", + description="Number of named parameters.", + ), + FieldSpec( + "NameID", + "FirstParamUILabelNameID", + description="First NameID of UI feature parameters.", + ), + FieldSpec( + "uint16", + "CharCount", + description="Count of characters this feature provides glyph variants for.", + ), + FieldSpec( + "uint24", + "Character", + repeat="CharCount", + aux=0, + description="Unicode characters for which this feature provides glyph variants.", + ), + ], + ), + ( + "LookupList", + [ + FieldSpec( + "uint16", "LookupCount", description="Number of lookups in this table" + ), + FieldSpec( + "Offset", + "Lookup", + repeat="LookupCount", + aux=0, + description="Array of offsets to Lookup tables-from beginning of LookupList -zero based (first lookup is Lookup index = 0)", + ), + ], + ), + ( + "Lookup", + [ + FieldSpec( + "uint16", + "LookupType", + description="Different enumerations for GSUB and GPOS", + ), + FieldSpec("LookupFlag", "LookupFlag", description="Lookup qualifiers"), + FieldSpec( + "uint16", + "SubTableCount", + description="Number of SubTables for this lookup", + ), + FieldSpec( + "Offset", + "SubTable", + repeat="SubTableCount", + aux=0, + description="Array of offsets to SubTables-from beginning of Lookup table", + ), + FieldSpec( + "uint16", + "MarkFilteringSet", + aux="LookupFlag & 0x0010", + description="If set, indicates that the lookup table structure is followed by a MarkFilteringSet field. The layout engine skips over all mark glyphs not in the mark filtering set indicated.", + ), + ], + ), + ( + "CoverageFormat1", + [ + FieldSpec( + "uint16", "CoverageFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "uint16", "GlyphCount", description="Number of glyphs in the GlyphArray" + ), + FieldSpec( + "GlyphID", + "GlyphArray", + repeat="GlyphCount", + aux=0, + description="Array of GlyphIDs-in numerical order", + ), + ], + ), + ( + "CoverageFormat2", + [ + FieldSpec( + "uint16", "CoverageFormat", description="Format identifier-format = 2" + ), + FieldSpec("uint16", "RangeCount", description="Number of RangeRecords"), + FieldSpec( + "struct", + "RangeRecord", + repeat="RangeCount", + aux=0, + description="Array of glyph ranges-ordered by Start GlyphID", + ), + ], + ), + ( + "RangeRecord", + [ + FieldSpec("GlyphID", "Start", description="First GlyphID in the range"), + FieldSpec("GlyphID", "End", description="Last GlyphID in the range"), + FieldSpec( + "uint16", + "StartCoverageIndex", + description="Coverage Index of first GlyphID in range", + ), + ], + ), + ( + "ClassDefFormat1", + [ + FieldSpec( + "uint16", "ClassFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "GlyphID", + "StartGlyph", + description="First GlyphID of the ClassValueArray", + ), + FieldSpec( + "uint16", "GlyphCount", description="Size of the ClassValueArray" + ), + FieldSpec( + "uint16", + "ClassValueArray", + repeat="GlyphCount", + aux=0, + description="Array of Class Values-one per GlyphID", + ), + ], + ), + ( + "ClassDefFormat2", + [ + FieldSpec( + "uint16", "ClassFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "uint16", "ClassRangeCount", description="Number of ClassRangeRecords" + ), + FieldSpec( + "struct", + "ClassRangeRecord", + repeat="ClassRangeCount", + aux=0, + description="Array of ClassRangeRecords-ordered by Start GlyphID", + ), + ], + ), + ( + "ClassRangeRecord", + [ + FieldSpec("GlyphID", "Start", description="First GlyphID in the range"), + FieldSpec("GlyphID", "End", description="Last GlyphID in the range"), + FieldSpec( + "uint16", "Class", description="Applied to all glyphs in the range" + ), + ], + ), + ( + "Device", + [ + FieldSpec( + "uint16", "StartSize", description="Smallest size to correct-in ppem" + ), + FieldSpec( + "uint16", "EndSize", description="Largest size to correct-in ppem" + ), + FieldSpec( + "uint16", + "DeltaFormat", + description="Format of DeltaValue array data: 1, 2, or 3", + ), + FieldSpec( + "DeltaValue", + "DeltaValue", + aux="DeltaFormat in (1,2,3)", + description="Array of compressed data", + ), + ], + ), + # + # gpos + # + ( + "GPOS", + [ + FieldSpec( + "Version", + "Version", + description="Version of the GPOS table- 0x00010000 or 0x00010001", + ), + FieldSpec( + "Offset", + "ScriptList", + description="Offset to ScriptList table-from beginning of GPOS table", + ), + FieldSpec( + "Offset", + "FeatureList", + description="Offset to FeatureList table-from beginning of GPOS table", + ), + FieldSpec( + "Offset", + "LookupList", + description="Offset to LookupList table-from beginning of GPOS table", + ), + FieldSpec( + "LOffset", + "FeatureVariations", + aux="Version >= 0x00010001", + description="Offset to FeatureVariations table-from beginning of GPOS table", + ), + ], + ), + ( + "SinglePosFormat1", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of SinglePos subtable", + ), + FieldSpec( + "uint16", + "ValueFormat", + description="Defines the types of data in the ValueRecord", + ), + FieldSpec( + "ValueRecord", + "Value", + description="Defines positioning value(s)-applied to all glyphs in the Coverage table", + ), + ], + ), + ( + "SinglePosFormat2", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of SinglePos subtable", + ), + FieldSpec( + "uint16", + "ValueFormat", + description="Defines the types of data in the ValueRecord", + ), + FieldSpec("uint16", "ValueCount", description="Number of ValueRecords"), + FieldSpec( + "ValueRecord", + "Value", + repeat="ValueCount", + aux=0, + description="Array of ValueRecords-positioning values applied to glyphs", + ), + ], + ), + ( + "PairPosFormat1", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of PairPos subtable-only the first glyph in each pair", + ), + FieldSpec( + "uint16", + "ValueFormat1", + description="Defines the types of data in ValueRecord1-for the first glyph in the pair -may be zero (0)", + ), + FieldSpec( + "uint16", + "ValueFormat2", + description="Defines the types of data in ValueRecord2-for the second glyph in the pair -may be zero (0)", + ), + FieldSpec("uint16", "PairSetCount", description="Number of PairSet tables"), + FieldSpec( + "Offset", + "PairSet", + repeat="PairSetCount", + aux=0, + description="Array of offsets to PairSet tables-from beginning of PairPos subtable-ordered by Coverage Index", + ), + ], + ), + ( + "PairSet", + [ + FieldSpec( + "uint16", "PairValueCount", description="Number of PairValueRecords" + ), + FieldSpec( + "struct", + "PairValueRecord", + repeat="PairValueCount", + aux=0, + description="Array of PairValueRecords-ordered by GlyphID of the second glyph", + ), + ], + ), + ( + "PairValueRecord", + [ + FieldSpec( + "GlyphID", + "SecondGlyph", + description="GlyphID of second glyph in the pair-first glyph is listed in the Coverage table", + ), + FieldSpec( + "ValueRecord", + "Value1", + description="Positioning data for the first glyph in the pair", + ), + FieldSpec( + "ValueRecord", + "Value2", + description="Positioning data for the second glyph in the pair", + ), + ], + ), + ( + "PairPosFormat2", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of PairPos subtable-for the first glyph of the pair", + ), + FieldSpec( + "uint16", + "ValueFormat1", + description="ValueRecord definition-for the first glyph of the pair-may be zero (0)", + ), + FieldSpec( + "uint16", + "ValueFormat2", + description="ValueRecord definition-for the second glyph of the pair-may be zero (0)", + ), + FieldSpec( + "Offset", + "ClassDef1", + description="Offset to ClassDef table-from beginning of PairPos subtable-for the first glyph of the pair", + ), + FieldSpec( + "Offset", + "ClassDef2", + description="Offset to ClassDef table-from beginning of PairPos subtable-for the second glyph of the pair", + ), + FieldSpec( + "uint16", + "Class1Count", + description="Number of classes in ClassDef1 table-includes Class0", + ), + FieldSpec( + "uint16", + "Class2Count", + description="Number of classes in ClassDef2 table-includes Class0", + ), + FieldSpec( + "struct", + "Class1Record", + repeat="Class1Count", + aux=0, + description="Array of Class1 records-ordered by Class1", + ), + ], + ), + ( + "Class1Record", + [ + FieldSpec( + "struct", + "Class2Record", + repeat="Class2Count", + aux=0, + description="Array of Class2 records-ordered by Class2", + ), + ], + ), + ( + "Class2Record", + [ + FieldSpec( + "ValueRecord", + "Value1", + description="Positioning for first glyph-empty if ValueFormat1 = 0", + ), + FieldSpec( + "ValueRecord", + "Value2", + description="Positioning for second glyph-empty if ValueFormat2 = 0", + ), + ], + ), + ( + "CursivePosFormat1", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of CursivePos subtable", + ), + FieldSpec( + "uint16", "EntryExitCount", description="Number of EntryExit records" + ), + FieldSpec( + "struct", + "EntryExitRecord", + repeat="EntryExitCount", + aux=0, + description="Array of EntryExit records-in Coverage Index order", + ), + ], + ), + ( + "EntryExitRecord", + [ + FieldSpec( + "Offset", + "EntryAnchor", + description="Offset to EntryAnchor table-from beginning of CursivePos subtable-may be NULL", + ), + FieldSpec( + "Offset", + "ExitAnchor", + description="Offset to ExitAnchor table-from beginning of CursivePos subtable-may be NULL", + ), + ], + ), + ( + "MarkBasePosFormat1", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "MarkCoverage", + description="Offset to MarkCoverage table-from beginning of MarkBasePos subtable", + ), + FieldSpec( + "Offset", + "BaseCoverage", + description="Offset to BaseCoverage table-from beginning of MarkBasePos subtable", + ), + FieldSpec( + "uint16", + "ClassCount", + description="Number of classes defined for marks", + ), + FieldSpec( + "Offset", + "MarkArray", + description="Offset to MarkArray table-from beginning of MarkBasePos subtable", + ), + FieldSpec( + "Offset", + "BaseArray", + description="Offset to BaseArray table-from beginning of MarkBasePos subtable", + ), + ], + ), + ( + "BaseArray", + [ + FieldSpec("uint16", "BaseCount", description="Number of BaseRecords"), + FieldSpec( + "struct", + "BaseRecord", + repeat="BaseCount", + aux=0, + description="Array of BaseRecords-in order of BaseCoverage Index", + ), + ], + ), + ( + "BaseRecord", + [ + FieldSpec( + "Offset", + "BaseAnchor", + repeat="ClassCount", + aux=0, + description="Array of offsets (one per class) to Anchor tables-from beginning of BaseArray table-ordered by class-zero-based", + ), + ], + ), + ( + "MarkLigPosFormat1", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "MarkCoverage", + description="Offset to Mark Coverage table-from beginning of MarkLigPos subtable", + ), + FieldSpec( + "Offset", + "LigatureCoverage", + description="Offset to Ligature Coverage table-from beginning of MarkLigPos subtable", + ), + FieldSpec( + "uint16", "ClassCount", description="Number of defined mark classes" + ), + FieldSpec( + "Offset", + "MarkArray", + description="Offset to MarkArray table-from beginning of MarkLigPos subtable", + ), + FieldSpec( + "Offset", + "LigatureArray", + description="Offset to LigatureArray table-from beginning of MarkLigPos subtable", + ), + ], + ), + ( + "LigatureArray", + [ + FieldSpec( + "uint16", + "LigatureCount", + description="Number of LigatureAttach table offsets", + ), + FieldSpec( + "Offset", + "LigatureAttach", + repeat="LigatureCount", + aux=0, + description="Array of offsets to LigatureAttach tables-from beginning of LigatureArray table-ordered by LigatureCoverage Index", + ), + ], + ), + ( + "LigatureAttach", + [ + FieldSpec( + "uint16", + "ComponentCount", + description="Number of ComponentRecords in this ligature", + ), + FieldSpec( + "struct", + "ComponentRecord", + repeat="ComponentCount", + aux=0, + description="Array of Component records-ordered in writing direction", + ), + ], + ), + ( + "ComponentRecord", + [ + FieldSpec( + "Offset", + "LigatureAnchor", + repeat="ClassCount", + aux=0, + description="Array of offsets (one per class) to Anchor tables-from beginning of LigatureAttach table-ordered by class-NULL if a component does not have an attachment for a class-zero-based array", + ), + ], + ), + ( + "MarkMarkPosFormat1", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Mark1Coverage", + description="Offset to Combining Mark Coverage table-from beginning of MarkMarkPos subtable", + ), + FieldSpec( + "Offset", + "Mark2Coverage", + description="Offset to Base Mark Coverage table-from beginning of MarkMarkPos subtable", + ), + FieldSpec( + "uint16", + "ClassCount", + description="Number of Combining Mark classes defined", + ), + FieldSpec( + "Offset", + "Mark1Array", + description="Offset to MarkArray table for Mark1-from beginning of MarkMarkPos subtable", + ), + FieldSpec( + "Offset", + "Mark2Array", + description="Offset to Mark2Array table for Mark2-from beginning of MarkMarkPos subtable", + ), + ], + ), + ( + "Mark2Array", + [ + FieldSpec("uint16", "Mark2Count", description="Number of Mark2 records"), + FieldSpec( + "struct", + "Mark2Record", + repeat="Mark2Count", + aux=0, + description="Array of Mark2 records-in Coverage order", + ), + ], + ), + ( + "Mark2Record", + [ + FieldSpec( + "Offset", + "Mark2Anchor", + repeat="ClassCount", + aux=0, + description="Array of offsets (one per class) to Anchor tables-from beginning of Mark2Array table-zero-based array", + ), + ], + ), + ( + "PosLookupRecord", + [ + FieldSpec( + "uint16", + "SequenceIndex", + description="Index to input glyph sequence-first glyph = 0", + ), + FieldSpec( + "uint16", + "LookupListIndex", + description="Lookup to apply to that position-zero-based", + ), + ], + ), + ( + "ContextPosFormat1", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of ContextPos subtable", + ), + FieldSpec( + "uint16", "PosRuleSetCount", description="Number of PosRuleSet tables" + ), + FieldSpec( + "Offset", + "PosRuleSet", + repeat="PosRuleSetCount", + aux=0, + description="Array of offsets to PosRuleSet tables-from beginning of ContextPos subtable-ordered by Coverage Index", + ), + ], + ), + ( + "PosRuleSet", + [ + FieldSpec("uint16", "PosRuleCount", description="Number of PosRule tables"), + FieldSpec( + "Offset", + "PosRule", + repeat="PosRuleCount", + aux=0, + description="Array of offsets to PosRule tables-from beginning of PosRuleSet-ordered by preference", + ), + ], + ), + ( + "PosRule", + [ + FieldSpec( + "uint16", + "GlyphCount", + description="Number of glyphs in the Input glyph sequence", + ), + FieldSpec("uint16", "PosCount", description="Number of PosLookupRecords"), + FieldSpec( + "GlyphID", + "Input", + repeat="GlyphCount", + aux=-1, + description="Array of input GlyphIDs-starting with the second glyph", + ), + FieldSpec( + "struct", + "PosLookupRecord", + repeat="PosCount", + aux=0, + description="Array of positioning lookups-in design order", + ), + ], + ), + ( + "ContextPosFormat2", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of ContextPos subtable", + ), + FieldSpec( + "Offset", + "ClassDef", + description="Offset to ClassDef table-from beginning of ContextPos subtable", + ), + FieldSpec( + "uint16", "PosClassSetCount", description="Number of PosClassSet tables" + ), + FieldSpec( + "Offset", + "PosClassSet", + repeat="PosClassSetCount", + aux=0, + description="Array of offsets to PosClassSet tables-from beginning of ContextPos subtable-ordered by class-may be NULL", + ), + ], + ), + ( + "PosClassSet", + [ + FieldSpec( + "uint16", + "PosClassRuleCount", + description="Number of PosClassRule tables", + ), + FieldSpec( + "Offset", + "PosClassRule", + repeat="PosClassRuleCount", + aux=0, + description="Array of offsets to PosClassRule tables-from beginning of PosClassSet-ordered by preference", + ), + ], + ), + ( + "PosClassRule", + [ + FieldSpec( + "uint16", "GlyphCount", description="Number of glyphs to be matched" + ), + FieldSpec("uint16", "PosCount", description="Number of PosLookupRecords"), + FieldSpec( + "uint16", + "Class", + repeat="GlyphCount", + aux=-1, + description="Array of classes-beginning with the second class-to be matched to the input glyph sequence", + ), + FieldSpec( + "struct", + "PosLookupRecord", + repeat="PosCount", + aux=0, + description="Array of positioning lookups-in design order", + ), + ], + ), + ( + "ContextPosFormat3", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 3" + ), + FieldSpec( + "uint16", + "GlyphCount", + description="Number of glyphs in the input sequence", + ), + FieldSpec("uint16", "PosCount", description="Number of PosLookupRecords"), + FieldSpec( + "Offset", + "Coverage", + repeat="GlyphCount", + aux=0, + description="Array of offsets to Coverage tables-from beginning of ContextPos subtable", + ), + FieldSpec( + "struct", + "PosLookupRecord", + repeat="PosCount", + aux=0, + description="Array of positioning lookups-in design order", + ), + ], + ), + ( + "ChainContextPosFormat1", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of ContextPos subtable", + ), + FieldSpec( + "uint16", + "ChainPosRuleSetCount", + description="Number of ChainPosRuleSet tables", + ), + FieldSpec( + "Offset", + "ChainPosRuleSet", + repeat="ChainPosRuleSetCount", + aux=0, + description="Array of offsets to ChainPosRuleSet tables-from beginning of ContextPos subtable-ordered by Coverage Index", + ), + ], + ), + ( + "ChainPosRuleSet", + [ + FieldSpec( + "uint16", + "ChainPosRuleCount", + description="Number of ChainPosRule tables", + ), + FieldSpec( + "Offset", + "ChainPosRule", + repeat="ChainPosRuleCount", + aux=0, + description="Array of offsets to ChainPosRule tables-from beginning of ChainPosRuleSet-ordered by preference", + ), + ], + ), + ( + "ChainPosRule", + [ + FieldSpec( + "uint16", + "BacktrackGlyphCount", + description="Total number of glyphs in the backtrack sequence (number of glyphs to be matched before the first glyph)", + ), + FieldSpec( + "GlyphID", + "Backtrack", + repeat="BacktrackGlyphCount", + aux=0, + description="Array of backtracking GlyphID's (to be matched before the input sequence)", + ), + FieldSpec( + "uint16", + "InputGlyphCount", + description="Total number of glyphs in the input sequence (includes the first glyph)", + ), + FieldSpec( + "GlyphID", + "Input", + repeat="InputGlyphCount", + aux=-1, + description="Array of input GlyphIDs (start with second glyph)", + ), + FieldSpec( + "uint16", + "LookAheadGlyphCount", + description="Total number of glyphs in the look ahead sequence (number of glyphs to be matched after the input sequence)", + ), + FieldSpec( + "GlyphID", + "LookAhead", + repeat="LookAheadGlyphCount", + aux=0, + description="Array of lookahead GlyphID's (to be matched after the input sequence)", + ), + FieldSpec("uint16", "PosCount", description="Number of PosLookupRecords"), + FieldSpec( + "struct", + "PosLookupRecord", + repeat="PosCount", + aux=0, + description="Array of PosLookupRecords (in design order)", + ), + ], + ), + ( + "ChainContextPosFormat2", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of ChainContextPos subtable", + ), + FieldSpec( + "Offset", + "BacktrackClassDef", + description="Offset to ClassDef table containing backtrack sequence context-from beginning of ChainContextPos subtable", + ), + FieldSpec( + "Offset", + "InputClassDef", + description="Offset to ClassDef table containing input sequence context-from beginning of ChainContextPos subtable", + ), + FieldSpec( + "Offset", + "LookAheadClassDef", + description="Offset to ClassDef table containing lookahead sequence context-from beginning of ChainContextPos subtable", + ), + FieldSpec( + "uint16", + "ChainPosClassSetCount", + description="Number of ChainPosClassSet tables", + ), + FieldSpec( + "Offset", + "ChainPosClassSet", + repeat="ChainPosClassSetCount", + aux=0, + description="Array of offsets to ChainPosClassSet tables-from beginning of ChainContextPos subtable-ordered by input class-may be NULL", + ), + ], + ), + ( + "ChainPosClassSet", + [ + FieldSpec( + "uint16", + "ChainPosClassRuleCount", + description="Number of ChainPosClassRule tables", + ), + FieldSpec( + "Offset", + "ChainPosClassRule", + repeat="ChainPosClassRuleCount", + aux=0, + description="Array of offsets to ChainPosClassRule tables-from beginning of ChainPosClassSet-ordered by preference", + ), + ], + ), + ( + "ChainPosClassRule", + [ + FieldSpec( + "uint16", + "BacktrackGlyphCount", + description="Total number of glyphs in the backtrack sequence (number of glyphs to be matched before the first glyph)", + ), + FieldSpec( + "uint16", + "Backtrack", + repeat="BacktrackGlyphCount", + aux=0, + description="Array of backtracking classes(to be matched before the input sequence)", + ), + FieldSpec( + "uint16", + "InputGlyphCount", + description="Total number of classes in the input sequence (includes the first class)", + ), + FieldSpec( + "uint16", + "Input", + repeat="InputGlyphCount", + aux=-1, + description="Array of input classes(start with second class; to be matched with the input glyph sequence)", + ), + FieldSpec( + "uint16", + "LookAheadGlyphCount", + description="Total number of classes in the look ahead sequence (number of classes to be matched after the input sequence)", + ), + FieldSpec( + "uint16", + "LookAhead", + repeat="LookAheadGlyphCount", + aux=0, + description="Array of lookahead classes(to be matched after the input sequence)", + ), + FieldSpec("uint16", "PosCount", description="Number of PosLookupRecords"), + FieldSpec( + "struct", + "PosLookupRecord", + repeat="PosCount", + aux=0, + description="Array of PosLookupRecords (in design order)", + ), + ], + ), + ( + "ChainContextPosFormat3", + [ + FieldSpec( + "uint16", "PosFormat", description="Format identifier-format = 3" + ), + FieldSpec( + "uint16", + "BacktrackGlyphCount", + description="Number of glyphs in the backtracking sequence", + ), + FieldSpec( + "Offset", + "BacktrackCoverage", + repeat="BacktrackGlyphCount", + aux=0, + description="Array of offsets to coverage tables in backtracking sequence, in glyph sequence order", + ), + FieldSpec( + "uint16", + "InputGlyphCount", + description="Number of glyphs in input sequence", + ), + FieldSpec( + "Offset", + "InputCoverage", + repeat="InputGlyphCount", + aux=0, + description="Array of offsets to coverage tables in input sequence, in glyph sequence order", + ), + FieldSpec( + "uint16", + "LookAheadGlyphCount", + description="Number of glyphs in lookahead sequence", + ), + FieldSpec( + "Offset", + "LookAheadCoverage", + repeat="LookAheadGlyphCount", + aux=0, + description="Array of offsets to coverage tables in lookahead sequence, in glyph sequence order", + ), + FieldSpec("uint16", "PosCount", description="Number of PosLookupRecords"), + FieldSpec( + "struct", + "PosLookupRecord", + repeat="PosCount", + aux=0, + description="Array of PosLookupRecords,in design order", + ), + ], + ), + ( + "ExtensionPosFormat1", + [ + FieldSpec( + "uint16", "ExtFormat", description="Format identifier. Set to 1." + ), + FieldSpec( + "uint16", + "ExtensionLookupType", + description="Lookup type of subtable referenced by ExtensionOffset (i.e. the extension subtable).", + ), + FieldSpec("LOffset", "ExtSubTable", description="Offset to SubTable"), + ], + ), + # ('ValueRecord', [ + # ('int16', 'XPlacement', None, None, 'Horizontal adjustment for placement-in design units'), + # ('int16', 'YPlacement', None, None, 'Vertical adjustment for placement-in design units'), + # ('int16', 'XAdvance', None, None, 'Horizontal adjustment for advance-in design units (only used for horizontal writing)'), + # ('int16', 'YAdvance', None, None, 'Vertical adjustment for advance-in design units (only used for vertical writing)'), + # ('Offset', 'XPlaDevice', None, None, 'Offset to Device table for horizontal placement-measured from beginning of PosTable (may be NULL)'), + # ('Offset', 'YPlaDevice', None, None, 'Offset to Device table for vertical placement-measured from beginning of PosTable (may be NULL)'), + # ('Offset', 'XAdvDevice', None, None, 'Offset to Device table for horizontal advance-measured from beginning of PosTable (may be NULL)'), + # ('Offset', 'YAdvDevice', None, None, 'Offset to Device table for vertical advance-measured from beginning of PosTable (may be NULL)'), + # ]), + ( + "AnchorFormat1", + [ + FieldSpec( + "uint16", "AnchorFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "int16", "XCoordinate", description="Horizontal value-in design units" + ), + FieldSpec( + "int16", "YCoordinate", description="Vertical value-in design units" + ), + ], + ), + ( + "AnchorFormat2", + [ + FieldSpec( + "uint16", "AnchorFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "int16", "XCoordinate", description="Horizontal value-in design units" + ), + FieldSpec( + "int16", "YCoordinate", description="Vertical value-in design units" + ), + FieldSpec( + "uint16", "AnchorPoint", description="Index to glyph contour point" + ), + ], + ), + ( + "AnchorFormat3", + [ + FieldSpec( + "uint16", "AnchorFormat", description="Format identifier-format = 3" + ), + FieldSpec( + "int16", "XCoordinate", description="Horizontal value-in design units" + ), + FieldSpec( + "int16", "YCoordinate", description="Vertical value-in design units" + ), + FieldSpec( + "Offset", + "XDeviceTable", + description="Offset to Device table for X coordinate- from beginning of Anchor table (may be NULL)", + ), + FieldSpec( + "Offset", + "YDeviceTable", + description="Offset to Device table for Y coordinate- from beginning of Anchor table (may be NULL)", + ), + ], + ), + ( + "MarkArray", + [ + FieldSpec("uint16", "MarkCount", description="Number of MarkRecords"), + FieldSpec( + "struct", + "MarkRecord", + repeat="MarkCount", + aux=0, + description="Array of MarkRecords-in Coverage order", + ), + ], + ), + ( + "MarkRecord", + [ + FieldSpec("uint16", "Class", description="Class defined for this mark"), + FieldSpec( + "Offset", + "MarkAnchor", + description="Offset to Anchor table-from beginning of MarkArray table", + ), + ], + ), + # + # gsub + # + ( + "GSUB", + [ + FieldSpec( + "Version", + "Version", + description="Version of the GSUB table- 0x00010000 or 0x00010001", + ), + FieldSpec( + "Offset", + "ScriptList", + description="Offset to ScriptList table-from beginning of GSUB table", + ), + FieldSpec( + "Offset", + "FeatureList", + description="Offset to FeatureList table-from beginning of GSUB table", + ), + FieldSpec( + "Offset", + "LookupList", + description="Offset to LookupList table-from beginning of GSUB table", + ), + FieldSpec( + "LOffset", + "FeatureVariations", + aux="Version >= 0x00010001", + description="Offset to FeatureVariations table-from beginning of GSUB table", + ), + ], + ), + ( + "SingleSubstFormat1", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of Substitution table", + ), + FieldSpec( + "uint16", + "DeltaGlyphID", + description="Add to original GlyphID modulo 65536 to get substitute GlyphID", + ), + ], + ), + ( + "SingleSubstFormat2", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of Substitution table", + ), + FieldSpec( + "uint16", + "GlyphCount", + description="Number of GlyphIDs in the Substitute array", + ), + FieldSpec( + "GlyphID", + "Substitute", + repeat="GlyphCount", + aux=0, + description="Array of substitute GlyphIDs-ordered by Coverage Index", + ), + ], + ), + ( + "MultipleSubstFormat1", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of Substitution table", + ), + FieldSpec( + "uint16", + "SequenceCount", + description="Number of Sequence table offsets in the Sequence array", + ), + FieldSpec( + "Offset", + "Sequence", + repeat="SequenceCount", + aux=0, + description="Array of offsets to Sequence tables-from beginning of Substitution table-ordered by Coverage Index", + ), + ], + ), + ( + "Sequence", + [ + FieldSpec( + "uint16", + "GlyphCount", + description="Number of GlyphIDs in the Substitute array. This should always be greater than 0.", + ), + FieldSpec( + "GlyphID", + "Substitute", + repeat="GlyphCount", + aux=0, + description="String of GlyphIDs to substitute", + ), + ], + ), + ( + "AlternateSubstFormat1", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of Substitution table", + ), + FieldSpec( + "uint16", + "AlternateSetCount", + description="Number of AlternateSet tables", + ), + FieldSpec( + "Offset", + "AlternateSet", + repeat="AlternateSetCount", + aux=0, + description="Array of offsets to AlternateSet tables-from beginning of Substitution table-ordered by Coverage Index", + ), + ], + ), + ( + "AlternateSet", + [ + FieldSpec( + "uint16", + "GlyphCount", + description="Number of GlyphIDs in the Alternate array", + ), + FieldSpec( + "GlyphID", + "Alternate", + repeat="GlyphCount", + aux=0, + description="Array of alternate GlyphIDs-in arbitrary order", + ), + ], + ), + ( + "LigatureSubstFormat1", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of Substitution table", + ), + FieldSpec( + "uint16", "LigSetCount", description="Number of LigatureSet tables" + ), + FieldSpec( + "Offset", + "LigatureSet", + repeat="LigSetCount", + aux=0, + description="Array of offsets to LigatureSet tables-from beginning of Substitution table-ordered by Coverage Index", + ), + ], + ), + ( + "LigatureSet", + [ + FieldSpec( + "uint16", "LigatureCount", description="Number of Ligature tables" + ), + FieldSpec( + "Offset", + "Ligature", + repeat="LigatureCount", + aux=0, + description="Array of offsets to Ligature tables-from beginning of LigatureSet table-ordered by preference", + ), + ], + ), + ( + "Ligature", + [ + FieldSpec( + "GlyphID", "LigGlyph", description="GlyphID of ligature to substitute" + ), + FieldSpec( + "uint16", + "CompCount", + description="Number of components in the ligature", + ), + FieldSpec( + "GlyphID", + "Component", + repeat="CompCount", + aux=-1, + description="Array of component GlyphIDs-start with the second component-ordered in writing direction", + ), + ], + ), + ( + "SubstLookupRecord", + [ + FieldSpec( + "uint16", + "SequenceIndex", + description="Index into current glyph sequence-first glyph = 0", + ), + FieldSpec( + "uint16", + "LookupListIndex", + description="Lookup to apply to that position-zero-based", + ), + ], + ), + ( + "ContextSubstFormat1", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of Substitution table", + ), + FieldSpec( + "uint16", + "SubRuleSetCount", + description="Number of SubRuleSet tables-must equal GlyphCount in Coverage table", + ), + FieldSpec( + "Offset", + "SubRuleSet", + repeat="SubRuleSetCount", + aux=0, + description="Array of offsets to SubRuleSet tables-from beginning of Substitution table-ordered by Coverage Index", + ), + ], + ), + ( + "SubRuleSet", + [ + FieldSpec("uint16", "SubRuleCount", description="Number of SubRule tables"), + FieldSpec( + "Offset", + "SubRule", + repeat="SubRuleCount", + aux=0, + description="Array of offsets to SubRule tables-from beginning of SubRuleSet table-ordered by preference", + ), + ], + ), + ( + "SubRule", + [ + FieldSpec( + "uint16", + "GlyphCount", + description="Total number of glyphs in input glyph sequence-includes the first glyph", + ), + FieldSpec( + "uint16", "SubstCount", description="Number of SubstLookupRecords" + ), + FieldSpec( + "GlyphID", + "Input", + repeat="GlyphCount", + aux=-1, + description="Array of input GlyphIDs-start with second glyph", + ), + FieldSpec( + "struct", + "SubstLookupRecord", + repeat="SubstCount", + aux=0, + description="Array of SubstLookupRecords-in design order", + ), + ], + ), + ( + "ContextSubstFormat2", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of Substitution table", + ), + FieldSpec( + "Offset", + "ClassDef", + description="Offset to glyph ClassDef table-from beginning of Substitution table", + ), + FieldSpec( + "uint16", "SubClassSetCount", description="Number of SubClassSet tables" + ), + FieldSpec( + "Offset", + "SubClassSet", + repeat="SubClassSetCount", + aux=0, + description="Array of offsets to SubClassSet tables-from beginning of Substitution table-ordered by class-may be NULL", + ), + ], + ), + ( + "SubClassSet", + [ + FieldSpec( + "uint16", + "SubClassRuleCount", + description="Number of SubClassRule tables", + ), + FieldSpec( + "Offset", + "SubClassRule", + repeat="SubClassRuleCount", + aux=0, + description="Array of offsets to SubClassRule tables-from beginning of SubClassSet-ordered by preference", + ), + ], + ), + ( + "SubClassRule", + [ + FieldSpec( + "uint16", + "GlyphCount", + description="Total number of classes specified for the context in the rule-includes the first class", + ), + FieldSpec( + "uint16", "SubstCount", description="Number of SubstLookupRecords" + ), + FieldSpec( + "uint16", + "Class", + repeat="GlyphCount", + aux=-1, + description="Array of classes-beginning with the second class-to be matched to the input glyph class sequence", + ), + FieldSpec( + "struct", + "SubstLookupRecord", + repeat="SubstCount", + aux=0, + description="Array of Substitution lookups-in design order", + ), + ], + ), + ( + "ContextSubstFormat3", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 3" + ), + FieldSpec( + "uint16", + "GlyphCount", + description="Number of glyphs in the input glyph sequence", + ), + FieldSpec( + "uint16", "SubstCount", description="Number of SubstLookupRecords" + ), + FieldSpec( + "Offset", + "Coverage", + repeat="GlyphCount", + aux=0, + description="Array of offsets to Coverage table-from beginning of Substitution table-in glyph sequence order", + ), + FieldSpec( + "struct", + "SubstLookupRecord", + repeat="SubstCount", + aux=0, + description="Array of SubstLookupRecords-in design order", + ), + ], + ), + ( + "ChainContextSubstFormat1", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of Substitution table", + ), + FieldSpec( + "uint16", + "ChainSubRuleSetCount", + description="Number of ChainSubRuleSet tables-must equal GlyphCount in Coverage table", + ), + FieldSpec( + "Offset", + "ChainSubRuleSet", + repeat="ChainSubRuleSetCount", + aux=0, + description="Array of offsets to ChainSubRuleSet tables-from beginning of Substitution table-ordered by Coverage Index", + ), + ], + ), + ( + "ChainSubRuleSet", + [ + FieldSpec( + "uint16", + "ChainSubRuleCount", + description="Number of ChainSubRule tables", + ), + FieldSpec( + "Offset", + "ChainSubRule", + repeat="ChainSubRuleCount", + aux=0, + description="Array of offsets to ChainSubRule tables-from beginning of ChainSubRuleSet table-ordered by preference", + ), + ], + ), + ( + "ChainSubRule", + [ + FieldSpec( + "uint16", + "BacktrackGlyphCount", + description="Total number of glyphs in the backtrack sequence (number of glyphs to be matched before the first glyph)", + ), + FieldSpec( + "GlyphID", + "Backtrack", + repeat="BacktrackGlyphCount", + aux=0, + description="Array of backtracking GlyphID's (to be matched before the input sequence)", + ), + FieldSpec( + "uint16", + "InputGlyphCount", + description="Total number of glyphs in the input sequence (includes the first glyph)", + ), + FieldSpec( + "GlyphID", + "Input", + repeat="InputGlyphCount", + aux=-1, + description="Array of input GlyphIDs (start with second glyph)", + ), + FieldSpec( + "uint16", + "LookAheadGlyphCount", + description="Total number of glyphs in the look ahead sequence (number of glyphs to be matched after the input sequence)", + ), + FieldSpec( + "GlyphID", + "LookAhead", + repeat="LookAheadGlyphCount", + aux=0, + description="Array of lookahead GlyphID's (to be matched after the input sequence)", + ), + FieldSpec( + "uint16", "SubstCount", description="Number of SubstLookupRecords" + ), + FieldSpec( + "struct", + "SubstLookupRecord", + repeat="SubstCount", + aux=0, + description="Array of SubstLookupRecords (in design order)", + ), + ], + ), + ( + "ChainContextSubstFormat2", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table-from beginning of Substitution table", + ), + FieldSpec( + "Offset", + "BacktrackClassDef", + description="Offset to glyph ClassDef table containing backtrack sequence data-from beginning of Substitution table", + ), + FieldSpec( + "Offset", + "InputClassDef", + description="Offset to glyph ClassDef table containing input sequence data-from beginning of Substitution table", + ), + FieldSpec( + "Offset", + "LookAheadClassDef", + description="Offset to glyph ClassDef table containing lookahead sequence data-from beginning of Substitution table", + ), + FieldSpec( + "uint16", + "ChainSubClassSetCount", + description="Number of ChainSubClassSet tables", + ), + FieldSpec( + "Offset", + "ChainSubClassSet", + repeat="ChainSubClassSetCount", + aux=0, + description="Array of offsets to ChainSubClassSet tables-from beginning of Substitution table-ordered by input class-may be NULL", + ), + ], + ), + ( + "ChainSubClassSet", + [ + FieldSpec( + "uint16", + "ChainSubClassRuleCount", + description="Number of ChainSubClassRule tables", + ), + FieldSpec( + "Offset", + "ChainSubClassRule", + repeat="ChainSubClassRuleCount", + aux=0, + description="Array of offsets to ChainSubClassRule tables-from beginning of ChainSubClassSet-ordered by preference", + ), + ], + ), + ( + "ChainSubClassRule", + [ + FieldSpec( + "uint16", + "BacktrackGlyphCount", + description="Total number of glyphs in the backtrack sequence (number of glyphs to be matched before the first glyph)", + ), + FieldSpec( + "uint16", + "Backtrack", + repeat="BacktrackGlyphCount", + aux=0, + description="Array of backtracking classes(to be matched before the input sequence)", + ), + FieldSpec( + "uint16", + "InputGlyphCount", + description="Total number of classes in the input sequence (includes the first class)", + ), + FieldSpec( + "uint16", + "Input", + repeat="InputGlyphCount", + aux=-1, + description="Array of input classes(start with second class; to be matched with the input glyph sequence)", + ), + FieldSpec( + "uint16", + "LookAheadGlyphCount", + description="Total number of classes in the look ahead sequence (number of classes to be matched after the input sequence)", + ), + FieldSpec( + "uint16", + "LookAhead", + repeat="LookAheadGlyphCount", + aux=0, + description="Array of lookahead classes(to be matched after the input sequence)", + ), + FieldSpec( + "uint16", "SubstCount", description="Number of SubstLookupRecords" + ), + FieldSpec( + "struct", + "SubstLookupRecord", + repeat="SubstCount", + aux=0, + description="Array of SubstLookupRecords (in design order)", + ), + ], + ), + ( + "ChainContextSubstFormat3", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 3" + ), + FieldSpec( + "uint16", + "BacktrackGlyphCount", + description="Number of glyphs in the backtracking sequence", + ), + FieldSpec( + "Offset", + "BacktrackCoverage", + repeat="BacktrackGlyphCount", + aux=0, + description="Array of offsets to coverage tables in backtracking sequence, in glyph sequence order", + ), + FieldSpec( + "uint16", + "InputGlyphCount", + description="Number of glyphs in input sequence", + ), + FieldSpec( + "Offset", + "InputCoverage", + repeat="InputGlyphCount", + aux=0, + description="Array of offsets to coverage tables in input sequence, in glyph sequence order", + ), + FieldSpec( + "uint16", + "LookAheadGlyphCount", + description="Number of glyphs in lookahead sequence", + ), + FieldSpec( + "Offset", + "LookAheadCoverage", + repeat="LookAheadGlyphCount", + aux=0, + description="Array of offsets to coverage tables in lookahead sequence, in glyph sequence order", + ), + FieldSpec( + "uint16", "SubstCount", description="Number of SubstLookupRecords" + ), + FieldSpec( + "struct", + "SubstLookupRecord", + repeat="SubstCount", + aux=0, + description="Array of SubstLookupRecords, in design order", + ), + ], + ), + ( + "ExtensionSubstFormat1", + [ + FieldSpec( + "uint16", "ExtFormat", description="Format identifier. Set to 1." + ), + FieldSpec( + "uint16", + "ExtensionLookupType", + description="Lookup type of subtable referenced by ExtensionOffset (i.e. the extension subtable).", + ), + FieldSpec( + "LOffset", + "ExtSubTable", + description="Array of offsets to Lookup tables-from beginning of LookupList -zero based (first lookup is Lookup index = 0)", + ), + ], + ), + ( + "ReverseChainSingleSubstFormat1", + [ + FieldSpec( + "uint16", "SubstFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "Offset", + "Coverage", + aux=0, + description="Offset to Coverage table - from beginning of Substitution table", + ), + FieldSpec( + "uint16", + "BacktrackGlyphCount", + description="Number of glyphs in the backtracking sequence", + ), + FieldSpec( + "Offset", + "BacktrackCoverage", + repeat="BacktrackGlyphCount", + aux=0, + description="Array of offsets to coverage tables in backtracking sequence, in glyph sequence order", + ), + FieldSpec( + "uint16", + "LookAheadGlyphCount", + description="Number of glyphs in lookahead sequence", + ), + FieldSpec( + "Offset", + "LookAheadCoverage", + repeat="LookAheadGlyphCount", + aux=0, + description="Array of offsets to coverage tables in lookahead sequence, in glyph sequence order", + ), + FieldSpec( + "uint16", + "GlyphCount", + description="Number of GlyphIDs in the Substitute array", + ), + FieldSpec( + "GlyphID", + "Substitute", + repeat="GlyphCount", + aux=0, + description="Array of substitute GlyphIDs-ordered by Coverage index", + ), + ], + ), + # + # gdef + # + ( + "GDEF", + [ + FieldSpec( + "Version", + "Version", + description="Version of the GDEF table- 0x00010000, 0x00010002, or 0x00010003", + ), + FieldSpec( + "Offset", + "GlyphClassDef", + description="Offset to class definition table for glyph type-from beginning of GDEF header (may be NULL)", + ), + FieldSpec( + "Offset", + "AttachList", + description="Offset to list of glyphs with attachment points-from beginning of GDEF header (may be NULL)", + ), + FieldSpec( + "Offset", + "LigCaretList", + description="Offset to list of positioning points for ligature carets-from beginning of GDEF header (may be NULL)", + ), + FieldSpec( + "Offset", + "MarkAttachClassDef", + description="Offset to class definition table for mark attachment type-from beginning of GDEF header (may be NULL)", + ), + FieldSpec( + "Offset", + "MarkGlyphSetsDef", + aux="Version >= 0x00010002", + description="Offset to the table of mark set definitions-from beginning of GDEF header (may be NULL)", + ), + FieldSpec( + "LOffset", + "VarStore", + aux="Version >= 0x00010003", + description="Offset to variation store (may be NULL)", + ), + ], + ), + ( + "AttachList", + [ + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table - from beginning of AttachList table", + ), + FieldSpec( + "uint16", + "GlyphCount", + description="Number of glyphs with attachment points", + ), + FieldSpec( + "Offset", + "AttachPoint", + repeat="GlyphCount", + aux=0, + description="Array of offsets to AttachPoint tables-from beginning of AttachList table-in Coverage Index order", + ), + ], + ), + ( + "AttachPoint", + [ + FieldSpec( + "uint16", + "PointCount", + description="Number of attachment points on this glyph", + ), + FieldSpec( + "uint16", + "PointIndex", + repeat="PointCount", + aux=0, + description="Array of contour point indices -in increasing numerical order", + ), + ], + ), + ( + "LigCaretList", + [ + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table - from beginning of LigCaretList table", + ), + FieldSpec( + "uint16", "LigGlyphCount", description="Number of ligature glyphs" + ), + FieldSpec( + "Offset", + "LigGlyph", + repeat="LigGlyphCount", + aux=0, + description="Array of offsets to LigGlyph tables-from beginning of LigCaretList table-in Coverage Index order", + ), + ], + ), + ( + "LigGlyph", + [ + FieldSpec( + "uint16", + "CaretCount", + description="Number of CaretValues for this ligature (components - 1)", + ), + FieldSpec( + "Offset", + "CaretValue", + repeat="CaretCount", + aux=0, + description="Array of offsets to CaretValue tables-from beginning of LigGlyph table-in increasing coordinate order", + ), + ], + ), + ( + "CaretValueFormat1", + [ + FieldSpec( + "uint16", "CaretValueFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "int16", "Coordinate", description="X or Y value, in design units" + ), + ], + ), + ( + "CaretValueFormat2", + [ + FieldSpec( + "uint16", "CaretValueFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "uint16", "CaretValuePoint", description="Contour point index on glyph" + ), + ], + ), + ( + "CaretValueFormat3", + [ + FieldSpec( + "uint16", "CaretValueFormat", description="Format identifier-format = 3" + ), + FieldSpec( + "int16", "Coordinate", description="X or Y value, in design units" + ), + FieldSpec( + "Offset", + "DeviceTable", + description="Offset to Device table for X or Y value-from beginning of CaretValue table", + ), + ], + ), + ( + "MarkGlyphSetsDef", + [ + FieldSpec( + "uint16", "MarkSetTableFormat", description="Format identifier == 1" + ), + FieldSpec( + "uint16", "MarkSetCount", description="Number of mark sets defined" + ), + FieldSpec( + "LOffset", + "Coverage", + repeat="MarkSetCount", + aux=0, + description="Array of offsets to mark set coverage tables.", + ), + ], + ), + # + # base + # + ( + "BASE", + [ + FieldSpec( + "Version", + "Version", + description="Version of the BASE table-initially 0x00010000", + ), + FieldSpec( + "Offset", + "HorizAxis", + description="Offset to horizontal Axis table-from beginning of BASE table-may be NULL", + ), + FieldSpec( + "Offset", + "VertAxis", + description="Offset to vertical Axis table-from beginning of BASE table-may be NULL", + ), + FieldSpec( + "LOffset", + "VarStore", + aux="Version >= 0x00010001", + description="Offset to variation store (may be NULL)", + ), + ], + ), + ( + "Axis", + [ + FieldSpec( + "Offset", + "BaseTagList", + description="Offset to BaseTagList table-from beginning of Axis table-may be NULL", + ), + FieldSpec( + "Offset", + "BaseScriptList", + description="Offset to BaseScriptList table-from beginning of Axis table", + ), + ], + ), + ( + "BaseTagList", + [ + FieldSpec( + "uint16", + "BaseTagCount", + description="Number of baseline identification tags in this text direction-may be zero (0)", + ), + FieldSpec( + "Tag", + "BaselineTag", + repeat="BaseTagCount", + aux=0, + description="Array of 4-byte baseline identification tags-must be in alphabetical order", + ), + ], + ), + ( + "BaseScriptList", + [ + FieldSpec( + "uint16", + "BaseScriptCount", + description="Number of BaseScriptRecords defined", + ), + FieldSpec( + "struct", + "BaseScriptRecord", + repeat="BaseScriptCount", + aux=0, + description="Array of BaseScriptRecords-in alphabetical order by BaseScriptTag", + ), + ], + ), + ( + "BaseScriptRecord", + [ + FieldSpec( + "Tag", "BaseScriptTag", description="4-byte script identification tag" + ), + FieldSpec( + "Offset", + "BaseScript", + description="Offset to BaseScript table-from beginning of BaseScriptList", + ), + ], + ), + ( + "BaseScript", + [ + FieldSpec( + "Offset", + "BaseValues", + description="Offset to BaseValues table-from beginning of BaseScript table-may be NULL", + ), + FieldSpec( + "Offset", + "DefaultMinMax", + description="Offset to MinMax table- from beginning of BaseScript table-may be NULL", + ), + FieldSpec( + "uint16", + "BaseLangSysCount", + description="Number of BaseLangSysRecords defined-may be zero (0)", + ), + FieldSpec( + "struct", + "BaseLangSysRecord", + repeat="BaseLangSysCount", + aux=0, + description="Array of BaseLangSysRecords-in alphabetical order by BaseLangSysTag", + ), + ], + ), + ( + "BaseLangSysRecord", + [ + FieldSpec( + "Tag", + "BaseLangSysTag", + description="4-byte language system identification tag", + ), + FieldSpec( + "Offset", + "MinMax", + description="Offset to MinMax table-from beginning of BaseScript table", + ), + ], + ), + ( + "BaseValues", + [ + FieldSpec( + "uint16", + "DefaultIndex", + description="Index number of default baseline for this script-equals index position of baseline tag in BaselineArray of the BaseTagList", + ), + FieldSpec( + "uint16", + "BaseCoordCount", + description="Number of BaseCoord tables defined-should equal BaseTagCount in the BaseTagList", + ), + FieldSpec( + "Offset", + "BaseCoord", + repeat="BaseCoordCount", + aux=0, + description="Array of offsets to BaseCoord-from beginning of BaseValues table-order matches BaselineTag array in the BaseTagList", + ), + ], + ), + ( + "MinMax", + [ + FieldSpec( + "Offset", + "MinCoord", + description="Offset to BaseCoord table-defines minimum extent value-from the beginning of MinMax table-may be NULL", + ), + FieldSpec( + "Offset", + "MaxCoord", + description="Offset to BaseCoord table-defines maximum extent value-from the beginning of MinMax table-may be NULL", + ), + FieldSpec( + "uint16", + "FeatMinMaxCount", + description="Number of FeatMinMaxRecords-may be zero (0)", + ), + FieldSpec( + "struct", + "FeatMinMaxRecord", + repeat="FeatMinMaxCount", + aux=0, + description="Array of FeatMinMaxRecords-in alphabetical order, by FeatureTableTag", + ), + ], + ), + ( + "FeatMinMaxRecord", + [ + FieldSpec( + "Tag", + "FeatureTableTag", + description="4-byte feature identification tag-must match FeatureTag in FeatureList", + ), + FieldSpec( + "Offset", + "MinCoord", + description="Offset to BaseCoord table-defines minimum extent value-from beginning of MinMax table-may be NULL", + ), + FieldSpec( + "Offset", + "MaxCoord", + description="Offset to BaseCoord table-defines maximum extent value-from beginning of MinMax table-may be NULL", + ), + ], + ), + ( + "BaseCoordFormat1", + [ + FieldSpec( + "uint16", "BaseCoordFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "int16", "Coordinate", description="X or Y value, in design units" + ), + ], + ), + ( + "BaseCoordFormat2", + [ + FieldSpec( + "uint16", "BaseCoordFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "int16", "Coordinate", description="X or Y value, in design units" + ), + FieldSpec( + "GlyphID", "ReferenceGlyph", description="GlyphID of control glyph" + ), + FieldSpec( + "uint16", + "BaseCoordPoint", + description="Index of contour point on the ReferenceGlyph", + ), + ], + ), + ( + "BaseCoordFormat3", + [ + FieldSpec( + "uint16", "BaseCoordFormat", description="Format identifier-format = 3" + ), + FieldSpec( + "int16", "Coordinate", description="X or Y value, in design units" + ), + FieldSpec( + "Offset", + "DeviceTable", + description="Offset to Device table for X or Y value", + ), + ], + ), + # + # jstf + # + ( + "JSTF", + [ + FieldSpec( + "Version", + "Version", + description="Version of the JSTF table-initially set to 0x00010000", + ), + FieldSpec( + "uint16", + "JstfScriptCount", + description="Number of JstfScriptRecords in this table", + ), + FieldSpec( + "struct", + "JstfScriptRecord", + repeat="JstfScriptCount", + aux=0, + description="Array of JstfScriptRecords-in alphabetical order, by JstfScriptTag", + ), + ], + ), + ( + "JstfScriptRecord", + [ + FieldSpec( + "Tag", "JstfScriptTag", description="4-byte JstfScript identification" + ), + FieldSpec( + "Offset", + "JstfScript", + description="Offset to JstfScript table-from beginning of JSTF Header", + ), + ], + ), + ( + "JstfScript", + [ + FieldSpec( + "Offset", + "ExtenderGlyph", + description="Offset to ExtenderGlyph table-from beginning of JstfScript table-may be NULL", + ), + FieldSpec( + "Offset", + "DefJstfLangSys", + description="Offset to Default JstfLangSys table-from beginning of JstfScript table-may be NULL", + ), + FieldSpec( + "uint16", + "JstfLangSysCount", + description="Number of JstfLangSysRecords in this table- may be zero (0)", + ), + FieldSpec( + "struct", + "JstfLangSysRecord", + repeat="JstfLangSysCount", + aux=0, + description="Array of JstfLangSysRecords-in alphabetical order, by JstfLangSysTag", + ), + ], + ), + ( + "JstfLangSysRecord", + [ + FieldSpec( + "Tag", "JstfLangSysTag", description="4-byte JstfLangSys identifier" + ), + FieldSpec( + "Offset", + "JstfLangSys", + description="Offset to JstfLangSys table-from beginning of JstfScript table", + ), + ], + ), + ( + "ExtenderGlyph", + [ + FieldSpec( + "uint16", + "GlyphCount", + description="Number of Extender Glyphs in this script", + ), + FieldSpec( + "GlyphID", + "ExtenderGlyph", + repeat="GlyphCount", + aux=0, + description="GlyphIDs-in increasing numerical order", + ), + ], + ), + ( + "JstfLangSys", + [ + FieldSpec( + "uint16", + "JstfPriorityCount", + description="Number of JstfPriority tables", + ), + FieldSpec( + "Offset", + "JstfPriority", + repeat="JstfPriorityCount", + aux=0, + description="Array of offsets to JstfPriority tables-from beginning of JstfLangSys table-in priority order", + ), + ], + ), + ( + "JstfPriority", + [ + FieldSpec( + "Offset", + "ShrinkageEnableGSUB", + description="Offset to Shrinkage Enable JstfGSUBModList table-from beginning of JstfPriority table-may be NULL", + ), + FieldSpec( + "Offset", + "ShrinkageDisableGSUB", + description="Offset to Shrinkage Disable JstfGSUBModList table-from beginning of JstfPriority table-may be NULL", + ), + FieldSpec( + "Offset", + "ShrinkageEnableGPOS", + description="Offset to Shrinkage Enable JstfGPOSModList table-from beginning of JstfPriority table-may be NULL", + ), + FieldSpec( + "Offset", + "ShrinkageDisableGPOS", + description="Offset to Shrinkage Disable JstfGPOSModList table-from beginning of JstfPriority table-may be NULL", + ), + FieldSpec( + "Offset", + "ShrinkageJstfMax", + description="Offset to Shrinkage JstfMax table-from beginning of JstfPriority table -may be NULL", + ), + FieldSpec( + "Offset", + "ExtensionEnableGSUB", + description="Offset to Extension Enable JstfGSUBModList table-may be NULL", + ), + FieldSpec( + "Offset", + "ExtensionDisableGSUB", + description="Offset to Extension Disable JstfGSUBModList table-from beginning of JstfPriority table-may be NULL", + ), + FieldSpec( + "Offset", + "ExtensionEnableGPOS", + description="Offset to Extension Enable JstfGSUBModList table-may be NULL", + ), + FieldSpec( + "Offset", + "ExtensionDisableGPOS", + description="Offset to Extension Disable JstfGSUBModList table-from beginning of JstfPriority table-may be NULL", + ), + FieldSpec( + "Offset", + "ExtensionJstfMax", + description="Offset to Extension JstfMax table-from beginning of JstfPriority table -may be NULL", + ), + ], + ), + ( + "JstfGSUBModList", + [ + FieldSpec( + "uint16", + "LookupCount", + description="Number of lookups for this modification", + ), + FieldSpec( + "uint16", + "GSUBLookupIndex", + repeat="LookupCount", + aux=0, + description="Array of LookupIndex identifiers in GSUB-in increasing numerical order", + ), + ], + ), + ( + "JstfGPOSModList", + [ + FieldSpec( + "uint16", + "LookupCount", + description="Number of lookups for this modification", + ), + FieldSpec( + "uint16", + "GPOSLookupIndex", + repeat="LookupCount", + aux=0, + description="Array of LookupIndex identifiers in GPOS-in increasing numerical order", + ), + ], + ), + ( + "JstfMax", + [ + FieldSpec( + "uint16", + "LookupCount", + description="Number of lookup Indices for this modification", + ), + FieldSpec( + "Offset", + "Lookup", + repeat="LookupCount", + aux=0, + description="Array of offsets to GPOS-type lookup tables-from beginning of JstfMax table-in design order", + ), + ], + ), + # + # STAT + # + ( + "STAT", + [ + FieldSpec( + "Version", + "Version", + description="Version of the table-initially set to 0x00010000, currently 0x00010002.", + ), + FieldSpec( + "uint16", + "DesignAxisRecordSize", + description="Size in bytes of each design axis record", + ), + FieldSpec( + "uint16", "DesignAxisCount", description="Number of design axis records" + ), + FieldSpec( + "LOffsetTo(AxisRecordArray)", + "DesignAxisRecord", + description="Offset in bytes from the beginning of the STAT table to the start of the design axes array", + ), + FieldSpec( + "uint16", "AxisValueCount", description="Number of axis value tables" + ), + FieldSpec( + "LOffsetTo(AxisValueArray)", + "AxisValueArray", + description="Offset in bytes from the beginning of the STAT table to the start of the axes value offset array", + ), + FieldSpec( + "NameID", + "ElidedFallbackNameID", + aux="Version >= 0x00010001", + description="NameID to use when all style attributes are elided.", + ), + ], + ), + ( + "AxisRecordArray", + [ + FieldSpec( + "AxisRecord", + "Axis", + repeat="DesignAxisCount", + aux=0, + description="Axis records", + ), + ], + ), + ( + "AxisRecord", + [ + FieldSpec( + "Tag", + "AxisTag", + description="A tag identifying the axis of design variation", + ), + FieldSpec( + "NameID", + "AxisNameID", + description='The name ID for entries in the "name" table that provide a display string for this axis', + ), + FieldSpec( + "uint16", + "AxisOrdering", + description="A value that applications can use to determine primary sorting of face names, or for ordering of descriptors when composing family or face names", + ), + FieldSpec( + "uint8", + "MoreBytes", + repeat="DesignAxisRecordSize", + aux=-8, + description="Extra bytes. Set to empty array.", + ), + ], + ), + ( + "AxisValueArray", + [ + FieldSpec( + "Offset", + "AxisValue", + repeat="AxisValueCount", + aux=0, + description="Axis values", + ), + ], + ), + ( + "AxisValueFormat1", + [ + FieldSpec("uint16", "Format", description="Format, = 1"), + FieldSpec( + "uint16", + "AxisIndex", + description="Index into the axis record array identifying the axis of design variation to which the axis value record applies.", + ), + FieldSpec("STATFlags", "Flags", description="Flags."), + FieldSpec("NameID", "ValueNameID"), + FieldSpec("Fixed", "Value"), + ], + ), + ( + "AxisValueFormat2", + [ + FieldSpec("uint16", "Format", description="Format, = 2"), + FieldSpec( + "uint16", + "AxisIndex", + description="Index into the axis record array identifying the axis of design variation to which the axis value record applies.", + ), + FieldSpec("STATFlags", "Flags", description="Flags."), + FieldSpec("NameID", "ValueNameID"), + FieldSpec("Fixed", "NominalValue"), + FieldSpec("Fixed", "RangeMinValue"), + FieldSpec("Fixed", "RangeMaxValue"), + ], + ), + ( + "AxisValueFormat3", + [ + FieldSpec("uint16", "Format", description="Format, = 3"), + FieldSpec( + "uint16", + "AxisIndex", + description="Index into the axis record array identifying the axis of design variation to which the axis value record applies.", + ), + FieldSpec("STATFlags", "Flags", description="Flags."), + FieldSpec("NameID", "ValueNameID"), + FieldSpec("Fixed", "Value"), + FieldSpec("Fixed", "LinkedValue"), + ], + ), + ( + "AxisValueFormat4", + [ + FieldSpec("uint16", "Format", description="Format, = 4"), + FieldSpec( + "uint16", + "AxisCount", + description="The total number of axes contributing to this axis-values combination.", + ), + FieldSpec("STATFlags", "Flags", description="Flags."), + FieldSpec("NameID", "ValueNameID"), + FieldSpec( + "struct", + "AxisValueRecord", + repeat="AxisCount", + aux=0, + description="Array of AxisValue records that provide the combination of axis values, one for each contributing axis. ", + ), + ], + ), + ( + "AxisValueRecord", + [ + FieldSpec( + "uint16", + "AxisIndex", + description="Index into the axis record array identifying the axis of design variation to which the axis value record applies.", + ), + FieldSpec( + "Fixed", + "Value", + description="A numeric value for this attribute value.", + ), + ], + ), + # + # Variation fonts + # + # GSUB/GPOS FeatureVariations + ( + "FeatureVariations", + [ + FieldSpec( + "Version", + "Version", + description="Version of the table-initially set to 0x00010000", + ), + FieldSpec( + "uint32", + "FeatureVariationCount", + description="Number of records in the FeatureVariationRecord array", + ), + FieldSpec( + "struct", + "FeatureVariationRecord", + repeat="FeatureVariationCount", + aux=0, + description="Array of FeatureVariationRecord", + ), + ], + ), + ( + "FeatureVariationRecord", + [ + FieldSpec( + "LOffset", + "ConditionSet", + description="Offset to a ConditionSet table, from beginning of the FeatureVariations table.", + ), + FieldSpec( + "LOffset", + "FeatureTableSubstitution", + description="Offset to a FeatureTableSubstitution table, from beginning of the FeatureVariations table", + ), + ], + ), + ( + "ConditionList", + [ + FieldSpec( + "uint32", + "ConditionCount", + description="Number of condition tables in the ConditionTable array", + ), + FieldSpec( + "LOffset", + "ConditionTable", + repeat="ConditionCount", + aux=0, + description="Array of offset to condition tables, from the beginning of the ConditionList table.", + ), + ], + ), + ( + "ConditionSet", + [ + FieldSpec( + "uint16", + "ConditionCount", + description="Number of condition tables in the ConditionTable array", + ), + FieldSpec( + "LOffset", + "ConditionTable", + repeat="ConditionCount", + aux=0, + description="Array of offset to condition tables, from the beginning of the ConditionSet table.", + ), + ], + ), + ( + "ConditionTableFormat1", + [ + FieldSpec("uint16", "Format", description="Format, = 1"), + FieldSpec( + "uint16", + "AxisIndex", + description="Index for the variation axis within the fvar table, base 0.", + ), + FieldSpec( + "F2Dot14", + "FilterRangeMinValue", + description="Minimum normalized axis value of the font variation instances that satisfy this condition.", + ), + FieldSpec( + "F2Dot14", + "FilterRangeMaxValue", + description="Maximum value that satisfies this condition.", + ), + ], + ), + ( + "ConditionTableFormat2", + [ + FieldSpec("uint16", "Format", description="Format, = 2"), + FieldSpec( + "int16", "DefaultValue", description="Value at default instance." + ), + FieldSpec( + "uint32", + "VarIdx", + description="Variation index to vary the value based on current designspace location.", + ), + ], + ), + ( + "ConditionTableFormat3", + [ + FieldSpec("uint16", "Format", description="Format, = 3"), + FieldSpec( + "uint8", + "ConditionCount", + description="Index for the variation axis within the fvar table, base 0.", + ), + FieldSpec( + "Offset24", + "ConditionTable", + repeat="ConditionCount", + aux=0, + description="Array of condition tables for this conjunction (AND) expression.", + ), + ], + ), + ( + "ConditionTableFormat4", + [ + FieldSpec("uint16", "Format", description="Format, = 4"), + FieldSpec( + "uint8", + "ConditionCount", + description="Index for the variation axis within the fvar table, base 0.", + ), + FieldSpec( + "Offset24", + "ConditionTable", + repeat="ConditionCount", + aux=0, + description="Array of condition tables for this disjunction (OR) expression.", + ), + ], + ), + ( + "ConditionTableFormat5", + [ + FieldSpec("uint16", "Format", description="Format, = 5"), + FieldSpec("Offset24", "ConditionTable", description="Condition to negate."), + ], + ), + ( + "FeatureTableSubstitution", + [ + FieldSpec( + "Version", + "Version", + description="Version of the table-initially set to 0x00010000", + ), + FieldSpec( + "uint16", + "SubstitutionCount", + description="Number of records in the FeatureVariationRecords array", + ), + FieldSpec( + "FeatureTableSubstitutionRecord", + "SubstitutionRecord", + repeat="SubstitutionCount", + aux=0, + description="Array of FeatureTableSubstitutionRecord", + ), + ], + ), + ( + "FeatureTableSubstitutionRecord", + [ + FieldSpec( + "uint16", + "FeatureIndex", + description="The feature table index to match.", + ), + FieldSpec( + "LOffset", + "Feature", + description="Offset to an alternate feature table, from start of the FeatureTableSubstitution table.", + ), + ], + ), + # VariationStore + ( + "VarRegionAxis", + [ + FieldSpec("F2Dot14", "StartCoord"), + FieldSpec("F2Dot14", "PeakCoord"), + FieldSpec("F2Dot14", "EndCoord"), + ], + ), + ( + "VarRegion", + [ + FieldSpec("struct", "VarRegionAxis", repeat="RegionAxisCount", aux=0), + ], + ), + ( + "VarRegionList", + [ + FieldSpec("uint16", "RegionAxisCount"), + FieldSpec("uint16", "RegionCount"), + FieldSpec("VarRegion", "Region", repeat="RegionCount", aux=0), + ], + ), + ( + "VarData", + [ + FieldSpec("uint16", "ItemCount"), + FieldSpec("uint16", "NumShorts"), + FieldSpec("uint16", "VarRegionCount"), + FieldSpec("uint16", "VarRegionIndex", repeat="VarRegionCount", aux=0), + FieldSpec("VarDataValue", "Item", repeat="ItemCount", aux=0), + ], + ), + ( + "VarStore", + [ + FieldSpec("uint16", "Format", description="Set to 1."), + FieldSpec("LOffset", "VarRegionList"), + FieldSpec("uint16", "VarDataCount"), + FieldSpec("LOffset", "VarData", repeat="VarDataCount", aux=0), + ], + ), + # Variation helpers + ( + "VarIdxMap", + [ + FieldSpec("uint16", "EntryFormat"), # Automatically computed + FieldSpec("uint16", "MappingCount"), # Automatically computed + FieldSpec( + "VarIdxMapValue", + "mapping", + repeat="", + aux=0, + description="Array of compressed data", + ), + ], + ), + ( + "DeltaSetIndexMapFormat0", + [ + FieldSpec( + "uint8", "Format", description="Format of the DeltaSetIndexMap = 0" + ), + FieldSpec("uint8", "EntryFormat"), # Automatically computed + FieldSpec("uint16", "MappingCount"), # Automatically computed + FieldSpec( + "VarIdxMapValue", + "mapping", + repeat="", + aux=0, + description="Array of compressed data", + ), + ], + ), + ( + "DeltaSetIndexMapFormat1", + [ + FieldSpec( + "uint8", "Format", description="Format of the DeltaSetIndexMap = 1" + ), + FieldSpec("uint8", "EntryFormat"), # Automatically computed + FieldSpec("uint32", "MappingCount"), # Automatically computed + FieldSpec( + "VarIdxMapValue", + "mapping", + repeat="", + aux=0, + description="Array of compressed data", + ), + ], + ), + # MultiVariationStore + ( + "SparseVarRegionAxis", + [ + FieldSpec("uint16", "AxisIndex"), + FieldSpec("F2Dot14", "StartCoord"), + FieldSpec("F2Dot14", "PeakCoord"), + FieldSpec("F2Dot14", "EndCoord"), + ], + ), + ( + "SparseVarRegion", + [ + FieldSpec("uint16", "SparseRegionCount"), + FieldSpec( + "struct", "SparseVarRegionAxis", repeat="SparseRegionCount", aux=0 + ), + ], + ), + ( + "SparseVarRegionList", + [ + FieldSpec("uint16", "RegionCount"), + FieldSpec( + "LOffsetTo(SparseVarRegion)", "Region", repeat="RegionCount", aux=0 + ), + ], + ), + ( + "MultiVarData", + [ + FieldSpec("uint8", "Format", description="Set to 1."), + FieldSpec("uint16", "VarRegionCount"), + FieldSpec("uint16", "VarRegionIndex", repeat="VarRegionCount", aux=0), + FieldSpec("TupleList", "Item", repeat="", aux=0), + ], + ), + ( + "MultiVarStore", + [ + FieldSpec("uint16", "Format", description="Set to 1."), + FieldSpec("LOffset", "SparseVarRegionList"), + FieldSpec("uint16", "MultiVarDataCount"), + FieldSpec("LOffset", "MultiVarData", repeat="MultiVarDataCount", aux=0), + ], + ), + # VariableComposites + ( + "VARC", + [ + FieldSpec( + "Version", + "Version", + description="Version of the HVAR table-initially = 0x00010000", + ), + FieldSpec("LOffset", "Coverage"), + FieldSpec("LOffset", "MultiVarStore", description="(may be NULL)"), + FieldSpec("LOffset", "ConditionList", description="(may be NULL)"), + FieldSpec("LOffset", "AxisIndicesList", description="(may be NULL)"), + FieldSpec("LOffset", "VarCompositeGlyphs"), + ], + ), + ( + "AxisIndicesList", + [ + FieldSpec("TupleList", "Item", repeat="", aux=0), + ], + ), + ( + "VarCompositeGlyphs", + [ + FieldSpec("VarCompositeGlyphList", "VarCompositeGlyph", repeat=""), + ], + ), + # Glyph advance variations + ( + "HVAR", + [ + FieldSpec( + "Version", + "Version", + description="Version of the HVAR table-initially = 0x00010000", + ), + FieldSpec("LOffset", "VarStore"), + FieldSpec("LOffsetTo(VarIdxMap)", "AdvWidthMap"), + FieldSpec("LOffsetTo(VarIdxMap)", "LsbMap"), + FieldSpec("LOffsetTo(VarIdxMap)", "RsbMap"), + ], + ), + ( + "VVAR", + [ + FieldSpec( + "Version", + "Version", + description="Version of the VVAR table-initially = 0x00010000", + ), + FieldSpec("LOffset", "VarStore"), + FieldSpec("LOffsetTo(VarIdxMap)", "AdvHeightMap"), + FieldSpec("LOffsetTo(VarIdxMap)", "TsbMap"), + FieldSpec("LOffsetTo(VarIdxMap)", "BsbMap"), + FieldSpec( + "LOffsetTo(VarIdxMap)", + "VOrgMap", + description="Vertical origin mapping.", + ), + ], + ), + # Font-wide metrics variations + ( + "MetricsValueRecord", + [ + FieldSpec( + "Tag", "ValueTag", description="4-byte font-wide measure identifier" + ), + FieldSpec( + "uint32", "VarIdx", description="Combined outer-inner variation index" + ), + FieldSpec( + "uint8", + "MoreBytes", + repeat="ValueRecordSize", + aux=-8, + description="Extra bytes. Set to empty array.", + ), + ], + ), + ( + "MVAR", + [ + FieldSpec( + "Version", + "Version", + description="Version of the MVAR table-initially = 0x00010000", + ), + FieldSpec("uint16", "Reserved", description="Set to 0"), + FieldSpec("uint16", "ValueRecordSize"), + FieldSpec("uint16", "ValueRecordCount"), + FieldSpec("Offset", "VarStore"), + FieldSpec( + "MetricsValueRecord", "ValueRecord", repeat="ValueRecordCount", aux=0 + ), + ], + ), + # + # math + # + ( + "MATH", + [ + FieldSpec( + "Version", + "Version", + description="Version of the MATH table-initially set to 0x00010000.", + ), + FieldSpec( + "Offset", + "MathConstants", + description="Offset to MathConstants table - from the beginning of MATH table.", + ), + FieldSpec( + "Offset", + "MathGlyphInfo", + description="Offset to MathGlyphInfo table - from the beginning of MATH table.", + ), + FieldSpec( + "Offset", + "MathVariants", + description="Offset to MathVariants table - from the beginning of MATH table.", + ), + ], + ), + ( + "MathValueRecord", + [ + FieldSpec( + "int16", "Value", description="The X or Y value in design units." + ), + FieldSpec( + "Offset", + "DeviceTable", + description="Offset to the device table - from the beginning of parent table. May be NULL. Suggested format for device table is 1.", + ), + ], + ), + ( + "MathConstants", + [ + FieldSpec( + "int16", + "ScriptPercentScaleDown", + description="Percentage of scaling down for script level 1. Suggested value: 80%.", + ), + FieldSpec( + "int16", + "ScriptScriptPercentScaleDown", + description="Percentage of scaling down for script level 2 (ScriptScript). Suggested value: 60%.", + ), + FieldSpec( + "uint16", + "DelimitedSubFormulaMinHeight", + description="Minimum height required for a delimited expression to be treated as a subformula. Suggested value: normal line height x1.5.", + ), + FieldSpec( + "uint16", + "DisplayOperatorMinHeight", + description="Minimum height of n-ary operators (such as integral and summation) for formulas in display mode.", + ), + FieldSpec( + "MathValueRecord", + "MathLeading", + description="White space to be left between math formulas to ensure proper line spacing. For example, for applications that treat line gap as a part of line ascender, formulas with ink going above (os2.sTypoAscender + os2.sTypoLineGap - MathLeading) or with ink going below os2.sTypoDescender will result in increasing line height.", + ), + FieldSpec( + "MathValueRecord", "AxisHeight", description="Axis height of the font." + ), + FieldSpec( + "MathValueRecord", + "AccentBaseHeight", + description="Maximum (ink) height of accent base that does not require raising the accents. Suggested: x-height of the font (os2.sxHeight) plus any possible overshots.", + ), + FieldSpec( + "MathValueRecord", + "FlattenedAccentBaseHeight", + description="Maximum (ink) height of accent base that does not require flattening the accents. Suggested: cap height of the font (os2.sCapHeight).", + ), + FieldSpec( + "MathValueRecord", + "SubscriptShiftDown", + description="The standard shift down applied to subscript elements. Positive for moving in the downward direction. Suggested: os2.ySubscriptYOffset.", + ), + FieldSpec( + "MathValueRecord", + "SubscriptTopMax", + description="Maximum allowed height of the (ink) top of subscripts that does not require moving subscripts further down. Suggested: 4/5 x-height.", + ), + FieldSpec( + "MathValueRecord", + "SubscriptBaselineDropMin", + description="Minimum allowed drop of the baseline of subscripts relative to the (ink) bottom of the base. Checked for bases that are treated as a box or extended shape. Positive for subscript baseline dropped below the base bottom.", + ), + FieldSpec( + "MathValueRecord", + "SuperscriptShiftUp", + description="Standard shift up applied to superscript elements. Suggested: os2.ySuperscriptYOffset.", + ), + FieldSpec( + "MathValueRecord", + "SuperscriptShiftUpCramped", + description="Standard shift of superscripts relative to the base, in cramped style.", + ), + FieldSpec( + "MathValueRecord", + "SuperscriptBottomMin", + description="Minimum allowed height of the (ink) bottom of superscripts that does not require moving subscripts further up. Suggested: 1/4 x-height.", + ), + FieldSpec( + "MathValueRecord", + "SuperscriptBaselineDropMax", + description="Maximum allowed drop of the baseline of superscripts relative to the (ink) top of the base. Checked for bases that are treated as a box or extended shape. Positive for superscript baseline below the base top.", + ), + FieldSpec( + "MathValueRecord", + "SubSuperscriptGapMin", + description="Minimum gap between the superscript and subscript ink. Suggested: 4x default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "SuperscriptBottomMaxWithSubscript", + description="The maximum level to which the (ink) bottom of superscript can be pushed to increase the gap between superscript and subscript, before subscript starts being moved down. Suggested: 4/5 x-height.", + ), + FieldSpec( + "MathValueRecord", + "SpaceAfterScript", + description="Extra white space to be added after each subscript and superscript. Suggested: 0.5pt for a 12 pt font.", + ), + FieldSpec( + "MathValueRecord", + "UpperLimitGapMin", + description="Minimum gap between the (ink) bottom of the upper limit, and the (ink) top of the base operator.", + ), + FieldSpec( + "MathValueRecord", + "UpperLimitBaselineRiseMin", + description="Minimum distance between baseline of upper limit and (ink) top of the base operator.", + ), + FieldSpec( + "MathValueRecord", + "LowerLimitGapMin", + description="Minimum gap between (ink) top of the lower limit, and (ink) bottom of the base operator.", + ), + FieldSpec( + "MathValueRecord", + "LowerLimitBaselineDropMin", + description="Minimum distance between baseline of the lower limit and (ink) bottom of the base operator.", + ), + FieldSpec( + "MathValueRecord", + "StackTopShiftUp", + description="Standard shift up applied to the top element of a stack.", + ), + FieldSpec( + "MathValueRecord", + "StackTopDisplayStyleShiftUp", + description="Standard shift up applied to the top element of a stack in display style.", + ), + FieldSpec( + "MathValueRecord", + "StackBottomShiftDown", + description="Standard shift down applied to the bottom element of a stack. Positive for moving in the downward direction.", + ), + FieldSpec( + "MathValueRecord", + "StackBottomDisplayStyleShiftDown", + description="Standard shift down applied to the bottom element of a stack in display style. Positive for moving in the downward direction.", + ), + FieldSpec( + "MathValueRecord", + "StackGapMin", + description="Minimum gap between (ink) bottom of the top element of a stack, and the (ink) top of the bottom element. Suggested: 3x default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "StackDisplayStyleGapMin", + description="Minimum gap between (ink) bottom of the top element of a stack, and the (ink) top of the bottom element in display style. Suggested: 7x default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "StretchStackTopShiftUp", + description="Standard shift up applied to the top element of the stretch stack.", + ), + FieldSpec( + "MathValueRecord", + "StretchStackBottomShiftDown", + description="Standard shift down applied to the bottom element of the stretch stack. Positive for moving in the downward direction.", + ), + FieldSpec( + "MathValueRecord", + "StretchStackGapAboveMin", + description="Minimum gap between the ink of the stretched element, and the (ink) bottom of the element above. Suggested: UpperLimitGapMin", + ), + FieldSpec( + "MathValueRecord", + "StretchStackGapBelowMin", + description="Minimum gap between the ink of the stretched element, and the (ink) top of the element below. Suggested: LowerLimitGapMin.", + ), + FieldSpec( + "MathValueRecord", + "FractionNumeratorShiftUp", + description="Standard shift up applied to the numerator.", + ), + FieldSpec( + "MathValueRecord", + "FractionNumeratorDisplayStyleShiftUp", + description="Standard shift up applied to the numerator in display style. Suggested: StackTopDisplayStyleShiftUp.", + ), + FieldSpec( + "MathValueRecord", + "FractionDenominatorShiftDown", + description="Standard shift down applied to the denominator. Positive for moving in the downward direction.", + ), + FieldSpec( + "MathValueRecord", + "FractionDenominatorDisplayStyleShiftDown", + description="Standard shift down applied to the denominator in display style. Positive for moving in the downward direction. Suggested: StackBottomDisplayStyleShiftDown.", + ), + FieldSpec( + "MathValueRecord", + "FractionNumeratorGapMin", + description="Minimum tolerated gap between the (ink) bottom of the numerator and the ink of the fraction bar. Suggested: default rule thickness", + ), + FieldSpec( + "MathValueRecord", + "FractionNumDisplayStyleGapMin", + description="Minimum tolerated gap between the (ink) bottom of the numerator and the ink of the fraction bar in display style. Suggested: 3x default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "FractionRuleThickness", + description="Thickness of the fraction bar. Suggested: default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "FractionDenominatorGapMin", + description="Minimum tolerated gap between the (ink) top of the denominator and the ink of the fraction bar. Suggested: default rule thickness", + ), + FieldSpec( + "MathValueRecord", + "FractionDenomDisplayStyleGapMin", + description="Minimum tolerated gap between the (ink) top of the denominator and the ink of the fraction bar in display style. Suggested: 3x default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "SkewedFractionHorizontalGap", + description="Horizontal distance between the top and bottom elements of a skewed fraction.", + ), + FieldSpec( + "MathValueRecord", + "SkewedFractionVerticalGap", + description="Vertical distance between the ink of the top and bottom elements of a skewed fraction.", + ), + FieldSpec( + "MathValueRecord", + "OverbarVerticalGap", + description="Distance between the overbar and the (ink) top of he base. Suggested: 3x default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "OverbarRuleThickness", + description="Thickness of overbar. Suggested: default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "OverbarExtraAscender", + description="Extra white space reserved above the overbar. Suggested: default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "UnderbarVerticalGap", + description="Distance between underbar and (ink) bottom of the base. Suggested: 3x default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "UnderbarRuleThickness", + description="Thickness of underbar. Suggested: default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "UnderbarExtraDescender", + description="Extra white space reserved below the underbar. Always positive. Suggested: default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "RadicalVerticalGap", + description="Space between the (ink) top of the expression and the bar over it. Suggested: 1 1/4 default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "RadicalDisplayStyleVerticalGap", + description="Space between the (ink) top of the expression and the bar over it. Suggested: default rule thickness + 1/4 x-height.", + ), + FieldSpec( + "MathValueRecord", + "RadicalRuleThickness", + description="Thickness of the radical rule. This is the thickness of the rule in designed or constructed radical signs. Suggested: default rule thickness.", + ), + FieldSpec( + "MathValueRecord", + "RadicalExtraAscender", + description="Extra white space reserved above the radical. Suggested: RadicalRuleThickness.", + ), + FieldSpec( + "MathValueRecord", + "RadicalKernBeforeDegree", + description="Extra horizontal kern before the degree of a radical, if such is present. Suggested: 5/18 of em.", + ), + FieldSpec( + "MathValueRecord", + "RadicalKernAfterDegree", + description="Negative kern after the degree of a radical, if such is present. Suggested: 10/18 of em.", + ), + FieldSpec( + "uint16", + "RadicalDegreeBottomRaisePercent", + description="Height of the bottom of the radical degree, if such is present, in proportion to the ascender of the radical sign. Suggested: 60%.", + ), + ], + ), + ( + "MathGlyphInfo", + [ + FieldSpec( + "Offset", + "MathItalicsCorrectionInfo", + description="Offset to MathItalicsCorrectionInfo table - from the beginning of MathGlyphInfo table.", + ), + FieldSpec( + "Offset", + "MathTopAccentAttachment", + description="Offset to MathTopAccentAttachment table - from the beginning of MathGlyphInfo table.", + ), + FieldSpec( + "Offset", + "ExtendedShapeCoverage", + description="Offset to coverage table for Extended Shape glyphs - from the beginning of MathGlyphInfo table. When the left or right glyph of a box is an extended shape variant, the (ink) box (and not the default position defined by values in MathConstants table) should be used for vertical positioning purposes. May be NULL.", + ), + FieldSpec( + "Offset", + "MathKernInfo", + description="Offset to MathKernInfo table - from the beginning of MathGlyphInfo table.", + ), + ], + ), + ( + "MathItalicsCorrectionInfo", + [ + FieldSpec( + "Offset", + "Coverage", + description="Offset to Coverage table - from the beginning of MathItalicsCorrectionInfo table.", + ), + FieldSpec( + "uint16", + "ItalicsCorrectionCount", + description="Number of italics correction values. Should coincide with the number of covered glyphs.", + ), + FieldSpec( + "MathValueRecord", + "ItalicsCorrection", + repeat="ItalicsCorrectionCount", + aux=0, + description="Array of MathValueRecords defining italics correction values for each covered glyph.", + ), + ], + ), + ( + "MathTopAccentAttachment", + [ + FieldSpec( + "Offset", + "TopAccentCoverage", + description="Offset to Coverage table - from the beginning of MathTopAccentAttachment table.", + ), + FieldSpec( + "uint16", + "TopAccentAttachmentCount", + description="Number of top accent attachment point values. Should coincide with the number of covered glyphs", + ), + FieldSpec( + "MathValueRecord", + "TopAccentAttachment", + repeat="TopAccentAttachmentCount", + aux=0, + description="Array of MathValueRecords defining top accent attachment points for each covered glyph", + ), + ], + ), + ( + "MathKernInfo", + [ + FieldSpec( + "Offset", + "MathKernCoverage", + description="Offset to Coverage table - from the beginning of the MathKernInfo table.", + ), + FieldSpec( + "uint16", "MathKernCount", description="Number of MathKernInfoRecords." + ), + FieldSpec( + "MathKernInfoRecord", + "MathKernInfoRecords", + repeat="MathKernCount", + aux=0, + description="Array of MathKernInfoRecords, per-glyph information for mathematical positioning of subscripts and superscripts.", + ), + ], + ), + ( + "MathKernInfoRecord", + [ + FieldSpec( + "Offset", + "TopRightMathKern", + description="Offset to MathKern table for top right corner - from the beginning of MathKernInfo table. May be NULL.", + ), + FieldSpec( + "Offset", + "TopLeftMathKern", + description="Offset to MathKern table for the top left corner - from the beginning of MathKernInfo table. May be NULL.", + ), + FieldSpec( + "Offset", + "BottomRightMathKern", + description="Offset to MathKern table for bottom right corner - from the beginning of MathKernInfo table. May be NULL.", + ), + FieldSpec( + "Offset", + "BottomLeftMathKern", + description="Offset to MathKern table for bottom left corner - from the beginning of MathKernInfo table. May be NULL.", + ), + ], + ), + ( + "MathKern", + [ + FieldSpec( + "uint16", + "HeightCount", + description="Number of heights on which the kern value changes.", + ), + FieldSpec( + "MathValueRecord", + "CorrectionHeight", + repeat="HeightCount", + aux=0, + description="Array of correction heights at which the kern value changes. Sorted by the height value in design units.", + ), + FieldSpec( + "MathValueRecord", + "KernValue", + repeat="HeightCount", + aux=1, + description="Array of kern values corresponding to heights. First value is the kern value for all heights less or equal than the first height in this table.Last value is the value to be applied for all heights greater than the last height in this table. Negative values are interpreted as move glyphs closer to each other.", + ), + ], + ), + ( + "MathVariants", + [ + FieldSpec( + "uint16", + "MinConnectorOverlap", + description="Minimum overlap of connecting glyphs during glyph construction, in design units.", + ), + FieldSpec( + "Offset", + "VertGlyphCoverage", + description="Offset to Coverage table - from the beginning of MathVariants table.", + ), + FieldSpec( + "Offset", + "HorizGlyphCoverage", + description="Offset to Coverage table - from the beginning of MathVariants table.", + ), + FieldSpec( + "uint16", + "VertGlyphCount", + description="Number of glyphs for which information is provided for vertically growing variants.", + ), + FieldSpec( + "uint16", + "HorizGlyphCount", + description="Number of glyphs for which information is provided for horizontally growing variants.", + ), + FieldSpec( + "Offset", + "VertGlyphConstruction", + repeat="VertGlyphCount", + aux=0, + description="Array of offsets to MathGlyphConstruction tables - from the beginning of the MathVariants table, for shapes growing in vertical direction.", + ), + FieldSpec( + "Offset", + "HorizGlyphConstruction", + repeat="HorizGlyphCount", + aux=0, + description="Array of offsets to MathGlyphConstruction tables - from the beginning of the MathVariants table, for shapes growing in horizontal direction.", + ), + ], + ), + ( + "MathGlyphConstruction", + [ + FieldSpec( + "Offset", + "GlyphAssembly", + description="Offset to GlyphAssembly table for this shape - from the beginning of MathGlyphConstruction table. May be NULL", + ), + FieldSpec( + "uint16", + "VariantCount", + description="Count of glyph growing variants for this glyph.", + ), + FieldSpec( + "MathGlyphVariantRecord", + "MathGlyphVariantRecord", + repeat="VariantCount", + aux=0, + description="MathGlyphVariantRecords for alternative variants of the glyphs.", + ), + ], + ), + ( + "MathGlyphVariantRecord", + [ + FieldSpec( + "GlyphID", "VariantGlyph", description="Glyph ID for the variant." + ), + FieldSpec( + "uint16", + "AdvanceMeasurement", + description="Advance width/height, in design units, of the variant, in the direction of requested glyph extension.", + ), + ], + ), + ( + "GlyphAssembly", + [ + FieldSpec( + "MathValueRecord", + "ItalicsCorrection", + description="Italics correction of this GlyphAssembly. Should not depend on the assembly size.", + ), + FieldSpec( + "uint16", "PartCount", description="Number of parts in this assembly." + ), + FieldSpec( + "GlyphPartRecord", + "PartRecords", + repeat="PartCount", + aux=0, + description="Array of part records, from left to right and bottom to top.", + ), + ], + ), + ( + "GlyphPartRecord", + [ + FieldSpec("GlyphID", "glyph", description="Glyph ID for the part."), + FieldSpec( + "uint16", + "StartConnectorLength", + description="Advance width/ height of the straight bar connector material, in design units, is at the beginning of the glyph, in the direction of the extension.", + ), + FieldSpec( + "uint16", + "EndConnectorLength", + description="Advance width/ height of the straight bar connector material, in design units, is at the end of the glyph, in the direction of the extension.", + ), + FieldSpec( + "uint16", + "FullAdvance", + description="Full advance width/height for this part, in the direction of the extension. In design units.", + ), + FieldSpec( + "uint16", + "PartFlags", + description="Part qualifiers. PartFlags enumeration currently uses only one bit: 0x0001 fExtender: If set, the part can be skipped or repeated. 0xFFFE Reserved", + ), + ], + ), + ## + ## Apple Advanced Typography (AAT) tables + ## + ( + "AATLookupSegment", + [ + FieldSpec( + "uint16", "lastGlyph", description="Last glyph index in this segment." + ), + FieldSpec( + "uint16", "firstGlyph", description="First glyph index in this segment." + ), + FieldSpec( + "uint16", + "value", + description="A 16-bit offset from the start of the table to the data.", + ), + ], + ), + # + # ankr + # + ( + "ankr", + [ + FieldSpec("struct", "AnchorPoints", description="Anchor points table."), + ], + ), + ( + "AnchorPointsFormat0", + [ + FieldSpec( + "uint16", + "Format", + description="Format of the anchor points table, = 0.", + ), + FieldSpec( + "uint16", "Flags", description="Flags. Currenty unused, set to zero." + ), + FieldSpec( + "AATLookupWithDataOffset(AnchorGlyphData)", + "Anchors", + description="Table of with anchor overrides for each glyph.", + ), + ], + ), + ( + "AnchorGlyphData", + [ + FieldSpec( + "uint32", + "AnchorPointCount", + description="Number of anchor points for this glyph.", + ), + FieldSpec( + "struct", + "AnchorPoint", + repeat="AnchorPointCount", + aux=0, + description="Individual anchor points.", + ), + ], + ), + ( + "AnchorPoint", + [ + FieldSpec( + "int16", "XCoordinate", description="X coordinate of this anchor point." + ), + FieldSpec( + "int16", "YCoordinate", description="Y coordinate of this anchor point." + ), + ], + ), + # + # bsln + # + ( + "bsln", + [ + FieldSpec( + "Version", + "Version", + description="Version number of the AAT baseline table (0x00010000 for the initial version).", + ), + FieldSpec("struct", "Baseline", description="Baseline table."), + ], + ), + ( + "BaselineFormat0", + [ + FieldSpec( + "uint16", "Format", description="Format of the baseline table, = 0." + ), + FieldSpec( + "uint16", + "DefaultBaseline", + description="Default baseline value for all glyphs. This value can be from 0 through 31.", + ), + FieldSpec( + "uint16", + "Delta", + repeat=32, + aux=0, + description="These are the FUnit distance deltas from the font’s natural baseline to the other baselines used in the font. A total of 32 deltas must be assigned.", + ), + ], + ), + ( + "BaselineFormat1", + [ + FieldSpec( + "uint16", "Format", description="Format of the baseline table, = 1." + ), + FieldSpec( + "uint16", + "DefaultBaseline", + description="Default baseline value for all glyphs. This value can be from 0 through 31.", + ), + FieldSpec( + "uint16", + "Delta", + repeat=32, + aux=0, + description="These are the FUnit distance deltas from the font’s natural baseline to the other baselines used in the font. A total of 32 deltas must be assigned.", + ), + FieldSpec( + "AATLookup(uint16)", + "BaselineValues", + description="Lookup table that maps glyphs to their baseline values.", + ), + ], + ), + ( + "BaselineFormat2", + [ + FieldSpec( + "uint16", "Format", description="Format of the baseline table, = 1." + ), + FieldSpec( + "uint16", + "DefaultBaseline", + description="Default baseline value for all glyphs. This value can be from 0 through 31.", + ), + FieldSpec( + "GlyphID", + "StandardGlyph", + description="Glyph index of the glyph in this font to be used to set the baseline values. This glyph must contain a set of control points (whose numbers are contained in the following field) that determines baseline distances.", + ), + FieldSpec( + "uint16", + "ControlPoint", + repeat=32, + aux=0, + description="Array of 32 control point numbers, associated with the standard glyph. A value of 0xFFFF means there is no corresponding control point in the standard glyph.", + ), + ], + ), + ( + "BaselineFormat3", + [ + FieldSpec( + "uint16", "Format", description="Format of the baseline table, = 1." + ), + FieldSpec( + "uint16", + "DefaultBaseline", + description="Default baseline value for all glyphs. This value can be from 0 through 31.", + ), + FieldSpec( + "GlyphID", + "StandardGlyph", + description="Glyph index of the glyph in this font to be used to set the baseline values. This glyph must contain a set of control points (whose numbers are contained in the following field) that determines baseline distances.", + ), + FieldSpec( + "uint16", + "ControlPoint", + repeat=32, + aux=0, + description="Array of 32 control point numbers, associated with the standard glyph. A value of 0xFFFF means there is no corresponding control point in the standard glyph.", + ), + FieldSpec( + "AATLookup(uint16)", + "BaselineValues", + description="Lookup table that maps glyphs to their baseline values.", + ), + ], + ), + # + # cidg + # + ( + "cidg", + [ + FieldSpec( + "struct", "CIDGlyphMapping", description="CID-to-glyph mapping table." + ), + ], + ), + ( + "CIDGlyphMappingFormat0", + [ + FieldSpec( + "uint16", + "Format", + description="Format of the CID-to-glyph mapping table, = 0.", + ), + FieldSpec( + "uint16", "DataFormat", description="Currenty unused, set to zero." + ), + FieldSpec( + "uint32", "StructLength", description="Size of the table in bytes." + ), + FieldSpec("uint16", "Registry", description="The registry ID."), + FieldSpec( + "char64", + "RegistryName", + description="The registry name in ASCII; unused bytes should be set to 0.", + ), + FieldSpec("uint16", "Order", description="The order ID."), + FieldSpec( + "char64", + "OrderName", + description="The order name in ASCII; unused bytes should be set to 0.", + ), + FieldSpec( + "uint16", "SupplementVersion", description="The supplement version." + ), + FieldSpec( + "CIDGlyphMap", + "Mapping", + description="A mapping from CIDs to the glyphs in the font, starting with CID 0. If a CID from the identified collection has no glyph in the font, 0xFFFF is used", + ), + ], + ), + # + # feat + # + ( + "feat", + [ + FieldSpec( + "Version", + "Version", + description="Version of the feat table-initially set to 0x00010000.", + ), + FieldSpec("FeatureNames", "FeatureNames", description="The feature names."), + ], + ), + ( + "FeatureNames", + [ + FieldSpec( + "uint16", + "FeatureNameCount", + description="Number of entries in the feature name array.", + ), + FieldSpec("uint16", "Reserved1", description="Reserved (set to zero)."), + FieldSpec("uint32", "Reserved2", description="Reserved (set to zero)."), + FieldSpec( + "FeatureName", + "FeatureName", + repeat="FeatureNameCount", + aux=0, + description="The feature name array.", + ), + ], + ), + ( + "FeatureName", + [ + FieldSpec("uint16", "FeatureType", description="Feature type."), + FieldSpec( + "uint16", + "SettingsCount", + description="The number of records in the setting name array.", + ), + FieldSpec( + "LOffset", + "Settings", + description="Offset to setting table for this feature.", + ), + FieldSpec( + "uint16", + "FeatureFlags", + description="Single-bit flags associated with the feature type.", + ), + FieldSpec( + "NameID", + "FeatureNameID", + description="The name table index for the feature name.", + ), + ], + ), + ( + "Settings", + [ + FieldSpec( + "Setting", + "Setting", + repeat="SettingsCount", + aux=0, + description="The setting array.", + ), + ], + ), + ( + "Setting", + [ + FieldSpec("uint16", "SettingValue", description="The setting."), + FieldSpec( + "NameID", + "SettingNameID", + description="The name table index for the setting name.", + ), + ], + ), + # + # gcid + # + ( + "gcid", + [ + FieldSpec( + "struct", "GlyphCIDMapping", description="Glyph to CID mapping table." + ), + ], + ), + ( + "GlyphCIDMappingFormat0", + [ + FieldSpec( + "uint16", + "Format", + description="Format of the glyph-to-CID mapping table, = 0.", + ), + FieldSpec( + "uint16", "DataFormat", description="Currenty unused, set to zero." + ), + FieldSpec( + "uint32", "StructLength", description="Size of the table in bytes." + ), + FieldSpec("uint16", "Registry", description="The registry ID."), + FieldSpec( + "char64", + "RegistryName", + description="The registry name in ASCII; unused bytes should be set to 0.", + ), + FieldSpec("uint16", "Order", description="The order ID."), + FieldSpec( + "char64", + "OrderName", + description="The order name in ASCII; unused bytes should be set to 0.", + ), + FieldSpec( + "uint16", "SupplementVersion", description="The supplement version." + ), + FieldSpec( + "GlyphCIDMap", + "Mapping", + description="The CIDs for the glyphs in the font, starting with glyph 0. If a glyph does not correspond to a CID in the identified collection, 0xFFFF is used", + ), + ], + ), + # + # lcar + # + ( + "lcar", + [ + FieldSpec( + "Version", + "Version", + description="Version number of the ligature caret table (0x00010000 for the initial version).", + ), + FieldSpec("struct", "LigatureCarets", description="Ligature carets table."), + ], + ), + ( + "LigatureCaretsFormat0", + [ + FieldSpec( + "uint16", + "Format", + description="Format of the ligature caret table. Format 0 indicates division points are distances in font units, Format 1 indicates division points are indexes of control points.", + ), + FieldSpec( + "AATLookup(LigCaretDistances)", + "Carets", + description="Lookup table associating ligature glyphs with their caret positions, in font unit distances.", + ), + ], + ), + ( + "LigatureCaretsFormat1", + [ + FieldSpec( + "uint16", + "Format", + description="Format of the ligature caret table. Format 0 indicates division points are distances in font units, Format 1 indicates division points are indexes of control points.", + ), + FieldSpec( + "AATLookup(LigCaretPoints)", + "Carets", + description="Lookup table associating ligature glyphs with their caret positions, as control points.", + ), + ], + ), + ( + "LigCaretDistances", + [ + FieldSpec( + "uint16", "DivsionPointCount", description="Number of division points." + ), + FieldSpec( + "int16", + "DivisionPoint", + repeat="DivsionPointCount", + aux=0, + description="Distance in font units through which a subdivision is made orthogonally to the baseline.", + ), + ], + ), + ( + "LigCaretPoints", + [ + FieldSpec( + "uint16", "DivsionPointCount", description="Number of division points." + ), + FieldSpec( + "int16", + "DivisionPoint", + repeat="DivsionPointCount", + aux=0, + description="The number of the control point through which a subdivision is made orthogonally to the baseline.", + ), + ], + ), + # + # mort + # + ( + "mort", + [ + FieldSpec("Version", "Version", description="Version of the mort table."), + FieldSpec( + "uint32", + "MorphChainCount", + description="Number of metamorphosis chains.", + ), + FieldSpec( + "MortChain", + "MorphChain", + repeat="MorphChainCount", + aux=0, + description="Array of metamorphosis chains.", + ), + ], + ), + ( + "MortChain", + [ + FieldSpec( + "Flags32", + "DefaultFlags", + description="The default specification for subtables.", + ), + FieldSpec( + "uint32", + "StructLength", + description="Total byte count, including this header; must be a multiple of 4.", + ), + FieldSpec( + "uint16", + "MorphFeatureCount", + description="Number of metamorphosis feature entries.", + ), + FieldSpec( + "uint16", + "MorphSubtableCount", + description="The number of subtables in the chain.", + ), + FieldSpec( + "struct", + "MorphFeature", + repeat="MorphFeatureCount", + aux=0, + description="Array of metamorphosis features.", + ), + FieldSpec( + "MortSubtable", + "MorphSubtable", + repeat="MorphSubtableCount", + aux=0, + description="Array of metamorphosis subtables.", + ), + ], + ), + ( + "MortSubtable", + [ + FieldSpec( + "uint16", + "StructLength", + description="Total subtable length, including this header.", + ), + FieldSpec( + "uint8", + "CoverageFlags", + description="Most significant byte of coverage flags.", + ), + FieldSpec("uint8", "MorphType", description="Subtable type."), + FieldSpec( + "Flags32", + "SubFeatureFlags", + description="The 32-bit mask identifying which subtable this is (the subtable being executed if the AND of this value and the processed defaultFlags is nonzero).", + ), + FieldSpec("SubStruct", "SubStruct", description="SubTable."), + ], + ), + # + # morx + # + ( + "morx", + [ + FieldSpec("uint16", "Version", description="Version of the morx table."), + FieldSpec("uint16", "Reserved", description="Reserved (set to zero)."), + FieldSpec( + "uint32", + "MorphChainCount", + description="Number of extended metamorphosis chains.", + ), + FieldSpec( + "MorxChain", + "MorphChain", + repeat="MorphChainCount", + aux=0, + description="Array of extended metamorphosis chains.", + ), + ], + ), + ( + "MorxChain", + [ + FieldSpec( + "Flags32", + "DefaultFlags", + description="The default specification for subtables.", + ), + FieldSpec( + "uint32", + "StructLength", + description="Total byte count, including this header; must be a multiple of 4.", + ), + FieldSpec( + "uint32", + "MorphFeatureCount", + description="Number of feature subtable entries.", + ), + FieldSpec( + "uint32", + "MorphSubtableCount", + description="The number of subtables in the chain.", + ), + FieldSpec( + "MorphFeature", + "MorphFeature", + repeat="MorphFeatureCount", + aux=0, + description="Array of metamorphosis features.", + ), + FieldSpec( + "MorxSubtable", + "MorphSubtable", + repeat="MorphSubtableCount", + aux=0, + description="Array of extended metamorphosis subtables.", + ), + ], + ), + ( + "MorphFeature", + [ + FieldSpec("uint16", "FeatureType", description="The type of feature."), + FieldSpec( + "uint16", + "FeatureSetting", + description="The feature's setting (aka selector).", + ), + FieldSpec( + "Flags32", + "EnableFlags", + description="Flags for the settings that this feature and setting enables.", + ), + FieldSpec( + "Flags32", + "DisableFlags", + description="Complement of flags for the settings that this feature and setting disable.", + ), + ], + ), + # Apple TrueType Reference Manual, chapter “The ‘morx’ table”, + # section “Metamorphosis Subtables”. + # https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6morx.html + ( + "MorxSubtable", + [ + FieldSpec( + "uint32", + "StructLength", + description="Total subtable length, including this header.", + ), + FieldSpec( + "uint8", + "CoverageFlags", + description="Most significant byte of coverage flags.", + ), + FieldSpec("uint16", "Reserved", description="Unused."), + FieldSpec("uint8", "MorphType", description="Subtable type."), + FieldSpec( + "Flags32", + "SubFeatureFlags", + description="The 32-bit mask identifying which subtable this is (the subtable being executed if the AND of this value and the processed defaultFlags is nonzero).", + ), + FieldSpec("SubStruct", "SubStruct", description="SubTable."), + ], + ), + ( + "StateHeader", + [ + FieldSpec( + "uint32", + "ClassCount", + description="Number of classes, which is the number of 16-bit entry indices in a single line in the state array.", + ), + FieldSpec( + "uint32", + "MorphClass", + description="Offset from the start of this state table header to the start of the class table.", + ), + FieldSpec( + "uint32", + "StateArrayOffset", + description="Offset from the start of this state table header to the start of the state array.", + ), + FieldSpec( + "uint32", + "EntryTableOffset", + description="Offset from the start of this state table header to the start of the entry table.", + ), + ], + ), + ( + "RearrangementMorph", + [ + FieldSpec( + "STXHeader(RearrangementMorphAction)", + "StateTable", + description="Finite-state transducer table for indic rearrangement.", + ), + ], + ), + ( + "ContextualMorph", + [ + FieldSpec( + "STXHeader(ContextualMorphAction)", + "StateTable", + description="Finite-state transducer for contextual glyph substitution.", + ), + ], + ), + ( + "LigatureMorph", + [ + FieldSpec( + "STXHeader(LigatureMorphAction)", + "StateTable", + description="Finite-state transducer for ligature substitution.", + ), + ], + ), + ( + "NoncontextualMorph", + [ + FieldSpec( + "AATLookup(GlyphID)", + "Substitution", + description="The noncontextual glyph substitution table.", + ), + ], + ), + ( + "InsertionMorph", + [ + FieldSpec( + "STXHeader(InsertionMorphAction)", + "StateTable", + description="Finite-state transducer for glyph insertion.", + ), + ], + ), + ( + "MorphClass", + [ + FieldSpec( + "uint16", + "FirstGlyph", + description="Glyph index of the first glyph in the class table.", + ), + # ('uint16', 'GlyphCount', None, None, 'Number of glyphs in class table.'), + # ('uint8', 'GlyphClass', 'GlyphCount', 0, 'The class codes (indexed by glyph index minus firstGlyph). Class codes range from 0 to the value of stateSize minus 1.'), + ], + ), + # If the 'morx' table version is 3 or greater, then the last subtable in the chain is followed by a subtableGlyphCoverageArray, as described below. + # ('Offset', 'MarkGlyphSetsDef', None, 'round(Version*0x10000) >= 0x00010002', 'Offset to the table of mark set definitions-from beginning of GDEF header (may be NULL)'), + # + # prop + # + ( + "prop", + [ + FieldSpec( + "Fixed", + "Version", + description="Version number of the AAT glyphs property table. Version 1.0 is the initial table version. Version 2.0, which is recognized by macOS 8.5 and later, adds support for the “attaches on right” bit. Version 3.0, which gets recognized by macOS X and iOS, adds support for the additional directional properties defined in Unicode 3.0.", + ), + FieldSpec("struct", "GlyphProperties", description="Glyph properties."), + ], + ), + ( + "GlyphPropertiesFormat0", + [ + FieldSpec("uint16", "Format", description="Format, = 0."), + FieldSpec( + "uint16", + "DefaultProperties", + description="Default properties applied to a glyph. Since there is no lookup table in prop format 0, the default properties get applied to every glyph in the font.", + ), + ], + ), + ( + "GlyphPropertiesFormat1", + [ + FieldSpec("uint16", "Format", description="Format, = 1."), + FieldSpec( + "uint16", + "DefaultProperties", + description="Default properties applied to a glyph if that glyph is not present in the Properties lookup table.", + ), + FieldSpec( + "AATLookup(uint16)", + "Properties", + description="Lookup data associating glyphs with their properties.", + ), + ], + ), + # + # opbd + # + ( + "opbd", + [ + FieldSpec( + "Version", + "Version", + description="Version number of the optical bounds table (0x00010000 for the initial version).", + ), + FieldSpec("struct", "OpticalBounds", description="Optical bounds table."), + ], + ), + ( + "OpticalBoundsFormat0", + [ + FieldSpec( + "uint16", + "Format", + description="Format of the optical bounds table, = 0.", + ), + FieldSpec( + "AATLookup(OpticalBoundsDeltas)", + "OpticalBoundsDeltas", + description="Lookup table associating glyphs with their optical bounds, given as deltas in font units.", + ), + ], + ), + ( + "OpticalBoundsFormat1", + [ + FieldSpec( + "uint16", + "Format", + description="Format of the optical bounds table, = 1.", + ), + FieldSpec( + "AATLookup(OpticalBoundsPoints)", + "OpticalBoundsPoints", + description="Lookup table associating glyphs with their optical bounds, given as references to control points.", + ), + ], + ), + ( + "OpticalBoundsDeltas", + [ + FieldSpec( + "int16", + "Left", + description="Delta value for the left-side optical edge.", + ), + FieldSpec( + "int16", "Top", description="Delta value for the top-side optical edge." + ), + FieldSpec( + "int16", + "Right", + description="Delta value for the right-side optical edge.", + ), + FieldSpec( + "int16", + "Bottom", + description="Delta value for the bottom-side optical edge.", + ), + ], + ), + ( + "OpticalBoundsPoints", + [ + FieldSpec( + "int16", + "Left", + description="Control point index for the left-side optical edge, or -1 if this glyph has none.", + ), + FieldSpec( + "int16", + "Top", + description="Control point index for the top-side optical edge, or -1 if this glyph has none.", + ), + FieldSpec( + "int16", + "Right", + description="Control point index for the right-side optical edge, or -1 if this glyph has none.", + ), + FieldSpec( + "int16", + "Bottom", + description="Control point index for the bottom-side optical edge, or -1 if this glyph has none.", + ), + ], + ), + # + # TSIC + # + ( + "TSIC", + [ + FieldSpec( + "Version", + "Version", + description="Version of table initially set to 0x00010000.", + ), + FieldSpec("uint16", "Flags", description="TSIC flags - set to 0"), + FieldSpec("uint16", "AxisCount", description="Axis count from fvar"), + FieldSpec("uint16", "RecordCount", description="TSIC record count"), + FieldSpec("uint16", "Reserved", description="Set to 0"), + FieldSpec( + "Tag", + "AxisArray", + repeat="AxisCount", + aux=0, + description="Array of axis tags in fvar order", + ), + FieldSpec( + "LocationRecord", + "RecordLocations", + repeat="RecordCount", + aux=0, + description="Location in variation space of TSIC record", + ), + FieldSpec( + "TSICRecord", + "Record", + repeat="RecordCount", + aux=0, + description="Array of TSIC records", + ), + ], + ), + ( + "LocationRecord", + [ + FieldSpec( + "F2Dot14", "Axis", repeat="AxisCount", aux=0, description="Axis record" + ), + ], + ), + ( + "TSICRecord", + [ + FieldSpec("uint16", "Flags", description="Record flags - set to 0"), + FieldSpec( + "uint16", + "NumCVTEntries", + description="Number of CVT number value pairs", + ), + FieldSpec( + "uint16", + "NameLength", + description="Length of optional user record name", + ), + FieldSpec( + "uint16", + "NameArray", + repeat="NameLength", + aux=0, + description="Unicode 16 name", + ), + FieldSpec( + "uint16", + "CVTArray", + repeat="NumCVTEntries", + aux=0, + description="CVT number array", + ), + FieldSpec( + "int16", + "CVTValueArray", + repeat="NumCVTEntries", + aux=0, + description="CVT value", + ), + ], + ), + # + # COLR + # + ( + "COLR", + [ + FieldSpec( + "uint16", "Version", description="Table version number (starts at 0)." + ), + FieldSpec( + "uint16", + "BaseGlyphRecordCount", + description="Number of Base Glyph Records.", + ), + FieldSpec( + "LOffset", + "BaseGlyphRecordArray", + description="Offset (from beginning of COLR table) to Base Glyph records.", + ), + FieldSpec( + "LOffset", + "LayerRecordArray", + description="Offset (from beginning of COLR table) to Layer Records.", + ), + FieldSpec( + "uint16", "LayerRecordCount", description="Number of Layer Records." + ), + FieldSpec( + "LOffset", + "BaseGlyphList", + aux="Version >= 1", + description="Offset (from beginning of COLR table) to array of Version-1 Base Glyph records.", + ), + FieldSpec( + "LOffset", + "LayerList", + aux="Version >= 1", + description="Offset (from beginning of COLR table) to LayerList.", + ), + FieldSpec( + "LOffset", + "ClipList", + aux="Version >= 1", + description="Offset to ClipList table (may be NULL)", + ), + FieldSpec( + "LOffsetTo(DeltaSetIndexMap)", + "VarIndexMap", + aux="Version >= 1", + description="Offset to DeltaSetIndexMap table (may be NULL)", + ), + FieldSpec( + "LOffset", + "VarStore", + aux="Version >= 1", + description="Offset to variation store (may be NULL)", + ), + ], + ), + ( + "BaseGlyphRecordArray", + [ + FieldSpec( + "BaseGlyphRecord", + "BaseGlyphRecord", + repeat="BaseGlyphRecordCount", + aux=0, + description="Base Glyph records.", + ), + ], + ), + ( + "BaseGlyphRecord", + [ + FieldSpec( + "GlyphID", + "BaseGlyph", + description="Glyph ID of reference glyph. This glyph is for reference only and is not rendered for color.", + ), + FieldSpec( + "uint16", + "FirstLayerIndex", + description="Index (from beginning of the Layer Records) to the layer record. There will be numLayers consecutive entries for this base glyph.", + ), + FieldSpec( + "uint16", + "NumLayers", + description="Number of color layers associated with this glyph.", + ), + ], + ), + ( + "LayerRecordArray", + [ + FieldSpec( + "LayerRecord", + "LayerRecord", + repeat="LayerRecordCount", + aux=0, + description="Layer records.", + ), + ], + ), + ( + "LayerRecord", + [ + FieldSpec( + "GlyphID", + "LayerGlyph", + description="Glyph ID of layer glyph (must be in z-order from bottom to top).", + ), + FieldSpec( + "uint16", + "PaletteIndex", + description="Index value to use with a selected color palette.", + ), + ], + ), + ( + "BaseGlyphList", + [ + FieldSpec( + "uint32", + "BaseGlyphCount", + description="Number of Version-1 Base Glyph records", + ), + FieldSpec( + "struct", + "BaseGlyphPaintRecord", + repeat="BaseGlyphCount", + aux=0, + description="Array of Version-1 Base Glyph records", + ), + ], + ), + ( + "BaseGlyphPaintRecord", + [ + FieldSpec( + "GlyphID", "BaseGlyph", description="Glyph ID of reference glyph." + ), + FieldSpec( + "LOffset", + "Paint", + description="Offset (from beginning of BaseGlyphPaintRecord) to Paint, typically a PaintColrLayers.", + ), + ], + ), + ( + "LayerList", + [ + FieldSpec("uint32", "LayerCount", description="Number of Version-1 Layers"), + FieldSpec( + "LOffset", + "Paint", + repeat="LayerCount", + aux=0, + description="Array of offsets to Paint tables, from the start of the LayerList table.", + ), + ], + ), + ( + "ClipListFormat1", + [ + FieldSpec( + "uint8", + "Format", + description="Format for ClipList with 16bit glyph IDs: 1", + ), + FieldSpec("uint32", "ClipCount", description="Number of Clip records."), + FieldSpec( + "struct", + "ClipRecord", + repeat="ClipCount", + aux=0, + description="Array of Clip records sorted by glyph ID.", + ), + ], + ), + ( + "ClipRecord", + [ + FieldSpec( + "uint16", "StartGlyphID", description="First glyph ID in the range." + ), + FieldSpec( + "uint16", "EndGlyphID", description="Last glyph ID in the range." + ), + FieldSpec("Offset24", "ClipBox", description="Offset to a ClipBox table."), + ], + ), + ( + "ClipBoxFormat1", + [ + FieldSpec( + "uint8", + "Format", + description="Format for ClipBox without variation: set to 1.", + ), + FieldSpec("int16", "xMin", description="Minimum x of clip box."), + FieldSpec("int16", "yMin", description="Minimum y of clip box."), + FieldSpec("int16", "xMax", description="Maximum x of clip box."), + FieldSpec("int16", "yMax", description="Maximum y of clip box."), + ], + ), + ( + "ClipBoxFormat2", + [ + FieldSpec( + "uint8", "Format", description="Format for variable ClipBox: set to 2." + ), + FieldSpec( + "int16", "xMin", description="Minimum x of clip box. VarIndexBase + 0." + ), + FieldSpec( + "int16", "yMin", description="Minimum y of clip box. VarIndexBase + 1." + ), + FieldSpec( + "int16", "xMax", description="Maximum x of clip box. VarIndexBase + 2." + ), + FieldSpec( + "int16", "yMax", description="Maximum y of clip box. VarIndexBase + 3." + ), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # COLRv1 Affine2x3 uses the same column-major order to serialize a 2D + # Affine Transformation as the one used by fontTools.misc.transform. + # However, for historical reasons, the labels 'xy' and 'yx' are swapped. + # Their fundamental meaning is the same though. + # COLRv1 Affine2x3 follows the names found in FreeType and Cairo. + # In all case, the second element in the 6-tuple correspond to the + # y-part of the x basis vector, and the third to the x-part of the y + # basis vector. + # See https://github.com/googlefonts/colr-gradients-spec/pull/85 + ( + "Affine2x3", + [ + FieldSpec("Fixed", "xx", description="x-part of x basis vector"), + FieldSpec("Fixed", "yx", description="y-part of x basis vector"), + FieldSpec("Fixed", "xy", description="x-part of y basis vector"), + FieldSpec("Fixed", "yy", description="y-part of y basis vector"), + FieldSpec("Fixed", "dx", description="Translation in x direction"), + FieldSpec("Fixed", "dy", description="Translation in y direction"), + ], + ), + ( + "VarAffine2x3", + [ + FieldSpec( + "Fixed", "xx", description="x-part of x basis vector. VarIndexBase + 0." + ), + FieldSpec( + "Fixed", "yx", description="y-part of x basis vector. VarIndexBase + 1." + ), + FieldSpec( + "Fixed", "xy", description="x-part of y basis vector. VarIndexBase + 2." + ), + FieldSpec( + "Fixed", "yy", description="y-part of y basis vector. VarIndexBase + 3." + ), + FieldSpec( + "Fixed", + "dx", + description="Translation in x direction. VarIndexBase + 4.", + ), + FieldSpec( + "Fixed", + "dy", + description="Translation in y direction. VarIndexBase + 5.", + ), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + ( + "ColorStop", + [ + FieldSpec("F2Dot14", "StopOffset"), + FieldSpec( + "uint16", "PaletteIndex", description="Index for a CPAL palette entry." + ), + FieldSpec( + "F2Dot14", "Alpha", description="Values outsided [0.,1.] reserved" + ), + ], + ), + ( + "VarColorStop", + [ + FieldSpec("F2Dot14", "StopOffset", description="VarIndexBase + 0."), + FieldSpec( + "uint16", "PaletteIndex", description="Index for a CPAL palette entry." + ), + FieldSpec( + "F2Dot14", + "Alpha", + description="Values outsided [0.,1.] reserved. VarIndexBase + 1.", + ), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + ( + "ColorLine", + [ + FieldSpec( + "ExtendMode", + "Extend", + description="Enum {PAD = 0, REPEAT = 1, REFLECT = 2}", + ), + FieldSpec("uint16", "StopCount", description="Number of Color stops."), + FieldSpec( + "ColorStop", + "ColorStop", + repeat="StopCount", + aux=0, + description="Array of Color stops.", + ), + ], + ), + ( + "VarColorLine", + [ + FieldSpec( + "ExtendMode", + "Extend", + description="Enum {PAD = 0, REPEAT = 1, REFLECT = 2}", + ), + FieldSpec("uint16", "StopCount", description="Number of Color stops."), + FieldSpec( + "VarColorStop", + "ColorStop", + repeat="StopCount", + aux=0, + description="Array of Color stops.", + ), + ], + ), + # PaintColrLayers + ( + "PaintFormat1", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 1" + ), + FieldSpec( + "uint8", + "NumLayers", + description="Number of offsets to Paint to read from LayerList.", + ), + FieldSpec("uint32", "FirstLayerIndex", description="Index into LayerList."), + ], + ), + # PaintSolid + ( + "PaintFormat2", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 2" + ), + FieldSpec( + "uint16", "PaletteIndex", description="Index for a CPAL palette entry." + ), + FieldSpec( + "F2Dot14", "Alpha", description="Values outsided [0.,1.] reserved" + ), + ], + ), + # PaintVarSolid + ( + "PaintFormat3", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 3" + ), + FieldSpec( + "uint16", "PaletteIndex", description="Index for a CPAL palette entry." + ), + FieldSpec( + "F2Dot14", + "Alpha", + description="Values outsided [0.,1.] reserved. VarIndexBase + 0.", + ), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintLinearGradient + ( + "PaintFormat4", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 4" + ), + FieldSpec( + "Offset24", + "ColorLine", + description="Offset (from beginning of PaintLinearGradient table) to ColorLine subtable.", + ), + FieldSpec("int16", "x0"), + FieldSpec("int16", "y0"), + FieldSpec("int16", "x1"), + FieldSpec("int16", "y1"), + FieldSpec("int16", "x2"), + FieldSpec("int16", "y2"), + ], + ), + # PaintVarLinearGradient + ( + "PaintFormat5", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 5" + ), + FieldSpec( + "LOffset24To(VarColorLine)", + "ColorLine", + description="Offset (from beginning of PaintVarLinearGradient table) to VarColorLine subtable.", + ), + FieldSpec("int16", "x0", description="VarIndexBase + 0."), + FieldSpec("int16", "y0", description="VarIndexBase + 1."), + FieldSpec("int16", "x1", description="VarIndexBase + 2."), + FieldSpec("int16", "y1", description="VarIndexBase + 3."), + FieldSpec("int16", "x2", description="VarIndexBase + 4."), + FieldSpec("int16", "y2", description="VarIndexBase + 5."), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintRadialGradient + ( + "PaintFormat6", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 6" + ), + FieldSpec( + "Offset24", + "ColorLine", + description="Offset (from beginning of PaintRadialGradient table) to ColorLine subtable.", + ), + FieldSpec("int16", "x0"), + FieldSpec("int16", "y0"), + FieldSpec("uint16", "r0"), + FieldSpec("int16", "x1"), + FieldSpec("int16", "y1"), + FieldSpec("uint16", "r1"), + ], + ), + # PaintVarRadialGradient + ( + "PaintFormat7", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 7" + ), + FieldSpec( + "LOffset24To(VarColorLine)", + "ColorLine", + description="Offset (from beginning of PaintVarRadialGradient table) to VarColorLine subtable.", + ), + FieldSpec("int16", "x0", description="VarIndexBase + 0."), + FieldSpec("int16", "y0", description="VarIndexBase + 1."), + FieldSpec("uint16", "r0", description="VarIndexBase + 2."), + FieldSpec("int16", "x1", description="VarIndexBase + 3."), + FieldSpec("int16", "y1", description="VarIndexBase + 4."), + FieldSpec("uint16", "r1", description="VarIndexBase + 5."), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintSweepGradient + ( + "PaintFormat8", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 8" + ), + FieldSpec( + "Offset24", + "ColorLine", + description="Offset (from beginning of PaintSweepGradient table) to ColorLine subtable.", + ), + FieldSpec("int16", "centerX", description="Center x coordinate."), + FieldSpec("int16", "centerY", description="Center y coordinate."), + FieldSpec( + "BiasedAngle", + "startAngle", + description="Start of the angular range of the gradient.", + ), + FieldSpec( + "BiasedAngle", + "endAngle", + description="End of the angular range of the gradient.", + ), + ], + ), + # PaintVarSweepGradient + ( + "PaintFormat9", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 9" + ), + FieldSpec( + "LOffset24To(VarColorLine)", + "ColorLine", + description="Offset (from beginning of PaintVarSweepGradient table) to VarColorLine subtable.", + ), + FieldSpec( + "int16", "centerX", description="Center x coordinate. VarIndexBase + 0." + ), + FieldSpec( + "int16", "centerY", description="Center y coordinate. VarIndexBase + 1." + ), + FieldSpec( + "BiasedAngle", + "startAngle", + description="Start of the angular range of the gradient. VarIndexBase + 2.", + ), + FieldSpec( + "BiasedAngle", + "endAngle", + description="End of the angular range of the gradient. VarIndexBase + 3.", + ), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintGlyph + ( + "PaintFormat10", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 10" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintGlyph table) to Paint subtable.", + ), + FieldSpec( + "GlyphID", "Glyph", description="Glyph ID for the source outline." + ), + ], + ), + # PaintColrGlyph + ( + "PaintFormat11", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 11" + ), + FieldSpec( + "GlyphID", + "Glyph", + description="Virtual glyph ID for a BaseGlyphList base glyph.", + ), + ], + ), + # PaintTransform + ( + "PaintFormat12", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 12" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintTransform table) to Paint subtable.", + ), + FieldSpec( + "LOffset24To(Affine2x3)", + "Transform", + description="2x3 matrix for 2D affine transformations.", + ), + ], + ), + # PaintVarTransform + ( + "PaintFormat13", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 13" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintVarTransform table) to Paint subtable.", + ), + FieldSpec( + "LOffset24To(VarAffine2x3)", + "Transform", + description="2x3 matrix for 2D affine transformations.", + ), + ], + ), + # PaintTranslate + ( + "PaintFormat14", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 14" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintTranslate table) to Paint subtable.", + ), + FieldSpec("int16", "dx", description="Translation in x direction."), + FieldSpec("int16", "dy", description="Translation in y direction."), + ], + ), + # PaintVarTranslate + ( + "PaintFormat15", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 15" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintVarTranslate table) to Paint subtable.", + ), + FieldSpec( + "int16", + "dx", + description="Translation in x direction. VarIndexBase + 0.", + ), + FieldSpec( + "int16", + "dy", + description="Translation in y direction. VarIndexBase + 1.", + ), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintScale + ( + "PaintFormat16", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 16" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintScale table) to Paint subtable.", + ), + FieldSpec("F2Dot14", "scaleX"), + FieldSpec("F2Dot14", "scaleY"), + ], + ), + # PaintVarScale + ( + "PaintFormat17", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 17" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintVarScale table) to Paint subtable.", + ), + FieldSpec("F2Dot14", "scaleX", description="VarIndexBase + 0."), + FieldSpec("F2Dot14", "scaleY", description="VarIndexBase + 1."), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintScaleAroundCenter + ( + "PaintFormat18", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 18" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintScaleAroundCenter table) to Paint subtable.", + ), + FieldSpec("F2Dot14", "scaleX"), + FieldSpec("F2Dot14", "scaleY"), + FieldSpec("int16", "centerX"), + FieldSpec("int16", "centerY"), + ], + ), + # PaintVarScaleAroundCenter + ( + "PaintFormat19", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 19" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintVarScaleAroundCenter table) to Paint subtable.", + ), + FieldSpec("F2Dot14", "scaleX", description="VarIndexBase + 0."), + FieldSpec("F2Dot14", "scaleY", description="VarIndexBase + 1."), + FieldSpec("int16", "centerX", description="VarIndexBase + 2."), + FieldSpec("int16", "centerY", description="VarIndexBase + 3."), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintScaleUniform + ( + "PaintFormat20", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 20" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintScaleUniform table) to Paint subtable.", + ), + FieldSpec("F2Dot14", "scale"), + ], + ), + # PaintVarScaleUniform + ( + "PaintFormat21", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 21" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintVarScaleUniform table) to Paint subtable.", + ), + FieldSpec("F2Dot14", "scale", description="VarIndexBase + 0."), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintScaleUniformAroundCenter + ( + "PaintFormat22", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 22" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintScaleUniformAroundCenter table) to Paint subtable.", + ), + FieldSpec("F2Dot14", "scale"), + FieldSpec("int16", "centerX"), + FieldSpec("int16", "centerY"), + ], + ), + # PaintVarScaleUniformAroundCenter + ( + "PaintFormat23", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 23" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintVarScaleUniformAroundCenter table) to Paint subtable.", + ), + FieldSpec("F2Dot14", "scale", description="VarIndexBase + 0"), + FieldSpec("int16", "centerX", description="VarIndexBase + 1"), + FieldSpec("int16", "centerY", description="VarIndexBase + 2"), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintRotate + ( + "PaintFormat24", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 24" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintRotate table) to Paint subtable.", + ), + FieldSpec("Angle", "angle"), + ], + ), + # PaintVarRotate + ( + "PaintFormat25", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 25" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintVarRotate table) to Paint subtable.", + ), + FieldSpec("Angle", "angle", description="VarIndexBase + 0."), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintRotateAroundCenter + ( + "PaintFormat26", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 26" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintRotateAroundCenter table) to Paint subtable.", + ), + FieldSpec("Angle", "angle"), + FieldSpec("int16", "centerX"), + FieldSpec("int16", "centerY"), + ], + ), + # PaintVarRotateAroundCenter + ( + "PaintFormat27", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 27" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintVarRotateAroundCenter table) to Paint subtable.", + ), + FieldSpec("Angle", "angle", description="VarIndexBase + 0."), + FieldSpec("int16", "centerX", description="VarIndexBase + 1."), + FieldSpec("int16", "centerY", description="VarIndexBase + 2."), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintSkew + ( + "PaintFormat28", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 28" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintSkew table) to Paint subtable.", + ), + FieldSpec("Angle", "xSkewAngle"), + FieldSpec("Angle", "ySkewAngle"), + ], + ), + # PaintVarSkew + ( + "PaintFormat29", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 29" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintVarSkew table) to Paint subtable.", + ), + FieldSpec("Angle", "xSkewAngle", description="VarIndexBase + 0."), + FieldSpec("Angle", "ySkewAngle", description="VarIndexBase + 1."), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintSkewAroundCenter + ( + "PaintFormat30", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 30" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintSkewAroundCenter table) to Paint subtable.", + ), + FieldSpec("Angle", "xSkewAngle"), + FieldSpec("Angle", "ySkewAngle"), + FieldSpec("int16", "centerX"), + FieldSpec("int16", "centerY"), + ], + ), + # PaintVarSkewAroundCenter + ( + "PaintFormat31", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 31" + ), + FieldSpec( + "Offset24", + "Paint", + description="Offset (from beginning of PaintVarSkewAroundCenter table) to Paint subtable.", + ), + FieldSpec("Angle", "xSkewAngle", description="VarIndexBase + 0."), + FieldSpec("Angle", "ySkewAngle", description="VarIndexBase + 1."), + FieldSpec("int16", "centerX", description="VarIndexBase + 2."), + FieldSpec("int16", "centerY", description="VarIndexBase + 3."), + FieldSpec( + "VarIndex", + "VarIndexBase", + description="Base index into DeltaSetIndexMap.", + ), + ], + ), + # PaintComposite + ( + "PaintFormat32", + [ + FieldSpec( + "uint8", "PaintFormat", description="Format identifier-format = 32" + ), + FieldSpec( + "LOffset24To(Paint)", + "SourcePaint", + description="Offset (from beginning of PaintComposite table) to source Paint subtable.", + ), + FieldSpec( + "CompositeMode", + "CompositeMode", + description="A CompositeMode enumeration value.", + ), + FieldSpec( + "LOffset24To(Paint)", + "BackdropPaint", + description="Offset (from beginning of PaintComposite table) to backdrop Paint subtable.", + ), + ], + ), + # + # avar + # + ( + "AxisValueMap", + [ + FieldSpec( + "F2Dot14", + "FromCoordinate", + description="A normalized coordinate value obtained using default normalization", + ), + FieldSpec( + "F2Dot14", + "ToCoordinate", + description="The modified, normalized coordinate value", + ), + ], + ), + ( + "AxisSegmentMap", + [ + FieldSpec( + "uint16", + "PositionMapCount", + description="The number of correspondence pairs for this axis", + ), + FieldSpec( + "AxisValueMap", + "AxisValueMap", + repeat="PositionMapCount", + aux=0, + description="The array of axis value map records for this axis", + ), + ], + ), + ( + "avar", + [ + FieldSpec( + "Version", + "Version", + description="Version of the avar table- 0x00010000 or 0x00020000", + ), + FieldSpec( + "uint16", "Reserved", description="Permanently reserved; set to zero" + ), + FieldSpec( + "uint16", + "AxisCount", + description='The number of variation axes for this font. This must be the same number as axisCount in the "fvar" table', + ), + FieldSpec( + "AxisSegmentMap", + "AxisSegmentMap", + repeat="AxisCount", + aux=0, + description='The segment maps array — one segment map for each axis, in the order of axes specified in the "fvar" table', + ), + FieldSpec( + "LOffsetTo(DeltaSetIndexMap)", "VarIdxMap", aux="Version >= 0x00020000" + ), + FieldSpec("LOffset", "VarStore", aux="Version >= 0x00020000"), + ], + ), + # + # IFT - Incremental Font Transfer tables + # https://w3c.github.io/IFT/Overview.html + # Reference: https://github.com/googlefonts/fontations/blob/main/read-fonts/src/tables/ift.rs + # + ( + "PatchMapFormat2", + [ + FieldSpec( + "uint8", "Format", description="Set to 2, identifies this as format 2." + ), + FieldSpec("uint24", "Reserved", description="Not used, set to 0."), + FieldSpec( + "uint8", + "Flags", + description="Bit 0: CffCharStringsOffset present. Bit 1: Cff2CharStringsOffset present.", + ), + FieldSpec( + "uint32", + "CompatibilityId", + repeat=4, + aux=0, + description="Unique ID to identify compatible patches (16 bytes).", + ), + FieldSpec( + "uint8", + "DefaultPatchFormat", + description="Default format of patches linked to by urlTemplate.", + ), + FieldSpec( + "uint24", + "NumEntries", + description="Number of entries encoded in this table.", + ), + FieldSpec( + "LOffset", + "MappingEntries", + description="Offset to a MappingEntries sub-table.", + ), + FieldSpec( + "LOffset", + "EntryIdStringData", + description="Offset to entry ID string data block. May be null (0).", + ), + FieldSpec( + "uint16", + "UrlTemplateLength", + description="Length of the urlTemplate byte array.", + ), + FieldSpec( + "uint8", + "UrlTemplate", + repeat="UrlTemplateLength", + aux=0, + description="URL Template bytes used to produce URL strings for each entry.", + ), + FieldSpec( + "uint32", + "CffCharStringsOffset", + aux="Flags & 0x01", + description="Offset from start of CFF table to CharStrings INDEX.", + ), + FieldSpec( + "uint32", + "Cff2CharStringsOffset", + aux="Flags & 0x02", + description="Offset from start of CFF2 table to CharStrings INDEX.", + ), + ], + ), + # 'MappingEntries' contains stateful, delta-encoded entry IDs. The custom + # 'MappingEntriesConverter' is used to do a stateful parsing of all records. + ( + "MappingEntries", + [ + FieldSpec( + "MappingEntries", + "entries", + description="Array of MappingEntry records.", + ) + ], + ), + ("EntryIdStringData", []), +] diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otDataSchema.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otDataSchema.py new file mode 100644 index 0000000000000000000000000000000000000000..2ea3481f6027bf50a487d0deebbea59afc8fc7ef --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otDataSchema.py @@ -0,0 +1,37 @@ +"""fontTools.ttLib.tables.otDataSchema -- contains the FieldSpec dataclass, which is +used to define the schema of OpenType tables in otData.py. +""" + +from typing import Any, NamedTuple + + +class FieldSpec(NamedTuple): + """Represents a single field in an OpenType table schema. + + These are defined in `otData.py` and used to construct converters in + `otConverters.py`. + + Attributes: + type: The type of the field. This string is used by `otConverters.py` to + select or construct the appropriate converter. It can be a simple + type name (e.g. 'uint16', 'Tag', 'Offset'), a specialized type (e.g. + 'ValueRecord', 'DeltaValue'), or a more complex expression (e.g. + 'OffsetTo(AatStateTable)', 'AATLookup(UShort)') which is evaluated + to create a converter factory. + name: The name of the field as defined in the OpenType specification. + repeat: If the field is an array, this defines how many times it + repeats. It can be an integer, or the name of another field (usually + a 'Count' field) that contains the number of repetitions. If it is a + string, it is used to lookup the count value from the table. + aux: Auxiliary data used by some converters. When `repeat` is not set + and `aux` is a string, it is often compiled into a Python + expression for evaluation. For repeating structs, it is often used + as a constant or an expression. + description: A human-readable description of the field's purpose. + """ + + type: str + name: str + repeat: Any = None + aux: Any = None + description: str = "" diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otTables.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otTables.py new file mode 100644 index 0000000000000000000000000000000000000000..0e80e72c217aeae3f9df6f193af85bc2f52bf109 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otTables.py @@ -0,0 +1,2733 @@ +# coding: utf-8 +"""fontTools.ttLib.tables.otTables -- A collection of classes representing the various +OpenType subtables. + +Most are constructed upon import from data in otData.py, all are populated with +converter objects from otConverters.py. +""" + +import copy +from enum import IntEnum +from functools import reduce +from math import radians +import itertools +from collections import defaultdict, namedtuple +from fontTools.ttLib import OPTIMIZE_FONT_SPEED +from fontTools.ttLib.tables.TupleVariation import TupleVariation +from fontTools.ttLib.tables.otTraverse import dfs_base_table +from fontTools.misc.arrayTools import quantizeRect +from fontTools.misc.roundTools import otRound +from fontTools.misc.transform import Transform, Identity, DecomposedTransform +from fontTools.misc.textTools import bytesjoin, pad, safeEval +from fontTools.misc.vector import Vector +from fontTools.pens.boundsPen import ControlBoundsPen +from fontTools.pens.transformPen import TransformPen +from .otBase import ( + BaseTable, + FormatSwitchingBaseTable, + ValueRecord, + CountReference, + getFormatSwitchingBaseTableClass, +) +from fontTools.misc.fixedTools import ( + fixedToFloat as fi2fl, + floatToFixed as fl2fi, + floatToFixedToStr as fl2str, + strToFixedToFloat as str2fl, +) +from fontTools.feaLib.lookupDebugInfo import LookupDebugInfo, LOOKUP_DEBUG_INFO_KEY +import logging +import struct +import array +import sys +from enum import IntFlag +from typing import TYPE_CHECKING, Iterator, List, Optional, Set + +if TYPE_CHECKING: + from fontTools.ttLib.ttGlyphSet import _TTGlyphSet + + +log = logging.getLogger(__name__) + + +class VarComponentFlags(IntFlag): + RESET_UNSPECIFIED_AXES = 1 << 0 + + HAVE_AXES = 1 << 1 + + AXIS_VALUES_HAVE_VARIATION = 1 << 2 + TRANSFORM_HAS_VARIATION = 1 << 3 + + HAVE_TRANSLATE_X = 1 << 4 + HAVE_TRANSLATE_Y = 1 << 5 + HAVE_ROTATION = 1 << 6 + + HAVE_CONDITION = 1 << 7 + + HAVE_SCALE_X = 1 << 8 + HAVE_SCALE_Y = 1 << 9 + HAVE_TCENTER_X = 1 << 10 + HAVE_TCENTER_Y = 1 << 11 + + GID_IS_24BIT = 1 << 12 + + HAVE_SKEW_X = 1 << 13 + HAVE_SKEW_Y = 1 << 14 + + RESERVED_MASK = (1 << 32) - (1 << 15) + + +VarTransformMappingValues = namedtuple( + "VarTransformMappingValues", + ["flag", "fractionalBits", "scale", "defaultValue"], +) + +VAR_TRANSFORM_MAPPING = { + "translateX": VarTransformMappingValues( + VarComponentFlags.HAVE_TRANSLATE_X, 0, 1, 0 + ), + "translateY": VarTransformMappingValues( + VarComponentFlags.HAVE_TRANSLATE_Y, 0, 1, 0 + ), + "rotation": VarTransformMappingValues(VarComponentFlags.HAVE_ROTATION, 12, 180, 0), + "scaleX": VarTransformMappingValues(VarComponentFlags.HAVE_SCALE_X, 10, 1, 1), + "scaleY": VarTransformMappingValues(VarComponentFlags.HAVE_SCALE_Y, 10, 1, 1), + "skewX": VarTransformMappingValues(VarComponentFlags.HAVE_SKEW_X, 12, -180, 0), + "skewY": VarTransformMappingValues(VarComponentFlags.HAVE_SKEW_Y, 12, 180, 0), + "tCenterX": VarTransformMappingValues(VarComponentFlags.HAVE_TCENTER_X, 0, 1, 0), + "tCenterY": VarTransformMappingValues(VarComponentFlags.HAVE_TCENTER_Y, 0, 1, 0), +} + +# Probably should be somewhere in fontTools.misc +_packer = { + 1: lambda v: struct.pack(">B", v), + 2: lambda v: struct.pack(">H", v), + 3: lambda v: struct.pack(">L", v)[1:], + 4: lambda v: struct.pack(">L", v), +} +_unpacker = { + 1: lambda v: struct.unpack(">B", v)[0], + 2: lambda v: struct.unpack(">H", v)[0], + 3: lambda v: struct.unpack(">L", b"\0" + v)[0], + 4: lambda v: struct.unpack(">L", v)[0], +} + + +def _read_uint32var(data, i): + """Read a variable-length number from data starting at index i. + + Return the number and the next index. + """ + + b0 = data[i] + if b0 < 0x80: + return b0, i + 1 + elif b0 < 0xC0: + return (b0 - 0x80) << 8 | data[i + 1], i + 2 + elif b0 < 0xE0: + return (b0 - 0xC0) << 16 | data[i + 1] << 8 | data[i + 2], i + 3 + elif b0 < 0xF0: + return (b0 - 0xE0) << 24 | data[i + 1] << 16 | data[i + 2] << 8 | data[ + i + 3 + ], i + 4 + else: + return (b0 - 0xF0) << 32 | data[i + 1] << 24 | data[i + 2] << 16 | data[ + i + 3 + ] << 8 | data[i + 4], i + 5 + + +def _write_uint32var(v): + """Write a variable-length number. + + Return the data. + """ + if v < 0x80: + return struct.pack(">B", v) + elif v < 0x4000: + return struct.pack(">H", (v | 0x8000)) + elif v < 0x200000: + return struct.pack(">L", (v | 0xC00000))[1:] + elif v < 0x10000000: + return struct.pack(">L", (v | 0xE0000000)) + else: + return struct.pack(">B", 0xF0) + struct.pack(">L", v) + + +class VarComponent: + def __init__(self): + self.populateDefaults() + + def populateDefaults(self, propagator=None): + self.flags = 0 + self.glyphName = None + self.conditionIndex = None + self.axisIndicesIndex = None + self.axisValues = () + self.axisValuesVarIndex = NO_VARIATION_INDEX + self.transformVarIndex = NO_VARIATION_INDEX + self.transform = DecomposedTransform() + + def decompile(self, data, font, localState): + i = 0 + self.flags, i = _read_uint32var(data, i) + flags = self.flags + + gidSize = 3 if flags & VarComponentFlags.GID_IS_24BIT else 2 + glyphID = _unpacker[gidSize](data[i : i + gidSize]) + i += gidSize + self.glyphName = font.glyphOrder[glyphID] + + if flags & VarComponentFlags.HAVE_CONDITION: + self.conditionIndex, i = _read_uint32var(data, i) + + if flags & VarComponentFlags.HAVE_AXES: + self.axisIndicesIndex, i = _read_uint32var(data, i) + else: + self.axisIndicesIndex = None + + if self.axisIndicesIndex is None: + numAxes = 0 + else: + axisIndices = localState["AxisIndicesList"].Item[self.axisIndicesIndex] + numAxes = len(axisIndices) + + if flags & VarComponentFlags.HAVE_AXES: + axisValues, i = TupleVariation.decompileDeltas_(numAxes, data, i) + self.axisValues = tuple(fi2fl(v, 14) for v in axisValues) + else: + self.axisValues = () + assert len(self.axisValues) == numAxes + + if flags & VarComponentFlags.AXIS_VALUES_HAVE_VARIATION: + self.axisValuesVarIndex, i = _read_uint32var(data, i) + else: + self.axisValuesVarIndex = NO_VARIATION_INDEX + if flags & VarComponentFlags.TRANSFORM_HAS_VARIATION: + self.transformVarIndex, i = _read_uint32var(data, i) + else: + self.transformVarIndex = NO_VARIATION_INDEX + + self.transform = DecomposedTransform() + + def read_transform_component(values): + nonlocal i + if flags & values.flag: + v = ( + fi2fl( + struct.unpack(">h", data[i : i + 2])[0], values.fractionalBits + ) + * values.scale + ) + i += 2 + return v + else: + return values.defaultValue + + for attr_name, mapping_values in VAR_TRANSFORM_MAPPING.items(): + value = read_transform_component(mapping_values) + setattr(self.transform, attr_name, value) + + if not (flags & VarComponentFlags.HAVE_SCALE_Y): + self.transform.scaleY = self.transform.scaleX + + n = flags & VarComponentFlags.RESERVED_MASK + while n: + _, i = _read_uint32var(data, i) + n &= n - 1 + + return data[i:] + + def compile(self, font): + optimizeSpeed = font.cfg[OPTIMIZE_FONT_SPEED] + + data = [] + + flags = self.flags + + glyphID = font.getGlyphID(self.glyphName) + if glyphID > 65535: + flags |= VarComponentFlags.GID_IS_24BIT + data.append(_packer[3](glyphID)) + else: + flags &= ~VarComponentFlags.GID_IS_24BIT + data.append(_packer[2](glyphID)) + + if self.conditionIndex is not None: + flags |= VarComponentFlags.HAVE_CONDITION + data.append(_write_uint32var(self.conditionIndex)) + + numAxes = len(self.axisValues) + + if numAxes: + flags |= VarComponentFlags.HAVE_AXES + data.append(_write_uint32var(self.axisIndicesIndex)) + data.append( + TupleVariation.compileDeltaValues_( + [fl2fi(v, 14) for v in self.axisValues], + optimizeSize=not optimizeSpeed, + ) + ) + else: + flags &= ~VarComponentFlags.HAVE_AXES + + if self.axisValuesVarIndex != NO_VARIATION_INDEX: + flags |= VarComponentFlags.AXIS_VALUES_HAVE_VARIATION + data.append(_write_uint32var(self.axisValuesVarIndex)) + else: + flags &= ~VarComponentFlags.AXIS_VALUES_HAVE_VARIATION + if self.transformVarIndex != NO_VARIATION_INDEX: + flags |= VarComponentFlags.TRANSFORM_HAS_VARIATION + data.append(_write_uint32var(self.transformVarIndex)) + else: + flags &= ~VarComponentFlags.TRANSFORM_HAS_VARIATION + + def write_transform_component(value, values): + if flags & values.flag: + return struct.pack( + ">h", fl2fi(value / values.scale, values.fractionalBits) + ) + else: + return b"" + + for attr_name, mapping_values in VAR_TRANSFORM_MAPPING.items(): + value = getattr(self.transform, attr_name) + data.append(write_transform_component(value, mapping_values)) + + return _write_uint32var(flags) + bytesjoin(data) + + def toXML(self, writer, ttFont, attrs): + writer.begintag("VarComponent", attrs) + writer.newline() + + def write(name, value, attrs=()): + if value is not None: + writer.simpletag(name, (("value", value),) + attrs) + writer.newline() + + write("glyphName", self.glyphName) + + if self.conditionIndex is not None: + write("conditionIndex", self.conditionIndex) + if self.axisIndicesIndex is not None: + write("axisIndicesIndex", self.axisIndicesIndex) + if ( + self.axisIndicesIndex is not None + or self.flags & VarComponentFlags.RESET_UNSPECIFIED_AXES + ): + if self.flags & VarComponentFlags.RESET_UNSPECIFIED_AXES: + attrs = (("resetUnspecifiedAxes", 1),) + else: + attrs = () + write("axisValues", [float(fl2str(v, 14)) for v in self.axisValues], attrs) + + if self.axisValuesVarIndex != NO_VARIATION_INDEX: + write("axisValuesVarIndex", self.axisValuesVarIndex) + if self.transformVarIndex != NO_VARIATION_INDEX: + write("transformVarIndex", self.transformVarIndex) + + # Only write transform components that are specified in the + # flags, even if they are the default value. + for attr_name, mapping in VAR_TRANSFORM_MAPPING.items(): + if not (self.flags & mapping.flag): + continue + v = getattr(self.transform, attr_name) + write(attr_name, fl2str(v, mapping.fractionalBits)) + + writer.endtag("VarComponent") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont): + content = [c for c in content if isinstance(c, tuple)] + + self.populateDefaults() + + for name, attrs, content in content: + assert not content + v = attrs["value"] + + if name == "glyphName": + self.glyphName = v + elif name == "conditionIndex": + self.conditionIndex = safeEval(v) + elif name == "axisIndicesIndex": + self.axisIndicesIndex = safeEval(v) + elif name == "axisValues": + self.axisValues = tuple(str2fl(v, 14) for v in safeEval(v)) + if safeEval(attrs.get("resetUnspecifiedAxes", "0")): + self.flags |= VarComponentFlags.RESET_UNSPECIFIED_AXES + elif name == "axisValuesVarIndex": + self.axisValuesVarIndex = safeEval(v) + elif name == "transformVarIndex": + self.transformVarIndex = safeEval(v) + elif name in VAR_TRANSFORM_MAPPING: + setattr( + self.transform, + name, + safeEval(v), + ) + self.flags |= VAR_TRANSFORM_MAPPING[name].flag + else: + assert False, name + + def applyTransformDeltas(self, deltas): + i = 0 + + def read_transform_component_delta(values): + nonlocal i + if self.flags & values.flag: + v = fi2fl(deltas[i], values.fractionalBits) * values.scale + i += 1 + return v + else: + return 0 + + for attr_name, mapping_values in VAR_TRANSFORM_MAPPING.items(): + value = read_transform_component_delta(mapping_values) + setattr( + self.transform, attr_name, getattr(self.transform, attr_name) + value + ) + + if not (self.flags & VarComponentFlags.HAVE_SCALE_Y): + self.transform.scaleY = self.transform.scaleX + + assert i == len(deltas), (i, len(deltas)) + + def __eq__(self, other): + if type(self) != type(other): + return NotImplemented + return self.__dict__ == other.__dict__ + + def __ne__(self, other): + result = self.__eq__(other) + return result if result is NotImplemented else not result + + +class VarCompositeGlyph: + def __init__(self, components=None): + self.components = components if components is not None else [] + + def decompile(self, data, font, localState): + self.components = [] + while data: + component = VarComponent() + data = component.decompile(data, font, localState) + self.components.append(component) + + def compile(self, font): + data = [] + for component in self.components: + data.append(component.compile(font)) + return bytesjoin(data) + + def toXML(self, xmlWriter, font, attrs, name): + xmlWriter.begintag("VarCompositeGlyph", attrs) + xmlWriter.newline() + for i, component in enumerate(self.components): + component.toXML(xmlWriter, font, [("index", i)]) + xmlWriter.endtag("VarCompositeGlyph") + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + content = [c for c in content if isinstance(c, tuple)] + for name, attrs, content in content: + assert name == "VarComponent" + component = VarComponent() + component.fromXML(name, attrs, content, font) + self.components.append(component) + + +class AATStateTable(object): + def __init__(self): + self.GlyphClasses = {} # GlyphID --> GlyphClass + self.States = [] # List of AATState, indexed by state number + self.PerGlyphLookups = [] # [{GlyphID:GlyphID}, ...] + + +class AATState(object): + def __init__(self): + self.Transitions = {} # GlyphClass --> AATAction + + +class AATAction(object): + _FLAGS = None + + @staticmethod + def compileActions(font, states): + return (None, None) + + def _writeFlagsToXML(self, xmlWriter): + flags = [f for f in self._FLAGS if self.__dict__[f]] + if flags: + xmlWriter.simpletag("Flags", value=",".join(flags)) + xmlWriter.newline() + if self.ReservedFlags != 0: + xmlWriter.simpletag("ReservedFlags", value="0x%04X" % self.ReservedFlags) + xmlWriter.newline() + + def _setFlag(self, flag): + assert flag in self._FLAGS, "unsupported flag %s" % flag + self.__dict__[flag] = True + + +class RearrangementMorphAction(AATAction): + staticSize = 4 + actionHeaderSize = 0 + _FLAGS = ["MarkFirst", "DontAdvance", "MarkLast"] + + _VERBS = { + 0: "no change", + 1: "Ax ⇒ xA", + 2: "xD ⇒ Dx", + 3: "AxD ⇒ DxA", + 4: "ABx ⇒ xAB", + 5: "ABx ⇒ xBA", + 6: "xCD ⇒ CDx", + 7: "xCD ⇒ DCx", + 8: "AxCD ⇒ CDxA", + 9: "AxCD ⇒ DCxA", + 10: "ABxD ⇒ DxAB", + 11: "ABxD ⇒ DxBA", + 12: "ABxCD ⇒ CDxAB", + 13: "ABxCD ⇒ CDxBA", + 14: "ABxCD ⇒ DCxAB", + 15: "ABxCD ⇒ DCxBA", + } + + def __init__(self): + self.NewState = 0 + self.Verb = 0 + self.MarkFirst = False + self.DontAdvance = False + self.MarkLast = False + self.ReservedFlags = 0 + + def compile(self, writer, font, actionIndex): + assert actionIndex is None + writer.writeUShort(self.NewState) + assert self.Verb >= 0 and self.Verb <= 15, self.Verb + flags = self.Verb | self.ReservedFlags + if self.MarkFirst: + flags |= 0x8000 + if self.DontAdvance: + flags |= 0x4000 + if self.MarkLast: + flags |= 0x2000 + writer.writeUShort(flags) + + def decompile(self, reader, font, actionReader): + assert actionReader is None + self.NewState = reader.readUShort() + flags = reader.readUShort() + self.Verb = flags & 0xF + self.MarkFirst = bool(flags & 0x8000) + self.DontAdvance = bool(flags & 0x4000) + self.MarkLast = bool(flags & 0x2000) + self.ReservedFlags = flags & 0x1FF0 + + def toXML(self, xmlWriter, font, attrs, name): + xmlWriter.begintag(name, **attrs) + xmlWriter.newline() + xmlWriter.simpletag("NewState", value=self.NewState) + xmlWriter.newline() + self._writeFlagsToXML(xmlWriter) + xmlWriter.simpletag("Verb", value=self.Verb) + verbComment = self._VERBS.get(self.Verb) + if verbComment is not None: + xmlWriter.comment(verbComment) + xmlWriter.newline() + xmlWriter.endtag(name) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + self.NewState = self.Verb = self.ReservedFlags = 0 + self.MarkFirst = self.DontAdvance = self.MarkLast = False + content = [t for t in content if isinstance(t, tuple)] + for eltName, eltAttrs, eltContent in content: + if eltName == "NewState": + self.NewState = safeEval(eltAttrs["value"]) + elif eltName == "Verb": + self.Verb = safeEval(eltAttrs["value"]) + elif eltName == "ReservedFlags": + self.ReservedFlags = safeEval(eltAttrs["value"]) + elif eltName == "Flags": + for flag in eltAttrs["value"].split(","): + self._setFlag(flag.strip()) + + +class ContextualMorphAction(AATAction): + staticSize = 8 + actionHeaderSize = 0 + _FLAGS = ["SetMark", "DontAdvance"] + + def __init__(self): + self.NewState = 0 + self.SetMark, self.DontAdvance = False, False + self.ReservedFlags = 0 + self.MarkIndex, self.CurrentIndex = 0xFFFF, 0xFFFF + + def compile(self, writer, font, actionIndex): + assert actionIndex is None + writer.writeUShort(self.NewState) + flags = self.ReservedFlags + if self.SetMark: + flags |= 0x8000 + if self.DontAdvance: + flags |= 0x4000 + writer.writeUShort(flags) + writer.writeUShort(self.MarkIndex) + writer.writeUShort(self.CurrentIndex) + + def decompile(self, reader, font, actionReader): + assert actionReader is None + self.NewState = reader.readUShort() + flags = reader.readUShort() + self.SetMark = bool(flags & 0x8000) + self.DontAdvance = bool(flags & 0x4000) + self.ReservedFlags = flags & 0x3FFF + self.MarkIndex = reader.readUShort() + self.CurrentIndex = reader.readUShort() + + def toXML(self, xmlWriter, font, attrs, name): + xmlWriter.begintag(name, **attrs) + xmlWriter.newline() + xmlWriter.simpletag("NewState", value=self.NewState) + xmlWriter.newline() + self._writeFlagsToXML(xmlWriter) + xmlWriter.simpletag("MarkIndex", value=self.MarkIndex) + xmlWriter.newline() + xmlWriter.simpletag("CurrentIndex", value=self.CurrentIndex) + xmlWriter.newline() + xmlWriter.endtag(name) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + self.NewState = self.ReservedFlags = 0 + self.SetMark = self.DontAdvance = False + self.MarkIndex, self.CurrentIndex = 0xFFFF, 0xFFFF + content = [t for t in content if isinstance(t, tuple)] + for eltName, eltAttrs, eltContent in content: + if eltName == "NewState": + self.NewState = safeEval(eltAttrs["value"]) + elif eltName == "Flags": + for flag in eltAttrs["value"].split(","): + self._setFlag(flag.strip()) + elif eltName == "ReservedFlags": + self.ReservedFlags = safeEval(eltAttrs["value"]) + elif eltName == "MarkIndex": + self.MarkIndex = safeEval(eltAttrs["value"]) + elif eltName == "CurrentIndex": + self.CurrentIndex = safeEval(eltAttrs["value"]) + + +class LigAction(object): + def __init__(self): + self.Store = False + # GlyphIndexDelta is a (possibly negative) delta that gets + # added to the glyph ID at the top of the AAT runtime + # execution stack. It is *not* a byte offset into the + # morx table. The result of the addition, which is performed + # at run time by the shaping engine, is an index into + # the ligature components table. See 'morx' specification. + # In the AAT specification, this field is called Offset; + # but its meaning is quite different from other offsets + # in either AAT or OpenType, so we use a different name. + self.GlyphIndexDelta = 0 + + +class LigatureMorphAction(AATAction): + staticSize = 6 + + # 4 bytes for each of {action,ligComponents,ligatures}Offset + actionHeaderSize = 12 + + _FLAGS = ["SetComponent", "DontAdvance"] + + def __init__(self): + self.NewState = 0 + self.SetComponent, self.DontAdvance = False, False + self.ReservedFlags = 0 + self.Actions = [] + + def compile(self, writer, font, actionIndex): + assert actionIndex is not None + writer.writeUShort(self.NewState) + flags = self.ReservedFlags + if self.SetComponent: + flags |= 0x8000 + if self.DontAdvance: + flags |= 0x4000 + if len(self.Actions) > 0: + flags |= 0x2000 + writer.writeUShort(flags) + if len(self.Actions) > 0: + actions = self.compileLigActions() + writer.writeUShort(actionIndex[actions]) + else: + writer.writeUShort(0) + + def decompile(self, reader, font, actionReader): + assert actionReader is not None + self.NewState = reader.readUShort() + flags = reader.readUShort() + self.SetComponent = bool(flags & 0x8000) + self.DontAdvance = bool(flags & 0x4000) + performAction = bool(flags & 0x2000) + # As of 2017-09-12, the 'morx' specification says that + # the reserved bitmask in ligature subtables is 0x3FFF. + # However, the specification also defines a flag 0x2000, + # so the reserved value should actually be 0x1FFF. + # TODO: Report this specification bug to Apple. + self.ReservedFlags = flags & 0x1FFF + actionIndex = reader.readUShort() + if performAction: + self.Actions = self._decompileLigActions(actionReader, actionIndex) + else: + self.Actions = [] + + @staticmethod + def compileActions(font, states): + result, actions, actionIndex = b"", set(), {} + for state in states: + for _glyphClass, trans in state.Transitions.items(): + actions.add(trans.compileLigActions()) + # Sort the compiled actions in decreasing order of + # length, so that the longer sequence come before the + # shorter ones. For each compiled action ABCD, its + # suffixes BCD, CD, and D do not be encoded separately + # (in case they occur); instead, we can just store an + # index that points into the middle of the longer + # sequence. Every compiled AAT ligature sequence is + # terminated with an end-of-sequence flag, which can + # only be set on the last element of the sequence. + # Therefore, it is sufficient to consider just the + # suffixes. + for a in sorted(actions, key=lambda x: (-len(x), x)): + if a not in actionIndex: + for i in range(0, len(a), 4): + suffix = a[i:] + suffixIndex = (len(result) + i) // 4 + actionIndex.setdefault(suffix, suffixIndex) + result += a + result = pad(result, 4) + return (result, actionIndex) + + def compileLigActions(self): + result = [] + for i, action in enumerate(self.Actions): + last = i == len(self.Actions) - 1 + value = action.GlyphIndexDelta & 0x3FFFFFFF + value |= 0x80000000 if last else 0 + value |= 0x40000000 if action.Store else 0 + result.append(struct.pack(">L", value)) + return bytesjoin(result) + + def _decompileLigActions(self, actionReader, actionIndex): + actions = [] + last = False + reader = actionReader.getSubReader(actionReader.pos + actionIndex * 4) + while not last: + value = reader.readULong() + last = bool(value & 0x80000000) + action = LigAction() + actions.append(action) + action.Store = bool(value & 0x40000000) + delta = value & 0x3FFFFFFF + if delta >= 0x20000000: # sign-extend 30-bit value + delta = -0x40000000 + delta + action.GlyphIndexDelta = delta + return actions + + def fromXML(self, name, attrs, content, font): + self.NewState = self.ReservedFlags = 0 + self.SetComponent = self.DontAdvance = False + self.ReservedFlags = 0 + self.Actions = [] + content = [t for t in content if isinstance(t, tuple)] + for eltName, eltAttrs, eltContent in content: + if eltName == "NewState": + self.NewState = safeEval(eltAttrs["value"]) + elif eltName == "Flags": + for flag in eltAttrs["value"].split(","): + self._setFlag(flag.strip()) + elif eltName == "ReservedFlags": + self.ReservedFlags = safeEval(eltAttrs["value"]) + elif eltName == "Action": + action = LigAction() + flags = eltAttrs.get("Flags", "").split(",") + flags = [f.strip() for f in flags] + action.Store = "Store" in flags + action.GlyphIndexDelta = safeEval(eltAttrs["GlyphIndexDelta"]) + self.Actions.append(action) + + def toXML(self, xmlWriter, font, attrs, name): + xmlWriter.begintag(name, **attrs) + xmlWriter.newline() + xmlWriter.simpletag("NewState", value=self.NewState) + xmlWriter.newline() + self._writeFlagsToXML(xmlWriter) + for action in self.Actions: + attribs = [("GlyphIndexDelta", action.GlyphIndexDelta)] + if action.Store: + attribs.append(("Flags", "Store")) + xmlWriter.simpletag("Action", attribs) + xmlWriter.newline() + xmlWriter.endtag(name) + xmlWriter.newline() + + +class InsertionMorphAction(AATAction): + staticSize = 8 + actionHeaderSize = 4 # 4 bytes for actionOffset + _FLAGS = [ + "SetMark", + "DontAdvance", + "CurrentIsKashidaLike", + "MarkedIsKashidaLike", + "CurrentInsertBefore", + "MarkedInsertBefore", + ] + + def __init__(self): + self.NewState = 0 + for flag in self._FLAGS: + setattr(self, flag, False) + self.ReservedFlags = 0 + self.CurrentInsertionAction, self.MarkedInsertionAction = [], [] + + def compile(self, writer, font, actionIndex): + assert actionIndex is not None + writer.writeUShort(self.NewState) + flags = self.ReservedFlags + if self.SetMark: + flags |= 0x8000 + if self.DontAdvance: + flags |= 0x4000 + if self.CurrentIsKashidaLike: + flags |= 0x2000 + if self.MarkedIsKashidaLike: + flags |= 0x1000 + if self.CurrentInsertBefore: + flags |= 0x0800 + if self.MarkedInsertBefore: + flags |= 0x0400 + flags |= len(self.CurrentInsertionAction) << 5 + flags |= len(self.MarkedInsertionAction) + writer.writeUShort(flags) + if len(self.CurrentInsertionAction) > 0: + currentIndex = actionIndex[tuple(self.CurrentInsertionAction)] + else: + currentIndex = 0xFFFF + writer.writeUShort(currentIndex) + if len(self.MarkedInsertionAction) > 0: + markedIndex = actionIndex[tuple(self.MarkedInsertionAction)] + else: + markedIndex = 0xFFFF + writer.writeUShort(markedIndex) + + def decompile(self, reader, font, actionReader): + assert actionReader is not None + self.NewState = reader.readUShort() + flags = reader.readUShort() + self.SetMark = bool(flags & 0x8000) + self.DontAdvance = bool(flags & 0x4000) + self.CurrentIsKashidaLike = bool(flags & 0x2000) + self.MarkedIsKashidaLike = bool(flags & 0x1000) + self.CurrentInsertBefore = bool(flags & 0x0800) + self.MarkedInsertBefore = bool(flags & 0x0400) + self.CurrentInsertionAction = self._decompileInsertionAction( + actionReader, font, index=reader.readUShort(), count=((flags & 0x03E0) >> 5) + ) + self.MarkedInsertionAction = self._decompileInsertionAction( + actionReader, font, index=reader.readUShort(), count=(flags & 0x001F) + ) + + def _decompileInsertionAction(self, actionReader, font, index, count): + if index == 0xFFFF or count == 0: + return [] + reader = actionReader.getSubReader(actionReader.pos + index * 2) + return font.getGlyphNameMany(reader.readUShortArray(count)) + + def toXML(self, xmlWriter, font, attrs, name): + xmlWriter.begintag(name, **attrs) + xmlWriter.newline() + xmlWriter.simpletag("NewState", value=self.NewState) + xmlWriter.newline() + self._writeFlagsToXML(xmlWriter) + for g in self.CurrentInsertionAction: + xmlWriter.simpletag("CurrentInsertionAction", glyph=g) + xmlWriter.newline() + for g in self.MarkedInsertionAction: + xmlWriter.simpletag("MarkedInsertionAction", glyph=g) + xmlWriter.newline() + xmlWriter.endtag(name) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + self.__init__() + content = [t for t in content if isinstance(t, tuple)] + for eltName, eltAttrs, eltContent in content: + if eltName == "NewState": + self.NewState = safeEval(eltAttrs["value"]) + elif eltName == "Flags": + for flag in eltAttrs["value"].split(","): + self._setFlag(flag.strip()) + elif eltName == "CurrentInsertionAction": + self.CurrentInsertionAction.append(eltAttrs["glyph"]) + elif eltName == "MarkedInsertionAction": + self.MarkedInsertionAction.append(eltAttrs["glyph"]) + else: + assert False, eltName + + @staticmethod + def compileActions(font, states): + actions, actionIndex, result = set(), {}, b"" + for state in states: + for _glyphClass, trans in state.Transitions.items(): + if trans.CurrentInsertionAction is not None: + actions.add(tuple(trans.CurrentInsertionAction)) + if trans.MarkedInsertionAction is not None: + actions.add(tuple(trans.MarkedInsertionAction)) + # Sort the compiled actions in decreasing order of + # length, so that the longer sequence come before the + # shorter ones. + for action in sorted(actions, key=lambda x: (-len(x), x)): + # We insert all sub-sequences of the action glyph sequence + # into actionIndex. For example, if one action triggers on + # glyph sequence [A, B, C, D, E] and another action triggers + # on [C, D], we return result=[A, B, C, D, E] (as list of + # encoded glyph IDs), and actionIndex={('A','B','C','D','E'): 0, + # ('C','D'): 2}. + if action in actionIndex: + continue + for start in range(0, len(action)): + startIndex = (len(result) // 2) + start + for limit in range(start, len(action)): + glyphs = action[start : limit + 1] + actionIndex.setdefault(glyphs, startIndex) + for glyph in action: + glyphID = font.getGlyphID(glyph) + result += struct.pack(">H", glyphID) + return result, actionIndex + + +class FeatureParams(BaseTable): + def compile(self, writer, font): + assert ( + featureParamTypes.get(writer["FeatureTag"]) == self.__class__ + ), "Wrong FeatureParams type for feature '%s': %s" % ( + writer["FeatureTag"], + self.__class__.__name__, + ) + BaseTable.compile(self, writer, font) + + def toXML(self, xmlWriter, font, attrs=None, name=None): + BaseTable.toXML(self, xmlWriter, font, attrs, name=self.__class__.__name__) + + +class FeatureParamsSize(FeatureParams): + pass + + +class FeatureParamsStylisticSet(FeatureParams): + pass + + +class FeatureParamsCharacterVariants(FeatureParams): + pass + + +class Coverage(FormatSwitchingBaseTable): + # manual implementation to get rid of glyphID dependencies + + def populateDefaults(self, propagator=None): + if not hasattr(self, "glyphs"): + self.glyphs = [] + + def postRead(self, rawTable, font): + if self.Format == 1: + self.glyphs = rawTable["GlyphArray"] + elif self.Format == 2: + glyphs = self.glyphs = [] + ranges = rawTable["RangeRecord"] + # Some SIL fonts have coverage entries that don't have sorted + # StartCoverageIndex. If it is so, fixup and warn. We undo + # this when writing font out. + sorted_ranges = sorted(ranges, key=lambda a: a.StartCoverageIndex) + if ranges != sorted_ranges: + log.warning("GSUB/GPOS Coverage is not sorted by glyph ids.") + ranges = sorted_ranges + del sorted_ranges + for r in ranges: + start = r.Start + end = r.End + startID = font.getGlyphID(start) + endID = font.getGlyphID(end) + 1 + glyphs.extend(font.getGlyphNameMany(range(startID, endID))) + else: + self.glyphs = [] + log.warning("Unknown Coverage format: %s", self.Format) + del self.Format # Don't need this anymore + + def preWrite(self, font): + glyphs = getattr(self, "glyphs", None) + if glyphs is None: + glyphs = self.glyphs = [] + format = 1 + rawTable = {"GlyphArray": glyphs} + if glyphs: + # find out whether Format 2 is more compact or not + glyphIDs = font.getGlyphIDMany(glyphs) + brokenOrder = sorted(glyphIDs) != glyphIDs + + last = glyphIDs[0] + ranges = [[last]] + for glyphID in glyphIDs[1:]: + if glyphID != last + 1: + ranges[-1].append(last) + ranges.append([glyphID]) + last = glyphID + ranges[-1].append(last) + + if brokenOrder or len(ranges) * 3 < len(glyphs): # 3 words vs. 1 word + # Format 2 is more compact + index = 0 + for i, (start, end) in enumerate(ranges): + r = RangeRecord() + r.StartID = start + r.Start = font.getGlyphName(start) + r.End = font.getGlyphName(end) + r.StartCoverageIndex = index + ranges[i] = r + index = index + end - start + 1 + if brokenOrder: + log.warning("GSUB/GPOS Coverage is not sorted by glyph ids.") + ranges.sort(key=lambda a: a.StartID) + for r in ranges: + del r.StartID + format = 2 + rawTable = {"RangeRecord": ranges} + # else: + # fallthrough; Format 1 is more compact + self.Format = format + return rawTable + + def toXML2(self, xmlWriter, font): + for glyphName in getattr(self, "glyphs", []): + xmlWriter.simpletag("Glyph", value=glyphName) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + glyphs = getattr(self, "glyphs", None) + if glyphs is None: + glyphs = [] + self.glyphs = glyphs + glyphs.append(attrs["value"]) + + +# The special 0xFFFFFFFF delta-set index is used to indicate that there +# is no variation data in the ItemVariationStore for a given variable field +NO_VARIATION_INDEX = 0xFFFFFFFF + + +class DeltaSetIndexMap(getFormatSwitchingBaseTableClass("uint8")): + def populateDefaults(self, propagator=None): + if not hasattr(self, "mapping"): + self.mapping = [] + + def postRead(self, rawTable, font): + assert (rawTable["EntryFormat"] & 0xFFC0) == 0 + self.mapping = rawTable["mapping"] + + @staticmethod + def getEntryFormat(mapping): + ored = 0 + for idx in mapping: + ored |= idx + + inner = ored & 0xFFFF + innerBits = 0 + while inner: + innerBits += 1 + inner >>= 1 + innerBits = max(innerBits, 1) + assert innerBits <= 16 + + ored = (ored >> (16 - innerBits)) | (ored & ((1 << innerBits) - 1)) + if ored <= 0x000000FF: + entrySize = 1 + elif ored <= 0x0000FFFF: + entrySize = 2 + elif ored <= 0x00FFFFFF: + entrySize = 3 + else: + entrySize = 4 + + return ((entrySize - 1) << 4) | (innerBits - 1) + + def preWrite(self, font): + mapping = getattr(self, "mapping", None) + if mapping is None: + mapping = self.mapping = [] + self.Format = 1 if len(mapping) > 0xFFFF else 0 + rawTable = self.__dict__.copy() + rawTable["MappingCount"] = len(mapping) + rawTable["EntryFormat"] = self.getEntryFormat(mapping) + return rawTable + + def toXML2(self, xmlWriter, font): + # Make xml dump less verbose, by omitting no-op entries like: + # <Map index="..." outer="65535" inner="65535"/> + xmlWriter.comment("Omitted values default to 0xFFFF/0xFFFF (no variations)") + xmlWriter.newline() + for i, value in enumerate(getattr(self, "mapping", [])): + attrs = [("index", i)] + if value != NO_VARIATION_INDEX: + attrs.extend( + [ + ("outer", value >> 16), + ("inner", value & 0xFFFF), + ] + ) + xmlWriter.simpletag("Map", attrs) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + mapping = getattr(self, "mapping", None) + if mapping is None: + self.mapping = mapping = [] + index = safeEval(attrs["index"]) + outer = safeEval(attrs.get("outer", "0xFFFF")) + inner = safeEval(attrs.get("inner", "0xFFFF")) + assert inner <= 0xFFFF + mapping.insert(index, (outer << 16) | inner) + + def __getitem__(self, i): + return self.mapping[i] if i < len(self.mapping) else NO_VARIATION_INDEX + + +class VarIdxMap(BaseTable): + def populateDefaults(self, propagator=None): + if not hasattr(self, "mapping"): + self.mapping = {} + + def postRead(self, rawTable, font): + assert (rawTable["EntryFormat"] & 0xFFC0) == 0 + glyphOrder = font.getGlyphOrder() + mapList = rawTable["mapping"] + mapList.extend([mapList[-1]] * (len(glyphOrder) - len(mapList))) + self.mapping = dict(zip(glyphOrder, mapList)) + + def preWrite(self, font): + mapping = getattr(self, "mapping", None) + if mapping is None: + mapping = self.mapping = {} + + glyphOrder = font.getGlyphOrder() + mapping = [mapping[g] for g in glyphOrder] + while len(mapping) > 1 and mapping[-2] == mapping[-1]: + del mapping[-1] + + rawTable = {"mapping": mapping} + rawTable["MappingCount"] = len(mapping) + rawTable["EntryFormat"] = DeltaSetIndexMap.getEntryFormat(mapping) + return rawTable + + def toXML2(self, xmlWriter, font): + for glyph, value in sorted(getattr(self, "mapping", {}).items()): + attrs = ( + ("glyph", glyph), + ("outer", value >> 16), + ("inner", value & 0xFFFF), + ) + xmlWriter.simpletag("Map", attrs) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + mapping = getattr(self, "mapping", None) + if mapping is None: + mapping = {} + self.mapping = mapping + try: + glyph = attrs["glyph"] + except: # https://github.com/fonttools/fonttools/commit/21cbab8ce9ded3356fef3745122da64dcaf314e9#commitcomment-27649836 + glyph = font.getGlyphOrder()[attrs["index"]] + outer = safeEval(attrs["outer"]) + inner = safeEval(attrs["inner"]) + assert inner <= 0xFFFF + mapping[glyph] = (outer << 16) | inner + + def __getitem__(self, glyphName): + return self.mapping.get(glyphName, NO_VARIATION_INDEX) + + +class VarRegionList(BaseTable): + def preWrite(self, font): + # The OT spec says VarStore.VarRegionList.RegionAxisCount should always + # be equal to the fvar.axisCount, and OTS < v8.0.0 enforces this rule + # even when the VarRegionList is empty. We can't treat RegionAxisCount + # like a normal propagated count (== len(Region[i].VarRegionAxis)), + # otherwise it would default to 0 if VarRegionList is empty. + # Thus, we force it to always be equal to fvar.axisCount. + # https://github.com/khaledhosny/ots/pull/192 + fvarTable = font.get("fvar") + if fvarTable: + self.RegionAxisCount = len(fvarTable.axes) + return { + **self.__dict__, + "RegionAxisCount": CountReference(self.__dict__, "RegionAxisCount"), + } + + +class SingleSubst(FormatSwitchingBaseTable): + def populateDefaults(self, propagator=None): + if not hasattr(self, "mapping"): + self.mapping = {} + + def postRead(self, rawTable, font): + mapping = {} + input = _getGlyphsFromCoverageTable(rawTable["Coverage"]) + if self.Format == 1: + delta = rawTable["DeltaGlyphID"] + inputGIDS = font.getGlyphIDMany(input) + outGIDS = [(glyphID + delta) % 65536 for glyphID in inputGIDS] + outNames = font.getGlyphNameMany(outGIDS) + for inp, out in zip(input, outNames): + mapping[inp] = out + elif self.Format == 2: + assert ( + len(input) == rawTable["GlyphCount"] + ), "invalid SingleSubstFormat2 table" + subst = rawTable["Substitute"] + for inp, sub in zip(input, subst): + mapping[inp] = sub + else: + assert 0, "unknown format: %s" % self.Format + self.mapping = mapping + del self.Format # Don't need this anymore + + def preWrite(self, font): + mapping = getattr(self, "mapping", None) + if mapping is None: + mapping = self.mapping = {} + items = list(mapping.items()) + getGlyphID = font.getGlyphID + gidItems = [(getGlyphID(a), getGlyphID(b)) for a, b in items] + sortableItems = sorted(zip(gidItems, items)) + + # figure out format + format = 2 + delta = None + for inID, outID in gidItems: + if delta is None: + delta = (outID - inID) % 65536 + + if (inID + delta) % 65536 != outID: + break + else: + if delta is None: + # the mapping is empty, better use format 2 + format = 2 + else: + format = 1 + + rawTable = {} + self.Format = format + cov = Coverage() + input = [item[1][0] for item in sortableItems] + subst = [item[1][1] for item in sortableItems] + cov.glyphs = input + rawTable["Coverage"] = cov + if format == 1: + assert delta is not None + rawTable["DeltaGlyphID"] = delta + else: + rawTable["Substitute"] = subst + return rawTable + + def toXML2(self, xmlWriter, font): + items = sorted(self.mapping.items()) + for inGlyph, outGlyph in items: + xmlWriter.simpletag("Substitution", [("in", inGlyph), ("out", outGlyph)]) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + mapping = getattr(self, "mapping", None) + if mapping is None: + mapping = {} + self.mapping = mapping + mapping[attrs["in"]] = attrs["out"] + + +class MultipleSubst(FormatSwitchingBaseTable): + def populateDefaults(self, propagator=None): + if not hasattr(self, "mapping"): + self.mapping = {} + + def postRead(self, rawTable, font): + mapping = {} + if self.Format == 1: + glyphs = _getGlyphsFromCoverageTable(rawTable["Coverage"]) + subst = [s.Substitute for s in rawTable["Sequence"]] + mapping = dict(zip(glyphs, subst)) + else: + assert 0, "unknown format: %s" % self.Format + self.mapping = mapping + del self.Format # Don't need this anymore + + def preWrite(self, font): + mapping = getattr(self, "mapping", None) + if mapping is None: + mapping = self.mapping = {} + cov = Coverage() + cov.glyphs = sorted(list(mapping.keys()), key=font.getGlyphID) + self.Format = 1 + rawTable = { + "Coverage": cov, + "Sequence": [self.makeSequence_(mapping[glyph]) for glyph in cov.glyphs], + } + return rawTable + + def toXML2(self, xmlWriter, font): + items = sorted(self.mapping.items()) + for inGlyph, outGlyphs in items: + out = ",".join(outGlyphs) + xmlWriter.simpletag("Substitution", [("in", inGlyph), ("out", out)]) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + mapping = getattr(self, "mapping", None) + if mapping is None: + mapping = {} + self.mapping = mapping + + # TTX v3.0 and earlier. + if name == "Coverage": + self.old_coverage_ = [] + for element in content: + if not isinstance(element, tuple): + continue + element_name, element_attrs, _ = element + if element_name == "Glyph": + self.old_coverage_.append(element_attrs["value"]) + return + if name == "Sequence": + index = int(attrs.get("index", len(mapping))) + glyph = self.old_coverage_[index] + glyph_mapping = mapping[glyph] = [] + for element in content: + if not isinstance(element, tuple): + continue + element_name, element_attrs, _ = element + if element_name == "Substitute": + glyph_mapping.append(element_attrs["value"]) + return + + # TTX v3.1 and later. + outGlyphs = attrs["out"].split(",") if attrs["out"] else [] + mapping[attrs["in"]] = [g.strip() for g in outGlyphs] + + @staticmethod + def makeSequence_(g): + seq = Sequence() + seq.Substitute = g + return seq + + +class ClassDef(FormatSwitchingBaseTable): + def populateDefaults(self, propagator=None): + if not hasattr(self, "classDefs"): + self.classDefs = {} + + def postRead(self, rawTable, font): + classDefs = {} + + if self.Format == 1: + start = rawTable["StartGlyph"] + classList = rawTable["ClassValueArray"] + startID = font.getGlyphID(start) + endID = startID + len(classList) + glyphNames = font.getGlyphNameMany(range(startID, endID)) + for glyphName, cls in zip(glyphNames, classList): + if cls: + classDefs[glyphName] = cls + + elif self.Format == 2: + records = rawTable["ClassRangeRecord"] + for rec in records: + cls = rec.Class + if not cls: + continue + start = rec.Start + end = rec.End + startID = font.getGlyphID(start) + endID = font.getGlyphID(end) + 1 + glyphNames = font.getGlyphNameMany(range(startID, endID)) + for glyphName in glyphNames: + classDefs[glyphName] = cls + else: + log.warning("Unknown ClassDef format: %s", self.Format) + self.classDefs = classDefs + del self.Format # Don't need this anymore + + def _getClassRanges(self, font): + classDefs = getattr(self, "classDefs", None) + if classDefs is None: + self.classDefs = {} + return + getGlyphID = font.getGlyphID + items = [] + for glyphName, cls in classDefs.items(): + if not cls: + continue + items.append((getGlyphID(glyphName), glyphName, cls)) + if items: + items.sort() + last, lastName, lastCls = items[0] + ranges = [[lastCls, last, lastName]] + for glyphID, glyphName, cls in items[1:]: + if glyphID != last + 1 or cls != lastCls: + ranges[-1].extend([last, lastName]) + ranges.append([cls, glyphID, glyphName]) + last = glyphID + lastName = glyphName + lastCls = cls + ranges[-1].extend([last, lastName]) + return ranges + + def preWrite(self, font): + format = 2 + rawTable = {"ClassRangeRecord": []} + ranges = self._getClassRanges(font) + if ranges: + startGlyph = ranges[0][1] + endGlyph = ranges[-1][3] + glyphCount = endGlyph - startGlyph + 1 + if len(ranges) * 3 < glyphCount + 1: + # Format 2 is more compact + for i, (cls, start, startName, end, endName) in enumerate(ranges): + rec = ClassRangeRecord() + rec.Start = startName + rec.End = endName + rec.Class = cls + ranges[i] = rec + format = 2 + rawTable = {"ClassRangeRecord": ranges} + else: + # Format 1 is more compact + startGlyphName = ranges[0][2] + classes = [0] * glyphCount + for cls, start, startName, end, endName in ranges: + for g in range(start - startGlyph, end - startGlyph + 1): + classes[g] = cls + format = 1 + rawTable = {"StartGlyph": startGlyphName, "ClassValueArray": classes} + self.Format = format + return rawTable + + def toXML2(self, xmlWriter, font): + items = sorted(self.classDefs.items()) + for glyphName, cls in items: + xmlWriter.simpletag("ClassDef", [("glyph", glyphName), ("class", cls)]) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + classDefs = getattr(self, "classDefs", None) + if classDefs is None: + classDefs = {} + self.classDefs = classDefs + classDefs[attrs["glyph"]] = int(attrs["class"]) + + +class AlternateSubst(FormatSwitchingBaseTable): + def populateDefaults(self, propagator=None): + if not hasattr(self, "alternates"): + self.alternates = {} + + def postRead(self, rawTable, font): + alternates = {} + if self.Format == 1: + input = _getGlyphsFromCoverageTable(rawTable["Coverage"]) + alts = rawTable["AlternateSet"] + assert len(input) == len(alts) + for inp, alt in zip(input, alts): + alternates[inp] = alt.Alternate + else: + assert 0, "unknown format: %s" % self.Format + self.alternates = alternates + del self.Format # Don't need this anymore + + def preWrite(self, font): + self.Format = 1 + alternates = getattr(self, "alternates", None) + if alternates is None: + alternates = self.alternates = {} + items = list(alternates.items()) + for i, (glyphName, set) in enumerate(items): + items[i] = font.getGlyphID(glyphName), glyphName, set + items.sort() + cov = Coverage() + cov.glyphs = [item[1] for item in items] + alternates = [] + setList = [item[-1] for item in items] + for set in setList: + alts = AlternateSet() + alts.Alternate = set + alternates.append(alts) + # a special case to deal with the fact that several hundred Adobe Japan1-5 + # CJK fonts will overflow an offset if the coverage table isn't pushed to the end. + # Also useful in that when splitting a sub-table because of an offset overflow + # I don't need to calculate the change in the subtable offset due to the change in the coverage table size. + # Allows packing more rules in subtable. + self.sortCoverageLast = 1 + return {"Coverage": cov, "AlternateSet": alternates} + + def toXML2(self, xmlWriter, font): + items = sorted(self.alternates.items()) + for glyphName, alternates in items: + xmlWriter.begintag("AlternateSet", glyph=glyphName) + xmlWriter.newline() + for alt in alternates: + xmlWriter.simpletag("Alternate", glyph=alt) + xmlWriter.newline() + xmlWriter.endtag("AlternateSet") + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + alternates = getattr(self, "alternates", None) + if alternates is None: + alternates = {} + self.alternates = alternates + glyphName = attrs["glyph"] + set = [] + alternates[glyphName] = set + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + set.append(attrs["glyph"]) + + +class LigatureSubst(FormatSwitchingBaseTable): + def populateDefaults(self, propagator=None): + if not hasattr(self, "ligatures"): + self.ligatures = {} + + def postRead(self, rawTable, font): + ligatures = {} + if self.Format == 1: + input = _getGlyphsFromCoverageTable(rawTable["Coverage"]) + ligSets = rawTable["LigatureSet"] + assert len(input) == len(ligSets) + for i, inp in enumerate(input): + ligatures[inp] = ligSets[i].Ligature + else: + assert 0, "unknown format: %s" % self.Format + self.ligatures = ligatures + del self.Format # Don't need this anymore + + @staticmethod + def _getLigatureSortKey(components): + # Computes a key for ordering ligatures in a GSUB Type-4 lookup. + + # When building the OpenType lookup, we need to make sure that + # the longest sequence of components is listed first, so we + # use the negative length as the key for sorting. + # Note, we no longer need to worry about deterministic order because the + # ligature mapping `dict` remembers the insertion order, and this in + # turn depends on the order in which the ligatures are written in the FEA. + # Since python sort algorithm is stable, the ligatures of equal length + # will keep the relative order in which they appear in the feature file. + # For example, given the following ligatures (all starting with 'f' and + # thus belonging to the same LigatureSet): + # + # feature liga { + # sub f i by f_i; + # sub f f f by f_f_f; + # sub f f by f_f; + # sub f f i by f_f_i; + # } liga; + # + # this should sort to: f_f_f, f_f_i, f_i, f_f + # This is also what fea-rs does, see: + # https://github.com/adobe-type-tools/afdko/issues/1727 + # https://github.com/fonttools/fonttools/issues/3428 + # https://github.com/googlefonts/fontc/pull/680 + return -len(components) + + def preWrite(self, font): + self.Format = 1 + ligatures = getattr(self, "ligatures", None) + if ligatures is None: + ligatures = self.ligatures = {} + + if ligatures and isinstance(next(iter(ligatures)), tuple): + # New high-level API in v3.1 and later. Note that we just support compiling this + # for now. We don't load to this API, and don't do XML with it. + + # ligatures is map from components-sequence to lig-glyph + newLigatures = dict() + for comps in sorted(ligatures.keys(), key=self._getLigatureSortKey): + ligature = Ligature() + ligature.Component = comps[1:] + ligature.CompCount = len(comps) + ligature.LigGlyph = ligatures[comps] + newLigatures.setdefault(comps[0], []).append(ligature) + ligatures = newLigatures + + items = list(ligatures.items()) + for i, (glyphName, set) in enumerate(items): + items[i] = font.getGlyphID(glyphName), glyphName, set + items.sort() + cov = Coverage() + cov.glyphs = [item[1] for item in items] + + ligSets = [] + setList = [item[-1] for item in items] + for set in setList: + ligSet = LigatureSet() + ligs = ligSet.Ligature = [] + for lig in set: + ligs.append(lig) + ligSets.append(ligSet) + # Useful in that when splitting a sub-table because of an offset overflow + # I don't need to calculate the change in subtabl offset due to the coverage table size. + # Allows packing more rules in subtable. + self.sortCoverageLast = 1 + return {"Coverage": cov, "LigatureSet": ligSets} + + def toXML2(self, xmlWriter, font): + items = sorted(self.ligatures.items()) + for glyphName, ligSets in items: + xmlWriter.begintag("LigatureSet", glyph=glyphName) + xmlWriter.newline() + for lig in ligSets: + xmlWriter.simpletag( + "Ligature", glyph=lig.LigGlyph, components=",".join(lig.Component) + ) + xmlWriter.newline() + xmlWriter.endtag("LigatureSet") + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + ligatures = getattr(self, "ligatures", None) + if ligatures is None: + ligatures = {} + self.ligatures = ligatures + glyphName = attrs["glyph"] + ligs = [] + ligatures[glyphName] = ligs + for element in content: + if not isinstance(element, tuple): + continue + name, attrs, content = element + lig = Ligature() + lig.LigGlyph = attrs["glyph"] + components = attrs["components"] + lig.Component = components.split(",") if components else [] + lig.CompCount = len(lig.Component) + ligs.append(lig) + + +class COLR(BaseTable): + def decompile(self, reader, font): + # COLRv0 is exceptional in that LayerRecordCount appears *after* the + # LayerRecordArray it counts, but the parser logic expects Count fields + # to always precede the arrays. Here we work around this by parsing the + # LayerRecordCount before the rest of the table, and storing it in + # the reader's local state. + subReader = reader.getSubReader(offset=0) + for conv in self.getConverters(): + if conv.name != "LayerRecordCount": + subReader.advance(conv.staticSize) + continue + reader[conv.name] = conv.read(subReader, font, tableDict={}) + break + else: + raise AssertionError("LayerRecordCount converter not found") + return BaseTable.decompile(self, reader, font) + + def preWrite(self, font): + # The writer similarly assumes Count values precede the things counted, + # thus here we pre-initialize a CountReference; the actual count value + # will be set to the lenght of the array by the time this is assembled. + self.LayerRecordCount = None + return { + **self.__dict__, + "LayerRecordCount": CountReference(self.__dict__, "LayerRecordCount"), + } + + def computeClipBoxes(self, glyphSet: "_TTGlyphSet", quantization: int = 1): + if self.Version == 0: + return + + clips = {} + for rec in self.BaseGlyphList.BaseGlyphPaintRecord: + try: + clipBox = rec.Paint.computeClipBox(self, glyphSet, quantization) + except Exception as e: + from fontTools.ttLib import TTLibError + + raise TTLibError( + f"Failed to compute COLR ClipBox for {rec.BaseGlyph!r}" + ) from e + + if clipBox is not None: + clips[rec.BaseGlyph] = clipBox + + hasClipList = hasattr(self, "ClipList") and self.ClipList is not None + if not clips: + if hasClipList: + self.ClipList = None + else: + if not hasClipList: + self.ClipList = ClipList() + self.ClipList.Format = 1 + self.ClipList.clips = clips + + +class LookupList(BaseTable): + @property + def table(self): + for l in self.Lookup: + for st in l.SubTable: + if type(st).__name__.endswith("Subst"): + return "GSUB" + if type(st).__name__.endswith("Pos"): + return "GPOS" + raise ValueError + + def toXML2(self, xmlWriter, font): + if ( + not font + or "Debg" not in font + or LOOKUP_DEBUG_INFO_KEY not in font["Debg"].data + ): + return super().toXML2(xmlWriter, font) + debugData = font["Debg"].data[LOOKUP_DEBUG_INFO_KEY][self.table] + for conv in self.getConverters(): + if conv.repeat: + value = getattr(self, conv.name, []) + for lookupIndex, item in enumerate(value): + if str(lookupIndex) in debugData: + info = LookupDebugInfo(*debugData[str(lookupIndex)]) + tag = info.location + if info.name: + tag = f"{info.name}: {tag}" + if info.feature: + script, language, feature = info.feature + tag = f"{tag} in {feature} ({script}/{language})" + xmlWriter.comment(tag) + xmlWriter.newline() + + conv.xmlWrite( + xmlWriter, font, item, conv.name, [("index", lookupIndex)] + ) + else: + if conv.aux and not eval(conv.aux, None, vars(self)): + continue + value = getattr( + self, conv.name, None + ) # TODO Handle defaults instead of defaulting to None! + conv.xmlWrite(xmlWriter, font, value, conv.name, []) + + +class BaseGlyphRecordArray(BaseTable): + def preWrite(self, font): + self.BaseGlyphRecord = sorted( + self.BaseGlyphRecord, key=lambda rec: font.getGlyphID(rec.BaseGlyph) + ) + return self.__dict__.copy() + + +class BaseGlyphList(BaseTable): + def preWrite(self, font): + self.BaseGlyphPaintRecord = sorted( + self.BaseGlyphPaintRecord, key=lambda rec: font.getGlyphID(rec.BaseGlyph) + ) + return self.__dict__.copy() + + +class ClipBoxFormat(IntEnum): + Static = 1 + Variable = 2 + + def is_variable(self): + return self is self.Variable + + def as_variable(self): + return self.Variable + + +class ClipBox(getFormatSwitchingBaseTableClass("uint8")): + formatEnum = ClipBoxFormat + + def as_tuple(self): + return tuple(getattr(self, conv.name) for conv in self.getConverters()) + + def __repr__(self): + return f"{self.__class__.__name__}{self.as_tuple()}" + + +class ClipList(getFormatSwitchingBaseTableClass("uint8")): + def populateDefaults(self, propagator=None): + if not hasattr(self, "clips"): + self.clips = {} + + def postRead(self, rawTable, font): + clips = {} + glyphOrder = font.getGlyphOrder() + for i, rec in enumerate(rawTable["ClipRecord"]): + if rec.StartGlyphID > rec.EndGlyphID: + log.warning( + "invalid ClipRecord[%i].StartGlyphID (%i) > " + "EndGlyphID (%i); skipped", + i, + rec.StartGlyphID, + rec.EndGlyphID, + ) + continue + redefinedGlyphs = [] + missingGlyphs = [] + for glyphID in range(rec.StartGlyphID, rec.EndGlyphID + 1): + try: + glyph = glyphOrder[glyphID] + except IndexError: + missingGlyphs.append(glyphID) + continue + if glyph not in clips: + clips[glyph] = copy.copy(rec.ClipBox) + else: + redefinedGlyphs.append(glyphID) + if redefinedGlyphs: + log.warning( + "ClipRecord[%i] overlaps previous records; " + "ignoring redefined clip boxes for the " + "following glyph ID range: [%i-%i]", + i, + min(redefinedGlyphs), + max(redefinedGlyphs), + ) + if missingGlyphs: + log.warning( + "ClipRecord[%i] range references missing " "glyph IDs: [%i-%i]", + i, + min(missingGlyphs), + max(missingGlyphs), + ) + self.clips = clips + + def groups(self): + glyphsByClip = defaultdict(list) + uniqueClips = {} + for glyphName, clipBox in self.clips.items(): + key = clipBox.as_tuple() + glyphsByClip[key].append(glyphName) + if key not in uniqueClips: + uniqueClips[key] = clipBox + return { + frozenset(glyphs): uniqueClips[key] for key, glyphs in glyphsByClip.items() + } + + def preWrite(self, font): + if not hasattr(self, "clips"): + self.clips = {} + clipBoxRanges = {} + glyphMap = font.getReverseGlyphMap() + for glyphs, clipBox in self.groups().items(): + glyphIDs = sorted( + glyphMap[glyphName] for glyphName in glyphs if glyphName in glyphMap + ) + if not glyphIDs: + continue + last = glyphIDs[0] + ranges = [[last]] + for glyphID in glyphIDs[1:]: + if glyphID != last + 1: + ranges[-1].append(last) + ranges.append([glyphID]) + last = glyphID + ranges[-1].append(last) + for start, end in ranges: + assert (start, end) not in clipBoxRanges + clipBoxRanges[(start, end)] = clipBox + + clipRecords = [] + for (start, end), clipBox in sorted(clipBoxRanges.items()): + record = ClipRecord() + record.StartGlyphID = start + record.EndGlyphID = end + record.ClipBox = clipBox + clipRecords.append(record) + rawTable = { + "ClipCount": len(clipRecords), + "ClipRecord": clipRecords, + } + return rawTable + + def toXML(self, xmlWriter, font, attrs=None, name=None): + tableName = name if name else self.__class__.__name__ + if attrs is None: + attrs = [] + if hasattr(self, "Format"): + attrs.append(("Format", self.Format)) + xmlWriter.begintag(tableName, attrs) + xmlWriter.newline() + # sort clips alphabetically to ensure deterministic XML dump + for glyphs, clipBox in sorted( + self.groups().items(), key=lambda item: min(item[0]) + ): + xmlWriter.begintag("Clip") + xmlWriter.newline() + for glyphName in sorted(glyphs): + xmlWriter.simpletag("Glyph", value=glyphName) + xmlWriter.newline() + xmlWriter.begintag("ClipBox", [("Format", clipBox.Format)]) + xmlWriter.newline() + clipBox.toXML2(xmlWriter, font) + xmlWriter.endtag("ClipBox") + xmlWriter.newline() + xmlWriter.endtag("Clip") + xmlWriter.newline() + xmlWriter.endtag(tableName) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + clips = getattr(self, "clips", None) + if clips is None: + self.clips = clips = {} + assert name == "Clip" + glyphs = [] + clipBox = None + for elem in content: + if not isinstance(elem, tuple): + continue + name, attrs, content = elem + if name == "Glyph": + glyphs.append(attrs["value"]) + elif name == "ClipBox": + clipBox = ClipBox() + clipBox.Format = safeEval(attrs["Format"]) + for elem in content: + if not isinstance(elem, tuple): + continue + name, attrs, content = elem + clipBox.fromXML(name, attrs, content, font) + if clipBox: + for glyphName in glyphs: + clips[glyphName] = clipBox + + +class ExtendMode(IntEnum): + PAD = 0 + REPEAT = 1 + REFLECT = 2 + + +# Porter-Duff modes for COLRv1 PaintComposite: +# https://github.com/googlefonts/colr-gradients-spec/tree/off_sub_1#compositemode-enumeration +class CompositeMode(IntEnum): + CLEAR = 0 + SRC = 1 + DEST = 2 + SRC_OVER = 3 + DEST_OVER = 4 + SRC_IN = 5 + DEST_IN = 6 + SRC_OUT = 7 + DEST_OUT = 8 + SRC_ATOP = 9 + DEST_ATOP = 10 + XOR = 11 + PLUS = 12 + SCREEN = 13 + OVERLAY = 14 + DARKEN = 15 + LIGHTEN = 16 + COLOR_DODGE = 17 + COLOR_BURN = 18 + HARD_LIGHT = 19 + SOFT_LIGHT = 20 + DIFFERENCE = 21 + EXCLUSION = 22 + MULTIPLY = 23 + HSL_HUE = 24 + HSL_SATURATION = 25 + HSL_COLOR = 26 + HSL_LUMINOSITY = 27 + + +class PaintFormat(IntEnum): + PaintColrLayers = 1 + PaintSolid = 2 + PaintVarSolid = 3 + PaintLinearGradient = 4 + PaintVarLinearGradient = 5 + PaintRadialGradient = 6 + PaintVarRadialGradient = 7 + PaintSweepGradient = 8 + PaintVarSweepGradient = 9 + PaintGlyph = 10 + PaintColrGlyph = 11 + PaintTransform = 12 + PaintVarTransform = 13 + PaintTranslate = 14 + PaintVarTranslate = 15 + PaintScale = 16 + PaintVarScale = 17 + PaintScaleAroundCenter = 18 + PaintVarScaleAroundCenter = 19 + PaintScaleUniform = 20 + PaintVarScaleUniform = 21 + PaintScaleUniformAroundCenter = 22 + PaintVarScaleUniformAroundCenter = 23 + PaintRotate = 24 + PaintVarRotate = 25 + PaintRotateAroundCenter = 26 + PaintVarRotateAroundCenter = 27 + PaintSkew = 28 + PaintVarSkew = 29 + PaintSkewAroundCenter = 30 + PaintVarSkewAroundCenter = 31 + PaintComposite = 32 + + def is_variable(self): + return self.name.startswith("PaintVar") + + def as_variable(self): + if self.is_variable(): + return self + try: + return PaintFormat.__members__[f"PaintVar{self.name[5:]}"] + except KeyError: + return None + + +class Paint(getFormatSwitchingBaseTableClass("uint8")): + formatEnum = PaintFormat + + def getFormatName(self): + try: + return self.formatEnum(self.Format).name + except ValueError: + raise NotImplementedError(f"Unknown Paint format: {self.Format}") + + def toXML(self, xmlWriter, font, attrs=None, name=None): + tableName = name if name else self.__class__.__name__ + if attrs is None: + attrs = [] + attrs.append(("Format", self.Format)) + xmlWriter.begintag(tableName, attrs) + xmlWriter.comment(self.getFormatName()) + xmlWriter.newline() + self.toXML2(xmlWriter, font) + xmlWriter.endtag(tableName) + xmlWriter.newline() + + def iterPaintSubTables(self, colr: COLR) -> Iterator[BaseTable.SubTableEntry]: + if self.Format == PaintFormat.PaintColrLayers: + # https://github.com/fonttools/fonttools/issues/2438: don't die when no LayerList exists + layers = [] + if colr.LayerList is not None: + layers = colr.LayerList.Paint + yield from ( + BaseTable.SubTableEntry(name="Layers", value=v, index=i) + for i, v in enumerate( + layers[self.FirstLayerIndex : self.FirstLayerIndex + self.NumLayers] + ) + ) + return + + if self.Format == PaintFormat.PaintColrGlyph: + for record in colr.BaseGlyphList.BaseGlyphPaintRecord: + if record.BaseGlyph == self.Glyph: + yield BaseTable.SubTableEntry(name="BaseGlyph", value=record.Paint) + return + else: + raise KeyError(f"{self.Glyph!r} not in colr.BaseGlyphList") + + for conv in self.getConverters(): + if conv.tableClass is not None and issubclass(conv.tableClass, type(self)): + value = getattr(self, conv.name) + yield BaseTable.SubTableEntry(name=conv.name, value=value) + + def getChildren(self, colr) -> List["Paint"]: + # this is kept for backward compatibility (e.g. it's used by the subsetter) + return [p.value for p in self.iterPaintSubTables(colr)] + + def traverse(self, colr: COLR, callback): + """Depth-first traversal of graph rooted at self, callback on each node.""" + if not callable(callback): + raise TypeError("callback must be callable") + + for path in dfs_base_table( + self, iter_subtables_fn=lambda paint: paint.iterPaintSubTables(colr) + ): + paint = path[-1].value + callback(paint) + + def getTransform(self) -> Transform: + if self.Format == PaintFormat.PaintTransform: + t = self.Transform + return Transform(t.xx, t.yx, t.xy, t.yy, t.dx, t.dy) + elif self.Format == PaintFormat.PaintTranslate: + return Identity.translate(self.dx, self.dy) + elif self.Format == PaintFormat.PaintScale: + return Identity.scale(self.scaleX, self.scaleY) + elif self.Format == PaintFormat.PaintScaleAroundCenter: + return ( + Identity.translate(self.centerX, self.centerY) + .scale(self.scaleX, self.scaleY) + .translate(-self.centerX, -self.centerY) + ) + elif self.Format == PaintFormat.PaintScaleUniform: + return Identity.scale(self.scale) + elif self.Format == PaintFormat.PaintScaleUniformAroundCenter: + return ( + Identity.translate(self.centerX, self.centerY) + .scale(self.scale) + .translate(-self.centerX, -self.centerY) + ) + elif self.Format == PaintFormat.PaintRotate: + return Identity.rotate(radians(self.angle)) + elif self.Format == PaintFormat.PaintRotateAroundCenter: + return ( + Identity.translate(self.centerX, self.centerY) + .rotate(radians(self.angle)) + .translate(-self.centerX, -self.centerY) + ) + elif self.Format == PaintFormat.PaintSkew: + return Identity.skew(radians(-self.xSkewAngle), radians(self.ySkewAngle)) + elif self.Format == PaintFormat.PaintSkewAroundCenter: + return ( + Identity.translate(self.centerX, self.centerY) + .skew(radians(-self.xSkewAngle), radians(self.ySkewAngle)) + .translate(-self.centerX, -self.centerY) + ) + if PaintFormat(self.Format).is_variable(): + raise NotImplementedError(f"Variable Paints not supported: {self.Format}") + + return Identity + + def computeClipBox( + self, colr: COLR, glyphSet: "_TTGlyphSet", quantization: int = 1 + ) -> Optional[ClipBox]: + pen = ControlBoundsPen(glyphSet) + for path in dfs_base_table( + self, iter_subtables_fn=lambda paint: paint.iterPaintSubTables(colr) + ): + paint = path[-1].value + if paint.Format == PaintFormat.PaintGlyph: + transformation = reduce( + Transform.transform, + (st.value.getTransform() for st in path), + Identity, + ) + glyphSet[paint.Glyph].draw(TransformPen(pen, transformation)) + + if pen.bounds is None: + return None + + cb = ClipBox() + cb.Format = int(ClipBoxFormat.Static) + cb.xMin, cb.yMin, cb.xMax, cb.yMax = quantizeRect(pen.bounds, quantization) + return cb + + +# For each subtable format there is a class. However, we don't really distinguish +# between "field name" and "format name": often these are the same. Yet there's +# a whole bunch of fields with different names. The following dict is a mapping +# from "format name" to "field name". _buildClasses() uses this to create a +# subclass for each alternate field name. +# +_equivalents = { + "PatchMap": ("IFT ", "IFTX"), + "MarkArray": ("Mark1Array",), + "LangSys": ("DefaultLangSys",), + "Coverage": ( + "MarkCoverage", + "BaseCoverage", + "LigatureCoverage", + "Mark1Coverage", + "Mark2Coverage", + "BacktrackCoverage", + "InputCoverage", + "LookAheadCoverage", + "VertGlyphCoverage", + "HorizGlyphCoverage", + "TopAccentCoverage", + "ExtendedShapeCoverage", + "MathKernCoverage", + ), + "ClassDef": ( + "ClassDef1", + "ClassDef2", + "BacktrackClassDef", + "InputClassDef", + "LookAheadClassDef", + "GlyphClassDef", + "MarkAttachClassDef", + ), + "Anchor": ( + "EntryAnchor", + "ExitAnchor", + "BaseAnchor", + "LigatureAnchor", + "Mark2Anchor", + "MarkAnchor", + ), + "Device": ( + "XPlaDevice", + "YPlaDevice", + "XAdvDevice", + "YAdvDevice", + "XDeviceTable", + "YDeviceTable", + "DeviceTable", + ), + "Axis": ( + "HorizAxis", + "VertAxis", + ), + "MinMax": ("DefaultMinMax",), + "BaseCoord": ( + "MinCoord", + "MaxCoord", + ), + "JstfLangSys": ("DefJstfLangSys",), + "JstfGSUBModList": ( + "ShrinkageEnableGSUB", + "ShrinkageDisableGSUB", + "ExtensionEnableGSUB", + "ExtensionDisableGSUB", + ), + "JstfGPOSModList": ( + "ShrinkageEnableGPOS", + "ShrinkageDisableGPOS", + "ExtensionEnableGPOS", + "ExtensionDisableGPOS", + ), + "JstfMax": ( + "ShrinkageJstfMax", + "ExtensionJstfMax", + ), + "MathKern": ( + "TopRightMathKern", + "TopLeftMathKern", + "BottomRightMathKern", + "BottomLeftMathKern", + ), + "MathGlyphConstruction": ("VertGlyphConstruction", "HorizGlyphConstruction"), +} + +# +# OverFlow logic, to automatically create ExtensionLookups +# XXX This should probably move to otBase.py +# + + +def fixLookupOverFlows(ttf, overflowRecord): + """Either the offset from the LookupList to a lookup overflowed, or + an offset from a lookup to a subtable overflowed. + + The table layout is:: + + GPSO/GUSB + Script List + Feature List + LookUpList + Lookup[0] and contents + SubTable offset list + SubTable[0] and contents + ... + SubTable[n] and contents + ... + Lookup[n] and contents + SubTable offset list + SubTable[0] and contents + ... + SubTable[n] and contents + + If the offset to a lookup overflowed (SubTableIndex is None) + we must promote the *previous* lookup to an Extension type. + + If the offset from a lookup to subtable overflowed, then we must promote it + to an Extension Lookup type. + """ + ok = 0 + lookupIndex = overflowRecord.LookupListIndex + if overflowRecord.SubTableIndex is None: + lookupIndex = lookupIndex - 1 + if lookupIndex < 0: + return ok + if overflowRecord.tableType == "GSUB": + extType = 7 + elif overflowRecord.tableType == "GPOS": + extType = 9 + + lookups = ttf[overflowRecord.tableType].table.LookupList.Lookup + lookup = lookups[lookupIndex] + # If the previous lookup is an extType, look further back. Very unlikely, but possible. + while lookup.SubTable[0].__class__.LookupType == extType: + lookupIndex = lookupIndex - 1 + if lookupIndex < 0: + return ok + lookup = lookups[lookupIndex] + + for lookupIndex in range(lookupIndex, len(lookups)): + lookup = lookups[lookupIndex] + if lookup.LookupType != extType: + lookup.LookupType = extType + for si, subTable in enumerate(lookup.SubTable): + extSubTableClass = lookupTypes[overflowRecord.tableType][extType] + extSubTable = extSubTableClass() + extSubTable.Format = 1 + extSubTable.ExtSubTable = subTable + lookup.SubTable[si] = extSubTable + ok = 1 + return ok + + +def splitMultipleSubst(oldSubTable, newSubTable, overflowRecord): + ok = 1 + oldMapping = sorted(oldSubTable.mapping.items()) + oldLen = len(oldMapping) + + if overflowRecord.itemName in ["Coverage", "RangeRecord"]: + # Coverage table is written last. Overflow is to or within the + # the coverage table. We will just cut the subtable in half. + newLen = oldLen // 2 + + elif overflowRecord.itemName == "Sequence": + # We just need to back up by two items from the overflowed + # Sequence index to make sure the offset to the Coverage table + # doesn't overflow. + newLen = overflowRecord.itemIndex - 1 + + newSubTable.mapping = {} + for i in range(newLen, oldLen): + item = oldMapping[i] + key = item[0] + newSubTable.mapping[key] = item[1] + del oldSubTable.mapping[key] + + return ok + + +def splitAlternateSubst(oldSubTable, newSubTable, overflowRecord): + ok = 1 + if hasattr(oldSubTable, "sortCoverageLast"): + newSubTable.sortCoverageLast = oldSubTable.sortCoverageLast + + oldAlts = sorted(oldSubTable.alternates.items()) + oldLen = len(oldAlts) + + if overflowRecord.itemName in ["Coverage", "RangeRecord"]: + # Coverage table is written last. overflow is to or within the + # the coverage table. We will just cut the subtable in half. + newLen = oldLen // 2 + + elif overflowRecord.itemName == "AlternateSet": + # We just need to back up by two items + # from the overflowed AlternateSet index to make sure the offset + # to the Coverage table doesn't overflow. + newLen = overflowRecord.itemIndex - 1 + + newSubTable.alternates = {} + for i in range(newLen, oldLen): + item = oldAlts[i] + key = item[0] + newSubTable.alternates[key] = item[1] + del oldSubTable.alternates[key] + + return ok + + +def splitLigatureSubst(oldSubTable, newSubTable, overflowRecord): + ok = 1 + oldLigs = sorted(oldSubTable.ligatures.items()) + oldLen = len(oldLigs) + + if overflowRecord.itemName in ["Coverage", "RangeRecord"]: + # Coverage table is written last. overflow is to or within the + # the coverage table. We will just cut the subtable in half. + newLen = oldLen // 2 + + elif overflowRecord.itemName == "LigatureSet": + # We just need to back up by two items + # from the overflowed AlternateSet index to make sure the offset + # to the Coverage table doesn't overflow. + newLen = overflowRecord.itemIndex - 1 + + newSubTable.ligatures = {} + for i in range(newLen, oldLen): + item = oldLigs[i] + key = item[0] + newSubTable.ligatures[key] = item[1] + del oldSubTable.ligatures[key] + + return ok + + +def splitPairPos(oldSubTable, newSubTable, overflowRecord): + st = oldSubTable + ok = False + newSubTable.Format = oldSubTable.Format + if oldSubTable.Format == 1 and len(oldSubTable.PairSet) > 1: + for name in "ValueFormat1", "ValueFormat2": + setattr(newSubTable, name, getattr(oldSubTable, name)) + + # Move top half of coverage to new subtable + + newSubTable.Coverage = oldSubTable.Coverage.__class__() + + coverage = oldSubTable.Coverage.glyphs + records = oldSubTable.PairSet + + oldCount = len(oldSubTable.PairSet) // 2 + + oldSubTable.Coverage.glyphs = coverage[:oldCount] + oldSubTable.PairSet = records[:oldCount] + + newSubTable.Coverage.glyphs = coverage[oldCount:] + newSubTable.PairSet = records[oldCount:] + + oldSubTable.PairSetCount = len(oldSubTable.PairSet) + newSubTable.PairSetCount = len(newSubTable.PairSet) + + ok = True + + elif oldSubTable.Format == 2 and len(oldSubTable.Class1Record) > 1: + if not hasattr(oldSubTable, "Class2Count"): + oldSubTable.Class2Count = len(oldSubTable.Class1Record[0].Class2Record) + for name in "Class2Count", "ClassDef2", "ValueFormat1", "ValueFormat2": + setattr(newSubTable, name, getattr(oldSubTable, name)) + + # The two subtables will still have the same ClassDef2 and the table + # sharing will still cause the sharing to overflow. As such, disable + # sharing on the one that is serialized second (that's oldSubTable). + oldSubTable.DontShare = True + + # Move top half of class numbers to new subtable + + newSubTable.Coverage = oldSubTable.Coverage.__class__() + newSubTable.ClassDef1 = oldSubTable.ClassDef1.__class__() + + coverage = oldSubTable.Coverage.glyphs + classDefs = oldSubTable.ClassDef1.classDefs + records = oldSubTable.Class1Record + + oldCount = len(oldSubTable.Class1Record) // 2 + newGlyphs = set(k for k, v in classDefs.items() if v >= oldCount) + + oldSubTable.Coverage.glyphs = [g for g in coverage if g not in newGlyphs] + oldSubTable.ClassDef1.classDefs = { + k: v for k, v in classDefs.items() if v < oldCount + } + oldSubTable.Class1Record = records[:oldCount] + + newSubTable.Coverage.glyphs = [g for g in coverage if g in newGlyphs] + newSubTable.ClassDef1.classDefs = { + k: (v - oldCount) for k, v in classDefs.items() if v > oldCount + } + newSubTable.Class1Record = records[oldCount:] + + oldSubTable.Class1Count = len(oldSubTable.Class1Record) + newSubTable.Class1Count = len(newSubTable.Class1Record) + + ok = True + + return ok + + +def splitMarkBasePos(oldSubTable, newSubTable, overflowRecord): + # split half of the mark classes to the new subtable + classCount = oldSubTable.ClassCount + if classCount < 2: + # oh well, not much left to split... + return False + + oldClassCount = classCount // 2 + newClassCount = classCount - oldClassCount + + oldMarkCoverage, oldMarkRecords = [], [] + newMarkCoverage, newMarkRecords = [], [] + for glyphName, markRecord in zip( + oldSubTable.MarkCoverage.glyphs, oldSubTable.MarkArray.MarkRecord + ): + if markRecord.Class < oldClassCount: + oldMarkCoverage.append(glyphName) + oldMarkRecords.append(markRecord) + else: + markRecord.Class -= oldClassCount + newMarkCoverage.append(glyphName) + newMarkRecords.append(markRecord) + + oldBaseRecords, newBaseRecords = [], [] + for rec in oldSubTable.BaseArray.BaseRecord: + oldBaseRecord, newBaseRecord = rec.__class__(), rec.__class__() + oldBaseRecord.BaseAnchor = rec.BaseAnchor[:oldClassCount] + newBaseRecord.BaseAnchor = rec.BaseAnchor[oldClassCount:] + oldBaseRecords.append(oldBaseRecord) + newBaseRecords.append(newBaseRecord) + + newSubTable.Format = oldSubTable.Format + + oldSubTable.MarkCoverage.glyphs = oldMarkCoverage + newSubTable.MarkCoverage = oldSubTable.MarkCoverage.__class__() + newSubTable.MarkCoverage.glyphs = newMarkCoverage + + # share the same BaseCoverage in both halves + newSubTable.BaseCoverage = oldSubTable.BaseCoverage + + oldSubTable.ClassCount = oldClassCount + newSubTable.ClassCount = newClassCount + + oldSubTable.MarkArray.MarkRecord = oldMarkRecords + newSubTable.MarkArray = oldSubTable.MarkArray.__class__() + newSubTable.MarkArray.MarkRecord = newMarkRecords + + oldSubTable.MarkArray.MarkCount = len(oldMarkRecords) + newSubTable.MarkArray.MarkCount = len(newMarkRecords) + + oldSubTable.BaseArray.BaseRecord = oldBaseRecords + newSubTable.BaseArray = oldSubTable.BaseArray.__class__() + newSubTable.BaseArray.BaseRecord = newBaseRecords + + oldSubTable.BaseArray.BaseCount = len(oldBaseRecords) + newSubTable.BaseArray.BaseCount = len(newBaseRecords) + + return True + + +splitTable = { + "GSUB": { + # 1: splitSingleSubst, + 2: splitMultipleSubst, + 3: splitAlternateSubst, + 4: splitLigatureSubst, + # 5: splitContextSubst, + # 6: splitChainContextSubst, + # 7: splitExtensionSubst, + # 8: splitReverseChainSingleSubst, + }, + "GPOS": { + # 1: splitSinglePos, + 2: splitPairPos, + # 3: splitCursivePos, + 4: splitMarkBasePos, + # 5: splitMarkLigPos, + # 6: splitMarkMarkPos, + # 7: splitContextPos, + # 8: splitChainContextPos, + # 9: splitExtensionPos, + }, +} + + +def fixSubTableOverFlows(ttf, overflowRecord): + """ + An offset has overflowed within a sub-table. We need to divide this subtable into smaller parts. + """ + table = ttf[overflowRecord.tableType].table + lookup = table.LookupList.Lookup[overflowRecord.LookupListIndex] + subIndex = overflowRecord.SubTableIndex + subtable = lookup.SubTable[subIndex] + + # First, try not sharing anything for this subtable... + if not hasattr(subtable, "DontShare"): + subtable.DontShare = True + return True + + if hasattr(subtable, "ExtSubTable"): + # We split the subtable of the Extension table, and add a new Extension table + # to contain the new subtable. + + subTableType = subtable.ExtSubTable.__class__.LookupType + extSubTable = subtable + subtable = extSubTable.ExtSubTable + newExtSubTableClass = lookupTypes[overflowRecord.tableType][ + extSubTable.__class__.LookupType + ] + newExtSubTable = newExtSubTableClass() + newExtSubTable.Format = extSubTable.Format + toInsert = newExtSubTable + + newSubTableClass = lookupTypes[overflowRecord.tableType][subTableType] + newSubTable = newSubTableClass() + newExtSubTable.ExtSubTable = newSubTable + else: + subTableType = subtable.__class__.LookupType + newSubTableClass = lookupTypes[overflowRecord.tableType][subTableType] + newSubTable = newSubTableClass() + toInsert = newSubTable + + if hasattr(lookup, "SubTableCount"): # may not be defined yet. + lookup.SubTableCount = lookup.SubTableCount + 1 + + try: + splitFunc = splitTable[overflowRecord.tableType][subTableType] + except KeyError: + log.error( + "Don't know how to split %s lookup type %s", + overflowRecord.tableType, + subTableType, + ) + return False + + ok = splitFunc(subtable, newSubTable, overflowRecord) + if ok: + lookup.SubTable.insert(subIndex + 1, toInsert) + return ok + + +# End of OverFlow logic + + +# --------------------------------------------------------------------------- +# IFT - Incremental Font Transfer tables +# --------------------------------------------------------------------------- + + +class EntryIdStringData(BaseTable): + """IFT entry ID string data block (raw bytes pool).""" + + def decompile(self, reader, font): + # This subtable is reached via an LOffset, so the reader's data buffer + # is already scoped to the bytes starting at the subtable's offset within + # the IFT table blob. Reading to the end of that buffer is correct; the + # actual bytes consumed are determined by the entryIdStringLength fields + # in the MappingEntries records. + self.data = bytes(reader.data[reader.pos :]) + + def compile(self, writer, font): + writer.writeData(self.data) + + def toXML2(self, xmlWriter, font): + xmlWriter.simpletag("data", value=self.data.hex()) + xmlWriter.newline() + + def fromXML(self, name, attrs, content, font): + if name == "data": + self.data = bytes.fromhex(attrs["value"]) + + +def _buildClasses(): + import re + from .otData import otData + + formatPat = re.compile(r"([A-Za-z0-9]+)Format(\d+)$") + namespace = globals() + + # populate module with classes + for name, fields in otData: + baseClass = BaseTable + m = formatPat.match(name) + if m: + # XxxFormatN subtable, we only add the "base" table + name = m.group(1) + # the first row of a format-switching otData table describes the Format; + # the first column defines the type of the Format field. + # Currently this can be either 'uint16' or 'uint8'. + formatType = fields[0].type + baseClass = getFormatSwitchingBaseTableClass(formatType) + if name not in namespace: + # the class doesn't exist yet, so the base implementation is used. + cls = type(name, (baseClass,), {}) + if name in ("GSUB", "GPOS"): + cls.DontShare = True + namespace[name] = cls + + # link Var{Table} <-> {Table} (e.g. ColorStop <-> VarColorStop, etc.) + for name, _ in otData: + if name.startswith("Var") and len(name) > 3 and name[3:] in namespace: + varType = namespace[name] + noVarType = namespace[name[3:]] + varType.NoVarType = noVarType + noVarType.VarType = varType + + for base, alts in _equivalents.items(): + base = namespace[base] + for alt in alts: + namespace[alt] = base + + global lookupTypes + lookupTypes = { + "GSUB": { + 1: SingleSubst, + 2: MultipleSubst, + 3: AlternateSubst, + 4: LigatureSubst, + 5: ContextSubst, + 6: ChainContextSubst, + 7: ExtensionSubst, + 8: ReverseChainSingleSubst, + }, + "GPOS": { + 1: SinglePos, + 2: PairPos, + 3: CursivePos, + 4: MarkBasePos, + 5: MarkLigPos, + 6: MarkMarkPos, + 7: ContextPos, + 8: ChainContextPos, + 9: ExtensionPos, + }, + "mort": { + 4: NoncontextualMorph, + }, + "morx": { + 0: RearrangementMorph, + 1: ContextualMorph, + 2: LigatureMorph, + # 3: Reserved, + 4: NoncontextualMorph, + 5: InsertionMorph, + }, + } + lookupTypes["JSTF"] = lookupTypes["GPOS"] # JSTF contains GPOS + for lookupEnum in lookupTypes.values(): + for enum, cls in lookupEnum.items(): + cls.LookupType = enum + + global featureParamTypes + featureParamTypes = { + "size": FeatureParamsSize, + } + for i in range(1, 20 + 1): + featureParamTypes["ss%02d" % i] = FeatureParamsStylisticSet + for i in range(1, 99 + 1): + featureParamTypes["cv%02d" % i] = FeatureParamsCharacterVariants + + # add converters to classes + from .otConverters import buildConverters + + for name, fields in otData: + m = formatPat.match(name) + if m: + # XxxFormatN subtable, add converter to "base" table + name, format = m.groups() + format = int(format) + cls = namespace[name] + if not hasattr(cls, "converters"): + cls.converters = {} + cls.convertersByName = {} + converters, convertersByName = buildConverters(fields[1:], namespace) + cls.converters[format] = converters + cls.convertersByName[format] = convertersByName + # XXX Add staticSize? + else: + cls = namespace[name] + cls.converters, cls.convertersByName = buildConverters(fields, namespace) + # XXX Add staticSize? + + +_buildClasses() + + +def _getGlyphsFromCoverageTable(coverage): + if coverage is None: + # empty coverage table + return [] + else: + return coverage.glyphs diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otTraverse.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otTraverse.py new file mode 100644 index 0000000000000000000000000000000000000000..9b624533f150cf16079b3acb8d5ae439236ef10a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/otTraverse.py @@ -0,0 +1,163 @@ +"""Methods for traversing trees of otData-driven OpenType tables.""" + +from collections import deque +from typing import Callable, Deque, Iterable, List, Optional, Tuple +from .otBase import BaseTable + + +__all__ = [ + "bfs_base_table", + "dfs_base_table", + "SubTablePath", +] + + +class SubTablePath(Tuple[BaseTable.SubTableEntry, ...]): + def __str__(self) -> str: + path_parts = [] + for entry in self: + path_part = entry.name + if entry.index is not None: + path_part += f"[{entry.index}]" + path_parts.append(path_part) + return ".".join(path_parts) + + +# Given f(current frontier, new entries) add new entries to frontier +AddToFrontierFn = Callable[[Deque[SubTablePath], List[SubTablePath]], None] + + +def dfs_base_table( + root: BaseTable, + root_accessor: Optional[str] = None, + skip_root: bool = False, + predicate: Optional[Callable[[SubTablePath], bool]] = None, + iter_subtables_fn: Optional[ + Callable[[BaseTable], Iterable[BaseTable.SubTableEntry]] + ] = None, +) -> Iterable[SubTablePath]: + """Depth-first search tree of BaseTables. + + Args: + root (BaseTable): the root of the tree. + root_accessor (Optional[str]): attribute name for the root table, if any (mostly + useful for debugging). + skip_root (Optional[bool]): if True, the root itself is not visited, only its + children. + predicate (Optional[Callable[[SubTablePath], bool]]): function to filter out + paths. If True, the path is yielded and its subtables are added to the + queue. If False, the path is skipped and its subtables are not traversed. + iter_subtables_fn (Optional[Callable[[BaseTable], Iterable[BaseTable.SubTableEntry]]]): + function to iterate over subtables of a table. If None, the default + BaseTable.iterSubTables() is used. + + Yields: + SubTablePath: tuples of BaseTable.SubTableEntry(name, table, index) namedtuples + for each of the nodes in the tree. The last entry in a path is the current + subtable, whereas preceding ones refer to its parent tables all the way up to + the root. + """ + yield from _traverse_ot_data( + root, + root_accessor, + skip_root, + predicate, + lambda frontier, new: frontier.extendleft(reversed(new)), + iter_subtables_fn, + ) + + +def bfs_base_table( + root: BaseTable, + root_accessor: Optional[str] = None, + skip_root: bool = False, + predicate: Optional[Callable[[SubTablePath], bool]] = None, + iter_subtables_fn: Optional[ + Callable[[BaseTable], Iterable[BaseTable.SubTableEntry]] + ] = None, +) -> Iterable[SubTablePath]: + """Breadth-first search tree of BaseTables. + + Args: + root + the root of the tree. + root_accessor (Optional[str]): attribute name for the root table, if any (mostly + useful for debugging). + skip_root (Optional[bool]): if True, the root itself is not visited, only its + children. + predicate (Optional[Callable[[SubTablePath], bool]]): function to filter out + paths. If True, the path is yielded and its subtables are added to the + queue. If False, the path is skipped and its subtables are not traversed. + iter_subtables_fn (Optional[Callable[[BaseTable], Iterable[BaseTable.SubTableEntry]]]): + function to iterate over subtables of a table. If None, the default + BaseTable.iterSubTables() is used. + + Yields: + SubTablePath: tuples of BaseTable.SubTableEntry(name, table, index) namedtuples + for each of the nodes in the tree. The last entry in a path is the current + subtable, whereas preceding ones refer to its parent tables all the way up to + the root. + """ + yield from _traverse_ot_data( + root, + root_accessor, + skip_root, + predicate, + lambda frontier, new: frontier.extend(new), + iter_subtables_fn, + ) + + +def _traverse_ot_data( + root: BaseTable, + root_accessor: Optional[str], + skip_root: bool, + predicate: Optional[Callable[[SubTablePath], bool]], + add_to_frontier_fn: AddToFrontierFn, + iter_subtables_fn: Optional[ + Callable[[BaseTable], Iterable[BaseTable.SubTableEntry]] + ] = None, +) -> Iterable[SubTablePath]: + # no visited because general otData cannot cycle (forward-offset only) + if root_accessor is None: + root_accessor = type(root).__name__ + + if predicate is None: + + def predicate(path): + return True + + if iter_subtables_fn is None: + + def iter_subtables_fn(table): + return table.iterSubTables() + + frontier: Deque[SubTablePath] = deque() + + root_entry = BaseTable.SubTableEntry(root_accessor, root) + if not skip_root: + frontier.append((root_entry,)) + else: + add_to_frontier_fn( + frontier, + [ + (root_entry, subtable_entry) + for subtable_entry in iter_subtables_fn(root) + ], + ) + + while frontier: + # path is (value, attr_name) tuples. attr_name is attr of parent to get value + path = frontier.popleft() + current = path[-1].value + + if not predicate(path): + continue + + yield SubTablePath(path) + + new_entries = [ + path + (subtable_entry,) for subtable_entry in iter_subtables_fn(current) + ] + + add_to_frontier_fn(frontier, new_entries) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/sbixGlyph.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/sbixGlyph.py new file mode 100644 index 0000000000000000000000000000000000000000..b744a2a3bc88907ef027ad7670b1b04f164dd44a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/sbixGlyph.py @@ -0,0 +1,149 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import readHex, safeEval +import struct + + +sbixGlyphHeaderFormat = """ + > + originOffsetX: h # The x-value of the point in the glyph relative to its + # lower-left corner which corresponds to the origin of + # the glyph on the screen, that is the point on the + # baseline at the left edge of the glyph. + originOffsetY: h # The y-value of the point in the glyph relative to its + # lower-left corner which corresponds to the origin of + # the glyph on the screen, that is the point on the + # baseline at the left edge of the glyph. + graphicType: 4s # e.g. "png " +""" + +sbixGlyphHeaderFormatSize = sstruct.calcsize(sbixGlyphHeaderFormat) + + +class Glyph(object): + def __init__( + self, + glyphName=None, + referenceGlyphName=None, + originOffsetX=0, + originOffsetY=0, + graphicType=None, + imageData=None, + rawdata=None, + gid=0, + ): + self.gid = gid + self.glyphName = glyphName + self.referenceGlyphName = referenceGlyphName + self.originOffsetX = originOffsetX + self.originOffsetY = originOffsetY + self.rawdata = rawdata + self.graphicType = graphicType + self.imageData = imageData + + # fix self.graphicType if it is null terminated or too short + if self.graphicType is not None: + if self.graphicType[-1] == "\0": + self.graphicType = self.graphicType[:-1] + if len(self.graphicType) > 4: + from fontTools import ttLib + + raise ttLib.TTLibError( + "Glyph.graphicType must not be longer than 4 characters." + ) + elif len(self.graphicType) < 4: + # pad with spaces + self.graphicType += " "[: (4 - len(self.graphicType))] + + def is_reference_type(self): + """Returns True if this glyph is a reference to another glyph's image data.""" + return self.graphicType == "dupe" or self.graphicType == "flip" + + def decompile(self, ttFont): + self.glyphName = ttFont.getGlyphName(self.gid) + if self.rawdata is None: + from fontTools import ttLib + + raise ttLib.TTLibError("No table data to decompile") + if len(self.rawdata) > 0: + if len(self.rawdata) < sbixGlyphHeaderFormatSize: + from fontTools import ttLib + + # print "Glyph %i header too short: Expected %x, got %x." % (self.gid, sbixGlyphHeaderFormatSize, len(self.rawdata)) + raise ttLib.TTLibError("Glyph header too short.") + + sstruct.unpack( + sbixGlyphHeaderFormat, self.rawdata[:sbixGlyphHeaderFormatSize], self + ) + + if self.is_reference_type(): + # this glyph is a reference to another glyph's image data + (gid,) = struct.unpack(">H", self.rawdata[sbixGlyphHeaderFormatSize:]) + self.referenceGlyphName = ttFont.getGlyphName(gid) + else: + self.imageData = self.rawdata[sbixGlyphHeaderFormatSize:] + self.referenceGlyphName = None + # clean up + del self.rawdata + del self.gid + + def compile(self, ttFont): + if self.glyphName is None: + from fontTools import ttLib + + raise ttLib.TTLibError("Can't compile Glyph without glyph name") + # TODO: if ttFont has no maxp, cmap etc., ignore glyph names and compile by index? + # (needed if you just want to compile the sbix table on its own) + self.gid = struct.pack(">H", ttFont.getGlyphID(self.glyphName)) + if self.graphicType is None: + rawdata = b"" + else: + rawdata = sstruct.pack(sbixGlyphHeaderFormat, self) + if self.is_reference_type(): + rawdata += struct.pack(">H", ttFont.getGlyphID(self.referenceGlyphName)) + else: + assert self.imageData is not None + rawdata += self.imageData + self.rawdata = rawdata + + def toXML(self, xmlWriter, ttFont): + if self.graphicType is None: + # TODO: ignore empty glyphs? + # a glyph data entry is required for each glyph, + # but empty ones can be calculated at compile time + xmlWriter.simpletag("glyph", name=self.glyphName) + xmlWriter.newline() + return + xmlWriter.begintag( + "glyph", + graphicType=self.graphicType, + name=self.glyphName, + originOffsetX=self.originOffsetX, + originOffsetY=self.originOffsetY, + ) + xmlWriter.newline() + if self.is_reference_type(): + # this glyph is a reference to another glyph id. + xmlWriter.simpletag("ref", glyphname=self.referenceGlyphName) + else: + xmlWriter.begintag("hexdata") + xmlWriter.newline() + xmlWriter.dumphex(self.imageData) + xmlWriter.endtag("hexdata") + xmlWriter.newline() + xmlWriter.endtag("glyph") + xmlWriter.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name == "ref": + # this glyph i.e. a reference to another glyph's image data. + # in this case imageData contains the glyph id of the reference glyph + # get glyph id from glyphname + glyphname = safeEval("'''" + attrs["glyphname"] + "'''") + self.imageData = struct.pack(">H", ttFont.getGlyphID(glyphname)) + self.referenceGlyphName = glyphname + elif name == "hexdata": + self.imageData = readHex(content) + else: + from fontTools import ttLib + + raise ttLib.TTLibError("can't handle '%s' element" % name) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/sbixStrike.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/sbixStrike.py new file mode 100644 index 0000000000000000000000000000000000000000..4dfba2e76e899a55b9fa88e2aaba0f5b8c5b142d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/sbixStrike.py @@ -0,0 +1,177 @@ +from fontTools.misc import sstruct +from fontTools.misc.textTools import safeEval +from .sbixGlyph import Glyph +import struct + +sbixStrikeHeaderFormat = """ + > + ppem: H # The PPEM for which this strike was designed (e.g., 9, + # 12, 24) + resolution: H # The screen resolution (in dpi) for which this strike + # was designed (e.g., 72) +""" + +sbixGlyphDataOffsetFormat = """ + > + glyphDataOffset: L # Offset from the beginning of the strike data record + # to data for the individual glyph +""" + +sbixStrikeHeaderFormatSize = sstruct.calcsize(sbixStrikeHeaderFormat) +sbixGlyphDataOffsetFormatSize = sstruct.calcsize(sbixGlyphDataOffsetFormat) + + +class Strike(object): + def __init__(self, rawdata=None, ppem=0, resolution=72): + self.data = rawdata + self.ppem = ppem + self.resolution = resolution + self.glyphs = {} + + def decompile(self, ttFont): + if self.data is None: + from fontTools import ttLib + + raise ttLib.TTLibError + if len(self.data) < sbixStrikeHeaderFormatSize: + from fontTools import ttLib + + raise ( + ttLib.TTLibError, + "Strike header too short: Expected %x, got %x.", + ) % (sbixStrikeHeaderFormatSize, len(self.data)) + + # read Strike header from raw data + sstruct.unpack( + sbixStrikeHeaderFormat, self.data[:sbixStrikeHeaderFormatSize], self + ) + + # calculate number of glyphs + (firstGlyphDataOffset,) = struct.unpack( + ">L", + self.data[ + sbixStrikeHeaderFormatSize : sbixStrikeHeaderFormatSize + + sbixGlyphDataOffsetFormatSize + ], + ) + self.numGlyphs = ( + firstGlyphDataOffset - sbixStrikeHeaderFormatSize + ) // sbixGlyphDataOffsetFormatSize - 1 + # ^ -1 because there's one more offset than glyphs + + # build offset list for single glyph data offsets + self.glyphDataOffsets = [] + for i in range( + self.numGlyphs + 1 + ): # + 1 because there's one more offset than glyphs + start = i * sbixGlyphDataOffsetFormatSize + sbixStrikeHeaderFormatSize + (current_offset,) = struct.unpack( + ">L", self.data[start : start + sbixGlyphDataOffsetFormatSize] + ) + self.glyphDataOffsets.append(current_offset) + + # iterate through offset list and slice raw data into glyph data records + for i in range(self.numGlyphs): + current_glyph = Glyph( + rawdata=self.data[ + self.glyphDataOffsets[i] : self.glyphDataOffsets[i + 1] + ], + gid=i, + ) + current_glyph.decompile(ttFont) + self.glyphs[current_glyph.glyphName] = current_glyph + del self.glyphDataOffsets + del self.numGlyphs + del self.data + + def compile(self, ttFont): + self.glyphDataOffsets = b"" + self.bitmapData = b"" + + glyphOrder = ttFont.getGlyphOrder() + + # first glyph starts right after the header + currentGlyphDataOffset = ( + sbixStrikeHeaderFormatSize + + sbixGlyphDataOffsetFormatSize * (len(glyphOrder) + 1) + ) + for glyphName in glyphOrder: + if glyphName in self.glyphs: + # we have glyph data for this glyph + current_glyph = self.glyphs[glyphName] + else: + # must add empty glyph data record for this glyph + current_glyph = Glyph(glyphName=glyphName) + current_glyph.compile(ttFont) + current_glyph.glyphDataOffset = currentGlyphDataOffset + self.bitmapData += current_glyph.rawdata + currentGlyphDataOffset += len(current_glyph.rawdata) + self.glyphDataOffsets += sstruct.pack( + sbixGlyphDataOffsetFormat, current_glyph + ) + + # add last "offset", really the end address of the last glyph data record + dummy = Glyph() + dummy.glyphDataOffset = currentGlyphDataOffset + self.glyphDataOffsets += sstruct.pack(sbixGlyphDataOffsetFormat, dummy) + + # pack header + self.data = sstruct.pack(sbixStrikeHeaderFormat, self) + # add offsets and image data after header + self.data += self.glyphDataOffsets + self.bitmapData + + def toXML(self, xmlWriter, ttFont): + xmlWriter.begintag("strike") + xmlWriter.newline() + xmlWriter.simpletag("ppem", value=self.ppem) + xmlWriter.newline() + xmlWriter.simpletag("resolution", value=self.resolution) + xmlWriter.newline() + glyphOrder = ttFont.getGlyphOrder() + for glyphName in glyphOrder: + if glyphName in self.glyphs: + self.glyphs[glyphName].toXML(xmlWriter, ttFont) + # TODO: what if there are more glyph data records than (glyf table) glyphs? + xmlWriter.endtag("strike") + xmlWriter.newline() + + def fromXML(self, name, attrs, content, ttFont): + if name in ["ppem", "resolution"]: + setattr(self, name, safeEval(attrs["value"])) + elif name == "glyph": + if "graphicType" in attrs: + myFormat = safeEval("'''" + attrs["graphicType"] + "'''") + else: + myFormat = None + if "glyphname" in attrs: + myGlyphName = safeEval("'''" + attrs["glyphname"] + "'''") + elif "name" in attrs: + myGlyphName = safeEval("'''" + attrs["name"] + "'''") + else: + from fontTools import ttLib + + raise ttLib.TTLibError("Glyph must have a glyph name.") + if "originOffsetX" in attrs: + myOffsetX = safeEval(attrs["originOffsetX"]) + else: + myOffsetX = 0 + if "originOffsetY" in attrs: + myOffsetY = safeEval(attrs["originOffsetY"]) + else: + myOffsetY = 0 + current_glyph = Glyph( + glyphName=myGlyphName, + graphicType=myFormat, + originOffsetX=myOffsetX, + originOffsetY=myOffsetY, + ) + for element in content: + if isinstance(element, tuple): + name, attrs, content = element + current_glyph.fromXML(name, attrs, content, ttFont) + current_glyph.compile(ttFont) + self.glyphs[current_glyph.glyphName] = current_glyph + else: + from fontTools import ttLib + + raise ttLib.TTLibError("can't handle '%s' element" % name) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/table_API_readme.txt b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/table_API_readme.txt new file mode 100644 index 0000000000000000000000000000000000000000..7719201a0ea2d4b8f28a3012ad63c1de8bbf10ef --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/table_API_readme.txt @@ -0,0 +1,91 @@ +This folder is a subpackage of ttLib. Each module here is a +specialized TT/OT table converter: they can convert raw data +to Python objects and vice versa. Usually you don't need to +use the modules directly: they are imported and used +automatically when needed by ttLib. + +If you are writing you own table converter the following is +important. + +The modules here have pretty strange names: this is due to the +fact that we need to map TT table tags (which are case sensitive) +to filenames (which on Mac and Win aren't case sensitive) as well +as to Python identifiers. The latter means it can only contain +[A-Za-z0-9_] and cannot start with a number. + +ttLib provides functions to expand a tag into the format used here: + +>>> from fontTools import ttLib +>>> ttLib.tagToIdentifier("FOO ") +'F_O_O_' +>>> ttLib.tagToIdentifier("cvt ") +'_c_v_t' +>>> ttLib.tagToIdentifier("OS/2") +'O_S_2f_2' +>>> ttLib.tagToIdentifier("glyf") +'_g_l_y_f' +>>> + +And vice versa: + +>>> ttLib.identifierToTag("F_O_O_") +'FOO ' +>>> ttLib.identifierToTag("_c_v_t") +'cvt ' +>>> ttLib.identifierToTag("O_S_2f_2") +'OS/2' +>>> ttLib.identifierToTag("_g_l_y_f") +'glyf' +>>> + +Eg. the 'glyf' table converter lives in a Python file called: + + _g_l_y_f.py + +The converter itself is a class, named "table_" + expandedtag. Eg: + + class table__g_l_y_f: + etc. + +Note that if you _do_ need to use such modules or classes manually, +there are two convenient API functions that let you find them by tag: + +>>> ttLib.getTableModule('glyf') +<module 'ttLib.tables._g_l_y_f'> +>>> ttLib.getTableClass('glyf') +<class ttLib.tables._g_l_y_f.table__g_l_y_f at 645f400> +>>> + +You must subclass from DefaultTable.DefaultTable. It provides some default +behavior, as well as a constructor method (__init__) that you don't need to +override. + +Your converter should minimally provide two methods: + +class table_F_O_O_(DefaultTable.DefaultTable): # converter for table 'FOO ' + + def decompile(self, data, ttFont): + # 'data' is the raw table data. Unpack it into a + # Python data structure. + # 'ttFont' is a ttLib.TTfile instance, enabling you to + # refer to other tables. Do ***not*** keep a reference to + # it: it will cause a circular reference (ttFont saves + # a reference to us), and that means we'll be leaking + # memory. If you need to use it in other methods, just + # pass it around as a method argument. + + def compile(self, ttFont): + # Return the raw data, as converted from the Python + # data structure. + # Again, 'ttFont' is there so you can access other tables. + # Same warning applies. + +If you want to support TTX import/export as well, you need to provide two +additional methods: + + def toXML(self, writer, ttFont): + # XXX + + def fromXML(self, (name, attrs, content), ttFont): + # XXX + diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/ttProgram.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/ttProgram.py new file mode 100644 index 0000000000000000000000000000000000000000..41ea098a86f8843c7cb6783f27810626a8584702 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/tables/ttProgram.py @@ -0,0 +1,596 @@ +"""ttLib.tables.ttProgram.py -- Assembler/disassembler for TrueType bytecode programs.""" + +from __future__ import annotations + +from fontTools.misc.textTools import num2binary, binary2num, readHex, strjoin +import array +from io import StringIO +from typing import List +import re +import logging + + +log = logging.getLogger(__name__) + +# fmt: off + +# first, the list of instructions that eat bytes or words from the instruction stream + +streamInstructions = [ +# +# opcode mnemonic argBits descriptive name pops pushes eats from instruction stream pushes +# + (0x40, 'NPUSHB', 0, 'PushNBytes', 0, -1), # n, b1, b2,...bn b1,b2...bn + (0x41, 'NPUSHW', 0, 'PushNWords', 0, -1), # n, w1, w2,...w w1,w2...wn + (0xb0, 'PUSHB', 3, 'PushBytes', 0, -1), # b0, b1,..bn b0, b1, ...,bn + (0xb8, 'PUSHW', 3, 'PushWords', 0, -1), # w0,w1,..wn w0 ,w1, ...wn +] + + +# next, the list of "normal" instructions + +instructions = [ +# +# opcode mnemonic argBits descriptive name pops pushes eats from instruction stream pushes +# + (0x7f, 'AA', 0, 'AdjustAngle', 1, 0), # p - + (0x64, 'ABS', 0, 'Absolute', 1, 1), # n |n| + (0x60, 'ADD', 0, 'Add', 2, 1), # n2, n1 (n1 + n2) + (0x27, 'ALIGNPTS', 0, 'AlignPts', 2, 0), # p2, p1 - + (0x3c, 'ALIGNRP', 0, 'AlignRelativePt', -1, 0), # p1, p2, ... , ploopvalue - + (0x5a, 'AND', 0, 'LogicalAnd', 2, 1), # e2, e1 b + (0x2b, 'CALL', 0, 'CallFunction', 1, 0), # f - + (0x67, 'CEILING', 0, 'Ceiling', 1, 1), # n ceil(n) + (0x25, 'CINDEX', 0, 'CopyXToTopStack', 1, 1), # k ek + (0x22, 'CLEAR', 0, 'ClearStack', -1, 0), # all items on the stack - + (0x4f, 'DEBUG', 0, 'DebugCall', 1, 0), # n - + (0x73, 'DELTAC1', 0, 'DeltaExceptionC1', -1, 0), # argn, cn, argn-1,cn-1, , arg1, c1 - + (0x74, 'DELTAC2', 0, 'DeltaExceptionC2', -1, 0), # argn, cn, argn-1,cn-1, , arg1, c1 - + (0x75, 'DELTAC3', 0, 'DeltaExceptionC3', -1, 0), # argn, cn, argn-1,cn-1, , arg1, c1 - + (0x5d, 'DELTAP1', 0, 'DeltaExceptionP1', -1, 0), # argn, pn, argn-1, pn-1, , arg1, p1 - + (0x71, 'DELTAP2', 0, 'DeltaExceptionP2', -1, 0), # argn, pn, argn-1, pn-1, , arg1, p1 - + (0x72, 'DELTAP3', 0, 'DeltaExceptionP3', -1, 0), # argn, pn, argn-1, pn-1, , arg1, p1 - + (0x24, 'DEPTH', 0, 'GetDepthStack', 0, 1), # - n + (0x62, 'DIV', 0, 'Divide', 2, 1), # n2, n1 (n1 * 64)/ n2 + (0x20, 'DUP', 0, 'DuplicateTopStack', 1, 2), # e e, e + (0x59, 'EIF', 0, 'EndIf', 0, 0), # - - + (0x1b, 'ELSE', 0, 'Else', 0, 0), # - - + (0x2d, 'ENDF', 0, 'EndFunctionDefinition', 0, 0), # - - + (0x54, 'EQ', 0, 'Equal', 2, 1), # e2, e1 b + (0x57, 'EVEN', 0, 'Even', 1, 1), # e b + (0x2c, 'FDEF', 0, 'FunctionDefinition', 1, 0), # f - + (0x4e, 'FLIPOFF', 0, 'SetAutoFlipOff', 0, 0), # - - + (0x4d, 'FLIPON', 0, 'SetAutoFlipOn', 0, 0), # - - + (0x80, 'FLIPPT', 0, 'FlipPoint', -1, 0), # p1, p2, ..., ploopvalue - + (0x82, 'FLIPRGOFF', 0, 'FlipRangeOff', 2, 0), # h, l - + (0x81, 'FLIPRGON', 0, 'FlipRangeOn', 2, 0), # h, l - + (0x66, 'FLOOR', 0, 'Floor', 1, 1), # n floor(n) + (0x46, 'GC', 1, 'GetCoordOnPVector', 1, 1), # p c + (0x88, 'GETINFO', 0, 'GetInfo', 1, 1), # selector result + (0x91, 'GETVARIATION', 0, 'GetVariation', 0, -1), # - a1,..,an + (0x0d, 'GFV', 0, 'GetFVector', 0, 2), # - px, py + (0x0c, 'GPV', 0, 'GetPVector', 0, 2), # - px, py + (0x52, 'GT', 0, 'GreaterThan', 2, 1), # e2, e1 b + (0x53, 'GTEQ', 0, 'GreaterThanOrEqual', 2, 1), # e2, e1 b + (0x89, 'IDEF', 0, 'InstructionDefinition', 1, 0), # f - + (0x58, 'IF', 0, 'If', 1, 0), # e - + (0x8e, 'INSTCTRL', 0, 'SetInstrExecControl', 2, 0), # s, v - + (0x39, 'IP', 0, 'InterpolatePts', -1, 0), # p1, p2, ... , ploopvalue - + (0x0f, 'ISECT', 0, 'MovePtToIntersect', 5, 0), # a1, a0, b1, b0, p - + (0x30, 'IUP', 1, 'InterpolateUntPts', 0, 0), # - - + (0x1c, 'JMPR', 0, 'Jump', 1, 0), # offset - + (0x79, 'JROF', 0, 'JumpRelativeOnFalse', 2, 0), # e, offset - + (0x78, 'JROT', 0, 'JumpRelativeOnTrue', 2, 0), # e, offset - + (0x2a, 'LOOPCALL', 0, 'LoopAndCallFunction', 2, 0), # f, count - + (0x50, 'LT', 0, 'LessThan', 2, 1), # e2, e1 b + (0x51, 'LTEQ', 0, 'LessThenOrEqual', 2, 1), # e2, e1 b + (0x8b, 'MAX', 0, 'Maximum', 2, 1), # e2, e1 max(e1, e2) + (0x49, 'MD', 1, 'MeasureDistance', 2, 1), # p2,p1 d + (0x2e, 'MDAP', 1, 'MoveDirectAbsPt', 1, 0), # p - + (0xc0, 'MDRP', 5, 'MoveDirectRelPt', 1, 0), # p - + (0x3e, 'MIAP', 1, 'MoveIndirectAbsPt', 2, 0), # n, p - + (0x8c, 'MIN', 0, 'Minimum', 2, 1), # e2, e1 min(e1, e2) + (0x26, 'MINDEX', 0, 'MoveXToTopStack', 1, 1), # k ek + (0xe0, 'MIRP', 5, 'MoveIndirectRelPt', 2, 0), # n, p - + (0x4b, 'MPPEM', 0, 'MeasurePixelPerEm', 0, 1), # - ppem + (0x4c, 'MPS', 0, 'MeasurePointSize', 0, 1), # - pointSize + (0x3a, 'MSIRP', 1, 'MoveStackIndirRelPt', 2, 0), # d, p - + (0x63, 'MUL', 0, 'Multiply', 2, 1), # n2, n1 (n1 * n2)/64 + (0x65, 'NEG', 0, 'Negate', 1, 1), # n -n + (0x55, 'NEQ', 0, 'NotEqual', 2, 1), # e2, e1 b + (0x5c, 'NOT', 0, 'LogicalNot', 1, 1), # e ( not e ) + (0x6c, 'NROUND', 2, 'NoRound', 1, 1), # n1 n2 + (0x56, 'ODD', 0, 'Odd', 1, 1), # e b + (0x5b, 'OR', 0, 'LogicalOr', 2, 1), # e2, e1 b + (0x21, 'POP', 0, 'PopTopStack', 1, 0), # e - + (0x45, 'RCVT', 0, 'ReadCVT', 1, 1), # location value + (0x7d, 'RDTG', 0, 'RoundDownToGrid', 0, 0), # - - + (0x7a, 'ROFF', 0, 'RoundOff', 0, 0), # - - + (0x8a, 'ROLL', 0, 'RollTopThreeStack', 3, 3), # a,b,c b,a,c + (0x68, 'ROUND', 2, 'Round', 1, 1), # n1 n2 + (0x43, 'RS', 0, 'ReadStore', 1, 1), # n v + (0x3d, 'RTDG', 0, 'RoundToDoubleGrid', 0, 0), # - - + (0x18, 'RTG', 0, 'RoundToGrid', 0, 0), # - - + (0x19, 'RTHG', 0, 'RoundToHalfGrid', 0, 0), # - - + (0x7c, 'RUTG', 0, 'RoundUpToGrid', 0, 0), # - - + (0x77, 'S45ROUND', 0, 'SuperRound45Degrees', 1, 0), # n - + (0x7e, 'SANGW', 0, 'SetAngleWeight', 1, 0), # weight - + (0x85, 'SCANCTRL', 0, 'ScanConversionControl', 1, 0), # n - + (0x8d, 'SCANTYPE', 0, 'ScanType', 1, 0), # n - + (0x48, 'SCFS', 0, 'SetCoordFromStackFP', 2, 0), # c, p - + (0x1d, 'SCVTCI', 0, 'SetCVTCutIn', 1, 0), # n - + (0x5e, 'SDB', 0, 'SetDeltaBaseInGState', 1, 0), # n - + (0x86, 'SDPVTL', 1, 'SetDualPVectorToLine', 2, 0), # p2, p1 - + (0x5f, 'SDS', 0, 'SetDeltaShiftInGState', 1, 0), # n - + (0x0b, 'SFVFS', 0, 'SetFVectorFromStack', 2, 0), # y, x - + (0x04, 'SFVTCA', 1, 'SetFVectorToAxis', 0, 0), # - - + (0x08, 'SFVTL', 1, 'SetFVectorToLine', 2, 0), # p2, p1 - + (0x0e, 'SFVTPV', 0, 'SetFVectorToPVector', 0, 0), # - - + (0x34, 'SHC', 1, 'ShiftContourByLastPt', 1, 0), # c - + (0x32, 'SHP', 1, 'ShiftPointByLastPoint', -1, 0), # p1, p2, ..., ploopvalue - + (0x38, 'SHPIX', 0, 'ShiftZoneByPixel', -1, 0), # d, p1, p2, ..., ploopvalue - + (0x36, 'SHZ', 1, 'ShiftZoneByLastPoint', 1, 0), # e - + (0x17, 'SLOOP', 0, 'SetLoopVariable', 1, 0), # n - + (0x1a, 'SMD', 0, 'SetMinimumDistance', 1, 0), # distance - + (0x0a, 'SPVFS', 0, 'SetPVectorFromStack', 2, 0), # y, x - + (0x02, 'SPVTCA', 1, 'SetPVectorToAxis', 0, 0), # - - + (0x06, 'SPVTL', 1, 'SetPVectorToLine', 2, 0), # p2, p1 - + (0x76, 'SROUND', 0, 'SuperRound', 1, 0), # n - + (0x10, 'SRP0', 0, 'SetRefPoint0', 1, 0), # p - + (0x11, 'SRP1', 0, 'SetRefPoint1', 1, 0), # p - + (0x12, 'SRP2', 0, 'SetRefPoint2', 1, 0), # p - + (0x1f, 'SSW', 0, 'SetSingleWidth', 1, 0), # n - + (0x1e, 'SSWCI', 0, 'SetSingleWidthCutIn', 1, 0), # n - + (0x61, 'SUB', 0, 'Subtract', 2, 1), # n2, n1 (n1 - n2) + (0x00, 'SVTCA', 1, 'SetFPVectorToAxis', 0, 0), # - - + (0x23, 'SWAP', 0, 'SwapTopStack', 2, 2), # e2, e1 e1, e2 + (0x13, 'SZP0', 0, 'SetZonePointer0', 1, 0), # n - + (0x14, 'SZP1', 0, 'SetZonePointer1', 1, 0), # n - + (0x15, 'SZP2', 0, 'SetZonePointer2', 1, 0), # n - + (0x16, 'SZPS', 0, 'SetZonePointerS', 1, 0), # n - + (0x29, 'UTP', 0, 'UnTouchPt', 1, 0), # p - + (0x70, 'WCVTF', 0, 'WriteCVTInFUnits', 2, 0), # n, l - + (0x44, 'WCVTP', 0, 'WriteCVTInPixels', 2, 0), # v, l - + (0x42, 'WS', 0, 'WriteStore', 2, 0), # v, l - +] + +# fmt: on + + +def bitRepr(value: int, bits: int) -> str: + s = "" + for i in range(bits): + s = "01"[value & 0x1] + s + value = value >> 1 + return s + + +_mnemonicPat = re.compile(r"[A-Z][A-Z0-9]*$") + + +def _makeDict( + instructionList: list[tuple[int, str, int, str, int, int]], +) -> tuple[dict, dict]: + opcodeDict = {} + mnemonicDict = {} + for op, mnemonic, argBits, name, pops, pushes in instructionList: + assert _mnemonicPat.match(mnemonic) + mnemonicDict[mnemonic] = op, argBits, name + if argBits: + argoffset = op + for i in range(1 << argBits): + opcodeDict[op + i] = mnemonic, argBits, argoffset, name + else: + opcodeDict[op] = mnemonic, 0, 0, name + return opcodeDict, mnemonicDict + + +streamOpcodeDict, streamMnemonicDict = _makeDict(streamInstructions) +opcodeDict, mnemonicDict = _makeDict(instructions) + + +class tt_instructions_error(Exception): + def __init__(self, error: str) -> None: + self.error = error + + def __str__(self) -> str: + return "TT instructions error: %s" % repr(self.error) + + +_comment = r"/\*.*?\*/" +_instruction = r"([A-Z][A-Z0-9]*)\s*\[(.*?)\]" +_number = r"-?[0-9]+" +_token = "(%s)|(%s)|(%s)" % (_instruction, _number, _comment) + +_tokenRE = re.compile(_token) +_whiteRE = re.compile(r"\s*") + +_pushCountPat = re.compile(r"[A-Z][A-Z0-9]*\s*\[.*?\]\s*/\* ([0-9]+).*?\*/") + +_indentRE = re.compile(r"^FDEF|IF|ELSE\[ \]\t.+") +_unindentRE = re.compile(r"^ELSE|ENDF|EIF\[ \]\t.+") + + +def _skipWhite(data: str, pos: int) -> int: + m = _whiteRE.match(data, pos) + newPos = m.regs[0][1] + assert newPos >= pos + return newPos + + +class Program: + def __init__(self) -> None: + pass + + def fromBytecode(self, bytecode: bytes) -> None: + self.bytecode = array.array("B", bytecode) + if hasattr(self, "assembly"): + del self.assembly + + def fromAssembly(self, assembly: list[str] | str) -> None: + if isinstance(assembly, list): + self.assembly = assembly + elif isinstance(assembly, str): + self.assembly = assembly.splitlines() + else: + raise TypeError(f"expected str or List[str], got {type(assembly).__name__}") + if hasattr(self, "bytecode"): + del self.bytecode + + def getBytecode(self) -> bytes: + if not hasattr(self, "bytecode"): + self._assemble() + return self.bytecode.tobytes() + + def getAssembly(self, preserve=True) -> List[str]: + if not hasattr(self, "assembly"): + self._disassemble(preserve=preserve) + return self.assembly + + def toXML(self, writer, ttFont) -> None: + if ( + not hasattr(ttFont, "disassembleInstructions") + or ttFont.disassembleInstructions + ): + try: + assembly = self.getAssembly() + except: + import traceback + + tmp = StringIO() + traceback.print_exc(file=tmp) + msg = "An exception occurred during the decompilation of glyph program:\n\n" + msg += tmp.getvalue() + log.error(msg) + writer.begintag("bytecode") + writer.newline() + writer.comment(msg.strip()) + writer.newline() + writer.dumphex(self.getBytecode()) + writer.endtag("bytecode") + writer.newline() + else: + if not assembly: + return + writer.begintag("assembly") + writer.newline() + i = 0 + indent = 0 + nInstr = len(assembly) + while i < nInstr: + instr = assembly[i] + if _unindentRE.match(instr): + indent -= 1 + writer.write(writer.indentwhite * indent) + writer.write(instr) + writer.newline() + m = _pushCountPat.match(instr) + i = i + 1 + if m: + nValues = int(m.group(1)) + line: List[str] = [] + j = 0 + for j in range(nValues): + if j and not (j % 25): + writer.write(writer.indentwhite * indent) + writer.write(" ".join(line)) + writer.newline() + line = [] + line.append(assembly[i + j]) + writer.write(writer.indentwhite * indent) + writer.write(" ".join(line)) + writer.newline() + i = i + j + 1 + if _indentRE.match(instr): + indent += 1 + writer.endtag("assembly") + writer.newline() + else: + bytecode = self.getBytecode() + if not bytecode: + return + writer.begintag("bytecode") + writer.newline() + writer.dumphex(bytecode) + writer.endtag("bytecode") + writer.newline() + + def fromXML(self, name, attrs, content, ttFont) -> None: + if name == "assembly": + self.fromAssembly(strjoin(content)) + self._assemble() + del self.assembly + else: + assert name == "bytecode" + self.fromBytecode(readHex(content)) + + def _assemble(self) -> None: + assembly = " ".join(getattr(self, "assembly", [])) + bytecode: List[int] = [] + push = bytecode.append + lenAssembly = len(assembly) + pos = _skipWhite(assembly, 0) + while pos < lenAssembly: + m = _tokenRE.match(assembly, pos) + if m is None: + raise tt_instructions_error( + "Syntax error in TT program (%s)" % assembly[pos - 5 : pos + 15] + ) + dummy, mnemonic, arg, number, comment = m.groups() + pos = m.regs[0][1] + if comment: + pos = _skipWhite(assembly, pos) + continue + + arg = arg.strip() + if mnemonic.startswith("INSTR"): + # Unknown instruction + op = int(mnemonic[5:]) + push(op) + elif mnemonic not in ("PUSH", "NPUSHB", "NPUSHW", "PUSHB", "PUSHW"): + op, argBits, name = mnemonicDict[mnemonic] + if len(arg) != argBits: + raise tt_instructions_error( + "Incorrect number of argument bits (%s[%s])" % (mnemonic, arg) + ) + if arg: + arg = binary2num(arg) + push(op + arg) + else: + push(op) + else: + args = [] + pos = _skipWhite(assembly, pos) + while pos < lenAssembly: + m = _tokenRE.match(assembly, pos) + if m is None: + raise tt_instructions_error( + "Syntax error in TT program (%s)" % assembly[pos : pos + 15] + ) + dummy, _mnemonic, arg, number, comment = m.groups() + if number is None and comment is None: + break + pos = m.regs[0][1] + pos = _skipWhite(assembly, pos) + if comment is not None: + continue + args.append(int(number)) + nArgs = len(args) + if mnemonic == "PUSH": + # Automatically choose the most compact representation + nWords = 0 + while nArgs: + while ( + nWords < nArgs + and nWords < 255 + and not (0 <= args[nWords] <= 255) + ): + nWords += 1 + nBytes = 0 + while ( + nWords + nBytes < nArgs + and nBytes < 255 + and 0 <= args[nWords + nBytes] <= 255 + ): + nBytes += 1 + if ( + nBytes < 2 + and nWords + nBytes < 255 + and nWords + nBytes != nArgs + ): + # Will write bytes as words + nWords += nBytes + continue + + # Write words + if nWords: + if nWords <= 8: + op, argBits, name = streamMnemonicDict["PUSHW"] + op = op + nWords - 1 + push(op) + else: + op, argBits, name = streamMnemonicDict["NPUSHW"] + push(op) + push(nWords) + for value in args[:nWords]: + assert -32768 <= value < 32768, ( + "PUSH value out of range %d" % value + ) + push((value >> 8) & 0xFF) + push(value & 0xFF) + + # Write bytes + if nBytes: + pass + if nBytes <= 8: + op, argBits, name = streamMnemonicDict["PUSHB"] + op = op + nBytes - 1 + push(op) + else: + op, argBits, name = streamMnemonicDict["NPUSHB"] + push(op) + push(nBytes) + for value in args[nWords : nWords + nBytes]: + push(value) + + nTotal = nWords + nBytes + args = args[nTotal:] + nArgs -= nTotal + nWords = 0 + else: + # Write exactly what we've been asked to + words = mnemonic[-1] == "W" + op, argBits, name = streamMnemonicDict[mnemonic] + if mnemonic[0] != "N": + assert nArgs <= 8, nArgs + op = op + nArgs - 1 + push(op) + else: + assert nArgs < 256 + push(op) + push(nArgs) + if words: + for value in args: + assert -32768 <= value < 32768, ( + "PUSHW value out of range %d" % value + ) + push((value >> 8) & 0xFF) + push(value & 0xFF) + else: + for value in args: + assert 0 <= value < 256, ( + "PUSHB value out of range %d" % value + ) + push(value) + + pos = _skipWhite(assembly, pos) + + if bytecode: + assert max(bytecode) < 256 and min(bytecode) >= 0 + self.bytecode = array.array("B", bytecode) + + def _disassemble(self, preserve=False) -> None: + assembly = [] + i = 0 + bytecode = getattr(self, "bytecode", []) + numBytecode = len(bytecode) + while i < numBytecode: + op = bytecode[i] + try: + mnemonic, argBits, argoffset, name = opcodeDict[op] + except KeyError: + if op in streamOpcodeDict: + values = [] + + # Merge consecutive PUSH operations + while bytecode[i] in streamOpcodeDict: + op = bytecode[i] + mnemonic, argBits, argoffset, name = streamOpcodeDict[op] + words = mnemonic[-1] == "W" + if argBits: + nValues = op - argoffset + 1 + else: + i = i + 1 + nValues = bytecode[i] + i = i + 1 + assert nValues > 0 + if not words: + for j in range(nValues): + value = bytecode[i] + values.append(repr(value)) + i = i + 1 + else: + for j in range(nValues): + # cast to signed int16 + value = (bytecode[i] << 8) | bytecode[i + 1] + if value >= 0x8000: + value = value - 0x10000 + values.append(repr(value)) + i = i + 2 + if preserve: + break + + if not preserve: + mnemonic = "PUSH" + nValues = len(values) + if nValues == 1: + assembly.append("%s[ ] /* 1 value pushed */" % mnemonic) + else: + assembly.append( + "%s[ ] /* %s values pushed */" % (mnemonic, nValues) + ) + assembly.extend(values) + else: + assembly.append("INSTR%d[ ]" % op) + i = i + 1 + else: + if argBits: + assembly.append( + mnemonic + + "[%s] /* %s */" % (num2binary(op - argoffset, argBits), name) + ) + else: + assembly.append(mnemonic + "[ ] /* %s */" % name) + i = i + 1 + self.assembly = assembly + + def __bool__(self) -> bool: + """ + >>> p = Program() + >>> bool(p) + False + >>> bc = array.array("B", [0]) + >>> p.fromBytecode(bc) + >>> bool(p) + True + >>> p.bytecode.pop() + 0 + >>> bool(p) + False + + >>> p = Program() + >>> asm = ['SVTCA[0]'] + >>> p.fromAssembly(asm) + >>> bool(p) + True + >>> p.assembly.pop() + 'SVTCA[0]' + >>> bool(p) + False + """ + return (hasattr(self, "assembly") and len(self.assembly) > 0) or ( + hasattr(self, "bytecode") and len(self.bytecode) > 0 + ) + + __nonzero__ = __bool__ + + def __eq__(self, other) -> bool: + if type(self) != type(other): + return NotImplemented + return self.__dict__ == other.__dict__ + + def __ne__(self, other) -> bool: + result = self.__eq__(other) + return result if result is NotImplemented else not result + + +def _test(): + """ + >>> _test() + True + """ + + bc = b"""@;:9876543210/.-,+*)(\'&%$#"! \037\036\035\034\033\032\031\030\027\026\025\024\023\022\021\020\017\016\015\014\013\012\011\010\007\006\005\004\003\002\001\000,\001\260\030CXEj\260\031C`\260F#D#\020 \260FN\360M/\260\000\022\033!#\0213Y-,\001\260\030CX\260\005+\260\000\023K\260\024PX\261\000@8Y\260\006+\033!#\0213Y-,\001\260\030CXN\260\003%\020\362!\260\000\022M\033 E\260\004%\260\004%#Jad\260(RX!#\020\326\033\260\003%\020\362!\260\000\022YY-,\260\032CX!!\033\260\002%\260\002%I\260\003%\260\003%Ja d\260\020PX!!!\033\260\003%\260\003%I\260\000PX\260\000PX\270\377\3428!\033\260\0208!Y\033\260\000RX\260\0368!\033\270\377\3608!YYYY-,\001\260\030CX\260\005+\260\000\023K\260\024PX\271\000\000\377\3008Y\260\006+\033!#\0213Y-,N\001\212\020\261F\031CD\260\000\024\261\000F\342\260\000\025\271\000\000\377\3608\000\260\000<\260(+\260\002%\020\260\000<-,\001\030\260\000/\260\001\024\362\260\001\023\260\001\025M\260\000\022-,\001\260\030CX\260\005+\260\000\023\271\000\000\377\3408\260\006+\033!#\0213Y-,\001\260\030CXEdj#Edi\260\031Cd``\260F#D#\020 \260F\360/\260\000\022\033!! \212 \212RX\0213\033!!YY-,\001\261\013\012C#Ce\012-,\000\261\012\013C#C\013-,\000\260F#p\261\001F>\001\260F#p\261\002FE:\261\002\000\010\015-,\260\022+\260\002%E\260\002%Ej\260@\213`\260\002%#D!!!-,\260\023+\260\002%E\260\002%Ej\270\377\300\214`\260\002%#D!!!-,\260\000\260\022+!!!-,\260\000\260\023+!!!-,\001\260\006C\260\007Ce\012-, i\260@a\260\000\213 \261,\300\212\214\270\020\000b`+\014d#da\\X\260\003aY-,\261\000\003%EhT\260\034KPZX\260\003%E\260\003%E`h \260\004%#D\260\004%#D\033\260\003% Eh \212#D\260\003%Eh`\260\003%#DY-,\260\003% Eh \212#D\260\003%Edhe`\260\004%\260\001`#D-,\260\011CX\207!\300\033\260\022CX\207E\260\021+\260G#D\260Gz\344\033\003\212E\030i \260G#D\212\212\207 \260\240QX\260\021+\260G#D\260Gz\344\033!\260Gz\344YYY\030-, \212E#Eh`D-,EjB-,\001\030/-,\001\260\030CX\260\004%\260\004%Id#Edi\260@\213a \260\200bj\260\002%\260\002%a\214\260\031C`\260F#D!\212\020\260F\366!\033!!!!Y-,\001\260\030CX\260\002%E\260\002%Ed`j\260\003%Eja \260\004%Ej \212\213e\260\004%#D\214\260\003%#D!!\033 EjD EjDY-,\001 E\260\000U\260\030CZXEh#Ei\260@\213a \260\200bj \212#a \260\003%\213e\260\004%#D\214\260\003%#D!!\033!!\260\031+Y-,\001\212\212Ed#EdadB-,\260\004%\260\004%\260\031+\260\030CX\260\004%\260\004%\260\003%\260\033+\001\260\002%C\260@T\260\002%C\260\000TZX\260\003% E\260@aDY\260\002%C\260\000T\260\002%C\260@TZX\260\004% E\260@`DYY!!!!-,\001KRXC\260\002%E#aD\033!!Y-,\001KRXC\260\002%E#`D\033!!Y-,KRXED\033!!Y-,\001 \260\003%#I\260@`\260 c \260\000RX#\260\002%8#\260\002%e8\000\212c8\033!!!!!Y\001-,KPXED\033!!Y-,\001\260\005%\020# \212\365\000\260\001`#\355\354-,\001\260\005%\020# \212\365\000\260\001a#\355\354-,\001\260\006%\020\365\000\355\354-,F#F`\212\212F# F\212`\212a\270\377\200b# \020#\212\261KK\212pE` \260\000PX\260\001a\270\377\272\213\033\260F\214Y\260\020`h\001:-, E\260\003%FRX\260\002%F ha\260\003%\260\003%?#!8\033!\021Y-, E\260\003%FPX\260\002%F ha\260\003%\260\003%?#!8\033!\021Y-,\000\260\007C\260\006C\013-,\212\020\354-,\260\014CX!\033 F\260\000RX\270\377\3608\033\260\0208YY-, \260\000UX\270\020\000c\260\003%Ed\260\003%Eda\260\000SX\260\002\033\260@a\260\003Y%EiSXED\033!!Y\033!\260\002%E\260\002%Ead\260(QXED\033!!YY-,!!\014d#d\213\270@\000b-,!\260\200QX\014d#d\213\270 \000b\033\262\000@/+Y\260\002`-,!\260\300QX\014d#d\213\270\025Ub\033\262\000\200/+Y\260\002`-,\014d#d\213\270@\000b`#!-,KSX\260\004%\260\004%Id#Edi\260@\213a \260\200bj\260\002%\260\002%a\214\260F#D!\212\020\260F\366!\033!\212\021#\022 9/Y-,\260\002%\260\002%Id\260\300TX\270\377\3708\260\0108\033!!Y-,\260\023CX\003\033\002Y-,\260\023CX\002\033\003Y-,\260\012+#\020 <\260\027+-,\260\002%\270\377\3608\260(+\212\020# \320#\260\020+\260\005CX\300\033<Y \020\021\260\000\022\001-,KS#KQZX8\033!!Y-,\001\260\002%\020\320#\311\001\260\001\023\260\000\024\020\260\001<\260\001\026-,\001\260\000\023\260\001\260\003%I\260\003\0278\260\001\023-,KS#KQZX E\212`D\033!!Y-, 9/-""" + + p = Program() + p.fromBytecode(bc) + asm = p.getAssembly(preserve=True) + p.fromAssembly(asm) + print(bc == p.getBytecode()) + + +if __name__ == "__main__": + import sys + import doctest + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttCollection.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttCollection.py new file mode 100644 index 0000000000000000000000000000000000000000..f01bc42be311904288070d8961270aca04029fe3 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttCollection.py @@ -0,0 +1,125 @@ +from fontTools.ttLib.ttFont import TTFont +from fontTools.ttLib.sfnt import readTTCHeader, writeTTCHeader +from io import BytesIO +import struct +import logging + +log = logging.getLogger(__name__) + + +class TTCollection(object): + """Object representing a TrueType Collection / OpenType Collection. + The main API is self.fonts being a list of TTFont instances. + + If shareTables is True, then different fonts in the collection + might point to the same table object if the data for the table was + the same in the font file. Note, however, that this might result + in suprises and incorrect behavior if the different fonts involved + have different GlyphOrder. Use only if you know what you are doing. + """ + + def __init__(self, file=None, shareTables=False, **kwargs): + fonts = self.fonts = [] + if file is None: + return + + assert "fontNumber" not in kwargs, kwargs + + closeStream = False + if not hasattr(file, "read"): + file = open(file, "rb") + closeStream = True + + tableCache = {} if shareTables else None + + header = readTTCHeader(file) + for i in range(header.numFonts): + font = TTFont(file, fontNumber=i, _tableCache=tableCache, **kwargs) + fonts.append(font) + + # don't close file if lazy=True, as the TTFont hold a reference to the original + # file; the file will be closed once the TTFonts are closed in the + # TTCollection.close(). We still want to close the file if lazy is None or + # False, because in that case the TTFont no longer need the original file + # and we want to avoid 'ResourceWarning: unclosed file'. + if not kwargs.get("lazy") and closeStream: + file.close() + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def close(self): + for font in self.fonts: + font.close() + + def save(self, file, shareTables=True): + """Save the font to disk. Similarly to the constructor, + the 'file' argument can be either a pathname or a writable + file object. + """ + if not hasattr(file, "write"): + final = None + file = open(file, "wb") + else: + # assume "file" is a writable file object + # write to a temporary stream to allow saving to unseekable streams + final = file + file = BytesIO() + + tableCache = {} if shareTables else None + + offsets_offset = writeTTCHeader(file, len(self.fonts)) + offsets = [] + for font in self.fonts: + offsets.append(file.tell()) + font._save(file, tableCache=tableCache) + file.seek(0, 2) + + file.seek(offsets_offset) + file.write(struct.pack(">%dL" % len(self.fonts), *offsets)) + + if final: + final.write(file.getvalue()) + file.close() + + def saveXML(self, fileOrPath, newlinestr="\n", writeVersion=True, **kwargs): + from fontTools.misc import xmlWriter + + writer = xmlWriter.XMLWriter(fileOrPath, newlinestr=newlinestr) + + if writeVersion: + from fontTools import version + + version = ".".join(version.split(".")[:2]) + writer.begintag("ttCollection", ttLibVersion=version) + else: + writer.begintag("ttCollection") + writer.newline() + writer.newline() + + for font in self.fonts: + font._saveXML(writer, writeVersion=False, **kwargs) + writer.newline() + + writer.endtag("ttCollection") + writer.newline() + + writer.close() + + def __getitem__(self, item): + return self.fonts[item] + + def __setitem__(self, item, value): + self.fonts[item] = value + + def __delitem__(self, item): + return self.fonts[item] + + def __len__(self): + return len(self.fonts) + + def __iter__(self): + return iter(self.fonts) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttFont.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttFont.py new file mode 100644 index 0000000000000000000000000000000000000000..507e2ce68aad47c0215e55e72b5fefdd3750d375 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttFont.py @@ -0,0 +1,1696 @@ +from __future__ import annotations + +import logging +import os +import traceback +from io import BytesIO, StringIO, UnsupportedOperation +from typing import TYPE_CHECKING, TypedDict, TypeVar, overload + +from fontTools.config import Config +from fontTools.misc import xmlWriter +from fontTools.misc.configTools import AbstractConfig +from fontTools.misc.loggingTools import deprecateArgument +from fontTools.misc.textTools import Tag, byteord, tostr +from fontTools.ttLib import TTLibError +from fontTools.ttLib.sfnt import SFNTReader, SFNTWriter +from fontTools.ttLib.ttGlyphSet import ( + _TTGlyph, # noqa: F401 + _TTGlyphSet, + _TTGlyphSetCFF, + _TTGlyphSetGlyf, + _TTGlyphSetVARC, +) + +if TYPE_CHECKING: + from collections.abc import Mapping, MutableMapping + from types import ModuleType, TracebackType + from typing import Any, BinaryIO, Literal, Sequence, TextIO + + from typing_extensions import Self, Unpack + + from fontTools.ttLib.tables import ( + B_A_S_E_, + C_B_D_T_, + C_B_L_C_, + C_F_F_, + C_F_F__2, + C_O_L_R_, + C_P_A_L_, + D_S_I_G_, + E_B_D_T_, + E_B_L_C_, + F_F_T_M_, + G_D_E_F_, + G_P_O_S_, + G_S_U_B_, + G_V_A_R_, + H_V_A_R_, + I_F_T_, + I_F_T_X_, + J_S_T_F_, + L_T_S_H_, + M_A_T_H_, + M_V_A_R_, + S_T_A_T_, + S_V_G_, + T_S_I__0, + T_S_I__1, + T_S_I__2, + T_S_I__3, + T_S_I__5, + T_S_I_B_, + T_S_I_C_, + T_S_I_D_, + T_S_I_J_, + T_S_I_P_, + T_S_I_S_, + T_S_I_V_, + T_T_F_A_, + V_A_R_C_, + V_D_M_X_, + V_O_R_G_, + V_V_A_R_, + D__e_b_g, + F__e_a_t, + G__l_a_t, + G__l_o_c, + O_S_2f_2, + S__i_l_f, + S__i_l_l, + _a_n_k_r, + _a_v_a_r, + _b_g_c_l, + _b_s_l_n, + _c_i_d_g, + _c_m_a_p, + _c_v_a_r, + _c_v_t, + _f_e_a_t, + _f_p_g_m, + _f_v_a_r, + _g_a_s_p, + _g_c_i_d, + _g_l_y_f, + _g_v_a_r, + _h_d_m_x, + _h_e_a_d, + _h_h_e_a, + _h_m_t_x, + _k_e_r_n, + _l_c_a_r, + _l_o_c_a, + _l_t_a_g, + _m_a_x_p, + _m_e_t_a, + _m_o_r_t, + _m_o_r_x, + _n_a_m_e, + _o_p_b_d, + _p_o_s_t, + _p_r_e_p, + _p_r_o_p, + _s_b_i_x, + _t_r_a_k, + _v_h_e_a, + _v_m_t_x, + ) + from fontTools.ttLib.tables.DefaultTable import DefaultTable + + _VT_co = TypeVar("_VT_co", covariant=True) # Value type covariant containers. + +log = logging.getLogger(__name__) + + +_NumberT = TypeVar("_NumberT", bound=float) + + +class TTFont(object): + """Represents a TrueType font. + + The object manages file input and output, and offers a convenient way of + accessing tables. Tables will be only decompiled when necessary, ie. when + they're actually accessed. This means that simple operations can be extremely fast. + + Example usage: + + .. code-block:: pycon + + >>> + >> from fontTools import ttLib + >> tt = ttLib.TTFont("afont.ttf") # Load an existing font file + >> tt['maxp'].numGlyphs + 242 + >> tt['OS/2'].achVendID + 'B&H\000' + >> tt['head'].unitsPerEm + 2048 + + For details of the objects returned when accessing each table, see the + :doc:`tables </ttLib/tables>` documentation. + To add a table to the font, use the :py:func:`newTable` function: + + .. code-block:: pycon + + >>> + >> os2 = newTable("OS/2") + >> os2.version = 4 + >> # set other attributes + >> font["OS/2"] = os2 + + TrueType fonts can also be serialized to and from XML format (see also the + :doc:`ttx </ttx>` binary): + + .. code-block:: pycon + + >> + >> tt.saveXML("afont.ttx") + Dumping 'LTSH' table... + Dumping 'OS/2' table... + [...] + + >> tt2 = ttLib.TTFont() # Create a new font object + >> tt2.importXML("afont.ttx") + >> tt2['maxp'].numGlyphs + 242 + + The TTFont object may be used as a context manager; this will cause the file + reader to be closed after the context ``with`` block is exited:: + + with TTFont(filename) as f: + # Do stuff + + Args: + file: When reading a font from disk, either a pathname pointing to a file, + or a readable file object. + res_name_or_index: If running on a Macintosh, either a sfnt resource name or + an sfnt resource index number. If the index number is zero, TTLib will + autodetect whether the file is a flat file or a suitcase. (If it is a suitcase, + only the first 'sfnt' resource will be read.) + sfntVersion (str): When constructing a font object from scratch, sets the four-byte + sfnt magic number to be used. Defaults to ``\0\1\0\0`` (TrueType). To create + an OpenType file, use ``OTTO``. + flavor (str): Set this to ``woff`` when creating a WOFF file or ``woff2`` for a WOFF2 + file. + checkChecksums (int): How checksum data should be treated. Default is 0 + (no checking). Set to 1 to check and warn on wrong checksums; set to 2 to + raise an exception if any wrong checksums are found. + recalcBBoxes (bool): If true (the default), recalculates ``glyf``, ``CFF ``, + ``head`` bounding box values and ``hhea``/``vhea`` min/max values on save. + Also compiles the glyphs on importing, which saves memory consumption and + time. + ignoreDecompileErrors (bool): If true, exceptions raised during table decompilation + will be ignored, and the binary data will be returned for those tables instead. + recalcTimestamp (bool): If true (the default), sets the ``modified`` timestamp in + the ``head`` table on save. + fontNumber (int): The index of the font in a TrueType Collection file. + lazy (bool): If lazy is set to True, many data structures are loaded lazily, upon + access only. If it is set to False, many data structures are loaded immediately. + The default is ``lazy=None`` which is somewhere in between. + """ + + tables: dict[Tag, DefaultTable | GlyphOrder] + reader: SFNTReader | None + sfntVersion: str + flavor: str | None + flavorData: Any | None + lazy: bool | None + recalcBBoxes: bool + recalcTimestamp: bool + ignoreDecompileErrors: bool + cfg: AbstractConfig + glyphOrder: list[str] + _reverseGlyphOrderDict: dict[str, int] + _tableCache: MutableMapping[tuple[Tag, bytes], DefaultTable] | None + disassembleInstructions: bool + bitmapGlyphDataFormat: str + # Deprecated attributes + verbose: bool | None + quiet: bool | None + + def __init__( + self, + file: str | os.PathLike[str] | BinaryIO | None = None, + res_name_or_index: str | int | None = None, + sfntVersion: str = "\000\001\000\000", + flavor: str | None = None, + checkChecksums: int = 0, + verbose: bool | None = None, # Deprecated + recalcBBoxes: bool = True, + allowVID: Any = NotImplemented, # Deprecated/Unused + ignoreDecompileErrors: bool = False, + recalcTimestamp: bool = True, + fontNumber: int = -1, + lazy: bool | None = None, + quiet: bool | None = None, # Deprecated + _tableCache: MutableMapping[tuple[Tag, bytes], DefaultTable] | None = None, + cfg: Mapping[str, Any] | AbstractConfig = {}, + ) -> None: + # Set deprecated attributes + for name in ("verbose", "quiet"): + val = locals().get(name) + if val is not None: + deprecateArgument(name, "configure logging instead") + setattr(self, name, val) + + self.lazy = lazy + self.recalcBBoxes = recalcBBoxes + self.recalcTimestamp = recalcTimestamp + self.tables = {} + self.reader = None + self.cfg = cfg.copy() if isinstance(cfg, AbstractConfig) else Config(cfg) + self.ignoreDecompileErrors = ignoreDecompileErrors + + if not file: + self.sfntVersion = sfntVersion + self.flavor = flavor + self.flavorData = None + return + seekable = True + if not hasattr(file, "read"): + if not isinstance(file, (str, os.PathLike)): + raise TypeError( + "fileOrPath must be a file path (str or PathLike) if it isn't an object with a `read` method." + ) + closeStream = True + # assume file is a string + if res_name_or_index is not None: + # see if it contains 'sfnt' resources in the resource or data fork + from . import macUtils + + if res_name_or_index == 0: + if macUtils.getSFNTResIndices(file): + # get the first available sfnt font. + file = macUtils.SFNTResourceReader(file, 1) + else: + file = open(file, "rb") + else: + file = macUtils.SFNTResourceReader(file, res_name_or_index) + else: + file = open(file, "rb") + else: + # assume "file" is a readable file object + assert not isinstance(file, (str, os.PathLike)) + closeStream = False + # SFNTReader wants the input file to be seekable. + # SpooledTemporaryFile has no seekable() on < 3.11, but still can seek: + # https://github.com/fonttools/fonttools/issues/3052 + if hasattr(file, "seekable"): + seekable = file.seekable() + elif hasattr(file, "seek"): + try: + file.seek(0) + except UnsupportedOperation: + seekable = False + + if not self.lazy: + # read input file in memory and wrap a stream around it to allow overwriting + if seekable: + file.seek(0) + tmp = BytesIO(file.read()) + if hasattr(file, "name"): + # save reference to input file name + tmp.name = file.name + if closeStream: + file.close() + file = tmp + elif not seekable: + raise TTLibError("Input file must be seekable when lazy=True") + self._tableCache = _tableCache + self.reader = SFNTReader(file, checkChecksums, fontNumber=fontNumber) + self.sfntVersion = self.reader.sfntVersion + self.flavor = self.reader.flavor + self.flavorData = self.reader.flavorData + + def __enter__(self) -> Self: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self.close() + + def close(self) -> None: + """If we still have a reader object, close it.""" + if self.reader is not None: + self.reader.close() + self.reader = None + + def save( + self, file: str | os.PathLike[str] | BinaryIO, reorderTables: bool | None = True + ) -> None: + """Save the font to disk. + + Args: + file: Similarly to the constructor, can be either a pathname or a writable + binary file object. + reorderTables (Option[bool]): If true (the default), reorder the tables, + sorting them by tag (recommended by the OpenType specification). If + false, retain the original font order. If None, reorder by table + dependency (fastest). + """ + if not hasattr(file, "write"): + if self.lazy and self.reader.file.name == file: + raise TTLibError("Can't overwrite TTFont when 'lazy' attribute is True") + createStream = True + else: + # assume "file" is a writable file object + createStream = False + + tmp = BytesIO() + + writer_reordersTables = self._save(tmp) + + if not ( + reorderTables is None + or writer_reordersTables + or (reorderTables is False and self.reader is None) + ): + if reorderTables is False: + # sort tables using the original font's order + if self.reader is None: + raise TTLibError( + "The original table order is unavailable because there isn't a font to read it from." + ) + tableOrder = list(self.reader.keys()) + else: + # use the recommended order from the OpenType specification + tableOrder = None + tmp.flush() + tmp2 = BytesIO() + reorderFontTables(tmp, tmp2, tableOrder) + tmp.close() + tmp = tmp2 + + if createStream: + # "file" is a path + assert isinstance(file, (str, os.PathLike)) + with open(file, "wb") as file: + file.write(tmp.getvalue()) + else: + assert not isinstance(file, (str, os.PathLike)) + file.write(tmp.getvalue()) + + tmp.close() + + def _save( + self, + file: BinaryIO, + tableCache: MutableMapping[tuple[Tag, bytes], Any] | None = None, + ) -> bool: + """Internal function, to be shared by save() and TTCollection.save()""" + + if self.recalcTimestamp and "head" in self: + # make sure 'head' is loaded so the recalculation is actually done + self["head"] + + tags = self.keys() + tags.pop(0) # skip GlyphOrder tag + numTables = len(tags) + # write to a temporary stream to allow saving to unseekable streams + writer = SFNTWriter( + file, numTables, self.sfntVersion, self.flavor, self.flavorData + ) + + done = [] + for tag in tags: + self._writeTable(tag, writer, done, tableCache) + + writer.close() + + return writer.reordersTables() + + class XMLSavingOptions(TypedDict): + writeVersion: bool + quiet: bool | None + tables: Sequence[str | bytes] | None + skipTables: Sequence[str] | None + splitTables: bool + splitGlyphs: bool + disassembleInstructions: bool + bitmapGlyphDataFormat: str + + def saveXML( + self, + fileOrPath: str | os.PathLike[str] | BinaryIO | TextIO, + newlinestr: str = "\n", + **kwargs: Unpack[XMLSavingOptions], + ) -> None: + """Export the font as TTX (an XML-based text file), or as a series of text + files when splitTables is true. In the latter case, the 'fileOrPath' + argument should be a path to a directory. + The 'tables' argument must either be falsy (None or empty list, meaning + dump all tables) or a non-empty list of tables to dump. The 'skipTables' + argument may be a list of tables to skip, but only when 'tables' is falsy. + """ + + writer = xmlWriter.XMLWriter(fileOrPath, newlinestr=newlinestr) + self._saveXML(writer, **kwargs) + writer.close() + + def _saveXML( + self, + writer: xmlWriter.XMLWriter, + writeVersion: bool = True, + quiet: bool | None = None, # Deprecated + tables: Sequence[str | bytes] | None = None, + skipTables: Sequence[str] | None = None, + splitTables: bool = False, + splitGlyphs: bool = False, + disassembleInstructions: bool = True, + bitmapGlyphDataFormat: str = "raw", + ) -> None: + if quiet is not None: + deprecateArgument("quiet", "configure logging instead") + + self.disassembleInstructions = disassembleInstructions + self.bitmapGlyphDataFormat = bitmapGlyphDataFormat + if not tables: + tables = self.keys() + if skipTables: + tables = [tag for tag in tables if tag not in skipTables] + + if writeVersion: + from fontTools import version + + version = ".".join(version.split(".")[:2]) + writer.begintag( + "ttFont", + sfntVersion=repr(tostr(self.sfntVersion))[1:-1], + ttLibVersion=version, + ) + else: + writer.begintag("ttFont", sfntVersion=repr(tostr(self.sfntVersion))[1:-1]) + writer.newline() + + # always splitTables if splitGlyphs is enabled + splitTables = splitTables or splitGlyphs + + if not splitTables: + writer.newline() + else: + if writer.filename is None: + raise TTLibError( + "splitTables requires the file name to be a file system path, not a stream." + ) + path, ext = os.path.splitext(writer.filename) + + for tag in tables: + if splitTables: + tablePath = path + "." + tagToIdentifier(tag) + ext + tableWriter = xmlWriter.XMLWriter( + tablePath, newlinestr=writer.newlinestr + ) + tableWriter.begintag("ttFont", ttLibVersion=version) + tableWriter.newline() + tableWriter.newline() + writer.simpletag(tagToXML(tag), src=os.path.basename(tablePath)) + writer.newline() + else: + tableWriter = writer + self._tableToXML(tableWriter, tag, splitGlyphs=splitGlyphs) + if splitTables: + tableWriter.endtag("ttFont") + tableWriter.newline() + tableWriter.close() + writer.endtag("ttFont") + writer.newline() + + def _tableToXML( + self, + writer: xmlWriter.XMLWriter, + tag: str | bytes, + quiet: bool | None = None, + splitGlyphs: bool = False, + ) -> None: + if quiet is not None: + deprecateArgument("quiet", "configure logging instead") + if tag in self: + table = self[tag] + report = "Dumping '%s' table..." % tag + else: + report = "No '%s' table found." % tag + log.info(report) + if tag not in self: + return + xmlTag = tagToXML(tag) + attrs: dict[str, Any] = {} + if hasattr(table, "ERROR"): + attrs["ERROR"] = "decompilation error" + from .tables.DefaultTable import DefaultTable + + if table.__class__ == DefaultTable: + attrs["raw"] = True + writer.begintag(xmlTag, **attrs) + writer.newline() + if tag == "glyf": + table.toXML(writer, self, splitGlyphs=splitGlyphs) + else: + table.toXML(writer, self) + writer.endtag(xmlTag) + writer.newline() + writer.newline() + + def importXML( + self, fileOrPath: str | os.PathLike[str] | BinaryIO, quiet: bool | None = None + ) -> None: + """Import a TTX file (an XML-based text format), so as to recreate + a font object. + """ + if quiet is not None: + deprecateArgument("quiet", "configure logging instead") + + if "maxp" in self and "post" in self: + # Make sure the glyph order is loaded, as it otherwise gets + # lost if the XML doesn't contain the glyph order, yet does + # contain the table which was originally used to extract the + # glyph names from (ie. 'post', 'cmap' or 'CFF '). + self.getGlyphOrder() + + from fontTools.misc import xmlReader + + reader = xmlReader.XMLReader(fileOrPath, self) + reader.read() + + def isLoaded(self, tag: str | bytes) -> bool: + """Return true if the table identified by ``tag`` has been + decompiled and loaded into memory.""" + return tag in self.tables + + def has_key(self, tag: str | bytes) -> bool: + """Test if the table identified by ``tag`` is present in the font. + + As well as this method, ``tag in font`` can also be used to determine the + presence of the table.""" + if self.isLoaded(tag): + return True + elif self.reader and tag in self.reader: + return True + elif tag == "GlyphOrder": + return True + else: + return False + + __contains__ = has_key + + def keys(self) -> list[str]: + """Returns the list of tables in the font, along with the ``GlyphOrder`` pseudo-table.""" + keys = list(self.tables.keys()) + if self.reader: + for key in list(self.reader.keys()): + if key not in keys: + keys.append(key) + + if "GlyphOrder" in keys: + keys.remove("GlyphOrder") + keys = sortedTagList(keys) + return ["GlyphOrder"] + keys + + def ensureDecompiled(self, recurse: bool | None = None) -> None: + """Decompile all the tables, even if a TTFont was opened in 'lazy' mode.""" + for tag in self.keys(): + table = self[tag] + if recurse is None: + recurse = self.lazy is not False + if recurse and hasattr(table, "ensureDecompiled"): + table.ensureDecompiled(recurse=recurse) + self.lazy = False + + def __len__(self) -> int: + return len(list(self.keys())) + + @overload + def __getitem__(self, tag: Literal["BASE"]) -> B_A_S_E_.table_B_A_S_E_: ... + @overload + def __getitem__(self, tag: Literal["CBDT"]) -> C_B_D_T_.table_C_B_D_T_: ... + @overload + def __getitem__(self, tag: Literal["CBLC"]) -> C_B_L_C_.table_C_B_L_C_: ... + @overload + def __getitem__(self, tag: Literal["CFF "]) -> C_F_F_.table_C_F_F_: ... + @overload + def __getitem__(self, tag: Literal["CFF2"]) -> C_F_F__2.table_C_F_F__2: ... + @overload + def __getitem__(self, tag: Literal["COLR"]) -> C_O_L_R_.table_C_O_L_R_: ... + @overload + def __getitem__(self, tag: Literal["CPAL"]) -> C_P_A_L_.table_C_P_A_L_: ... + @overload + def __getitem__(self, tag: Literal["DSIG"]) -> D_S_I_G_.table_D_S_I_G_: ... + @overload + def __getitem__(self, tag: Literal["EBDT"]) -> E_B_D_T_.table_E_B_D_T_: ... + @overload + def __getitem__(self, tag: Literal["EBLC"]) -> E_B_L_C_.table_E_B_L_C_: ... + @overload + def __getitem__(self, tag: Literal["FFTM"]) -> F_F_T_M_.table_F_F_T_M_: ... + @overload + def __getitem__(self, tag: Literal["GDEF"]) -> G_D_E_F_.table_G_D_E_F_: ... + @overload + def __getitem__(self, tag: Literal["GPOS"]) -> G_P_O_S_.table_G_P_O_S_: ... + @overload + def __getitem__(self, tag: Literal["GSUB"]) -> G_S_U_B_.table_G_S_U_B_: ... + @overload + def __getitem__(self, tag: Literal["GVAR"]) -> G_V_A_R_.table_G_V_A_R_: ... + @overload + def __getitem__(self, tag: Literal["HVAR"]) -> H_V_A_R_.table_H_V_A_R_: ... + @overload + def __getitem__(self, tag: Literal["IFT "]) -> I_F_T_.table_I_F_T_: ... + @overload + def __getitem__(self, tag: Literal["IFTX"]) -> I_F_T_X_.table_I_F_T_X_: ... + @overload + def __getitem__(self, tag: Literal["JSTF"]) -> J_S_T_F_.table_J_S_T_F_: ... + @overload + def __getitem__(self, tag: Literal["LTSH"]) -> L_T_S_H_.table_L_T_S_H_: ... + @overload + def __getitem__(self, tag: Literal["MATH"]) -> M_A_T_H_.table_M_A_T_H_: ... + @overload + def __getitem__(self, tag: Literal["MVAR"]) -> M_V_A_R_.table_M_V_A_R_: ... + @overload + def __getitem__(self, tag: Literal["STAT"]) -> S_T_A_T_.table_S_T_A_T_: ... + @overload + def __getitem__(self, tag: Literal["SVG "]) -> S_V_G_.table_S_V_G_: ... + @overload + def __getitem__(self, tag: Literal["TSI0"]) -> T_S_I__0.table_T_S_I__0: ... + @overload + def __getitem__(self, tag: Literal["TSI1"]) -> T_S_I__1.table_T_S_I__1: ... + @overload + def __getitem__(self, tag: Literal["TSI2"]) -> T_S_I__2.table_T_S_I__2: ... + @overload + def __getitem__(self, tag: Literal["TSI3"]) -> T_S_I__3.table_T_S_I__3: ... + @overload + def __getitem__(self, tag: Literal["TSI5"]) -> T_S_I__5.table_T_S_I__5: ... + @overload + def __getitem__(self, tag: Literal["TSIB"]) -> T_S_I_B_.table_T_S_I_B_: ... + @overload + def __getitem__(self, tag: Literal["TSIC"]) -> T_S_I_C_.table_T_S_I_C_: ... + @overload + def __getitem__(self, tag: Literal["TSID"]) -> T_S_I_D_.table_T_S_I_D_: ... + @overload + def __getitem__(self, tag: Literal["TSIJ"]) -> T_S_I_J_.table_T_S_I_J_: ... + @overload + def __getitem__(self, tag: Literal["TSIP"]) -> T_S_I_P_.table_T_S_I_P_: ... + @overload + def __getitem__(self, tag: Literal["TSIS"]) -> T_S_I_S_.table_T_S_I_S_: ... + @overload + def __getitem__(self, tag: Literal["TSIV"]) -> T_S_I_V_.table_T_S_I_V_: ... + @overload + def __getitem__(self, tag: Literal["TTFA"]) -> T_T_F_A_.table_T_T_F_A_: ... + @overload + def __getitem__(self, tag: Literal["VARC"]) -> V_A_R_C_.table_V_A_R_C_: ... + @overload + def __getitem__(self, tag: Literal["VDMX"]) -> V_D_M_X_.table_V_D_M_X_: ... + @overload + def __getitem__(self, tag: Literal["VORG"]) -> V_O_R_G_.table_V_O_R_G_: ... + @overload + def __getitem__(self, tag: Literal["VVAR"]) -> V_V_A_R_.table_V_V_A_R_: ... + @overload + def __getitem__(self, tag: Literal["Debg"]) -> D__e_b_g.table_D__e_b_g: ... + @overload + def __getitem__(self, tag: Literal["Feat"]) -> F__e_a_t.table_F__e_a_t: ... + @overload + def __getitem__(self, tag: Literal["Glat"]) -> G__l_a_t.table_G__l_a_t: ... + @overload + def __getitem__(self, tag: Literal["Gloc"]) -> G__l_o_c.table_G__l_o_c: ... + @overload + def __getitem__(self, tag: Literal["OS/2"]) -> O_S_2f_2.table_O_S_2f_2: ... + @overload + def __getitem__(self, tag: Literal["Silf"]) -> S__i_l_f.table_S__i_l_f: ... + @overload + def __getitem__(self, tag: Literal["Sill"]) -> S__i_l_l.table_S__i_l_l: ... + @overload + def __getitem__(self, tag: Literal["ankr"]) -> _a_n_k_r.table__a_n_k_r: ... + @overload + def __getitem__(self, tag: Literal["avar"]) -> _a_v_a_r.table__a_v_a_r: ... + @overload + def __getitem__(self, tag: Literal["bsln"]) -> _b_s_l_n.table__b_s_l_n: ... + @overload + def __getitem__(self, tag: Literal["bgcl"]) -> _b_g_c_l.table__b_g_c_l: ... + @overload + def __getitem__(self, tag: Literal["cidg"]) -> _c_i_d_g.table__c_i_d_g: ... + @overload + def __getitem__(self, tag: Literal["cmap"]) -> _c_m_a_p.table__c_m_a_p: ... + @overload + def __getitem__(self, tag: Literal["cvar"]) -> _c_v_a_r.table__c_v_a_r: ... + @overload + def __getitem__(self, tag: Literal["cvt "]) -> _c_v_t.table__c_v_t: ... + @overload + def __getitem__(self, tag: Literal["feat"]) -> _f_e_a_t.table__f_e_a_t: ... + @overload + def __getitem__(self, tag: Literal["fpgm"]) -> _f_p_g_m.table__f_p_g_m: ... + @overload + def __getitem__(self, tag: Literal["fvar"]) -> _f_v_a_r.table__f_v_a_r: ... + @overload + def __getitem__(self, tag: Literal["gasp"]) -> _g_a_s_p.table__g_a_s_p: ... + @overload + def __getitem__(self, tag: Literal["gcid"]) -> _g_c_i_d.table__g_c_i_d: ... + @overload + def __getitem__(self, tag: Literal["glyf"]) -> _g_l_y_f.table__g_l_y_f: ... + @overload + def __getitem__(self, tag: Literal["gvar"]) -> _g_v_a_r.table__g_v_a_r: ... + @overload + def __getitem__(self, tag: Literal["hdmx"]) -> _h_d_m_x.table__h_d_m_x: ... + @overload + def __getitem__(self, tag: Literal["head"]) -> _h_e_a_d.table__h_e_a_d: ... + @overload + def __getitem__(self, tag: Literal["hhea"]) -> _h_h_e_a.table__h_h_e_a: ... + @overload + def __getitem__(self, tag: Literal["hmtx"]) -> _h_m_t_x.table__h_m_t_x: ... + @overload + def __getitem__(self, tag: Literal["kern"]) -> _k_e_r_n.table__k_e_r_n: ... + @overload + def __getitem__(self, tag: Literal["lcar"]) -> _l_c_a_r.table__l_c_a_r: ... + @overload + def __getitem__(self, tag: Literal["loca"]) -> _l_o_c_a.table__l_o_c_a: ... + @overload + def __getitem__(self, tag: Literal["ltag"]) -> _l_t_a_g.table__l_t_a_g: ... + @overload + def __getitem__(self, tag: Literal["maxp"]) -> _m_a_x_p.table__m_a_x_p: ... + @overload + def __getitem__(self, tag: Literal["meta"]) -> _m_e_t_a.table__m_e_t_a: ... + @overload + def __getitem__(self, tag: Literal["mort"]) -> _m_o_r_t.table__m_o_r_t: ... + @overload + def __getitem__(self, tag: Literal["morx"]) -> _m_o_r_x.table__m_o_r_x: ... + @overload + def __getitem__(self, tag: Literal["name"]) -> _n_a_m_e.table__n_a_m_e: ... + @overload + def __getitem__(self, tag: Literal["opbd"]) -> _o_p_b_d.table__o_p_b_d: ... + @overload + def __getitem__(self, tag: Literal["post"]) -> _p_o_s_t.table__p_o_s_t: ... + @overload + def __getitem__(self, tag: Literal["prep"]) -> _p_r_e_p.table__p_r_e_p: ... + @overload + def __getitem__(self, tag: Literal["prop"]) -> _p_r_o_p.table__p_r_o_p: ... + @overload + def __getitem__(self, tag: Literal["sbix"]) -> _s_b_i_x.table__s_b_i_x: ... + @overload + def __getitem__(self, tag: Literal["trak"]) -> _t_r_a_k.table__t_r_a_k: ... + @overload + def __getitem__(self, tag: Literal["vhea"]) -> _v_h_e_a.table__v_h_e_a: ... + @overload + def __getitem__(self, tag: Literal["vmtx"]) -> _v_m_t_x.table__v_m_t_x: ... + @overload + def __getitem__(self, tag: Literal["GlyphOrder"]) -> GlyphOrder: ... + @overload + def __getitem__(self, tag: str | bytes) -> DefaultTable | GlyphOrder: ... + + def __getitem__(self, tag: str | bytes) -> DefaultTable | GlyphOrder: + tag = Tag(tag) + table = self.tables.get(tag) + if table is None: + if tag == "GlyphOrder": + table = GlyphOrder(tag) + self.tables[tag] = table + elif self.reader is not None: + table = self._readTable(tag) + else: + raise KeyError("'%s' table not found" % tag) + return table + + def _readTable(self, tag: Tag) -> DefaultTable: + log.debug("Reading '%s' table from disk", tag) + assert self.reader is not None + data = self.reader[tag] + if self._tableCache is not None: + table = self._tableCache.get((tag, data)) + if table is not None: + return table + tableClass = getTableClass(tag) + table = tableClass(tag) + self.tables[tag] = table + log.debug("Decompiling '%s' table", tag) + try: + table.decompile(data, self) + except Exception: + if not self.ignoreDecompileErrors: + raise + # fall back to DefaultTable, retaining the binary table data + log.exception( + "An exception occurred during the decompilation of the '%s' table", tag + ) + from .tables.DefaultTable import DefaultTable + + file = StringIO() + traceback.print_exc(file=file) + table = DefaultTable(tag) + table.ERROR = file.getvalue() + self.tables[tag] = table + table.decompile(data, self) + if self._tableCache is not None: + self._tableCache[(tag, data)] = table + return table + + def __setitem__(self, tag: str | bytes, table: DefaultTable) -> None: + self.tables[Tag(tag)] = table + + def __delitem__(self, tag: str | bytes) -> None: + if tag not in self: + raise KeyError("'%s' table not found" % tag) + if tag in self.tables: + del self.tables[tag] + if self.reader and tag in self.reader: + del self.reader[tag] + + @overload + def get(self, tag: Literal["BASE"]) -> B_A_S_E_.table_B_A_S_E_ | None: ... + @overload + def get(self, tag: Literal["CBDT"]) -> C_B_D_T_.table_C_B_D_T_ | None: ... + @overload + def get(self, tag: Literal["CBLC"]) -> C_B_L_C_.table_C_B_L_C_ | None: ... + @overload + def get(self, tag: Literal["CFF "]) -> C_F_F_.table_C_F_F_ | None: ... + @overload + def get(self, tag: Literal["CFF2"]) -> C_F_F__2.table_C_F_F__2 | None: ... + @overload + def get(self, tag: Literal["COLR"]) -> C_O_L_R_.table_C_O_L_R_ | None: ... + @overload + def get(self, tag: Literal["CPAL"]) -> C_P_A_L_.table_C_P_A_L_ | None: ... + @overload + def get(self, tag: Literal["DSIG"]) -> D_S_I_G_.table_D_S_I_G_ | None: ... + @overload + def get(self, tag: Literal["EBDT"]) -> E_B_D_T_.table_E_B_D_T_ | None: ... + @overload + def get(self, tag: Literal["EBLC"]) -> E_B_L_C_.table_E_B_L_C_ | None: ... + @overload + def get(self, tag: Literal["FFTM"]) -> F_F_T_M_.table_F_F_T_M_ | None: ... + @overload + def get(self, tag: Literal["GDEF"]) -> G_D_E_F_.table_G_D_E_F_ | None: ... + @overload + def get(self, tag: Literal["GPOS"]) -> G_P_O_S_.table_G_P_O_S_ | None: ... + @overload + def get(self, tag: Literal["GSUB"]) -> G_S_U_B_.table_G_S_U_B_ | None: ... + @overload + def get(self, tag: Literal["GVAR"]) -> G_V_A_R_.table_G_V_A_R_ | None: ... + @overload + def get(self, tag: Literal["HVAR"]) -> H_V_A_R_.table_H_V_A_R_ | None: ... + @overload + def get(self, tag: Literal["IFT "]) -> I_F_T_.table_I_F_T_ | None: ... + @overload + def get(self, tag: Literal["IFTX"]) -> I_F_T_X_.table_I_F_T_X_ | None: ... + @overload + def get(self, tag: Literal["JSTF"]) -> J_S_T_F_.table_J_S_T_F_ | None: ... + @overload + def get(self, tag: Literal["LTSH"]) -> L_T_S_H_.table_L_T_S_H_ | None: ... + @overload + def get(self, tag: Literal["MATH"]) -> M_A_T_H_.table_M_A_T_H_ | None: ... + @overload + def get(self, tag: Literal["MVAR"]) -> M_V_A_R_.table_M_V_A_R_ | None: ... + @overload + def get(self, tag: Literal["STAT"]) -> S_T_A_T_.table_S_T_A_T_ | None: ... + @overload + def get(self, tag: Literal["SVG "]) -> S_V_G_.table_S_V_G_ | None: ... + @overload + def get(self, tag: Literal["TSI0"]) -> T_S_I__0.table_T_S_I__0 | None: ... + @overload + def get(self, tag: Literal["TSI1"]) -> T_S_I__1.table_T_S_I__1 | None: ... + @overload + def get(self, tag: Literal["TSI2"]) -> T_S_I__2.table_T_S_I__2 | None: ... + @overload + def get(self, tag: Literal["TSI3"]) -> T_S_I__3.table_T_S_I__3 | None: ... + @overload + def get(self, tag: Literal["TSI5"]) -> T_S_I__5.table_T_S_I__5 | None: ... + @overload + def get(self, tag: Literal["TSIB"]) -> T_S_I_B_.table_T_S_I_B_ | None: ... + @overload + def get(self, tag: Literal["TSIC"]) -> T_S_I_C_.table_T_S_I_C_ | None: ... + @overload + def get(self, tag: Literal["TSID"]) -> T_S_I_D_.table_T_S_I_D_ | None: ... + @overload + def get(self, tag: Literal["TSIJ"]) -> T_S_I_J_.table_T_S_I_J_ | None: ... + @overload + def get(self, tag: Literal["TSIP"]) -> T_S_I_P_.table_T_S_I_P_ | None: ... + @overload + def get(self, tag: Literal["TSIS"]) -> T_S_I_S_.table_T_S_I_S_ | None: ... + @overload + def get(self, tag: Literal["TSIV"]) -> T_S_I_V_.table_T_S_I_V_ | None: ... + @overload + def get(self, tag: Literal["TTFA"]) -> T_T_F_A_.table_T_T_F_A_ | None: ... + @overload + def get(self, tag: Literal["VARC"]) -> V_A_R_C_.table_V_A_R_C_ | None: ... + @overload + def get(self, tag: Literal["VDMX"]) -> V_D_M_X_.table_V_D_M_X_ | None: ... + @overload + def get(self, tag: Literal["VORG"]) -> V_O_R_G_.table_V_O_R_G_ | None: ... + @overload + def get(self, tag: Literal["VVAR"]) -> V_V_A_R_.table_V_V_A_R_ | None: ... + @overload + def get(self, tag: Literal["Debg"]) -> D__e_b_g.table_D__e_b_g | None: ... + @overload + def get(self, tag: Literal["Feat"]) -> F__e_a_t.table_F__e_a_t | None: ... + @overload + def get(self, tag: Literal["Glat"]) -> G__l_a_t.table_G__l_a_t | None: ... + @overload + def get(self, tag: Literal["Gloc"]) -> G__l_o_c.table_G__l_o_c | None: ... + @overload + def get(self, tag: Literal["OS/2"]) -> O_S_2f_2.table_O_S_2f_2 | None: ... + @overload + def get(self, tag: Literal["Silf"]) -> S__i_l_f.table_S__i_l_f | None: ... + @overload + def get(self, tag: Literal["Sill"]) -> S__i_l_l.table_S__i_l_l | None: ... + @overload + def get(self, tag: Literal["ankr"]) -> _a_n_k_r.table__a_n_k_r | None: ... + @overload + def get(self, tag: Literal["avar"]) -> _a_v_a_r.table__a_v_a_r | None: ... + @overload + def get(self, tag: Literal["bsln"]) -> _b_s_l_n.table__b_s_l_n | None: ... + @overload + def get(self, tag: Literal["bgcl"]) -> _b_g_c_l.table__b_g_c_l | None: ... + @overload + def get(self, tag: Literal["cidg"]) -> _c_i_d_g.table__c_i_d_g | None: ... + @overload + def get(self, tag: Literal["cmap"]) -> _c_m_a_p.table__c_m_a_p | None: ... + @overload + def get(self, tag: Literal["cvar"]) -> _c_v_a_r.table__c_v_a_r | None: ... + @overload + def get(self, tag: Literal["cvt "]) -> _c_v_t.table__c_v_t | None: ... + @overload + def get(self, tag: Literal["feat"]) -> _f_e_a_t.table__f_e_a_t | None: ... + @overload + def get(self, tag: Literal["fpgm"]) -> _f_p_g_m.table__f_p_g_m | None: ... + @overload + def get(self, tag: Literal["fvar"]) -> _f_v_a_r.table__f_v_a_r | None: ... + @overload + def get(self, tag: Literal["gasp"]) -> _g_a_s_p.table__g_a_s_p | None: ... + @overload + def get(self, tag: Literal["gcid"]) -> _g_c_i_d.table__g_c_i_d | None: ... + @overload + def get(self, tag: Literal["glyf"]) -> _g_l_y_f.table__g_l_y_f | None: ... + @overload + def get(self, tag: Literal["gvar"]) -> _g_v_a_r.table__g_v_a_r | None: ... + @overload + def get(self, tag: Literal["hdmx"]) -> _h_d_m_x.table__h_d_m_x | None: ... + @overload + def get(self, tag: Literal["head"]) -> _h_e_a_d.table__h_e_a_d | None: ... + @overload + def get(self, tag: Literal["hhea"]) -> _h_h_e_a.table__h_h_e_a | None: ... + @overload + def get(self, tag: Literal["hmtx"]) -> _h_m_t_x.table__h_m_t_x | None: ... + @overload + def get(self, tag: Literal["kern"]) -> _k_e_r_n.table__k_e_r_n | None: ... + @overload + def get(self, tag: Literal["lcar"]) -> _l_c_a_r.table__l_c_a_r | None: ... + @overload + def get(self, tag: Literal["loca"]) -> _l_o_c_a.table__l_o_c_a | None: ... + @overload + def get(self, tag: Literal["ltag"]) -> _l_t_a_g.table__l_t_a_g | None: ... + @overload + def get(self, tag: Literal["maxp"]) -> _m_a_x_p.table__m_a_x_p | None: ... + @overload + def get(self, tag: Literal["meta"]) -> _m_e_t_a.table__m_e_t_a | None: ... + @overload + def get(self, tag: Literal["mort"]) -> _m_o_r_t.table__m_o_r_t | None: ... + @overload + def get(self, tag: Literal["morx"]) -> _m_o_r_x.table__m_o_r_x | None: ... + @overload + def get(self, tag: Literal["name"]) -> _n_a_m_e.table__n_a_m_e | None: ... + @overload + def get(self, tag: Literal["opbd"]) -> _o_p_b_d.table__o_p_b_d | None: ... + @overload + def get(self, tag: Literal["post"]) -> _p_o_s_t.table__p_o_s_t | None: ... + @overload + def get(self, tag: Literal["prep"]) -> _p_r_e_p.table__p_r_e_p | None: ... + @overload + def get(self, tag: Literal["prop"]) -> _p_r_o_p.table__p_r_o_p | None: ... + @overload + def get(self, tag: Literal["sbix"]) -> _s_b_i_x.table__s_b_i_x | None: ... + @overload + def get(self, tag: Literal["trak"]) -> _t_r_a_k.table__t_r_a_k | None: ... + @overload + def get(self, tag: Literal["vhea"]) -> _v_h_e_a.table__v_h_e_a | None: ... + @overload + def get(self, tag: Literal["vmtx"]) -> _v_m_t_x.table__v_m_t_x | None: ... + @overload + def get(self, tag: Literal["GlyphOrder"]) -> GlyphOrder: ... + @overload + def get(self, tag: str | bytes) -> DefaultTable | GlyphOrder | Any | None: ... + @overload + def get( + self, tag: str | bytes, default: _VT_co + ) -> DefaultTable | GlyphOrder | Any | _VT_co: ... + + def get( + self, tag: str | bytes, default: Any | None = None + ) -> DefaultTable | GlyphOrder | Any | None: + """Returns the table if it exists or (optionally) a default if it doesn't.""" + try: + return self[tag] + except KeyError: + return default + + def setGlyphOrder(self, glyphOrder: list[str]) -> None: + """Set the glyph order + + Args: + glyphOrder ([str]): List of glyph names in order. + """ + self.glyphOrder = glyphOrder + if hasattr(self, "_reverseGlyphOrderDict"): + del self._reverseGlyphOrderDict + if self.isLoaded("glyf"): + self["glyf"].setGlyphOrder(glyphOrder) + + def getGlyphOrder(self) -> list[str]: + """Returns a list of glyph names ordered by their position in the font.""" + try: + return self.glyphOrder + except AttributeError: + pass + if "CFF " in self: + cff = self["CFF "] + self.glyphOrder = cff.getGlyphOrder() + elif "post" in self: + # TrueType font + glyphOrder = self["post"].getGlyphOrder() + if glyphOrder is None: + # + # No names found in the 'post' table. + # Try to create glyph names from the unicode cmap (if available) + # in combination with the Adobe Glyph List (AGL). + # + self._getGlyphNamesFromCmap() + elif len(glyphOrder) < self["maxp"].numGlyphs: + # + # Not enough names found in the 'post' table. + # Can happen when 'post' format 1 is improperly used on a font that + # has more than 258 glyphs (the length of 'standardGlyphOrder'). + # + log.warning( + "Not enough names found in the 'post' table, generating them from cmap instead" + ) + self._getGlyphNamesFromCmap() + else: + self.glyphOrder = glyphOrder + else: + self._getGlyphNamesFromCmap() + return self.glyphOrder + + def _getGlyphNamesFromCmap(self) -> None: + # + # This is rather convoluted, but then again, it's an interesting problem: + # - we need to use the unicode values found in the cmap table to + # build glyph names (eg. because there is only a minimal post table, + # or none at all). + # - but the cmap parser also needs glyph names to work with... + # So here's what we do: + # - make up glyph names based on glyphID + # - load a temporary cmap table based on those names + # - extract the unicode values, build the "real" glyph names + # - unload the temporary cmap table + # + if self.isLoaded("cmap"): + # Bootstrapping: we're getting called by the cmap parser + # itself. This means self.tables['cmap'] contains a partially + # loaded cmap, making it impossible to get at a unicode + # subtable here. We remove the partially loaded cmap and + # restore it later. + # This only happens if the cmap table is loaded before any + # other table that does f.getGlyphOrder() or f.getGlyphName(). + cmapLoading = self.tables["cmap"] + del self.tables["cmap"] + else: + cmapLoading = None + # Make up glyph names based on glyphID, which will be used by the + # temporary cmap and by the real cmap in case we don't find a unicode + # cmap. + numGlyphs = int(self["maxp"].numGlyphs) + glyphOrder = ["glyph%.5d" % i for i in range(numGlyphs)] + glyphOrder[0] = ".notdef" + # Set the glyph order, so the cmap parser has something + # to work with (so we don't get called recursively). + self.glyphOrder = glyphOrder + + # Make up glyph names based on the reversed cmap table. Because some + # glyphs (eg. ligatures or alternates) may not be reachable via cmap, + # this naming table will usually not cover all glyphs in the font. + # If the font has no Unicode cmap table, reversecmap will be empty. + if "cmap" in self: + reversecmap = self["cmap"].buildReversedMin() + else: + reversecmap = {} + useCount = {} + for i, tempName in enumerate(glyphOrder): + if tempName in reversecmap: + # If a font maps both U+0041 LATIN CAPITAL LETTER A and + # U+0391 GREEK CAPITAL LETTER ALPHA to the same glyph, + # we prefer naming the glyph as "A". + glyphName = self._makeGlyphName(reversecmap[tempName]) + numUses = useCount[glyphName] = useCount.get(glyphName, 0) + 1 + if numUses > 1: + glyphName = "%s.alt%d" % (glyphName, numUses - 1) + glyphOrder[i] = glyphName + + if "cmap" in self: + # Delete the temporary cmap table from the cache, so it can + # be parsed again with the right names. + del self.tables["cmap"] + self.glyphOrder = glyphOrder + if cmapLoading: + # restore partially loaded cmap, so it can continue loading + # using the proper names. + self.tables["cmap"] = cmapLoading + + @staticmethod + def _makeGlyphName(codepoint: int) -> str: + from fontTools import agl # Adobe Glyph List + + if codepoint in agl.UV2AGL: + return agl.UV2AGL[codepoint] + elif codepoint <= 0xFFFF: + return "uni%04X" % codepoint + else: + return "u%X" % codepoint + + def getGlyphNames(self) -> list[str]: + """Get a list of glyph names, sorted alphabetically.""" + glyphNames = sorted(self.getGlyphOrder()) + return glyphNames + + def getGlyphNames2(self) -> list[str]: + """Get a list of glyph names, sorted alphabetically, + but not case sensitive. + """ + from fontTools.misc import textTools + + return textTools.caselessSort(self.getGlyphOrder()) + + def getGlyphName(self, glyphID: int) -> str: + """Returns the name for the glyph with the given ID. + + If no name is available, synthesises one with the form ``glyphXXXXX``` where + ```XXXXX`` is the zero-padded glyph ID. + """ + try: + return self.getGlyphOrder()[glyphID] + except IndexError: + return "glyph%.5d" % glyphID + + def getGlyphNameMany(self, lst: Sequence[int]) -> list[str]: + """Converts a list of glyph IDs into a list of glyph names.""" + glyphOrder = self.getGlyphOrder() + cnt = len(glyphOrder) + return [glyphOrder[gid] if gid < cnt else "glyph%.5d" % gid for gid in lst] + + def getGlyphID(self, glyphName: str) -> int: + """Returns the ID of the glyph with the given name.""" + try: + return self.getReverseGlyphMap()[glyphName] + except KeyError: + if glyphName[:5] == "glyph": + try: + return int(glyphName[5:]) + except (NameError, ValueError): + raise KeyError(glyphName) + raise + + def getGlyphIDMany(self, lst: Sequence[str]) -> list[int]: + """Converts a list of glyph names into a list of glyph IDs.""" + d = self.getReverseGlyphMap() + try: + return [d[glyphName] for glyphName in lst] + except KeyError: + getGlyphID = self.getGlyphID + return [getGlyphID(glyphName) for glyphName in lst] + + def getReverseGlyphMap(self, rebuild: bool = False) -> dict[str, int]: + """Returns a mapping of glyph names to glyph IDs.""" + if rebuild or not hasattr(self, "_reverseGlyphOrderDict"): + self._buildReverseGlyphOrderDict() + return self._reverseGlyphOrderDict + + def _buildReverseGlyphOrderDict(self) -> dict[str, int]: + self._reverseGlyphOrderDict = d = {} + for glyphID, glyphName in enumerate(self.getGlyphOrder()): + d[glyphName] = glyphID + return d + + def _writeTable( + self, + tag: str | bytes, + writer: SFNTWriter, + done: list[str | bytes], # Use list as original + tableCache: MutableMapping[tuple[Tag, bytes], DefaultTable] | None = None, + ) -> None: + """Internal helper function for self.save(). Keeps track of + inter-table dependencies. + """ + if tag in done: + return + tableClass = getTableClass(tag) + for masterTable in tableClass.dependencies: + if masterTable not in done: + if masterTable in self: + self._writeTable(masterTable, writer, done, tableCache) + else: + done.append(masterTable) + done.append(tag) + tabledata = self.getTableData(tag) + if tableCache is not None: + entry = tableCache.get((Tag(tag), tabledata)) + if entry is not None: + log.debug("reusing '%s' table", tag) + writer.setEntry(tag, entry) + return + log.debug("Writing '%s' table to disk", tag) + writer[tag] = tabledata + if tableCache is not None: + tableCache[(Tag(tag), tabledata)] = writer[tag] + + def getTableData(self, tag: str | bytes) -> bytes: + """Returns the binary representation of a table. + + If the table is currently loaded and in memory, the data is compiled to + binary and returned; if it is not currently loaded, the binary data is + read from the font file and returned. + """ + tag = Tag(tag) + if self.isLoaded(tag): + log.debug("Compiling '%s' table", tag) + return self.tables[tag].compile(self) + elif self.reader and tag in self.reader: + log.debug("Reading '%s' table from disk", tag) + return self.reader[tag] + else: + raise KeyError(tag) + + def getGlyphSet( + self, + preferCFF: bool = True, + location: Mapping[str, _NumberT] | None = None, + normalized: bool = False, + recalcBounds: bool = True, + ) -> _TTGlyphSet: + """Return a generic GlyphSet, which is a dict-like object + mapping glyph names to glyph objects. The returned glyph objects + have a ``.draw()`` method that supports the Pen protocol, and will + have an attribute named 'width'. + + If the font is CFF-based, the outlines will be taken from the ``CFF `` + or ``CFF2`` tables. Otherwise the outlines will be taken from the + ``glyf`` table. + + If the font contains both a ``CFF ``/``CFF2`` and a ``glyf`` table, you + can use the ``preferCFF`` argument to specify which one should be taken. + If the font contains both a ``CFF `` and a ``CFF2`` table, the latter is + taken. + + If the ``location`` parameter is set, it should be a dictionary mapping + four-letter variation tags to their float values, and the returned + glyph-set will represent an instance of a variable font at that + location. + + If the ``normalized`` variable is set to True, that location is + interpreted as in the normalized (-1..+1) space, otherwise it is in the + font's defined axes space. + """ + if location and "fvar" not in self: + location = None + if location and not normalized: + location = self.normalizeLocation(location) + glyphSet = None + if ("CFF " in self or "CFF2" in self) and (preferCFF or "glyf" not in self): + glyphSet = _TTGlyphSetCFF(self, location) + elif "glyf" in self: + glyphSet = _TTGlyphSetGlyf(self, location, recalcBounds=recalcBounds) + else: + raise TTLibError("Font contains no outlines") + if "VARC" in self: + glyphSet = _TTGlyphSetVARC(self, location, glyphSet) + return glyphSet + + def normalizeLocation(self, location: Mapping[str, float]) -> dict[str, float]: + """Normalize a ``location`` from the font's defined axes space (also + known as user space) into the normalized (-1..+1) space. It applies + ``avar`` mapping if the font contains an ``avar`` table. + + The ``location`` parameter should be a dictionary mapping four-letter + variation tags to their float values. + + Raises ``TTLibError`` if the font is not a variable font. + """ + from fontTools.varLib.models import normalizeLocation + + if "fvar" not in self: + raise TTLibError("Not a variable font") + + axes = self["fvar"].getAxes() + location = normalizeLocation(location, axes) + if "avar" in self: + location = self["avar"].renormalizeLocation(location, self) + return location + + def getBestCmap( + self, + cmapPreferences: Sequence[tuple[int, int]] = ( + (3, 10), + (0, 6), + (0, 4), + (3, 1), + (0, 3), + (0, 2), + (0, 1), + (0, 0), + ), + ) -> dict[int, str] | None: + """Returns the 'best' Unicode cmap dictionary available in the font + or ``None``, if no Unicode cmap subtable is available. + + By default it will search for the following (platformID, platEncID) + pairs in order:: + + (3, 10), # Windows Unicode full repertoire + (0, 6), # Unicode full repertoire (format 13 subtable) + (0, 4), # Unicode 2.0 full repertoire + (3, 1), # Windows Unicode BMP + (0, 3), # Unicode 2.0 BMP + (0, 2), # Unicode ISO/IEC 10646 + (0, 1), # Unicode 1.1 + (0, 0) # Unicode 1.0 + + This particular order matches what HarfBuzz uses to choose what + subtable to use by default. This order prefers the largest-repertoire + subtable, and among those, prefers the Windows-platform over the + Unicode-platform as the former has wider support. + + This order can be customized via the ``cmapPreferences`` argument. + """ + return self["cmap"].getBestCmap(cmapPreferences=cmapPreferences) + + def reorderGlyphs(self, new_glyph_order: list[str]) -> None: + from .reorderGlyphs import reorderGlyphs + + reorderGlyphs(self, new_glyph_order) + + +class GlyphOrder(object): + """A pseudo table. The glyph order isn't in the font as a separate + table, but it's nice to present it as such in the TTX format. + """ + + def __init__(self, tag: str | None = None) -> None: + pass + + def toXML(self, writer: xmlWriter.XMLWriter, ttFont: TTFont) -> None: + glyphOrder = ttFont.getGlyphOrder() + writer.comment( + "The 'id' attribute is only for humans; it is ignored when parsed." + ) + writer.newline() + for i, glyphName in enumerate(glyphOrder): + writer.simpletag("GlyphID", id=i, name=glyphName) + writer.newline() + + def fromXML( + self, name: str, attrs: dict[str, str], content: list[Any], ttFont: TTFont + ) -> None: + if not hasattr(self, "glyphOrder"): + self.glyphOrder = [] + if name == "GlyphID": + self.glyphOrder.append(attrs["name"]) + ttFont.setGlyphOrder(self.glyphOrder) + + +def getTableModule(tag: str | bytes) -> ModuleType | None: + """Fetch the packer/unpacker module for a table. + Return None when no module is found. + """ + from . import tables + + pyTag = tagToIdentifier(tag) + try: + __import__("fontTools.ttLib.tables." + pyTag) + except ImportError as err: + # If pyTag is found in the ImportError message, + # means table is not implemented. If it's not + # there, then some other module is missing, don't + # suppress the error. + if str(err).find(pyTag) >= 0: + return None + else: + raise err + else: + return getattr(tables, pyTag) + + +# Registry for custom table packer/unpacker classes. Keys are table +# tags, values are (moduleName, className) tuples. +# See registerCustomTableClass() and getCustomTableClass() +_customTableRegistry: dict[str | bytes, tuple[str, str]] = {} + + +def registerCustomTableClass( + tag: str | bytes, moduleName: str, className: str | None = None +) -> None: + """Register a custom packer/unpacker class for a table. + + The 'moduleName' must be an importable module. If no 'className' + is given, it is derived from the tag, for example it will be + ``table_C_U_S_T_`` for a 'CUST' tag. + + The registered table class should be a subclass of + :py:class:`fontTools.ttLib.tables.DefaultTable.DefaultTable` + """ + if className is None: + className = "table_" + tagToIdentifier(tag) + _customTableRegistry[tag] = (moduleName, className) + + +def unregisterCustomTableClass(tag: str | bytes) -> None: + """Unregister the custom packer/unpacker class for a table.""" + del _customTableRegistry[tag] + + +def getCustomTableClass(tag: str | bytes) -> type[DefaultTable] | None: + """Return the custom table class for tag, if one has been registered + with 'registerCustomTableClass()'. Else return None. + """ + if tag not in _customTableRegistry: + return None + import importlib + + moduleName, className = _customTableRegistry[tag] + module = importlib.import_module(moduleName) + return getattr(module, className) + + +def getTableClass(tag: str | bytes) -> type[DefaultTable]: + """Fetch the packer/unpacker class for a table.""" + tableClass = getCustomTableClass(tag) + if tableClass is not None: + return tableClass + module = getTableModule(tag) + if module is None: + from .tables.DefaultTable import DefaultTable + + return DefaultTable + pyTag = tagToIdentifier(tag) + tableClass = getattr(module, "table_" + pyTag) + return tableClass + + +def getClassTag(klass: type[DefaultTable]) -> str | bytes: + """Fetch the table tag for a class object.""" + name = klass.__name__ + assert name[:6] == "table_" + name = name[6:] # Chop 'table_' + return identifierToTag(name) + + +def newTable(tag: str | bytes) -> DefaultTable: + """Return a new instance of a table.""" + tableClass = getTableClass(tag) + return tableClass(tag) + + +def _escapechar(c: str) -> str: + """Helper function for tagToIdentifier()""" + import re + + if re.match("[a-z0-9]", c): + return "_" + c + elif re.match("[A-Z]", c): + return c + "_" + else: + return hex(byteord(c))[2:] + + +def tagToIdentifier(tag: str | bytes) -> str: + """Convert a table tag to a valid (but UGLY) python identifier, + as well as a filename that's guaranteed to be unique even on a + caseless file system. Each character is mapped to two characters. + Lowercase letters get an underscore before the letter, uppercase + letters get an underscore after the letter. Trailing spaces are + trimmed. Illegal characters are escaped as two hex bytes. If the + result starts with a number (as the result of a hex escape), an + extra underscore is prepended. Examples: + .. code-block:: pycon + + >>> + >> tagToIdentifier('glyf') + '_g_l_y_f' + >> tagToIdentifier('cvt ') + '_c_v_t' + >> tagToIdentifier('OS/2') + 'O_S_2f_2' + """ + import re + + tag = Tag(tag) + if tag == "GlyphOrder": + return tag + assert len(tag) == 4, "tag should be 4 characters long" + while len(tag) > 1 and tag[-1] == " ": + tag = tag[:-1] + ident = "" + for c in tag: + ident = ident + _escapechar(c) + if re.match("[0-9]", ident): + ident = "_" + ident + return ident + + +def identifierToTag(ident: str) -> str: + """the opposite of tagToIdentifier()""" + if ident == "GlyphOrder": + return ident + if len(ident) % 2 and ident[0] == "_": + ident = ident[1:] + assert not (len(ident) % 2) + tag = "" + for i in range(0, len(ident), 2): + if ident[i] == "_": + tag = tag + ident[i + 1] + elif ident[i + 1] == "_": + tag = tag + ident[i] + else: + # assume hex + tag = tag + chr(int(ident[i : i + 2], 16)) + # append trailing spaces + tag = tag + (4 - len(tag)) * " " + return Tag(tag) + + +def tagToXML(tag: str | bytes) -> str: + """Similarly to tagToIdentifier(), this converts a TT tag + to a valid XML element name. Since XML element names are + case sensitive, this is a fairly simple/readable translation. + """ + import re + + tag = Tag(tag) + if tag == "OS/2": + return "OS_2" + elif tag == "GlyphOrder": + return tag + if re.match("[A-Za-z_][A-Za-z_0-9]* *$", tag): + return tag.strip() + else: + return tagToIdentifier(tag) + + +def xmlToTag(tag: str) -> str: + """The opposite of tagToXML()""" + if tag == "OS_2": + return Tag("OS/2") + if len(tag) == 8: + return identifierToTag(tag) + else: + return Tag(tag + " " * (4 - len(tag))) + + +# Table order as recommended in the OpenType specification 1.4 +TTFTableOrder = [ + "head", + "hhea", + "maxp", + "OS/2", + "hmtx", + "LTSH", + "VDMX", + "hdmx", + "cmap", + "fpgm", + "prep", + "cvt ", + "loca", + "glyf", + "kern", + "name", + "post", + "gasp", + "PCLT", +] + +OTFTableOrder = ["head", "hhea", "maxp", "OS/2", "name", "cmap", "post", "CFF "] + + +def sortedTagList( + tagList: Sequence[str], tableOrder: Sequence[str] | None = None +) -> list[str]: + """Return a sorted copy of tagList, sorted according to the OpenType + specification, or according to a custom tableOrder. If given and not + None, tableOrder needs to be a list of tag names. + """ + tagList = sorted(tagList) + if tableOrder is None: + if "DSIG" in tagList: + # DSIG should be last (XXX spec reference?) + tagList.remove("DSIG") + tagList.append("DSIG") + if "CFF " in tagList: + tableOrder = OTFTableOrder + else: + tableOrder = TTFTableOrder + orderedTables = [] + for tag in tableOrder: + if tag in tagList: + orderedTables.append(tag) + tagList.remove(tag) + orderedTables.extend(tagList) + return orderedTables + + +def reorderFontTables( + inFile: BinaryIO, # Takes file-like object as per original + outFile: BinaryIO, # Takes file-like object + tableOrder: Sequence[str] | None = None, + checkChecksums: bool = False, # Keep param even if reader handles it +) -> None: + """Rewrite a font file, ordering the tables as recommended by the + OpenType specification 1.4. + """ + inFile.seek(0) + outFile.seek(0) + reader = SFNTReader(inFile, checkChecksums=checkChecksums) + writer = SFNTWriter( + outFile, + len(reader.tables), + reader.sfntVersion, + reader.flavor, + reader.flavorData, + ) + tables = list(reader.keys()) + for tag in sortedTagList(tables, tableOrder): + writer[tag] = reader[tag] + writer.close() + + +def maxPowerOfTwo(x: int) -> int: + """Return the highest exponent of two, so that + (2 ** exponent) <= x. Return 0 if x is 0. + """ + exponent = 0 + while x: + x = x >> 1 + exponent = exponent + 1 + return max(exponent - 1, 0) + + +def getSearchRange(n: int, itemSize: int = 16) -> tuple[int, int, int]: + """Calculate searchRange, entrySelector, rangeShift.""" + # itemSize defaults to 16, for backward compatibility + # with upstream fonttools. + exponent = maxPowerOfTwo(n) + searchRange = (2**exponent) * itemSize + entrySelector = exponent + rangeShift = max(0, n * itemSize - searchRange) + return searchRange, entrySelector, rangeShift diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttGlyphSet.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttGlyphSet.py new file mode 100644 index 0000000000000000000000000000000000000000..3a9a0cb9960434d28fd98bd6bf9fdd128522d89f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttGlyphSet.py @@ -0,0 +1,490 @@ +"""GlyphSets returned by a TTFont.""" + +from abc import ABC, abstractmethod +from collections.abc import Mapping +from contextlib import contextmanager +from copy import copy, deepcopy +from types import SimpleNamespace +from fontTools.misc.vector import Vector +from fontTools.misc.fixedTools import otRound, fixedToFloat as fi2fl +from fontTools.misc.loggingTools import deprecateFunction +from fontTools.misc.transform import Transform, DecomposedTransform +from fontTools.pens.transformPen import TransformPen, TransformPointPen +from fontTools.pens.recordingPen import ( + DecomposingRecordingPen, + lerpRecordings, + replayRecording, +) + + +class _TTGlyphSet(Mapping): + """Generic dict-like GlyphSet class that pulls metrics from hmtx and + glyph shape from TrueType or CFF. + """ + + def __init__(self, font, location, glyphsMapping, *, recalcBounds=True): + self.recalcBounds = recalcBounds + self.font = font + self.defaultLocationNormalized = ( + {axis.axisTag: 0 for axis in self.font["fvar"].axes} + if "fvar" in self.font + else {} + ) + self.location = location if location is not None else {} + self.rawLocation = {} # VarComponent-only location + self.originalLocation = location if location is not None else {} + self.depth = 0 + self.locationStack = [] + self.rawLocationStack = [] + self.glyphsMapping = glyphsMapping + self.hMetrics = font["hmtx"].metrics + self.vMetrics = getattr(font.get("vmtx"), "metrics", None) + self.hvarTable = None + if location: + from fontTools.varLib.varStore import VarStoreInstancer + + self.hvarTable = getattr(font.get("HVAR"), "table", None) + if self.hvarTable is not None: + self.hvarInstancer = VarStoreInstancer( + self.hvarTable.VarStore, font["fvar"].axes, location + ) + # TODO VVAR, VORG + + @contextmanager + def pushLocation(self, location, reset: bool): + self.locationStack.append(self.location) + self.rawLocationStack.append(self.rawLocation) + if reset: + self.location = self.originalLocation.copy() + self.rawLocation = self.defaultLocationNormalized.copy() + else: + self.location = self.location.copy() + self.rawLocation = {} + self.location.update(location) + self.rawLocation.update(location) + + try: + yield None + finally: + self.location = self.locationStack.pop() + self.rawLocation = self.rawLocationStack.pop() + + @contextmanager + def pushDepth(self): + try: + depth = self.depth + self.depth += 1 + yield depth + finally: + self.depth -= 1 + + def __contains__(self, glyphName): + return glyphName in self.glyphsMapping + + def __iter__(self): + return iter(self.glyphsMapping.keys()) + + def __len__(self): + return len(self.glyphsMapping) + + @deprecateFunction( + "use 'glyphName in glyphSet' instead", category=DeprecationWarning + ) + def has_key(self, glyphName): + return glyphName in self.glyphsMapping + + +class _TTGlyphSetGlyf(_TTGlyphSet): + def __init__(self, font, location, recalcBounds=True): + self.glyfTable = font["glyf"] + super().__init__(font, location, self.glyfTable, recalcBounds=recalcBounds) + self.gvarTable = font.get("gvar") + + def __getitem__(self, glyphName): + return _TTGlyphGlyf(self, glyphName, recalcBounds=self.recalcBounds) + + +class _TTGlyphSetCFF(_TTGlyphSet): + def __init__(self, font, location): + tableTag = "CFF2" if "CFF2" in font else "CFF " + self.charStrings = list(font[tableTag].cff.values())[0].CharStrings + super().__init__(font, location, self.charStrings) + self.setLocation(location) + + def __getitem__(self, glyphName): + return _TTGlyphCFF(self, glyphName) + + def setLocation(self, location): + self.blender = None + if location: + # TODO Optimize by using instancer.setLocation() + + from fontTools.varLib.varStore import VarStoreInstancer + + varStore = getattr(self.charStrings, "varStore", None) + if varStore is not None: + instancer = VarStoreInstancer( + varStore.otVarStore, self.font["fvar"].axes, location + ) + self.blender = instancer.interpolateFromDeltas + else: + self.blender = None + + @contextmanager + def pushLocation(self, location, reset: bool): + self.setLocation(location) + with _TTGlyphSet.pushLocation(self, location, reset) as value: + try: + yield value + finally: + self.setLocation(self.location) + + +class _TTGlyphSetVARC(_TTGlyphSet): + def __init__(self, font, location, glyphSet): + self.glyphSet = glyphSet + super().__init__(font, location, glyphSet) + self.varcTable = font["VARC"].table + + def __getitem__(self, glyphName): + varc = self.varcTable + if glyphName not in varc.Coverage.glyphs: + return self.glyphSet[glyphName] + return _TTGlyphVARC(self, glyphName) + + +class _TTGlyph(ABC): + """Glyph object that supports the Pen protocol, meaning that it has + .draw() and .drawPoints() methods that take a pen object as their only + argument. Additionally there are 'width' and 'lsb' attributes, read from + the 'hmtx' table. + + If the font contains a 'vmtx' table, there will also be 'height' and 'tsb' + attributes. + """ + + def __init__(self, glyphSet, glyphName, *, recalcBounds=True): + self.glyphSet = glyphSet + self.name = glyphName + self.recalcBounds = recalcBounds + self.width, self.lsb = glyphSet.hMetrics[glyphName] + if glyphSet.vMetrics is not None: + self.height, self.tsb = glyphSet.vMetrics[glyphName] + else: + self.height, self.tsb = None, None + if glyphSet.location and glyphSet.hvarTable is not None: + varidx = ( + glyphSet.font.getGlyphID(glyphName) + if glyphSet.hvarTable.AdvWidthMap is None + else glyphSet.hvarTable.AdvWidthMap.mapping[glyphName] + ) + self.width += glyphSet.hvarInstancer[varidx] + # TODO: VVAR/VORG + + @abstractmethod + def draw(self, pen): + """Draw the glyph onto ``pen``. See fontTools.pens.basePen for details + how that works. + """ + raise NotImplementedError + + def drawPoints(self, pen): + """Draw the glyph onto ``pen``. See fontTools.pens.pointPen for details + how that works. + """ + from fontTools.pens.pointPen import SegmentToPointPen + + self.draw(SegmentToPointPen(pen)) + + +class _TTGlyphGlyf(_TTGlyph): + def draw(self, pen): + """Draw the glyph onto ``pen``. See fontTools.pens.basePen for details + how that works. + """ + glyph, offset = self._getGlyphAndOffset() + + with self.glyphSet.pushDepth() as depth: + if depth: + offset = 0 # Offset should only apply at top-level + + glyph.draw(pen, self.glyphSet.glyfTable, offset) + + def drawPoints(self, pen): + """Draw the glyph onto ``pen``. See fontTools.pens.pointPen for details + how that works. + """ + glyph, offset = self._getGlyphAndOffset() + + with self.glyphSet.pushDepth() as depth: + if depth: + offset = 0 # Offset should only apply at top-level + + glyph.drawPoints(pen, self.glyphSet.glyfTable, offset) + + def _getGlyphAndOffset(self): + if self.glyphSet.location and self.glyphSet.gvarTable is not None: + glyph = self._getGlyphInstance() + else: + glyph = self.glyphSet.glyfTable[self.name] + + offset = self.lsb - glyph.xMin if hasattr(glyph, "xMin") else 0 + return glyph, offset + + def _getGlyphInstance(self): + from fontTools.varLib.iup import iup_delta + from fontTools.ttLib.tables._g_l_y_f import GlyphCoordinates + from fontTools.varLib.models import supportScalar + + glyphSet = self.glyphSet + glyfTable = glyphSet.glyfTable + variations = glyphSet.gvarTable.variations[self.name] + hMetrics = glyphSet.hMetrics + vMetrics = glyphSet.vMetrics + coordinates, _ = glyfTable._getCoordinatesAndControls( + self.name, hMetrics, vMetrics + ) + origCoords, endPts = None, None + for var in variations: + scalar = supportScalar(glyphSet.location, var.axes) + if not scalar: + continue + delta = var.coordinates + if None in delta: + if origCoords is None: + origCoords, control = glyfTable._getCoordinatesAndControls( + self.name, hMetrics, vMetrics + ) + endPts = ( + control[1] if control[0] >= 1 else list(range(len(control[1]))) + ) + delta = iup_delta(delta, origCoords, endPts) + coordinates += GlyphCoordinates(delta) * scalar + + glyph = copy(glyfTable[self.name]) # Shallow copy + width, lsb, height, tsb = _setCoordinates( + glyph, coordinates, glyfTable, recalcBounds=self.recalcBounds + ) + self.lsb = lsb + self.tsb = tsb + if glyphSet.hvarTable is None: + # no HVAR: let's set metrics from the phantom points + self.width = width + self.height = height + return glyph + + +class _TTGlyphCFF(_TTGlyph): + def draw(self, pen): + """Draw the glyph onto ``pen``. See fontTools.pens.basePen for details + how that works. + """ + self.glyphSet.charStrings[self.name].draw(pen, self.glyphSet.blender) + + +def _evaluateCondition(condition, fvarAxes, location, instancer): + if condition.Format == 1: + # ConditionAxisRange + axisIndex = condition.AxisIndex + axisTag = fvarAxes[axisIndex].axisTag + axisValue = location.get(axisTag, 0) + minValue = condition.FilterRangeMinValue + maxValue = condition.FilterRangeMaxValue + return minValue <= axisValue <= maxValue + elif condition.Format == 2: + # ConditionValue + value = condition.DefaultValue + value += instancer[condition.VarIdx][0] + return value > 0 + elif condition.Format == 3: + # ConditionAnd + for subcondition in condition.ConditionTable: + if not _evaluateCondition(subcondition, fvarAxes, location, instancer): + return False + return True + elif condition.Format == 4: + # ConditionOr + for subcondition in condition.ConditionTable: + if _evaluateCondition(subcondition, fvarAxes, location, instancer): + return True + return False + elif condition.Format == 5: + # ConditionNegate + return not _evaluateCondition( + condition.conditionTable, fvarAxes, location, instancer + ) + else: + return False # Unkonwn condition format + + +class _TTGlyphVARC(_TTGlyph): + def _draw(self, pen, isPointPen): + """Draw the glyph onto ``pen``. See fontTools.pens.basePen for details + how that works. + """ + from fontTools.ttLib.tables.otTables import ( + VarComponentFlags, + NO_VARIATION_INDEX, + ) + + glyphSet = self.glyphSet + varc = glyphSet.varcTable + idx = varc.Coverage.glyphs.index(self.name) + glyph = varc.VarCompositeGlyphs.VarCompositeGlyph[idx] + + from fontTools.varLib.multiVarStore import MultiVarStoreInstancer + from fontTools.varLib.varStore import VarStoreInstancer + + fvarAxes = glyphSet.font["fvar"].axes + instancer = MultiVarStoreInstancer( + varc.MultiVarStore, fvarAxes, self.glyphSet.location + ) + + for comp in glyph.components: + if comp.flags & VarComponentFlags.HAVE_CONDITION: + condition = varc.ConditionList.ConditionTable[comp.conditionIndex] + if not _evaluateCondition( + condition, fvarAxes, self.glyphSet.location, instancer + ): + continue + + location = {} + if comp.axisIndicesIndex is not None: + axisIndices = varc.AxisIndicesList.Item[comp.axisIndicesIndex] + axisValues = Vector(comp.axisValues) + if comp.axisValuesVarIndex != NO_VARIATION_INDEX: + axisValues += fi2fl(instancer[comp.axisValuesVarIndex], 14) + assert len(axisIndices) == len(axisValues), ( + len(axisIndices), + len(axisValues), + ) + location = { + fvarAxes[i].axisTag: v for i, v in zip(axisIndices, axisValues) + } + + if comp.transformVarIndex != NO_VARIATION_INDEX: + deltas = instancer[comp.transformVarIndex] + comp = deepcopy(comp) + comp.applyTransformDeltas(deltas) + transform = comp.transform + + reset = comp.flags & VarComponentFlags.RESET_UNSPECIFIED_AXES + with self.glyphSet.glyphSet.pushLocation(location, reset): + with self.glyphSet.pushLocation(location, reset): + shouldDecompose = self.name == comp.glyphName + + if not shouldDecompose: + try: + pen.addVarComponent( + comp.glyphName, transform, self.glyphSet.rawLocation + ) + except AttributeError: + shouldDecompose = True + + if shouldDecompose: + t = transform.toTransform() + compGlyphSet = ( + self.glyphSet + if comp.glyphName != self.name + else glyphSet.glyphSet + ) + g = compGlyphSet[comp.glyphName] + if isPointPen: + tPen = TransformPointPen(pen, t) + g.drawPoints(tPen) + else: + tPen = TransformPen(pen, t) + g.draw(tPen) + + def draw(self, pen): + self._draw(pen, False) + + def drawPoints(self, pen): + self._draw(pen, True) + + +def _setCoordinates(glyph, coord, glyfTable, *, recalcBounds=True): + # Handle phantom points for (left, right, top, bottom) positions. + assert len(coord) >= 4 + leftSideX = coord[-4][0] + rightSideX = coord[-3][0] + topSideY = coord[-2][1] + bottomSideY = coord[-1][1] + + for _ in range(4): + del coord[-1] + + if glyph.isComposite(): + assert len(coord) == len(glyph.components) + glyph.components = [copy(comp) for comp in glyph.components] # Shallow copy + for p, comp in zip(coord, glyph.components): + if hasattr(comp, "x"): + comp.x, comp.y = p + elif glyph.numberOfContours == 0: + assert len(coord) == 0 + else: + assert len(coord) == len(glyph.coordinates) + glyph.coordinates = coord + + if recalcBounds: + glyph.recalcBounds(glyfTable) + + horizontalAdvanceWidth = otRound(rightSideX - leftSideX) + verticalAdvanceWidth = otRound(topSideY - bottomSideY) + leftSideBearing = otRound(glyph.xMin - leftSideX) + topSideBearing = otRound(topSideY - glyph.yMax) + return ( + horizontalAdvanceWidth, + leftSideBearing, + verticalAdvanceWidth, + topSideBearing, + ) + + +class LerpGlyphSet(Mapping): + """A glyphset that interpolates between two other glyphsets. + + Factor is typically between 0 and 1. 0 means the first glyphset, + 1 means the second glyphset, and 0.5 means the average of the + two glyphsets. Other values are possible, and can be useful to + extrapolate. Defaults to 0.5. + """ + + def __init__(self, glyphset1, glyphset2, factor=0.5): + self.glyphset1 = glyphset1 + self.glyphset2 = glyphset2 + self.factor = factor + + def __getitem__(self, glyphname): + if glyphname in self.glyphset1 and glyphname in self.glyphset2: + return LerpGlyph(glyphname, self) + raise KeyError(glyphname) + + def __contains__(self, glyphname): + return glyphname in self.glyphset1 and glyphname in self.glyphset2 + + def __iter__(self): + set1 = set(self.glyphset1) + set2 = set(self.glyphset2) + return iter(set1.intersection(set2)) + + def __len__(self): + set1 = set(self.glyphset1) + set2 = set(self.glyphset2) + return len(set1.intersection(set2)) + + +class LerpGlyph: + def __init__(self, glyphname, glyphset): + self.glyphset = glyphset + self.glyphname = glyphname + + def draw(self, pen): + recording1 = DecomposingRecordingPen(self.glyphset.glyphset1) + self.glyphset.glyphset1[self.glyphname].draw(recording1) + recording2 = DecomposingRecordingPen(self.glyphset.glyphset2) + self.glyphset.glyphset2[self.glyphname].draw(recording2) + + factor = self.glyphset.factor + + replayRecording(lerpRecordings(recording1.value, recording2.value, factor), pen) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttVisitor.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttVisitor.py new file mode 100644 index 0000000000000000000000000000000000000000..54db61b1e0b1be5e2d36fd72008230de7fc35401 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/ttVisitor.py @@ -0,0 +1,32 @@ +"""Specialization of fontTools.misc.visitor to work with TTFont.""" + +from fontTools.misc.visitor import Visitor +from fontTools.ttLib import TTFont + + +class TTVisitor(Visitor): + def visitAttr(self, obj, attr, value, *args, **kwargs): + if isinstance(value, TTFont): + return False + super().visitAttr(obj, attr, value, *args, **kwargs) + + def visit(self, obj, *args, **kwargs): + if hasattr(obj, "ensureDecompiled"): + obj.ensureDecompiled(recurse=False) + super().visit(obj, *args, **kwargs) + + +@TTVisitor.register(TTFont) +def visit(visitor, font, *args, **kwargs): + # Some objects have links back to TTFont; even though we + # have a check in visitAttr to stop them from recursing + # onto TTFont, sometimes they still do, for example when + # someone overrides visitAttr. + if hasattr(visitor, "font"): + return False + + visitor.font = font + for tag in font.keys(): + visitor.visit(font[tag], *args, **kwargs) + del visitor.font + return False diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttLib/woff2.py b/.venv/lib/python3.11/site-packages/fontTools/ttLib/woff2.py new file mode 100644 index 0000000000000000000000000000000000000000..c11aeb2e54349b82f86213d598f98ee5eb763238 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttLib/woff2.py @@ -0,0 +1,1680 @@ +from io import BytesIO +import sys +import array +import struct +from collections import OrderedDict +from fontTools.misc import sstruct +from fontTools.misc.arrayTools import calcIntBounds +from fontTools.misc.textTools import Tag, bytechr, byteord, bytesjoin, pad +from fontTools.ttLib import ( + TTFont, + TTLibError, + getTableModule, + getTableClass, + getSearchRange, +) +from fontTools.ttLib.sfnt import ( + SFNTReader, + SFNTWriter, + DirectoryEntry, + WOFFFlavorData, + sfntDirectoryFormat, + sfntDirectorySize, + SFNTDirectoryEntry, + sfntDirectoryEntrySize, + calcChecksum, +) +from fontTools.ttLib.tables import ttProgram, _g_l_y_f +import logging + + +log = logging.getLogger("fontTools.ttLib.woff2") + +haveBrotli = False +try: + try: + import brotlicffi as brotli + except ImportError: + import brotli + haveBrotli = True +except ImportError: + pass + + +class WOFF2Reader(SFNTReader): + flavor = "woff2" + + def __init__(self, file, checkChecksums=0, fontNumber=-1): + if not haveBrotli: + log.error( + "The WOFF2 decoder requires the Brotli Python extension, available at: " + "https://github.com/google/brotli" + ) + raise ImportError("No module named brotli") + + self.file = file + + signature = Tag(self.file.read(4)) + if signature != b"wOF2": + raise TTLibError("Not a WOFF2 font (bad signature)") + + self.file.seek(0) + self.DirectoryEntry = WOFF2DirectoryEntry + data = self.file.read(woff2DirectorySize) + if len(data) != woff2DirectorySize: + raise TTLibError("Not a WOFF2 font (not enough data)") + sstruct.unpack(woff2DirectoryFormat, data, self) + + self.tables = OrderedDict() + offset = 0 + for i in range(self.numTables): + entry = self.DirectoryEntry() + entry.fromFile(self.file) + tag = Tag(entry.tag) + self.tables[tag] = entry + entry.offset = offset + offset += entry.length + + totalUncompressedSize = offset + compressedData = self.file.read(self.totalCompressedSize) + decompressedData = brotli.decompress(compressedData) + if len(decompressedData) != totalUncompressedSize: + raise TTLibError( + "unexpected size for decompressed font data: expected %d, found %d" + % (totalUncompressedSize, len(decompressedData)) + ) + self.transformBuffer = BytesIO(decompressedData) + + self.file.seek(0, 2) + if self.length != self.file.tell(): + raise TTLibError("reported 'length' doesn't match the actual file size") + + self.flavorData = WOFF2FlavorData(self) + + # make empty TTFont to store data while reconstructing tables + self.ttFont = TTFont(recalcBBoxes=False, recalcTimestamp=False) + + def __getitem__(self, tag): + """Fetch the raw table data. Reconstruct transformed tables.""" + entry = self.tables[Tag(tag)] + if not hasattr(entry, "data"): + if entry.transformed: + entry.data = self.reconstructTable(tag) + else: + entry.data = entry.loadData(self.transformBuffer) + return entry.data + + def reconstructTable(self, tag): + """Reconstruct table named 'tag' from transformed data.""" + entry = self.tables[Tag(tag)] + rawData = entry.loadData(self.transformBuffer) + if tag == "glyf": + # no need to pad glyph data when reconstructing + padding = self.padding if hasattr(self, "padding") else None + data = self._reconstructGlyf(rawData, padding) + elif tag == "loca": + data = self._reconstructLoca() + elif tag == "hmtx": + data = self._reconstructHmtx(rawData) + else: + raise TTLibError("transform for table '%s' is unknown" % tag) + return data + + def _reconstructGlyf(self, data, padding=None): + """Return recostructed glyf table data, and set the corresponding loca's + locations. Optionally pad glyph offsets to the specified number of bytes. + """ + self.ttFont["loca"] = WOFF2LocaTable() + glyfTable = self.ttFont["glyf"] = WOFF2GlyfTable() + glyfTable.reconstruct(data, self.ttFont) + if padding: + glyfTable.padding = padding + data = glyfTable.compile(self.ttFont) + return data + + def _reconstructLoca(self): + """Return reconstructed loca table data.""" + if "loca" not in self.ttFont: + # make sure glyf is reconstructed first + self.tables["glyf"].data = self.reconstructTable("glyf") + locaTable = self.ttFont["loca"] + data = locaTable.compile(self.ttFont) + if len(data) != self.tables["loca"].origLength: + raise TTLibError( + "reconstructed 'loca' table doesn't match original size: " + "expected %d, found %d" % (self.tables["loca"].origLength, len(data)) + ) + return data + + def _reconstructHmtx(self, data): + """Return reconstructed hmtx table data.""" + # Before reconstructing 'hmtx' table we need to parse other tables: + # 'glyf' is required for reconstructing the sidebearings from the glyphs' + # bounding box; 'hhea' is needed for the numberOfHMetrics field. + if "glyf" in self.flavorData.transformedTables: + # transformed 'glyf' table is self-contained, thus 'loca' not needed + tableDependencies = ("maxp", "hhea", "glyf") + else: + # decompiling untransformed 'glyf' requires 'loca', which requires 'head' + tableDependencies = ("maxp", "head", "hhea", "loca", "glyf") + for tag in tableDependencies: + self._decompileTable(tag) + hmtxTable = self.ttFont["hmtx"] = WOFF2HmtxTable() + hmtxTable.reconstruct(data, self.ttFont) + data = hmtxTable.compile(self.ttFont) + return data + + def _decompileTable(self, tag): + """Decompile table data and store it inside self.ttFont.""" + data = self[tag] + if self.ttFont.isLoaded(tag): + return self.ttFont[tag] + tableClass = getTableClass(tag) + table = tableClass(tag) + self.ttFont.tables[tag] = table + table.decompile(data, self.ttFont) + + +class WOFF2Writer(SFNTWriter): + flavor = "woff2" + + def __init__( + self, + file, + numTables, + sfntVersion="\000\001\000\000", + flavor=None, + flavorData=None, + ): + if not haveBrotli: + log.error( + "The WOFF2 encoder requires the Brotli Python extension, available at: " + "https://github.com/google/brotli" + ) + raise ImportError("No module named brotli") + + self.file = file + self.numTables = numTables + self.sfntVersion = Tag(sfntVersion) + self.flavorData = WOFF2FlavorData(data=flavorData) + + self.directoryFormat = woff2DirectoryFormat + self.directorySize = woff2DirectorySize + self.DirectoryEntry = WOFF2DirectoryEntry + + self.signature = Tag("wOF2") + + self.nextTableOffset = 0 + self.transformBuffer = BytesIO() + + self.tables = OrderedDict() + + # make empty TTFont to store data while normalising and transforming tables + self.ttFont = TTFont(recalcBBoxes=False, recalcTimestamp=False) + + def __setitem__(self, tag, data): + """Associate new entry named 'tag' with raw table data.""" + if tag in self.tables: + raise TTLibError("cannot rewrite '%s' table" % tag) + if tag == "DSIG": + # always drop DSIG table, since the encoding process can invalidate it + self.numTables -= 1 + return + + entry = self.DirectoryEntry() + entry.tag = Tag(tag) + entry.flags = getKnownTagIndex(entry.tag) + # WOFF2 table data are written to disk only on close(), after all tags + # have been specified + entry.data = data + + self.tables[tag] = entry + + def close(self): + """All tags must have been specified. Now write the table data and directory.""" + if len(self.tables) != self.numTables: + raise TTLibError( + "wrong number of tables; expected %d, found %d" + % (self.numTables, len(self.tables)) + ) + + if self.sfntVersion in ("\x00\x01\x00\x00", "true"): + isTrueType = True + elif self.sfntVersion == "OTTO": + isTrueType = False + else: + raise TTLibError("Not a TrueType or OpenType font (bad sfntVersion)") + + # The WOFF2 spec no longer requires the glyph offsets to be 4-byte aligned. + # However, the reference WOFF2 implementation still fails to reconstruct + # 'unpadded' glyf tables, therefore we need to 'normalise' them. + # See: + # https://github.com/khaledhosny/ots/issues/60 + # https://github.com/google/woff2/issues/15 + if ( + isTrueType + and "glyf" in self.flavorData.transformedTables + and "glyf" in self.tables + ): + self._normaliseGlyfAndLoca(padding=4) + self._setHeadTransformFlag() + + # To pass the legacy OpenType Sanitiser currently included in browsers, + # we must sort the table directory and data alphabetically by tag. + # See: + # https://github.com/google/woff2/pull/3 + # https://lists.w3.org/Archives/Public/public-webfonts-wg/2015Mar/0000.html + # + # 2023: We rely on this in _transformTables where we expect that + # "loca" comes after "glyf" table. + self.tables = OrderedDict(sorted(self.tables.items())) + + self.totalSfntSize = self._calcSFNTChecksumsLengthsAndOffsets() + + fontData = self._transformTables() + compressedFont = brotli.compress(fontData, mode=brotli.MODE_FONT) + + self.totalCompressedSize = len(compressedFont) + self.length = self._calcTotalSize() + self.majorVersion, self.minorVersion = self._getVersion() + self.reserved = 0 + + directory = self._packTableDirectory() + self.file.seek(0) + self.file.write(pad(directory + compressedFont, size=4)) + self._writeFlavorData() + + def _normaliseGlyfAndLoca(self, padding=4): + """Recompile glyf and loca tables, aligning glyph offsets to multiples of + 'padding' size. Update the head table's 'indexToLocFormat' accordingly while + compiling loca. + """ + if self.sfntVersion == "OTTO": + return + + for tag in ("maxp", "head", "loca", "glyf", "fvar"): + if tag in self.tables: + self._decompileTable(tag) + self.ttFont["glyf"].padding = padding + for tag in ("glyf", "loca"): + self._compileTable(tag) + + def _setHeadTransformFlag(self): + """Set bit 11 of 'head' table flags to indicate that the font has undergone + a lossless modifying transform. Re-compile head table data.""" + self._decompileTable("head") + self.ttFont["head"].flags |= 1 << 11 + self._compileTable("head") + + def _decompileTable(self, tag): + """Fetch table data, decompile it, and store it inside self.ttFont.""" + tag = Tag(tag) + if tag not in self.tables: + raise TTLibError("missing required table: %s" % tag) + if self.ttFont.isLoaded(tag): + return + data = self.tables[tag].data + if tag == "loca": + tableClass = WOFF2LocaTable + elif tag == "glyf": + tableClass = WOFF2GlyfTable + elif tag == "hmtx": + tableClass = WOFF2HmtxTable + else: + tableClass = getTableClass(tag) + table = tableClass(tag) + self.ttFont.tables[tag] = table + table.decompile(data, self.ttFont) + + def _compileTable(self, tag): + """Compile table and store it in its 'data' attribute.""" + self.tables[tag].data = self.ttFont[tag].compile(self.ttFont) + + def _calcSFNTChecksumsLengthsAndOffsets(self): + """Compute the 'original' SFNT checksums, lengths and offsets for checksum + adjustment calculation. Return the total size of the uncompressed font. + """ + offset = sfntDirectorySize + sfntDirectoryEntrySize * len(self.tables) + for tag, entry in self.tables.items(): + data = entry.data + entry.origOffset = offset + entry.origLength = len(data) + if tag == "head": + entry.checkSum = calcChecksum(data[:8] + b"\0\0\0\0" + data[12:]) + else: + entry.checkSum = calcChecksum(data) + offset += (entry.origLength + 3) & ~3 + return offset + + def _transformTables(self): + """Return transformed font data.""" + transformedTables = self.flavorData.transformedTables + for tag, entry in self.tables.items(): + data = None + if tag in transformedTables: + data = self.transformTable(tag) + if data is not None: + entry.transformed = True + if data is None: + if tag == "glyf": + # Currently we always sort table tags so + # 'loca' comes after 'glyf'. + transformedTables.discard("loca") + # pass-through the table data without transformation + data = entry.data + entry.transformed = False + entry.offset = self.nextTableOffset + entry.saveData(self.transformBuffer, data) + self.nextTableOffset += entry.length + self.writeMasterChecksum() + fontData = self.transformBuffer.getvalue() + return fontData + + def transformTable(self, tag): + """Return transformed table data, or None if some pre-conditions aren't + met -- in which case, the non-transformed table data will be used. + """ + if tag == "loca": + data = b"" + elif tag == "glyf": + for tag in ("maxp", "head", "loca", "glyf"): + self._decompileTable(tag) + glyfTable = self.ttFont["glyf"] + data = glyfTable.transform(self.ttFont) + elif tag == "hmtx": + if "glyf" not in self.tables: + return + for tag in ("maxp", "head", "hhea", "loca", "glyf", "hmtx"): + self._decompileTable(tag) + hmtxTable = self.ttFont["hmtx"] + data = hmtxTable.transform(self.ttFont) # can be None + else: + raise TTLibError("Transform for table '%s' is unknown" % tag) + return data + + def _calcMasterChecksum(self): + """Calculate checkSumAdjustment.""" + checksums = [] + for tag in self.tables.keys(): + checksums.append(self.tables[tag].checkSum) + + # Create a SFNT directory for checksum calculation purposes + self.searchRange, self.entrySelector, self.rangeShift = getSearchRange( + self.numTables, 16 + ) + directory = sstruct.pack(sfntDirectoryFormat, self) + tables = sorted(self.tables.items()) + for tag, entry in tables: + sfntEntry = SFNTDirectoryEntry() + sfntEntry.tag = entry.tag + sfntEntry.checkSum = entry.checkSum + sfntEntry.offset = entry.origOffset + sfntEntry.length = entry.origLength + directory = directory + sfntEntry.toString() + + directory_end = sfntDirectorySize + len(self.tables) * sfntDirectoryEntrySize + assert directory_end == len(directory) + + checksums.append(calcChecksum(directory)) + checksum = sum(checksums) & 0xFFFFFFFF + # BiboAfba! + checksumadjustment = (0xB1B0AFBA - checksum) & 0xFFFFFFFF + return checksumadjustment + + def writeMasterChecksum(self): + """Write checkSumAdjustment to the transformBuffer.""" + checksumadjustment = self._calcMasterChecksum() + self.transformBuffer.seek(self.tables["head"].offset + 8) + self.transformBuffer.write(struct.pack(">L", checksumadjustment)) + + def _calcTotalSize(self): + """Calculate total size of WOFF2 font, including any meta- and/or private data.""" + offset = self.directorySize + for entry in self.tables.values(): + offset += len(entry.toString()) + offset += self.totalCompressedSize + offset = (offset + 3) & ~3 + offset = self._calcFlavorDataOffsetsAndSize(offset) + return offset + + def _calcFlavorDataOffsetsAndSize(self, start): + """Calculate offsets and lengths for any meta- and/or private data.""" + offset = start + data = self.flavorData + if data.metaData: + self.metaOrigLength = len(data.metaData) + self.metaOffset = offset + self.compressedMetaData = brotli.compress( + data.metaData, mode=brotli.MODE_TEXT + ) + self.metaLength = len(self.compressedMetaData) + offset += self.metaLength + else: + self.metaOffset = self.metaLength = self.metaOrigLength = 0 + self.compressedMetaData = b"" + if data.privData: + # make sure private data is padded to 4-byte boundary + offset = (offset + 3) & ~3 + self.privOffset = offset + self.privLength = len(data.privData) + offset += self.privLength + else: + self.privOffset = self.privLength = 0 + return offset + + def _getVersion(self): + """Return the WOFF2 font's (majorVersion, minorVersion) tuple.""" + data = self.flavorData + if data.majorVersion is not None and data.minorVersion is not None: + return data.majorVersion, data.minorVersion + else: + # if None, return 'fontRevision' from 'head' table + if "head" in self.tables: + return struct.unpack(">HH", self.tables["head"].data[4:8]) + else: + return 0, 0 + + def _packTableDirectory(self): + """Return WOFF2 table directory data.""" + directory = sstruct.pack(self.directoryFormat, self) + for entry in self.tables.values(): + directory = directory + entry.toString() + return directory + + def _writeFlavorData(self): + """Write metadata and/or private data using appropiate padding.""" + compressedMetaData = self.compressedMetaData + privData = self.flavorData.privData + if compressedMetaData and privData: + compressedMetaData = pad(compressedMetaData, size=4) + if compressedMetaData: + self.file.seek(self.metaOffset) + assert self.file.tell() == self.metaOffset + self.file.write(compressedMetaData) + if privData: + self.file.seek(self.privOffset) + assert self.file.tell() == self.privOffset + self.file.write(privData) + + def reordersTables(self): + return True + + +# -- woff2 directory helpers and cruft + +woff2DirectoryFormat = """ + > # big endian + signature: 4s # "wOF2" + sfntVersion: 4s + length: L # total woff2 file size + numTables: H # number of tables + reserved: H # set to 0 + totalSfntSize: L # uncompressed size + totalCompressedSize: L # compressed size + majorVersion: H # major version of WOFF file + minorVersion: H # minor version of WOFF file + metaOffset: L # offset to metadata block + metaLength: L # length of compressed metadata + metaOrigLength: L # length of uncompressed metadata + privOffset: L # offset to private data block + privLength: L # length of private data block +""" + +woff2DirectorySize = sstruct.calcsize(woff2DirectoryFormat) + +woff2KnownTags = ( + "cmap", + "head", + "hhea", + "hmtx", + "maxp", + "name", + "OS/2", + "post", + "cvt ", + "fpgm", + "glyf", + "loca", + "prep", + "CFF ", + "VORG", + "EBDT", + "EBLC", + "gasp", + "hdmx", + "kern", + "LTSH", + "PCLT", + "VDMX", + "vhea", + "vmtx", + "BASE", + "GDEF", + "GPOS", + "GSUB", + "EBSC", + "JSTF", + "MATH", + "CBDT", + "CBLC", + "COLR", + "CPAL", + "SVG ", + "sbix", + "acnt", + "avar", + "bdat", + "bloc", + "bsln", + "cvar", + "fdsc", + "feat", + "fmtx", + "fvar", + "gvar", + "hsty", + "just", + "lcar", + "mort", + "morx", + "opbd", + "prop", + "trak", + "Zapf", + "Silf", + "Glat", + "Gloc", + "Feat", + "Sill", +) + +woff2FlagsFormat = """ + > # big endian + flags: B # table type and flags +""" + +woff2FlagsSize = sstruct.calcsize(woff2FlagsFormat) + +woff2UnknownTagFormat = """ + > # big endian + tag: 4s # 4-byte tag (optional) +""" + +woff2UnknownTagSize = sstruct.calcsize(woff2UnknownTagFormat) + +woff2UnknownTagIndex = 0x3F + +woff2Base128MaxSize = 5 +woff2DirectoryEntryMaxSize = ( + woff2FlagsSize + woff2UnknownTagSize + 2 * woff2Base128MaxSize +) + +woff2TransformedTableTags = ("glyf", "loca") + +woff2GlyfTableFormat = """ + > # big endian + version: H # = 0x0000 + optionFlags: H # Bit 0: we have overlapSimpleBitmap[], Bits 1-15: reserved + numGlyphs: H # Number of glyphs + indexFormat: H # Offset format for loca table + nContourStreamSize: L # Size of nContour stream + nPointsStreamSize: L # Size of nPoints stream + flagStreamSize: L # Size of flag stream + glyphStreamSize: L # Size of glyph stream + compositeStreamSize: L # Size of composite stream + bboxStreamSize: L # Comnined size of bboxBitmap and bboxStream + instructionStreamSize: L # Size of instruction stream +""" + +woff2GlyfTableFormatSize = sstruct.calcsize(woff2GlyfTableFormat) + +bboxFormat = """ + > # big endian + xMin: h + yMin: h + xMax: h + yMax: h +""" + +woff2OverlapSimpleBitmapFlag = 0x0001 + + +def getKnownTagIndex(tag): + """Return index of 'tag' in woff2KnownTags list. Return 63 if not found.""" + try: + return woff2KnownTags.index(tag) + except ValueError: + return woff2UnknownTagIndex + + +class WOFF2DirectoryEntry(DirectoryEntry): + def fromFile(self, file): + pos = file.tell() + data = file.read(woff2DirectoryEntryMaxSize) + left = self.fromString(data) + consumed = len(data) - len(left) + file.seek(pos + consumed) + + def fromString(self, data): + if len(data) < 1: + raise TTLibError("can't read table 'flags': not enough data") + dummy, data = sstruct.unpack2(woff2FlagsFormat, data, self) + if self.flags & 0x3F == 0x3F: + # if bits [0..5] of the flags byte == 63, read a 4-byte arbitrary tag value + if len(data) < woff2UnknownTagSize: + raise TTLibError("can't read table 'tag': not enough data") + dummy, data = sstruct.unpack2(woff2UnknownTagFormat, data, self) + else: + # otherwise, tag is derived from a fixed 'Known Tags' table + self.tag = woff2KnownTags[self.flags & 0x3F] + self.tag = Tag(self.tag) + self.origLength, data = unpackBase128(data) + self.length = self.origLength + if self.transformed: + self.length, data = unpackBase128(data) + if self.tag == "loca" and self.length != 0: + raise TTLibError("the transformLength of the 'loca' table must be 0") + # return left over data + return data + + def toString(self): + data = bytechr(self.flags) + if (self.flags & 0x3F) == 0x3F: + data += struct.pack(">4s", self.tag.tobytes()) + data += packBase128(self.origLength) + if self.transformed: + data += packBase128(self.length) + return data + + @property + def transformVersion(self): + """Return bits 6-7 of table entry's flags, which indicate the preprocessing + transformation version number (between 0 and 3). + """ + return self.flags >> 6 + + @transformVersion.setter + def transformVersion(self, value): + assert 0 <= value <= 3 + self.flags |= value << 6 + + @property + def transformed(self): + """Return True if the table has any transformation, else return False.""" + # For all tables in a font, except for 'glyf' and 'loca', the transformation + # version 0 indicates the null transform (where the original table data is + # passed directly to the Brotli compressor). For 'glyf' and 'loca' tables, + # transformation version 3 indicates the null transform + if self.tag in {"glyf", "loca"}: + return self.transformVersion != 3 + else: + return self.transformVersion != 0 + + @transformed.setter + def transformed(self, booleanValue): + # here we assume that a non-null transform means version 0 for 'glyf' and + # 'loca' and 1 for every other table (e.g. hmtx); but that may change as + # new transformation formats are introduced in the future (if ever). + if self.tag in {"glyf", "loca"}: + self.transformVersion = 3 if not booleanValue else 0 + else: + self.transformVersion = int(booleanValue) + + +class WOFF2LocaTable(getTableClass("loca")): + """Same as parent class. The only difference is that it attempts to preserve + the 'indexFormat' as encoded in the WOFF2 glyf table. + """ + + def __init__(self, tag=None): + self.tableTag = Tag(tag or "loca") + + def compile(self, ttFont): + try: + max_location = max(self.locations) + except AttributeError: + self.set([]) + max_location = 0 + if "glyf" in ttFont and hasattr(ttFont["glyf"], "indexFormat"): + # copile loca using the indexFormat specified in the WOFF2 glyf table + indexFormat = ttFont["glyf"].indexFormat + if indexFormat == 0: + if max_location >= 0x20000: + raise TTLibError("indexFormat is 0 but local offsets > 0x20000") + if not all(l % 2 == 0 for l in self.locations): + raise TTLibError( + "indexFormat is 0 but local offsets not multiples of 2" + ) + locations = array.array("H") + for location in self.locations: + locations.append(location // 2) + else: + locations = array.array("I", self.locations) + if sys.byteorder != "big": + locations.byteswap() + data = locations.tobytes() + else: + # use the most compact indexFormat given the current glyph offsets + data = super(WOFF2LocaTable, self).compile(ttFont) + return data + + +class WOFF2GlyfTable(getTableClass("glyf")): + """Decoder/Encoder for WOFF2 'glyf' table transform.""" + + subStreams = ( + "nContourStream", + "nPointsStream", + "flagStream", + "glyphStream", + "compositeStream", + "bboxStream", + "instructionStream", + ) + + def __init__(self, tag=None): + self.tableTag = Tag(tag or "glyf") + + def reconstruct(self, data, ttFont): + """Decompile transformed 'glyf' data.""" + inputDataSize = len(data) + + if inputDataSize < woff2GlyfTableFormatSize: + raise TTLibError("not enough 'glyf' data") + dummy, data = sstruct.unpack2(woff2GlyfTableFormat, data, self) + offset = woff2GlyfTableFormatSize + + for stream in self.subStreams: + size = getattr(self, stream + "Size") + setattr(self, stream, data[:size]) + data = data[size:] + offset += size + + hasOverlapSimpleBitmap = self.optionFlags & woff2OverlapSimpleBitmapFlag + self.overlapSimpleBitmap = None + if hasOverlapSimpleBitmap: + overlapSimpleBitmapSize = (self.numGlyphs + 7) >> 3 + self.overlapSimpleBitmap = array.array("B", data[:overlapSimpleBitmapSize]) + offset += overlapSimpleBitmapSize + + if offset != inputDataSize: + raise TTLibError( + "incorrect size of transformed 'glyf' table: expected %d, received %d bytes" + % (offset, inputDataSize) + ) + + bboxBitmapSize = ((self.numGlyphs + 31) >> 5) << 2 + bboxBitmap = self.bboxStream[:bboxBitmapSize] + self.bboxBitmap = array.array("B", bboxBitmap) + self.bboxStream = self.bboxStream[bboxBitmapSize:] + + self.nContourStream = array.array("h", self.nContourStream) + if sys.byteorder != "big": + self.nContourStream.byteswap() + assert len(self.nContourStream) == self.numGlyphs + + if "head" in ttFont: + ttFont["head"].indexToLocFormat = self.indexFormat + try: + self.glyphOrder = ttFont.getGlyphOrder() + except: + self.glyphOrder = None + if self.glyphOrder is None: + self.glyphOrder = [".notdef"] + self.glyphOrder.extend(["glyph%.5d" % i for i in range(1, self.numGlyphs)]) + else: + if len(self.glyphOrder) != self.numGlyphs: + raise TTLibError( + "incorrect glyphOrder: expected %d glyphs, found %d" + % (len(self.glyphOrder), self.numGlyphs) + ) + + glyphs = self.glyphs = {} + for glyphID, glyphName in enumerate(self.glyphOrder): + glyph = self._decodeGlyph(glyphID) + glyphs[glyphName] = glyph + + def transform(self, ttFont): + """Return transformed 'glyf' data""" + self.numGlyphs = len(self.glyphs) + assert len(self.glyphOrder) == self.numGlyphs + if "maxp" in ttFont: + ttFont["maxp"].numGlyphs = self.numGlyphs + self.indexFormat = ttFont["head"].indexToLocFormat + + for stream in self.subStreams: + setattr(self, stream, b"") + bboxBitmapSize = ((self.numGlyphs + 31) >> 5) << 2 + self.bboxBitmap = array.array("B", [0] * bboxBitmapSize) + + self.overlapSimpleBitmap = array.array("B", [0] * ((self.numGlyphs + 7) >> 3)) + for glyphID in range(self.numGlyphs): + try: + self._encodeGlyph(glyphID) + except NotImplementedError: + return None + hasOverlapSimpleBitmap = any(self.overlapSimpleBitmap) + + self.bboxStream = self.bboxBitmap.tobytes() + self.bboxStream + for stream in self.subStreams: + setattr(self, stream + "Size", len(getattr(self, stream))) + self.version = 0 + self.optionFlags = 0 + if hasOverlapSimpleBitmap: + self.optionFlags |= woff2OverlapSimpleBitmapFlag + data = sstruct.pack(woff2GlyfTableFormat, self) + data += bytesjoin([getattr(self, s) for s in self.subStreams]) + if hasOverlapSimpleBitmap: + data += self.overlapSimpleBitmap.tobytes() + return data + + def _decodeGlyph(self, glyphID): + glyph = getTableModule("glyf").Glyph() + glyph.numberOfContours = self.nContourStream[glyphID] + if glyph.numberOfContours == 0: + return glyph + elif glyph.isComposite(): + self._decodeComponents(glyph) + else: + self._decodeCoordinates(glyph) + self._decodeOverlapSimpleFlag(glyph, glyphID) + self._decodeBBox(glyphID, glyph) + return glyph + + def _decodeComponents(self, glyph): + data = self.compositeStream + glyph.components = [] + more = 1 + haveInstructions = 0 + while more: + component = getTableModule("glyf").GlyphComponent() + more, haveInstr, data = component.decompile(data, self) + haveInstructions = haveInstructions | haveInstr + glyph.components.append(component) + self.compositeStream = data + if haveInstructions: + self._decodeInstructions(glyph) + + def _decodeCoordinates(self, glyph): + data = self.nPointsStream + endPtsOfContours = [] + endPoint = -1 + for i in range(glyph.numberOfContours): + ptsOfContour, data = unpack255UShort(data) + endPoint += ptsOfContour + endPtsOfContours.append(endPoint) + glyph.endPtsOfContours = endPtsOfContours + self.nPointsStream = data + self._decodeTriplets(glyph) + self._decodeInstructions(glyph) + + def _decodeOverlapSimpleFlag(self, glyph, glyphID): + if self.overlapSimpleBitmap is None or glyph.numberOfContours <= 0: + return + byte = glyphID >> 3 + bit = glyphID & 7 + if self.overlapSimpleBitmap[byte] & (0x80 >> bit): + glyph.flags[0] |= _g_l_y_f.flagOverlapSimple + + def _decodeInstructions(self, glyph): + glyphStream = self.glyphStream + instructionStream = self.instructionStream + instructionLength, glyphStream = unpack255UShort(glyphStream) + glyph.program = ttProgram.Program() + glyph.program.fromBytecode(instructionStream[:instructionLength]) + self.glyphStream = glyphStream + self.instructionStream = instructionStream[instructionLength:] + + def _decodeBBox(self, glyphID, glyph): + haveBBox = bool(self.bboxBitmap[glyphID >> 3] & (0x80 >> (glyphID & 7))) + if glyph.isComposite() and not haveBBox: + raise TTLibError("no bbox values for composite glyph %d" % glyphID) + if haveBBox: + dummy, self.bboxStream = sstruct.unpack2(bboxFormat, self.bboxStream, glyph) + else: + glyph.recalcBounds(self) + + def _decodeTriplets(self, glyph): + def withSign(flag, baseval): + assert 0 <= baseval and baseval < 65536, "integer overflow" + return baseval if flag & 1 else -baseval + + nPoints = glyph.endPtsOfContours[-1] + 1 + flagSize = nPoints + if flagSize > len(self.flagStream): + raise TTLibError("not enough 'flagStream' data") + flagsData = self.flagStream[:flagSize] + self.flagStream = self.flagStream[flagSize:] + flags = array.array("B", flagsData) + + triplets = array.array("B", self.glyphStream) + nTriplets = len(triplets) + assert nPoints <= nTriplets + + x = 0 + y = 0 + glyph.coordinates = getTableModule("glyf").GlyphCoordinates.zeros(nPoints) + glyph.flags = array.array("B") + tripletIndex = 0 + for i in range(nPoints): + flag = flags[i] + onCurve = not bool(flag >> 7) + flag &= 0x7F + if flag < 84: + nBytes = 1 + elif flag < 120: + nBytes = 2 + elif flag < 124: + nBytes = 3 + else: + nBytes = 4 + assert (tripletIndex + nBytes) <= nTriplets + if flag < 10: + dx = 0 + dy = withSign(flag, ((flag & 14) << 7) + triplets[tripletIndex]) + elif flag < 20: + dx = withSign(flag, (((flag - 10) & 14) << 7) + triplets[tripletIndex]) + dy = 0 + elif flag < 84: + b0 = flag - 20 + b1 = triplets[tripletIndex] + dx = withSign(flag, 1 + (b0 & 0x30) + (b1 >> 4)) + dy = withSign(flag >> 1, 1 + ((b0 & 0x0C) << 2) + (b1 & 0x0F)) + elif flag < 120: + b0 = flag - 84 + dx = withSign(flag, 1 + ((b0 // 12) << 8) + triplets[tripletIndex]) + dy = withSign( + flag >> 1, 1 + (((b0 % 12) >> 2) << 8) + triplets[tripletIndex + 1] + ) + elif flag < 124: + b2 = triplets[tripletIndex + 1] + dx = withSign(flag, (triplets[tripletIndex] << 4) + (b2 >> 4)) + dy = withSign( + flag >> 1, ((b2 & 0x0F) << 8) + triplets[tripletIndex + 2] + ) + else: + dx = withSign( + flag, (triplets[tripletIndex] << 8) + triplets[tripletIndex + 1] + ) + dy = withSign( + flag >> 1, + (triplets[tripletIndex + 2] << 8) + triplets[tripletIndex + 3], + ) + tripletIndex += nBytes + x += dx + y += dy + glyph.coordinates[i] = (x, y) + glyph.flags.append(int(onCurve)) + bytesConsumed = tripletIndex + self.glyphStream = self.glyphStream[bytesConsumed:] + + def _encodeGlyph(self, glyphID): + glyphName = self.getGlyphName(glyphID) + glyph = self[glyphName] + self.nContourStream += struct.pack(">h", glyph.numberOfContours) + if glyph.numberOfContours == 0: + return + elif glyph.isComposite(): + self._encodeComponents(glyph) + else: + self._encodeCoordinates(glyph) + self._encodeOverlapSimpleFlag(glyph, glyphID) + self._encodeBBox(glyphID, glyph) + + def _encodeComponents(self, glyph): + lastcomponent = len(glyph.components) - 1 + more = 1 + haveInstructions = 0 + for i, component in enumerate(glyph.components): + if i == lastcomponent: + haveInstructions = hasattr(glyph, "program") + more = 0 + self.compositeStream += component.compile(more, haveInstructions, self) + if haveInstructions: + self._encodeInstructions(glyph) + + def _encodeCoordinates(self, glyph): + lastEndPoint = -1 + if _g_l_y_f.flagCubic in glyph.flags: + raise NotImplementedError + for endPoint in glyph.endPtsOfContours: + ptsOfContour = endPoint - lastEndPoint + self.nPointsStream += pack255UShort(ptsOfContour) + lastEndPoint = endPoint + self._encodeTriplets(glyph) + self._encodeInstructions(glyph) + + def _encodeOverlapSimpleFlag(self, glyph, glyphID): + if glyph.numberOfContours <= 0: + return + if glyph.flags[0] & _g_l_y_f.flagOverlapSimple: + byte = glyphID >> 3 + bit = glyphID & 7 + self.overlapSimpleBitmap[byte] |= 0x80 >> bit + + def _encodeInstructions(self, glyph): + instructions = glyph.program.getBytecode() + self.glyphStream += pack255UShort(len(instructions)) + self.instructionStream += instructions + + def _encodeBBox(self, glyphID, glyph): + assert glyph.numberOfContours != 0, "empty glyph has no bbox" + if not glyph.isComposite(): + # for simple glyphs, compare the encoded bounding box info with the calculated + # values, and if they match omit the bounding box info + currentBBox = glyph.xMin, glyph.yMin, glyph.xMax, glyph.yMax + calculatedBBox = calcIntBounds(glyph.coordinates) + if currentBBox == calculatedBBox: + return + self.bboxBitmap[glyphID >> 3] |= 0x80 >> (glyphID & 7) + self.bboxStream += sstruct.pack(bboxFormat, glyph) + + def _encodeTriplets(self, glyph): + assert len(glyph.coordinates) == len(glyph.flags) + coordinates = glyph.coordinates.copy() + coordinates.absoluteToRelative() + + flags = array.array("B") + triplets = array.array("B") + for i, (x, y) in enumerate(coordinates): + onCurve = glyph.flags[i] & _g_l_y_f.flagOnCurve + absX = abs(x) + absY = abs(y) + onCurveBit = 0 if onCurve else 128 + xSignBit = 0 if (x < 0) else 1 + ySignBit = 0 if (y < 0) else 1 + xySignBits = xSignBit + 2 * ySignBit + + if x == 0 and absY < 1280: + flags.append(onCurveBit + ((absY & 0xF00) >> 7) + ySignBit) + triplets.append(absY & 0xFF) + elif y == 0 and absX < 1280: + flags.append(onCurveBit + 10 + ((absX & 0xF00) >> 7) + xSignBit) + triplets.append(absX & 0xFF) + elif absX < 65 and absY < 65: + flags.append( + onCurveBit + + 20 + + ((absX - 1) & 0x30) + + (((absY - 1) & 0x30) >> 2) + + xySignBits + ) + triplets.append((((absX - 1) & 0xF) << 4) | ((absY - 1) & 0xF)) + elif absX < 769 and absY < 769: + flags.append( + onCurveBit + + 84 + + 12 * (((absX - 1) & 0x300) >> 8) + + (((absY - 1) & 0x300) >> 6) + + xySignBits + ) + triplets.append((absX - 1) & 0xFF) + triplets.append((absY - 1) & 0xFF) + elif absX < 4096 and absY < 4096: + flags.append(onCurveBit + 120 + xySignBits) + triplets.append(absX >> 4) + triplets.append(((absX & 0xF) << 4) | (absY >> 8)) + triplets.append(absY & 0xFF) + else: + flags.append(onCurveBit + 124 + xySignBits) + triplets.append(absX >> 8) + triplets.append(absX & 0xFF) + triplets.append(absY >> 8) + triplets.append(absY & 0xFF) + + self.flagStream += flags.tobytes() + self.glyphStream += triplets.tobytes() + + +class WOFF2HmtxTable(getTableClass("hmtx")): + def __init__(self, tag=None): + self.tableTag = Tag(tag or "hmtx") + + def reconstruct(self, data, ttFont): + (flags,) = struct.unpack(">B", data[:1]) + data = data[1:] + if flags & 0b11111100 != 0: + raise TTLibError("Bits 2-7 of '%s' flags are reserved" % self.tableTag) + + # When bit 0 is _not_ set, the lsb[] array is present + hasLsbArray = flags & 1 == 0 + # When bit 1 is _not_ set, the leftSideBearing[] array is present + hasLeftSideBearingArray = flags & 2 == 0 + if hasLsbArray and hasLeftSideBearingArray: + raise TTLibError( + "either bits 0 or 1 (or both) must set in transformed '%s' flags" + % self.tableTag + ) + + glyfTable = ttFont["glyf"] + headerTable = ttFont["hhea"] + glyphOrder = glyfTable.glyphOrder + numGlyphs = len(glyphOrder) + numberOfHMetrics = min(int(headerTable.numberOfHMetrics), numGlyphs) + + assert len(data) >= 2 * numberOfHMetrics + advanceWidthArray = array.array("H", data[: 2 * numberOfHMetrics]) + if sys.byteorder != "big": + advanceWidthArray.byteswap() + data = data[2 * numberOfHMetrics :] + + if hasLsbArray: + assert len(data) >= 2 * numberOfHMetrics + lsbArray = array.array("h", data[: 2 * numberOfHMetrics]) + if sys.byteorder != "big": + lsbArray.byteswap() + data = data[2 * numberOfHMetrics :] + else: + # compute (proportional) glyphs' lsb from their xMin + lsbArray = array.array("h") + for i, glyphName in enumerate(glyphOrder): + if i >= numberOfHMetrics: + break + glyph = glyfTable[glyphName] + xMin = getattr(glyph, "xMin", 0) + lsbArray.append(xMin) + + numberOfSideBearings = numGlyphs - numberOfHMetrics + if hasLeftSideBearingArray: + assert len(data) >= 2 * numberOfSideBearings + leftSideBearingArray = array.array("h", data[: 2 * numberOfSideBearings]) + if sys.byteorder != "big": + leftSideBearingArray.byteswap() + data = data[2 * numberOfSideBearings :] + else: + # compute (monospaced) glyphs' leftSideBearing from their xMin + leftSideBearingArray = array.array("h") + for i, glyphName in enumerate(glyphOrder): + if i < numberOfHMetrics: + continue + glyph = glyfTable[glyphName] + xMin = getattr(glyph, "xMin", 0) + leftSideBearingArray.append(xMin) + + if data: + raise TTLibError("too much '%s' table data" % self.tableTag) + + self.metrics = {} + for i in range(numberOfHMetrics): + glyphName = glyphOrder[i] + advanceWidth, lsb = advanceWidthArray[i], lsbArray[i] + self.metrics[glyphName] = (advanceWidth, lsb) + lastAdvance = advanceWidthArray[-1] + for i in range(numberOfSideBearings): + glyphName = glyphOrder[i + numberOfHMetrics] + self.metrics[glyphName] = (lastAdvance, leftSideBearingArray[i]) + + def transform(self, ttFont): + glyphOrder = ttFont.getGlyphOrder() + glyf = ttFont["glyf"] + hhea = ttFont["hhea"] + numberOfHMetrics = hhea.numberOfHMetrics + + # check if any of the proportional glyphs has left sidebearings that + # differ from their xMin bounding box values. + hasLsbArray = False + for i in range(numberOfHMetrics): + glyphName = glyphOrder[i] + lsb = self.metrics[glyphName][1] + if lsb != getattr(glyf[glyphName], "xMin", 0): + hasLsbArray = True + break + + # do the same for the monospaced glyphs (if any) at the end of hmtx table + hasLeftSideBearingArray = False + for i in range(numberOfHMetrics, len(glyphOrder)): + glyphName = glyphOrder[i] + lsb = self.metrics[glyphName][1] + if lsb != getattr(glyf[glyphName], "xMin", 0): + hasLeftSideBearingArray = True + break + + # if we need to encode both sidebearings arrays, then no transformation is + # applicable, and we must use the untransformed hmtx data + if hasLsbArray and hasLeftSideBearingArray: + return + + # set bit 0 and 1 when the respective arrays are _not_ present + flags = 0 + if not hasLsbArray: + flags |= 1 << 0 + if not hasLeftSideBearingArray: + flags |= 1 << 1 + + data = struct.pack(">B", flags) + + advanceWidthArray = array.array( + "H", + [ + self.metrics[glyphName][0] + for i, glyphName in enumerate(glyphOrder) + if i < numberOfHMetrics + ], + ) + if sys.byteorder != "big": + advanceWidthArray.byteswap() + data += advanceWidthArray.tobytes() + + if hasLsbArray: + lsbArray = array.array( + "h", + [ + self.metrics[glyphName][1] + for i, glyphName in enumerate(glyphOrder) + if i < numberOfHMetrics + ], + ) + if sys.byteorder != "big": + lsbArray.byteswap() + data += lsbArray.tobytes() + + if hasLeftSideBearingArray: + leftSideBearingArray = array.array( + "h", + [ + self.metrics[glyphOrder[i]][1] + for i in range(numberOfHMetrics, len(glyphOrder)) + ], + ) + if sys.byteorder != "big": + leftSideBearingArray.byteswap() + data += leftSideBearingArray.tobytes() + + return data + + +class WOFF2FlavorData(WOFFFlavorData): + Flavor = "woff2" + + def __init__(self, reader=None, data=None, transformedTables=None): + """Data class that holds the WOFF2 header major/minor version, any + metadata or private data (as bytes strings), and the set of + table tags that have transformations applied (if reader is not None), + or will have once the WOFF2 font is compiled. + + Args: + reader: an SFNTReader (or subclass) object to read flavor data from. + data: another WOFFFlavorData object to initialise data from. + transformedTables: set of strings containing table tags to be transformed. + + Raises: + ImportError if the brotli module is not installed. + + NOTE: The 'reader' argument, on the one hand, and the 'data' and + 'transformedTables' arguments, on the other hand, are mutually exclusive. + """ + if not haveBrotli: + raise ImportError("No module named brotli") + + if reader is not None: + if data is not None: + raise TypeError("'reader' and 'data' arguments are mutually exclusive") + if transformedTables is not None: + raise TypeError( + "'reader' and 'transformedTables' arguments are mutually exclusive" + ) + + if transformedTables is not None and ( + "glyf" in transformedTables + and "loca" not in transformedTables + or "loca" in transformedTables + and "glyf" not in transformedTables + ): + raise ValueError("'glyf' and 'loca' must be transformed (or not) together") + super(WOFF2FlavorData, self).__init__(reader=reader) + if reader: + transformedTables = [ + tag for tag, entry in reader.tables.items() if entry.transformed + ] + elif data: + self.majorVersion = data.majorVersion + self.majorVersion = data.minorVersion + self.metaData = data.metaData + self.privData = data.privData + if transformedTables is None and hasattr(data, "transformedTables"): + transformedTables = data.transformedTables + + if transformedTables is None: + transformedTables = woff2TransformedTableTags + + self.transformedTables = set(transformedTables) + + def _decompress(self, rawData): + return brotli.decompress(rawData) + + +def unpackBase128(data): + r"""Read one to five bytes from UIntBase128-encoded input string, and return + a tuple containing the decoded integer plus any leftover data. + + >>> unpackBase128(b'\x3f\x00\x00') == (63, b"\x00\x00") + True + >>> unpackBase128(b'\x8f\xff\xff\xff\x7f')[0] == 4294967295 + True + >>> unpackBase128(b'\x80\x80\x3f') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + File "<stdin>", line 1, in ? + TTLibError: UIntBase128 value must not start with leading zeros + >>> unpackBase128(b'\x8f\xff\xff\xff\xff\x7f')[0] # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + File "<stdin>", line 1, in ? + TTLibError: UIntBase128-encoded sequence is longer than 5 bytes + >>> unpackBase128(b'\x90\x80\x80\x80\x00')[0] # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + File "<stdin>", line 1, in ? + TTLibError: UIntBase128 value exceeds 2**32-1 + """ + if len(data) == 0: + raise TTLibError("not enough data to unpack UIntBase128") + result = 0 + if byteord(data[0]) == 0x80: + # font must be rejected if UIntBase128 value starts with 0x80 + raise TTLibError("UIntBase128 value must not start with leading zeros") + for i in range(woff2Base128MaxSize): + if len(data) == 0: + raise TTLibError("not enough data to unpack UIntBase128") + code = byteord(data[0]) + data = data[1:] + # if any of the top seven bits are set then we're about to overflow + if result & 0xFE000000: + raise TTLibError("UIntBase128 value exceeds 2**32-1") + # set current value = old value times 128 bitwise-or (byte bitwise-and 127) + result = (result << 7) | (code & 0x7F) + # repeat until the most significant bit of byte is false + if (code & 0x80) == 0: + # return result plus left over data + return result, data + # make sure not to exceed the size bound + raise TTLibError("UIntBase128-encoded sequence is longer than 5 bytes") + + +def base128Size(n): + """Return the length in bytes of a UIntBase128-encoded sequence with value n. + + >>> base128Size(0) + 1 + >>> base128Size(24567) + 3 + >>> base128Size(2**32-1) + 5 + """ + assert n >= 0 + size = 1 + while n >= 128: + size += 1 + n >>= 7 + return size + + +def packBase128(n): + r"""Encode unsigned integer in range 0 to 2**32-1 (inclusive) to a string of + bytes using UIntBase128 variable-length encoding. Produce the shortest possible + encoding. + + >>> packBase128(63) == b"\x3f" + True + >>> packBase128(2**32-1) == b'\x8f\xff\xff\xff\x7f' + True + """ + if n < 0 or n >= 2**32: + raise TTLibError("UIntBase128 format requires 0 <= integer <= 2**32-1") + data = b"" + size = base128Size(n) + for i in range(size): + b = (n >> (7 * (size - i - 1))) & 0x7F + if i < size - 1: + b |= 0x80 + data += struct.pack("B", b) + return data + + +def unpack255UShort(data): + """Read one to three bytes from 255UInt16-encoded input string, and return a + tuple containing the decoded integer plus any leftover data. + + >>> unpack255UShort(bytechr(252))[0] + 252 + + Note that some numbers (e.g. 506) can have multiple encodings: + >>> unpack255UShort(struct.pack("BB", 254, 0))[0] + 506 + >>> unpack255UShort(struct.pack("BB", 255, 253))[0] + 506 + >>> unpack255UShort(struct.pack("BBB", 253, 1, 250))[0] + 506 + """ + code = byteord(data[:1]) + data = data[1:] + if code == 253: + # read two more bytes as an unsigned short + if len(data) < 2: + raise TTLibError("not enough data to unpack 255UInt16") + (result,) = struct.unpack(">H", data[:2]) + data = data[2:] + elif code == 254: + # read another byte, plus 253 * 2 + if len(data) == 0: + raise TTLibError("not enough data to unpack 255UInt16") + result = byteord(data[:1]) + result += 506 + data = data[1:] + elif code == 255: + # read another byte, plus 253 + if len(data) == 0: + raise TTLibError("not enough data to unpack 255UInt16") + result = byteord(data[:1]) + result += 253 + data = data[1:] + else: + # leave as is if lower than 253 + result = code + # return result plus left over data + return result, data + + +def pack255UShort(value): + r"""Encode unsigned integer in range 0 to 65535 (inclusive) to a bytestring + using 255UInt16 variable-length encoding. + + >>> pack255UShort(252) == b'\xfc' + True + >>> pack255UShort(506) == b'\xfe\x00' + True + >>> pack255UShort(762) == b'\xfd\x02\xfa' + True + """ + if value < 0 or value > 0xFFFF: + raise TTLibError("255UInt16 format requires 0 <= integer <= 65535") + if value < 253: + return struct.pack(">B", value) + elif value < 506: + return struct.pack(">BB", 255, value - 253) + elif value < 762: + return struct.pack(">BB", 254, value - 506) + else: + return struct.pack(">BH", 253, value) + + +def compress(input_file, output_file, transform_tables=None): + """Compress OpenType font to WOFF2. + + Args: + input_file: a file path, file or file-like object (open in binary mode) + containing an OpenType font (either CFF- or TrueType-flavored). + output_file: a file path, file or file-like object where to save the + compressed WOFF2 font. + transform_tables: Optional[Iterable[str]]: a set of table tags for which + to enable preprocessing transformations. By default, only 'glyf' + and 'loca' tables are transformed. An empty set means disable all + transformations. + """ + log.info("Processing %s => %s" % (input_file, output_file)) + + font = TTFont(input_file, recalcBBoxes=False, recalcTimestamp=False) + font.flavor = "woff2" + + if transform_tables is not None: + font.flavorData = WOFF2FlavorData( + data=font.flavorData, transformedTables=transform_tables + ) + + font.save(output_file, reorderTables=False) + + +def decompress(input_file, output_file): + """Decompress WOFF2 font to OpenType font. + + Args: + input_file: a file path, file or file-like object (open in binary mode) + containing a compressed WOFF2 font. + output_file: a file path, file or file-like object where to save the + decompressed OpenType font. + """ + log.info("Processing %s => %s" % (input_file, output_file)) + + font = TTFont(input_file, recalcBBoxes=False, recalcTimestamp=False) + font.flavor = None + font.flavorData = None + font.save(output_file, reorderTables=True) + + +def main(args=None): + """Compress and decompress WOFF2 fonts""" + import argparse + from fontTools import configLogger + from fontTools.ttx import makeOutputFileName + + class _HelpAction(argparse._HelpAction): + def __call__(self, parser, namespace, values, option_string=None): + subparsers_actions = [ + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ] + for subparsers_action in subparsers_actions: + for choice, subparser in subparsers_action.choices.items(): + print(subparser.format_help()) + parser.exit() + + class _NoGlyfTransformAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.transform_tables.difference_update({"glyf", "loca"}) + + class _HmtxTransformAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.transform_tables.add("hmtx") + + parser = argparse.ArgumentParser( + prog="fonttools ttLib.woff2", description=main.__doc__, add_help=False + ) + + parser.add_argument( + "-h", "--help", action=_HelpAction, help="show this help message and exit" + ) + + parser_group = parser.add_subparsers(title="sub-commands") + parser_compress = parser_group.add_parser( + "compress", description="Compress a TTF or OTF font to WOFF2" + ) + parser_decompress = parser_group.add_parser( + "decompress", description="Decompress a WOFF2 font to OTF" + ) + + for subparser in (parser_compress, parser_decompress): + group = subparser.add_mutually_exclusive_group(required=False) + group.add_argument( + "-v", + "--verbose", + action="store_true", + help="print more messages to console", + ) + group.add_argument( + "-q", + "--quiet", + action="store_true", + help="do not print messages to console", + ) + + parser_compress.add_argument( + "input_file", + metavar="INPUT", + help="the input OpenType font (.ttf or .otf)", + ) + parser_decompress.add_argument( + "input_file", + metavar="INPUT", + help="the input WOFF2 font", + ) + + parser_compress.add_argument( + "-o", + "--output-file", + metavar="OUTPUT", + help="the output WOFF2 font", + ) + parser_decompress.add_argument( + "-o", + "--output-file", + metavar="OUTPUT", + help="the output OpenType font", + ) + + transform_group = parser_compress.add_argument_group() + transform_group.add_argument( + "--no-glyf-transform", + dest="transform_tables", + nargs=0, + action=_NoGlyfTransformAction, + help="Do not transform glyf (and loca) tables", + ) + transform_group.add_argument( + "--hmtx-transform", + dest="transform_tables", + nargs=0, + action=_HmtxTransformAction, + help="Enable optional transformation for 'hmtx' table", + ) + + parser_compress.set_defaults( + subcommand=compress, + transform_tables={"glyf", "loca"}, + ) + parser_decompress.set_defaults(subcommand=decompress) + + options = vars(parser.parse_args(args)) + + subcommand = options.pop("subcommand", None) + if not subcommand: + parser.print_help() + return + + quiet = options.pop("quiet") + verbose = options.pop("verbose") + configLogger( + level=("ERROR" if quiet else "DEBUG" if verbose else "INFO"), + ) + + if not options["output_file"]: + if subcommand is compress: + extension = ".woff2" + elif subcommand is decompress: + # choose .ttf/.otf file extension depending on sfntVersion + with open(options["input_file"], "rb") as f: + f.seek(4) # skip 'wOF2' signature + sfntVersion = f.read(4) + assert len(sfntVersion) == 4, "not enough data" + extension = ".otf" if sfntVersion == b"OTTO" else ".ttf" + else: + raise AssertionError(subcommand) + options["output_file"] = makeOutputFileName( + options["input_file"], outputDir=None, extension=extension + ) + + try: + subcommand(**options) + except TTLibError as e: + parser.error(e) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ttx.py b/.venv/lib/python3.11/site-packages/fontTools/ttx.py new file mode 100644 index 0000000000000000000000000000000000000000..1a6330cd4658ab78c3948a44543a41ed38a990f0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ttx.py @@ -0,0 +1,479 @@ +"""\ +usage: ttx [options] inputfile1 [... inputfileN] + +TTX -- From OpenType To XML And Back + +If an input file is a TrueType or OpenType font file, it will be +decompiled to a TTX file (an XML-based text format). +If an input file is a TTX file, it will be compiled to whatever +format the data is in, a TrueType or OpenType/CFF font file. +A special input value of - means read from the standard input. + +Output files are created so they are unique: an existing file is +never overwritten. + +General options +=============== + +-h Help print this message. +--version show version and exit. +-d <outputfolder> Specify a directory where the output files are + to be created. +-o <outputfile> Specify a file to write the output to. A special + value of - would use the standard output. +-f Overwrite existing output file(s), ie. don't append + numbers. +-v Verbose: more messages will be written to stdout + about what is being done. +-q Quiet: No messages will be written to stdout about + what is being done. +-a allow virtual glyphs ID's on compile or decompile. + +Dump options +============ + +-l List table info: instead of dumping to a TTX file, list + some minimal info about each table. +-t <table> Specify a table to dump. Multiple -t options + are allowed. When no -t option is specified, all tables + will be dumped. +-x <table> Specify a table to exclude from the dump. Multiple + -x options are allowed. -t and -x are mutually exclusive. +-s Split tables: save the TTX data into separate TTX files per + table and write one small TTX file that contains references + to the individual table dumps. This file can be used as + input to ttx, as long as the table files are in the + same directory. +-g Split glyf table: Save the glyf data into separate TTX files + per glyph and write a small TTX for the glyf table which + contains references to the individual TTGlyph elements. + NOTE: specifying -g implies -s (no need for -s together + with -g) +-i Do NOT disassemble TT instructions: when this option is + given, all TrueType programs (glyph programs, the font + program and the pre-program) will be written to the TTX + file as hex data instead of assembly. This saves some time + and makes the TTX file smaller. +-z <format> Specify a bitmap data export option for EBDT: + {'raw', 'row', 'bitwise', 'extfile'} or for the CBDT: + {'raw', 'extfile'} Each option does one of the following: + + -z raw + export the bitmap data as a hex dump + -z row + export each row as hex data + -z bitwise + export each row as binary in an ASCII art style + -z extfile + export the data as external files with XML references + + If no export format is specified 'raw' format is used. +-e Don't ignore decompilation errors, but show a full traceback + and abort. +-y <number> Select font number for TrueType Collection (.ttc/.otc), + starting from 0. +--unicodedata <UnicodeData.txt> + Use custom database file to write character names in the + comments of the cmap TTX output. +--newline <value> + Control how line endings are written in the XML file. It + can be 'LF', 'CR', or 'CRLF'. If not specified, the + default platform-specific line endings are used. + +Compile options +=============== + +-m Merge with TrueType-input-file: specify a TrueType or + OpenType font file to be merged with the TTX file. This + option is only valid when at most one TTX file is specified. +-b Don't recalc glyph bounding boxes: use the values in the + TTX file as-is. +--recalc-timestamp + Set font 'modified' timestamp to current time. + By default, the modification time of the TTX file will be + used. +--no-recalc-timestamp + Keep the original font 'modified' timestamp. +--flavor <type> + Specify flavor of output font file. May be 'woff' or 'woff2'. + Note that WOFF2 requires the Brotli Python extension, + available at https://github.com/google/brotli +--with-zopfli + Use Zopfli instead of Zlib to compress WOFF. The Python + extension is available at https://pypi.python.org/pypi/zopfli +--optimize-font-speed + Enable optimizations that prioritize speed over file size. + This mainly affects how glyf t able and gvar / VARC tables are + compiled. The produced fonts will be larger, but rendering + performance will be improved with HarfBuzz and other text + layout engines. +""" + +from fontTools.ttLib import OPTIMIZE_FONT_SPEED, TTFont, TTLibError +from fontTools.misc.macCreatorType import getMacCreatorAndType +from fontTools.unicode import setUnicodeData +from fontTools.misc.textTools import Tag, tostr +from fontTools.misc.timeTools import timestampSinceEpoch +from fontTools.misc.loggingTools import Timer +from fontTools.misc.cliTools import makeOutputFileName +import os +import sys +import getopt +import re +import logging + + +log = logging.getLogger("fontTools.ttx") + +opentypeheaderRE = re.compile("""sfntVersion=['"]OTTO["']""") + + +class Options(object): + listTables = False + outputDir = None + outputFile = None + overWrite = False + verbose = False + quiet = False + splitTables = False + splitGlyphs = False + disassembleInstructions = True + mergeFile = None + recalcBBoxes = True + ignoreDecompileErrors = True + bitmapGlyphDataFormat = "raw" + unicodedata = None + newlinestr = "\n" + recalcTimestamp = None + flavor = None + useZopfli = False + optimizeFontSpeed = False + + def __init__(self, rawOptions, numFiles): + self.onlyTables = [] + self.skipTables = [] + self.fontNumber = -1 + for option, value in rawOptions: + # general options + if option == "-h": + print(__doc__) + sys.exit(0) + elif option == "--version": + from fontTools import version + + print(version) + sys.exit(0) + elif option == "-d": + if not os.path.isdir(value): + raise getopt.GetoptError( + "The -d option value must be an existing directory" + ) + self.outputDir = value + elif option == "-o": + self.outputFile = value + elif option == "-f": + self.overWrite = True + elif option == "-v": + self.verbose = True + elif option == "-q": + self.quiet = True + # dump options + elif option == "-l": + self.listTables = True + elif option == "-t": + # pad with space if table tag length is less than 4 + value = value.ljust(4) + self.onlyTables.append(value) + elif option == "-x": + # pad with space if table tag length is less than 4 + value = value.ljust(4) + self.skipTables.append(value) + elif option == "-s": + self.splitTables = True + elif option == "-g": + # -g implies (and forces) splitTables + self.splitGlyphs = True + self.splitTables = True + elif option == "-i": + self.disassembleInstructions = False + elif option == "-z": + validOptions = ("raw", "row", "bitwise", "extfile") + if value not in validOptions: + raise getopt.GetoptError( + "-z does not allow %s as a format. Use %s" + % (option, validOptions) + ) + self.bitmapGlyphDataFormat = value + elif option == "-y": + self.fontNumber = int(value) + # compile options + elif option == "-m": + self.mergeFile = value + elif option == "-b": + self.recalcBBoxes = False + elif option == "-e": + self.ignoreDecompileErrors = False + elif option == "--unicodedata": + self.unicodedata = value + elif option == "--newline": + validOptions = ("LF", "CR", "CRLF") + if value == "LF": + self.newlinestr = "\n" + elif value == "CR": + self.newlinestr = "\r" + elif value == "CRLF": + self.newlinestr = "\r\n" + else: + raise getopt.GetoptError( + "Invalid choice for --newline: %r (choose from %s)" + % (value, ", ".join(map(repr, validOptions))) + ) + elif option == "--recalc-timestamp": + self.recalcTimestamp = True + elif option == "--no-recalc-timestamp": + self.recalcTimestamp = False + elif option == "--flavor": + self.flavor = value + elif option == "--with-zopfli": + self.useZopfli = True + elif option == "--optimize-font-speed": + self.optimizeFontSpeed = True + if self.verbose and self.quiet: + raise getopt.GetoptError("-q and -v options are mutually exclusive") + if self.verbose: + self.logLevel = logging.DEBUG + elif self.quiet: + self.logLevel = logging.WARNING + else: + self.logLevel = logging.INFO + if self.mergeFile and self.flavor: + raise getopt.GetoptError("-m and --flavor options are mutually exclusive") + if self.onlyTables and self.skipTables: + raise getopt.GetoptError("-t and -x options are mutually exclusive") + if self.mergeFile and numFiles > 1: + raise getopt.GetoptError( + "Must specify exactly one TTX source file when using -m" + ) + if self.flavor != "woff" and self.useZopfli: + raise getopt.GetoptError("--with-zopfli option requires --flavor 'woff'") + + +def ttList(input, output, options): + ttf = TTFont(input, fontNumber=options.fontNumber, lazy=True) + reader = ttf.reader + tags = sorted(reader.keys()) + print('Listing table info for "%s":' % input) + format = " %4s %10s %8s %8s" + print(format % ("tag ", " checksum", " length", " offset")) + print(format % ("----", "----------", "--------", "--------")) + for tag in tags: + entry = reader.tables[tag] + if ttf.flavor == "woff2": + # WOFF2 doesn't store table checksums, so they must be calculated + from fontTools.ttLib.sfnt import calcChecksum + + data = entry.loadData(reader.transformBuffer) + checkSum = calcChecksum(data) + else: + checkSum = int(entry.checkSum) + if checkSum < 0: + checkSum = checkSum + 0x100000000 + checksum = "0x%08X" % checkSum + print(format % (tag, checksum, entry.length, entry.offset)) + print() + ttf.close() + + +@Timer(log, "Done dumping TTX in %(time).3f seconds") +def ttDump(input, output, options): + input_name = input + if input == "-": + input, input_name = sys.stdin.buffer, sys.stdin.name + output_name = output + if output == "-": + output, output_name = sys.stdout, sys.stdout.name + log.info('Dumping "%s" to "%s"...', input_name, output_name) + if options.unicodedata: + setUnicodeData(options.unicodedata) + ttf = TTFont( + input, + 0, + ignoreDecompileErrors=options.ignoreDecompileErrors, + fontNumber=options.fontNumber, + ) + ttf.saveXML( + output, + tables=options.onlyTables, + skipTables=options.skipTables, + splitTables=options.splitTables, + splitGlyphs=options.splitGlyphs, + disassembleInstructions=options.disassembleInstructions, + bitmapGlyphDataFormat=options.bitmapGlyphDataFormat, + newlinestr=options.newlinestr, + ) + ttf.close() + + +@Timer(log, "Done compiling TTX in %(time).3f seconds") +def ttCompile(input, output, options): + input_name = input + if input == "-": + input, input_name = sys.stdin, sys.stdin.name + output_name = output + if output == "-": + output, output_name = sys.stdout.buffer, sys.stdout.name + log.info('Compiling "%s" to "%s"...' % (input_name, output)) + if options.useZopfli: + from fontTools.ttLib import sfnt + + sfnt.USE_ZOPFLI = True + ttf = TTFont( + options.mergeFile, + flavor=options.flavor, + recalcBBoxes=options.recalcBBoxes, + recalcTimestamp=options.recalcTimestamp, + ) + if options.optimizeFontSpeed: + ttf.cfg[OPTIMIZE_FONT_SPEED] = options.optimizeFontSpeed + ttf.importXML(input) + + if options.recalcTimestamp is None and "head" in ttf and input is not sys.stdin: + # use TTX file modification time for head "modified" timestamp + mtime = os.path.getmtime(input) + ttf["head"].modified = timestampSinceEpoch(mtime) + + ttf.save(output) + + +def guessFileType(fileName): + if fileName == "-": + header = sys.stdin.buffer.peek(256) + ext = "" + else: + base, ext = os.path.splitext(fileName) + try: + with open(fileName, "rb") as f: + header = f.read(256) + except IOError: + return None + + if header.startswith(b"\xef\xbb\xbf<?xml"): + header = header.lstrip(b"\xef\xbb\xbf") + cr, tp = getMacCreatorAndType(fileName) + if tp in ("sfnt", "FFIL"): + return "TTF" + if ext == ".dfont": + return "TTF" + head = Tag(header[:4]) + if head == "OTTO": + return "OTF" + elif head == "ttcf": + return "TTC" + elif head in ("\0\1\0\0", "true"): + return "TTF" + elif head == "wOFF": + return "WOFF" + elif head == "wOF2": + return "WOFF2" + elif head == "<?xm": + # Use 'latin1' because that can't fail. + header = tostr(header, "latin1") + if opentypeheaderRE.search(header): + return "OTX" + else: + return "TTX" + return None + + +def parseOptions(args): + rawOptions, files = getopt.gnu_getopt( + args, + "ld:o:fvqht:x:sgim:z:baey:", + [ + "unicodedata=", + "recalc-timestamp", + "no-recalc-timestamp", + "flavor=", + "version", + "with-zopfli", + "newline=", + "optimize-font-speed", + ], + ) + + options = Options(rawOptions, len(files)) + jobs = [] + + if not files: + raise getopt.GetoptError("Must specify at least one input file") + + for input in files: + if input != "-" and not os.path.isfile(input): + raise getopt.GetoptError('File not found: "%s"' % input) + tp = guessFileType(input) + if tp in ("OTF", "TTF", "TTC", "WOFF", "WOFF2"): + extension = ".ttx" + if options.listTables: + action = ttList + else: + action = ttDump + elif tp == "TTX": + extension = "." + options.flavor if options.flavor else ".ttf" + action = ttCompile + elif tp == "OTX": + extension = "." + options.flavor if options.flavor else ".otf" + action = ttCompile + else: + raise getopt.GetoptError('Unknown file type: "%s"' % input) + + if options.outputFile: + output = options.outputFile + else: + if input == "-": + raise getopt.GetoptError("Must provide -o when reading from stdin") + output = makeOutputFileName( + input, options.outputDir, extension, options.overWrite + ) + # 'touch' output file to avoid race condition in choosing file names + if action != ttList: + open(output, "a").close() + jobs.append((action, input, output)) + return jobs, options + + +def process(jobs, options): + for action, input, output in jobs: + action(input, output, options) + + +def main(args=None): + """Convert OpenType fonts to XML and back""" + from fontTools import configLogger + + if args is None: + args = sys.argv[1:] + try: + jobs, options = parseOptions(args) + except getopt.GetoptError as e: + print("%s\nERROR: %s" % (__doc__, e), file=sys.stderr) + sys.exit(2) + + configLogger(level=options.logLevel) + + try: + process(jobs, options) + except KeyboardInterrupt: + log.error("(Cancelled.)") + sys.exit(1) + except SystemExit: + raise + except TTLibError as e: + log.error(e) + sys.exit(1) + except: + log.exception("Unhandled exception has occurred") + sys.exit(1) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/__init__.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b0e425ff06543d0cc4de40b48184ffb350596835 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/__init__.py @@ -0,0 +1,2575 @@ +""" +A library for importing .ufo files and their descendants. +Refer to http://unifiedfontobject.org for the UFO specification. + +The main interfaces are the :class:`.UFOReader` and :class:`.UFOWriter` +classes, which support versions 1, 2, and 3 of the UFO specification. + +Set variables are available for external use that list the font +info attribute names for the `fontinfo.plist` formats. These are: + +- :obj:`.fontInfoAttributesVersion1` +- :obj:`.fontInfoAttributesVersion2` +- :obj:`.fontInfoAttributesVersion3` + +A set listing the `fontinfo.plist` attributes that were deprecated +in version 2 is available for external use: + +- :obj:`.deprecatedFontInfoAttributesVersion2` + +Functions that do basic validation on values for `fontinfo.plist` +are available for external use. These are + +- :func:`.validateFontInfoVersion2ValueForAttribute` +- :func:`.validateFontInfoVersion3ValueForAttribute` + +Value conversion functions are available for converting +`fontinfo.plist` values between the possible format versions. + +- :func:`.convertFontInfoValueForAttributeFromVersion1ToVersion2` +- :func:`.convertFontInfoValueForAttributeFromVersion2ToVersion1` +- :func:`.convertFontInfoValueForAttributeFromVersion2ToVersion3` +- :func:`.convertFontInfoValueForAttributeFromVersion3ToVersion2` +""" + +from __future__ import annotations + +import enum +import logging +import os +import zipfile +from collections import OrderedDict +from copy import deepcopy +from os import fsdecode +from typing import IO, TYPE_CHECKING, Any, Optional, Union, cast + +from fontTools.misc import filesystem as fs +from fontTools.misc import plistlib +from fontTools.ufoLib.converters import convertUFO1OrUFO2KerningToUFO3Kerning +from fontTools.ufoLib.errors import UFOLibError +from fontTools.ufoLib.filenames import userNameToFileName +from fontTools.ufoLib.utils import ( + BaseFormatVersion, + normalizeFormatVersion, + numberTypes, +) +from fontTools.ufoLib.validators import * + +if TYPE_CHECKING: + from logging import Logger + + from fontTools.annotations import ( + GlyphNameToFileNameFunc, + K, + KerningDict, + KerningGroups, + KerningNested, + PathOrFS, + PathStr, + UFOFormatVersionInput, + V, + ) + from fontTools.misc.filesystem._base import FS + from fontTools.ufoLib.glifLib import GlyphSet + +KerningGroupRenameMaps = dict[str, dict[str, str]] +LibDict = dict[str, Any] +LayerOrderList = Optional[list[Optional[str]]] +AttributeDataDict = dict[str, Any] +FontInfoAttributes = dict[str, AttributeDataDict] + +# client code can check this to see if the upstream `fs` package is being used +haveFS = fs._haveFS + +__all__: list[str] = [ + "haveFS", + "makeUFOPath", + "UFOLibError", + "UFOReader", + "UFOWriter", + "UFOReaderWriter", + "UFOFileStructure", + "fontInfoAttributesVersion1", + "fontInfoAttributesVersion2", + "fontInfoAttributesVersion3", + "deprecatedFontInfoAttributesVersion2", + "validateFontInfoVersion2ValueForAttribute", + "validateFontInfoVersion3ValueForAttribute", + "convertFontInfoValueForAttributeFromVersion1ToVersion2", + "convertFontInfoValueForAttributeFromVersion2ToVersion1", +] + +__version__: str = "3.0.0" + + +logger: Logger = logging.getLogger(__name__) + + +# --------- +# Constants +# --------- + +DEFAULT_GLYPHS_DIRNAME: str = "glyphs" +DATA_DIRNAME: str = "data" +IMAGES_DIRNAME: str = "images" +METAINFO_FILENAME: str = "metainfo.plist" +FONTINFO_FILENAME: str = "fontinfo.plist" +LIB_FILENAME: str = "lib.plist" +GROUPS_FILENAME: str = "groups.plist" +KERNING_FILENAME: str = "kerning.plist" +FEATURES_FILENAME: str = "features.fea" +LAYERCONTENTS_FILENAME: str = "layercontents.plist" +LAYERINFO_FILENAME: str = "layerinfo.plist" + +DEFAULT_LAYER_NAME: str = "public.default" + + +class UFOFormatVersion(BaseFormatVersion): + FORMAT_1_0 = (1, 0) + FORMAT_2_0 = (2, 0) + FORMAT_3_0 = (3, 0) + + +class UFOFileStructure(enum.Enum): + ZIP = "zip" + PACKAGE = "package" + + +# -------------- +# Shared Methods +# -------------- + + +class _UFOBaseIO: + if TYPE_CHECKING: + fs: FS + _havePreviousFile: bool + + def getFileModificationTime(self, path: PathStr) -> Optional[float]: + """ + Returns the modification time for the file at the given path, as a + floating point number giving the number of seconds since the epoch. + The path must be relative to the UFO path. + Returns None if the file does not exist. + """ + try: + dt = self.fs.getinfo(fsdecode(path), namespaces=["details"]).modified + except (fs.errors.MissingInfoNamespace, fs.errors.ResourceNotFound): + return None + else: + if dt is not None: + return dt.timestamp() + return None + + def _getPlist(self, fileName: str, default: Optional[Any] = None) -> Any: + """ + Read a property list relative to the UFO filesystem's root. + Raises UFOLibError if the file is missing and default is None, + otherwise default is returned. + + The errors that could be raised during the reading of a plist are + unpredictable and/or too large to list, so, a blind try: except: + is done. If an exception occurs, a UFOLibError will be raised. + """ + try: + with self.fs.open(fileName, "rb") as f: + return plistlib.load(f) + except fs.errors.ResourceNotFound: + if default is None: + raise UFOLibError( + "'%s' is missing on %s. This file is required" % (fileName, self.fs) + ) + else: + return default + except Exception as e: + # TODO(anthrotype): try to narrow this down a little + raise UFOLibError(f"'{fileName}' could not be read on {self.fs}: {e}") + + def _writePlist(self, fileName: str, obj: Any) -> None: + """ + Write a property list to a file relative to the UFO filesystem's root. + + Do this sort of atomically, making it harder to corrupt existing files, + for example when plistlib encounters an error halfway during write. + This also checks to see if text matches the text that is already in the + file at path. If so, the file is not rewritten so that the modification + date is preserved. + + The errors that could be raised during the writing of a plist are + unpredictable and/or too large to list, so, a blind try: except: is done. + If an exception occurs, a UFOLibError will be raised. + """ + if self._havePreviousFile: + try: + data = plistlib.dumps(obj) + except Exception as e: + raise UFOLibError( + "'%s' could not be written on %s because " + "the data is not properly formatted: %s" % (fileName, self.fs, e) + ) + if self.fs.exists(fileName) and data == self.fs.readbytes(fileName): + return + self.fs.writebytes(fileName, data) + else: + with self.fs.open(fileName, mode="wb") as fp: + try: + plistlib.dump(obj, fp) + except Exception as e: + raise UFOLibError( + "'%s' could not be written on %s because " + "the data is not properly formatted: %s" + % (fileName, self.fs, e) + ) + + +# ---------- +# UFO Reader +# ---------- + + +class UFOReader(_UFOBaseIO): + """Read the various components of a .ufo. + + Attributes: + path: An :class:`os.PathLike` object pointing to the .ufo. + validate: A boolean indicating if the data read should be + validated. Defaults to `True`. + + By default read data is validated. Set ``validate`` to + ``False`` to not validate the data. + """ + + def __init__(self, path: PathOrFS, validate: bool = True) -> None: + # Only call __fspath__ if path is not already a str or FS object + if not isinstance(path, (str, fs.base.FS)) and hasattr(path, "__fspath__"): + path = path.__fspath__() + + if isinstance(path, str): + structure = _sniffFileStructure(path) + parentFS: FS + try: + if structure is UFOFileStructure.ZIP: + parentFS = fs.zipfs.ZipFS(path, write=False, encoding="utf-8") # type: ignore[abstract] + else: + parentFS = fs.osfs.OSFS(path) + except fs.errors.CreateFailed as e: + raise UFOLibError(f"unable to open '{path}': {e}") + + if structure is UFOFileStructure.ZIP: + # .ufoz zip files must contain a single root directory, with arbitrary + # name, containing all the UFO files + rootDirs = [ + p.name + for p in parentFS.scandir("/") + # exclude macOS metadata contained in zip file + if p.is_dir and p.name != "__MACOSX" + ] + if len(rootDirs) == 1: + # 'ClosingSubFS' ensures that the parent zip file is closed when + # its root subdirectory is closed + self.fs: FS = parentFS.opendir( + rootDirs[0], factory=fs.subfs.ClosingSubFS + ) + else: + raise UFOLibError( + "Expected exactly 1 root directory, found %d" % len(rootDirs) + ) + else: + # normal UFO 'packages' are just a single folder + self.fs = parentFS + # when passed a path string, we make sure we close the newly opened fs + # upon calling UFOReader.close method or context manager's __exit__ + self._shouldClose: bool = True + self._fileStructure = structure + elif isinstance(path, fs.base.FS): + filesystem: FS = path + try: + filesystem.check() + except fs.errors.FilesystemClosed: + raise UFOLibError("the filesystem '%s' is closed" % path) + else: + self.fs = filesystem + try: + path = filesystem.getsyspath("/") + except fs.errors.NoSysPath: + # network or in-memory FS may not map to the local one + path = str(filesystem) + # when user passed an already initialized fs instance, it is her + # responsibility to close it, thus UFOReader.close/__exit__ are no-op + self._shouldClose = False + # default to a 'package' structure + self._fileStructure = UFOFileStructure.PACKAGE + else: + raise TypeError( + "Expected a path string or fs.base.FS object, found '%s'" + % type(path).__name__ + ) + self._path: str = fsdecode(path) + self._validate: bool = validate + self._upConvertedKerningData: Optional[dict[str, Any]] = None + + try: + self.readMetaInfo(validate=validate) + except UFOLibError: + self.close() + raise + + # properties + + def _get_path(self) -> str: + import warnings + + warnings.warn( + "The 'path' attribute is deprecated; use the 'fs' attribute instead", + DeprecationWarning, + stacklevel=2, + ) + return self._path + + path: property = property(_get_path, doc="The path of the UFO (DEPRECATED).") + + def _get_formatVersion(self) -> int: + import warnings + + warnings.warn( + "The 'formatVersion' attribute is deprecated; use the 'formatVersionTuple'", + DeprecationWarning, + stacklevel=2, + ) + return self._formatVersion.major + + formatVersion = property( + _get_formatVersion, + doc="The (major) format version of the UFO. DEPRECATED: Use formatVersionTuple", + ) + + @property + def formatVersionTuple(self) -> tuple[int, int]: + """The (major, minor) format version of the UFO. + This is determined by reading metainfo.plist during __init__. + """ + return self._formatVersion + + def _get_fileStructure(self) -> Any: + return self._fileStructure + + fileStructure: property = property( + _get_fileStructure, + doc=( + "The file structure of the UFO: " + "either UFOFileStructure.ZIP or UFOFileStructure.PACKAGE" + ), + ) + + # up conversion + + def _upConvertKerning(self, validate: bool) -> None: + """ + Up convert kerning and groups in UFO 1 and 2. + The data will be held internally until each bit of data + has been retrieved. The conversion of both must be done + at once, so the raw data is cached and an error is raised + if one bit of data becomes obsolete before it is called. + + ``validate`` will validate the data. + """ + if self._upConvertedKerningData: + testKerning = self._readKerning() + if testKerning != self._upConvertedKerningData["originalKerning"]: + raise UFOLibError( + "The data in kerning.plist has been modified since it was converted to UFO 3 format." + ) + testGroups = self._readGroups() + if testGroups != self._upConvertedKerningData["originalGroups"]: + raise UFOLibError( + "The data in groups.plist has been modified since it was converted to UFO 3 format." + ) + else: + groups = self._readGroups() + if validate: + invalidFormatMessage = "groups.plist is not properly formatted." + if not isinstance(groups, dict): + raise UFOLibError(invalidFormatMessage) + for groupName, glyphList in groups.items(): + if not isinstance(groupName, str): + raise UFOLibError(invalidFormatMessage) + elif not isinstance(glyphList, list): + raise UFOLibError(invalidFormatMessage) + for glyphName in glyphList: + if not isinstance(glyphName, str): + raise UFOLibError(invalidFormatMessage) + self._upConvertedKerningData = dict( + kerning={}, + originalKerning=self._readKerning(), + groups={}, + originalGroups=groups, + ) + # convert kerning and groups + kerning, groups, conversionMaps = convertUFO1OrUFO2KerningToUFO3Kerning( + self._upConvertedKerningData["originalKerning"], + deepcopy(self._upConvertedKerningData["originalGroups"]), + self.getGlyphSet(), + ) + # store + self._upConvertedKerningData["kerning"] = kerning + self._upConvertedKerningData["groups"] = groups + self._upConvertedKerningData["groupRenameMaps"] = conversionMaps + + # support methods + + def readBytesFromPath(self, path: PathStr) -> Optional[bytes]: + """ + Returns the bytes in the file at the given path. + The path must be relative to the UFO's filesystem root. + Returns None if the file does not exist. + """ + try: + return self.fs.readbytes(fsdecode(path)) + except fs.errors.ResourceNotFound: + return None + + def getReadFileForPath( + self, path: PathStr, encoding: Optional[str] = None + ) -> Optional[Union[IO[bytes], IO[str]]]: + """ + Returns a file (or file-like) object for the file at the given path. + The path must be relative to the UFO path. + Returns None if the file does not exist. + By default the file is opened in binary mode (reads bytes). + If encoding is passed, the file is opened in text mode (reads str). + + Note: The caller is responsible for closing the open file. + """ + path = fsdecode(path) + try: + if encoding is None: + return self.fs.open(path, mode="rb") + else: + return self.fs.open(path, mode="r", encoding=encoding) + except fs.errors.ResourceNotFound: + return None + + # metainfo.plist + + def _readMetaInfo(self, validate: Optional[bool] = None) -> dict[str, Any]: + """ + Read metainfo.plist and return raw data. Only used for internal operations. + + ``validate`` will validate the read data, by default it is set + to the class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + data = self._getPlist(METAINFO_FILENAME) + if validate and not isinstance(data, dict): + raise UFOLibError("metainfo.plist is not properly formatted.") + try: + formatVersionMajor = data["formatVersion"] + except KeyError: + raise UFOLibError( + f"Missing required formatVersion in '{METAINFO_FILENAME}' on {self.fs}" + ) + formatVersionMinor = data.setdefault("formatVersionMinor", 0) + + try: + formatVersion = UFOFormatVersion((formatVersionMajor, formatVersionMinor)) + except ValueError as e: + unsupportedMsg = ( + f"Unsupported UFO format ({formatVersionMajor}.{formatVersionMinor}) " + f"in '{METAINFO_FILENAME}' on {self.fs}" + ) + if validate: + from fontTools.ufoLib.errors import UnsupportedUFOFormat + + raise UnsupportedUFOFormat(unsupportedMsg) from e + + formatVersion = UFOFormatVersion.default() + logger.warning( + "%s. Assuming the latest supported version (%s). " + "Some data may be skipped or parsed incorrectly", + unsupportedMsg, + formatVersion, + ) + data["formatVersionTuple"] = formatVersion + return data + + def readMetaInfo(self, validate: Optional[bool] = None) -> None: + """ + Read metainfo.plist and set formatVersion. Only used for internal operations. + + ``validate`` will validate the read data, by default it is set + to the class's validate value, can be overridden. + """ + data = self._readMetaInfo(validate=validate) + self._formatVersion = data["formatVersionTuple"] + + # groups.plist + + def _readGroups(self) -> dict[str, list[str]]: + groups = self._getPlist(GROUPS_FILENAME, {}) + # remove any duplicate glyphs in a kerning group + for groupName, glyphList in groups.items(): + if groupName.startswith(("public.kern1.", "public.kern2.")): + groups[groupName] = list(OrderedDict.fromkeys(glyphList)) + return groups + + def readGroups(self, validate: Optional[bool] = None) -> dict[str, list[str]]: + """ + Read groups.plist. Returns a dict. + ``validate`` will validate the read data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + # handle up conversion + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + self._upConvertKerning(validate) + groups = cast(dict, self._upConvertedKerningData)["groups"] + # normal + else: + groups = self._readGroups() + if validate: + valid, message = groupsValidator(groups) + if not valid: + raise UFOLibError(message) + return groups + + def getKerningGroupConversionRenameMaps( + self, validate: Optional[bool] = None + ) -> KerningGroupRenameMaps: + """ + Get maps defining the renaming that was done during any + needed kerning group conversion. This method returns a + dictionary of this form:: + + { + "side1" : {"old group name" : "new group name"}, + "side2" : {"old group name" : "new group name"} + } + + When no conversion has been performed, the side1 and side2 + dictionaries will be empty. + + ``validate`` will validate the groups, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + if self._formatVersion >= UFOFormatVersion.FORMAT_3_0: + return dict(side1={}, side2={}) + # use the public group reader to force the load and + # conversion of the data if it hasn't happened yet. + self.readGroups(validate=validate) + return cast(dict, self._upConvertedKerningData)["groupRenameMaps"] + + # fontinfo.plist + + def _readInfo(self, validate: bool) -> dict[str, Any]: + data = self._getPlist(FONTINFO_FILENAME, {}) + if validate and not isinstance(data, dict): + raise UFOLibError("fontinfo.plist is not properly formatted.") + return data + + def readInfo(self, info: Any, validate: Optional[bool] = None) -> None: + """ + Read fontinfo.plist. It requires an object that allows + setting attributes with names that follow the fontinfo.plist + version 3 specification. This will write the attributes + defined in the file into the object. + + ``validate`` will validate the read data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + infoDict = self._readInfo(validate) + infoDataToSet = {} + # version 1 + if self._formatVersion == UFOFormatVersion.FORMAT_1_0: + for attr in fontInfoAttributesVersion1: + value = infoDict.get(attr) + if value is not None: + infoDataToSet[attr] = value + infoDataToSet = _convertFontInfoDataVersion1ToVersion2(infoDataToSet) + infoDataToSet = _convertFontInfoDataVersion2ToVersion3(infoDataToSet) + # version 2 + elif self._formatVersion == UFOFormatVersion.FORMAT_2_0: + for attr, dataValidationDict in list( + fontInfoAttributesVersion2ValueData.items() + ): + value = infoDict.get(attr) + if value is None: + continue + infoDataToSet[attr] = value + infoDataToSet = _convertFontInfoDataVersion2ToVersion3(infoDataToSet) + # version 3.x + elif self._formatVersion.major == UFOFormatVersion.FORMAT_3_0.major: + for attr, dataValidationDict in list( + fontInfoAttributesVersion3ValueData.items() + ): + value = infoDict.get(attr) + if value is None: + continue + infoDataToSet[attr] = value + # unsupported version + else: + raise NotImplementedError(self._formatVersion) + # validate data + if validate: + infoDataToSet = validateInfoVersion3Data(infoDataToSet) + # populate the object + for attr, value in list(infoDataToSet.items()): + try: + setattr(info, attr, value) + except AttributeError: + raise UFOLibError( + "The supplied info object does not support setting a necessary attribute (%s)." + % attr + ) + + # kerning.plist + + def _readKerning(self) -> KerningNested: + data = self._getPlist(KERNING_FILENAME, {}) + return data + + def readKerning(self, validate: Optional[bool] = None) -> KerningDict: + """ + Read kerning.plist. Returns a dict. + + ``validate`` will validate the kerning data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + # handle up conversion + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + self._upConvertKerning(validate) + kerningNested = cast(dict, self._upConvertedKerningData)["kerning"] + # normal + else: + kerningNested = self._readKerning() + if validate: + valid, message = kerningValidator(kerningNested) + if not valid: + raise UFOLibError(message) + # flatten + kerning = {} + for left in kerningNested: + for right in kerningNested[left]: + value = kerningNested[left][right] + kerning[left, right] = value + return kerning + + # lib.plist + + def readLib(self, validate: Optional[bool] = None) -> dict[str, Any]: + """ + Read lib.plist. Returns a dict. + + ``validate`` will validate the data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + data = self._getPlist(LIB_FILENAME, {}) + if validate: + valid, message = fontLibValidator(data) + if not valid: + raise UFOLibError(message) + return data + + # features.fea + + def readFeatures(self) -> str: + """ + Read features.fea. Return a string. + The returned string is empty if the file is missing. + """ + try: + with self.fs.open(FEATURES_FILENAME, "r", encoding="utf-8-sig") as f: + return f.read() + except fs.errors.ResourceNotFound: + return "" + + # glyph sets & layers + + def _readLayerContents(self, validate: bool) -> list[tuple[str, str]]: + """ + Rebuild the layer contents list by checking what glyphsets + are available on disk. + + ``validate`` will validate the layer contents. + """ + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + return [(DEFAULT_LAYER_NAME, DEFAULT_GLYPHS_DIRNAME)] + contents = self._getPlist(LAYERCONTENTS_FILENAME) + if validate: + valid, error = layerContentsValidator(contents, self.fs) + if not valid: + raise UFOLibError(error) + return contents + + def getLayerNames(self, validate: Optional[bool] = None) -> list[str]: + """ + Get the ordered layer names from layercontents.plist. + + ``validate`` will validate the data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + layerContents = self._readLayerContents(validate) + layerNames = [layerName for layerName, directoryName in layerContents] + return layerNames + + def getDefaultLayerName(self, validate: Optional[bool] = None) -> str: + """ + Get the default layer name from layercontents.plist. + + ``validate`` will validate the data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + layerContents = self._readLayerContents(validate) + for layerName, layerDirectory in layerContents: + if layerDirectory == DEFAULT_GLYPHS_DIRNAME: + return layerName + # this will already have been raised during __init__ + raise UFOLibError("The default layer is not defined in layercontents.plist.") + + def getGlyphSet( + self, + layerName: Optional[str] = None, + validateRead: Optional[bool] = None, + validateWrite: Optional[bool] = None, + ) -> GlyphSet: + """ + Return the GlyphSet associated with the + glyphs directory mapped to layerName + in the UFO. If layerName is not provided, + the name retrieved with getDefaultLayerName + will be used. + + ``validateRead`` will validate the read data, by default it is set to the + class's validate value, can be overridden. + ``validateWrite`` will validate the written data, by default it is set to the + class's validate value, can be overridden. + """ + from fontTools.ufoLib.glifLib import GlyphSet + + if validateRead is None: + validateRead = self._validate + if validateWrite is None: + validateWrite = self._validate + if layerName is None: + layerName = self.getDefaultLayerName(validate=validateRead) + directory = None + layerContents = self._readLayerContents(validateRead) + for storedLayerName, storedLayerDirectory in layerContents: + if layerName == storedLayerName: + directory = storedLayerDirectory + break + if directory is None: + raise UFOLibError('No glyphs directory is mapped to "%s".' % layerName) + try: + glyphSubFS = self.fs.opendir(directory) + except fs.errors.ResourceNotFound: + raise UFOLibError(f"No '{directory}' directory for layer '{layerName}'") + return GlyphSet( + glyphSubFS, + ufoFormatVersion=self._formatVersion, + validateRead=validateRead, + validateWrite=validateWrite, + expectContentsFile=True, + ) + + def getCharacterMapping( + self, layerName: Optional[str] = None, validate: Optional[bool] = None + ) -> dict[int, list[str]]: + """ + Return a dictionary that maps unicode values (ints) to + lists of glyph names. + """ + if validate is None: + validate = self._validate + glyphSet = self.getGlyphSet( + layerName, validateRead=validate, validateWrite=True + ) + allUnicodes = glyphSet.getUnicodes() + cmap: dict[int, list[str]] = {} + for glyphName, unicodes in allUnicodes.items(): + for code in unicodes: + if code in cmap: + cmap[code].append(glyphName) + else: + cmap[code] = [glyphName] + return cmap + + # /data + + def getDataDirectoryListing(self) -> list[str]: + """ + Returns a list of all files in the data directory. + The returned paths will be relative to the UFO. + This will not list directory names, only file names. + Thus, empty directories will be skipped. + """ + try: + self._dataFS = self.fs.opendir(DATA_DIRNAME) + except fs.errors.ResourceNotFound: + return [] + except fs.errors.DirectoryExpected: + raise UFOLibError('The UFO contains a "data" file instead of a directory.') + try: + # fs Walker.files method returns "absolute" paths (in terms of the + # root of the 'data' SubFS), so we strip the leading '/' to make + # them relative + return [p.lstrip("/") for p in self._dataFS.walk.files()] + except fs.errors.ResourceError: + return [] + + def getImageDirectoryListing(self, validate: Optional[bool] = None) -> list[str]: + """ + Returns a list of all image file names in + the images directory. Each of the images will + have been verified to have the PNG signature. + + ``validate`` will validate the data, by default it is set to the + class's validate value, can be overridden. + """ + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + return [] + if validate is None: + validate = self._validate + try: + self._imagesFS = imagesFS = self.fs.opendir(IMAGES_DIRNAME) + except fs.errors.ResourceNotFound: + return [] + except fs.errors.DirectoryExpected: + raise UFOLibError( + 'The UFO contains an "images" file instead of a directory.' + ) + result = [] + for path in imagesFS.scandir("/"): + if path.is_dir: + # silently skip this as version control + # systems often have hidden directories + continue + if validate: + with imagesFS.open(path.name, "rb") as fp: + valid, error = pngValidator(fileObj=fp) + if valid: + result.append(path.name) + else: + result.append(path.name) + return result + + def readData(self, fileName: PathStr) -> bytes: + """ + Return bytes for the file named 'fileName' inside the 'data/' directory. + """ + fileName = fsdecode(fileName) + try: + try: + dataFS = self._dataFS + except AttributeError: + # in case readData is called before getDataDirectoryListing + dataFS = self.fs.opendir(DATA_DIRNAME) + data = dataFS.readbytes(fileName) + except fs.errors.ResourceNotFound: + raise UFOLibError(f"No data file named '{fileName}' on {self.fs}") + return data + + def readImage(self, fileName: PathStr, validate: Optional[bool] = None) -> bytes: + """ + Return image data for the file named fileName. + + ``validate`` will validate the data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + raise UFOLibError( + f"Reading images is not allowed in UFO {self._formatVersion.major}." + ) + fileName = fsdecode(fileName) + try: + try: + imagesFS = self._imagesFS + except AttributeError: + # in case readImage is called before getImageDirectoryListing + imagesFS = self.fs.opendir(IMAGES_DIRNAME) + data = imagesFS.readbytes(fileName) + except fs.errors.ResourceNotFound: + raise UFOLibError(f"No image file named '{fileName}' on {self.fs}") + if validate: + valid, error = pngValidator(data=data) + if not valid: + raise UFOLibError(error) + return data + + def close(self) -> None: + if self._shouldClose: + self.fs.close() + + def __enter__(self) -> UFOReader: + return self + + def __exit__(self, exc_type: Any, exc_value: Any, exc_tb: Any) -> None: + self.close() + + +# ---------- +# UFO Writer +# ---------- + + +class UFOWriter(UFOReader): + """Write the various components of a .ufo. + + Attributes: + path: An :class:`os.PathLike` object pointing to the .ufo. + formatVersion: the UFO format version as a tuple of integers (major, minor), + or as a single integer for the major digit only (minor is implied to be 0). + By default, the latest formatVersion will be used; currently it is 3.0, + which is equivalent to formatVersion=(3, 0). + fileCreator: The creator of the .ufo file. Defaults to + `com.github.fonttools.ufoLib`. + structure: The internal structure of the .ufo file: either `ZIP` or `PACKAGE`. + validate: A boolean indicating if the data read should be validated. Defaults + to `True`. + + By default, the written data will be validated before writing. Set ``validate`` to + ``False`` if you do not want to validate the data. Validation can also be overriden + on a per-method level if desired. + + Raises: + UnsupportedUFOFormat: An exception indicating that the requested UFO + formatVersion is not supported. + """ + + def __init__( + self, + path: PathOrFS, + formatVersion: UFOFormatVersionInput = None, + fileCreator: str = "com.github.fonttools.ufoLib", + structure: Optional[UFOFileStructure] = None, + validate: bool = True, + ) -> None: + try: + formatVersion = normalizeFormatVersion(formatVersion, UFOFormatVersion) + except ValueError as e: + from fontTools.ufoLib.errors import UnsupportedUFOFormat + + raise UnsupportedUFOFormat( + f"Unsupported UFO format: {formatVersion!r}" + ) from e + + if hasattr(path, "__fspath__"): # support os.PathLike objects + path = path.__fspath__() + + if isinstance(path, str): + # normalize path by removing trailing or double slashes + path = os.path.normpath(path) + havePreviousFile = os.path.exists(path) + if havePreviousFile: + # ensure we use the same structure as the destination + existingStructure = _sniffFileStructure(path) + if structure is not None: + try: + structure = UFOFileStructure(structure) + except ValueError: + raise UFOLibError( + "Invalid or unsupported structure: '%s'" % structure + ) + if structure is not existingStructure: + raise UFOLibError( + "A UFO with a different structure (%s) already exists " + "at the given path: '%s'" % (existingStructure, path) + ) + else: + structure = existingStructure + else: + # if not exists, default to 'package' structure + if structure is None: + structure = UFOFileStructure.PACKAGE + dirName = os.path.dirname(path) + if dirName and not os.path.isdir(dirName): + raise UFOLibError( + "Cannot write to '%s': directory does not exist" % path + ) + if structure is UFOFileStructure.ZIP: + if havePreviousFile: + # we can't write a zip in-place, so we have to copy its + # contents to a temporary location and work from there, then + # upon closing UFOWriter we create the final zip file + parentFS: FS = fs.tempfs.TempFS() + with fs.zipfs.ZipFS(path, encoding="utf-8") as origFS: # type: ignore[abstract] + fs.copy.copy_fs(origFS, parentFS) + # if output path is an existing zip, we require that it contains + # one, and only one, root directory (with arbitrary name), in turn + # containing all the existing UFO contents + rootDirs = [ + p.name + for p in parentFS.scandir("/") + # exclude macOS metadata contained in zip file + if p.is_dir and p.name != "__MACOSX" + ] + if len(rootDirs) != 1: + raise UFOLibError( + "Expected exactly 1 root directory, found %d" + % len(rootDirs) + ) + else: + rootDir = rootDirs[0] + else: + # if the output zip file didn't exist, we create the root folder; + # we name it the same as input 'path', but with '.ufo' extension + rootDir = os.path.splitext(os.path.basename(path))[0] + ".ufo" + parentFS = fs.zipfs.ZipFS(path, write=True, encoding="utf-8") # type: ignore[abstract] + parentFS.makedir(rootDir) + # 'ClosingSubFS' ensures that the parent filesystem is closed + # when its root subdirectory is closed + self.fs = parentFS.opendir(rootDir, factory=fs.subfs.ClosingSubFS) + else: + self.fs = fs.osfs.OSFS(path, create=True) + self._fileStructure = structure + self._havePreviousFile = havePreviousFile + self._shouldClose = True + elif isinstance(path, fs.base.FS): + filesystem: FS = path + try: + filesystem.check() + except fs.errors.FilesystemClosed: + raise UFOLibError("the filesystem '%s' is closed" % path) + else: + self.fs = filesystem + try: + path = filesystem.getsyspath("/") + except fs.errors.NoSysPath: + # network or in-memory FS may not map to the local one + path = str(filesystem) + # if passed an FS object, always use 'package' structure + if structure and structure is not UFOFileStructure.PACKAGE: + import warnings + + warnings.warn( + "The 'structure' argument is not used when input is an FS object", + UserWarning, + stacklevel=2, + ) + self._fileStructure = UFOFileStructure.PACKAGE + # if FS contains a "metainfo.plist", we consider it non-empty + self._havePreviousFile = filesystem.exists(METAINFO_FILENAME) + # the user is responsible for closing the FS object + self._shouldClose = False + else: + raise TypeError( + "Expected a path string or fs object, found %s" % type(path).__name__ + ) + + # establish some basic stuff + self._path = fsdecode(path) + self._formatVersion = formatVersion + self._fileCreator = fileCreator + self._downConversionKerningData: Optional[KerningGroupRenameMaps] = None + self._validate = validate + # if the file already exists, get the format version. + # this will be needed for up and down conversion. + previousFormatVersion = None + if self._havePreviousFile: + metaInfo = self._readMetaInfo(validate=validate) + previousFormatVersion = metaInfo["formatVersionTuple"] + # catch down conversion + if previousFormatVersion > formatVersion: + from fontTools.ufoLib.errors import UnsupportedUFOFormat + + raise UnsupportedUFOFormat( + "The UFO located at this path is a higher version " + f"({previousFormatVersion}) than the version ({formatVersion}) " + "that is trying to be written. This is not supported." + ) + # handle the layer contents + self.layerContents: Union[dict[str, str], OrderedDict[str, str]] = {} + if previousFormatVersion is not None and previousFormatVersion.major >= 3: + # already exists + self.layerContents = OrderedDict(self._readLayerContents(validate)) + else: + # previous < 3 + # imply the layer contents + if self.fs.exists(DEFAULT_GLYPHS_DIRNAME): + self.layerContents = {DEFAULT_LAYER_NAME: DEFAULT_GLYPHS_DIRNAME} + # write the new metainfo + self._writeMetaInfo() + + # properties + + def _get_fileCreator(self) -> str: + return self._fileCreator + + fileCreator: property = property( + _get_fileCreator, + doc="The file creator of the UFO. This is set into metainfo.plist during __init__.", + ) + + # support methods for file system interaction + + def copyFromReader( + self, reader: UFOReader, sourcePath: PathStr, destPath: PathStr + ) -> None: + """ + Copy the sourcePath in the provided UFOReader to destPath + in this writer. The paths must be relative. This works with + both individual files and directories. + """ + if not isinstance(reader, UFOReader): + raise UFOLibError("The reader must be an instance of UFOReader.") + sourcePath = fsdecode(sourcePath) + destPath = fsdecode(destPath) + if not reader.fs.exists(sourcePath): + raise UFOLibError( + 'The reader does not have data located at "%s".' % sourcePath + ) + if self.fs.exists(destPath): + raise UFOLibError('A file named "%s" already exists.' % destPath) + # create the destination directory if it doesn't exist + self.fs.makedirs(fs.path.dirname(destPath), recreate=True) + if reader.fs.isdir(sourcePath): + fs.copy.copy_dir(reader.fs, sourcePath, self.fs, destPath) + else: + fs.copy.copy_file(reader.fs, sourcePath, self.fs, destPath) + + def writeBytesToPath(self, path: PathStr, data: bytes) -> None: + """ + Write bytes to a path relative to the UFO filesystem's root. + If writing to an existing UFO, check to see if data matches the data + that is already in the file at path; if so, the file is not rewritten + so that the modification date is preserved. + If needed, the directory tree for the given path will be built. + """ + path = fsdecode(path) + if self._havePreviousFile: + if self.fs.isfile(path) and data == self.fs.readbytes(path): + return + try: + self.fs.writebytes(path, data) + except fs.errors.FileExpected: + raise UFOLibError("A directory exists at '%s'" % path) + except fs.errors.ResourceNotFound: + self.fs.makedirs(fs.path.dirname(path), recreate=True) + self.fs.writebytes(path, data) + + def getFileObjectForPath( + self, + path: PathStr, + mode: str = "w", + encoding: Optional[str] = None, + ) -> Optional[IO[Any]]: + """ + Returns a file (or file-like) object for the + file at the given path. The path must be relative + to the UFO path. Returns None if the file does + not exist and the mode is "r" or "rb. + An encoding may be passed if the file is opened in text mode. + + Note: The caller is responsible for closing the open file. + """ + path = fsdecode(path) + try: + return self.fs.open(path, mode=mode, encoding=encoding) + except fs.errors.ResourceNotFound as e: + m = mode[0] + if m == "r": + # XXX I think we should just let it raise. The docstring, + # however, says that this returns None if mode is 'r' + return None + elif m == "w" or m == "a" or m == "x": + self.fs.makedirs(fs.path.dirname(path), recreate=True) + return self.fs.open(path, mode=mode, encoding=encoding) + except fs.errors.ResourceError as e: + raise UFOLibError(f"unable to open '{path}' on {self.fs}: {e}") + return None + + def removePath( + self, path: PathStr, force: bool = False, removeEmptyParents: bool = True + ) -> None: + """ + Remove the file (or directory) at path. The path + must be relative to the UFO. + Raises UFOLibError if the path doesn't exist. + If force=True, ignore non-existent paths. + If the directory where 'path' is located becomes empty, it will + be automatically removed, unless 'removeEmptyParents' is False. + """ + path = fsdecode(path) + try: + self.fs.remove(path) + except fs.errors.FileExpected: + self.fs.removetree(path) + except fs.errors.ResourceNotFound: + if not force: + raise UFOLibError(f"'{path}' does not exist on {self.fs}") + if removeEmptyParents: + parent = fs.path.dirname(path) + if parent: + fs.tools.remove_empty(self.fs, parent) + + # alias kept for backward compatibility with old API + removeFileForPath = removePath + + # UFO mod time + + def setModificationTime(self) -> None: + """ + Set the UFO modification time to the current time. + This is never called automatically. It is up to the + caller to call this when finished working on the UFO. + """ + path = self._path + if path is not None and os.path.exists(path): + try: + # this may fail on some filesystems (e.g. SMB servers) + os.utime(path, None) + except OSError as e: + logger.warning("Failed to set modified time: %s", e) + + # metainfo.plist + + def _writeMetaInfo(self) -> None: + metaInfo = dict( + creator=self._fileCreator, + formatVersion=self._formatVersion.major, + ) + if self._formatVersion.minor != 0: + metaInfo["formatVersionMinor"] = self._formatVersion.minor + self._writePlist(METAINFO_FILENAME, metaInfo) + + # groups.plist + + def setKerningGroupConversionRenameMaps(self, maps: KerningGroupRenameMaps) -> None: + """ + Set maps defining the renaming that should be done + when writing groups and kerning in UFO 1 and UFO 2. + This will effectively undo the conversion done when + UFOReader reads this data. The dictionary should have + this form:: + + { + "side1" : {"group name to use when writing" : "group name in data"}, + "side2" : {"group name to use when writing" : "group name in data"} + } + + This is the same form returned by UFOReader's + getKerningGroupConversionRenameMaps method. + """ + if self._formatVersion >= UFOFormatVersion.FORMAT_3_0: + return # XXX raise an error here + # flip the dictionaries + remap = {} + for side in ("side1", "side2"): + for writeName, dataName in list(maps[side].items()): + remap[dataName] = writeName + self._downConversionKerningData = dict(groupRenameMap=remap) + + def writeGroups( + self, groups: KerningGroups, validate: Optional[bool] = None + ) -> None: + """ + Write groups.plist. This method requires a + dict of glyph groups as an argument. + + ``validate`` will validate the data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + # validate the data structure + if validate: + valid, message = groupsValidator(groups) + if not valid: + raise UFOLibError(message) + # down convert + if ( + self._formatVersion < UFOFormatVersion.FORMAT_3_0 + and self._downConversionKerningData is not None + ): + remap = self._downConversionKerningData["groupRenameMap"] + remappedGroups = {} + # there are some edge cases here that are ignored: + # 1. if a group is being renamed to a name that + # already exists, the existing group is always + # overwritten. (this is why there are two loops + # below.) there doesn't seem to be a logical + # solution to groups mismatching and overwriting + # with the specifiecd group seems like a better + # solution than throwing an error. + # 2. if side 1 and side 2 groups are being renamed + # to the same group name there is no check to + # ensure that the contents are identical. that + # is left up to the caller. + for name, contents in list(groups.items()): + if name in remap: + continue + remappedGroups[name] = contents + for name, contents in list(groups.items()): + if name not in remap: + continue + name = remap[name] + remappedGroups[name] = contents + groups = remappedGroups + # pack and write + groupsNew = {} + for key, value in groups.items(): + groupsNew[key] = list(value) + if groupsNew: + self._writePlist(GROUPS_FILENAME, groupsNew) + elif self._havePreviousFile: + self.removePath(GROUPS_FILENAME, force=True, removeEmptyParents=False) + + # fontinfo.plist + + def writeInfo(self, info: Any, validate: Optional[bool] = None) -> None: + """ + Write info.plist. This method requires an object + that supports getting attributes that follow the + fontinfo.plist version 2 specification. Attributes + will be taken from the given object and written + into the file. + + ``validate`` will validate the data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + # gather version 3 data + infoData = {} + for attr in list(fontInfoAttributesVersion3ValueData.keys()): + if hasattr(info, attr): + try: + value = getattr(info, attr) + except AttributeError: + raise UFOLibError( + "The supplied info object does not support getting a necessary attribute (%s)." + % attr + ) + if value is None: + continue + infoData[attr] = value + # down convert data if necessary and validate + if self._formatVersion == UFOFormatVersion.FORMAT_3_0: + if validate: + infoData = validateInfoVersion3Data(infoData) + elif self._formatVersion == UFOFormatVersion.FORMAT_2_0: + infoData = _convertFontInfoDataVersion3ToVersion2(infoData) + if validate: + infoData = validateInfoVersion2Data(infoData) + elif self._formatVersion == UFOFormatVersion.FORMAT_1_0: + infoData = _convertFontInfoDataVersion3ToVersion2(infoData) + if validate: + infoData = validateInfoVersion2Data(infoData) + infoData = _convertFontInfoDataVersion2ToVersion1(infoData) + # write file if there is anything to write + if infoData: + self._writePlist(FONTINFO_FILENAME, infoData) + + # kerning.plist + + def writeKerning( + self, kerning: KerningDict, validate: Optional[bool] = None + ) -> None: + """ + Write kerning.plist. This method requires a + dict of kerning pairs as an argument. + + This performs basic structural validation of the kerning, + but it does not check for compliance with the spec in + regards to conflicting pairs. The assumption is that the + kerning data being passed is standards compliant. + + ``validate`` will validate the data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + # validate the data structure + if validate: + invalidFormatMessage = "The kerning is not properly formatted." + if not isDictEnough(kerning): + raise UFOLibError(invalidFormatMessage) + for pair, value in list(kerning.items()): + if not isinstance(pair, (list, tuple)): + raise UFOLibError(invalidFormatMessage) + if not len(pair) == 2: + raise UFOLibError(invalidFormatMessage) + if not isinstance(pair[0], str): + raise UFOLibError(invalidFormatMessage) + if not isinstance(pair[1], str): + raise UFOLibError(invalidFormatMessage) + if not isinstance(value, numberTypes): + raise UFOLibError(invalidFormatMessage) + # down convert + if ( + self._formatVersion < UFOFormatVersion.FORMAT_3_0 + and self._downConversionKerningData is not None + ): + remap = self._downConversionKerningData["groupRenameMap"] + remappedKerning = {} + for (side1, side2), value in list(kerning.items()): + side1 = remap.get(side1, side1) + side2 = remap.get(side2, side2) + remappedKerning[side1, side2] = value + kerning = remappedKerning + # pack and write + kerningDict: KerningNested = {} + for left, right in kerning.keys(): + value = kerning[left, right] + if left not in kerningDict: + kerningDict[left] = {} + kerningDict[left][right] = value + if kerningDict: + self._writePlist(KERNING_FILENAME, kerningDict) + elif self._havePreviousFile: + self.removePath(KERNING_FILENAME, force=True, removeEmptyParents=False) + + # lib.plist + + def writeLib(self, libDict: LibDict, validate: Optional[bool] = None) -> None: + """ + Write lib.plist. This method requires a + lib dict as an argument. + + ``validate`` will validate the data, by default it is set to the + class's validate value, can be overridden. + """ + if validate is None: + validate = self._validate + if validate: + valid, message = fontLibValidator(libDict) + if not valid: + raise UFOLibError(message) + if libDict: + self._writePlist(LIB_FILENAME, libDict) + elif self._havePreviousFile: + self.removePath(LIB_FILENAME, force=True, removeEmptyParents=False) + + # features.fea + + def writeFeatures(self, features: str, validate: Optional[bool] = None) -> None: + """ + Write features.fea. This method requires a + features string as an argument. + """ + if validate is None: + validate = self._validate + if self._formatVersion == UFOFormatVersion.FORMAT_1_0: + raise UFOLibError("features.fea is not allowed in UFO Format Version 1.") + if validate: + if not isinstance(features, str): + raise UFOLibError("The features are not text.") + if features: + self.writeBytesToPath(FEATURES_FILENAME, features.encode("utf8")) + elif self._havePreviousFile: + self.removePath(FEATURES_FILENAME, force=True, removeEmptyParents=False) + + # glyph sets & layers + + def writeLayerContents( + self, layerOrder: LayerOrderList = None, validate: Optional[bool] = None + ) -> None: + """ + Write the layercontents.plist file. This method *must* be called + after all glyph sets have been written. + """ + if validate is None: + validate = self._validate + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + return + if layerOrder is not None: + newOrder: list[Optional[str]] = [] + for layerName in layerOrder: + if layerName is None: + layerName = DEFAULT_LAYER_NAME + newOrder.append(layerName) + layerOrder = newOrder + else: + layerOrder = list(self.layerContents.keys()) + if validate and set(layerOrder) != set(self.layerContents.keys()): + raise UFOLibError( + "The layer order content does not match the glyph sets that have been created." + ) + layerContents = [ + (layerName, self.layerContents[layerName]) + for layerName in layerOrder + if layerName is not None + ] + self._writePlist(LAYERCONTENTS_FILENAME, layerContents) + + def _findDirectoryForLayerName(self, layerName: Optional[str]) -> str: + foundDirectory = None + for existingLayerName, directoryName in list(self.layerContents.items()): + if layerName is None and directoryName == DEFAULT_GLYPHS_DIRNAME: + foundDirectory = directoryName + break + elif existingLayerName == layerName: + foundDirectory = directoryName + break + if not foundDirectory: + raise UFOLibError( + "Could not locate a glyph set directory for the layer named %s." + % layerName + ) + return foundDirectory + + def getGlyphSet( # type: ignore[override] + self, + layerName: Optional[str] = None, + defaultLayer: bool = True, + glyphNameToFileNameFunc: GlyphNameToFileNameFunc = None, + validateRead: Optional[bool] = None, + validateWrite: Optional[bool] = None, + expectContentsFile: bool = False, + ) -> GlyphSet: + """ + Return the GlyphSet object associated with the + appropriate glyph directory in the .ufo. + If layerName is None, the default glyph set + will be used. The defaultLayer flag indicates + that the layer should be saved into the default + glyphs directory. + + ``validateRead`` will validate the read data, by default it is set to the + class's validate value, can be overridden. + ``validateWrte`` will validate the written data, by default it is set to the + class's validate value, can be overridden. + ``expectContentsFile`` will raise a GlifLibError if a contents.plist file is + not found on the glyph set file system. This should be set to ``True`` if you + are reading an existing UFO and ``False`` if you use ``getGlyphSet`` to create + a fresh glyph set. + """ + if validateRead is None: + validateRead = self._validate + if validateWrite is None: + validateWrite = self._validate + # only default can be written in < 3 + if self._formatVersion < UFOFormatVersion.FORMAT_3_0 and ( + not defaultLayer or layerName is not None + ): + raise UFOLibError( + f"Only the default layer can be writen in UFO {self._formatVersion.major}." + ) + # locate a layer name when None has been given + if layerName is None and defaultLayer: + for existingLayerName, directory in self.layerContents.items(): + if directory == DEFAULT_GLYPHS_DIRNAME: + layerName = existingLayerName + if layerName is None: + layerName = DEFAULT_LAYER_NAME + elif layerName is None and not defaultLayer: + raise UFOLibError("A layer name must be provided for non-default layers.") + # move along to format specific writing + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + return self._getDefaultGlyphSet( + validateRead, + validateWrite, + glyphNameToFileNameFunc=glyphNameToFileNameFunc, + expectContentsFile=expectContentsFile, + ) + elif self._formatVersion.major == UFOFormatVersion.FORMAT_3_0.major: + return self._getGlyphSetFormatVersion3( + validateRead, + validateWrite, + layerName=layerName, + defaultLayer=defaultLayer, + glyphNameToFileNameFunc=glyphNameToFileNameFunc, + expectContentsFile=expectContentsFile, + ) + else: + raise NotImplementedError(self._formatVersion) + + def _getDefaultGlyphSet( + self, + validateRead: bool, + validateWrite: bool, + glyphNameToFileNameFunc: GlyphNameToFileNameFunc = None, + expectContentsFile: bool = False, + ) -> GlyphSet: + from fontTools.ufoLib.glifLib import GlyphSet + + glyphSubFS = self.fs.makedir(DEFAULT_GLYPHS_DIRNAME, recreate=True) + return GlyphSet( + glyphSubFS, + glyphNameToFileNameFunc=glyphNameToFileNameFunc, + ufoFormatVersion=self._formatVersion, + validateRead=validateRead, + validateWrite=validateWrite, + expectContentsFile=expectContentsFile, + ) + + def _getGlyphSetFormatVersion3( + self, + validateRead: bool, + validateWrite: bool, + layerName: Optional[str] = None, + defaultLayer: bool = True, + glyphNameToFileNameFunc: GlyphNameToFileNameFunc = None, + expectContentsFile: bool = False, + ) -> GlyphSet: + from fontTools.ufoLib.glifLib import GlyphSet + + # if the default flag is on, make sure that the default in the file + # matches the default being written. also make sure that this layer + # name is not already linked to a non-default layer. + if defaultLayer: + for existingLayerName, directory in self.layerContents.items(): + if directory == DEFAULT_GLYPHS_DIRNAME: + if existingLayerName != layerName: + raise UFOLibError( + "Another layer ('%s') is already mapped to the default directory." + % existingLayerName + ) + elif existingLayerName == layerName: + raise UFOLibError( + "The layer name is already mapped to a non-default layer." + ) + + # handle layerName is None to avoid MyPy errors + if layerName is None: + raise TypeError("'leyerName' cannot be None.") + + # get an existing directory name + if layerName in self.layerContents: + directory = self.layerContents[layerName] + # get a new directory name + else: + if defaultLayer: + directory = DEFAULT_GLYPHS_DIRNAME + else: + # not caching this could be slightly expensive, + # but caching it will be cumbersome + existing = {d.lower() for d in self.layerContents.values()} + directory = userNameToFileName( + layerName, existing=existing, prefix="glyphs." + ) + # make the directory + glyphSubFS = self.fs.makedir(directory, recreate=True) + # store the mapping + self.layerContents[layerName] = directory + # load the glyph set + return GlyphSet( + glyphSubFS, + glyphNameToFileNameFunc=glyphNameToFileNameFunc, + ufoFormatVersion=self._formatVersion, + validateRead=validateRead, + validateWrite=validateWrite, + expectContentsFile=expectContentsFile, + ) + + def renameGlyphSet( + self, + layerName: Optional[str], + newLayerName: Optional[str], + defaultLayer: bool = False, + ) -> None: + """ + Rename a glyph set. + + Note: if a GlyphSet object has already been retrieved for + layerName, it is up to the caller to inform that object that + the directory it represents has changed. + """ + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + # ignore renaming glyph sets for UFO1 UFO2 + # just write the data from the default layer + return + # the new and old names can be the same + # as long as the default is being switched + if layerName is not None and layerName == newLayerName: + # if the default is off and the layer is already not the default, skip + if ( + self.layerContents[layerName] != DEFAULT_GLYPHS_DIRNAME + and not defaultLayer + ): + return + # if the default is on and the layer is already the default, skip + if self.layerContents[layerName] == DEFAULT_GLYPHS_DIRNAME and defaultLayer: + return + else: + # make sure the new layer name doesn't already exist + if newLayerName is None: + newLayerName = DEFAULT_LAYER_NAME + if newLayerName in self.layerContents: + raise UFOLibError("A layer named %s already exists." % newLayerName) + # make sure the default layer doesn't already exist + if defaultLayer and DEFAULT_GLYPHS_DIRNAME in self.layerContents.values(): + raise UFOLibError("A default layer already exists.") + # get the paths + oldDirectory = self._findDirectoryForLayerName(layerName) + if defaultLayer: + newDirectory = DEFAULT_GLYPHS_DIRNAME + else: + existing = {name.lower() for name in self.layerContents.values()} + newDirectory = userNameToFileName( + newLayerName, existing=existing, prefix="glyphs." + ) + # update the internal mapping + if layerName is not None: + del self.layerContents[layerName] + self.layerContents[newLayerName] = newDirectory + # do the file system copy + self.fs.movedir(oldDirectory, newDirectory, create=True) + + def deleteGlyphSet(self, layerName: Optional[str]) -> None: + """ + Remove the glyph set matching layerName. + """ + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + # ignore deleting glyph sets for UFO1 UFO2 as there are no layers + # just write the data from the default layer + return + foundDirectory = self._findDirectoryForLayerName(layerName) + self.removePath(foundDirectory, removeEmptyParents=False) + if layerName is not None: + del self.layerContents[layerName] + + def writeData(self, fileName: PathStr, data: bytes) -> None: + """ + Write data to fileName in the 'data' directory. + The data must be a bytes string. + """ + self.writeBytesToPath(f"{DATA_DIRNAME}/{fsdecode(fileName)}", data) + + def removeData(self, fileName: PathStr) -> None: + """ + Remove the file named fileName from the data directory. + """ + self.removePath(f"{DATA_DIRNAME}/{fsdecode(fileName)}") + + # /images + + def writeImage( + self, + fileName: PathStr, + data: bytes, + validate: Optional[bool] = None, + ) -> None: + """ + Write data to fileName in the images directory. + The data must be a valid PNG. + """ + if validate is None: + validate = self._validate + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + raise UFOLibError( + f"Images are not allowed in UFO {self._formatVersion.major}." + ) + fileName = fsdecode(fileName) + if validate: + valid, error = pngValidator(data=data) + if not valid: + raise UFOLibError(error) + self.writeBytesToPath(f"{IMAGES_DIRNAME}/{fileName}", data) + + def removeImage( + self, + fileName: PathStr, + validate: Optional[bool] = None, + ) -> None: # XXX remove unused 'validate'? + """ + Remove the file named fileName from the + images directory. + """ + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + raise UFOLibError( + f"Images are not allowed in UFO {self._formatVersion.major}." + ) + self.removePath(f"{IMAGES_DIRNAME}/{fsdecode(fileName)}") + + def copyImageFromReader( + self, + reader: UFOReader, + sourceFileName: PathStr, + destFileName: PathStr, + validate: Optional[bool] = None, + ) -> None: + """ + Copy the sourceFileName in the provided UFOReader to destFileName + in this writer. This uses the most memory efficient method possible + for copying the data possible. + """ + if validate is None: + validate = self._validate + if self._formatVersion < UFOFormatVersion.FORMAT_3_0: + raise UFOLibError( + f"Images are not allowed in UFO {self._formatVersion.major}." + ) + sourcePath = f"{IMAGES_DIRNAME}/{fsdecode(sourceFileName)}" + destPath = f"{IMAGES_DIRNAME}/{fsdecode(destFileName)}" + self.copyFromReader(reader, sourcePath, destPath) + + def close(self) -> None: + if self._havePreviousFile and self._fileStructure is UFOFileStructure.ZIP: + # if we are updating an existing zip file, we can now compress the + # contents of the temporary filesystem in the destination path + rootDir = os.path.splitext(os.path.basename(self._path))[0] + ".ufo" + with fs.zipfs.ZipFS(self._path, write=True, encoding="utf-8") as destFS: # type: ignore[abstract] + fs.copy.copy_fs(self.fs, destFS.makedir(rootDir)) + super().close() + + +# just an alias, makes it more explicit +UFOReaderWriter = UFOWriter + + +# ---------------- +# Helper Functions +# ---------------- + + +def _sniffFileStructure(ufo_path: PathStr) -> UFOFileStructure: + """Return UFOFileStructure.ZIP if the UFO at path 'ufo_path' (str) + is a zip file, else return UFOFileStructure.PACKAGE if 'ufo_path' is a + directory. + Raise UFOLibError if it is a file with unknown structure, or if the path + does not exist. + """ + if zipfile.is_zipfile(ufo_path): + return UFOFileStructure.ZIP + elif os.path.isdir(ufo_path): + return UFOFileStructure.PACKAGE + elif os.path.isfile(ufo_path): + raise UFOLibError( + "The specified UFO does not have a known structure: '%s'" % ufo_path + ) + else: + raise UFOLibError("No such file or directory: '%s'" % ufo_path) + + +def makeUFOPath(path: PathStr) -> str: + """ + Return a .ufo pathname. + + >>> makeUFOPath("directory/something.ext") == ( + ... os.path.join('directory', 'something.ufo')) + True + >>> makeUFOPath("directory/something.another.thing.ext") == ( + ... os.path.join('directory', 'something.another.thing.ufo')) + True + """ + dir, name = os.path.split(path) + name = ".".join([".".join(name.split(".")[:-1]), "ufo"]) + return os.path.join(dir, name) + + +# ---------------------- +# fontinfo.plist Support +# ---------------------- + +# Version Validators + +# There is no version 1 validator and there shouldn't be. +# The version 1 spec was very loose and there were numerous +# cases of invalid values. + + +def validateFontInfoVersion2ValueForAttribute(attr: str, value: Any) -> bool: + """ + This performs very basic validation of the value for attribute + following the UFO 2 fontinfo.plist specification. The results + of this should not be interpretted as *correct* for the font + that they are part of. This merely indicates that the value + is of the proper type and, where the specification defines + a set range of possible values for an attribute, that the + value is in the accepted range. + """ + dataValidationDict = fontInfoAttributesVersion2ValueData[attr] + valueType = dataValidationDict.get("type") + validator = dataValidationDict.get("valueValidator", genericTypeValidator) + valueOptions = dataValidationDict.get("valueOptions") + # have specific options for the validator + if valueOptions is not None: + isValidValue = validator(value, valueOptions) + # no specific options + else: + if validator == genericTypeValidator: + isValidValue = validator(value, valueType) + else: + isValidValue = validator(value) + return isValidValue + + +def validateInfoVersion2Data(infoData: dict[str, Any]) -> dict[str, Any]: + """ + This performs very basic validation of the value for infoData + following the UFO 2 fontinfo.plist specification. The results + of this should not be interpretted as *correct* for the font + that they are part of. This merely indicates that the values + are of the proper type and, where the specification defines + a set range of possible values for an attribute, that the + value is in the accepted range. + """ + validInfoData = {} + for attr, value in list(infoData.items()): + isValidValue = validateFontInfoVersion2ValueForAttribute(attr, value) + if not isValidValue: + raise UFOLibError(f"Invalid value for attribute {attr} ({value!r}).") + else: + validInfoData[attr] = value + return validInfoData + + +def validateFontInfoVersion3ValueForAttribute(attr: str, value: Any) -> bool: + """ + This performs very basic validation of the value for attribute + following the UFO 3 fontinfo.plist specification. The results + of this should not be interpretted as *correct* for the font + that they are part of. This merely indicates that the value + is of the proper type and, where the specification defines + a set range of possible values for an attribute, that the + value is in the accepted range. + """ + dataValidationDict = fontInfoAttributesVersion3ValueData[attr] + valueType = dataValidationDict.get("type") + validator = dataValidationDict.get("valueValidator", genericTypeValidator) + valueOptions = dataValidationDict.get("valueOptions") + # have specific options for the validator + if valueOptions is not None: + isValidValue = validator(value, valueOptions) + # no specific options + else: + if validator == genericTypeValidator: + isValidValue = validator(value, valueType) + else: + isValidValue = validator(value) + return isValidValue + + +def validateInfoVersion3Data(infoData: dict[str, Any]) -> dict[str, Any]: + """ + This performs very basic validation of the value for infoData + following the UFO 3 fontinfo.plist specification. The results + of this should not be interpretted as *correct* for the font + that they are part of. This merely indicates that the values + are of the proper type and, where the specification defines + a set range of possible values for an attribute, that the + value is in the accepted range. + """ + validInfoData = {} + for attr, value in list(infoData.items()): + isValidValue = validateFontInfoVersion3ValueForAttribute(attr, value) + if not isValidValue: + raise UFOLibError(f"Invalid value for attribute {attr} ({value!r}).") + else: + validInfoData[attr] = value + return validInfoData + + +# Value Options + +fontInfoOpenTypeHeadFlagsOptions: list[int] = list(range(0, 15)) +fontInfoOpenTypeOS2SelectionOptions: list[int] = [1, 2, 3, 4, 7, 8, 9] +fontInfoOpenTypeOS2UnicodeRangesOptions: list[int] = list(range(0, 128)) +fontInfoOpenTypeOS2CodePageRangesOptions: list[int] = list(range(0, 64)) +fontInfoOpenTypeOS2TypeOptions: list[int] = [0, 1, 2, 3, 8, 9] + +# Version Attribute Definitions +# This defines the attributes, types and, in some +# cases the possible values, that can exist is +# fontinfo.plist. + +fontInfoAttributesVersion1: set[str] = { + "familyName", + "styleName", + "fullName", + "fontName", + "menuName", + "fontStyle", + "note", + "versionMajor", + "versionMinor", + "year", + "copyright", + "notice", + "trademark", + "license", + "licenseURL", + "createdBy", + "designer", + "designerURL", + "vendorURL", + "unitsPerEm", + "ascender", + "descender", + "capHeight", + "xHeight", + "defaultWidth", + "slantAngle", + "italicAngle", + "widthName", + "weightName", + "weightValue", + "fondName", + "otFamilyName", + "otStyleName", + "otMacName", + "msCharSet", + "fondID", + "uniqueID", + "ttVendor", + "ttUniqueID", + "ttVersion", +} + +fontInfoAttributesVersion2ValueData: FontInfoAttributes = { + "familyName": dict(type=str), + "styleName": dict(type=str), + "styleMapFamilyName": dict(type=str), + "styleMapStyleName": dict( + type=str, valueValidator=fontInfoStyleMapStyleNameValidator + ), + "versionMajor": dict(type=int), + "versionMinor": dict(type=int), + "year": dict(type=int), + "copyright": dict(type=str), + "trademark": dict(type=str), + "unitsPerEm": dict(type=(int, float)), + "descender": dict(type=(int, float)), + "xHeight": dict(type=(int, float)), + "capHeight": dict(type=(int, float)), + "ascender": dict(type=(int, float)), + "italicAngle": dict(type=(float, int)), + "note": dict(type=str), + "openTypeHeadCreated": dict( + type=str, valueValidator=fontInfoOpenTypeHeadCreatedValidator + ), + "openTypeHeadLowestRecPPEM": dict(type=(int, float)), + "openTypeHeadFlags": dict( + type="integerList", + valueValidator=genericIntListValidator, + valueOptions=fontInfoOpenTypeHeadFlagsOptions, + ), + "openTypeHheaAscender": dict(type=(int, float)), + "openTypeHheaDescender": dict(type=(int, float)), + "openTypeHheaLineGap": dict(type=(int, float)), + "openTypeHheaCaretSlopeRise": dict(type=int), + "openTypeHheaCaretSlopeRun": dict(type=int), + "openTypeHheaCaretOffset": dict(type=(int, float)), + "openTypeNameDesigner": dict(type=str), + "openTypeNameDesignerURL": dict(type=str), + "openTypeNameManufacturer": dict(type=str), + "openTypeNameManufacturerURL": dict(type=str), + "openTypeNameLicense": dict(type=str), + "openTypeNameLicenseURL": dict(type=str), + "openTypeNameVersion": dict(type=str), + "openTypeNameUniqueID": dict(type=str), + "openTypeNameDescription": dict(type=str), + "openTypeNamePreferredFamilyName": dict(type=str), + "openTypeNamePreferredSubfamilyName": dict(type=str), + "openTypeNameCompatibleFullName": dict(type=str), + "openTypeNameSampleText": dict(type=str), + "openTypeNameWWSFamilyName": dict(type=str), + "openTypeNameWWSSubfamilyName": dict(type=str), + "openTypeOS2WidthClass": dict( + type=int, valueValidator=fontInfoOpenTypeOS2WidthClassValidator + ), + "openTypeOS2WeightClass": dict( + type=int, valueValidator=fontInfoOpenTypeOS2WeightClassValidator + ), + "openTypeOS2Selection": dict( + type="integerList", + valueValidator=genericIntListValidator, + valueOptions=fontInfoOpenTypeOS2SelectionOptions, + ), + "openTypeOS2VendorID": dict(type=str), + "openTypeOS2Panose": dict( + type="integerList", valueValidator=fontInfoVersion2OpenTypeOS2PanoseValidator + ), + "openTypeOS2FamilyClass": dict( + type="integerList", valueValidator=fontInfoOpenTypeOS2FamilyClassValidator + ), + "openTypeOS2UnicodeRanges": dict( + type="integerList", + valueValidator=genericIntListValidator, + valueOptions=fontInfoOpenTypeOS2UnicodeRangesOptions, + ), + "openTypeOS2CodePageRanges": dict( + type="integerList", + valueValidator=genericIntListValidator, + valueOptions=fontInfoOpenTypeOS2CodePageRangesOptions, + ), + "openTypeOS2TypoAscender": dict(type=(int, float)), + "openTypeOS2TypoDescender": dict(type=(int, float)), + "openTypeOS2TypoLineGap": dict(type=(int, float)), + "openTypeOS2WinAscent": dict(type=(int, float)), + "openTypeOS2WinDescent": dict(type=(int, float)), + "openTypeOS2Type": dict( + type="integerList", + valueValidator=genericIntListValidator, + valueOptions=fontInfoOpenTypeOS2TypeOptions, + ), + "openTypeOS2SubscriptXSize": dict(type=(int, float)), + "openTypeOS2SubscriptYSize": dict(type=(int, float)), + "openTypeOS2SubscriptXOffset": dict(type=(int, float)), + "openTypeOS2SubscriptYOffset": dict(type=(int, float)), + "openTypeOS2SuperscriptXSize": dict(type=(int, float)), + "openTypeOS2SuperscriptYSize": dict(type=(int, float)), + "openTypeOS2SuperscriptXOffset": dict(type=(int, float)), + "openTypeOS2SuperscriptYOffset": dict(type=(int, float)), + "openTypeOS2StrikeoutSize": dict(type=(int, float)), + "openTypeOS2StrikeoutPosition": dict(type=(int, float)), + "openTypeVheaVertTypoAscender": dict(type=(int, float)), + "openTypeVheaVertTypoDescender": dict(type=(int, float)), + "openTypeVheaVertTypoLineGap": dict(type=(int, float)), + "openTypeVheaCaretSlopeRise": dict(type=int), + "openTypeVheaCaretSlopeRun": dict(type=int), + "openTypeVheaCaretOffset": dict(type=(int, float)), + "postscriptFontName": dict(type=str), + "postscriptFullName": dict(type=str), + "postscriptSlantAngle": dict(type=(float, int)), + "postscriptUniqueID": dict(type=int), + "postscriptUnderlineThickness": dict(type=(int, float)), + "postscriptUnderlinePosition": dict(type=(int, float)), + "postscriptIsFixedPitch": dict(type=bool), + "postscriptBlueValues": dict( + type="integerList", valueValidator=fontInfoPostscriptBluesValidator + ), + "postscriptOtherBlues": dict( + type="integerList", valueValidator=fontInfoPostscriptOtherBluesValidator + ), + "postscriptFamilyBlues": dict( + type="integerList", valueValidator=fontInfoPostscriptBluesValidator + ), + "postscriptFamilyOtherBlues": dict( + type="integerList", valueValidator=fontInfoPostscriptOtherBluesValidator + ), + "postscriptStemSnapH": dict( + type="integerList", valueValidator=fontInfoPostscriptStemsValidator + ), + "postscriptStemSnapV": dict( + type="integerList", valueValidator=fontInfoPostscriptStemsValidator + ), + "postscriptBlueFuzz": dict(type=(int, float)), + "postscriptBlueShift": dict(type=(int, float)), + "postscriptBlueScale": dict(type=(float, int)), + "postscriptForceBold": dict(type=bool), + "postscriptDefaultWidthX": dict(type=(int, float)), + "postscriptNominalWidthX": dict(type=(int, float)), + "postscriptWeightName": dict(type=str), + "postscriptDefaultCharacter": dict(type=str), + "postscriptWindowsCharacterSet": dict( + type=int, valueValidator=fontInfoPostscriptWindowsCharacterSetValidator + ), + "macintoshFONDFamilyID": dict(type=int), + "macintoshFONDName": dict(type=str), +} +fontInfoAttributesVersion2: set[str] = set(fontInfoAttributesVersion2ValueData.keys()) + +fontInfoAttributesVersion3ValueData: FontInfoAttributes = deepcopy( + fontInfoAttributesVersion2ValueData +) +fontInfoAttributesVersion3ValueData.update( + { + "versionMinor": dict(type=int, valueValidator=genericNonNegativeIntValidator), + "unitsPerEm": dict( + type=(int, float), valueValidator=genericNonNegativeNumberValidator + ), + "openTypeHeadLowestRecPPEM": dict( + type=int, valueValidator=genericNonNegativeNumberValidator + ), + "openTypeHheaAscender": dict(type=int), + "openTypeHheaDescender": dict(type=int), + "openTypeHheaLineGap": dict(type=int), + "openTypeHheaCaretOffset": dict(type=int), + "openTypeOS2Panose": dict( + type="integerList", + valueValidator=fontInfoVersion3OpenTypeOS2PanoseValidator, + ), + "openTypeOS2TypoAscender": dict(type=int), + "openTypeOS2TypoDescender": dict(type=int), + "openTypeOS2TypoLineGap": dict(type=int), + "openTypeOS2WinAscent": dict( + type=int, valueValidator=genericNonNegativeNumberValidator + ), + "openTypeOS2WinDescent": dict( + type=int, valueValidator=genericNonNegativeNumberValidator + ), + "openTypeOS2SubscriptXSize": dict(type=int), + "openTypeOS2SubscriptYSize": dict(type=int), + "openTypeOS2SubscriptXOffset": dict(type=int), + "openTypeOS2SubscriptYOffset": dict(type=int), + "openTypeOS2SuperscriptXSize": dict(type=int), + "openTypeOS2SuperscriptYSize": dict(type=int), + "openTypeOS2SuperscriptXOffset": dict(type=int), + "openTypeOS2SuperscriptYOffset": dict(type=int), + "openTypeOS2StrikeoutSize": dict(type=int), + "openTypeOS2StrikeoutPosition": dict(type=int), + "openTypeGaspRangeRecords": dict( + type="dictList", valueValidator=fontInfoOpenTypeGaspRangeRecordsValidator + ), + "openTypeNameRecords": dict( + type="dictList", valueValidator=fontInfoOpenTypeNameRecordsValidator + ), + "openTypeVheaVertTypoAscender": dict(type=int), + "openTypeVheaVertTypoDescender": dict(type=int), + "openTypeVheaVertTypoLineGap": dict(type=int), + "openTypeVheaCaretOffset": dict(type=int), + "woffMajorVersion": dict( + type=int, valueValidator=genericNonNegativeIntValidator + ), + "woffMinorVersion": dict( + type=int, valueValidator=genericNonNegativeIntValidator + ), + "woffMetadataUniqueID": dict( + type=dict, valueValidator=fontInfoWOFFMetadataUniqueIDValidator + ), + "woffMetadataVendor": dict( + type=dict, valueValidator=fontInfoWOFFMetadataVendorValidator + ), + "woffMetadataCredits": dict( + type=dict, valueValidator=fontInfoWOFFMetadataCreditsValidator + ), + "woffMetadataDescription": dict( + type=dict, valueValidator=fontInfoWOFFMetadataDescriptionValidator + ), + "woffMetadataLicense": dict( + type=dict, valueValidator=fontInfoWOFFMetadataLicenseValidator + ), + "woffMetadataCopyright": dict( + type=dict, valueValidator=fontInfoWOFFMetadataCopyrightValidator + ), + "woffMetadataTrademark": dict( + type=dict, valueValidator=fontInfoWOFFMetadataTrademarkValidator + ), + "woffMetadataLicensee": dict( + type=dict, valueValidator=fontInfoWOFFMetadataLicenseeValidator + ), + "woffMetadataExtensions": dict( + type=list, valueValidator=fontInfoWOFFMetadataExtensionsValidator + ), + "guidelines": dict(type=list, valueValidator=guidelinesValidator), + } +) +fontInfoAttributesVersion3: set[str] = set(fontInfoAttributesVersion3ValueData.keys()) + +# insert the type validator for all attrs that +# have no defined validator. +for attr, dataDict in list(fontInfoAttributesVersion2ValueData.items()): + if "valueValidator" not in dataDict: + dataDict["valueValidator"] = genericTypeValidator + +for attr, dataDict in list(fontInfoAttributesVersion3ValueData.items()): + if "valueValidator" not in dataDict: + dataDict["valueValidator"] = genericTypeValidator + +# Version Conversion Support +# These are used from converting from version 1 +# to version 2 or vice-versa. + + +def _flipDict(d: dict[K, V]) -> dict[V, K]: + flipped = {} + for key, value in list(d.items()): + flipped[value] = key + return flipped + + +fontInfoAttributesVersion1To2: dict[str, str] = { + "menuName": "styleMapFamilyName", + "designer": "openTypeNameDesigner", + "designerURL": "openTypeNameDesignerURL", + "createdBy": "openTypeNameManufacturer", + "vendorURL": "openTypeNameManufacturerURL", + "license": "openTypeNameLicense", + "licenseURL": "openTypeNameLicenseURL", + "ttVersion": "openTypeNameVersion", + "ttUniqueID": "openTypeNameUniqueID", + "notice": "openTypeNameDescription", + "otFamilyName": "openTypeNamePreferredFamilyName", + "otStyleName": "openTypeNamePreferredSubfamilyName", + "otMacName": "openTypeNameCompatibleFullName", + "weightName": "postscriptWeightName", + "weightValue": "openTypeOS2WeightClass", + "ttVendor": "openTypeOS2VendorID", + "uniqueID": "postscriptUniqueID", + "fontName": "postscriptFontName", + "fondID": "macintoshFONDFamilyID", + "fondName": "macintoshFONDName", + "defaultWidth": "postscriptDefaultWidthX", + "slantAngle": "postscriptSlantAngle", + "fullName": "postscriptFullName", + # require special value conversion + "fontStyle": "styleMapStyleName", + "widthName": "openTypeOS2WidthClass", + "msCharSet": "postscriptWindowsCharacterSet", +} +fontInfoAttributesVersion2To1 = _flipDict(fontInfoAttributesVersion1To2) +deprecatedFontInfoAttributesVersion2 = set(fontInfoAttributesVersion1To2.keys()) + +_fontStyle1To2: dict[int, str] = { + 64: "regular", + 1: "italic", + 32: "bold", + 33: "bold italic", +} +_fontStyle2To1: dict[str, int] = _flipDict(_fontStyle1To2) +# Some UFO 1 files have 0 +_fontStyle1To2[0] = "regular" + +_widthName1To2: dict[str, int] = { + "Ultra-condensed": 1, + "Extra-condensed": 2, + "Condensed": 3, + "Semi-condensed": 4, + "Medium (normal)": 5, + "Semi-expanded": 6, + "Expanded": 7, + "Extra-expanded": 8, + "Ultra-expanded": 9, +} +_widthName2To1: dict[int, str] = _flipDict(_widthName1To2) +# FontLab's default width value is "Normal". +# Many format version 1 UFOs will have this. +_widthName1To2["Normal"] = 5 +# FontLab has an "All" width value. In UFO 1 +# move this up to "Normal". +_widthName1To2["All"] = 5 +# "medium" appears in a lot of UFO 1 files. +_widthName1To2["medium"] = 5 +# "Medium" appears in a lot of UFO 1 files. +_widthName1To2["Medium"] = 5 + +_msCharSet1To2: dict[int, int] = { + 0: 1, + 1: 2, + 2: 3, + 77: 4, + 128: 5, + 129: 6, + 130: 7, + 134: 8, + 136: 9, + 161: 10, + 162: 11, + 163: 12, + 177: 13, + 178: 14, + 186: 15, + 200: 16, + 204: 17, + 222: 18, + 238: 19, + 255: 20, +} +_msCharSet2To1: dict[int, int] = _flipDict(_msCharSet1To2) + +# 1 <-> 2 + + +def convertFontInfoValueForAttributeFromVersion1ToVersion2( + attr: str, value: Any +) -> tuple[str, Any]: + """ + Convert value from version 1 to version 2 format. + Returns the new attribute name and the converted value. + If the value is None, None will be returned for the new value. + """ + # convert floats to ints if possible + if isinstance(value, float): + if int(value) == value: + value = int(value) + if value is not None: + if attr == "fontStyle": + v: Optional[Union[str, int]] = _fontStyle1To2.get(value) + if v is None: + raise UFOLibError( + f"Cannot convert value ({value!r}) for attribute {attr}." + ) + value = v + elif attr == "widthName": + v = _widthName1To2.get(value) + if v is None: + raise UFOLibError( + f"Cannot convert value ({value!r}) for attribute {attr}." + ) + value = v + elif attr == "msCharSet": + v = _msCharSet1To2.get(value) + if v is None: + raise UFOLibError( + f"Cannot convert value ({value!r}) for attribute {attr}." + ) + value = v + attr = fontInfoAttributesVersion1To2.get(attr, attr) + return attr, value + + +def convertFontInfoValueForAttributeFromVersion2ToVersion1( + attr: str, value: Any +) -> tuple[str, Any]: + """ + Convert value from version 2 to version 1 format. + Returns the new attribute name and the converted value. + If the value is None, None will be returned for the new value. + """ + if value is not None: + if attr == "styleMapStyleName": + value = _fontStyle2To1.get(value) + elif attr == "openTypeOS2WidthClass": + value = _widthName2To1.get(value) + elif attr == "postscriptWindowsCharacterSet": + value = _msCharSet2To1.get(value) + attr = fontInfoAttributesVersion2To1.get(attr, attr) + return attr, value + + +def _convertFontInfoDataVersion1ToVersion2(data: dict[str, Any]) -> dict[str, Any]: + converted = {} + for attr, value in list(data.items()): + # FontLab gives -1 for the weightValue + # for fonts wil no defined value. Many + # format version 1 UFOs will have this. + if attr == "weightValue" and value == -1: + continue + newAttr, newValue = convertFontInfoValueForAttributeFromVersion1ToVersion2( + attr, value + ) + # skip if the attribute is not part of version 2 + if newAttr not in fontInfoAttributesVersion2: + continue + # catch values that can't be converted + if value is None: + raise UFOLibError( + f"Cannot convert value ({value!r}) for attribute {newAttr}." + ) + # store + converted[newAttr] = newValue + return converted + + +def _convertFontInfoDataVersion2ToVersion1(data: dict[str, Any]) -> dict[str, Any]: + converted = {} + for attr, value in list(data.items()): + newAttr, newValue = convertFontInfoValueForAttributeFromVersion2ToVersion1( + attr, value + ) + # only take attributes that are registered for version 1 + if newAttr not in fontInfoAttributesVersion1: + continue + # catch values that can't be converted + if value is None: + raise UFOLibError( + f"Cannot convert value ({value!r}) for attribute {newAttr}." + ) + # store + converted[newAttr] = newValue + return converted + + +# 2 <-> 3 + +_ufo2To3NonNegativeInt: set[str] = { + "versionMinor", + "openTypeHeadLowestRecPPEM", + "openTypeOS2WinAscent", + "openTypeOS2WinDescent", +} +_ufo2To3NonNegativeIntOrFloat: set[str] = { + "unitsPerEm", +} +_ufo2To3FloatToInt: set[str] = { + "openTypeHeadLowestRecPPEM", + "openTypeHheaAscender", + "openTypeHheaDescender", + "openTypeHheaLineGap", + "openTypeHheaCaretOffset", + "openTypeOS2TypoAscender", + "openTypeOS2TypoDescender", + "openTypeOS2TypoLineGap", + "openTypeOS2WinAscent", + "openTypeOS2WinDescent", + "openTypeOS2SubscriptXSize", + "openTypeOS2SubscriptYSize", + "openTypeOS2SubscriptXOffset", + "openTypeOS2SubscriptYOffset", + "openTypeOS2SuperscriptXSize", + "openTypeOS2SuperscriptYSize", + "openTypeOS2SuperscriptXOffset", + "openTypeOS2SuperscriptYOffset", + "openTypeOS2StrikeoutSize", + "openTypeOS2StrikeoutPosition", + "openTypeVheaVertTypoAscender", + "openTypeVheaVertTypoDescender", + "openTypeVheaVertTypoLineGap", + "openTypeVheaCaretOffset", +} + + +def convertFontInfoValueForAttributeFromVersion2ToVersion3( + attr: str, value: Any +) -> tuple[str, Any]: + """ + Convert value from version 2 to version 3 format. + Returns the new attribute name and the converted value. + If the value is None, None will be returned for the new value. + """ + if attr in _ufo2To3FloatToInt: + try: + value = round(value) + except (ValueError, TypeError): + raise UFOLibError("Could not convert value for %s." % attr) + if attr in _ufo2To3NonNegativeInt: + try: + value = int(abs(value)) + except (ValueError, TypeError): + raise UFOLibError("Could not convert value for %s." % attr) + elif attr in _ufo2To3NonNegativeIntOrFloat: + try: + v = float(abs(value)) + except (ValueError, TypeError): + raise UFOLibError("Could not convert value for %s." % attr) + if v == int(v): + v = int(v) + if v != value: + value = v + return attr, value + + +def convertFontInfoValueForAttributeFromVersion3ToVersion2( + attr: str, value: Any +) -> tuple[str, Any]: + """ + Convert value from version 3 to version 2 format. + Returns the new attribute name and the converted value. + If the value is None, None will be returned for the new value. + """ + return attr, value + + +def _convertFontInfoDataVersion3ToVersion2(data: dict[str, Any]) -> dict[str, Any]: + converted = {} + for attr, value in list(data.items()): + newAttr, newValue = convertFontInfoValueForAttributeFromVersion3ToVersion2( + attr, value + ) + if newAttr not in fontInfoAttributesVersion2: + continue + converted[newAttr] = newValue + return converted + + +def _convertFontInfoDataVersion2ToVersion3(data: dict[str, Any]) -> dict[str, Any]: + converted = {} + for attr, value in list(data.items()): + attr, value = convertFontInfoValueForAttributeFromVersion2ToVersion3( + attr, value + ) + converted[attr] = value + return converted + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/converters.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/converters.py new file mode 100644 index 0000000000000000000000000000000000000000..94a229f48d2f89bf0fc51b3b1a2a0f083b97f81d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/converters.py @@ -0,0 +1,407 @@ +from __future__ import annotations + +from typing import Mapping, Any +from collections.abc import Container + +from fontTools.annotations import KerningNested + +""" +Functions for converting UFO1 or UFO2 files into UFO3 format. + +Currently provides functionality for converting kerning rules +and kerning groups. Conversion is only supported _from_ UFO1 +or UFO2, and _to_ UFO3. +""" + +# adapted from the UFO spec + + +def convertUFO1OrUFO2KerningToUFO3Kerning( + kerning: KerningNested, groups: dict[str, list[str]], glyphSet: Container[str] = () +) -> tuple[KerningNested, dict[str, list[str]], dict[str, dict[str, str]]]: + """Convert kerning data in UFO1 or UFO2 syntax into UFO3 syntax. + + Args: + kerning: + A dictionary containing the kerning rules defined in + the UFO font, as used in :class:`.UFOReader` objects. + groups: + A dictionary containing the groups defined in the UFO + font, as used in :class:`.UFOReader` objects. + glyphSet: + Optional; a set of glyph objects to skip (default: None). + + Returns: + 1. A dictionary representing the converted kerning data. + 2. A copy of the groups dictionary, with all groups renamed to UFO3 syntax. + 3. A dictionary containing the mapping of old group names to new group names. + + """ + # gather known kerning groups based on the prefixes + firstReferencedGroups, secondReferencedGroups = findKnownKerningGroups(groups) + # Make lists of groups referenced in kerning pairs. + for first, seconds in list(kerning.items()): + if first in groups and first not in glyphSet: + if not first.startswith("public.kern1."): + firstReferencedGroups.add(first) + for second in list(seconds.keys()): + if second in groups and second not in glyphSet: + if not second.startswith("public.kern2."): + secondReferencedGroups.add(second) + # Create new names for these groups. + firstRenamedGroups: dict[str, str] = {} + for first in firstReferencedGroups: + # Make a list of existing group names. + existingGroupNames = list(groups.keys()) + list(firstRenamedGroups.keys()) + # Remove the old prefix from the name + newName = first.replace("@MMK_L_", "") + # Add the new prefix to the name. + newName = "public.kern1." + newName + # Make a unique group name. + newName = makeUniqueGroupName(newName, existingGroupNames) + # Store for use later. + firstRenamedGroups[first] = newName + secondRenamedGroups: dict[str, str] = {} + for second in secondReferencedGroups: + # Make a list of existing group names. + existingGroupNames = list(groups.keys()) + list(secondRenamedGroups.keys()) + # Remove the old prefix from the name + newName = second.replace("@MMK_R_", "") + # Add the new prefix to the name. + newName = "public.kern2." + newName + # Make a unique group name. + newName = makeUniqueGroupName(newName, existingGroupNames) + # Store for use later. + secondRenamedGroups[second] = newName + # Populate the new group names into the kerning dictionary as needed. + newKerning = {} + for first, seconds in list(kerning.items()): + first = firstRenamedGroups.get(first, first) + newSeconds = {} + for second, value in list(seconds.items()): + second = secondRenamedGroups.get(second, second) + newSeconds[second] = value + newKerning[first] = newSeconds + # Make copies of the referenced groups and store them + # under the new names in the overall groups dictionary. + allRenamedGroups = list(firstRenamedGroups.items()) + allRenamedGroups += list(secondRenamedGroups.items()) + for oldName, newName in allRenamedGroups: + group = list(groups[oldName]) + groups[newName] = group + # Return the kerning and the groups. + return newKerning, groups, dict(side1=firstRenamedGroups, side2=secondRenamedGroups) + + +def findKnownKerningGroups(groups: Mapping[str, Any]) -> tuple[set[str], set[str]]: + """Find all kerning groups in a UFO1 or UFO2 font that use known prefixes. + + In some cases, not all kerning groups will be referenced + by the kerning pairs in a UFO. The algorithm for locating + groups in :func:`convertUFO1OrUFO2KerningToUFO3Kerning` will + miss these unreferenced groups. By scanning for known prefixes, + this function will catch all of the prefixed groups. + + The prefixes and sides by this function are: + + @MMK_L_ - side 1 + @MMK_R_ - side 2 + + as defined in the UFO1 specification. + + Args: + groups: + A dictionary containing the groups defined in the UFO + font, as read by :class:`.UFOReader`. + + Returns: + Two sets; the first containing the names of all + first-side kerning groups identified in the ``groups`` + dictionary, and the second containing the names of all + second-side kerning groups identified. + + "First-side" and "second-side" are with respect to the + writing direction of the script. + + Example:: + + >>> testGroups = { + ... "@MMK_L_1" : None, + ... "@MMK_L_2" : None, + ... "@MMK_L_3" : None, + ... "@MMK_R_1" : None, + ... "@MMK_R_2" : None, + ... "@MMK_R_3" : None, + ... "@MMK_l_1" : None, + ... "@MMK_r_1" : None, + ... "@MMK_X_1" : None, + ... "foo" : None, + ... } + >>> first, second = findKnownKerningGroups(testGroups) + >>> sorted(first) == ['@MMK_L_1', '@MMK_L_2', '@MMK_L_3'] + True + >>> sorted(second) == ['@MMK_R_1', '@MMK_R_2', '@MMK_R_3'] + True + """ + knownFirstGroupPrefixes = ["@MMK_L_"] + knownSecondGroupPrefixes = ["@MMK_R_"] + firstGroups = set() + secondGroups = set() + for groupName in list(groups.keys()): + for firstPrefix in knownFirstGroupPrefixes: + if groupName.startswith(firstPrefix): + firstGroups.add(groupName) + break + for secondPrefix in knownSecondGroupPrefixes: + if groupName.startswith(secondPrefix): + secondGroups.add(groupName) + break + return firstGroups, secondGroups + + +def makeUniqueGroupName(name: str, groupNames: list[str], counter: int = 0) -> str: + """Make a kerning group name that will be unique within the set of group names. + + If the requested kerning group name already exists within the set, this + will return a new name by adding an incremented counter to the end + of the requested name. + + Args: + name: + The requested kerning group name. + groupNames: + A list of the existing kerning group names. + counter: + Optional; a counter of group names already seen (default: 0). If + :attr:`.counter` is not provided, the function will recurse, + incrementing the value of :attr:`.counter` until it finds the + first unused ``name+counter`` combination, and return that result. + + Returns: + A unique kerning group name composed of the requested name suffixed + by the smallest available integer counter. + """ + # Add a number to the name if the counter is higher than zero. + newName = name + if counter > 0: + newName = "%s%d" % (newName, counter) + # If the new name is in the existing group names, recurse. + if newName in groupNames: + return makeUniqueGroupName(name, groupNames, counter + 1) + # Otherwise send back the new name. + return newName + + +def test(): + """ + Tests for :func:`.convertUFO1OrUFO2KerningToUFO3Kerning`. + + No known prefixes. + + >>> testKerning = { + ... "A" : { + ... "A" : 1, + ... "B" : 2, + ... "CGroup" : 3, + ... "DGroup" : 4 + ... }, + ... "BGroup" : { + ... "A" : 5, + ... "B" : 6, + ... "CGroup" : 7, + ... "DGroup" : 8 + ... }, + ... "CGroup" : { + ... "A" : 9, + ... "B" : 10, + ... "CGroup" : 11, + ... "DGroup" : 12 + ... }, + ... } + >>> testGroups = { + ... "BGroup" : ["B"], + ... "CGroup" : ["C"], + ... "DGroup" : ["D"], + ... } + >>> kerning, groups, maps = convertUFO1OrUFO2KerningToUFO3Kerning( + ... testKerning, testGroups, []) + >>> expected = { + ... "A" : { + ... "A": 1, + ... "B": 2, + ... "public.kern2.CGroup": 3, + ... "public.kern2.DGroup": 4 + ... }, + ... "public.kern1.BGroup": { + ... "A": 5, + ... "B": 6, + ... "public.kern2.CGroup": 7, + ... "public.kern2.DGroup": 8 + ... }, + ... "public.kern1.CGroup": { + ... "A": 9, + ... "B": 10, + ... "public.kern2.CGroup": 11, + ... "public.kern2.DGroup": 12 + ... } + ... } + >>> kerning == expected + True + >>> expected = { + ... "BGroup": ["B"], + ... "CGroup": ["C"], + ... "DGroup": ["D"], + ... "public.kern1.BGroup": ["B"], + ... "public.kern1.CGroup": ["C"], + ... "public.kern2.CGroup": ["C"], + ... "public.kern2.DGroup": ["D"], + ... } + >>> groups == expected + True + + Known prefixes. + + >>> testKerning = { + ... "A" : { + ... "A" : 1, + ... "B" : 2, + ... "@MMK_R_CGroup" : 3, + ... "@MMK_R_DGroup" : 4 + ... }, + ... "@MMK_L_BGroup" : { + ... "A" : 5, + ... "B" : 6, + ... "@MMK_R_CGroup" : 7, + ... "@MMK_R_DGroup" : 8 + ... }, + ... "@MMK_L_CGroup" : { + ... "A" : 9, + ... "B" : 10, + ... "@MMK_R_CGroup" : 11, + ... "@MMK_R_DGroup" : 12 + ... }, + ... } + >>> testGroups = { + ... "@MMK_L_BGroup" : ["B"], + ... "@MMK_L_CGroup" : ["C"], + ... "@MMK_L_XGroup" : ["X"], + ... "@MMK_R_CGroup" : ["C"], + ... "@MMK_R_DGroup" : ["D"], + ... "@MMK_R_XGroup" : ["X"], + ... } + >>> kerning, groups, maps = convertUFO1OrUFO2KerningToUFO3Kerning( + ... testKerning, testGroups, []) + >>> expected = { + ... "A" : { + ... "A": 1, + ... "B": 2, + ... "public.kern2.CGroup": 3, + ... "public.kern2.DGroup": 4 + ... }, + ... "public.kern1.BGroup": { + ... "A": 5, + ... "B": 6, + ... "public.kern2.CGroup": 7, + ... "public.kern2.DGroup": 8 + ... }, + ... "public.kern1.CGroup": { + ... "A": 9, + ... "B": 10, + ... "public.kern2.CGroup": 11, + ... "public.kern2.DGroup": 12 + ... } + ... } + >>> kerning == expected + True + >>> expected = { + ... "@MMK_L_BGroup": ["B"], + ... "@MMK_L_CGroup": ["C"], + ... "@MMK_L_XGroup": ["X"], + ... "@MMK_R_CGroup": ["C"], + ... "@MMK_R_DGroup": ["D"], + ... "@MMK_R_XGroup": ["X"], + ... "public.kern1.BGroup": ["B"], + ... "public.kern1.CGroup": ["C"], + ... "public.kern1.XGroup": ["X"], + ... "public.kern2.CGroup": ["C"], + ... "public.kern2.DGroup": ["D"], + ... "public.kern2.XGroup": ["X"], + ... } + >>> groups == expected + True + + >>> from .validators import kerningValidator + >>> kerningValidator(kerning) + (True, None) + + Mixture of known prefixes and groups without prefixes. + + >>> testKerning = { + ... "A" : { + ... "A" : 1, + ... "B" : 2, + ... "@MMK_R_CGroup" : 3, + ... "DGroup" : 4 + ... }, + ... "BGroup" : { + ... "A" : 5, + ... "B" : 6, + ... "@MMK_R_CGroup" : 7, + ... "DGroup" : 8 + ... }, + ... "@MMK_L_CGroup" : { + ... "A" : 9, + ... "B" : 10, + ... "@MMK_R_CGroup" : 11, + ... "DGroup" : 12 + ... }, + ... } + >>> testGroups = { + ... "BGroup" : ["B"], + ... "@MMK_L_CGroup" : ["C"], + ... "@MMK_R_CGroup" : ["C"], + ... "DGroup" : ["D"], + ... } + >>> kerning, groups, maps = convertUFO1OrUFO2KerningToUFO3Kerning( + ... testKerning, testGroups, []) + >>> expected = { + ... "A" : { + ... "A": 1, + ... "B": 2, + ... "public.kern2.CGroup": 3, + ... "public.kern2.DGroup": 4 + ... }, + ... "public.kern1.BGroup": { + ... "A": 5, + ... "B": 6, + ... "public.kern2.CGroup": 7, + ... "public.kern2.DGroup": 8 + ... }, + ... "public.kern1.CGroup": { + ... "A": 9, + ... "B": 10, + ... "public.kern2.CGroup": 11, + ... "public.kern2.DGroup": 12 + ... } + ... } + >>> kerning == expected + True + >>> expected = { + ... "BGroup": ["B"], + ... "@MMK_L_CGroup": ["C"], + ... "@MMK_R_CGroup": ["C"], + ... "DGroup": ["D"], + ... "public.kern1.BGroup": ["B"], + ... "public.kern1.CGroup": ["C"], + ... "public.kern2.CGroup": ["C"], + ... "public.kern2.DGroup": ["D"], + ... } + >>> groups == expected + True + """ + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/errors.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..6cc9fec399471c1dff9141187650c2caabc53d58 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/errors.py @@ -0,0 +1,30 @@ +from __future__ import annotations + + +class UFOLibError(Exception): + pass + + +class UnsupportedUFOFormat(UFOLibError): + pass + + +class GlifLibError(UFOLibError): + """An error raised by glifLib. + + This class is a loose backport of PEP 678, adding a :attr:`.note` + attribute that can hold additional context for errors encountered. + + It will be maintained until only Python 3.11-and-later are supported. + """ + + def _add_note(self, note: str) -> None: + # Loose backport of PEP 678 until we only support Python 3.11+, used for + # adding additional context to errors. + # TODO: Replace with https://docs.python.org/3.11/library/exceptions.html#BaseException.add_note + (message, *rest) = self.args + self.args = ((message + "\n" + note), *rest) + + +class UnsupportedGLIFFormat(GlifLibError): + pass diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/etree.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/etree.py new file mode 100644 index 0000000000000000000000000000000000000000..07b924ac8523c0680b175f0269c42fa1959abc47 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/etree.py @@ -0,0 +1,6 @@ +"""DEPRECATED - This module is kept here only as a backward compatibility shim +for the old ufoLib.etree module, which was moved to :mod:`fontTools.misc.etree`. +Please use the latter instead. +""" + +from fontTools.misc.etree import * diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/filenames.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/filenames.py new file mode 100644 index 0000000000000000000000000000000000000000..6a4090a75536a0dc1e3d7b7e096982d3e6287726 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/filenames.py @@ -0,0 +1,356 @@ +from __future__ import annotations + +from collections.abc import Iterable + +""" +Convert user-provided internal UFO names to spec-compliant filenames. + +This module implements the algorithm for converting between a "user name" - +something that a user can choose arbitrarily inside a font editor - and a file +name suitable for use in a wide range of operating systems and filesystems. + +The `UFO 3 specification <http://unifiedfontobject.org/versions/ufo3/conventions/>`_ +provides an example of an algorithm for such conversion, which avoids illegal +characters, reserved file names, ambiguity between upper- and lower-case +characters, and clashes with existing files. + +This code was originally copied from +`ufoLib <https://github.com/unified-font-object/ufoLib/blob/8747da7/Lib/ufoLib/filenames.py>`_ +by Tal Leming and is copyright (c) 2005-2016, The RoboFab Developers: + +- Erik van Blokland +- Tal Leming +- Just van Rossum +""" + +# Restrictions are taken mostly from +# https://docs.microsoft.com/en-gb/windows/win32/fileio/naming-a-file#naming-conventions. +# +# 1. Integer value zero, sometimes referred to as the ASCII NUL character. +# 2. Characters whose integer representations are in the range 1 to 31, +# inclusive. +# 3. Various characters that (mostly) Windows and POSIX-y filesystems don't +# allow, plus "(" and ")", as per the specification. +illegalCharacters: set[str] = { + "\x00", + "\x01", + "\x02", + "\x03", + "\x04", + "\x05", + "\x06", + "\x07", + "\x08", + "\t", + "\n", + "\x0b", + "\x0c", + "\r", + "\x0e", + "\x0f", + "\x10", + "\x11", + "\x12", + "\x13", + "\x14", + "\x15", + "\x16", + "\x17", + "\x18", + "\x19", + "\x1a", + "\x1b", + "\x1c", + "\x1d", + "\x1e", + "\x1f", + '"', + "*", + "+", + "/", + ":", + "<", + ">", + "?", + "[", + "\\", + "]", + "(", + ")", + "|", + "\x7f", +} +reservedFileNames: set[str] = { + "aux", + "clock$", + "com1", + "com2", + "com3", + "com4", + "com5", + "com6", + "com7", + "com8", + "com9", + "con", + "lpt1", + "lpt2", + "lpt3", + "lpt4", + "lpt5", + "lpt6", + "lpt7", + "lpt8", + "lpt9", + "nul", + "prn", +} +maxFileNameLength: int = 255 + + +class NameTranslationError(Exception): + pass + + +def userNameToFileName( + userName: str, existing: Iterable[str] = (), prefix: str = "", suffix: str = "" +) -> str: + """Converts from a user name to a file name. + + Takes care to avoid illegal characters, reserved file names, ambiguity between + upper- and lower-case characters, and clashes with existing files. + + Args: + userName (str): The input file name. + existing: A case-insensitive list of all existing file names. + prefix: Prefix to be prepended to the file name. + suffix: Suffix to be appended to the file name. + + Returns: + A suitable filename. + + Raises: + NameTranslationError: If no suitable name could be generated. + + Examples:: + + >>> userNameToFileName("a") == "a" + True + >>> userNameToFileName("A") == "A_" + True + >>> userNameToFileName("AE") == "A_E_" + True + >>> userNameToFileName("Ae") == "A_e" + True + >>> userNameToFileName("ae") == "ae" + True + >>> userNameToFileName("aE") == "aE_" + True + >>> userNameToFileName("a.alt") == "a.alt" + True + >>> userNameToFileName("A.alt") == "A_.alt" + True + >>> userNameToFileName("A.Alt") == "A_.A_lt" + True + >>> userNameToFileName("A.aLt") == "A_.aL_t" + True + >>> userNameToFileName(u"A.alT") == "A_.alT_" + True + >>> userNameToFileName("T_H") == "T__H_" + True + >>> userNameToFileName("T_h") == "T__h" + True + >>> userNameToFileName("t_h") == "t_h" + True + >>> userNameToFileName("F_F_I") == "F__F__I_" + True + >>> userNameToFileName("f_f_i") == "f_f_i" + True + >>> userNameToFileName("Aacute_V.swash") == "A_acute_V_.swash" + True + >>> userNameToFileName(".notdef") == "_notdef" + True + >>> userNameToFileName("con") == "_con" + True + >>> userNameToFileName("CON") == "C_O_N_" + True + >>> userNameToFileName("con.alt") == "_con.alt" + True + >>> userNameToFileName("alt.con") == "alt._con" + True + """ + # the incoming name must be a string + if not isinstance(userName, str): + raise ValueError("The value for userName must be a string.") + # establish the prefix and suffix lengths + prefixLength = len(prefix) + suffixLength = len(suffix) + # replace an initial period with an _ + # if no prefix is to be added + if not prefix and userName[0] == ".": + userName = "_" + userName[1:] + # filter the user name + filteredUserName = [] + for character in userName: + # replace illegal characters with _ + if character in illegalCharacters: + character = "_" + # add _ to all non-lower characters + elif character != character.lower(): + character += "_" + filteredUserName.append(character) + userName = "".join(filteredUserName) + # clip to 255 + sliceLength = maxFileNameLength - prefixLength - suffixLength + userName = userName[:sliceLength] + # test for illegal files names + parts = [] + for part in userName.split("."): + if part.lower() in reservedFileNames: + part = "_" + part + parts.append(part) + userName = ".".join(parts) + # test for clash + fullName = prefix + userName + suffix + if fullName.lower() in existing: + fullName = handleClash1(userName, existing, prefix, suffix) + # finished + return fullName + + +def handleClash1( + userName: str, existing: Iterable[str] = [], prefix: str = "", suffix: str = "" +) -> str: + """A helper function that resolves collisions with existing names when choosing a filename. + + This function attempts to append an unused integer counter to the filename. + + Args: + userName (str): The input file name. + existing: A case-insensitive list of all existing file names. + prefix: Prefix to be prepended to the file name. + suffix: Suffix to be appended to the file name. + + Returns: + A suitable filename. + + >>> prefix = ("0" * 5) + "." + >>> suffix = "." + ("0" * 10) + >>> existing = ["a" * 5] + + >>> e = list(existing) + >>> handleClash1(userName="A" * 5, existing=e, + ... prefix=prefix, suffix=suffix) == ( + ... '00000.AAAAA000000000000001.0000000000') + True + + >>> e = list(existing) + >>> e.append(prefix + "aaaaa" + "1".zfill(15) + suffix) + >>> handleClash1(userName="A" * 5, existing=e, + ... prefix=prefix, suffix=suffix) == ( + ... '00000.AAAAA000000000000002.0000000000') + True + + >>> e = list(existing) + >>> e.append(prefix + "AAAAA" + "2".zfill(15) + suffix) + >>> handleClash1(userName="A" * 5, existing=e, + ... prefix=prefix, suffix=suffix) == ( + ... '00000.AAAAA000000000000001.0000000000') + True + """ + # if the prefix length + user name length + suffix length + 15 is at + # or past the maximum length, silce 15 characters off of the user name + prefixLength = len(prefix) + suffixLength = len(suffix) + if prefixLength + len(userName) + suffixLength + 15 > maxFileNameLength: + l = prefixLength + len(userName) + suffixLength + 15 + sliceLength = maxFileNameLength - l + userName = userName[:sliceLength] + finalName = None + # try to add numbers to create a unique name + counter = 1 + while finalName is None: + name = userName + str(counter).zfill(15) + fullName = prefix + name + suffix + if fullName.lower() not in existing: + finalName = fullName + break + else: + counter += 1 + if counter >= 999999999999999: + break + # if there is a clash, go to the next fallback + if finalName is None: + finalName = handleClash2(existing, prefix, suffix) + # finished + return finalName + + +def handleClash2( + existing: Iterable[str] = [], prefix: str = "", suffix: str = "" +) -> str: + """A helper function that resolves collisions with existing names when choosing a filename. + + This function is a fallback to :func:`handleClash1`. It attempts to append an unused integer counter to the filename. + + Args: + userName (str): The input file name. + existing: A case-insensitive list of all existing file names. + prefix: Prefix to be prepended to the file name. + suffix: Suffix to be appended to the file name. + + Returns: + A suitable filename. + + Raises: + NameTranslationError: If no suitable name could be generated. + + Examples:: + + >>> prefix = ("0" * 5) + "." + >>> suffix = "." + ("0" * 10) + >>> existing = [prefix + str(i) + suffix for i in range(100)] + + >>> e = list(existing) + >>> handleClash2(existing=e, prefix=prefix, suffix=suffix) == ( + ... '00000.100.0000000000') + True + + >>> e = list(existing) + >>> e.remove(prefix + "1" + suffix) + >>> handleClash2(existing=e, prefix=prefix, suffix=suffix) == ( + ... '00000.1.0000000000') + True + + >>> e = list(existing) + >>> e.remove(prefix + "2" + suffix) + >>> handleClash2(existing=e, prefix=prefix, suffix=suffix) == ( + ... '00000.2.0000000000') + True + """ + # calculate the longest possible string + maxLength = maxFileNameLength - len(prefix) - len(suffix) + maxValue = int("9" * maxLength) + # try to find a number + finalName = None + counter = 1 + while finalName is None: + fullName = prefix + str(counter) + suffix + if fullName.lower() not in existing: + finalName = fullName + break + else: + counter += 1 + if counter >= maxValue: + break + # raise an error if nothing has been found + if finalName is None: + raise NameTranslationError("No unique name could be found.") + # finished + return finalName + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/glifLib.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/glifLib.py new file mode 100644 index 0000000000000000000000000000000000000000..c9929b2f69bcd7c5e93e4c65e24361e87d9d534a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/glifLib.py @@ -0,0 +1,2123 @@ +""" +Generic module for reading and writing the .glif format. + +More info about the .glif format (GLyphInterchangeFormat) can be found here: + + http://unifiedfontobject.org + +The main class in this module is :class:`GlyphSet`. It manages a set of .glif files +in a folder. It offers two ways to read glyph data, and one way to write +glyph data. See the class doc string for details. +""" + +from __future__ import annotations + +import logging +from collections import OrderedDict +from typing import TYPE_CHECKING, Any, Optional, Union, cast +from warnings import warn + +import fontTools.misc.filesystem as fs +from fontTools.misc import etree, plistlib +from fontTools.misc.textTools import tobytes +from fontTools.pens.pointPen import AbstractPointPen, PointToSegmentPen +from fontTools.ufoLib import UFOFormatVersion, _UFOBaseIO +from fontTools.ufoLib.errors import GlifLibError +from fontTools.ufoLib.filenames import userNameToFileName +from fontTools.ufoLib.utils import ( + BaseFormatVersion, + normalizeFormatVersion, + numberTypes, +) +from fontTools.ufoLib.validators import ( + anchorsValidator, + colorValidator, + genericTypeValidator, + glyphLibValidator, + guidelinesValidator, + identifierValidator, + imageValidator, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Set + from logging import Logger + + from fontTools.annotations import ( + ElementType, + FormatVersion, + FormatVersions, + GLIFFormatVersionInput, + GlyphNameToFileNameFunc, + IntFloat, + PathOrFS, + UFOFormatVersionInput, + ) + from fontTools.misc.filesystem._base import FS + + +__all__: list[str] = [ + "GlyphSet", + "GlifLibError", + "readGlyphFromString", + "writeGlyphToString", + "glyphNameToFileName", +] + +logger: Logger = logging.getLogger(__name__) + + +# --------- +# Constants +# --------- + +CONTENTS_FILENAME = "contents.plist" +LAYERINFO_FILENAME = "layerinfo.plist" + + +class GLIFFormatVersion(BaseFormatVersion): + """Class representing the versions of the .glif format supported by the UFO version in use. + + For a given :mod:`fontTools.ufoLib.UFOFormatVersion`, the :func:`supported_versions` method will + return the supported versions of the GLIF file format. If the UFO version is unspecified, the + :func:`supported_versions` method will return all available GLIF format versions. + """ + + FORMAT_1_0 = (1, 0) + FORMAT_2_0 = (2, 0) + + @classmethod + def default( + cls, ufoFormatVersion: Optional[UFOFormatVersion] = None + ) -> GLIFFormatVersion: + if ufoFormatVersion is not None: + return max(cls.supported_versions(ufoFormatVersion)) + return super().default() + + @classmethod + def supported_versions( + cls, ufoFormatVersion: Optional[UFOFormatVersion] = None + ) -> frozenset[GLIFFormatVersion]: + if ufoFormatVersion is None: + # if ufo format unspecified, return all the supported GLIF formats + return super().supported_versions() + # else only return the GLIF formats supported by the given UFO format + versions = {cls.FORMAT_1_0} + if ufoFormatVersion >= UFOFormatVersion.FORMAT_3_0: + versions.add(cls.FORMAT_2_0) + return frozenset(versions) + + +# ------------ +# Simple Glyph +# ------------ + + +class Glyph: + """ + Minimal glyph object. It has no glyph attributes until either + the draw() or the drawPoints() method has been called. + """ + + def __init__(self, glyphName: str, glyphSet: GlyphSet) -> None: + self.glyphName: str = glyphName + self.glyphSet: GlyphSet = glyphSet + + def draw(self, pen: Any, outputImpliedClosingLine: bool = False) -> None: + """ + Draw this glyph onto a *FontTools* Pen. + """ + pointPen = PointToSegmentPen( + pen, outputImpliedClosingLine=outputImpliedClosingLine + ) + self.drawPoints(pointPen) + + def drawPoints(self, pointPen: AbstractPointPen) -> None: + """ + Draw this glyph onto a PointPen. + """ + self.glyphSet.readGlyph(self.glyphName, self, pointPen) + + +# --------- +# Glyph Set +# --------- + + +class GlyphSet(_UFOBaseIO): + """ + GlyphSet manages a set of .glif files inside one directory. + + GlyphSet's constructor takes a path to an existing directory as it's + first argument. Reading glyph data can either be done through the + readGlyph() method, or by using GlyphSet's dictionary interface, where + the keys are glyph names and the values are (very) simple glyph objects. + + To write a glyph to the glyph set, you use the writeGlyph() method. + The simple glyph objects returned through the dict interface do not + support writing, they are just a convenient way to get at the glyph data. + """ + + glyphClass = Glyph + + def __init__( + self, + path: PathOrFS, + glyphNameToFileNameFunc: GlyphNameToFileNameFunc = None, + ufoFormatVersion: UFOFormatVersionInput = None, + validateRead: bool = True, + validateWrite: bool = True, + expectContentsFile: bool = False, + ) -> None: + """ + 'path' should be a path (string) to an existing local directory, or + an instance of fs.base.FS class. + + The optional 'glyphNameToFileNameFunc' argument must be a callback + function that takes two arguments: a glyph name and a list of all + existing filenames (if any exist). It should return a file name + (including the .glif extension). The glyphNameToFileName function + is called whenever a file name is created for a given glyph name. + + ``validateRead`` will validate read operations. Its default is ``True``. + ``validateWrite`` will validate write operations. Its default is ``True``. + ``expectContentsFile`` will raise a GlifLibError if a contents.plist file is + not found on the glyph set file system. This should be set to ``True`` if you + are reading an existing UFO and ``False`` if you create a fresh glyph set. + """ + try: + ufoFormatVersion = normalizeFormatVersion( + ufoFormatVersion, UFOFormatVersion + ) + except ValueError as e: + from fontTools.ufoLib.errors import UnsupportedUFOFormat + + raise UnsupportedUFOFormat( + f"Unsupported UFO format: {ufoFormatVersion!r}" + ) from e + + if hasattr(path, "__fspath__"): # support os.PathLike objects + path = path.__fspath__() + + if isinstance(path, str): + try: + filesystem: FS = fs.osfs.OSFS(path) + except fs.errors.CreateFailed: + raise GlifLibError("No glyphs directory '%s'" % path) + self._shouldClose: bool = True + elif isinstance(path, fs.base.FS): + filesystem = path + try: + filesystem.check() + except fs.errors.FilesystemClosed: + raise GlifLibError("the filesystem '%s' is closed" % filesystem) + self._shouldClose = False + else: + raise TypeError( + "Expected a path string or fs object, found %s" % type(path).__name__ + ) + try: + path = filesystem.getsyspath("/") + except fs.errors.NoSysPath: + # network or in-memory FS may not map to the local one + path = str(filesystem) + # 'dirName' is kept for backward compatibility only, but it's DEPRECATED + # as it's not guaranteed that it maps to an existing OSFS directory. + # Client could use the FS api via the `self.fs` attribute instead. + self.dirName: str = fs.path.basename(path) + self.fs: FS = filesystem + # if glyphSet contains no 'contents.plist', we consider it empty + self._havePreviousFile: bool = filesystem.exists(CONTENTS_FILENAME) + if expectContentsFile and not self._havePreviousFile: + raise GlifLibError(f"{CONTENTS_FILENAME} is missing.") + # attribute kept for backward compatibility + self.ufoFormatVersion: int = ufoFormatVersion.major + self.ufoFormatVersionTuple: UFOFormatVersion = ufoFormatVersion + if glyphNameToFileNameFunc is None: + glyphNameToFileNameFunc = glyphNameToFileName + self.glyphNameToFileName: Callable[[str, set[str]], str] = ( + glyphNameToFileNameFunc + ) + self._validateRead: bool = validateRead + self._validateWrite: bool = validateWrite + self._existingFileNames: set[str] | None = None + self._reverseContents: Optional[dict[str, str]] = None + + self.rebuildContents() + + def rebuildContents(self, validateRead: bool = False) -> None: + """ + Rebuild the contents dict by loading contents.plist. + + ``validateRead`` will validate the data, by default it is set to the + class's ``validateRead`` value, can be overridden. + """ + if validateRead is None: + validateRead = self._validateRead + contents = self._getPlist(CONTENTS_FILENAME, {}) + # validate the contents + if validateRead: + invalidFormat = False + if not isinstance(contents, dict): + invalidFormat = True + else: + for name, fileName in contents.items(): + if not isinstance(name, str): + invalidFormat = True + if not isinstance(fileName, str): + invalidFormat = True + elif not self.fs.exists(fileName): + raise GlifLibError( + "%s references a file that does not exist: %s" + % (CONTENTS_FILENAME, fileName) + ) + if invalidFormat: + raise GlifLibError("%s is not properly formatted" % CONTENTS_FILENAME) + self.contents: dict[str, str] = contents + self._existingFileNames = None + self._reverseContents = None + + def getReverseContents(self) -> dict[str, str]: + """ + Return a reversed dict of self.contents, mapping file names to + glyph names. This is primarily an aid for custom glyph name to file + name schemes that want to make sure they don't generate duplicate + file names. The file names are converted to lowercase so we can + reliably check for duplicates that only differ in case, which is + important for case-insensitive file systems. + """ + if self._reverseContents is None: + d = {} + for k, v in self.contents.items(): + d[v.lower()] = k + self._reverseContents = d + return self._reverseContents + + def writeContents(self) -> None: + """ + Write the contents.plist file out to disk. Call this method when + you're done writing glyphs. + """ + self._writePlist(CONTENTS_FILENAME, self.contents) + + # layer info + + def readLayerInfo(self, info: Any, validateRead: Optional[bool] = None) -> None: + """ + ``validateRead`` will validate the data, by default it is set to the + class's ``validateRead`` value, can be overridden. + """ + if validateRead is None: + validateRead = self._validateRead + infoDict = self._getPlist(LAYERINFO_FILENAME, {}) + if validateRead: + if not isinstance(infoDict, dict): + raise GlifLibError("layerinfo.plist is not properly formatted.") + infoDict = validateLayerInfoVersion3Data(infoDict) + # populate the object + for attr, value in infoDict.items(): + try: + setattr(info, attr, value) + except AttributeError: + raise GlifLibError( + "The supplied layer info object does not support setting a necessary attribute (%s)." + % attr + ) + + def writeLayerInfo(self, info: Any, validateWrite: Optional[bool] = None) -> None: + """ + ``validateWrite`` will validate the data, by default it is set to the + class's ``validateWrite`` value, can be overridden. + """ + if validateWrite is None: + validateWrite = self._validateWrite + if self.ufoFormatVersionTuple.major < 3: + raise GlifLibError( + "layerinfo.plist is not allowed in UFO %d." + % self.ufoFormatVersionTuple.major + ) + # gather data + infoData = {} + for attr in layerInfoVersion3ValueData.keys(): + if hasattr(info, attr): + try: + value = getattr(info, attr) + except AttributeError: + raise GlifLibError( + "The supplied info object does not support getting a necessary attribute (%s)." + % attr + ) + if value is None or (attr == "lib" and not value): + continue + infoData[attr] = value + if infoData: + # validate + if validateWrite: + infoData = validateLayerInfoVersion3Data(infoData) + # write file + self._writePlist(LAYERINFO_FILENAME, infoData) + elif self._havePreviousFile and self.fs.exists(LAYERINFO_FILENAME): + # data empty, remove existing file + self.fs.remove(LAYERINFO_FILENAME) + + def getGLIF(self, glyphName: str) -> bytes: + """ + Get the raw GLIF text for a given glyph name. This only works + for GLIF files that are already on disk. + + This method is useful in situations when the raw XML needs to be + read from a glyph set for a particular glyph before fully parsing + it into an object structure via the readGlyph method. + + Raises KeyError if 'glyphName' is not in contents.plist, or + GlifLibError if the file associated with can't be found. + """ + fileName = self.contents[glyphName] + try: + return self.fs.readbytes(fileName) + except fs.errors.ResourceNotFound: + raise GlifLibError( + "The file '%s' associated with glyph '%s' in contents.plist " + "does not exist on %s" % (fileName, glyphName, self.fs) + ) + + def getGLIFModificationTime(self, glyphName: str) -> Optional[float]: + """ + Returns the modification time for the GLIF file with 'glyphName', as + a floating point number giving the number of seconds since the epoch. + Return None if the associated file does not exist or the underlying + filesystem does not support getting modified times. + Raises KeyError if the glyphName is not in contents.plist. + """ + fileName = self.contents[glyphName] + return self.getFileModificationTime(fileName) + + # reading/writing API + + def readGlyph( + self, + glyphName: str, + glyphObject: Optional[Any] = None, + pointPen: Optional[AbstractPointPen] = None, + validate: Optional[bool] = None, + ) -> None: + """ + Read a .glif file for 'glyphName' from the glyph set. The + 'glyphObject' argument can be any kind of object (even None); + the readGlyph() method will attempt to set the following + attributes on it: + + width + the advance width of the glyph + height + the advance height of the glyph + unicodes + a list of unicode values for this glyph + note + a string + lib + a dictionary containing custom data + image + a dictionary containing image data + guidelines + a list of guideline data dictionaries + anchors + a list of anchor data dictionaries + + All attributes are optional, in two ways: + + 1) An attribute *won't* be set if the .glif file doesn't + contain data for it. 'glyphObject' will have to deal + with default values itself. + 2) If setting the attribute fails with an AttributeError + (for example if the 'glyphObject' attribute is read- + only), readGlyph() will not propagate that exception, + but ignore that attribute. + + To retrieve outline information, you need to pass an object + conforming to the PointPen protocol as the 'pointPen' argument. + This argument may be None if you don't need the outline data. + + readGlyph() will raise KeyError if the glyph is not present in + the glyph set. + + ``validate`` will validate the data, by default it is set to the + class's ``validateRead`` value, can be overridden. + """ + if validate is None: + validate = self._validateRead + text = self.getGLIF(glyphName) + try: + tree = _glifTreeFromString(text) + formatVersions = GLIFFormatVersion.supported_versions( + self.ufoFormatVersionTuple + ) + _readGlyphFromTree( + tree, + glyphObject, + pointPen, + formatVersions=formatVersions, + validate=validate, + ) + except GlifLibError as glifLibError: + # Re-raise with a note that gives extra context, describing where + # the error occurred. + fileName = self.contents[glyphName] + try: + glifLocation = f"'{self.fs.getsyspath(fileName)}'" + except fs.errors.NoSysPath: + # Network or in-memory FS may not map to a local path, so use + # the best string representation we have. + glifLocation = f"'{fileName}' from '{str(self.fs)}'" + + glifLibError._add_note( + f"The issue is in glyph '{glyphName}', located in {glifLocation}." + ) + raise + + def writeGlyph( + self, + glyphName: str, + glyphObject: Optional[Any] = None, + drawPointsFunc: Optional[Callable[[AbstractPointPen], None]] = None, + formatVersion: GLIFFormatVersionInput = None, + validate: Optional[bool] = None, + ) -> None: + """ + Write a .glif file for 'glyphName' to the glyph set. The + 'glyphObject' argument can be any kind of object (even None); + the writeGlyph() method will attempt to get the following + attributes from it: + + width + the advance width of the glyph + height + the advance height of the glyph + unicodes + a list of unicode values for this glyph + note + a string + lib + a dictionary containing custom data + image + a dictionary containing image data + guidelines + a list of guideline data dictionaries + anchors + a list of anchor data dictionaries + + All attributes are optional: if 'glyphObject' doesn't + have the attribute, it will simply be skipped. + + To write outline data to the .glif file, writeGlyph() needs + a function (any callable object actually) that will take one + argument: an object that conforms to the PointPen protocol. + The function will be called by writeGlyph(); it has to call the + proper PointPen methods to transfer the outline to the .glif file. + + The GLIF format version will be chosen based on the ufoFormatVersion + passed during the creation of this object. If a particular format + version is desired, it can be passed with the formatVersion argument. + The formatVersion argument accepts either a tuple of integers for + (major, minor), or a single integer for the major digit only (with + minor digit implied as 0). + + An UnsupportedGLIFFormat exception is raised if the requested GLIF + formatVersion is not supported. + + ``validate`` will validate the data, by default it is set to the + class's ``validateWrite`` value, can be overridden. + """ + if formatVersion is None: + formatVersion = GLIFFormatVersion.default(self.ufoFormatVersionTuple) + else: + try: + formatVersion = normalizeFormatVersion(formatVersion, GLIFFormatVersion) + except ValueError as e: + from fontTools.ufoLib.errors import UnsupportedGLIFFormat + + raise UnsupportedGLIFFormat( + f"Unsupported GLIF format version: {formatVersion!r}" + ) from e + if formatVersion not in GLIFFormatVersion.supported_versions( + self.ufoFormatVersionTuple + ): + from fontTools.ufoLib.errors import UnsupportedGLIFFormat + + raise UnsupportedGLIFFormat( + f"Unsupported GLIF format version ({formatVersion!s}) " + f"for UFO format version {self.ufoFormatVersionTuple!s}." + ) + if validate is None: + validate = self._validateWrite + fileName = self.contents.get(glyphName) + if fileName is None: + if self._existingFileNames is None: + self._existingFileNames = { + fileName.lower() for fileName in self.contents.values() + } + fileName = self.glyphNameToFileName(glyphName, self._existingFileNames) + self.contents[glyphName] = fileName + self._existingFileNames.add(fileName.lower()) + if self._reverseContents is not None: + self._reverseContents[fileName.lower()] = glyphName + data = _writeGlyphToBytes( + glyphName, + glyphObject, + drawPointsFunc, + formatVersion=formatVersion, + validate=validate, + ) + if ( + self._havePreviousFile + and self.fs.exists(fileName) + and data == self.fs.readbytes(fileName) + ): + return + self.fs.writebytes(fileName, data) + + def deleteGlyph(self, glyphName: str) -> None: + """Permanently delete the glyph from the glyph set on disk. Will + raise KeyError if the glyph is not present in the glyph set. + """ + fileName = self.contents[glyphName] + self.fs.remove(fileName) + if self._existingFileNames is not None: + self._existingFileNames.remove(fileName.lower()) + if self._reverseContents is not None: + del self._reverseContents[fileName.lower()] + del self.contents[glyphName] + + # dict-like support + + def keys(self) -> list[str]: + return list(self.contents.keys()) + + def has_key(self, glyphName: str) -> bool: + return glyphName in self.contents + + __contains__ = has_key + + def __len__(self) -> int: + return len(self.contents) + + def __getitem__(self, glyphName: str) -> Any: + if glyphName not in self.contents: + raise KeyError(glyphName) + return self.glyphClass(glyphName, self) + + # quickly fetch unicode values + + def getUnicodes( + self, glyphNames: Optional[Iterable[str]] = None + ) -> dict[str, list[int]]: + """ + Return a dictionary that maps glyph names to lists containing + the unicode value[s] for that glyph, if any. This parses the .glif + files partially, so it is a lot faster than parsing all files completely. + By default this checks all glyphs, but a subset can be passed with glyphNames. + """ + unicodes = {} + if glyphNames is None: + glyphNames = self.contents.keys() + for glyphName in glyphNames: + text = self.getGLIF(glyphName) + unicodes[glyphName] = _fetchUnicodes(text) + return unicodes + + def getComponentReferences( + self, glyphNames: Optional[Iterable[str]] = None + ) -> dict[str, list[str]]: + """ + Return a dictionary that maps glyph names to lists containing the + base glyph name of components in the glyph. This parses the .glif + files partially, so it is a lot faster than parsing all files completely. + By default this checks all glyphs, but a subset can be passed with glyphNames. + """ + components = {} + if glyphNames is None: + glyphNames = self.contents.keys() + for glyphName in glyphNames: + text = self.getGLIF(glyphName) + components[glyphName] = _fetchComponentBases(text) + return components + + def getImageReferences( + self, glyphNames: Optional[Iterable[str]] = None + ) -> dict[str, Optional[str]]: + """ + Return a dictionary that maps glyph names to the file name of the image + referenced by the glyph. This parses the .glif files partially, so it is a + lot faster than parsing all files completely. + By default this checks all glyphs, but a subset can be passed with glyphNames. + """ + images = {} + if glyphNames is None: + glyphNames = self.contents.keys() + for glyphName in glyphNames: + text = self.getGLIF(glyphName) + images[glyphName] = _fetchImageFileName(text) + return images + + def close(self) -> None: + if self._shouldClose: + self.fs.close() + + def __enter__(self) -> GlyphSet: + return self + + def __exit__(self, exc_type: Any, exc_value: Any, exc_tb: Any) -> None: + self.close() + + +# ----------------------- +# Glyph Name to File Name +# ----------------------- + + +def glyphNameToFileName(glyphName: str, existingFileNames: Optional[set[str]]) -> str: + """ + Wrapper around the userNameToFileName function in filenames.py + + Note that existingFileNames should be a set for large glyphsets + or performance will suffer. + """ + if existingFileNames is None: + existingFileNames = set() + return userNameToFileName(glyphName, existing=existingFileNames, suffix=".glif") + + +# ----------------------- +# GLIF To and From String +# ----------------------- + + +def readGlyphFromString( + aString: Union[str, bytes], + glyphObject: Optional[Any] = None, + pointPen: Optional[Any] = None, + formatVersions: FormatVersions = None, + validate: bool = True, +) -> None: + """ + Read .glif data from a string into a glyph object. + + The 'glyphObject' argument can be any kind of object (even None); + the readGlyphFromString() method will attempt to set the following + attributes on it: + + width + the advance width of the glyph + height + the advance height of the glyph + unicodes + a list of unicode values for this glyph + note + a string + lib + a dictionary containing custom data + image + a dictionary containing image data + guidelines + a list of guideline data dictionaries + anchors + a list of anchor data dictionaries + + All attributes are optional, in two ways: + + 1) An attribute *won't* be set if the .glif file doesn't + contain data for it. 'glyphObject' will have to deal + with default values itself. + 2) If setting the attribute fails with an AttributeError + (for example if the 'glyphObject' attribute is read- + only), readGlyphFromString() will not propagate that + exception, but ignore that attribute. + + To retrieve outline information, you need to pass an object + conforming to the PointPen protocol as the 'pointPen' argument. + This argument may be None if you don't need the outline data. + + The formatVersions optional argument define the GLIF format versions + that are allowed to be read. + The type is Optional[Iterable[tuple[int, int], int]]. It can contain + either integers (for the major versions to be allowed, with minor + digits defaulting to 0), or tuples of integers to specify both + (major, minor) versions. + By default when formatVersions is None all the GLIF format versions + currently defined are allowed to be read. + + ``validate`` will validate the read data. It is set to ``True`` by default. + """ + tree = _glifTreeFromString(aString) + + if formatVersions is None: + validFormatVersions: Set[GLIFFormatVersion] = ( + GLIFFormatVersion.supported_versions() + ) + else: + validFormatVersions, invalidFormatVersions = set(), set() + for v in formatVersions: + try: + formatVersion = normalizeFormatVersion(v, GLIFFormatVersion) + except ValueError: + invalidFormatVersions.add(v) + else: + validFormatVersions.add(formatVersion) + if not validFormatVersions: + raise ValueError( + "None of the requested GLIF formatVersions are supported: " + f"{formatVersions!r}" + ) + + _readGlyphFromTree( + tree, + glyphObject, + pointPen, + formatVersions=validFormatVersions, + validate=validate, + ) + + +def _writeGlyphToBytes( + glyphName: str, + glyphObject: Optional[Any] = None, + drawPointsFunc: Optional[Callable[[Any], None]] = None, + writer: Optional[Any] = None, + formatVersion: Optional[FormatVersion] = None, + validate: bool = True, +) -> bytes: + """Return .glif data for a glyph as a UTF-8 encoded bytes string.""" + try: + formatVersion = normalizeFormatVersion(formatVersion, GLIFFormatVersion) + except ValueError: + from fontTools.ufoLib.errors import UnsupportedGLIFFormat + + raise UnsupportedGLIFFormat( + "Unsupported GLIF format version: {formatVersion!r}" + ) + # start + if validate and not isinstance(glyphName, str): + raise GlifLibError("The glyph name is not properly formatted.") + if validate and len(glyphName) == 0: + raise GlifLibError("The glyph name is empty.") + glyphAttrs = OrderedDict( + [("name", glyphName), ("format", repr(formatVersion.major))] + ) + if formatVersion.minor != 0: + glyphAttrs["formatMinor"] = repr(formatVersion.minor) + root = etree.Element("glyph", glyphAttrs) + identifiers: set[str] = set() + # advance + _writeAdvance(glyphObject, root, validate) + # unicodes + if getattr(glyphObject, "unicodes", None): + _writeUnicodes(glyphObject, root, validate) + # note + if getattr(glyphObject, "note", None): + _writeNote(glyphObject, root, validate) + # image + if formatVersion.major >= 2 and getattr(glyphObject, "image", None): + _writeImage(glyphObject, root, validate) + # guidelines + if formatVersion.major >= 2 and getattr(glyphObject, "guidelines", None): + _writeGuidelines(glyphObject, root, identifiers, validate) + # anchors + anchors = getattr(glyphObject, "anchors", None) + if formatVersion.major >= 2 and anchors: + _writeAnchors(glyphObject, root, identifiers, validate) + # outline + if drawPointsFunc is not None: + outline = etree.SubElement(root, "outline") + pen = GLIFPointPen(outline, identifiers=identifiers, validate=validate) + drawPointsFunc(pen) + if formatVersion.major == 1 and anchors: + _writeAnchorsFormat1(pen, anchors, validate) + # prevent lxml from writing self-closing tags + if not len(outline): + outline.text = "\n " + # lib + if getattr(glyphObject, "lib", None): + _writeLib(glyphObject, root, validate) + # return the text + data = etree.tostring( + root, encoding="UTF-8", xml_declaration=True, pretty_print=True + ) + return data + + +def writeGlyphToString( + glyphName: str, + glyphObject: Optional[Any] = None, + drawPointsFunc: Optional[Callable[[Any], None]] = None, + formatVersion: Optional[FormatVersion] = None, + validate: bool = True, +) -> str: + """ + Return .glif data for a glyph as a string. The XML declaration's + encoding is always set to "UTF-8". + The 'glyphObject' argument can be any kind of object (even None); + the writeGlyphToString() method will attempt to get the following + attributes from it: + + width + the advance width of the glyph + height + the advance height of the glyph + unicodes + a list of unicode values for this glyph + note + a string + lib + a dictionary containing custom data + image + a dictionary containing image data + guidelines + a list of guideline data dictionaries + anchors + a list of anchor data dictionaries + + All attributes are optional: if 'glyphObject' doesn't + have the attribute, it will simply be skipped. + + To write outline data to the .glif file, writeGlyphToString() needs + a function (any callable object actually) that will take one + argument: an object that conforms to the PointPen protocol. + The function will be called by writeGlyphToString(); it has to call the + proper PointPen methods to transfer the outline to the .glif file. + + The GLIF format version can be specified with the formatVersion argument. + This accepts either a tuple of integers for (major, minor), or a single + integer for the major digit only (with minor digit implied as 0). + By default when formatVesion is None the latest GLIF format version will + be used; currently it's 2.0, which is equivalent to formatVersion=(2, 0). + + An UnsupportedGLIFFormat exception is raised if the requested UFO + formatVersion is not supported. + + ``validate`` will validate the written data. It is set to ``True`` by default. + """ + data = _writeGlyphToBytes( + glyphName, + glyphObject=glyphObject, + drawPointsFunc=drawPointsFunc, + formatVersion=formatVersion, + validate=validate, + ) + return data.decode("utf-8") + + +def _writeAdvance(glyphObject: Any, element: ElementType, validate: bool) -> None: + width = getattr(glyphObject, "width", None) + if width is not None: + if validate and not isinstance(width, numberTypes): + raise GlifLibError("width attribute must be int or float") + if width == 0: + width = None + height = getattr(glyphObject, "height", None) + if height is not None: + if validate and not isinstance(height, numberTypes): + raise GlifLibError("height attribute must be int or float") + if height == 0: + height = None + if width is not None and height is not None: + etree.SubElement( + element, + "advance", + OrderedDict([("height", repr(height)), ("width", repr(width))]), + ) + elif width is not None: + etree.SubElement(element, "advance", dict(width=repr(width))) + elif height is not None: + etree.SubElement(element, "advance", dict(height=repr(height))) + + +def _writeUnicodes(glyphObject: Any, element: ElementType, validate: bool) -> None: + unicodes = getattr(glyphObject, "unicodes", []) + if validate and isinstance(unicodes, int): + unicodes = [unicodes] + seen = set() + for code in unicodes: + if validate and not isinstance(code, int): + raise GlifLibError("unicode values must be int") + if code in seen: + continue + seen.add(code) + hexCode = "%04X" % code + etree.SubElement(element, "unicode", dict(hex=hexCode)) + + +def _writeNote(glyphObject: Any, element: ElementType, validate: bool) -> None: + note = getattr(glyphObject, "note", None) + if validate and not isinstance(note, str): + raise GlifLibError("note attribute must be str") + if isinstance(note, str): + note = note.strip() + note = "\n" + note + "\n" + etree.SubElement(element, "note").text = note + + +def _writeImage(glyphObject: Any, element: ElementType, validate: bool) -> None: + image = getattr(glyphObject, "image", None) + if image is None: + return + + if validate and not imageValidator(image): + raise GlifLibError( + "image attribute must be a dict or dict-like object with the proper structure." + ) + attrs = OrderedDict([("fileName", image["fileName"])]) + for attr, default in _transformationInfo: + value = image.get(attr, default) + if value != default: + attrs[attr] = repr(value) + color = image.get("color") + if color is not None: + attrs["color"] = color + etree.SubElement(element, "image", attrs) + + +def _writeGuidelines( + glyphObject: Any, element: ElementType, identifiers: set[str], validate: bool +) -> None: + guidelines = getattr(glyphObject, "guidelines", []) + if validate and not guidelinesValidator(guidelines): + raise GlifLibError("guidelines attribute does not have the proper structure.") + for guideline in guidelines: + attrs = OrderedDict() + x = guideline.get("x") + if x is not None: + attrs["x"] = repr(x) + y = guideline.get("y") + if y is not None: + attrs["y"] = repr(y) + angle = guideline.get("angle") + if angle is not None: + attrs["angle"] = repr(angle) + name = guideline.get("name") + if name is not None: + attrs["name"] = name + color = guideline.get("color") + if color is not None: + attrs["color"] = color + identifier = guideline.get("identifier") + if identifier is not None: + if validate and identifier in identifiers: + raise GlifLibError("identifier used more than once: %s" % identifier) + attrs["identifier"] = identifier + identifiers.add(identifier) + etree.SubElement(element, "guideline", attrs) + + +def _writeAnchorsFormat1(pen: Any, anchors: Any, validate: bool) -> None: + if validate and not anchorsValidator(anchors): + raise GlifLibError("anchors attribute does not have the proper structure.") + for anchor in anchors: + attrs = {} + x = anchor["x"] + attrs["x"] = repr(x) + y = anchor["y"] + attrs["y"] = repr(y) + name = anchor.get("name") + if name is not None: + attrs["name"] = name + pen.beginPath() + pen.addPoint((x, y), segmentType="move", name=name) + pen.endPath() + + +def _writeAnchors( + glyphObject: Any, + element: ElementType, + identifiers: set[str], + validate: bool, +) -> None: + anchors = getattr(glyphObject, "anchors", []) + if validate and not anchorsValidator(anchors): + raise GlifLibError("anchors attribute does not have the proper structure.") + for anchor in anchors: + attrs = OrderedDict() + x = anchor["x"] + attrs["x"] = repr(x) + y = anchor["y"] + attrs["y"] = repr(y) + name = anchor.get("name") + if name is not None: + attrs["name"] = name + color = anchor.get("color") + if color is not None: + attrs["color"] = color + identifier = anchor.get("identifier") + if identifier is not None: + if validate and identifier in identifiers: + raise GlifLibError("identifier used more than once: %s" % identifier) + attrs["identifier"] = identifier + identifiers.add(identifier) + etree.SubElement(element, "anchor", attrs) + + +def _writeLib(glyphObject: Any, element: ElementType, validate: bool) -> None: + lib = getattr(glyphObject, "lib", None) + if not lib: + # don't write empty lib + return + if validate: + valid, message = glyphLibValidator(lib) + if not valid: + raise GlifLibError(message) + if not isinstance(lib, dict): + lib = dict(lib) + # plist inside GLIF begins with 2 levels of indentation + e = plistlib.totree(lib, indent_level=2) + etree.SubElement(element, "lib").append(e) + + +# ----------------------- +# layerinfo.plist Support +# ----------------------- + +layerInfoVersion3ValueData = { + "color": dict(type=str, valueValidator=colorValidator), + "lib": dict(type=dict, valueValidator=genericTypeValidator), +} + + +def validateLayerInfoVersion3ValueForAttribute(attr: str, value: Any) -> bool: + """ + This performs very basic validation of the value for attribute + following the UFO 3 fontinfo.plist specification. The results + of this should not be interpretted as *correct* for the font + that they are part of. This merely indicates that the value + is of the proper type and, where the specification defines + a set range of possible values for an attribute, that the + value is in the accepted range. + """ + if attr not in layerInfoVersion3ValueData: + return False + dataValidationDict = layerInfoVersion3ValueData[attr] + valueType = dataValidationDict.get("type") + validator = dataValidationDict.get("valueValidator") + valueOptions = dataValidationDict.get("valueOptions") + # have specific options for the validator + assert callable(validator) + if valueOptions is not None: + isValidValue = validator(value, valueOptions) + # no specific options + else: + if validator == genericTypeValidator: + isValidValue = validator(value, valueType) + else: + isValidValue = validator(value) + return isValidValue + + +def validateLayerInfoVersion3Data(infoData: dict[str, Any]) -> dict[str, Any]: + """ + This performs very basic validation of the value for infoData + following the UFO 3 layerinfo.plist specification. The results + of this should not be interpretted as *correct* for the font + that they are part of. This merely indicates that the values + are of the proper type and, where the specification defines + a set range of possible values for an attribute, that the + value is in the accepted range. + """ + for attr, value in infoData.items(): + if attr not in layerInfoVersion3ValueData: + raise GlifLibError("Unknown attribute %s." % attr) + isValidValue = validateLayerInfoVersion3ValueForAttribute(attr, value) + if not isValidValue: + raise GlifLibError(f"Invalid value for attribute {attr} ({value!r}).") + return infoData + + +# ----------------- +# GLIF Tree Support +# ----------------- + + +def _glifTreeFromFile(aFile: Union[str, bytes, int]) -> ElementType: + if etree._have_lxml: + tree = etree.parse(aFile, parser=etree.XMLParser(remove_comments=True)) + else: + tree = etree.parse(aFile) + root = tree.getroot() + if root.tag != "glyph": + raise GlifLibError("The GLIF is not properly formatted.") + if root.text and root.text.strip() != "": + raise GlifLibError("Invalid GLIF structure.") + return root + + +def _glifTreeFromString(aString: Union[str, bytes]) -> ElementType: + data = tobytes(aString, encoding="utf-8") + try: + if etree._have_lxml: + root = etree.fromstring(data, parser=etree.XMLParser(remove_comments=True)) + else: + root = etree.fromstring(data) + except Exception as etree_exception: + raise GlifLibError("GLIF contains invalid XML.") from etree_exception + + if root.tag != "glyph": + raise GlifLibError("The GLIF is not properly formatted.") + if root.text and root.text.strip() != "": + raise GlifLibError("Invalid GLIF structure.") + return root + + +def _readGlyphFromTree( + tree: ElementType, + glyphObject: Optional[Any] = None, + pointPen: Optional[AbstractPointPen] = None, + formatVersions: Set[GLIFFormatVersion] = GLIFFormatVersion.supported_versions(), + validate: bool = True, +) -> None: + # check the format version + formatVersionMajor = tree.get("format") + if formatVersionMajor is None: + if validate: + raise GlifLibError("Unspecified format version in GLIF.") + formatVersionMajor = 0 + formatVersionMinor = tree.get("formatMinor", 0) + try: + formatVersion = GLIFFormatVersion( + (int(formatVersionMajor), int(formatVersionMinor)) + ) + except ValueError as e: + msg = "Unsupported GLIF format: %s.%s" % ( + formatVersionMajor, + formatVersionMinor, + ) + if validate: + from fontTools.ufoLib.errors import UnsupportedGLIFFormat + + raise UnsupportedGLIFFormat(msg) from e + # warn but continue using the latest supported format + formatVersion = GLIFFormatVersion.default() + logger.warning( + "%s. Assuming the latest supported version (%s). " + "Some data may be skipped or parsed incorrectly.", + msg, + formatVersion, + ) + + if validate and formatVersion not in formatVersions: + raise GlifLibError(f"Forbidden GLIF format version: {formatVersion!s}") + + try: + readGlyphFromTree = _READ_GLYPH_FROM_TREE_FUNCS[formatVersion] + except KeyError: + raise NotImplementedError(formatVersion) + + readGlyphFromTree( + tree=tree, + glyphObject=glyphObject, + pointPen=pointPen, + validate=validate, + formatMinor=formatVersion.minor, + ) + + +def _readGlyphFromTreeFormat1( + tree: ElementType, + glyphObject: Optional[Any] = None, + pointPen: Optional[AbstractPointPen] = None, + validate: bool = False, + **kwargs: Any, +) -> None: + # get the name + _readName(glyphObject, tree, validate) + # populate the sub elements + unicodes = [] + haveSeenAdvance = haveSeenOutline = haveSeenLib = haveSeenNote = False + for element in tree: + if element.tag == "outline": + if validate: + if haveSeenOutline: + raise GlifLibError("The outline element occurs more than once.") + if element.attrib: + raise GlifLibError( + "The outline element contains unknown attributes." + ) + if element.text and element.text.strip() != "": + raise GlifLibError("Invalid outline structure.") + haveSeenOutline = True + buildOutlineFormat1(glyphObject, pointPen, element, validate) + elif glyphObject is None: + # Skip remaining elements if no glyphObject, but outline is still + # processed above to allow drawing via pointPen without a glyphObject. + continue + elif element.tag == "advance": + if validate and haveSeenAdvance: + raise GlifLibError("The advance element occurs more than once.") + haveSeenAdvance = True + _readAdvance(glyphObject, element) + elif element.tag == "unicode": + v = element.get("hex") + if v is None: + raise GlifLibError( + "A unicode element is missing its required hex attribute." + ) + try: + v = int(v, 16) + if v not in unicodes: + unicodes.append(v) + except ValueError: + raise GlifLibError( + "Illegal value for hex attribute of unicode element." + ) + elif element.tag == "note": + if validate and haveSeenNote: + raise GlifLibError("The note element occurs more than once.") + haveSeenNote = True + _readNote(glyphObject, element) + elif element.tag == "lib": + if validate and haveSeenLib: + raise GlifLibError("The lib element occurs more than once.") + haveSeenLib = True + _readLib(glyphObject, element, validate) + else: + raise GlifLibError("Unknown element in GLIF: %s" % element) + # set the collected unicodes + if unicodes: + _relaxedSetattr(glyphObject, "unicodes", unicodes) + + +def _readGlyphFromTreeFormat2( + tree: ElementType, + glyphObject: Optional[Any] = None, + pointPen: Optional[AbstractPointPen] = None, + validate: bool = False, + formatMinor: int = 0, +) -> None: + # get the name + _readName(glyphObject, tree, validate) + # populate the sub elements + unicodes = [] + guidelines = [] + anchors = [] + haveSeenAdvance = haveSeenImage = haveSeenOutline = haveSeenLib = haveSeenNote = ( + False + ) + identifiers: set[str] = set() + for element in tree: + if element.tag == "outline": + if validate: + if haveSeenOutline: + raise GlifLibError("The outline element occurs more than once.") + if element.attrib: + raise GlifLibError( + "The outline element contains unknown attributes." + ) + if element.text and element.text.strip() != "": + raise GlifLibError("Invalid outline structure.") + haveSeenOutline = True + if pointPen is not None: + buildOutlineFormat2( + glyphObject, pointPen, element, identifiers, validate + ) + elif glyphObject is None: + # Skip remaining elements if no glyphObject, but outline is still + # processed above to allow drawing via pointPen without a glyphObject. + continue + elif element.tag == "advance": + if validate and haveSeenAdvance: + raise GlifLibError("The advance element occurs more than once.") + haveSeenAdvance = True + _readAdvance(glyphObject, element) + elif element.tag == "unicode": + v = element.get("hex") + if v is None: + raise GlifLibError( + "A unicode element is missing its required hex attribute." + ) + try: + v = int(v, 16) + if v not in unicodes: + unicodes.append(v) + except ValueError: + raise GlifLibError( + "Illegal value for hex attribute of unicode element." + ) + elif element.tag == "guideline": + if validate and len(element): + raise GlifLibError("Unknown children in guideline element.") + attrib = dict(element.attrib) + for attr in ("x", "y", "angle"): + if attr in attrib: + attrib[attr] = _number(attrib[attr]) + guidelines.append(attrib) + elif element.tag == "anchor": + if validate and len(element): + raise GlifLibError("Unknown children in anchor element.") + attrib = dict(element.attrib) + for attr in ("x", "y"): + if attr in element.attrib: + attrib[attr] = _number(attrib[attr]) + anchors.append(attrib) + elif element.tag == "image": + if validate: + if haveSeenImage: + raise GlifLibError("The image element occurs more than once.") + if len(element): + raise GlifLibError("Unknown children in image element.") + haveSeenImage = True + _readImage(glyphObject, element, validate) + elif element.tag == "note": + if validate and haveSeenNote: + raise GlifLibError("The note element occurs more than once.") + haveSeenNote = True + _readNote(glyphObject, element) + elif element.tag == "lib": + if validate and haveSeenLib: + raise GlifLibError("The lib element occurs more than once.") + haveSeenLib = True + _readLib(glyphObject, element, validate) + else: + raise GlifLibError("Unknown element in GLIF: %s" % element) + # set the collected unicodes + if unicodes: + _relaxedSetattr(glyphObject, "unicodes", unicodes) + # set the collected guidelines + if guidelines: + if validate and not guidelinesValidator(guidelines, identifiers): + raise GlifLibError("The guidelines are improperly formatted.") + _relaxedSetattr(glyphObject, "guidelines", guidelines) + # set the collected anchors + if anchors: + if validate and not anchorsValidator(anchors, identifiers): + raise GlifLibError("The anchors are improperly formatted.") + _relaxedSetattr(glyphObject, "anchors", anchors) + + +_READ_GLYPH_FROM_TREE_FUNCS: dict[GLIFFormatVersion, Callable[..., Any]] = { + GLIFFormatVersion.FORMAT_1_0: _readGlyphFromTreeFormat1, + GLIFFormatVersion.FORMAT_2_0: _readGlyphFromTreeFormat2, +} + + +def _readName(glyphObject: Optional[Any], root: ElementType, validate: bool) -> None: + glyphName = root.get("name") + if validate and not glyphName: + raise GlifLibError("Empty glyph name in GLIF.") + if glyphName and glyphObject is not None: + _relaxedSetattr(glyphObject, "name", glyphName) + + +def _readAdvance(glyphObject: Optional[Any], advance: ElementType) -> None: + width = _number(advance.get("width", 0)) + _relaxedSetattr(glyphObject, "width", width) + height = _number(advance.get("height", 0)) + _relaxedSetattr(glyphObject, "height", height) + + +def _readNote(glyphObject: Optional[Any], note: ElementType) -> None: + if note.text is None: + return + lines = note.text.split("\n") + note = "\n".join(line.strip() for line in lines if line.strip()) + _relaxedSetattr(glyphObject, "note", note) + + +def _readLib(glyphObject: Optional[Any], lib: ElementType, validate: bool) -> None: + assert len(lib) == 1 + child = lib[0] + plist = plistlib.fromtree(child) + if validate: + valid, message = glyphLibValidator(plist) + if not valid: + raise GlifLibError(message) + _relaxedSetattr(glyphObject, "lib", plist) + + +def _readImage(glyphObject: Optional[Any], image: ElementType, validate: bool) -> None: + imageData = dict(image.attrib) + for attr, default in _transformationInfo: + value = imageData.get(attr, default) + imageData[attr] = _number(value) + if validate and not imageValidator(imageData): + raise GlifLibError("The image element is not properly formatted.") + _relaxedSetattr(glyphObject, "image", imageData) + + +# ---------------- +# GLIF to PointPen +# ---------------- + +contourAttributesFormat2: set[str] = {"identifier"} +componentAttributesFormat1: set[str] = { + "base", + "xScale", + "xyScale", + "yxScale", + "yScale", + "xOffset", + "yOffset", +} +componentAttributesFormat2: set[str] = componentAttributesFormat1 | {"identifier"} +pointAttributesFormat1: set[str] = {"x", "y", "type", "smooth", "name"} +pointAttributesFormat2: set[str] = pointAttributesFormat1 | {"identifier"} +pointSmoothOptions: set[str] = {"no", "yes"} +pointTypeOptions: set[str] = {"move", "line", "offcurve", "curve", "qcurve"} + +# format 1 + + +def buildOutlineFormat1( + glyphObject: Any, + pen: Optional[AbstractPointPen], + outline: Iterable[ElementType], + validate: bool, +) -> None: + anchors = [] + for element in outline: + if element.tag == "contour": + if len(element) == 1: + point = element[0] + if point.tag == "point": + anchor = _buildAnchorFormat1(point, validate) + if anchor is not None: + anchors.append(anchor) + continue + if pen is not None: + _buildOutlineContourFormat1(pen, element, validate) + elif element.tag == "component": + if pen is not None: + _buildOutlineComponentFormat1(pen, element, validate) + else: + raise GlifLibError("Unknown element in outline element: %s" % element) + if glyphObject is not None and anchors: + if validate and not anchorsValidator(anchors): + raise GlifLibError("GLIF 1 anchors are not properly formatted.") + _relaxedSetattr(glyphObject, "anchors", anchors) + + +def _buildAnchorFormat1(point: ElementType, validate: bool) -> Optional[dict[str, Any]]: + if point.get("type") != "move": + return None + name = point.get("name") + if name is None: + return None + x = point.get("x") + y = point.get("y") + if validate and x is None: + raise GlifLibError("Required x attribute is missing in point element.") + assert x is not None + if validate and y is None: + raise GlifLibError("Required y attribute is missing in point element.") + assert y is not None + x = _number(x) + y = _number(y) + anchor = dict(x=x, y=y, name=name) + return anchor + + +def _buildOutlineContourFormat1( + pen: AbstractPointPen, contour: ElementType, validate: bool +) -> None: + if validate and contour.attrib: + raise GlifLibError("Unknown attributes in contour element.") + pen.beginPath() + if len(contour): + massaged = _validateAndMassagePointStructures( + contour, + pointAttributesFormat1, + openContourOffCurveLeniency=True, + validate=validate, + ) + _buildOutlinePointsFormat1(pen, massaged) + pen.endPath() + + +def _buildOutlinePointsFormat1( + pen: AbstractPointPen, contour: list[dict[str, Any]] +) -> None: + for point in contour: + x = point["x"] + y = point["y"] + segmentType = point["segmentType"] + smooth = point["smooth"] + name = point["name"] + pen.addPoint((x, y), segmentType=segmentType, smooth=smooth, name=name) + + +def _buildOutlineComponentFormat1( + pen: AbstractPointPen, component: ElementType, validate: bool +) -> None: + if validate: + if len(component): + raise GlifLibError("Unknown child elements of component element.") + for attr in component.attrib.keys(): + if attr not in componentAttributesFormat1: + raise GlifLibError("Unknown attribute in component element: %s" % attr) + baseGlyphName = component.get("base") + if validate and baseGlyphName is None: + raise GlifLibError("The base attribute is not defined in the component.") + assert baseGlyphName is not None + transformation = tuple( + _number(component.get(attr) or default) for attr, default in _transformationInfo + ) + transformation = cast( + tuple[float, float, float, float, float, float], transformation + ) + pen.addComponent(baseGlyphName, transformation) + + +# format 2 + + +def buildOutlineFormat2( + glyphObject: Any, + pen: AbstractPointPen, + outline: Iterable[ElementType], + identifiers: set[str], + validate: bool, +) -> None: + for element in outline: + if element.tag == "contour": + _buildOutlineContourFormat2(pen, element, identifiers, validate) + elif element.tag == "component": + _buildOutlineComponentFormat2(pen, element, identifiers, validate) + else: + raise GlifLibError("Unknown element in outline element: %s" % element.tag) + + +def _buildOutlineContourFormat2( + pen: AbstractPointPen, contour: ElementType, identifiers: set[str], validate: bool +) -> None: + if validate: + for attr in contour.attrib.keys(): + if attr not in contourAttributesFormat2: + raise GlifLibError("Unknown attribute in contour element: %s" % attr) + identifier = contour.get("identifier") + if identifier is not None: + if validate: + if identifier in identifiers: + raise GlifLibError( + "The identifier %s is used more than once." % identifier + ) + if not identifierValidator(identifier): + raise GlifLibError( + "The contour identifier %s is not valid." % identifier + ) + identifiers.add(identifier) + try: + pen.beginPath(identifier=identifier) + except TypeError: + pen.beginPath() + warn( + "The beginPath method needs an identifier kwarg. The contour's identifier value has been discarded.", + DeprecationWarning, + ) + if len(contour): + massaged = _validateAndMassagePointStructures( + contour, pointAttributesFormat2, validate=validate + ) + _buildOutlinePointsFormat2(pen, massaged, identifiers, validate) + pen.endPath() + + +def _buildOutlinePointsFormat2( + pen: AbstractPointPen, + contour: list[dict[str, Any]], + identifiers: set[str], + validate: bool, +) -> None: + for point in contour: + x = point["x"] + y = point["y"] + segmentType = point["segmentType"] + smooth = point["smooth"] + name = point["name"] + identifier = point.get("identifier") + if identifier is not None: + if validate: + if identifier in identifiers: + raise GlifLibError( + "The identifier %s is used more than once." % identifier + ) + if not identifierValidator(identifier): + raise GlifLibError("The identifier %s is not valid." % identifier) + identifiers.add(identifier) + try: + pen.addPoint( + (x, y), + segmentType=segmentType, + smooth=smooth, + name=name, + identifier=identifier, + ) + except TypeError: + pen.addPoint((x, y), segmentType=segmentType, smooth=smooth, name=name) + warn( + "The addPoint method needs an identifier kwarg. The point's identifier value has been discarded.", + DeprecationWarning, + ) + + +def _buildOutlineComponentFormat2( + pen: AbstractPointPen, component: ElementType, identifiers: set[str], validate: bool +) -> None: + if validate: + if len(component): + raise GlifLibError("Unknown child elements of component element.") + for attr in component.attrib.keys(): + if attr not in componentAttributesFormat2: + raise GlifLibError("Unknown attribute in component element: %s" % attr) + baseGlyphName = component.get("base") + if validate and baseGlyphName is None: + raise GlifLibError("The base attribute is not defined in the component.") + assert baseGlyphName is not None + transformation = tuple( + _number(component.get(attr) or default) for attr, default in _transformationInfo + ) + transformation = cast( + tuple[float, float, float, float, float, float], transformation + ) + identifier = component.get("identifier") + if identifier is not None: + if validate: + if identifier in identifiers: + raise GlifLibError( + "The identifier %s is used more than once." % identifier + ) + if validate and not identifierValidator(identifier): + raise GlifLibError("The identifier %s is not valid." % identifier) + identifiers.add(identifier) + try: + pen.addComponent(baseGlyphName, transformation, identifier=identifier) + except TypeError: + pen.addComponent(baseGlyphName, transformation) + warn( + "The addComponent method needs an identifier kwarg. The component's identifier value has been discarded.", + DeprecationWarning, + ) + + +# all formats + + +def _validateAndMassagePointStructures( + contour, pointAttributes, openContourOffCurveLeniency=False, validate=True +): + if not len(contour): + return + # store some data for later validation + lastOnCurvePoint = None + haveOffCurvePoint = False + # validate and massage the individual point elements + massaged = [] + for index, element in enumerate(contour): + # not <point> + if element.tag != "point": + raise GlifLibError( + "Unknown child element (%s) of contour element." % element.tag + ) + point = dict(element.attrib) + massaged.append(point) + if validate: + # unknown attributes + for attr in point.keys(): + if attr not in pointAttributes: + raise GlifLibError("Unknown attribute in point element: %s" % attr) + # search for unknown children + if len(element): + raise GlifLibError("Unknown child elements in point element.") + # x and y are required + for attr in ("x", "y"): + try: + point[attr] = _number(point[attr]) + except KeyError as e: + raise GlifLibError( + f"Required {attr} attribute is missing in point element." + ) from e + # segment type + pointType = point.pop("type", "offcurve") + if validate and pointType not in pointTypeOptions: + raise GlifLibError("Unknown point type: %s" % pointType) + if pointType == "offcurve": + pointType = None + point["segmentType"] = pointType + if pointType is None: + haveOffCurvePoint = True + else: + lastOnCurvePoint = index + # move can only occur as the first point + if validate and pointType == "move" and index != 0: + raise GlifLibError( + "A move point occurs after the first point in the contour." + ) + # smooth is optional + smooth = point.get("smooth", "no") + if validate and smooth is not None: + if smooth not in pointSmoothOptions: + raise GlifLibError("Unknown point smooth value: %s" % smooth) + smooth = smooth == "yes" + point["smooth"] = smooth + # smooth can only be applied to curve and qcurve + if validate and smooth and pointType is None: + raise GlifLibError("smooth attribute set in an offcurve point.") + # name is optional + if "name" not in element.attrib: + point["name"] = None + if openContourOffCurveLeniency: + # remove offcurves that precede a move. this is technically illegal, + # but we let it slide because there are fonts out there in the wild like this. + if massaged[0]["segmentType"] == "move": + count = 0 + for point in reversed(massaged): + if point["segmentType"] is None: + count += 1 + else: + break + if count: + massaged = massaged[:-count] + # validate the off-curves in the segments + if validate and haveOffCurvePoint and lastOnCurvePoint is not None: + # we only care about how many offCurves there are before an onCurve + # filter out the trailing offCurves + offCurvesCount = len(massaged) - 1 - lastOnCurvePoint + for point in massaged: + segmentType = point["segmentType"] + if segmentType is None: + offCurvesCount += 1 + else: + if offCurvesCount: + # move and line can't be preceded by off-curves + if segmentType == "move": + # this will have been filtered out already + raise GlifLibError("move can not have an offcurve.") + elif segmentType == "line": + raise GlifLibError("line can not have an offcurve.") + elif segmentType == "curve": + if offCurvesCount > 2: + raise GlifLibError("Too many offcurves defined for curve.") + elif segmentType == "qcurve": + pass + else: + # unknown segment type. it'll be caught later. + pass + offCurvesCount = 0 + return massaged + + +# --------------------- +# Misc Helper Functions +# --------------------- + + +def _relaxedSetattr(object: Any, attr: str, value: Any) -> None: + try: + setattr(object, attr, value) + except AttributeError: + pass + + +def _number(s: Union[str, int, float]) -> IntFloat: + """ + Given a numeric string, return an integer or a float, whichever + the string indicates. _number("1") will return the integer 1, + _number("1.0") will return the float 1.0. + + >>> _number("1") + 1 + >>> _number("1.0") + 1.0 + >>> _number("a") # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + GlifLibError: Could not convert a to an int or float. + """ + try: + n: IntFloat = int(s) + return n + except ValueError: + pass + try: + n = float(s) + return n + except ValueError: + raise GlifLibError("Could not convert %s to an int or float." % s) + + +# -------------------- +# Rapid Value Fetching +# -------------------- + +# base + + +class _DoneParsing(Exception): + pass + + +class _BaseParser: + def __init__(self) -> None: + self._elementStack: list[str] = [] + + def parse(self, text: bytes): + from xml.parsers.expat import ParserCreate + + parser = ParserCreate() + parser.StartElementHandler = self.startElementHandler + parser.EndElementHandler = self.endElementHandler + parser.Parse(text, True) + + def startElementHandler(self, name: str, attrs: Any) -> None: + self._elementStack.append(name) + + def endElementHandler(self, name: str) -> None: + other = self._elementStack.pop(-1) + assert other == name + + +# unicodes + + +def _fetchUnicodes(glif: bytes) -> list[int]: + """ + Get a list of unicodes listed in glif. + """ + parser = _FetchUnicodesParser() + parser.parse(glif) + return parser.unicodes + + +class _FetchUnicodesParser(_BaseParser): + def __init__(self) -> None: + self.unicodes: list[int] = [] + super().__init__() + + def startElementHandler(self, name: str, attrs: dict[str, str]) -> None: + if ( + name == "unicode" + and self._elementStack + and self._elementStack[-1] == "glyph" + ): + value = attrs.get("hex") + if value is not None: + try: + intValue = int(value, 16) + if intValue not in self.unicodes: + self.unicodes.append(intValue) + except ValueError: + pass + super().startElementHandler(name, attrs) + + +# image + + +def _fetchImageFileName(glif: bytes) -> Optional[str]: + """ + The image file name (if any) from glif. + """ + parser = _FetchImageFileNameParser() + try: + parser.parse(glif) + except _DoneParsing: + pass + return parser.fileName + + +class _FetchImageFileNameParser(_BaseParser): + def __init__(self) -> None: + self.fileName: Optional[str] = None + super().__init__() + + def startElementHandler(self, name: str, attrs: dict[str, str]) -> None: + if name == "image" and self._elementStack and self._elementStack[-1] == "glyph": + self.fileName = attrs.get("fileName") + raise _DoneParsing + super().startElementHandler(name, attrs) + + +# component references + + +def _fetchComponentBases(glif: bytes) -> list[str]: + """ + Get a list of component base glyphs listed in glif. + """ + parser = _FetchComponentBasesParser() + try: + parser.parse(glif) + except _DoneParsing: + pass + return list(parser.bases) + + +class _FetchComponentBasesParser(_BaseParser): + def __init__(self) -> None: + self.bases: list[str] = [] + super().__init__() + + def startElementHandler(self, name: str, attrs: dict[str, str]) -> None: + if ( + name == "component" + and self._elementStack + and self._elementStack[-1] == "outline" + ): + base = attrs.get("base") + if base is not None: + self.bases.append(base) + super().startElementHandler(name, attrs) + + def endElementHandler(self, name: str) -> None: + if name == "outline": + raise _DoneParsing + super().endElementHandler(name) + + +# -------------- +# GLIF Point Pen +# -------------- + +_transformationInfo: list[tuple[str, int]] = [ + # field name, default value + ("xScale", 1), + ("xyScale", 0), + ("yxScale", 0), + ("yScale", 1), + ("xOffset", 0), + ("yOffset", 0), +] + + +class GLIFPointPen(AbstractPointPen): + """ + Helper class using the PointPen protocol to write the <outline> + part of .glif files. + """ + + def __init__( + self, + element: ElementType, + formatVersion: Optional[FormatVersion] = None, + identifiers: Optional[set[str]] = None, + validate: bool = True, + ) -> None: + if identifiers is None: + identifiers = set() + self.formatVersion = normalizeFormatVersion(formatVersion, GLIFFormatVersion) + self.identifiers = identifiers + self.outline = element + self.contour = None + self.prevOffCurveCount = 0 + self.prevPointTypes: list[str] = [] + self.validate = validate + + def beginPath(self, identifier=None, **kwargs): + attrs = OrderedDict() + if identifier is not None and self.formatVersion.major >= 2: + if self.validate: + if identifier in self.identifiers: + raise GlifLibError( + "identifier used more than once: %s" % identifier + ) + if not identifierValidator(identifier): + raise GlifLibError( + "identifier not formatted properly: %s" % identifier + ) + attrs["identifier"] = identifier + self.identifiers.add(identifier) + self.contour = etree.SubElement(self.outline, "contour", attrs) + self.prevOffCurveCount = 0 + + def endPath(self): + if self.prevPointTypes and self.prevPointTypes[0] == "move": + if self.validate and self.prevPointTypes[-1] == "offcurve": + raise GlifLibError("open contour has loose offcurve point") + # prevent lxml from writing self-closing tags + if not len(self.contour): + self.contour.text = "\n " + self.contour = None + self.prevPointType = None + self.prevOffCurveCount = 0 + self.prevPointTypes = [] + + def addPoint( + self, pt, segmentType=None, smooth=None, name=None, identifier=None, **kwargs + ): + attrs = OrderedDict() + # coordinates + if pt is not None: + if self.validate: + for coord in pt: + if not isinstance(coord, numberTypes): + raise GlifLibError("coordinates must be int or float") + attrs["x"] = repr(pt[0]) + attrs["y"] = repr(pt[1]) + # segment type + if segmentType == "offcurve": + segmentType = None + if self.validate: + if segmentType == "move" and self.prevPointTypes: + raise GlifLibError( + "move occurs after a point has already been added to the contour." + ) + if ( + segmentType in ("move", "line") + and self.prevPointTypes + and self.prevPointTypes[-1] == "offcurve" + ): + raise GlifLibError("offcurve occurs before %s point." % segmentType) + if segmentType == "curve" and self.prevOffCurveCount > 2: + raise GlifLibError("too many offcurve points before curve point.") + if segmentType is not None: + attrs["type"] = segmentType + else: + segmentType = "offcurve" + if segmentType == "offcurve": + self.prevOffCurveCount += 1 + else: + self.prevOffCurveCount = 0 + self.prevPointTypes.append(segmentType) + # smooth + if smooth: + if self.validate and segmentType == "offcurve": + raise GlifLibError("can't set smooth in an offcurve point.") + attrs["smooth"] = "yes" + # name + if name is not None: + attrs["name"] = name + # identifier + if identifier is not None and self.formatVersion.major >= 2: + if self.validate: + if identifier in self.identifiers: + raise GlifLibError( + "identifier used more than once: %s" % identifier + ) + if not identifierValidator(identifier): + raise GlifLibError( + "identifier not formatted properly: %s" % identifier + ) + attrs["identifier"] = identifier + self.identifiers.add(identifier) + etree.SubElement(self.contour, "point", attrs) + + def addComponent(self, glyphName, transformation, identifier=None, **kwargs): + attrs = OrderedDict([("base", glyphName)]) + for (attr, default), value in zip(_transformationInfo, transformation): + if self.validate and not isinstance(value, numberTypes): + raise GlifLibError("transformation values must be int or float") + if value != default: + attrs[attr] = repr(value) + if identifier is not None and self.formatVersion.major >= 2: + if self.validate: + if identifier in self.identifiers: + raise GlifLibError( + "identifier used more than once: %s" % identifier + ) + if self.validate and not identifierValidator(identifier): + raise GlifLibError( + "identifier not formatted properly: %s" % identifier + ) + attrs["identifier"] = identifier + self.identifiers.add(identifier) + etree.SubElement(self.outline, "component", attrs) + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/kerning.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/kerning.py new file mode 100644 index 0000000000000000000000000000000000000000..01ae55c062cad4ef8e6f7c0a400c51f17cb29d58 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/kerning.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +from typing import Optional + +from fontTools.annotations import KerningPair, KerningDict, KerningGroups, IntFloat + +StrDict = dict[str, str] + + +def lookupKerningValue( + pair: KerningPair, + kerning: KerningDict, + groups: KerningGroups, + fallback: IntFloat = 0, + glyphToFirstGroup: Optional[StrDict] = None, + glyphToSecondGroup: Optional[StrDict] = None, +) -> IntFloat: + """Retrieve the kerning value (if any) between a pair of elements. + + The elments can be either individual glyphs (by name) or kerning + groups (by name), or any combination of the two. + + Args: + pair: + A tuple, in logical order (first, second) with respect + to the reading direction, to query the font for kerning + information on. Each element in the tuple can be either + a glyph name or a kerning group name. + kerning: + A dictionary of kerning pairs. + groups: + A set of kerning groups. + fallback: + The fallback value to return if no kern is found between + the elements in ``pair``. Defaults to 0. + glyphToFirstGroup: + A dictionary mapping glyph names to the first-glyph kerning + groups to which they belong. Defaults to ``None``. + glyphToSecondGroup: + A dictionary mapping glyph names to the second-glyph kerning + groups to which they belong. Defaults to ``None``. + + Returns: + The kerning value between the element pair. If no kerning for + the pair is found, the fallback value is returned. + + Note: This function expects the ``kerning`` argument to be a flat + dictionary of kerning pairs, not the nested structure used in a + kerning.plist file. + + Examples:: + + >>> groups = { + ... "public.kern1.O" : ["O", "D", "Q"], + ... "public.kern2.E" : ["E", "F"] + ... } + >>> kerning = { + ... ("public.kern1.O", "public.kern2.E") : -100, + ... ("public.kern1.O", "F") : -200, + ... ("D", "F") : -300 + ... } + >>> lookupKerningValue(("D", "F"), kerning, groups) + -300 + >>> lookupKerningValue(("O", "F"), kerning, groups) + -200 + >>> lookupKerningValue(("O", "E"), kerning, groups) + -100 + >>> lookupKerningValue(("O", "O"), kerning, groups) + 0 + >>> lookupKerningValue(("E", "E"), kerning, groups) + 0 + >>> lookupKerningValue(("E", "O"), kerning, groups) + 0 + >>> lookupKerningValue(("X", "X"), kerning, groups) + 0 + >>> lookupKerningValue(("public.kern1.O", "public.kern2.E"), + ... kerning, groups) + -100 + >>> lookupKerningValue(("public.kern1.O", "F"), kerning, groups) + -200 + >>> lookupKerningValue(("O", "public.kern2.E"), kerning, groups) + -100 + >>> lookupKerningValue(("public.kern1.X", "public.kern2.X"), kerning, groups) + 0 + """ + # quickly check to see if the pair is in the kerning dictionary + if pair in kerning: + return kerning[pair] + # ensure both or no glyph-to-group mappings are provided + if (glyphToFirstGroup is None) != (glyphToSecondGroup is None): + raise ValueError( + "Must provide both 'glyphToFirstGroup' and 'glyphToSecondGroup', or neither." + ) + # create glyph to group mapping + if glyphToFirstGroup is None: + glyphToFirstGroup = {} + glyphToSecondGroup = {} + for group, groupMembers in groups.items(): + if group.startswith("public.kern1."): + for glyph in groupMembers: + glyphToFirstGroup[glyph] = group + elif group.startswith("public.kern2."): + for glyph in groupMembers: + glyphToSecondGroup[glyph] = group + # ensure type safety for mappings + assert glyphToFirstGroup is not None + assert glyphToSecondGroup is not None + # get group names and make sure first and second are glyph names + first, second = pair + firstGroup = secondGroup = None + if first.startswith("public.kern1."): + firstGroup = first + firstGlyph = None + else: + firstGroup = glyphToFirstGroup.get(first) + firstGlyph = first + if second.startswith("public.kern2."): + secondGroup = second + secondGlyph = None + else: + secondGroup = glyphToSecondGroup.get(second) + secondGlyph = second + # make an ordered list of pairs to look up + pairs = [ + (a, b) + for a in (firstGlyph, firstGroup) + for b in (secondGlyph, secondGroup) + if a is not None and b is not None + ] + # look up the pairs and return any matches + for pair in pairs: + if pair in kerning: + return kerning[pair] + # use the fallback value + return fallback + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/plistlib.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/plistlib.py new file mode 100644 index 0000000000000000000000000000000000000000..0242e9d26fce6b5ec4def58ac6aee44a5738db98 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/plistlib.py @@ -0,0 +1,47 @@ +"""DEPRECATED - This module is kept here only as a backward compatibility shim +for the old `ufoLib.plistlib` module, which was moved to :class:`fontTools.misc.plistlib`. +Please use the latter instead. +""" + +from fontTools.misc.plistlib import dump, dumps, load, loads +from fontTools.misc.textTools import tobytes + +# The following functions were part of the old py2-like ufoLib.plistlib API. +# They are kept only for backward compatiblity. +from fontTools.ufoLib.utils import deprecated + + +@deprecated("Use 'fontTools.misc.plistlib.load' instead") +def readPlist(path_or_file): + did_open = False + if isinstance(path_or_file, str): + path_or_file = open(path_or_file, "rb") + did_open = True + try: + return load(path_or_file, use_builtin_types=False) + finally: + if did_open: + path_or_file.close() + + +@deprecated("Use 'fontTools.misc.plistlib.dump' instead") +def writePlist(value, path_or_file): + did_open = False + if isinstance(path_or_file, str): + path_or_file = open(path_or_file, "wb") + did_open = True + try: + dump(value, path_or_file, use_builtin_types=False) + finally: + if did_open: + path_or_file.close() + + +@deprecated("Use 'fontTools.misc.plistlib.loads' instead") +def readPlistFromString(data): + return loads(tobytes(data, encoding="utf-8"), use_builtin_types=False) + + +@deprecated("Use 'fontTools.misc.plistlib.dumps' instead") +def writePlistToString(value): + return dumps(value, use_builtin_types=False) diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/pointPen.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/pointPen.py new file mode 100644 index 0000000000000000000000000000000000000000..4a8126cd6481a39b5bfeec7348c8efd168118236 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/pointPen.py @@ -0,0 +1,6 @@ +"""DEPRECATED - This module is kept here only as a backward compatibility shim +for the old `ufoLib.pointPen` module, which was moved to :class:`fontTools.pens.pointPen`. +Please use the latter instead. +""" + +from fontTools.pens.pointPen import * diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/utils.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3930d8931da14db0815d885028304d3905c0d680 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/utils.py @@ -0,0 +1,107 @@ +"""This module contains miscellaneous helpers. + +It is not considered part of the public ufoLib API. It does, however, +define the :py:obj:`.deprecated` decorator that is used elsewhere in +the module. +""" + +from __future__ import annotations + +from typing import Optional, Type, TypeVar, Union, cast +from collections.abc import Callable +import enum +import functools +import warnings + +F = TypeVar("F", bound=Callable[..., object]) +FormatVersion = TypeVar("FormatVersion", bound="BaseFormatVersion") +FormatVersionInput = Optional[Union[int, tuple[int, int], FormatVersion]] + +numberTypes = (int, float) + + +def deprecated(msg: str = "") -> Callable[[F], F]: + """Decorator factory to mark functions as deprecated with given message. + + >>> @deprecated("Enough!") + ... def some_function(): + ... "I just print 'hello world'." + ... print("hello world") + >>> some_function() + hello world + >>> some_function.__doc__ == "I just print 'hello world'." + True + """ + + def deprecated_decorator(func: F) -> F: + @functools.wraps(func) + def wrapper(*args, **kwargs): + warnings.warn( + f"{func.__name__} function is a deprecated. {msg}", + category=DeprecationWarning, + stacklevel=2, + ) + return func(*args, **kwargs) + + return cast(F, wrapper) + + return deprecated_decorator + + +def normalizeFormatVersion( + value: FormatVersionInput, cls: Type[FormatVersion] +) -> FormatVersion: + # Needed for type safety of UFOFormatVersion and GLIFFormatVersion input + if value is None: + return cls.default() + if isinstance(value, cls): + return value + if isinstance(value, int): + return cls((value, 0)) + if isinstance(value, tuple) and len(value) == 2: + return cls(value) + raise ValueError(f"Unsupported format version: {value!r}") + + +# Base class for UFOFormatVersion and GLIFFormatVersion +class BaseFormatVersion(tuple[int, int], enum.Enum): + value: tuple[int, int] + + def __new__(cls: Type[FormatVersion], value: tuple[int, int]) -> BaseFormatVersion: + return super().__new__(cls, value) + + @property + def major(self) -> int: + return self.value[0] + + @property + def minor(self) -> int: + return self.value[1] + + @classmethod + def _missing_(cls, value: object) -> BaseFormatVersion: + # allow to initialize a version enum from a single (major) integer + if isinstance(value, int): + return cls((value, 0)) + # or from None to obtain the current default version + if value is None: + return cls.default() + raise ValueError(f"{value!r} is not a valid {cls.__name__}") + + def __str__(self) -> str: + return f"{self.major}.{self.minor}" + + @classmethod + def default(cls: Type[FormatVersion]) -> FormatVersion: + # get the latest defined version (i.e. the max of all versions) + return max(cls.__members__.values()) + + @classmethod + def supported_versions(cls: Type[FormatVersion]) -> frozenset[FormatVersion]: + return frozenset(cls.__members__.values()) + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/.venv/lib/python3.11/site-packages/fontTools/ufoLib/validators.py b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/validators.py new file mode 100644 index 0000000000000000000000000000000000000000..baed5688d040eaa96382ffb0c0a98cdf930ff6a7 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/ufoLib/validators.py @@ -0,0 +1,1197 @@ +"""Various low level data validators.""" + +from __future__ import annotations + +import calendar +from collections.abc import Mapping, Sequence +from io import open + +import fontTools.misc.filesystem as fs +from typing import Any, Type, Optional, Union + +from fontTools.annotations import IntFloat +from fontTools.ufoLib.utils import numberTypes + +GenericDict = dict[str, tuple[Union[type, tuple[Type[Any], ...]], bool]] + +# ------- +# Generic +# ------- + + +def isDictEnough(value: Any) -> bool: + """ + Some objects will likely come in that aren't + dicts but are dict-ish enough. + """ + if isinstance(value, Mapping): + return True + for attr in ("keys", "values", "items"): + if not hasattr(value, attr): + return False + return True + + +def genericTypeValidator(value: Any, typ: Type[Any]) -> bool: + """ + Generic. (Added at version 2.) + """ + return isinstance(value, typ) + + +def genericIntListValidator(values: Any, validValues: Sequence[int]) -> bool: + """ + Generic. (Added at version 2.) + """ + if not isinstance(values, (list, tuple)): + return False + valuesSet = set(values) + validValuesSet = set(validValues) + if valuesSet - validValuesSet: + return False + for value in values: + if not isinstance(value, int): + return False + return True + + +def genericNonNegativeIntValidator(value: Any) -> bool: + """ + Generic. (Added at version 3.) + """ + if not isinstance(value, int): + return False + if value < 0: + return False + return True + + +def genericNonNegativeNumberValidator(value: Any) -> bool: + """ + Generic. (Added at version 3.) + """ + if not isinstance(value, numberTypes): + return False + if value < 0: + return False + return True + + +def genericDictValidator(value: Any, prototype: GenericDict) -> bool: + """ + Generic. (Added at version 3.) + """ + # not a dict + if not isinstance(value, Mapping): + return False + # missing required keys + for key, (typ, required) in prototype.items(): + if not required: + continue + if key not in value: + return False + # unknown keys + for key in value.keys(): + if key not in prototype: + return False + # incorrect types + for key, v in value.items(): + prototypeType, required = prototype[key] + if v is None and not required: + continue + if not isinstance(v, prototypeType): + return False + return True + + +# -------------- +# fontinfo.plist +# -------------- + +# Data Validators + + +def fontInfoStyleMapStyleNameValidator(value: Any) -> bool: + """ + Version 2+. + """ + options = ["regular", "italic", "bold", "bold italic"] + return value in options + + +def fontInfoOpenTypeGaspRangeRecordsValidator(value: Any) -> bool: + """ + Version 3+. + """ + if not isinstance(value, list): + return False + if len(value) == 0: + return True + validBehaviors = [0, 1, 2, 3] + dictPrototype: GenericDict = dict( + rangeMaxPPEM=(int, True), rangeGaspBehavior=(list, True) + ) + ppemOrder = [] + for rangeRecord in value: + if not genericDictValidator(rangeRecord, dictPrototype): + return False + ppem = rangeRecord["rangeMaxPPEM"] + behavior = rangeRecord["rangeGaspBehavior"] + ppemValidity = genericNonNegativeIntValidator(ppem) + if not ppemValidity: + return False + behaviorValidity = genericIntListValidator(behavior, validBehaviors) + if not behaviorValidity: + return False + ppemOrder.append(ppem) + if ppemOrder != sorted(ppemOrder): + return False + return True + + +def fontInfoOpenTypeHeadCreatedValidator(value: Any) -> bool: + """ + Version 2+. + """ + # format: 0000/00/00 00:00:00 + if not isinstance(value, str): + return False + # basic formatting + if not len(value) == 19: + return False + if value.count(" ") != 1: + return False + strDate, strTime = value.split(" ") + if strDate.count("/") != 2: + return False + if strTime.count(":") != 2: + return False + # date + strYear, strMonth, strDay = strDate.split("/") + if len(strYear) != 4: + return False + if len(strMonth) != 2: + return False + if len(strDay) != 2: + return False + try: + intYear = int(strYear) + intMonth = int(strMonth) + intDay = int(strDay) + except ValueError: + return False + if intMonth < 1 or intMonth > 12: + return False + monthMaxDay = calendar.monthrange(intYear, intMonth)[1] + if intDay < 1 or intDay > monthMaxDay: + return False + # time + strHour, strMinute, strSecond = strTime.split(":") + if len(strHour) != 2: + return False + if len(strMinute) != 2: + return False + if len(strSecond) != 2: + return False + try: + intHour = int(strHour) + intMinute = int(strMinute) + intSecond = int(strSecond) + except ValueError: + return False + if intHour < 0 or intHour > 23: + return False + if intMinute < 0 or intMinute > 59: + return False + if intSecond < 0 or intSecond > 59: + return False + # fallback + return True + + +def fontInfoOpenTypeNameRecordsValidator(value: Any) -> bool: + """ + Version 3+. + """ + if not isinstance(value, list): + return False + dictPrototype: GenericDict = dict( + nameID=(int, True), + platformID=(int, True), + encodingID=(int, True), + languageID=(int, True), + string=(str, True), + ) + for nameRecord in value: + if not genericDictValidator(nameRecord, dictPrototype): + return False + return True + + +def fontInfoOpenTypeOS2WeightClassValidator(value: Any) -> bool: + """ + Version 2+. + """ + if not isinstance(value, int): + return False + if value < 0: + return False + return True + + +def fontInfoOpenTypeOS2WidthClassValidator(value: Any) -> bool: + """ + Version 2+. + """ + if not isinstance(value, int): + return False + if value < 1: + return False + if value > 9: + return False + return True + + +def fontInfoVersion2OpenTypeOS2PanoseValidator(values: Any) -> bool: + """ + Version 2. + """ + if not isinstance(values, (list, tuple)): + return False + if len(values) != 10: + return False + for value in values: + if not isinstance(value, int): + return False + # XXX further validation? + return True + + +def fontInfoVersion3OpenTypeOS2PanoseValidator(values: Any) -> bool: + """ + Version 3+. + """ + if not isinstance(values, (list, tuple)): + return False + if len(values) != 10: + return False + for value in values: + if not isinstance(value, int): + return False + if value < 0: + return False + # XXX further validation? + return True + + +def fontInfoOpenTypeOS2FamilyClassValidator(values: Any) -> bool: + """ + Version 2+. + """ + if not isinstance(values, (list, tuple)): + return False + if len(values) != 2: + return False + for value in values: + if not isinstance(value, int): + return False + classID, subclassID = values + if classID < 0 or classID > 14: + return False + if subclassID < 0 or subclassID > 15: + return False + return True + + +def fontInfoPostscriptBluesValidator(values: Any) -> bool: + """ + Version 2+. + """ + if not isinstance(values, (list, tuple)): + return False + if len(values) > 14: + return False + if len(values) % 2: + return False + for value in values: + if not isinstance(value, numberTypes): + return False + return True + + +def fontInfoPostscriptOtherBluesValidator(values: Any) -> bool: + """ + Version 2+. + """ + if not isinstance(values, (list, tuple)): + return False + if len(values) > 10: + return False + if len(values) % 2: + return False + for value in values: + if not isinstance(value, numberTypes): + return False + return True + + +def fontInfoPostscriptStemsValidator(values: Any) -> bool: + """ + Version 2+. + """ + if not isinstance(values, (list, tuple)): + return False + if len(values) > 12: + return False + for value in values: + if not isinstance(value, numberTypes): + return False + return True + + +def fontInfoPostscriptWindowsCharacterSetValidator(value: Any) -> bool: + """ + Version 2+. + """ + validValues = list(range(1, 21)) + if value not in validValues: + return False + return True + + +def fontInfoWOFFMetadataUniqueIDValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = dict(id=(str, True)) + if not genericDictValidator(value, dictPrototype): + return False + return True + + +def fontInfoWOFFMetadataVendorValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = { + "name": (str, True), + "url": (str, False), + "dir": (str, False), + "class": (str, False), + } + if not genericDictValidator(value, dictPrototype): + return False + if "dir" in value and value.get("dir") not in ("ltr", "rtl"): + return False + return True + + +def fontInfoWOFFMetadataCreditsValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = dict(credits=(list, True)) + if not genericDictValidator(value, dictPrototype): + return False + if not len(value["credits"]): + return False + dictPrototype = { + "name": (str, True), + "url": (str, False), + "role": (str, False), + "dir": (str, False), + "class": (str, False), + } + for credit in value["credits"]: + if not genericDictValidator(credit, dictPrototype): + return False + if "dir" in credit and credit.get("dir") not in ("ltr", "rtl"): + return False + return True + + +def fontInfoWOFFMetadataDescriptionValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = dict(url=(str, False), text=(list, True)) + if not genericDictValidator(value, dictPrototype): + return False + for text in value["text"]: + if not fontInfoWOFFMetadataTextValue(text): + return False + return True + + +def fontInfoWOFFMetadataLicenseValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = dict( + url=(str, False), text=(list, False), id=(str, False) + ) + if not genericDictValidator(value, dictPrototype): + return False + if "text" in value: + for text in value["text"]: + if not fontInfoWOFFMetadataTextValue(text): + return False + return True + + +def fontInfoWOFFMetadataTrademarkValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = dict(text=(list, True)) + if not genericDictValidator(value, dictPrototype): + return False + for text in value["text"]: + if not fontInfoWOFFMetadataTextValue(text): + return False + return True + + +def fontInfoWOFFMetadataCopyrightValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = dict(text=(list, True)) + if not genericDictValidator(value, dictPrototype): + return False + for text in value["text"]: + if not fontInfoWOFFMetadataTextValue(text): + return False + return True + + +def fontInfoWOFFMetadataLicenseeValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = { + "name": (str, True), + "dir": (str, False), + "class": (str, False), + } + if not genericDictValidator(value, dictPrototype): + return False + if "dir" in value and value.get("dir") not in ("ltr", "rtl"): + return False + return True + + +def fontInfoWOFFMetadataTextValue(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = { + "text": (str, True), + "language": (str, False), + "dir": (str, False), + "class": (str, False), + } + if not genericDictValidator(value, dictPrototype): + return False + if "dir" in value and value.get("dir") not in ("ltr", "rtl"): + return False + return True + + +def fontInfoWOFFMetadataExtensionsValidator(value: Any) -> bool: + """ + Version 3+. + """ + if not isinstance(value, list): + return False + if not value: + return False + for extension in value: + if not fontInfoWOFFMetadataExtensionValidator(extension): + return False + return True + + +def fontInfoWOFFMetadataExtensionValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = dict( + names=(list, False), items=(list, True), id=(str, False) + ) + if not genericDictValidator(value, dictPrototype): + return False + if "names" in value: + for name in value["names"]: + if not fontInfoWOFFMetadataExtensionNameValidator(name): + return False + for item in value["items"]: + if not fontInfoWOFFMetadataExtensionItemValidator(item): + return False + return True + + +def fontInfoWOFFMetadataExtensionItemValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = dict( + id=(str, False), names=(list, True), values=(list, True) + ) + if not genericDictValidator(value, dictPrototype): + return False + for name in value["names"]: + if not fontInfoWOFFMetadataExtensionNameValidator(name): + return False + for val in value["values"]: + if not fontInfoWOFFMetadataExtensionValueValidator(val): + return False + return True + + +def fontInfoWOFFMetadataExtensionNameValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = { + "text": (str, True), + "language": (str, False), + "dir": (str, False), + "class": (str, False), + } + if not genericDictValidator(value, dictPrototype): + return False + if "dir" in value and value.get("dir") not in ("ltr", "rtl"): + return False + return True + + +def fontInfoWOFFMetadataExtensionValueValidator(value: Any) -> bool: + """ + Version 3+. + """ + dictPrototype: GenericDict = { + "text": (str, True), + "language": (str, False), + "dir": (str, False), + "class": (str, False), + } + if not genericDictValidator(value, dictPrototype): + return False + if "dir" in value and value.get("dir") not in ("ltr", "rtl"): + return False + return True + + +# ---------- +# Guidelines +# ---------- + + +def guidelinesValidator(value: Any, identifiers: Optional[set[str]] = None) -> bool: + """ + Version 3+. + """ + if not isinstance(value, list): + return False + if identifiers is None: + identifiers = set() + for guide in value: + if not guidelineValidator(guide): + return False + identifier = guide.get("identifier") + if identifier is not None: + if identifier in identifiers: + return False + identifiers.add(identifier) + return True + + +_guidelineDictPrototype: GenericDict = dict( + x=((int, float), False), + y=((int, float), False), + angle=((int, float), False), + name=(str, False), + color=(str, False), + identifier=(str, False), +) + + +def guidelineValidator(value: Any) -> bool: + """ + Version 3+. + """ + if not genericDictValidator(value, _guidelineDictPrototype): + return False + + angle = value.get("angle") + # angle must be between 0 and 360 + if angle is not None: + if angle < 0: + return False + if angle > 360: + return False + # identifier must be 1 or more characters + identifier = value.get("identifier") + if identifier is not None and not identifierValidator(identifier): + return False + # color must follow the proper format + color = value.get("color") + if color is not None and not colorValidator(color): + return False + return True + + +# ------- +# Anchors +# ------- + + +def anchorsValidator(value: Any, identifiers: Optional[set[str]] = None) -> bool: + """ + Version 3+. + """ + if not isinstance(value, list): + return False + if identifiers is None: + identifiers = set() + for anchor in value: + if not anchorValidator(anchor): + return False + identifier = anchor.get("identifier") + if identifier is not None: + if identifier in identifiers: + return False + identifiers.add(identifier) + return True + + +_anchorDictPrototype: GenericDict = dict( + x=((int, float), False), + y=((int, float), False), + name=(str, False), + color=(str, False), + identifier=(str, False), +) + + +def anchorValidator(value: Any) -> bool: + """ + Version 3+. + """ + if not genericDictValidator(value, _anchorDictPrototype): + return False + x = value.get("x") + y = value.get("y") + # x and y must be present + if x is None or y is None: + return False + # identifier must be 1 or more characters + identifier = value.get("identifier") + if identifier is not None and not identifierValidator(identifier): + return False + # color must follow the proper format + color = value.get("color") + if color is not None and not colorValidator(color): + return False + return True + + +# ---------- +# Identifier +# ---------- + + +def identifierValidator(value: Any) -> bool: + """ + Version 3+. + + >>> identifierValidator("a") + True + >>> identifierValidator("") + False + >>> identifierValidator("a" * 101) + False + """ + validCharactersMin = 0x20 + validCharactersMax = 0x7E + if not isinstance(value, str): + return False + if not value: + return False + if len(value) > 100: + return False + for c in value: + i = ord(c) + if i < validCharactersMin or i > validCharactersMax: + return False + return True + + +# ----- +# Color +# ----- + + +def colorValidator(value: Any) -> bool: + """ + Version 3+. + + >>> colorValidator("0,0,0,0") + True + >>> colorValidator(".5,.5,.5,.5") + True + >>> colorValidator("0.5,0.5,0.5,0.5") + True + >>> colorValidator("1,1,1,1") + True + + >>> colorValidator("2,0,0,0") + False + >>> colorValidator("0,2,0,0") + False + >>> colorValidator("0,0,2,0") + False + >>> colorValidator("0,0,0,2") + False + + >>> colorValidator("1r,1,1,1") + False + >>> colorValidator("1,1g,1,1") + False + >>> colorValidator("1,1,1b,1") + False + >>> colorValidator("1,1,1,1a") + False + + >>> colorValidator("1 1 1 1") + False + >>> colorValidator("1 1,1,1") + False + >>> colorValidator("1,1 1,1") + False + >>> colorValidator("1,1,1 1") + False + + >>> colorValidator("1, 1, 1, 1") + True + """ + if not isinstance(value, str): + return False + parts = value.split(",") + if len(parts) != 4: + return False + for part in parts: + part = part.strip() + converted = False + number: IntFloat + try: + number = int(part) + converted = True + except ValueError: + pass + if not converted: + try: + number = float(part) + converted = True + except ValueError: + pass + if not converted: + return False + if not 0 <= number <= 1: + return False + return True + + +# ----- +# image +# ----- + +pngSignature: bytes = b"\x89PNG\r\n\x1a\n" + +_imageDictPrototype: GenericDict = dict( + fileName=(str, True), + xScale=((int, float), False), + xyScale=((int, float), False), + yxScale=((int, float), False), + yScale=((int, float), False), + xOffset=((int, float), False), + yOffset=((int, float), False), + color=(str, False), +) + + +def imageValidator(value): + """ + Version 3+. + """ + if not genericDictValidator(value, _imageDictPrototype): + return False + # fileName must be one or more characters + if not value["fileName"]: + return False + # color must follow the proper format + color = value.get("color") + if color is not None and not colorValidator(color): + return False + return True + + +def pngValidator( + path: Optional[str] = None, + data: Optional[bytes] = None, + fileObj: Optional[Any] = None, +) -> tuple[bool, Any]: + """ + Version 3+. + + This checks the signature of the image data. + """ + assert path is not None or data is not None or fileObj is not None + if path is not None: + with open(path, "rb") as f: + signature = f.read(8) + elif data is not None: + signature = data[:8] + elif fileObj is not None: + pos = fileObj.tell() + signature = fileObj.read(8) + fileObj.seek(pos) + if signature != pngSignature: + return False, "Image does not begin with the PNG signature." + return True, None + + +# ------------------- +# layercontents.plist +# ------------------- + + +def layerContentsValidator( + value: Any, ufoPathOrFileSystem: Union[str, fs.base.FS] +) -> tuple[bool, Optional[str]]: + """ + Check the validity of layercontents.plist. + Version 3+. + """ + if isinstance(ufoPathOrFileSystem, fs.base.FS): + fileSystem = ufoPathOrFileSystem + else: + fileSystem = fs.osfs.OSFS(ufoPathOrFileSystem) + + bogusFileMessage = "layercontents.plist in not in the correct format." + # file isn't in the right format + if not isinstance(value, list): + return False, bogusFileMessage + # work through each entry + usedLayerNames = set() + usedDirectories = set() + contents = {} + for entry in value: + # layer entry in the incorrect format + if not isinstance(entry, list): + return False, bogusFileMessage + if not len(entry) == 2: + return False, bogusFileMessage + for i in entry: + if not isinstance(i, str): + return False, bogusFileMessage + layerName, directoryName = entry + # check directory naming + if directoryName != "glyphs": + if not directoryName.startswith("glyphs."): + return ( + False, + "Invalid directory name (%s) in layercontents.plist." + % directoryName, + ) + if len(layerName) == 0: + return False, "Empty layer name in layercontents.plist." + # directory doesn't exist + if not fileSystem.exists(directoryName): + return False, "A glyphset does not exist at %s." % directoryName + # default layer name + if layerName == "public.default" and directoryName != "glyphs": + return ( + False, + "The name public.default is being used by a layer that is not the default.", + ) + # check usage + if layerName in usedLayerNames: + return ( + False, + "The layer name %s is used by more than one layer." % layerName, + ) + usedLayerNames.add(layerName) + if directoryName in usedDirectories: + return ( + False, + "The directory %s is used by more than one layer." % directoryName, + ) + usedDirectories.add(directoryName) + # store + contents[layerName] = directoryName + # missing default layer + foundDefault = "glyphs" in contents.values() + if not foundDefault: + return False, "The required default glyph set is not in the UFO." + return True, None + + +# ------------ +# groups.plist +# ------------ + + +def groupsValidator(value: Any) -> tuple[bool, Optional[str]]: + """ + Check the validity of the groups. + Version 3+ (though it's backwards compatible with UFO 1 and UFO 2). + + >>> groups = {"A" : ["A", "A"], "A2" : ["A"]} + >>> groupsValidator(groups) + (True, None) + + >>> groups = {"" : ["A"]} + >>> valid, msg = groupsValidator(groups) + >>> valid + False + >>> print(msg) + A group has an empty name. + + >>> groups = {"public.awesome" : ["A"]} + >>> groupsValidator(groups) + (True, None) + + >>> groups = {"public.kern1." : ["A"]} + >>> valid, msg = groupsValidator(groups) + >>> valid + False + >>> print(msg) + The group data contains a kerning group with an incomplete name. + >>> groups = {"public.kern2." : ["A"]} + >>> valid, msg = groupsValidator(groups) + >>> valid + False + >>> print(msg) + The group data contains a kerning group with an incomplete name. + + >>> groups = {"public.kern1.A" : ["A"], "public.kern2.A" : ["A"]} + >>> groupsValidator(groups) + (True, None) + + >>> groups = {"public.kern1.A1" : ["A"], "public.kern1.A2" : ["A"]} + >>> valid, msg = groupsValidator(groups) + >>> valid + False + >>> print(msg) + The glyph "A" occurs in too many kerning groups. + """ + bogusFormatMessage = "The group data is not in the correct format." + if not isDictEnough(value): + return False, bogusFormatMessage + firstSideMapping: dict[str, str] = {} + secondSideMapping: dict[str, str] = {} + for groupName, glyphList in value.items(): + if not isinstance(groupName, (str)): + return False, bogusFormatMessage + if not isinstance(glyphList, (list, tuple)): + return False, bogusFormatMessage + if not groupName: + return False, "A group has an empty name." + if groupName.startswith("public."): + if not groupName.startswith("public.kern1.") and not groupName.startswith( + "public.kern2." + ): + # unknown public.* name. silently skip. + continue + else: + if len("public.kernN.") == len(groupName): + return ( + False, + "The group data contains a kerning group with an incomplete name.", + ) + if groupName.startswith("public.kern1."): + d = firstSideMapping + else: + d = secondSideMapping + for glyphName in glyphList: + if not isinstance(glyphName, str): + return ( + False, + "The group data %s contains an invalid member." % groupName, + ) + if glyphName in d: + return ( + False, + 'The glyph "%s" occurs in too many kerning groups.' % glyphName, + ) + d[glyphName] = groupName + return True, None + + +# ------------- +# kerning.plist +# ------------- + + +def kerningValidator(data: Any) -> tuple[bool, Optional[str]]: + """ + Check the validity of the kerning data structure. + Version 3+ (though it's backwards compatible with UFO 1 and UFO 2). + + >>> kerning = {"A" : {"B" : 100}} + >>> kerningValidator(kerning) + (True, None) + + >>> kerning = {"A" : ["B"]} + >>> valid, msg = kerningValidator(kerning) + >>> valid + False + >>> print(msg) + The kerning data is not in the correct format. + + >>> kerning = {"A" : {"B" : "100"}} + >>> valid, msg = kerningValidator(kerning) + >>> valid + False + >>> print(msg) + The kerning data is not in the correct format. + """ + bogusFormatMessage = "The kerning data is not in the correct format." + if not isinstance(data, Mapping): + return False, bogusFormatMessage + for first, secondDict in data.items(): + if not isinstance(first, str): + return False, bogusFormatMessage + elif not isinstance(secondDict, Mapping): + return False, bogusFormatMessage + for second, value in secondDict.items(): + if not isinstance(second, str): + return False, bogusFormatMessage + elif not isinstance(value, numberTypes): + return False, bogusFormatMessage + return True, None + + +# ------------- +# lib.plist/lib +# ------------- + +_bogusLibFormatMessage = "The lib data is not in the correct format: %s" + + +def fontLibValidator(value: Any) -> tuple[bool, Optional[str]]: + """ + Check the validity of the lib. + Version 3+ (though it's backwards compatible with UFO 1 and UFO 2). + + >>> lib = {"foo" : "bar"} + >>> fontLibValidator(lib) + (True, None) + + >>> lib = {"public.awesome" : "hello"} + >>> fontLibValidator(lib) + (True, None) + + >>> lib = {"public.glyphOrder" : ["A", "C", "B"]} + >>> fontLibValidator(lib) + (True, None) + + >>> lib = "hello" + >>> valid, msg = fontLibValidator(lib) + >>> valid + False + >>> print(msg) # doctest: +ELLIPSIS + The lib data is not in the correct format: expected a dictionary, ... + + >>> lib = {1: "hello"} + >>> valid, msg = fontLibValidator(lib) + >>> valid + False + >>> print(msg) + The lib key is not properly formatted: expected str, found int: 1 + + >>> lib = {"public.glyphOrder" : "hello"} + >>> valid, msg = fontLibValidator(lib) + >>> valid + False + >>> print(msg) # doctest: +ELLIPSIS + public.glyphOrder is not properly formatted: expected list or tuple,... + + >>> lib = {"public.glyphOrder" : ["A", 1, "B"]} + >>> valid, msg = fontLibValidator(lib) + >>> valid + False + >>> print(msg) # doctest: +ELLIPSIS + public.glyphOrder is not properly formatted: expected str,... + """ + if not isDictEnough(value): + reason = "expected a dictionary, found %s" % type(value).__name__ + return False, _bogusLibFormatMessage % reason + for key, value in value.items(): + if not isinstance(key, str): + return False, ( + "The lib key is not properly formatted: expected str, found %s: %r" + % (type(key).__name__, key) + ) + # public.glyphOrder + if key == "public.glyphOrder": + bogusGlyphOrderMessage = "public.glyphOrder is not properly formatted: %s" + if not isinstance(value, (list, tuple)): + reason = "expected list or tuple, found %s" % type(value).__name__ + return False, bogusGlyphOrderMessage % reason + for glyphName in value: + if not isinstance(glyphName, str): + reason = "expected str, found %s" % type(glyphName).__name__ + return False, bogusGlyphOrderMessage % reason + return True, None + + +# -------- +# GLIF lib +# -------- + + +def glyphLibValidator(value: Any) -> tuple[bool, Optional[str]]: + """ + Check the validity of the lib. + Version 3+ (though it's backwards compatible with UFO 1 and UFO 2). + + >>> lib = {"foo" : "bar"} + >>> glyphLibValidator(lib) + (True, None) + + >>> lib = {"public.awesome" : "hello"} + >>> glyphLibValidator(lib) + (True, None) + + >>> lib = {"public.markColor" : "1,0,0,0.5"} + >>> glyphLibValidator(lib) + (True, None) + + >>> lib = {"public.markColor" : 1} + >>> valid, msg = glyphLibValidator(lib) + >>> valid + False + >>> print(msg) + public.markColor is not properly formatted. + """ + if not isDictEnough(value): + reason = "expected a dictionary, found %s" % type(value).__name__ + return False, _bogusLibFormatMessage % reason + for key, value in value.items(): + if not isinstance(key, str): + reason = "key (%s) should be a string" % key + return False, _bogusLibFormatMessage % reason + # public.markColor + if key == "public.markColor": + if not colorValidator(value): + return False, "public.markColor is not properly formatted." + return True, None + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/.venv/lib/python3.11/site-packages/fontTools/unicode.py b/.venv/lib/python3.11/site-packages/fontTools/unicode.py new file mode 100644 index 0000000000000000000000000000000000000000..a9ffeefac1c9e553c53bc12346e49e7ece8d364a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/unicode.py @@ -0,0 +1,50 @@ +def _makeunicodes(f): + lines = iter(f.readlines()) + unicodes = {} + for line in lines: + if not line: + continue + num, name = line.split(";")[:2] + if name[0] == "<": + continue # "<control>", etc. + num = int(num, 16) + unicodes[num] = name + return unicodes + + +class _UnicodeCustom(object): + def __init__(self, f): + if isinstance(f, str): + with open(f) as fd: + codes = _makeunicodes(fd) + else: + codes = _makeunicodes(f) + self.codes = codes + + def __getitem__(self, charCode): + try: + return self.codes[charCode] + except KeyError: + return "????" + + +class _UnicodeBuiltin(object): + def __getitem__(self, charCode): + try: + # use unicodedata backport to python2, if available: + # https://github.com/mikekap/unicodedata2 + import unicodedata2 as unicodedata + except ImportError: + import unicodedata + try: + return unicodedata.name(chr(charCode)) + except ValueError: + return "????" + + +Unicode = _UnicodeBuiltin() + + +def setUnicodeData(f): + global Unicode + Unicode = _UnicodeCustom(f) diff --git a/.venv/lib/python3.11/site-packages/fontTools/unicodedata/Blocks.py b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/Blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..c333c4ee3bfdd4d20e5b89854bac3ba35769b975 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/Blocks.py @@ -0,0 +1,817 @@ +# -*- coding: utf-8 -*- +# +# NOTE: This file was auto-generated with MetaTools/buildUCD.py. +# Source: https://unicode.org/Public/UNIDATA/Blocks.txt +# License: http://unicode.org/copyright.html#License +# +# Blocks-17.0.0.txt +# Date: 2025-08-01 +# © 2025 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see https://www.unicode.org/reports/tr44/ +# +# Format: +# Start Code..End Code; Block Name + +RANGES = [ + 0x0000, # .. 0x007F ; Basic Latin + 0x0080, # .. 0x00FF ; Latin-1 Supplement + 0x0100, # .. 0x017F ; Latin Extended-A + 0x0180, # .. 0x024F ; Latin Extended-B + 0x0250, # .. 0x02AF ; IPA Extensions + 0x02B0, # .. 0x02FF ; Spacing Modifier Letters + 0x0300, # .. 0x036F ; Combining Diacritical Marks + 0x0370, # .. 0x03FF ; Greek and Coptic + 0x0400, # .. 0x04FF ; Cyrillic + 0x0500, # .. 0x052F ; Cyrillic Supplement + 0x0530, # .. 0x058F ; Armenian + 0x0590, # .. 0x05FF ; Hebrew + 0x0600, # .. 0x06FF ; Arabic + 0x0700, # .. 0x074F ; Syriac + 0x0750, # .. 0x077F ; Arabic Supplement + 0x0780, # .. 0x07BF ; Thaana + 0x07C0, # .. 0x07FF ; NKo + 0x0800, # .. 0x083F ; Samaritan + 0x0840, # .. 0x085F ; Mandaic + 0x0860, # .. 0x086F ; Syriac Supplement + 0x0870, # .. 0x089F ; Arabic Extended-B + 0x08A0, # .. 0x08FF ; Arabic Extended-A + 0x0900, # .. 0x097F ; Devanagari + 0x0980, # .. 0x09FF ; Bengali + 0x0A00, # .. 0x0A7F ; Gurmukhi + 0x0A80, # .. 0x0AFF ; Gujarati + 0x0B00, # .. 0x0B7F ; Oriya + 0x0B80, # .. 0x0BFF ; Tamil + 0x0C00, # .. 0x0C7F ; Telugu + 0x0C80, # .. 0x0CFF ; Kannada + 0x0D00, # .. 0x0D7F ; Malayalam + 0x0D80, # .. 0x0DFF ; Sinhala + 0x0E00, # .. 0x0E7F ; Thai + 0x0E80, # .. 0x0EFF ; Lao + 0x0F00, # .. 0x0FFF ; Tibetan + 0x1000, # .. 0x109F ; Myanmar + 0x10A0, # .. 0x10FF ; Georgian + 0x1100, # .. 0x11FF ; Hangul Jamo + 0x1200, # .. 0x137F ; Ethiopic + 0x1380, # .. 0x139F ; Ethiopic Supplement + 0x13A0, # .. 0x13FF ; Cherokee + 0x1400, # .. 0x167F ; Unified Canadian Aboriginal Syllabics + 0x1680, # .. 0x169F ; Ogham + 0x16A0, # .. 0x16FF ; Runic + 0x1700, # .. 0x171F ; Tagalog + 0x1720, # .. 0x173F ; Hanunoo + 0x1740, # .. 0x175F ; Buhid + 0x1760, # .. 0x177F ; Tagbanwa + 0x1780, # .. 0x17FF ; Khmer + 0x1800, # .. 0x18AF ; Mongolian + 0x18B0, # .. 0x18FF ; Unified Canadian Aboriginal Syllabics Extended + 0x1900, # .. 0x194F ; Limbu + 0x1950, # .. 0x197F ; Tai Le + 0x1980, # .. 0x19DF ; New Tai Lue + 0x19E0, # .. 0x19FF ; Khmer Symbols + 0x1A00, # .. 0x1A1F ; Buginese + 0x1A20, # .. 0x1AAF ; Tai Tham + 0x1AB0, # .. 0x1AFF ; Combining Diacritical Marks Extended + 0x1B00, # .. 0x1B7F ; Balinese + 0x1B80, # .. 0x1BBF ; Sundanese + 0x1BC0, # .. 0x1BFF ; Batak + 0x1C00, # .. 0x1C4F ; Lepcha + 0x1C50, # .. 0x1C7F ; Ol Chiki + 0x1C80, # .. 0x1C8F ; Cyrillic Extended-C + 0x1C90, # .. 0x1CBF ; Georgian Extended + 0x1CC0, # .. 0x1CCF ; Sundanese Supplement + 0x1CD0, # .. 0x1CFF ; Vedic Extensions + 0x1D00, # .. 0x1D7F ; Phonetic Extensions + 0x1D80, # .. 0x1DBF ; Phonetic Extensions Supplement + 0x1DC0, # .. 0x1DFF ; Combining Diacritical Marks Supplement + 0x1E00, # .. 0x1EFF ; Latin Extended Additional + 0x1F00, # .. 0x1FFF ; Greek Extended + 0x2000, # .. 0x206F ; General Punctuation + 0x2070, # .. 0x209F ; Superscripts and Subscripts + 0x20A0, # .. 0x20CF ; Currency Symbols + 0x20D0, # .. 0x20FF ; Combining Diacritical Marks for Symbols + 0x2100, # .. 0x214F ; Letterlike Symbols + 0x2150, # .. 0x218F ; Number Forms + 0x2190, # .. 0x21FF ; Arrows + 0x2200, # .. 0x22FF ; Mathematical Operators + 0x2300, # .. 0x23FF ; Miscellaneous Technical + 0x2400, # .. 0x243F ; Control Pictures + 0x2440, # .. 0x245F ; Optical Character Recognition + 0x2460, # .. 0x24FF ; Enclosed Alphanumerics + 0x2500, # .. 0x257F ; Box Drawing + 0x2580, # .. 0x259F ; Block Elements + 0x25A0, # .. 0x25FF ; Geometric Shapes + 0x2600, # .. 0x26FF ; Miscellaneous Symbols + 0x2700, # .. 0x27BF ; Dingbats + 0x27C0, # .. 0x27EF ; Miscellaneous Mathematical Symbols-A + 0x27F0, # .. 0x27FF ; Supplemental Arrows-A + 0x2800, # .. 0x28FF ; Braille Patterns + 0x2900, # .. 0x297F ; Supplemental Arrows-B + 0x2980, # .. 0x29FF ; Miscellaneous Mathematical Symbols-B + 0x2A00, # .. 0x2AFF ; Supplemental Mathematical Operators + 0x2B00, # .. 0x2BFF ; Miscellaneous Symbols and Arrows + 0x2C00, # .. 0x2C5F ; Glagolitic + 0x2C60, # .. 0x2C7F ; Latin Extended-C + 0x2C80, # .. 0x2CFF ; Coptic + 0x2D00, # .. 0x2D2F ; Georgian Supplement + 0x2D30, # .. 0x2D7F ; Tifinagh + 0x2D80, # .. 0x2DDF ; Ethiopic Extended + 0x2DE0, # .. 0x2DFF ; Cyrillic Extended-A + 0x2E00, # .. 0x2E7F ; Supplemental Punctuation + 0x2E80, # .. 0x2EFF ; CJK Radicals Supplement + 0x2F00, # .. 0x2FDF ; Kangxi Radicals + 0x2FE0, # .. 0x2FEF ; No_Block + 0x2FF0, # .. 0x2FFF ; Ideographic Description Characters + 0x3000, # .. 0x303F ; CJK Symbols and Punctuation + 0x3040, # .. 0x309F ; Hiragana + 0x30A0, # .. 0x30FF ; Katakana + 0x3100, # .. 0x312F ; Bopomofo + 0x3130, # .. 0x318F ; Hangul Compatibility Jamo + 0x3190, # .. 0x319F ; Kanbun + 0x31A0, # .. 0x31BF ; Bopomofo Extended + 0x31C0, # .. 0x31EF ; CJK Strokes + 0x31F0, # .. 0x31FF ; Katakana Phonetic Extensions + 0x3200, # .. 0x32FF ; Enclosed CJK Letters and Months + 0x3300, # .. 0x33FF ; CJK Compatibility + 0x3400, # .. 0x4DBF ; CJK Unified Ideographs Extension A + 0x4DC0, # .. 0x4DFF ; Yijing Hexagram Symbols + 0x4E00, # .. 0x9FFF ; CJK Unified Ideographs + 0xA000, # .. 0xA48F ; Yi Syllables + 0xA490, # .. 0xA4CF ; Yi Radicals + 0xA4D0, # .. 0xA4FF ; Lisu + 0xA500, # .. 0xA63F ; Vai + 0xA640, # .. 0xA69F ; Cyrillic Extended-B + 0xA6A0, # .. 0xA6FF ; Bamum + 0xA700, # .. 0xA71F ; Modifier Tone Letters + 0xA720, # .. 0xA7FF ; Latin Extended-D + 0xA800, # .. 0xA82F ; Syloti Nagri + 0xA830, # .. 0xA83F ; Common Indic Number Forms + 0xA840, # .. 0xA87F ; Phags-pa + 0xA880, # .. 0xA8DF ; Saurashtra + 0xA8E0, # .. 0xA8FF ; Devanagari Extended + 0xA900, # .. 0xA92F ; Kayah Li + 0xA930, # .. 0xA95F ; Rejang + 0xA960, # .. 0xA97F ; Hangul Jamo Extended-A + 0xA980, # .. 0xA9DF ; Javanese + 0xA9E0, # .. 0xA9FF ; Myanmar Extended-B + 0xAA00, # .. 0xAA5F ; Cham + 0xAA60, # .. 0xAA7F ; Myanmar Extended-A + 0xAA80, # .. 0xAADF ; Tai Viet + 0xAAE0, # .. 0xAAFF ; Meetei Mayek Extensions + 0xAB00, # .. 0xAB2F ; Ethiopic Extended-A + 0xAB30, # .. 0xAB6F ; Latin Extended-E + 0xAB70, # .. 0xABBF ; Cherokee Supplement + 0xABC0, # .. 0xABFF ; Meetei Mayek + 0xAC00, # .. 0xD7AF ; Hangul Syllables + 0xD7B0, # .. 0xD7FF ; Hangul Jamo Extended-B + 0xD800, # .. 0xDB7F ; High Surrogates + 0xDB80, # .. 0xDBFF ; High Private Use Surrogates + 0xDC00, # .. 0xDFFF ; Low Surrogates + 0xE000, # .. 0xF8FF ; Private Use Area + 0xF900, # .. 0xFAFF ; CJK Compatibility Ideographs + 0xFB00, # .. 0xFB4F ; Alphabetic Presentation Forms + 0xFB50, # .. 0xFDFF ; Arabic Presentation Forms-A + 0xFE00, # .. 0xFE0F ; Variation Selectors + 0xFE10, # .. 0xFE1F ; Vertical Forms + 0xFE20, # .. 0xFE2F ; Combining Half Marks + 0xFE30, # .. 0xFE4F ; CJK Compatibility Forms + 0xFE50, # .. 0xFE6F ; Small Form Variants + 0xFE70, # .. 0xFEFF ; Arabic Presentation Forms-B + 0xFF00, # .. 0xFFEF ; Halfwidth and Fullwidth Forms + 0xFFF0, # .. 0xFFFF ; Specials + 0x10000, # .. 0x1007F ; Linear B Syllabary + 0x10080, # .. 0x100FF ; Linear B Ideograms + 0x10100, # .. 0x1013F ; Aegean Numbers + 0x10140, # .. 0x1018F ; Ancient Greek Numbers + 0x10190, # .. 0x101CF ; Ancient Symbols + 0x101D0, # .. 0x101FF ; Phaistos Disc + 0x10200, # .. 0x1027F ; No_Block + 0x10280, # .. 0x1029F ; Lycian + 0x102A0, # .. 0x102DF ; Carian + 0x102E0, # .. 0x102FF ; Coptic Epact Numbers + 0x10300, # .. 0x1032F ; Old Italic + 0x10330, # .. 0x1034F ; Gothic + 0x10350, # .. 0x1037F ; Old Permic + 0x10380, # .. 0x1039F ; Ugaritic + 0x103A0, # .. 0x103DF ; Old Persian + 0x103E0, # .. 0x103FF ; No_Block + 0x10400, # .. 0x1044F ; Deseret + 0x10450, # .. 0x1047F ; Shavian + 0x10480, # .. 0x104AF ; Osmanya + 0x104B0, # .. 0x104FF ; Osage + 0x10500, # .. 0x1052F ; Elbasan + 0x10530, # .. 0x1056F ; Caucasian Albanian + 0x10570, # .. 0x105BF ; Vithkuqi + 0x105C0, # .. 0x105FF ; Todhri + 0x10600, # .. 0x1077F ; Linear A + 0x10780, # .. 0x107BF ; Latin Extended-F + 0x107C0, # .. 0x107FF ; No_Block + 0x10800, # .. 0x1083F ; Cypriot Syllabary + 0x10840, # .. 0x1085F ; Imperial Aramaic + 0x10860, # .. 0x1087F ; Palmyrene + 0x10880, # .. 0x108AF ; Nabataean + 0x108B0, # .. 0x108DF ; No_Block + 0x108E0, # .. 0x108FF ; Hatran + 0x10900, # .. 0x1091F ; Phoenician + 0x10920, # .. 0x1093F ; Lydian + 0x10940, # .. 0x1095F ; Sidetic + 0x10960, # .. 0x1097F ; No_Block + 0x10980, # .. 0x1099F ; Meroitic Hieroglyphs + 0x109A0, # .. 0x109FF ; Meroitic Cursive + 0x10A00, # .. 0x10A5F ; Kharoshthi + 0x10A60, # .. 0x10A7F ; Old South Arabian + 0x10A80, # .. 0x10A9F ; Old North Arabian + 0x10AA0, # .. 0x10ABF ; No_Block + 0x10AC0, # .. 0x10AFF ; Manichaean + 0x10B00, # .. 0x10B3F ; Avestan + 0x10B40, # .. 0x10B5F ; Inscriptional Parthian + 0x10B60, # .. 0x10B7F ; Inscriptional Pahlavi + 0x10B80, # .. 0x10BAF ; Psalter Pahlavi + 0x10BB0, # .. 0x10BFF ; No_Block + 0x10C00, # .. 0x10C4F ; Old Turkic + 0x10C50, # .. 0x10C7F ; No_Block + 0x10C80, # .. 0x10CFF ; Old Hungarian + 0x10D00, # .. 0x10D3F ; Hanifi Rohingya + 0x10D40, # .. 0x10D8F ; Garay + 0x10D90, # .. 0x10E5F ; No_Block + 0x10E60, # .. 0x10E7F ; Rumi Numeral Symbols + 0x10E80, # .. 0x10EBF ; Yezidi + 0x10EC0, # .. 0x10EFF ; Arabic Extended-C + 0x10F00, # .. 0x10F2F ; Old Sogdian + 0x10F30, # .. 0x10F6F ; Sogdian + 0x10F70, # .. 0x10FAF ; Old Uyghur + 0x10FB0, # .. 0x10FDF ; Chorasmian + 0x10FE0, # .. 0x10FFF ; Elymaic + 0x11000, # .. 0x1107F ; Brahmi + 0x11080, # .. 0x110CF ; Kaithi + 0x110D0, # .. 0x110FF ; Sora Sompeng + 0x11100, # .. 0x1114F ; Chakma + 0x11150, # .. 0x1117F ; Mahajani + 0x11180, # .. 0x111DF ; Sharada + 0x111E0, # .. 0x111FF ; Sinhala Archaic Numbers + 0x11200, # .. 0x1124F ; Khojki + 0x11250, # .. 0x1127F ; No_Block + 0x11280, # .. 0x112AF ; Multani + 0x112B0, # .. 0x112FF ; Khudawadi + 0x11300, # .. 0x1137F ; Grantha + 0x11380, # .. 0x113FF ; Tulu-Tigalari + 0x11400, # .. 0x1147F ; Newa + 0x11480, # .. 0x114DF ; Tirhuta + 0x114E0, # .. 0x1157F ; No_Block + 0x11580, # .. 0x115FF ; Siddham + 0x11600, # .. 0x1165F ; Modi + 0x11660, # .. 0x1167F ; Mongolian Supplement + 0x11680, # .. 0x116CF ; Takri + 0x116D0, # .. 0x116FF ; Myanmar Extended-C + 0x11700, # .. 0x1174F ; Ahom + 0x11750, # .. 0x117FF ; No_Block + 0x11800, # .. 0x1184F ; Dogra + 0x11850, # .. 0x1189F ; No_Block + 0x118A0, # .. 0x118FF ; Warang Citi + 0x11900, # .. 0x1195F ; Dives Akuru + 0x11960, # .. 0x1199F ; No_Block + 0x119A0, # .. 0x119FF ; Nandinagari + 0x11A00, # .. 0x11A4F ; Zanabazar Square + 0x11A50, # .. 0x11AAF ; Soyombo + 0x11AB0, # .. 0x11ABF ; Unified Canadian Aboriginal Syllabics Extended-A + 0x11AC0, # .. 0x11AFF ; Pau Cin Hau + 0x11B00, # .. 0x11B5F ; Devanagari Extended-A + 0x11B60, # .. 0x11B7F ; Sharada Supplement + 0x11B80, # .. 0x11BBF ; No_Block + 0x11BC0, # .. 0x11BFF ; Sunuwar + 0x11C00, # .. 0x11C6F ; Bhaiksuki + 0x11C70, # .. 0x11CBF ; Marchen + 0x11CC0, # .. 0x11CFF ; No_Block + 0x11D00, # .. 0x11D5F ; Masaram Gondi + 0x11D60, # .. 0x11DAF ; Gunjala Gondi + 0x11DB0, # .. 0x11DEF ; Tolong Siki + 0x11DF0, # .. 0x11EDF ; No_Block + 0x11EE0, # .. 0x11EFF ; Makasar + 0x11F00, # .. 0x11F5F ; Kawi + 0x11F60, # .. 0x11FAF ; No_Block + 0x11FB0, # .. 0x11FBF ; Lisu Supplement + 0x11FC0, # .. 0x11FFF ; Tamil Supplement + 0x12000, # .. 0x123FF ; Cuneiform + 0x12400, # .. 0x1247F ; Cuneiform Numbers and Punctuation + 0x12480, # .. 0x1254F ; Early Dynastic Cuneiform + 0x12550, # .. 0x12F8F ; No_Block + 0x12F90, # .. 0x12FFF ; Cypro-Minoan + 0x13000, # .. 0x1342F ; Egyptian Hieroglyphs + 0x13430, # .. 0x1345F ; Egyptian Hieroglyph Format Controls + 0x13460, # .. 0x143FF ; Egyptian Hieroglyphs Extended-A + 0x14400, # .. 0x1467F ; Anatolian Hieroglyphs + 0x14680, # .. 0x160FF ; No_Block + 0x16100, # .. 0x1613F ; Gurung Khema + 0x16140, # .. 0x167FF ; No_Block + 0x16800, # .. 0x16A3F ; Bamum Supplement + 0x16A40, # .. 0x16A6F ; Mro + 0x16A70, # .. 0x16ACF ; Tangsa + 0x16AD0, # .. 0x16AFF ; Bassa Vah + 0x16B00, # .. 0x16B8F ; Pahawh Hmong + 0x16B90, # .. 0x16D3F ; No_Block + 0x16D40, # .. 0x16D7F ; Kirat Rai + 0x16D80, # .. 0x16E3F ; No_Block + 0x16E40, # .. 0x16E9F ; Medefaidrin + 0x16EA0, # .. 0x16EDF ; Beria Erfe + 0x16EE0, # .. 0x16EFF ; No_Block + 0x16F00, # .. 0x16F9F ; Miao + 0x16FA0, # .. 0x16FDF ; No_Block + 0x16FE0, # .. 0x16FFF ; Ideographic Symbols and Punctuation + 0x17000, # .. 0x187FF ; Tangut + 0x18800, # .. 0x18AFF ; Tangut Components + 0x18B00, # .. 0x18CFF ; Khitan Small Script + 0x18D00, # .. 0x18D7F ; Tangut Supplement + 0x18D80, # .. 0x18DFF ; Tangut Components Supplement + 0x18E00, # .. 0x1AFEF ; No_Block + 0x1AFF0, # .. 0x1AFFF ; Kana Extended-B + 0x1B000, # .. 0x1B0FF ; Kana Supplement + 0x1B100, # .. 0x1B12F ; Kana Extended-A + 0x1B130, # .. 0x1B16F ; Small Kana Extension + 0x1B170, # .. 0x1B2FF ; Nushu + 0x1B300, # .. 0x1BBFF ; No_Block + 0x1BC00, # .. 0x1BC9F ; Duployan + 0x1BCA0, # .. 0x1BCAF ; Shorthand Format Controls + 0x1BCB0, # .. 0x1CBFF ; No_Block + 0x1CC00, # .. 0x1CEBF ; Symbols for Legacy Computing Supplement + 0x1CEC0, # .. 0x1CEFF ; Miscellaneous Symbols Supplement + 0x1CF00, # .. 0x1CFCF ; Znamenny Musical Notation + 0x1CFD0, # .. 0x1CFFF ; No_Block + 0x1D000, # .. 0x1D0FF ; Byzantine Musical Symbols + 0x1D100, # .. 0x1D1FF ; Musical Symbols + 0x1D200, # .. 0x1D24F ; Ancient Greek Musical Notation + 0x1D250, # .. 0x1D2BF ; No_Block + 0x1D2C0, # .. 0x1D2DF ; Kaktovik Numerals + 0x1D2E0, # .. 0x1D2FF ; Mayan Numerals + 0x1D300, # .. 0x1D35F ; Tai Xuan Jing Symbols + 0x1D360, # .. 0x1D37F ; Counting Rod Numerals + 0x1D380, # .. 0x1D3FF ; No_Block + 0x1D400, # .. 0x1D7FF ; Mathematical Alphanumeric Symbols + 0x1D800, # .. 0x1DAAF ; Sutton SignWriting + 0x1DAB0, # .. 0x1DEFF ; No_Block + 0x1DF00, # .. 0x1DFFF ; Latin Extended-G + 0x1E000, # .. 0x1E02F ; Glagolitic Supplement + 0x1E030, # .. 0x1E08F ; Cyrillic Extended-D + 0x1E090, # .. 0x1E0FF ; No_Block + 0x1E100, # .. 0x1E14F ; Nyiakeng Puachue Hmong + 0x1E150, # .. 0x1E28F ; No_Block + 0x1E290, # .. 0x1E2BF ; Toto + 0x1E2C0, # .. 0x1E2FF ; Wancho + 0x1E300, # .. 0x1E4CF ; No_Block + 0x1E4D0, # .. 0x1E4FF ; Nag Mundari + 0x1E500, # .. 0x1E5CF ; No_Block + 0x1E5D0, # .. 0x1E5FF ; Ol Onal + 0x1E600, # .. 0x1E6BF ; No_Block + 0x1E6C0, # .. 0x1E6FF ; Tai Yo + 0x1E700, # .. 0x1E7DF ; No_Block + 0x1E7E0, # .. 0x1E7FF ; Ethiopic Extended-B + 0x1E800, # .. 0x1E8DF ; Mende Kikakui + 0x1E8E0, # .. 0x1E8FF ; No_Block + 0x1E900, # .. 0x1E95F ; Adlam + 0x1E960, # .. 0x1EC6F ; No_Block + 0x1EC70, # .. 0x1ECBF ; Indic Siyaq Numbers + 0x1ECC0, # .. 0x1ECFF ; No_Block + 0x1ED00, # .. 0x1ED4F ; Ottoman Siyaq Numbers + 0x1ED50, # .. 0x1EDFF ; No_Block + 0x1EE00, # .. 0x1EEFF ; Arabic Mathematical Alphabetic Symbols + 0x1EF00, # .. 0x1EFFF ; No_Block + 0x1F000, # .. 0x1F02F ; Mahjong Tiles + 0x1F030, # .. 0x1F09F ; Domino Tiles + 0x1F0A0, # .. 0x1F0FF ; Playing Cards + 0x1F100, # .. 0x1F1FF ; Enclosed Alphanumeric Supplement + 0x1F200, # .. 0x1F2FF ; Enclosed Ideographic Supplement + 0x1F300, # .. 0x1F5FF ; Miscellaneous Symbols and Pictographs + 0x1F600, # .. 0x1F64F ; Emoticons + 0x1F650, # .. 0x1F67F ; Ornamental Dingbats + 0x1F680, # .. 0x1F6FF ; Transport and Map Symbols + 0x1F700, # .. 0x1F77F ; Alchemical Symbols + 0x1F780, # .. 0x1F7FF ; Geometric Shapes Extended + 0x1F800, # .. 0x1F8FF ; Supplemental Arrows-C + 0x1F900, # .. 0x1F9FF ; Supplemental Symbols and Pictographs + 0x1FA00, # .. 0x1FA6F ; Chess Symbols + 0x1FA70, # .. 0x1FAFF ; Symbols and Pictographs Extended-A + 0x1FB00, # .. 0x1FBFF ; Symbols for Legacy Computing + 0x1FC00, # .. 0x1FFFF ; No_Block + 0x20000, # .. 0x2A6DF ; CJK Unified Ideographs Extension B + 0x2A6E0, # .. 0x2A6FF ; No_Block + 0x2A700, # .. 0x2B73F ; CJK Unified Ideographs Extension C + 0x2B740, # .. 0x2B81F ; CJK Unified Ideographs Extension D + 0x2B820, # .. 0x2CEAF ; CJK Unified Ideographs Extension E + 0x2CEB0, # .. 0x2EBEF ; CJK Unified Ideographs Extension F + 0x2EBF0, # .. 0x2EE5F ; CJK Unified Ideographs Extension I + 0x2EE60, # .. 0x2F7FF ; No_Block + 0x2F800, # .. 0x2FA1F ; CJK Compatibility Ideographs Supplement + 0x2FA20, # .. 0x2FFFF ; No_Block + 0x30000, # .. 0x3134F ; CJK Unified Ideographs Extension G + 0x31350, # .. 0x323AF ; CJK Unified Ideographs Extension H + 0x323B0, # .. 0x3347F ; CJK Unified Ideographs Extension J + 0x33480, # .. 0xDFFFF ; No_Block + 0xE0000, # .. 0xE007F ; Tags + 0xE0080, # .. 0xE00FF ; No_Block + 0xE0100, # .. 0xE01EF ; Variation Selectors Supplement + 0xE01F0, # .. 0xEFFFF ; No_Block + 0xF0000, # .. 0xFFFFF ; Supplementary Private Use Area-A + 0x100000, # .. 0x10FFFF ; Supplementary Private Use Area-B +] + +VALUES = [ + "Basic Latin", # 0000..007F + "Latin-1 Supplement", # 0080..00FF + "Latin Extended-A", # 0100..017F + "Latin Extended-B", # 0180..024F + "IPA Extensions", # 0250..02AF + "Spacing Modifier Letters", # 02B0..02FF + "Combining Diacritical Marks", # 0300..036F + "Greek and Coptic", # 0370..03FF + "Cyrillic", # 0400..04FF + "Cyrillic Supplement", # 0500..052F + "Armenian", # 0530..058F + "Hebrew", # 0590..05FF + "Arabic", # 0600..06FF + "Syriac", # 0700..074F + "Arabic Supplement", # 0750..077F + "Thaana", # 0780..07BF + "NKo", # 07C0..07FF + "Samaritan", # 0800..083F + "Mandaic", # 0840..085F + "Syriac Supplement", # 0860..086F + "Arabic Extended-B", # 0870..089F + "Arabic Extended-A", # 08A0..08FF + "Devanagari", # 0900..097F + "Bengali", # 0980..09FF + "Gurmukhi", # 0A00..0A7F + "Gujarati", # 0A80..0AFF + "Oriya", # 0B00..0B7F + "Tamil", # 0B80..0BFF + "Telugu", # 0C00..0C7F + "Kannada", # 0C80..0CFF + "Malayalam", # 0D00..0D7F + "Sinhala", # 0D80..0DFF + "Thai", # 0E00..0E7F + "Lao", # 0E80..0EFF + "Tibetan", # 0F00..0FFF + "Myanmar", # 1000..109F + "Georgian", # 10A0..10FF + "Hangul Jamo", # 1100..11FF + "Ethiopic", # 1200..137F + "Ethiopic Supplement", # 1380..139F + "Cherokee", # 13A0..13FF + "Unified Canadian Aboriginal Syllabics", # 1400..167F + "Ogham", # 1680..169F + "Runic", # 16A0..16FF + "Tagalog", # 1700..171F + "Hanunoo", # 1720..173F + "Buhid", # 1740..175F + "Tagbanwa", # 1760..177F + "Khmer", # 1780..17FF + "Mongolian", # 1800..18AF + "Unified Canadian Aboriginal Syllabics Extended", # 18B0..18FF + "Limbu", # 1900..194F + "Tai Le", # 1950..197F + "New Tai Lue", # 1980..19DF + "Khmer Symbols", # 19E0..19FF + "Buginese", # 1A00..1A1F + "Tai Tham", # 1A20..1AAF + "Combining Diacritical Marks Extended", # 1AB0..1AFF + "Balinese", # 1B00..1B7F + "Sundanese", # 1B80..1BBF + "Batak", # 1BC0..1BFF + "Lepcha", # 1C00..1C4F + "Ol Chiki", # 1C50..1C7F + "Cyrillic Extended-C", # 1C80..1C8F + "Georgian Extended", # 1C90..1CBF + "Sundanese Supplement", # 1CC0..1CCF + "Vedic Extensions", # 1CD0..1CFF + "Phonetic Extensions", # 1D00..1D7F + "Phonetic Extensions Supplement", # 1D80..1DBF + "Combining Diacritical Marks Supplement", # 1DC0..1DFF + "Latin Extended Additional", # 1E00..1EFF + "Greek Extended", # 1F00..1FFF + "General Punctuation", # 2000..206F + "Superscripts and Subscripts", # 2070..209F + "Currency Symbols", # 20A0..20CF + "Combining Diacritical Marks for Symbols", # 20D0..20FF + "Letterlike Symbols", # 2100..214F + "Number Forms", # 2150..218F + "Arrows", # 2190..21FF + "Mathematical Operators", # 2200..22FF + "Miscellaneous Technical", # 2300..23FF + "Control Pictures", # 2400..243F + "Optical Character Recognition", # 2440..245F + "Enclosed Alphanumerics", # 2460..24FF + "Box Drawing", # 2500..257F + "Block Elements", # 2580..259F + "Geometric Shapes", # 25A0..25FF + "Miscellaneous Symbols", # 2600..26FF + "Dingbats", # 2700..27BF + "Miscellaneous Mathematical Symbols-A", # 27C0..27EF + "Supplemental Arrows-A", # 27F0..27FF + "Braille Patterns", # 2800..28FF + "Supplemental Arrows-B", # 2900..297F + "Miscellaneous Mathematical Symbols-B", # 2980..29FF + "Supplemental Mathematical Operators", # 2A00..2AFF + "Miscellaneous Symbols and Arrows", # 2B00..2BFF + "Glagolitic", # 2C00..2C5F + "Latin Extended-C", # 2C60..2C7F + "Coptic", # 2C80..2CFF + "Georgian Supplement", # 2D00..2D2F + "Tifinagh", # 2D30..2D7F + "Ethiopic Extended", # 2D80..2DDF + "Cyrillic Extended-A", # 2DE0..2DFF + "Supplemental Punctuation", # 2E00..2E7F + "CJK Radicals Supplement", # 2E80..2EFF + "Kangxi Radicals", # 2F00..2FDF + "No_Block", # 2FE0..2FEF + "Ideographic Description Characters", # 2FF0..2FFF + "CJK Symbols and Punctuation", # 3000..303F + "Hiragana", # 3040..309F + "Katakana", # 30A0..30FF + "Bopomofo", # 3100..312F + "Hangul Compatibility Jamo", # 3130..318F + "Kanbun", # 3190..319F + "Bopomofo Extended", # 31A0..31BF + "CJK Strokes", # 31C0..31EF + "Katakana Phonetic Extensions", # 31F0..31FF + "Enclosed CJK Letters and Months", # 3200..32FF + "CJK Compatibility", # 3300..33FF + "CJK Unified Ideographs Extension A", # 3400..4DBF + "Yijing Hexagram Symbols", # 4DC0..4DFF + "CJK Unified Ideographs", # 4E00..9FFF + "Yi Syllables", # A000..A48F + "Yi Radicals", # A490..A4CF + "Lisu", # A4D0..A4FF + "Vai", # A500..A63F + "Cyrillic Extended-B", # A640..A69F + "Bamum", # A6A0..A6FF + "Modifier Tone Letters", # A700..A71F + "Latin Extended-D", # A720..A7FF + "Syloti Nagri", # A800..A82F + "Common Indic Number Forms", # A830..A83F + "Phags-pa", # A840..A87F + "Saurashtra", # A880..A8DF + "Devanagari Extended", # A8E0..A8FF + "Kayah Li", # A900..A92F + "Rejang", # A930..A95F + "Hangul Jamo Extended-A", # A960..A97F + "Javanese", # A980..A9DF + "Myanmar Extended-B", # A9E0..A9FF + "Cham", # AA00..AA5F + "Myanmar Extended-A", # AA60..AA7F + "Tai Viet", # AA80..AADF + "Meetei Mayek Extensions", # AAE0..AAFF + "Ethiopic Extended-A", # AB00..AB2F + "Latin Extended-E", # AB30..AB6F + "Cherokee Supplement", # AB70..ABBF + "Meetei Mayek", # ABC0..ABFF + "Hangul Syllables", # AC00..D7AF + "Hangul Jamo Extended-B", # D7B0..D7FF + "High Surrogates", # D800..DB7F + "High Private Use Surrogates", # DB80..DBFF + "Low Surrogates", # DC00..DFFF + "Private Use Area", # E000..F8FF + "CJK Compatibility Ideographs", # F900..FAFF + "Alphabetic Presentation Forms", # FB00..FB4F + "Arabic Presentation Forms-A", # FB50..FDFF + "Variation Selectors", # FE00..FE0F + "Vertical Forms", # FE10..FE1F + "Combining Half Marks", # FE20..FE2F + "CJK Compatibility Forms", # FE30..FE4F + "Small Form Variants", # FE50..FE6F + "Arabic Presentation Forms-B", # FE70..FEFF + "Halfwidth and Fullwidth Forms", # FF00..FFEF + "Specials", # FFF0..FFFF + "Linear B Syllabary", # 10000..1007F + "Linear B Ideograms", # 10080..100FF + "Aegean Numbers", # 10100..1013F + "Ancient Greek Numbers", # 10140..1018F + "Ancient Symbols", # 10190..101CF + "Phaistos Disc", # 101D0..101FF + "No_Block", # 10200..1027F + "Lycian", # 10280..1029F + "Carian", # 102A0..102DF + "Coptic Epact Numbers", # 102E0..102FF + "Old Italic", # 10300..1032F + "Gothic", # 10330..1034F + "Old Permic", # 10350..1037F + "Ugaritic", # 10380..1039F + "Old Persian", # 103A0..103DF + "No_Block", # 103E0..103FF + "Deseret", # 10400..1044F + "Shavian", # 10450..1047F + "Osmanya", # 10480..104AF + "Osage", # 104B0..104FF + "Elbasan", # 10500..1052F + "Caucasian Albanian", # 10530..1056F + "Vithkuqi", # 10570..105BF + "Todhri", # 105C0..105FF + "Linear A", # 10600..1077F + "Latin Extended-F", # 10780..107BF + "No_Block", # 107C0..107FF + "Cypriot Syllabary", # 10800..1083F + "Imperial Aramaic", # 10840..1085F + "Palmyrene", # 10860..1087F + "Nabataean", # 10880..108AF + "No_Block", # 108B0..108DF + "Hatran", # 108E0..108FF + "Phoenician", # 10900..1091F + "Lydian", # 10920..1093F + "Sidetic", # 10940..1095F + "No_Block", # 10960..1097F + "Meroitic Hieroglyphs", # 10980..1099F + "Meroitic Cursive", # 109A0..109FF + "Kharoshthi", # 10A00..10A5F + "Old South Arabian", # 10A60..10A7F + "Old North Arabian", # 10A80..10A9F + "No_Block", # 10AA0..10ABF + "Manichaean", # 10AC0..10AFF + "Avestan", # 10B00..10B3F + "Inscriptional Parthian", # 10B40..10B5F + "Inscriptional Pahlavi", # 10B60..10B7F + "Psalter Pahlavi", # 10B80..10BAF + "No_Block", # 10BB0..10BFF + "Old Turkic", # 10C00..10C4F + "No_Block", # 10C50..10C7F + "Old Hungarian", # 10C80..10CFF + "Hanifi Rohingya", # 10D00..10D3F + "Garay", # 10D40..10D8F + "No_Block", # 10D90..10E5F + "Rumi Numeral Symbols", # 10E60..10E7F + "Yezidi", # 10E80..10EBF + "Arabic Extended-C", # 10EC0..10EFF + "Old Sogdian", # 10F00..10F2F + "Sogdian", # 10F30..10F6F + "Old Uyghur", # 10F70..10FAF + "Chorasmian", # 10FB0..10FDF + "Elymaic", # 10FE0..10FFF + "Brahmi", # 11000..1107F + "Kaithi", # 11080..110CF + "Sora Sompeng", # 110D0..110FF + "Chakma", # 11100..1114F + "Mahajani", # 11150..1117F + "Sharada", # 11180..111DF + "Sinhala Archaic Numbers", # 111E0..111FF + "Khojki", # 11200..1124F + "No_Block", # 11250..1127F + "Multani", # 11280..112AF + "Khudawadi", # 112B0..112FF + "Grantha", # 11300..1137F + "Tulu-Tigalari", # 11380..113FF + "Newa", # 11400..1147F + "Tirhuta", # 11480..114DF + "No_Block", # 114E0..1157F + "Siddham", # 11580..115FF + "Modi", # 11600..1165F + "Mongolian Supplement", # 11660..1167F + "Takri", # 11680..116CF + "Myanmar Extended-C", # 116D0..116FF + "Ahom", # 11700..1174F + "No_Block", # 11750..117FF + "Dogra", # 11800..1184F + "No_Block", # 11850..1189F + "Warang Citi", # 118A0..118FF + "Dives Akuru", # 11900..1195F + "No_Block", # 11960..1199F + "Nandinagari", # 119A0..119FF + "Zanabazar Square", # 11A00..11A4F + "Soyombo", # 11A50..11AAF + "Unified Canadian Aboriginal Syllabics Extended-A", # 11AB0..11ABF + "Pau Cin Hau", # 11AC0..11AFF + "Devanagari Extended-A", # 11B00..11B5F + "Sharada Supplement", # 11B60..11B7F + "No_Block", # 11B80..11BBF + "Sunuwar", # 11BC0..11BFF + "Bhaiksuki", # 11C00..11C6F + "Marchen", # 11C70..11CBF + "No_Block", # 11CC0..11CFF + "Masaram Gondi", # 11D00..11D5F + "Gunjala Gondi", # 11D60..11DAF + "Tolong Siki", # 11DB0..11DEF + "No_Block", # 11DF0..11EDF + "Makasar", # 11EE0..11EFF + "Kawi", # 11F00..11F5F + "No_Block", # 11F60..11FAF + "Lisu Supplement", # 11FB0..11FBF + "Tamil Supplement", # 11FC0..11FFF + "Cuneiform", # 12000..123FF + "Cuneiform Numbers and Punctuation", # 12400..1247F + "Early Dynastic Cuneiform", # 12480..1254F + "No_Block", # 12550..12F8F + "Cypro-Minoan", # 12F90..12FFF + "Egyptian Hieroglyphs", # 13000..1342F + "Egyptian Hieroglyph Format Controls", # 13430..1345F + "Egyptian Hieroglyphs Extended-A", # 13460..143FF + "Anatolian Hieroglyphs", # 14400..1467F + "No_Block", # 14680..160FF + "Gurung Khema", # 16100..1613F + "No_Block", # 16140..167FF + "Bamum Supplement", # 16800..16A3F + "Mro", # 16A40..16A6F + "Tangsa", # 16A70..16ACF + "Bassa Vah", # 16AD0..16AFF + "Pahawh Hmong", # 16B00..16B8F + "No_Block", # 16B90..16D3F + "Kirat Rai", # 16D40..16D7F + "No_Block", # 16D80..16E3F + "Medefaidrin", # 16E40..16E9F + "Beria Erfe", # 16EA0..16EDF + "No_Block", # 16EE0..16EFF + "Miao", # 16F00..16F9F + "No_Block", # 16FA0..16FDF + "Ideographic Symbols and Punctuation", # 16FE0..16FFF + "Tangut", # 17000..187FF + "Tangut Components", # 18800..18AFF + "Khitan Small Script", # 18B00..18CFF + "Tangut Supplement", # 18D00..18D7F + "Tangut Components Supplement", # 18D80..18DFF + "No_Block", # 18E00..1AFEF + "Kana Extended-B", # 1AFF0..1AFFF + "Kana Supplement", # 1B000..1B0FF + "Kana Extended-A", # 1B100..1B12F + "Small Kana Extension", # 1B130..1B16F + "Nushu", # 1B170..1B2FF + "No_Block", # 1B300..1BBFF + "Duployan", # 1BC00..1BC9F + "Shorthand Format Controls", # 1BCA0..1BCAF + "No_Block", # 1BCB0..1CBFF + "Symbols for Legacy Computing Supplement", # 1CC00..1CEBF + "Miscellaneous Symbols Supplement", # 1CEC0..1CEFF + "Znamenny Musical Notation", # 1CF00..1CFCF + "No_Block", # 1CFD0..1CFFF + "Byzantine Musical Symbols", # 1D000..1D0FF + "Musical Symbols", # 1D100..1D1FF + "Ancient Greek Musical Notation", # 1D200..1D24F + "No_Block", # 1D250..1D2BF + "Kaktovik Numerals", # 1D2C0..1D2DF + "Mayan Numerals", # 1D2E0..1D2FF + "Tai Xuan Jing Symbols", # 1D300..1D35F + "Counting Rod Numerals", # 1D360..1D37F + "No_Block", # 1D380..1D3FF + "Mathematical Alphanumeric Symbols", # 1D400..1D7FF + "Sutton SignWriting", # 1D800..1DAAF + "No_Block", # 1DAB0..1DEFF + "Latin Extended-G", # 1DF00..1DFFF + "Glagolitic Supplement", # 1E000..1E02F + "Cyrillic Extended-D", # 1E030..1E08F + "No_Block", # 1E090..1E0FF + "Nyiakeng Puachue Hmong", # 1E100..1E14F + "No_Block", # 1E150..1E28F + "Toto", # 1E290..1E2BF + "Wancho", # 1E2C0..1E2FF + "No_Block", # 1E300..1E4CF + "Nag Mundari", # 1E4D0..1E4FF + "No_Block", # 1E500..1E5CF + "Ol Onal", # 1E5D0..1E5FF + "No_Block", # 1E600..1E6BF + "Tai Yo", # 1E6C0..1E6FF + "No_Block", # 1E700..1E7DF + "Ethiopic Extended-B", # 1E7E0..1E7FF + "Mende Kikakui", # 1E800..1E8DF + "No_Block", # 1E8E0..1E8FF + "Adlam", # 1E900..1E95F + "No_Block", # 1E960..1EC6F + "Indic Siyaq Numbers", # 1EC70..1ECBF + "No_Block", # 1ECC0..1ECFF + "Ottoman Siyaq Numbers", # 1ED00..1ED4F + "No_Block", # 1ED50..1EDFF + "Arabic Mathematical Alphabetic Symbols", # 1EE00..1EEFF + "No_Block", # 1EF00..1EFFF + "Mahjong Tiles", # 1F000..1F02F + "Domino Tiles", # 1F030..1F09F + "Playing Cards", # 1F0A0..1F0FF + "Enclosed Alphanumeric Supplement", # 1F100..1F1FF + "Enclosed Ideographic Supplement", # 1F200..1F2FF + "Miscellaneous Symbols and Pictographs", # 1F300..1F5FF + "Emoticons", # 1F600..1F64F + "Ornamental Dingbats", # 1F650..1F67F + "Transport and Map Symbols", # 1F680..1F6FF + "Alchemical Symbols", # 1F700..1F77F + "Geometric Shapes Extended", # 1F780..1F7FF + "Supplemental Arrows-C", # 1F800..1F8FF + "Supplemental Symbols and Pictographs", # 1F900..1F9FF + "Chess Symbols", # 1FA00..1FA6F + "Symbols and Pictographs Extended-A", # 1FA70..1FAFF + "Symbols for Legacy Computing", # 1FB00..1FBFF + "No_Block", # 1FC00..1FFFF + "CJK Unified Ideographs Extension B", # 20000..2A6DF + "No_Block", # 2A6E0..2A6FF + "CJK Unified Ideographs Extension C", # 2A700..2B73F + "CJK Unified Ideographs Extension D", # 2B740..2B81F + "CJK Unified Ideographs Extension E", # 2B820..2CEAF + "CJK Unified Ideographs Extension F", # 2CEB0..2EBEF + "CJK Unified Ideographs Extension I", # 2EBF0..2EE5F + "No_Block", # 2EE60..2F7FF + "CJK Compatibility Ideographs Supplement", # 2F800..2FA1F + "No_Block", # 2FA20..2FFFF + "CJK Unified Ideographs Extension G", # 30000..3134F + "CJK Unified Ideographs Extension H", # 31350..323AF + "CJK Unified Ideographs Extension J", # 323B0..3347F + "No_Block", # 33480..DFFFF + "Tags", # E0000..E007F + "No_Block", # E0080..E00FF + "Variation Selectors Supplement", # E0100..E01EF + "No_Block", # E01F0..EFFFF + "Supplementary Private Use Area-A", # F0000..FFFFF + "Supplementary Private Use Area-B", # 100000..10FFFF +] diff --git a/.venv/lib/python3.11/site-packages/fontTools/unicodedata/Mirrored.py b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/Mirrored.py new file mode 100644 index 0000000000000000000000000000000000000000..75b51a901230cce5d3886a6497dbb13da51c86b7 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/Mirrored.py @@ -0,0 +1,446 @@ +# -*- coding: utf-8 -*- +# +# NOTE: The mappings in this file were generated from the command line: +# cat BidiMirroring.txt | grep "^[0-9A-F]" | sed "s/;//" | awk '{print " 0x"$1": 0x"$2","}' +# +# Source: http://www.unicode.org/Public/UNIDATA/BidiMirroring.txt +# License: http://unicode.org/copyright.html#License +# +# BidiMirroring-16.0.0.txt +# Date: 2024-01-30 +# © 2024 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see https://www.unicode.org/reports/tr44/ +MIRRORED = { + 0x0028: 0x0029, + 0x0029: 0x0028, + 0x003C: 0x003E, + 0x003E: 0x003C, + 0x005B: 0x005D, + 0x005D: 0x005B, + 0x007B: 0x007D, + 0x007D: 0x007B, + 0x00AB: 0x00BB, + 0x00BB: 0x00AB, + 0x0F3A: 0x0F3B, + 0x0F3B: 0x0F3A, + 0x0F3C: 0x0F3D, + 0x0F3D: 0x0F3C, + 0x169B: 0x169C, + 0x169C: 0x169B, + 0x2039: 0x203A, + 0x203A: 0x2039, + 0x2045: 0x2046, + 0x2046: 0x2045, + 0x207D: 0x207E, + 0x207E: 0x207D, + 0x208D: 0x208E, + 0x208E: 0x208D, + 0x2208: 0x220B, + 0x2209: 0x220C, + 0x220A: 0x220D, + 0x220B: 0x2208, + 0x220C: 0x2209, + 0x220D: 0x220A, + 0x2215: 0x29F5, + 0x221F: 0x2BFE, + 0x2220: 0x29A3, + 0x2221: 0x299B, + 0x2222: 0x29A0, + 0x2224: 0x2AEE, + 0x223C: 0x223D, + 0x223D: 0x223C, + 0x2243: 0x22CD, + 0x2245: 0x224C, + 0x224C: 0x2245, + 0x2252: 0x2253, + 0x2253: 0x2252, + 0x2254: 0x2255, + 0x2255: 0x2254, + 0x2264: 0x2265, + 0x2265: 0x2264, + 0x2266: 0x2267, + 0x2267: 0x2266, + 0x2268: 0x2269, + 0x2269: 0x2268, + 0x226A: 0x226B, + 0x226B: 0x226A, + 0x226E: 0x226F, + 0x226F: 0x226E, + 0x2270: 0x2271, + 0x2271: 0x2270, + 0x2272: 0x2273, + 0x2273: 0x2272, + 0x2274: 0x2275, + 0x2275: 0x2274, + 0x2276: 0x2277, + 0x2277: 0x2276, + 0x2278: 0x2279, + 0x2279: 0x2278, + 0x227A: 0x227B, + 0x227B: 0x227A, + 0x227C: 0x227D, + 0x227D: 0x227C, + 0x227E: 0x227F, + 0x227F: 0x227E, + 0x2280: 0x2281, + 0x2281: 0x2280, + 0x2282: 0x2283, + 0x2283: 0x2282, + 0x2284: 0x2285, + 0x2285: 0x2284, + 0x2286: 0x2287, + 0x2287: 0x2286, + 0x2288: 0x2289, + 0x2289: 0x2288, + 0x228A: 0x228B, + 0x228B: 0x228A, + 0x228F: 0x2290, + 0x2290: 0x228F, + 0x2291: 0x2292, + 0x2292: 0x2291, + 0x2298: 0x29B8, + 0x22A2: 0x22A3, + 0x22A3: 0x22A2, + 0x22A6: 0x2ADE, + 0x22A8: 0x2AE4, + 0x22A9: 0x2AE3, + 0x22AB: 0x2AE5, + 0x22B0: 0x22B1, + 0x22B1: 0x22B0, + 0x22B2: 0x22B3, + 0x22B3: 0x22B2, + 0x22B4: 0x22B5, + 0x22B5: 0x22B4, + 0x22B6: 0x22B7, + 0x22B7: 0x22B6, + 0x22B8: 0x27DC, + 0x22C9: 0x22CA, + 0x22CA: 0x22C9, + 0x22CB: 0x22CC, + 0x22CC: 0x22CB, + 0x22CD: 0x2243, + 0x22D0: 0x22D1, + 0x22D1: 0x22D0, + 0x22D6: 0x22D7, + 0x22D7: 0x22D6, + 0x22D8: 0x22D9, + 0x22D9: 0x22D8, + 0x22DA: 0x22DB, + 0x22DB: 0x22DA, + 0x22DC: 0x22DD, + 0x22DD: 0x22DC, + 0x22DE: 0x22DF, + 0x22DF: 0x22DE, + 0x22E0: 0x22E1, + 0x22E1: 0x22E0, + 0x22E2: 0x22E3, + 0x22E3: 0x22E2, + 0x22E4: 0x22E5, + 0x22E5: 0x22E4, + 0x22E6: 0x22E7, + 0x22E7: 0x22E6, + 0x22E8: 0x22E9, + 0x22E9: 0x22E8, + 0x22EA: 0x22EB, + 0x22EB: 0x22EA, + 0x22EC: 0x22ED, + 0x22ED: 0x22EC, + 0x22F0: 0x22F1, + 0x22F1: 0x22F0, + 0x22F2: 0x22FA, + 0x22F3: 0x22FB, + 0x22F4: 0x22FC, + 0x22F6: 0x22FD, + 0x22F7: 0x22FE, + 0x22FA: 0x22F2, + 0x22FB: 0x22F3, + 0x22FC: 0x22F4, + 0x22FD: 0x22F6, + 0x22FE: 0x22F7, + 0x2308: 0x2309, + 0x2309: 0x2308, + 0x230A: 0x230B, + 0x230B: 0x230A, + 0x2329: 0x232A, + 0x232A: 0x2329, + 0x2768: 0x2769, + 0x2769: 0x2768, + 0x276A: 0x276B, + 0x276B: 0x276A, + 0x276C: 0x276D, + 0x276D: 0x276C, + 0x276E: 0x276F, + 0x276F: 0x276E, + 0x2770: 0x2771, + 0x2771: 0x2770, + 0x2772: 0x2773, + 0x2773: 0x2772, + 0x2774: 0x2775, + 0x2775: 0x2774, + 0x27C3: 0x27C4, + 0x27C4: 0x27C3, + 0x27C5: 0x27C6, + 0x27C6: 0x27C5, + 0x27C8: 0x27C9, + 0x27C9: 0x27C8, + 0x27CB: 0x27CD, + 0x27CD: 0x27CB, + 0x27D5: 0x27D6, + 0x27D6: 0x27D5, + 0x27DC: 0x22B8, + 0x27DD: 0x27DE, + 0x27DE: 0x27DD, + 0x27E2: 0x27E3, + 0x27E3: 0x27E2, + 0x27E4: 0x27E5, + 0x27E5: 0x27E4, + 0x27E6: 0x27E7, + 0x27E7: 0x27E6, + 0x27E8: 0x27E9, + 0x27E9: 0x27E8, + 0x27EA: 0x27EB, + 0x27EB: 0x27EA, + 0x27EC: 0x27ED, + 0x27ED: 0x27EC, + 0x27EE: 0x27EF, + 0x27EF: 0x27EE, + 0x2983: 0x2984, + 0x2984: 0x2983, + 0x2985: 0x2986, + 0x2986: 0x2985, + 0x2987: 0x2988, + 0x2988: 0x2987, + 0x2989: 0x298A, + 0x298A: 0x2989, + 0x298B: 0x298C, + 0x298C: 0x298B, + 0x298D: 0x2990, + 0x298E: 0x298F, + 0x298F: 0x298E, + 0x2990: 0x298D, + 0x2991: 0x2992, + 0x2992: 0x2991, + 0x2993: 0x2994, + 0x2994: 0x2993, + 0x2995: 0x2996, + 0x2996: 0x2995, + 0x2997: 0x2998, + 0x2998: 0x2997, + 0x299B: 0x2221, + 0x29A0: 0x2222, + 0x29A3: 0x2220, + 0x29A4: 0x29A5, + 0x29A5: 0x29A4, + 0x29A8: 0x29A9, + 0x29A9: 0x29A8, + 0x29AA: 0x29AB, + 0x29AB: 0x29AA, + 0x29AC: 0x29AD, + 0x29AD: 0x29AC, + 0x29AE: 0x29AF, + 0x29AF: 0x29AE, + 0x29B8: 0x2298, + 0x29C0: 0x29C1, + 0x29C1: 0x29C0, + 0x29C4: 0x29C5, + 0x29C5: 0x29C4, + 0x29CF: 0x29D0, + 0x29D0: 0x29CF, + 0x29D1: 0x29D2, + 0x29D2: 0x29D1, + 0x29D4: 0x29D5, + 0x29D5: 0x29D4, + 0x29D8: 0x29D9, + 0x29D9: 0x29D8, + 0x29DA: 0x29DB, + 0x29DB: 0x29DA, + 0x29E8: 0x29E9, + 0x29E9: 0x29E8, + 0x29F5: 0x2215, + 0x29F8: 0x29F9, + 0x29F9: 0x29F8, + 0x29FC: 0x29FD, + 0x29FD: 0x29FC, + 0x2A2B: 0x2A2C, + 0x2A2C: 0x2A2B, + 0x2A2D: 0x2A2E, + 0x2A2E: 0x2A2D, + 0x2A34: 0x2A35, + 0x2A35: 0x2A34, + 0x2A3C: 0x2A3D, + 0x2A3D: 0x2A3C, + 0x2A64: 0x2A65, + 0x2A65: 0x2A64, + 0x2A79: 0x2A7A, + 0x2A7A: 0x2A79, + 0x2A7B: 0x2A7C, + 0x2A7C: 0x2A7B, + 0x2A7D: 0x2A7E, + 0x2A7E: 0x2A7D, + 0x2A7F: 0x2A80, + 0x2A80: 0x2A7F, + 0x2A81: 0x2A82, + 0x2A82: 0x2A81, + 0x2A83: 0x2A84, + 0x2A84: 0x2A83, + 0x2A85: 0x2A86, + 0x2A86: 0x2A85, + 0x2A87: 0x2A88, + 0x2A88: 0x2A87, + 0x2A89: 0x2A8A, + 0x2A8A: 0x2A89, + 0x2A8B: 0x2A8C, + 0x2A8C: 0x2A8B, + 0x2A8D: 0x2A8E, + 0x2A8E: 0x2A8D, + 0x2A8F: 0x2A90, + 0x2A90: 0x2A8F, + 0x2A91: 0x2A92, + 0x2A92: 0x2A91, + 0x2A93: 0x2A94, + 0x2A94: 0x2A93, + 0x2A95: 0x2A96, + 0x2A96: 0x2A95, + 0x2A97: 0x2A98, + 0x2A98: 0x2A97, + 0x2A99: 0x2A9A, + 0x2A9A: 0x2A99, + 0x2A9B: 0x2A9C, + 0x2A9C: 0x2A9B, + 0x2A9D: 0x2A9E, + 0x2A9E: 0x2A9D, + 0x2A9F: 0x2AA0, + 0x2AA0: 0x2A9F, + 0x2AA1: 0x2AA2, + 0x2AA2: 0x2AA1, + 0x2AA6: 0x2AA7, + 0x2AA7: 0x2AA6, + 0x2AA8: 0x2AA9, + 0x2AA9: 0x2AA8, + 0x2AAA: 0x2AAB, + 0x2AAB: 0x2AAA, + 0x2AAC: 0x2AAD, + 0x2AAD: 0x2AAC, + 0x2AAF: 0x2AB0, + 0x2AB0: 0x2AAF, + 0x2AB1: 0x2AB2, + 0x2AB2: 0x2AB1, + 0x2AB3: 0x2AB4, + 0x2AB4: 0x2AB3, + 0x2AB5: 0x2AB6, + 0x2AB6: 0x2AB5, + 0x2AB7: 0x2AB8, + 0x2AB8: 0x2AB7, + 0x2AB9: 0x2ABA, + 0x2ABA: 0x2AB9, + 0x2ABB: 0x2ABC, + 0x2ABC: 0x2ABB, + 0x2ABD: 0x2ABE, + 0x2ABE: 0x2ABD, + 0x2ABF: 0x2AC0, + 0x2AC0: 0x2ABF, + 0x2AC1: 0x2AC2, + 0x2AC2: 0x2AC1, + 0x2AC3: 0x2AC4, + 0x2AC4: 0x2AC3, + 0x2AC5: 0x2AC6, + 0x2AC6: 0x2AC5, + 0x2AC7: 0x2AC8, + 0x2AC8: 0x2AC7, + 0x2AC9: 0x2ACA, + 0x2ACA: 0x2AC9, + 0x2ACB: 0x2ACC, + 0x2ACC: 0x2ACB, + 0x2ACD: 0x2ACE, + 0x2ACE: 0x2ACD, + 0x2ACF: 0x2AD0, + 0x2AD0: 0x2ACF, + 0x2AD1: 0x2AD2, + 0x2AD2: 0x2AD1, + 0x2AD3: 0x2AD4, + 0x2AD4: 0x2AD3, + 0x2AD5: 0x2AD6, + 0x2AD6: 0x2AD5, + 0x2ADE: 0x22A6, + 0x2AE3: 0x22A9, + 0x2AE4: 0x22A8, + 0x2AE5: 0x22AB, + 0x2AEC: 0x2AED, + 0x2AED: 0x2AEC, + 0x2AEE: 0x2224, + 0x2AF7: 0x2AF8, + 0x2AF8: 0x2AF7, + 0x2AF9: 0x2AFA, + 0x2AFA: 0x2AF9, + 0x2BFE: 0x221F, + 0x2E02: 0x2E03, + 0x2E03: 0x2E02, + 0x2E04: 0x2E05, + 0x2E05: 0x2E04, + 0x2E09: 0x2E0A, + 0x2E0A: 0x2E09, + 0x2E0C: 0x2E0D, + 0x2E0D: 0x2E0C, + 0x2E1C: 0x2E1D, + 0x2E1D: 0x2E1C, + 0x2E20: 0x2E21, + 0x2E21: 0x2E20, + 0x2E22: 0x2E23, + 0x2E23: 0x2E22, + 0x2E24: 0x2E25, + 0x2E25: 0x2E24, + 0x2E26: 0x2E27, + 0x2E27: 0x2E26, + 0x2E28: 0x2E29, + 0x2E29: 0x2E28, + 0x2E55: 0x2E56, + 0x2E56: 0x2E55, + 0x2E57: 0x2E58, + 0x2E58: 0x2E57, + 0x2E59: 0x2E5A, + 0x2E5A: 0x2E59, + 0x2E5B: 0x2E5C, + 0x2E5C: 0x2E5B, + 0x3008: 0x3009, + 0x3009: 0x3008, + 0x300A: 0x300B, + 0x300B: 0x300A, + 0x300C: 0x300D, + 0x300D: 0x300C, + 0x300E: 0x300F, + 0x300F: 0x300E, + 0x3010: 0x3011, + 0x3011: 0x3010, + 0x3014: 0x3015, + 0x3015: 0x3014, + 0x3016: 0x3017, + 0x3017: 0x3016, + 0x3018: 0x3019, + 0x3019: 0x3018, + 0x301A: 0x301B, + 0x301B: 0x301A, + 0xFE59: 0xFE5A, + 0xFE5A: 0xFE59, + 0xFE5B: 0xFE5C, + 0xFE5C: 0xFE5B, + 0xFE5D: 0xFE5E, + 0xFE5E: 0xFE5D, + 0xFE64: 0xFE65, + 0xFE65: 0xFE64, + 0xFF08: 0xFF09, + 0xFF09: 0xFF08, + 0xFF1C: 0xFF1E, + 0xFF1E: 0xFF1C, + 0xFF3B: 0xFF3D, + 0xFF3D: 0xFF3B, + 0xFF5B: 0xFF5D, + 0xFF5D: 0xFF5B, + 0xFF5F: 0xFF60, + 0xFF60: 0xFF5F, + 0xFF62: 0xFF63, + 0xFF63: 0xFF62, +} diff --git a/.venv/lib/python3.11/site-packages/fontTools/unicodedata/OTTags.py b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/OTTags.py new file mode 100644 index 0000000000000000000000000000000000000000..859a3bcdcf29bcdda827edad766ffeab2f0b636a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/OTTags.py @@ -0,0 +1,50 @@ +# Data updated to OpenType 1.8.2 as of January 2018. + +# Complete list of OpenType script tags at: +# https://www.microsoft.com/typography/otspec/scripttags.htm + +# Most of the script tags are the same as the ISO 15924 tag but lowercased, +# so we only have to handle the exceptional cases: +# - KATAKANA and HIRAGANA both map to 'kana'; +# - spaces at the end are preserved, unlike ISO 15924; +# - we map special script codes for Inherited, Common and Unknown to DFLT. + +DEFAULT_SCRIPT = "DFLT" + +SCRIPT_ALIASES = { + "jamo": "hang", +} + +SCRIPT_EXCEPTIONS = { + "Hira": "kana", + "Hrkt": "kana", + "Laoo": "lao ", + "Yiii": "yi ", + "Nkoo": "nko ", + "Vaii": "vai ", + "Zmth": "math", + "Zinh": DEFAULT_SCRIPT, + "Zyyy": DEFAULT_SCRIPT, + "Zzzz": DEFAULT_SCRIPT, +} + +SCRIPT_EXCEPTIONS_REVERSED = { + "math": "Zmth", +} + +NEW_SCRIPT_TAGS = { + "Beng": ("bng2",), + "Deva": ("dev2",), + "Gujr": ("gjr2",), + "Guru": ("gur2",), + "Knda": ("knd2",), + "Mlym": ("mlm2",), + "Orya": ("ory2",), + "Taml": ("tml2",), + "Telu": ("tel2",), + "Mymr": ("mym2",), +} + +NEW_SCRIPT_TAGS_REVERSED = { + value: key for key, values in NEW_SCRIPT_TAGS.items() for value in values +} diff --git a/.venv/lib/python3.11/site-packages/fontTools/unicodedata/ScriptExtensions.py b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/ScriptExtensions.py new file mode 100644 index 0000000000000000000000000000000000000000..ec2dbac16fa45de8aec3f05a0f5dead90555b222 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/ScriptExtensions.py @@ -0,0 +1,832 @@ +# -*- coding: utf-8 -*- +# +# NOTE: This file was auto-generated with MetaTools/buildUCD.py. +# Source: https://unicode.org/Public/UNIDATA/ScriptExtensions.txt +# License: http://unicode.org/copyright.html#License +# +# ScriptExtensions-17.0.0.txt +# Date: 2025-08-01, 21:42:00 GMT +# © 2025 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see https://www.unicode.org/reports/tr44/ +# +# The Script_Extensions property indicates which characters are commonly used +# with more than one script, but with a limited number of scripts. +# For each code point, there is one or more property values. Each such value is a Script property value. +# For more information, see: +# UAX #24, Unicode Script Property: https://www.unicode.org/reports/tr24/ +# Especially the sections: +# https://www.unicode.org/reports/tr24/#Assignment_Script_Values +# https://www.unicode.org/reports/tr24/#Assignment_ScriptX_Values +# +# Each Script_Extensions value in this file consists of a set +# of one or more abbreviated Script property values. The ordering of the +# values in that set is not material, but for stability in presentation +# it is given here as alphabetical. +# +# All code points not explicitly listed for Script_Extensions +# have as their value the corresponding Script property value. +# +# @missing: 0000..10FFFF; <script> + +RANGES = [ + 0x0000, # .. 0x00B6 ; None + 0x00B7, # .. 0x00B7 ; {'Avst', 'Cari', 'Copt', 'Dupl', 'Elba', 'Geor', 'Glag', 'Gong', 'Goth', 'Grek', 'Hani', 'Latn', 'Lydi', 'Mahj', 'Perm', 'Shaw'} + 0x00B8, # .. 0x02BB ; None + 0x02BC, # .. 0x02BC ; {'Beng', 'Cyrl', 'Deva', 'Latn', 'Lisu', 'Thai', 'Toto'} + 0x02BD, # .. 0x02C6 ; None + 0x02C7, # .. 0x02C7 ; {'Bopo', 'Latn'} + 0x02C8, # .. 0x02C8 ; None + 0x02C9, # .. 0x02CB ; {'Bopo', 'Latn'} + 0x02CC, # .. 0x02CC ; None + 0x02CD, # .. 0x02CD ; {'Latn', 'Lisu'} + 0x02CE, # .. 0x02D6 ; None + 0x02D7, # .. 0x02D7 ; {'Latn', 'Thai'} + 0x02D8, # .. 0x02D8 ; None + 0x02D9, # .. 0x02D9 ; {'Bopo', 'Latn'} + 0x02DA, # .. 0x02FF ; None + 0x0300, # .. 0x0300 ; {'Cher', 'Copt', 'Cyrl', 'Grek', 'Latn', 'Perm', 'Sunu', 'Tale'} + 0x0301, # .. 0x0301 ; {'Cher', 'Cyrl', 'Grek', 'Latn', 'Osge', 'Sunu', 'Tale', 'Todr'} + 0x0302, # .. 0x0302 ; {'Cher', 'Cyrl', 'Latn', 'Tfng'} + 0x0303, # .. 0x0303 ; {'Glag', 'Latn', 'Sunu', 'Syrc', 'Thai'} + 0x0304, # .. 0x0304 ; {'Aghb', 'Cher', 'Copt', 'Cyrl', 'Goth', 'Grek', 'Latn', 'Osge', 'Syrc', 'Tfng', 'Todr'} + 0x0305, # .. 0x0305 ; {'Copt', 'Elba', 'Glag', 'Goth', 'Kana', 'Latn'} + 0x0306, # .. 0x0306 ; {'Cyrl', 'Grek', 'Latn', 'Perm', 'Tfng'} + 0x0307, # .. 0x0307 ; {'Copt', 'Dupl', 'Hebr', 'Latn', 'Perm', 'Syrc', 'Tale', 'Tfng', 'Todr'} + 0x0308, # .. 0x0308 ; {'Armn', 'Cyrl', 'Dupl', 'Goth', 'Grek', 'Hebr', 'Latn', 'Perm', 'Syrc', 'Tale', 'Tfng'} + 0x0309, # .. 0x0309 ; {'Latn', 'Tfng'} + 0x030A, # .. 0x030A ; {'Dupl', 'Latn', 'Syrc'} + 0x030B, # .. 0x030B ; {'Cher', 'Cyrl', 'Latn', 'Osge'} + 0x030C, # .. 0x030C ; {'Cher', 'Latn', 'Tale'} + 0x030D, # .. 0x030D ; {'Latn', 'Sunu'} + 0x030E, # .. 0x030E ; {'Ethi', 'Latn'} + 0x030F, # .. 0x030F ; None + 0x0310, # .. 0x0310 ; {'Latn', 'Sunu'} + 0x0311, # .. 0x0311 ; {'Cyrl', 'Latn', 'Todr'} + 0x0312, # .. 0x0312 ; None + 0x0313, # .. 0x0313 ; {'Grek', 'Latn', 'Perm', 'Todr'} + 0x0314, # .. 0x0322 ; None + 0x0323, # .. 0x0323 ; {'Cher', 'Dupl', 'Kana', 'Latn', 'Syrc', 'Tfng'} + 0x0324, # .. 0x0324 ; {'Cher', 'Dupl', 'Latn', 'Syrc'} + 0x0325, # .. 0x0325 ; {'Latn', 'Syrc'} + 0x0326, # .. 0x032C ; None + 0x032D, # .. 0x032D ; {'Latn', 'Sunu', 'Syrc'} + 0x032E, # .. 0x032E ; {'Latn', 'Syrc'} + 0x032F, # .. 0x032F ; None + 0x0330, # .. 0x0330 ; {'Cher', 'Latn', 'Syrc'} + 0x0331, # .. 0x0331 ; {'Aghb', 'Cher', 'Goth', 'Latn', 'Sunu', 'Syrc', 'Thai'} + 0x0332, # .. 0x0341 ; None + 0x0342, # .. 0x0342 ; {'Grek'} + 0x0343, # .. 0x0344 ; None + 0x0345, # .. 0x0345 ; {'Grek'} + 0x0346, # .. 0x0357 ; None + 0x0358, # .. 0x0358 ; {'Latn', 'Osge'} + 0x0359, # .. 0x035D ; None + 0x035E, # .. 0x035E ; {'Aghb', 'Latn', 'Todr'} + 0x035F, # .. 0x0362 ; None + 0x0363, # .. 0x036F ; {'Latn'} + 0x0370, # .. 0x0373 ; None + 0x0374, # .. 0x0375 ; {'Copt', 'Grek'} + 0x0376, # .. 0x0482 ; None + 0x0483, # .. 0x0483 ; {'Cyrl', 'Perm'} + 0x0484, # .. 0x0484 ; {'Cyrl', 'Glag'} + 0x0485, # .. 0x0486 ; {'Cyrl', 'Latn'} + 0x0487, # .. 0x0487 ; {'Cyrl', 'Glag'} + 0x0488, # .. 0x0588 ; None + 0x0589, # .. 0x0589 ; {'Armn', 'Geor', 'Glag'} + 0x058A, # .. 0x060B ; None + 0x060C, # .. 0x060C ; {'Arab', 'Gara', 'Nkoo', 'Rohg', 'Syrc', 'Thaa', 'Yezi'} + 0x060D, # .. 0x061A ; None + 0x061B, # .. 0x061B ; {'Arab', 'Gara', 'Nkoo', 'Rohg', 'Syrc', 'Thaa', 'Yezi'} + 0x061C, # .. 0x061C ; {'Arab', 'Syrc', 'Thaa'} + 0x061D, # .. 0x061E ; None + 0x061F, # .. 0x061F ; {'Adlm', 'Arab', 'Gara', 'Nkoo', 'Rohg', 'Syrc', 'Thaa', 'Yezi'} + 0x0620, # .. 0x063F ; None + 0x0640, # .. 0x0640 ; {'Adlm', 'Arab', 'Mand', 'Mani', 'Ougr', 'Phlp', 'Rohg', 'Sogd', 'Syrc'} + 0x0641, # .. 0x064A ; None + 0x064B, # .. 0x0655 ; {'Arab', 'Syrc'} + 0x0656, # .. 0x065F ; None + 0x0660, # .. 0x0669 ; {'Arab', 'Thaa', 'Yezi'} + 0x066A, # .. 0x066F ; None + 0x0670, # .. 0x0670 ; {'Arab', 'Syrc'} + 0x0671, # .. 0x06D3 ; None + 0x06D4, # .. 0x06D4 ; {'Arab', 'Rohg'} + 0x06D5, # .. 0x0950 ; None + 0x0951, # .. 0x0951 ; {'Beng', 'Deva', 'Gran', 'Gujr', 'Guru', 'Knda', 'Latn', 'Mlym', 'Nand', 'Newa', 'Orya', 'Shrd', 'Taml', 'Telu', 'Tirh'} + 0x0952, # .. 0x0952 ; {'Beng', 'Deva', 'Gran', 'Gujr', 'Guru', 'Knda', 'Latn', 'Mlym', 'Newa', 'Orya', 'Taml', 'Telu', 'Tirh'} + 0x0953, # .. 0x0963 ; None + 0x0964, # .. 0x0964 ; {'Beng', 'Deva', 'Dogr', 'Gong', 'Gonm', 'Gran', 'Gujr', 'Guru', 'Knda', 'Mahj', 'Mlym', 'Nand', 'Onao', 'Orya', 'Sind', 'Sinh', 'Sylo', 'Takr', 'Taml', 'Telu', 'Tirh'} + 0x0965, # .. 0x0965 ; {'Beng', 'Deva', 'Dogr', 'Gong', 'Gonm', 'Gran', 'Gujr', 'Gukh', 'Guru', 'Knda', 'Limb', 'Mahj', 'Mlym', 'Nand', 'Onao', 'Orya', 'Sind', 'Sinh', 'Sylo', 'Takr', 'Taml', 'Telu', 'Tirh'} + 0x0966, # .. 0x096F ; {'Deva', 'Dogr', 'Kthi', 'Mahj'} + 0x0970, # .. 0x09E5 ; None + 0x09E6, # .. 0x09EF ; {'Beng', 'Cakm', 'Sylo'} + 0x09F0, # .. 0x0A65 ; None + 0x0A66, # .. 0x0A6F ; {'Guru', 'Mult'} + 0x0A70, # .. 0x0AE5 ; None + 0x0AE6, # .. 0x0AEF ; {'Gujr', 'Khoj'} + 0x0AF0, # .. 0x0BE5 ; None + 0x0BE6, # .. 0x0BF3 ; {'Gran', 'Taml'} + 0x0BF4, # .. 0x0CE5 ; None + 0x0CE6, # .. 0x0CEF ; {'Knda', 'Nand', 'Tutg'} + 0x0CF0, # .. 0x103F ; None + 0x1040, # .. 0x1049 ; {'Cakm', 'Mymr', 'Tale'} + 0x104A, # .. 0x10FA ; None + 0x10FB, # .. 0x10FB ; {'Geor', 'Glag', 'Latn'} + 0x10FC, # .. 0x16EA ; None + 0x16EB, # .. 0x16ED ; {'Runr'} + 0x16EE, # .. 0x1734 ; None + 0x1735, # .. 0x1736 ; {'Buhd', 'Hano', 'Tagb', 'Tglg'} + 0x1737, # .. 0x1801 ; None + 0x1802, # .. 0x1803 ; {'Mong', 'Phag'} + 0x1804, # .. 0x1804 ; None + 0x1805, # .. 0x1805 ; {'Mong', 'Phag'} + 0x1806, # .. 0x1CCF ; None + 0x1CD0, # .. 0x1CD0 ; {'Beng', 'Deva', 'Gran', 'Knda'} + 0x1CD1, # .. 0x1CD1 ; {'Deva'} + 0x1CD2, # .. 0x1CD2 ; {'Beng', 'Deva', 'Gran', 'Knda'} + 0x1CD3, # .. 0x1CD3 ; {'Deva', 'Gran', 'Knda'} + 0x1CD4, # .. 0x1CD4 ; {'Deva'} + 0x1CD5, # .. 0x1CD5 ; {'Beng', 'Deva', 'Newa', 'Telu', 'Tirh'} + 0x1CD6, # .. 0x1CD6 ; {'Beng', 'Deva', 'Telu'} + 0x1CD7, # .. 0x1CD7 ; {'Deva', 'Newa', 'Shrd'} + 0x1CD8, # .. 0x1CD8 ; {'Beng', 'Deva', 'Newa', 'Telu'} + 0x1CD9, # .. 0x1CD9 ; {'Deva', 'Shrd'} + 0x1CDA, # .. 0x1CDA ; {'Deva', 'Knda', 'Mlym', 'Orya', 'Taml', 'Telu'} + 0x1CDB, # .. 0x1CDB ; {'Deva'} + 0x1CDC, # .. 0x1CDD ; {'Deva', 'Shrd'} + 0x1CDE, # .. 0x1CDF ; {'Deva'} + 0x1CE0, # .. 0x1CE0 ; {'Deva', 'Shrd'} + 0x1CE1, # .. 0x1CE1 ; {'Beng', 'Deva'} + 0x1CE2, # .. 0x1CE2 ; {'Deva', 'Newa', 'Tirh'} + 0x1CE3, # .. 0x1CE8 ; {'Deva'} + 0x1CE9, # .. 0x1CE9 ; {'Deva', 'Nand', 'Newa'} + 0x1CEA, # .. 0x1CEA ; {'Beng', 'Deva', 'Shrd'} + 0x1CEB, # .. 0x1CEB ; {'Deva', 'Newa'} + 0x1CEC, # .. 0x1CEC ; {'Deva'} + 0x1CED, # .. 0x1CED ; {'Beng', 'Deva', 'Newa', 'Shrd'} + 0x1CEE, # .. 0x1CF1 ; {'Deva'} + 0x1CF2, # .. 0x1CF2 ; {'Beng', 'Deva', 'Gran', 'Knda', 'Mlym', 'Nand', 'Orya', 'Sinh', 'Telu', 'Tirh', 'Tutg'} + 0x1CF3, # .. 0x1CF3 ; {'Deva', 'Gran'} + 0x1CF4, # .. 0x1CF4 ; {'Deva', 'Gran', 'Knda', 'Tutg'} + 0x1CF5, # .. 0x1CF6 ; {'Beng', 'Deva'} + 0x1CF7, # .. 0x1CF7 ; {'Beng'} + 0x1CF8, # .. 0x1CF9 ; {'Deva', 'Gran'} + 0x1CFA, # .. 0x1CFA ; {'Nand'} + 0x1CFB, # .. 0x1DBF ; None + 0x1DC0, # .. 0x1DC1 ; {'Grek'} + 0x1DC2, # .. 0x1DF7 ; None + 0x1DF8, # .. 0x1DF8 ; {'Cyrl', 'Latn', 'Syrc'} + 0x1DF9, # .. 0x1DF9 ; None + 0x1DFA, # .. 0x1DFA ; {'Syrc'} + 0x1DFB, # .. 0x202E ; None + 0x202F, # .. 0x202F ; {'Latn', 'Mong', 'Phag'} + 0x2030, # .. 0x204E ; None + 0x204F, # .. 0x204F ; {'Adlm', 'Arab'} + 0x2050, # .. 0x2059 ; None + 0x205A, # .. 0x205A ; {'Cari', 'Geor', 'Glag', 'Hung', 'Lyci', 'Orkh'} + 0x205B, # .. 0x205C ; None + 0x205D, # .. 0x205D ; {'Cari', 'Grek', 'Hung', 'Mero'} + 0x205E, # .. 0x20EF ; None + 0x20F0, # .. 0x20F0 ; {'Deva', 'Gran', 'Latn'} + 0x20F1, # .. 0x2E16 ; None + 0x2E17, # .. 0x2E17 ; {'Copt', 'Latn'} + 0x2E18, # .. 0x2E2F ; None + 0x2E30, # .. 0x2E30 ; {'Avst', 'Orkh'} + 0x2E31, # .. 0x2E31 ; {'Avst', 'Cari', 'Geor', 'Hung', 'Kthi', 'Lydi', 'Samr'} + 0x2E32, # .. 0x2E3B ; None + 0x2E3C, # .. 0x2E3C ; {'Dupl'} + 0x2E3D, # .. 0x2E40 ; None + 0x2E41, # .. 0x2E41 ; {'Adlm', 'Arab', 'Hung'} + 0x2E42, # .. 0x2E42 ; None + 0x2E43, # .. 0x2E43 ; {'Cyrl', 'Glag'} + 0x2E44, # .. 0x2FEF ; None + 0x2FF0, # .. 0x2FFF ; {'Hani', 'Tang'} + 0x3000, # .. 0x3000 ; None + 0x3001, # .. 0x3001 ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana', 'Mong', 'Yiii'} + 0x3002, # .. 0x3002 ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana', 'Mong', 'Phag', 'Yiii'} + 0x3003, # .. 0x3003 ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana'} + 0x3004, # .. 0x3005 ; None + 0x3006, # .. 0x3006 ; {'Hani'} + 0x3007, # .. 0x3007 ; None + 0x3008, # .. 0x3009 ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana', 'Mong', 'Tibt', 'Yiii'} + 0x300A, # .. 0x300B ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana', 'Lisu', 'Mong', 'Tibt', 'Yiii'} + 0x300C, # .. 0x3011 ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana', 'Yiii'} + 0x3012, # .. 0x3012 ; None + 0x3013, # .. 0x3013 ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana'} + 0x3014, # .. 0x301B ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana', 'Yiii'} + 0x301C, # .. 0x301F ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana'} + 0x3020, # .. 0x3029 ; None + 0x302A, # .. 0x302D ; {'Bopo', 'Hani'} + 0x302E, # .. 0x302F ; None + 0x3030, # .. 0x3030 ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana'} + 0x3031, # .. 0x3035 ; {'Hira', 'Kana'} + 0x3036, # .. 0x3036 ; None + 0x3037, # .. 0x3037 ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana'} + 0x3038, # .. 0x303B ; None + 0x303C, # .. 0x303D ; {'Hani', 'Hira', 'Kana'} + 0x303E, # .. 0x303F ; {'Hani'} + 0x3040, # .. 0x3098 ; None + 0x3099, # .. 0x309C ; {'Hira', 'Kana'} + 0x309D, # .. 0x309F ; None + 0x30A0, # .. 0x30A0 ; {'Hira', 'Kana'} + 0x30A1, # .. 0x30FA ; None + 0x30FB, # .. 0x30FB ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana', 'Yiii'} + 0x30FC, # .. 0x30FC ; {'Hira', 'Kana'} + 0x30FD, # .. 0x318F ; None + 0x3190, # .. 0x319F ; {'Hani'} + 0x31A0, # .. 0x31BF ; None + 0x31C0, # .. 0x31E5 ; {'Hani'} + 0x31E6, # .. 0x31EE ; None + 0x31EF, # .. 0x31EF ; {'Hani', 'Tang'} + 0x31F0, # .. 0x321F ; None + 0x3220, # .. 0x3247 ; {'Hani'} + 0x3248, # .. 0x327F ; None + 0x3280, # .. 0x32B0 ; {'Hani'} + 0x32B1, # .. 0x32BF ; None + 0x32C0, # .. 0x32CB ; {'Hani'} + 0x32CC, # .. 0x32FE ; None + 0x32FF, # .. 0x32FF ; {'Hani'} + 0x3300, # .. 0x3357 ; None + 0x3358, # .. 0x3370 ; {'Hani'} + 0x3371, # .. 0x337A ; None + 0x337B, # .. 0x337F ; {'Hani'} + 0x3380, # .. 0x33DF ; None + 0x33E0, # .. 0x33FE ; {'Hani'} + 0x33FF, # .. 0xA66E ; None + 0xA66F, # .. 0xA66F ; {'Cyrl', 'Glag'} + 0xA670, # .. 0xA6FF ; None + 0xA700, # .. 0xA707 ; {'Hani', 'Latn'} + 0xA708, # .. 0xA82F ; None + 0xA830, # .. 0xA832 ; {'Deva', 'Dogr', 'Gujr', 'Guru', 'Khoj', 'Knda', 'Kthi', 'Mahj', 'Mlym', 'Modi', 'Nand', 'Shrd', 'Sind', 'Takr', 'Tirh', 'Tutg'} + 0xA833, # .. 0xA835 ; {'Deva', 'Dogr', 'Gujr', 'Guru', 'Khoj', 'Knda', 'Kthi', 'Mahj', 'Modi', 'Nand', 'Shrd', 'Sind', 'Takr', 'Tirh', 'Tutg'} + 0xA836, # .. 0xA837 ; {'Deva', 'Dogr', 'Gujr', 'Guru', 'Khoj', 'Kthi', 'Mahj', 'Modi', 'Sind', 'Takr', 'Tirh'} + 0xA838, # .. 0xA838 ; {'Deva', 'Dogr', 'Gujr', 'Guru', 'Khoj', 'Kthi', 'Mahj', 'Modi', 'Shrd', 'Sind', 'Takr', 'Tirh'} + 0xA839, # .. 0xA839 ; {'Deva', 'Dogr', 'Gujr', 'Guru', 'Khoj', 'Kthi', 'Mahj', 'Modi', 'Sind', 'Takr', 'Tirh'} + 0xA83A, # .. 0xA8F0 ; None + 0xA8F1, # .. 0xA8F1 ; {'Beng', 'Deva', 'Tutg'} + 0xA8F2, # .. 0xA8F2 ; None + 0xA8F3, # .. 0xA8F3 ; {'Deva', 'Taml'} + 0xA8F4, # .. 0xA92D ; None + 0xA92E, # .. 0xA92E ; {'Kali', 'Latn', 'Mymr'} + 0xA92F, # .. 0xA9CE ; None + 0xA9CF, # .. 0xA9CF ; {'Bugi', 'Java'} + 0xA9D0, # .. 0xFD3D ; None + 0xFD3E, # .. 0xFD3F ; {'Arab', 'Nkoo'} + 0xFD40, # .. 0xFDF1 ; None + 0xFDF2, # .. 0xFDF2 ; {'Arab', 'Thaa'} + 0xFDF3, # .. 0xFDFC ; None + 0xFDFD, # .. 0xFDFD ; {'Arab', 'Thaa'} + 0xFDFE, # .. 0xFE44 ; None + 0xFE45, # .. 0xFE46 ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana'} + 0xFE47, # .. 0xFF60 ; None + 0xFF61, # .. 0xFF65 ; {'Bopo', 'Hang', 'Hani', 'Hira', 'Kana', 'Yiii'} + 0xFF66, # .. 0xFF6F ; None + 0xFF70, # .. 0xFF70 ; {'Hira', 'Kana'} + 0xFF71, # .. 0xFF9D ; None + 0xFF9E, # .. 0xFF9F ; {'Hira', 'Kana'} + 0xFFA0, # .. 0x100FF ; None + 0x10100, # .. 0x10101 ; {'Cpmn', 'Cprt', 'Linb'} + 0x10102, # .. 0x10102 ; {'Cprt', 'Linb'} + 0x10103, # .. 0x10106 ; None + 0x10107, # .. 0x10133 ; {'Cprt', 'Lina', 'Linb'} + 0x10134, # .. 0x10136 ; None + 0x10137, # .. 0x1013F ; {'Cprt', 'Linb'} + 0x10140, # .. 0x102DF ; None + 0x102E0, # .. 0x102FB ; {'Arab', 'Copt'} + 0x102FC, # .. 0x10AF1 ; None + 0x10AF2, # .. 0x10AF2 ; {'Mani', 'Ougr'} + 0x10AF3, # .. 0x11300 ; None + 0x11301, # .. 0x11301 ; {'Gran', 'Taml'} + 0x11302, # .. 0x11302 ; None + 0x11303, # .. 0x11303 ; {'Gran', 'Taml'} + 0x11304, # .. 0x1133A ; None + 0x1133B, # .. 0x1133C ; {'Gran', 'Taml'} + 0x1133D, # .. 0x11FCF ; None + 0x11FD0, # .. 0x11FD1 ; {'Gran', 'Taml'} + 0x11FD2, # .. 0x11FD2 ; None + 0x11FD3, # .. 0x11FD3 ; {'Gran', 'Taml'} + 0x11FD4, # .. 0x1BC9F ; None + 0x1BCA0, # .. 0x1BCA3 ; {'Dupl'} + 0x1BCA4, # .. 0x1D35F ; None + 0x1D360, # .. 0x1D371 ; {'Hani'} + 0x1D372, # .. 0x1F24F ; None + 0x1F250, # .. 0x1F251 ; {'Hani'} + 0x1F252, # .. 0x10FFFF ; None +] + +VALUES = [ + None, # 0000..00B6 + { + "Avst", + "Cari", + "Copt", + "Dupl", + "Elba", + "Geor", + "Glag", + "Gong", + "Goth", + "Grek", + "Hani", + "Latn", + "Lydi", + "Mahj", + "Perm", + "Shaw", + }, # 00B7..00B7 + None, # 00B8..02BB + {"Beng", "Cyrl", "Deva", "Latn", "Lisu", "Thai", "Toto"}, # 02BC..02BC + None, # 02BD..02C6 + {"Bopo", "Latn"}, # 02C7..02C7 + None, # 02C8..02C8 + {"Bopo", "Latn"}, # 02C9..02CB + None, # 02CC..02CC + {"Latn", "Lisu"}, # 02CD..02CD + None, # 02CE..02D6 + {"Latn", "Thai"}, # 02D7..02D7 + None, # 02D8..02D8 + {"Bopo", "Latn"}, # 02D9..02D9 + None, # 02DA..02FF + {"Cher", "Copt", "Cyrl", "Grek", "Latn", "Perm", "Sunu", "Tale"}, # 0300..0300 + {"Cher", "Cyrl", "Grek", "Latn", "Osge", "Sunu", "Tale", "Todr"}, # 0301..0301 + {"Cher", "Cyrl", "Latn", "Tfng"}, # 0302..0302 + {"Glag", "Latn", "Sunu", "Syrc", "Thai"}, # 0303..0303 + { + "Aghb", + "Cher", + "Copt", + "Cyrl", + "Goth", + "Grek", + "Latn", + "Osge", + "Syrc", + "Tfng", + "Todr", + }, # 0304..0304 + {"Copt", "Elba", "Glag", "Goth", "Kana", "Latn"}, # 0305..0305 + {"Cyrl", "Grek", "Latn", "Perm", "Tfng"}, # 0306..0306 + { + "Copt", + "Dupl", + "Hebr", + "Latn", + "Perm", + "Syrc", + "Tale", + "Tfng", + "Todr", + }, # 0307..0307 + { + "Armn", + "Cyrl", + "Dupl", + "Goth", + "Grek", + "Hebr", + "Latn", + "Perm", + "Syrc", + "Tale", + "Tfng", + }, # 0308..0308 + {"Latn", "Tfng"}, # 0309..0309 + {"Dupl", "Latn", "Syrc"}, # 030A..030A + {"Cher", "Cyrl", "Latn", "Osge"}, # 030B..030B + {"Cher", "Latn", "Tale"}, # 030C..030C + {"Latn", "Sunu"}, # 030D..030D + {"Ethi", "Latn"}, # 030E..030E + None, # 030F..030F + {"Latn", "Sunu"}, # 0310..0310 + {"Cyrl", "Latn", "Todr"}, # 0311..0311 + None, # 0312..0312 + {"Grek", "Latn", "Perm", "Todr"}, # 0313..0313 + None, # 0314..0322 + {"Cher", "Dupl", "Kana", "Latn", "Syrc", "Tfng"}, # 0323..0323 + {"Cher", "Dupl", "Latn", "Syrc"}, # 0324..0324 + {"Latn", "Syrc"}, # 0325..0325 + None, # 0326..032C + {"Latn", "Sunu", "Syrc"}, # 032D..032D + {"Latn", "Syrc"}, # 032E..032E + None, # 032F..032F + {"Cher", "Latn", "Syrc"}, # 0330..0330 + {"Aghb", "Cher", "Goth", "Latn", "Sunu", "Syrc", "Thai"}, # 0331..0331 + None, # 0332..0341 + {"Grek"}, # 0342..0342 + None, # 0343..0344 + {"Grek"}, # 0345..0345 + None, # 0346..0357 + {"Latn", "Osge"}, # 0358..0358 + None, # 0359..035D + {"Aghb", "Latn", "Todr"}, # 035E..035E + None, # 035F..0362 + {"Latn"}, # 0363..036F + None, # 0370..0373 + {"Copt", "Grek"}, # 0374..0375 + None, # 0376..0482 + {"Cyrl", "Perm"}, # 0483..0483 + {"Cyrl", "Glag"}, # 0484..0484 + {"Cyrl", "Latn"}, # 0485..0486 + {"Cyrl", "Glag"}, # 0487..0487 + None, # 0488..0588 + {"Armn", "Geor", "Glag"}, # 0589..0589 + None, # 058A..060B + {"Arab", "Gara", "Nkoo", "Rohg", "Syrc", "Thaa", "Yezi"}, # 060C..060C + None, # 060D..061A + {"Arab", "Gara", "Nkoo", "Rohg", "Syrc", "Thaa", "Yezi"}, # 061B..061B + {"Arab", "Syrc", "Thaa"}, # 061C..061C + None, # 061D..061E + {"Adlm", "Arab", "Gara", "Nkoo", "Rohg", "Syrc", "Thaa", "Yezi"}, # 061F..061F + None, # 0620..063F + { + "Adlm", + "Arab", + "Mand", + "Mani", + "Ougr", + "Phlp", + "Rohg", + "Sogd", + "Syrc", + }, # 0640..0640 + None, # 0641..064A + {"Arab", "Syrc"}, # 064B..0655 + None, # 0656..065F + {"Arab", "Thaa", "Yezi"}, # 0660..0669 + None, # 066A..066F + {"Arab", "Syrc"}, # 0670..0670 + None, # 0671..06D3 + {"Arab", "Rohg"}, # 06D4..06D4 + None, # 06D5..0950 + { + "Beng", + "Deva", + "Gran", + "Gujr", + "Guru", + "Knda", + "Latn", + "Mlym", + "Nand", + "Newa", + "Orya", + "Shrd", + "Taml", + "Telu", + "Tirh", + }, # 0951..0951 + { + "Beng", + "Deva", + "Gran", + "Gujr", + "Guru", + "Knda", + "Latn", + "Mlym", + "Newa", + "Orya", + "Taml", + "Telu", + "Tirh", + }, # 0952..0952 + None, # 0953..0963 + { + "Beng", + "Deva", + "Dogr", + "Gong", + "Gonm", + "Gran", + "Gujr", + "Guru", + "Knda", + "Mahj", + "Mlym", + "Nand", + "Onao", + "Orya", + "Sind", + "Sinh", + "Sylo", + "Takr", + "Taml", + "Telu", + "Tirh", + }, # 0964..0964 + { + "Beng", + "Deva", + "Dogr", + "Gong", + "Gonm", + "Gran", + "Gujr", + "Gukh", + "Guru", + "Knda", + "Limb", + "Mahj", + "Mlym", + "Nand", + "Onao", + "Orya", + "Sind", + "Sinh", + "Sylo", + "Takr", + "Taml", + "Telu", + "Tirh", + }, # 0965..0965 + {"Deva", "Dogr", "Kthi", "Mahj"}, # 0966..096F + None, # 0970..09E5 + {"Beng", "Cakm", "Sylo"}, # 09E6..09EF + None, # 09F0..0A65 + {"Guru", "Mult"}, # 0A66..0A6F + None, # 0A70..0AE5 + {"Gujr", "Khoj"}, # 0AE6..0AEF + None, # 0AF0..0BE5 + {"Gran", "Taml"}, # 0BE6..0BF3 + None, # 0BF4..0CE5 + {"Knda", "Nand", "Tutg"}, # 0CE6..0CEF + None, # 0CF0..103F + {"Cakm", "Mymr", "Tale"}, # 1040..1049 + None, # 104A..10FA + {"Geor", "Glag", "Latn"}, # 10FB..10FB + None, # 10FC..16EA + {"Runr"}, # 16EB..16ED + None, # 16EE..1734 + {"Buhd", "Hano", "Tagb", "Tglg"}, # 1735..1736 + None, # 1737..1801 + {"Mong", "Phag"}, # 1802..1803 + None, # 1804..1804 + {"Mong", "Phag"}, # 1805..1805 + None, # 1806..1CCF + {"Beng", "Deva", "Gran", "Knda"}, # 1CD0..1CD0 + {"Deva"}, # 1CD1..1CD1 + {"Beng", "Deva", "Gran", "Knda"}, # 1CD2..1CD2 + {"Deva", "Gran", "Knda"}, # 1CD3..1CD3 + {"Deva"}, # 1CD4..1CD4 + {"Beng", "Deva", "Newa", "Telu", "Tirh"}, # 1CD5..1CD5 + {"Beng", "Deva", "Telu"}, # 1CD6..1CD6 + {"Deva", "Newa", "Shrd"}, # 1CD7..1CD7 + {"Beng", "Deva", "Newa", "Telu"}, # 1CD8..1CD8 + {"Deva", "Shrd"}, # 1CD9..1CD9 + {"Deva", "Knda", "Mlym", "Orya", "Taml", "Telu"}, # 1CDA..1CDA + {"Deva"}, # 1CDB..1CDB + {"Deva", "Shrd"}, # 1CDC..1CDD + {"Deva"}, # 1CDE..1CDF + {"Deva", "Shrd"}, # 1CE0..1CE0 + {"Beng", "Deva"}, # 1CE1..1CE1 + {"Deva", "Newa", "Tirh"}, # 1CE2..1CE2 + {"Deva"}, # 1CE3..1CE8 + {"Deva", "Nand", "Newa"}, # 1CE9..1CE9 + {"Beng", "Deva", "Shrd"}, # 1CEA..1CEA + {"Deva", "Newa"}, # 1CEB..1CEB + {"Deva"}, # 1CEC..1CEC + {"Beng", "Deva", "Newa", "Shrd"}, # 1CED..1CED + {"Deva"}, # 1CEE..1CF1 + { + "Beng", + "Deva", + "Gran", + "Knda", + "Mlym", + "Nand", + "Orya", + "Sinh", + "Telu", + "Tirh", + "Tutg", + }, # 1CF2..1CF2 + {"Deva", "Gran"}, # 1CF3..1CF3 + {"Deva", "Gran", "Knda", "Tutg"}, # 1CF4..1CF4 + {"Beng", "Deva"}, # 1CF5..1CF6 + {"Beng"}, # 1CF7..1CF7 + {"Deva", "Gran"}, # 1CF8..1CF9 + {"Nand"}, # 1CFA..1CFA + None, # 1CFB..1DBF + {"Grek"}, # 1DC0..1DC1 + None, # 1DC2..1DF7 + {"Cyrl", "Latn", "Syrc"}, # 1DF8..1DF8 + None, # 1DF9..1DF9 + {"Syrc"}, # 1DFA..1DFA + None, # 1DFB..202E + {"Latn", "Mong", "Phag"}, # 202F..202F + None, # 2030..204E + {"Adlm", "Arab"}, # 204F..204F + None, # 2050..2059 + {"Cari", "Geor", "Glag", "Hung", "Lyci", "Orkh"}, # 205A..205A + None, # 205B..205C + {"Cari", "Grek", "Hung", "Mero"}, # 205D..205D + None, # 205E..20EF + {"Deva", "Gran", "Latn"}, # 20F0..20F0 + None, # 20F1..2E16 + {"Copt", "Latn"}, # 2E17..2E17 + None, # 2E18..2E2F + {"Avst", "Orkh"}, # 2E30..2E30 + {"Avst", "Cari", "Geor", "Hung", "Kthi", "Lydi", "Samr"}, # 2E31..2E31 + None, # 2E32..2E3B + {"Dupl"}, # 2E3C..2E3C + None, # 2E3D..2E40 + {"Adlm", "Arab", "Hung"}, # 2E41..2E41 + None, # 2E42..2E42 + {"Cyrl", "Glag"}, # 2E43..2E43 + None, # 2E44..2FEF + {"Hani", "Tang"}, # 2FF0..2FFF + None, # 3000..3000 + {"Bopo", "Hang", "Hani", "Hira", "Kana", "Mong", "Yiii"}, # 3001..3001 + {"Bopo", "Hang", "Hani", "Hira", "Kana", "Mong", "Phag", "Yiii"}, # 3002..3002 + {"Bopo", "Hang", "Hani", "Hira", "Kana"}, # 3003..3003 + None, # 3004..3005 + {"Hani"}, # 3006..3006 + None, # 3007..3007 + {"Bopo", "Hang", "Hani", "Hira", "Kana", "Mong", "Tibt", "Yiii"}, # 3008..3009 + { + "Bopo", + "Hang", + "Hani", + "Hira", + "Kana", + "Lisu", + "Mong", + "Tibt", + "Yiii", + }, # 300A..300B + {"Bopo", "Hang", "Hani", "Hira", "Kana", "Yiii"}, # 300C..3011 + None, # 3012..3012 + {"Bopo", "Hang", "Hani", "Hira", "Kana"}, # 3013..3013 + {"Bopo", "Hang", "Hani", "Hira", "Kana", "Yiii"}, # 3014..301B + {"Bopo", "Hang", "Hani", "Hira", "Kana"}, # 301C..301F + None, # 3020..3029 + {"Bopo", "Hani"}, # 302A..302D + None, # 302E..302F + {"Bopo", "Hang", "Hani", "Hira", "Kana"}, # 3030..3030 + {"Hira", "Kana"}, # 3031..3035 + None, # 3036..3036 + {"Bopo", "Hang", "Hani", "Hira", "Kana"}, # 3037..3037 + None, # 3038..303B + {"Hani", "Hira", "Kana"}, # 303C..303D + {"Hani"}, # 303E..303F + None, # 3040..3098 + {"Hira", "Kana"}, # 3099..309C + None, # 309D..309F + {"Hira", "Kana"}, # 30A0..30A0 + None, # 30A1..30FA + {"Bopo", "Hang", "Hani", "Hira", "Kana", "Yiii"}, # 30FB..30FB + {"Hira", "Kana"}, # 30FC..30FC + None, # 30FD..318F + {"Hani"}, # 3190..319F + None, # 31A0..31BF + {"Hani"}, # 31C0..31E5 + None, # 31E6..31EE + {"Hani", "Tang"}, # 31EF..31EF + None, # 31F0..321F + {"Hani"}, # 3220..3247 + None, # 3248..327F + {"Hani"}, # 3280..32B0 + None, # 32B1..32BF + {"Hani"}, # 32C0..32CB + None, # 32CC..32FE + {"Hani"}, # 32FF..32FF + None, # 3300..3357 + {"Hani"}, # 3358..3370 + None, # 3371..337A + {"Hani"}, # 337B..337F + None, # 3380..33DF + {"Hani"}, # 33E0..33FE + None, # 33FF..A66E + {"Cyrl", "Glag"}, # A66F..A66F + None, # A670..A6FF + {"Hani", "Latn"}, # A700..A707 + None, # A708..A82F + { + "Deva", + "Dogr", + "Gujr", + "Guru", + "Khoj", + "Knda", + "Kthi", + "Mahj", + "Mlym", + "Modi", + "Nand", + "Shrd", + "Sind", + "Takr", + "Tirh", + "Tutg", + }, # A830..A832 + { + "Deva", + "Dogr", + "Gujr", + "Guru", + "Khoj", + "Knda", + "Kthi", + "Mahj", + "Modi", + "Nand", + "Shrd", + "Sind", + "Takr", + "Tirh", + "Tutg", + }, # A833..A835 + { + "Deva", + "Dogr", + "Gujr", + "Guru", + "Khoj", + "Kthi", + "Mahj", + "Modi", + "Sind", + "Takr", + "Tirh", + }, # A836..A837 + { + "Deva", + "Dogr", + "Gujr", + "Guru", + "Khoj", + "Kthi", + "Mahj", + "Modi", + "Shrd", + "Sind", + "Takr", + "Tirh", + }, # A838..A838 + { + "Deva", + "Dogr", + "Gujr", + "Guru", + "Khoj", + "Kthi", + "Mahj", + "Modi", + "Sind", + "Takr", + "Tirh", + }, # A839..A839 + None, # A83A..A8F0 + {"Beng", "Deva", "Tutg"}, # A8F1..A8F1 + None, # A8F2..A8F2 + {"Deva", "Taml"}, # A8F3..A8F3 + None, # A8F4..A92D + {"Kali", "Latn", "Mymr"}, # A92E..A92E + None, # A92F..A9CE + {"Bugi", "Java"}, # A9CF..A9CF + None, # A9D0..FD3D + {"Arab", "Nkoo"}, # FD3E..FD3F + None, # FD40..FDF1 + {"Arab", "Thaa"}, # FDF2..FDF2 + None, # FDF3..FDFC + {"Arab", "Thaa"}, # FDFD..FDFD + None, # FDFE..FE44 + {"Bopo", "Hang", "Hani", "Hira", "Kana"}, # FE45..FE46 + None, # FE47..FF60 + {"Bopo", "Hang", "Hani", "Hira", "Kana", "Yiii"}, # FF61..FF65 + None, # FF66..FF6F + {"Hira", "Kana"}, # FF70..FF70 + None, # FF71..FF9D + {"Hira", "Kana"}, # FF9E..FF9F + None, # FFA0..100FF + {"Cpmn", "Cprt", "Linb"}, # 10100..10101 + {"Cprt", "Linb"}, # 10102..10102 + None, # 10103..10106 + {"Cprt", "Lina", "Linb"}, # 10107..10133 + None, # 10134..10136 + {"Cprt", "Linb"}, # 10137..1013F + None, # 10140..102DF + {"Arab", "Copt"}, # 102E0..102FB + None, # 102FC..10AF1 + {"Mani", "Ougr"}, # 10AF2..10AF2 + None, # 10AF3..11300 + {"Gran", "Taml"}, # 11301..11301 + None, # 11302..11302 + {"Gran", "Taml"}, # 11303..11303 + None, # 11304..1133A + {"Gran", "Taml"}, # 1133B..1133C + None, # 1133D..11FCF + {"Gran", "Taml"}, # 11FD0..11FD1 + None, # 11FD2..11FD2 + {"Gran", "Taml"}, # 11FD3..11FD3 + None, # 11FD4..1BC9F + {"Dupl"}, # 1BCA0..1BCA3 + None, # 1BCA4..1D35F + {"Hani"}, # 1D360..1D371 + None, # 1D372..1F24F + {"Hani"}, # 1F250..1F251 + None, # 1F252..10FFFF +] diff --git a/.venv/lib/python3.11/site-packages/fontTools/unicodedata/Scripts.py b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/Scripts.py new file mode 100644 index 0000000000000000000000000000000000000000..dac10667b423a6a6681e0a67234de37db00611b9 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/Scripts.py @@ -0,0 +1,3639 @@ +# -*- coding: utf-8 -*- +# +# NOTE: This file was auto-generated with MetaTools/buildUCD.py. +# Source: https://unicode.org/Public/UNIDATA/Scripts.txt +# License: http://unicode.org/copyright.html#License +# +# Scripts-17.0.0.txt +# Date: 2025-07-24, 13:28:55 GMT +# © 2025 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see https://www.unicode.org/reports/tr44/ +# For more information, see: +# UAX #24, Unicode Script Property: https://www.unicode.org/reports/tr24/ +# Especially the sections: +# https://www.unicode.org/reports/tr24/#Assignment_Script_Values +# https://www.unicode.org/reports/tr24/#Assignment_ScriptX_Values +# + +RANGES = [ + 0x0000, # .. 0x0040 ; Common + 0x0041, # .. 0x005A ; Latin + 0x005B, # .. 0x0060 ; Common + 0x0061, # .. 0x007A ; Latin + 0x007B, # .. 0x00A9 ; Common + 0x00AA, # .. 0x00AA ; Latin + 0x00AB, # .. 0x00B9 ; Common + 0x00BA, # .. 0x00BA ; Latin + 0x00BB, # .. 0x00BF ; Common + 0x00C0, # .. 0x00D6 ; Latin + 0x00D7, # .. 0x00D7 ; Common + 0x00D8, # .. 0x00F6 ; Latin + 0x00F7, # .. 0x00F7 ; Common + 0x00F8, # .. 0x02B8 ; Latin + 0x02B9, # .. 0x02DF ; Common + 0x02E0, # .. 0x02E4 ; Latin + 0x02E5, # .. 0x02E9 ; Common + 0x02EA, # .. 0x02EB ; Bopomofo + 0x02EC, # .. 0x02FF ; Common + 0x0300, # .. 0x036F ; Inherited + 0x0370, # .. 0x0373 ; Greek + 0x0374, # .. 0x0374 ; Common + 0x0375, # .. 0x0377 ; Greek + 0x0378, # .. 0x0379 ; Unknown + 0x037A, # .. 0x037D ; Greek + 0x037E, # .. 0x037E ; Common + 0x037F, # .. 0x037F ; Greek + 0x0380, # .. 0x0383 ; Unknown + 0x0384, # .. 0x0384 ; Greek + 0x0385, # .. 0x0385 ; Common + 0x0386, # .. 0x0386 ; Greek + 0x0387, # .. 0x0387 ; Common + 0x0388, # .. 0x038A ; Greek + 0x038B, # .. 0x038B ; Unknown + 0x038C, # .. 0x038C ; Greek + 0x038D, # .. 0x038D ; Unknown + 0x038E, # .. 0x03A1 ; Greek + 0x03A2, # .. 0x03A2 ; Unknown + 0x03A3, # .. 0x03E1 ; Greek + 0x03E2, # .. 0x03EF ; Coptic + 0x03F0, # .. 0x03FF ; Greek + 0x0400, # .. 0x0484 ; Cyrillic + 0x0485, # .. 0x0486 ; Inherited + 0x0487, # .. 0x052F ; Cyrillic + 0x0530, # .. 0x0530 ; Unknown + 0x0531, # .. 0x0556 ; Armenian + 0x0557, # .. 0x0558 ; Unknown + 0x0559, # .. 0x058A ; Armenian + 0x058B, # .. 0x058C ; Unknown + 0x058D, # .. 0x058F ; Armenian + 0x0590, # .. 0x0590 ; Unknown + 0x0591, # .. 0x05C7 ; Hebrew + 0x05C8, # .. 0x05CF ; Unknown + 0x05D0, # .. 0x05EA ; Hebrew + 0x05EB, # .. 0x05EE ; Unknown + 0x05EF, # .. 0x05F4 ; Hebrew + 0x05F5, # .. 0x05FF ; Unknown + 0x0600, # .. 0x0604 ; Arabic + 0x0605, # .. 0x0605 ; Common + 0x0606, # .. 0x060B ; Arabic + 0x060C, # .. 0x060C ; Common + 0x060D, # .. 0x061A ; Arabic + 0x061B, # .. 0x061B ; Common + 0x061C, # .. 0x061E ; Arabic + 0x061F, # .. 0x061F ; Common + 0x0620, # .. 0x063F ; Arabic + 0x0640, # .. 0x0640 ; Common + 0x0641, # .. 0x064A ; Arabic + 0x064B, # .. 0x0655 ; Inherited + 0x0656, # .. 0x066F ; Arabic + 0x0670, # .. 0x0670 ; Inherited + 0x0671, # .. 0x06DC ; Arabic + 0x06DD, # .. 0x06DD ; Common + 0x06DE, # .. 0x06FF ; Arabic + 0x0700, # .. 0x070D ; Syriac + 0x070E, # .. 0x070E ; Unknown + 0x070F, # .. 0x074A ; Syriac + 0x074B, # .. 0x074C ; Unknown + 0x074D, # .. 0x074F ; Syriac + 0x0750, # .. 0x077F ; Arabic + 0x0780, # .. 0x07B1 ; Thaana + 0x07B2, # .. 0x07BF ; Unknown + 0x07C0, # .. 0x07FA ; Nko + 0x07FB, # .. 0x07FC ; Unknown + 0x07FD, # .. 0x07FF ; Nko + 0x0800, # .. 0x082D ; Samaritan + 0x082E, # .. 0x082F ; Unknown + 0x0830, # .. 0x083E ; Samaritan + 0x083F, # .. 0x083F ; Unknown + 0x0840, # .. 0x085B ; Mandaic + 0x085C, # .. 0x085D ; Unknown + 0x085E, # .. 0x085E ; Mandaic + 0x085F, # .. 0x085F ; Unknown + 0x0860, # .. 0x086A ; Syriac + 0x086B, # .. 0x086F ; Unknown + 0x0870, # .. 0x0891 ; Arabic + 0x0892, # .. 0x0896 ; Unknown + 0x0897, # .. 0x08E1 ; Arabic + 0x08E2, # .. 0x08E2 ; Common + 0x08E3, # .. 0x08FF ; Arabic + 0x0900, # .. 0x0950 ; Devanagari + 0x0951, # .. 0x0954 ; Inherited + 0x0955, # .. 0x0963 ; Devanagari + 0x0964, # .. 0x0965 ; Common + 0x0966, # .. 0x097F ; Devanagari + 0x0980, # .. 0x0983 ; Bengali + 0x0984, # .. 0x0984 ; Unknown + 0x0985, # .. 0x098C ; Bengali + 0x098D, # .. 0x098E ; Unknown + 0x098F, # .. 0x0990 ; Bengali + 0x0991, # .. 0x0992 ; Unknown + 0x0993, # .. 0x09A8 ; Bengali + 0x09A9, # .. 0x09A9 ; Unknown + 0x09AA, # .. 0x09B0 ; Bengali + 0x09B1, # .. 0x09B1 ; Unknown + 0x09B2, # .. 0x09B2 ; Bengali + 0x09B3, # .. 0x09B5 ; Unknown + 0x09B6, # .. 0x09B9 ; Bengali + 0x09BA, # .. 0x09BB ; Unknown + 0x09BC, # .. 0x09C4 ; Bengali + 0x09C5, # .. 0x09C6 ; Unknown + 0x09C7, # .. 0x09C8 ; Bengali + 0x09C9, # .. 0x09CA ; Unknown + 0x09CB, # .. 0x09CE ; Bengali + 0x09CF, # .. 0x09D6 ; Unknown + 0x09D7, # .. 0x09D7 ; Bengali + 0x09D8, # .. 0x09DB ; Unknown + 0x09DC, # .. 0x09DD ; Bengali + 0x09DE, # .. 0x09DE ; Unknown + 0x09DF, # .. 0x09E3 ; Bengali + 0x09E4, # .. 0x09E5 ; Unknown + 0x09E6, # .. 0x09FE ; Bengali + 0x09FF, # .. 0x0A00 ; Unknown + 0x0A01, # .. 0x0A03 ; Gurmukhi + 0x0A04, # .. 0x0A04 ; Unknown + 0x0A05, # .. 0x0A0A ; Gurmukhi + 0x0A0B, # .. 0x0A0E ; Unknown + 0x0A0F, # .. 0x0A10 ; Gurmukhi + 0x0A11, # .. 0x0A12 ; Unknown + 0x0A13, # .. 0x0A28 ; Gurmukhi + 0x0A29, # .. 0x0A29 ; Unknown + 0x0A2A, # .. 0x0A30 ; Gurmukhi + 0x0A31, # .. 0x0A31 ; Unknown + 0x0A32, # .. 0x0A33 ; Gurmukhi + 0x0A34, # .. 0x0A34 ; Unknown + 0x0A35, # .. 0x0A36 ; Gurmukhi + 0x0A37, # .. 0x0A37 ; Unknown + 0x0A38, # .. 0x0A39 ; Gurmukhi + 0x0A3A, # .. 0x0A3B ; Unknown + 0x0A3C, # .. 0x0A3C ; Gurmukhi + 0x0A3D, # .. 0x0A3D ; Unknown + 0x0A3E, # .. 0x0A42 ; Gurmukhi + 0x0A43, # .. 0x0A46 ; Unknown + 0x0A47, # .. 0x0A48 ; Gurmukhi + 0x0A49, # .. 0x0A4A ; Unknown + 0x0A4B, # .. 0x0A4D ; Gurmukhi + 0x0A4E, # .. 0x0A50 ; Unknown + 0x0A51, # .. 0x0A51 ; Gurmukhi + 0x0A52, # .. 0x0A58 ; Unknown + 0x0A59, # .. 0x0A5C ; Gurmukhi + 0x0A5D, # .. 0x0A5D ; Unknown + 0x0A5E, # .. 0x0A5E ; Gurmukhi + 0x0A5F, # .. 0x0A65 ; Unknown + 0x0A66, # .. 0x0A76 ; Gurmukhi + 0x0A77, # .. 0x0A80 ; Unknown + 0x0A81, # .. 0x0A83 ; Gujarati + 0x0A84, # .. 0x0A84 ; Unknown + 0x0A85, # .. 0x0A8D ; Gujarati + 0x0A8E, # .. 0x0A8E ; Unknown + 0x0A8F, # .. 0x0A91 ; Gujarati + 0x0A92, # .. 0x0A92 ; Unknown + 0x0A93, # .. 0x0AA8 ; Gujarati + 0x0AA9, # .. 0x0AA9 ; Unknown + 0x0AAA, # .. 0x0AB0 ; Gujarati + 0x0AB1, # .. 0x0AB1 ; Unknown + 0x0AB2, # .. 0x0AB3 ; Gujarati + 0x0AB4, # .. 0x0AB4 ; Unknown + 0x0AB5, # .. 0x0AB9 ; Gujarati + 0x0ABA, # .. 0x0ABB ; Unknown + 0x0ABC, # .. 0x0AC5 ; Gujarati + 0x0AC6, # .. 0x0AC6 ; Unknown + 0x0AC7, # .. 0x0AC9 ; Gujarati + 0x0ACA, # .. 0x0ACA ; Unknown + 0x0ACB, # .. 0x0ACD ; Gujarati + 0x0ACE, # .. 0x0ACF ; Unknown + 0x0AD0, # .. 0x0AD0 ; Gujarati + 0x0AD1, # .. 0x0ADF ; Unknown + 0x0AE0, # .. 0x0AE3 ; Gujarati + 0x0AE4, # .. 0x0AE5 ; Unknown + 0x0AE6, # .. 0x0AF1 ; Gujarati + 0x0AF2, # .. 0x0AF8 ; Unknown + 0x0AF9, # .. 0x0AFF ; Gujarati + 0x0B00, # .. 0x0B00 ; Unknown + 0x0B01, # .. 0x0B03 ; Oriya + 0x0B04, # .. 0x0B04 ; Unknown + 0x0B05, # .. 0x0B0C ; Oriya + 0x0B0D, # .. 0x0B0E ; Unknown + 0x0B0F, # .. 0x0B10 ; Oriya + 0x0B11, # .. 0x0B12 ; Unknown + 0x0B13, # .. 0x0B28 ; Oriya + 0x0B29, # .. 0x0B29 ; Unknown + 0x0B2A, # .. 0x0B30 ; Oriya + 0x0B31, # .. 0x0B31 ; Unknown + 0x0B32, # .. 0x0B33 ; Oriya + 0x0B34, # .. 0x0B34 ; Unknown + 0x0B35, # .. 0x0B39 ; Oriya + 0x0B3A, # .. 0x0B3B ; Unknown + 0x0B3C, # .. 0x0B44 ; Oriya + 0x0B45, # .. 0x0B46 ; Unknown + 0x0B47, # .. 0x0B48 ; Oriya + 0x0B49, # .. 0x0B4A ; Unknown + 0x0B4B, # .. 0x0B4D ; Oriya + 0x0B4E, # .. 0x0B54 ; Unknown + 0x0B55, # .. 0x0B57 ; Oriya + 0x0B58, # .. 0x0B5B ; Unknown + 0x0B5C, # .. 0x0B5D ; Oriya + 0x0B5E, # .. 0x0B5E ; Unknown + 0x0B5F, # .. 0x0B63 ; Oriya + 0x0B64, # .. 0x0B65 ; Unknown + 0x0B66, # .. 0x0B77 ; Oriya + 0x0B78, # .. 0x0B81 ; Unknown + 0x0B82, # .. 0x0B83 ; Tamil + 0x0B84, # .. 0x0B84 ; Unknown + 0x0B85, # .. 0x0B8A ; Tamil + 0x0B8B, # .. 0x0B8D ; Unknown + 0x0B8E, # .. 0x0B90 ; Tamil + 0x0B91, # .. 0x0B91 ; Unknown + 0x0B92, # .. 0x0B95 ; Tamil + 0x0B96, # .. 0x0B98 ; Unknown + 0x0B99, # .. 0x0B9A ; Tamil + 0x0B9B, # .. 0x0B9B ; Unknown + 0x0B9C, # .. 0x0B9C ; Tamil + 0x0B9D, # .. 0x0B9D ; Unknown + 0x0B9E, # .. 0x0B9F ; Tamil + 0x0BA0, # .. 0x0BA2 ; Unknown + 0x0BA3, # .. 0x0BA4 ; Tamil + 0x0BA5, # .. 0x0BA7 ; Unknown + 0x0BA8, # .. 0x0BAA ; Tamil + 0x0BAB, # .. 0x0BAD ; Unknown + 0x0BAE, # .. 0x0BB9 ; Tamil + 0x0BBA, # .. 0x0BBD ; Unknown + 0x0BBE, # .. 0x0BC2 ; Tamil + 0x0BC3, # .. 0x0BC5 ; Unknown + 0x0BC6, # .. 0x0BC8 ; Tamil + 0x0BC9, # .. 0x0BC9 ; Unknown + 0x0BCA, # .. 0x0BCD ; Tamil + 0x0BCE, # .. 0x0BCF ; Unknown + 0x0BD0, # .. 0x0BD0 ; Tamil + 0x0BD1, # .. 0x0BD6 ; Unknown + 0x0BD7, # .. 0x0BD7 ; Tamil + 0x0BD8, # .. 0x0BE5 ; Unknown + 0x0BE6, # .. 0x0BFA ; Tamil + 0x0BFB, # .. 0x0BFF ; Unknown + 0x0C00, # .. 0x0C0C ; Telugu + 0x0C0D, # .. 0x0C0D ; Unknown + 0x0C0E, # .. 0x0C10 ; Telugu + 0x0C11, # .. 0x0C11 ; Unknown + 0x0C12, # .. 0x0C28 ; Telugu + 0x0C29, # .. 0x0C29 ; Unknown + 0x0C2A, # .. 0x0C39 ; Telugu + 0x0C3A, # .. 0x0C3B ; Unknown + 0x0C3C, # .. 0x0C44 ; Telugu + 0x0C45, # .. 0x0C45 ; Unknown + 0x0C46, # .. 0x0C48 ; Telugu + 0x0C49, # .. 0x0C49 ; Unknown + 0x0C4A, # .. 0x0C4D ; Telugu + 0x0C4E, # .. 0x0C54 ; Unknown + 0x0C55, # .. 0x0C56 ; Telugu + 0x0C57, # .. 0x0C57 ; Unknown + 0x0C58, # .. 0x0C5A ; Telugu + 0x0C5B, # .. 0x0C5B ; Unknown + 0x0C5C, # .. 0x0C5D ; Telugu + 0x0C5E, # .. 0x0C5F ; Unknown + 0x0C60, # .. 0x0C63 ; Telugu + 0x0C64, # .. 0x0C65 ; Unknown + 0x0C66, # .. 0x0C6F ; Telugu + 0x0C70, # .. 0x0C76 ; Unknown + 0x0C77, # .. 0x0C7F ; Telugu + 0x0C80, # .. 0x0C8C ; Kannada + 0x0C8D, # .. 0x0C8D ; Unknown + 0x0C8E, # .. 0x0C90 ; Kannada + 0x0C91, # .. 0x0C91 ; Unknown + 0x0C92, # .. 0x0CA8 ; Kannada + 0x0CA9, # .. 0x0CA9 ; Unknown + 0x0CAA, # .. 0x0CB3 ; Kannada + 0x0CB4, # .. 0x0CB4 ; Unknown + 0x0CB5, # .. 0x0CB9 ; Kannada + 0x0CBA, # .. 0x0CBB ; Unknown + 0x0CBC, # .. 0x0CC4 ; Kannada + 0x0CC5, # .. 0x0CC5 ; Unknown + 0x0CC6, # .. 0x0CC8 ; Kannada + 0x0CC9, # .. 0x0CC9 ; Unknown + 0x0CCA, # .. 0x0CCD ; Kannada + 0x0CCE, # .. 0x0CD4 ; Unknown + 0x0CD5, # .. 0x0CD6 ; Kannada + 0x0CD7, # .. 0x0CDB ; Unknown + 0x0CDC, # .. 0x0CDE ; Kannada + 0x0CDF, # .. 0x0CDF ; Unknown + 0x0CE0, # .. 0x0CE3 ; Kannada + 0x0CE4, # .. 0x0CE5 ; Unknown + 0x0CE6, # .. 0x0CEF ; Kannada + 0x0CF0, # .. 0x0CF0 ; Unknown + 0x0CF1, # .. 0x0CF3 ; Kannada + 0x0CF4, # .. 0x0CFF ; Unknown + 0x0D00, # .. 0x0D0C ; Malayalam + 0x0D0D, # .. 0x0D0D ; Unknown + 0x0D0E, # .. 0x0D10 ; Malayalam + 0x0D11, # .. 0x0D11 ; Unknown + 0x0D12, # .. 0x0D44 ; Malayalam + 0x0D45, # .. 0x0D45 ; Unknown + 0x0D46, # .. 0x0D48 ; Malayalam + 0x0D49, # .. 0x0D49 ; Unknown + 0x0D4A, # .. 0x0D4F ; Malayalam + 0x0D50, # .. 0x0D53 ; Unknown + 0x0D54, # .. 0x0D63 ; Malayalam + 0x0D64, # .. 0x0D65 ; Unknown + 0x0D66, # .. 0x0D7F ; Malayalam + 0x0D80, # .. 0x0D80 ; Unknown + 0x0D81, # .. 0x0D83 ; Sinhala + 0x0D84, # .. 0x0D84 ; Unknown + 0x0D85, # .. 0x0D96 ; Sinhala + 0x0D97, # .. 0x0D99 ; Unknown + 0x0D9A, # .. 0x0DB1 ; Sinhala + 0x0DB2, # .. 0x0DB2 ; Unknown + 0x0DB3, # .. 0x0DBB ; Sinhala + 0x0DBC, # .. 0x0DBC ; Unknown + 0x0DBD, # .. 0x0DBD ; Sinhala + 0x0DBE, # .. 0x0DBF ; Unknown + 0x0DC0, # .. 0x0DC6 ; Sinhala + 0x0DC7, # .. 0x0DC9 ; Unknown + 0x0DCA, # .. 0x0DCA ; Sinhala + 0x0DCB, # .. 0x0DCE ; Unknown + 0x0DCF, # .. 0x0DD4 ; Sinhala + 0x0DD5, # .. 0x0DD5 ; Unknown + 0x0DD6, # .. 0x0DD6 ; Sinhala + 0x0DD7, # .. 0x0DD7 ; Unknown + 0x0DD8, # .. 0x0DDF ; Sinhala + 0x0DE0, # .. 0x0DE5 ; Unknown + 0x0DE6, # .. 0x0DEF ; Sinhala + 0x0DF0, # .. 0x0DF1 ; Unknown + 0x0DF2, # .. 0x0DF4 ; Sinhala + 0x0DF5, # .. 0x0E00 ; Unknown + 0x0E01, # .. 0x0E3A ; Thai + 0x0E3B, # .. 0x0E3E ; Unknown + 0x0E3F, # .. 0x0E3F ; Common + 0x0E40, # .. 0x0E5B ; Thai + 0x0E5C, # .. 0x0E80 ; Unknown + 0x0E81, # .. 0x0E82 ; Lao + 0x0E83, # .. 0x0E83 ; Unknown + 0x0E84, # .. 0x0E84 ; Lao + 0x0E85, # .. 0x0E85 ; Unknown + 0x0E86, # .. 0x0E8A ; Lao + 0x0E8B, # .. 0x0E8B ; Unknown + 0x0E8C, # .. 0x0EA3 ; Lao + 0x0EA4, # .. 0x0EA4 ; Unknown + 0x0EA5, # .. 0x0EA5 ; Lao + 0x0EA6, # .. 0x0EA6 ; Unknown + 0x0EA7, # .. 0x0EBD ; Lao + 0x0EBE, # .. 0x0EBF ; Unknown + 0x0EC0, # .. 0x0EC4 ; Lao + 0x0EC5, # .. 0x0EC5 ; Unknown + 0x0EC6, # .. 0x0EC6 ; Lao + 0x0EC7, # .. 0x0EC7 ; Unknown + 0x0EC8, # .. 0x0ECE ; Lao + 0x0ECF, # .. 0x0ECF ; Unknown + 0x0ED0, # .. 0x0ED9 ; Lao + 0x0EDA, # .. 0x0EDB ; Unknown + 0x0EDC, # .. 0x0EDF ; Lao + 0x0EE0, # .. 0x0EFF ; Unknown + 0x0F00, # .. 0x0F47 ; Tibetan + 0x0F48, # .. 0x0F48 ; Unknown + 0x0F49, # .. 0x0F6C ; Tibetan + 0x0F6D, # .. 0x0F70 ; Unknown + 0x0F71, # .. 0x0F97 ; Tibetan + 0x0F98, # .. 0x0F98 ; Unknown + 0x0F99, # .. 0x0FBC ; Tibetan + 0x0FBD, # .. 0x0FBD ; Unknown + 0x0FBE, # .. 0x0FCC ; Tibetan + 0x0FCD, # .. 0x0FCD ; Unknown + 0x0FCE, # .. 0x0FD4 ; Tibetan + 0x0FD5, # .. 0x0FD8 ; Common + 0x0FD9, # .. 0x0FDA ; Tibetan + 0x0FDB, # .. 0x0FFF ; Unknown + 0x1000, # .. 0x109F ; Myanmar + 0x10A0, # .. 0x10C5 ; Georgian + 0x10C6, # .. 0x10C6 ; Unknown + 0x10C7, # .. 0x10C7 ; Georgian + 0x10C8, # .. 0x10CC ; Unknown + 0x10CD, # .. 0x10CD ; Georgian + 0x10CE, # .. 0x10CF ; Unknown + 0x10D0, # .. 0x10FA ; Georgian + 0x10FB, # .. 0x10FB ; Common + 0x10FC, # .. 0x10FF ; Georgian + 0x1100, # .. 0x11FF ; Hangul + 0x1200, # .. 0x1248 ; Ethiopic + 0x1249, # .. 0x1249 ; Unknown + 0x124A, # .. 0x124D ; Ethiopic + 0x124E, # .. 0x124F ; Unknown + 0x1250, # .. 0x1256 ; Ethiopic + 0x1257, # .. 0x1257 ; Unknown + 0x1258, # .. 0x1258 ; Ethiopic + 0x1259, # .. 0x1259 ; Unknown + 0x125A, # .. 0x125D ; Ethiopic + 0x125E, # .. 0x125F ; Unknown + 0x1260, # .. 0x1288 ; Ethiopic + 0x1289, # .. 0x1289 ; Unknown + 0x128A, # .. 0x128D ; Ethiopic + 0x128E, # .. 0x128F ; Unknown + 0x1290, # .. 0x12B0 ; Ethiopic + 0x12B1, # .. 0x12B1 ; Unknown + 0x12B2, # .. 0x12B5 ; Ethiopic + 0x12B6, # .. 0x12B7 ; Unknown + 0x12B8, # .. 0x12BE ; Ethiopic + 0x12BF, # .. 0x12BF ; Unknown + 0x12C0, # .. 0x12C0 ; Ethiopic + 0x12C1, # .. 0x12C1 ; Unknown + 0x12C2, # .. 0x12C5 ; Ethiopic + 0x12C6, # .. 0x12C7 ; Unknown + 0x12C8, # .. 0x12D6 ; Ethiopic + 0x12D7, # .. 0x12D7 ; Unknown + 0x12D8, # .. 0x1310 ; Ethiopic + 0x1311, # .. 0x1311 ; Unknown + 0x1312, # .. 0x1315 ; Ethiopic + 0x1316, # .. 0x1317 ; Unknown + 0x1318, # .. 0x135A ; Ethiopic + 0x135B, # .. 0x135C ; Unknown + 0x135D, # .. 0x137C ; Ethiopic + 0x137D, # .. 0x137F ; Unknown + 0x1380, # .. 0x1399 ; Ethiopic + 0x139A, # .. 0x139F ; Unknown + 0x13A0, # .. 0x13F5 ; Cherokee + 0x13F6, # .. 0x13F7 ; Unknown + 0x13F8, # .. 0x13FD ; Cherokee + 0x13FE, # .. 0x13FF ; Unknown + 0x1400, # .. 0x167F ; Canadian_Aboriginal + 0x1680, # .. 0x169C ; Ogham + 0x169D, # .. 0x169F ; Unknown + 0x16A0, # .. 0x16EA ; Runic + 0x16EB, # .. 0x16ED ; Common + 0x16EE, # .. 0x16F8 ; Runic + 0x16F9, # .. 0x16FF ; Unknown + 0x1700, # .. 0x1715 ; Tagalog + 0x1716, # .. 0x171E ; Unknown + 0x171F, # .. 0x171F ; Tagalog + 0x1720, # .. 0x1734 ; Hanunoo + 0x1735, # .. 0x1736 ; Common + 0x1737, # .. 0x173F ; Unknown + 0x1740, # .. 0x1753 ; Buhid + 0x1754, # .. 0x175F ; Unknown + 0x1760, # .. 0x176C ; Tagbanwa + 0x176D, # .. 0x176D ; Unknown + 0x176E, # .. 0x1770 ; Tagbanwa + 0x1771, # .. 0x1771 ; Unknown + 0x1772, # .. 0x1773 ; Tagbanwa + 0x1774, # .. 0x177F ; Unknown + 0x1780, # .. 0x17DD ; Khmer + 0x17DE, # .. 0x17DF ; Unknown + 0x17E0, # .. 0x17E9 ; Khmer + 0x17EA, # .. 0x17EF ; Unknown + 0x17F0, # .. 0x17F9 ; Khmer + 0x17FA, # .. 0x17FF ; Unknown + 0x1800, # .. 0x1801 ; Mongolian + 0x1802, # .. 0x1803 ; Common + 0x1804, # .. 0x1804 ; Mongolian + 0x1805, # .. 0x1805 ; Common + 0x1806, # .. 0x1819 ; Mongolian + 0x181A, # .. 0x181F ; Unknown + 0x1820, # .. 0x1878 ; Mongolian + 0x1879, # .. 0x187F ; Unknown + 0x1880, # .. 0x18AA ; Mongolian + 0x18AB, # .. 0x18AF ; Unknown + 0x18B0, # .. 0x18F5 ; Canadian_Aboriginal + 0x18F6, # .. 0x18FF ; Unknown + 0x1900, # .. 0x191E ; Limbu + 0x191F, # .. 0x191F ; Unknown + 0x1920, # .. 0x192B ; Limbu + 0x192C, # .. 0x192F ; Unknown + 0x1930, # .. 0x193B ; Limbu + 0x193C, # .. 0x193F ; Unknown + 0x1940, # .. 0x1940 ; Limbu + 0x1941, # .. 0x1943 ; Unknown + 0x1944, # .. 0x194F ; Limbu + 0x1950, # .. 0x196D ; Tai_Le + 0x196E, # .. 0x196F ; Unknown + 0x1970, # .. 0x1974 ; Tai_Le + 0x1975, # .. 0x197F ; Unknown + 0x1980, # .. 0x19AB ; New_Tai_Lue + 0x19AC, # .. 0x19AF ; Unknown + 0x19B0, # .. 0x19C9 ; New_Tai_Lue + 0x19CA, # .. 0x19CF ; Unknown + 0x19D0, # .. 0x19DA ; New_Tai_Lue + 0x19DB, # .. 0x19DD ; Unknown + 0x19DE, # .. 0x19DF ; New_Tai_Lue + 0x19E0, # .. 0x19FF ; Khmer + 0x1A00, # .. 0x1A1B ; Buginese + 0x1A1C, # .. 0x1A1D ; Unknown + 0x1A1E, # .. 0x1A1F ; Buginese + 0x1A20, # .. 0x1A5E ; Tai_Tham + 0x1A5F, # .. 0x1A5F ; Unknown + 0x1A60, # .. 0x1A7C ; Tai_Tham + 0x1A7D, # .. 0x1A7E ; Unknown + 0x1A7F, # .. 0x1A89 ; Tai_Tham + 0x1A8A, # .. 0x1A8F ; Unknown + 0x1A90, # .. 0x1A99 ; Tai_Tham + 0x1A9A, # .. 0x1A9F ; Unknown + 0x1AA0, # .. 0x1AAD ; Tai_Tham + 0x1AAE, # .. 0x1AAF ; Unknown + 0x1AB0, # .. 0x1ADD ; Inherited + 0x1ADE, # .. 0x1ADF ; Unknown + 0x1AE0, # .. 0x1AEB ; Inherited + 0x1AEC, # .. 0x1AFF ; Unknown + 0x1B00, # .. 0x1B4C ; Balinese + 0x1B4D, # .. 0x1B4D ; Unknown + 0x1B4E, # .. 0x1B7F ; Balinese + 0x1B80, # .. 0x1BBF ; Sundanese + 0x1BC0, # .. 0x1BF3 ; Batak + 0x1BF4, # .. 0x1BFB ; Unknown + 0x1BFC, # .. 0x1BFF ; Batak + 0x1C00, # .. 0x1C37 ; Lepcha + 0x1C38, # .. 0x1C3A ; Unknown + 0x1C3B, # .. 0x1C49 ; Lepcha + 0x1C4A, # .. 0x1C4C ; Unknown + 0x1C4D, # .. 0x1C4F ; Lepcha + 0x1C50, # .. 0x1C7F ; Ol_Chiki + 0x1C80, # .. 0x1C8A ; Cyrillic + 0x1C8B, # .. 0x1C8F ; Unknown + 0x1C90, # .. 0x1CBA ; Georgian + 0x1CBB, # .. 0x1CBC ; Unknown + 0x1CBD, # .. 0x1CBF ; Georgian + 0x1CC0, # .. 0x1CC7 ; Sundanese + 0x1CC8, # .. 0x1CCF ; Unknown + 0x1CD0, # .. 0x1CD2 ; Inherited + 0x1CD3, # .. 0x1CD3 ; Common + 0x1CD4, # .. 0x1CE0 ; Inherited + 0x1CE1, # .. 0x1CE1 ; Common + 0x1CE2, # .. 0x1CE8 ; Inherited + 0x1CE9, # .. 0x1CEC ; Common + 0x1CED, # .. 0x1CED ; Inherited + 0x1CEE, # .. 0x1CF3 ; Common + 0x1CF4, # .. 0x1CF4 ; Inherited + 0x1CF5, # .. 0x1CF7 ; Common + 0x1CF8, # .. 0x1CF9 ; Inherited + 0x1CFA, # .. 0x1CFA ; Common + 0x1CFB, # .. 0x1CFF ; Unknown + 0x1D00, # .. 0x1D25 ; Latin + 0x1D26, # .. 0x1D2A ; Greek + 0x1D2B, # .. 0x1D2B ; Cyrillic + 0x1D2C, # .. 0x1D5C ; Latin + 0x1D5D, # .. 0x1D61 ; Greek + 0x1D62, # .. 0x1D65 ; Latin + 0x1D66, # .. 0x1D6A ; Greek + 0x1D6B, # .. 0x1D77 ; Latin + 0x1D78, # .. 0x1D78 ; Cyrillic + 0x1D79, # .. 0x1DBE ; Latin + 0x1DBF, # .. 0x1DBF ; Greek + 0x1DC0, # .. 0x1DFF ; Inherited + 0x1E00, # .. 0x1EFF ; Latin + 0x1F00, # .. 0x1F15 ; Greek + 0x1F16, # .. 0x1F17 ; Unknown + 0x1F18, # .. 0x1F1D ; Greek + 0x1F1E, # .. 0x1F1F ; Unknown + 0x1F20, # .. 0x1F45 ; Greek + 0x1F46, # .. 0x1F47 ; Unknown + 0x1F48, # .. 0x1F4D ; Greek + 0x1F4E, # .. 0x1F4F ; Unknown + 0x1F50, # .. 0x1F57 ; Greek + 0x1F58, # .. 0x1F58 ; Unknown + 0x1F59, # .. 0x1F59 ; Greek + 0x1F5A, # .. 0x1F5A ; Unknown + 0x1F5B, # .. 0x1F5B ; Greek + 0x1F5C, # .. 0x1F5C ; Unknown + 0x1F5D, # .. 0x1F5D ; Greek + 0x1F5E, # .. 0x1F5E ; Unknown + 0x1F5F, # .. 0x1F7D ; Greek + 0x1F7E, # .. 0x1F7F ; Unknown + 0x1F80, # .. 0x1FB4 ; Greek + 0x1FB5, # .. 0x1FB5 ; Unknown + 0x1FB6, # .. 0x1FC4 ; Greek + 0x1FC5, # .. 0x1FC5 ; Unknown + 0x1FC6, # .. 0x1FD3 ; Greek + 0x1FD4, # .. 0x1FD5 ; Unknown + 0x1FD6, # .. 0x1FDB ; Greek + 0x1FDC, # .. 0x1FDC ; Unknown + 0x1FDD, # .. 0x1FEF ; Greek + 0x1FF0, # .. 0x1FF1 ; Unknown + 0x1FF2, # .. 0x1FF4 ; Greek + 0x1FF5, # .. 0x1FF5 ; Unknown + 0x1FF6, # .. 0x1FFE ; Greek + 0x1FFF, # .. 0x1FFF ; Unknown + 0x2000, # .. 0x200B ; Common + 0x200C, # .. 0x200D ; Inherited + 0x200E, # .. 0x2064 ; Common + 0x2065, # .. 0x2065 ; Unknown + 0x2066, # .. 0x2070 ; Common + 0x2071, # .. 0x2071 ; Latin + 0x2072, # .. 0x2073 ; Unknown + 0x2074, # .. 0x207E ; Common + 0x207F, # .. 0x207F ; Latin + 0x2080, # .. 0x208E ; Common + 0x208F, # .. 0x208F ; Unknown + 0x2090, # .. 0x209C ; Latin + 0x209D, # .. 0x209F ; Unknown + 0x20A0, # .. 0x20C1 ; Common + 0x20C2, # .. 0x20CF ; Unknown + 0x20D0, # .. 0x20F0 ; Inherited + 0x20F1, # .. 0x20FF ; Unknown + 0x2100, # .. 0x2125 ; Common + 0x2126, # .. 0x2126 ; Greek + 0x2127, # .. 0x2129 ; Common + 0x212A, # .. 0x212B ; Latin + 0x212C, # .. 0x2131 ; Common + 0x2132, # .. 0x2132 ; Latin + 0x2133, # .. 0x214D ; Common + 0x214E, # .. 0x214E ; Latin + 0x214F, # .. 0x215F ; Common + 0x2160, # .. 0x2188 ; Latin + 0x2189, # .. 0x218B ; Common + 0x218C, # .. 0x218F ; Unknown + 0x2190, # .. 0x2429 ; Common + 0x242A, # .. 0x243F ; Unknown + 0x2440, # .. 0x244A ; Common + 0x244B, # .. 0x245F ; Unknown + 0x2460, # .. 0x27FF ; Common + 0x2800, # .. 0x28FF ; Braille + 0x2900, # .. 0x2B73 ; Common + 0x2B74, # .. 0x2B75 ; Unknown + 0x2B76, # .. 0x2BFF ; Common + 0x2C00, # .. 0x2C5F ; Glagolitic + 0x2C60, # .. 0x2C7F ; Latin + 0x2C80, # .. 0x2CF3 ; Coptic + 0x2CF4, # .. 0x2CF8 ; Unknown + 0x2CF9, # .. 0x2CFF ; Coptic + 0x2D00, # .. 0x2D25 ; Georgian + 0x2D26, # .. 0x2D26 ; Unknown + 0x2D27, # .. 0x2D27 ; Georgian + 0x2D28, # .. 0x2D2C ; Unknown + 0x2D2D, # .. 0x2D2D ; Georgian + 0x2D2E, # .. 0x2D2F ; Unknown + 0x2D30, # .. 0x2D67 ; Tifinagh + 0x2D68, # .. 0x2D6E ; Unknown + 0x2D6F, # .. 0x2D70 ; Tifinagh + 0x2D71, # .. 0x2D7E ; Unknown + 0x2D7F, # .. 0x2D7F ; Tifinagh + 0x2D80, # .. 0x2D96 ; Ethiopic + 0x2D97, # .. 0x2D9F ; Unknown + 0x2DA0, # .. 0x2DA6 ; Ethiopic + 0x2DA7, # .. 0x2DA7 ; Unknown + 0x2DA8, # .. 0x2DAE ; Ethiopic + 0x2DAF, # .. 0x2DAF ; Unknown + 0x2DB0, # .. 0x2DB6 ; Ethiopic + 0x2DB7, # .. 0x2DB7 ; Unknown + 0x2DB8, # .. 0x2DBE ; Ethiopic + 0x2DBF, # .. 0x2DBF ; Unknown + 0x2DC0, # .. 0x2DC6 ; Ethiopic + 0x2DC7, # .. 0x2DC7 ; Unknown + 0x2DC8, # .. 0x2DCE ; Ethiopic + 0x2DCF, # .. 0x2DCF ; Unknown + 0x2DD0, # .. 0x2DD6 ; Ethiopic + 0x2DD7, # .. 0x2DD7 ; Unknown + 0x2DD8, # .. 0x2DDE ; Ethiopic + 0x2DDF, # .. 0x2DDF ; Unknown + 0x2DE0, # .. 0x2DFF ; Cyrillic + 0x2E00, # .. 0x2E5D ; Common + 0x2E5E, # .. 0x2E7F ; Unknown + 0x2E80, # .. 0x2E99 ; Han + 0x2E9A, # .. 0x2E9A ; Unknown + 0x2E9B, # .. 0x2EF3 ; Han + 0x2EF4, # .. 0x2EFF ; Unknown + 0x2F00, # .. 0x2FD5 ; Han + 0x2FD6, # .. 0x2FEF ; Unknown + 0x2FF0, # .. 0x3004 ; Common + 0x3005, # .. 0x3005 ; Han + 0x3006, # .. 0x3006 ; Common + 0x3007, # .. 0x3007 ; Han + 0x3008, # .. 0x3020 ; Common + 0x3021, # .. 0x3029 ; Han + 0x302A, # .. 0x302D ; Inherited + 0x302E, # .. 0x302F ; Hangul + 0x3030, # .. 0x3037 ; Common + 0x3038, # .. 0x303B ; Han + 0x303C, # .. 0x303F ; Common + 0x3040, # .. 0x3040 ; Unknown + 0x3041, # .. 0x3096 ; Hiragana + 0x3097, # .. 0x3098 ; Unknown + 0x3099, # .. 0x309A ; Inherited + 0x309B, # .. 0x309C ; Common + 0x309D, # .. 0x309F ; Hiragana + 0x30A0, # .. 0x30A0 ; Common + 0x30A1, # .. 0x30FA ; Katakana + 0x30FB, # .. 0x30FC ; Common + 0x30FD, # .. 0x30FF ; Katakana + 0x3100, # .. 0x3104 ; Unknown + 0x3105, # .. 0x312F ; Bopomofo + 0x3130, # .. 0x3130 ; Unknown + 0x3131, # .. 0x318E ; Hangul + 0x318F, # .. 0x318F ; Unknown + 0x3190, # .. 0x319F ; Common + 0x31A0, # .. 0x31BF ; Bopomofo + 0x31C0, # .. 0x31E5 ; Common + 0x31E6, # .. 0x31EE ; Unknown + 0x31EF, # .. 0x31EF ; Common + 0x31F0, # .. 0x31FF ; Katakana + 0x3200, # .. 0x321E ; Hangul + 0x321F, # .. 0x321F ; Unknown + 0x3220, # .. 0x325F ; Common + 0x3260, # .. 0x327E ; Hangul + 0x327F, # .. 0x32CF ; Common + 0x32D0, # .. 0x32FE ; Katakana + 0x32FF, # .. 0x32FF ; Common + 0x3300, # .. 0x3357 ; Katakana + 0x3358, # .. 0x33FF ; Common + 0x3400, # .. 0x4DBF ; Han + 0x4DC0, # .. 0x4DFF ; Common + 0x4E00, # .. 0x9FFF ; Han + 0xA000, # .. 0xA48C ; Yi + 0xA48D, # .. 0xA48F ; Unknown + 0xA490, # .. 0xA4C6 ; Yi + 0xA4C7, # .. 0xA4CF ; Unknown + 0xA4D0, # .. 0xA4FF ; Lisu + 0xA500, # .. 0xA62B ; Vai + 0xA62C, # .. 0xA63F ; Unknown + 0xA640, # .. 0xA69F ; Cyrillic + 0xA6A0, # .. 0xA6F7 ; Bamum + 0xA6F8, # .. 0xA6FF ; Unknown + 0xA700, # .. 0xA721 ; Common + 0xA722, # .. 0xA787 ; Latin + 0xA788, # .. 0xA78A ; Common + 0xA78B, # .. 0xA7DC ; Latin + 0xA7DD, # .. 0xA7F0 ; Unknown + 0xA7F1, # .. 0xA7FF ; Latin + 0xA800, # .. 0xA82C ; Syloti_Nagri + 0xA82D, # .. 0xA82F ; Unknown + 0xA830, # .. 0xA839 ; Common + 0xA83A, # .. 0xA83F ; Unknown + 0xA840, # .. 0xA877 ; Phags_Pa + 0xA878, # .. 0xA87F ; Unknown + 0xA880, # .. 0xA8C5 ; Saurashtra + 0xA8C6, # .. 0xA8CD ; Unknown + 0xA8CE, # .. 0xA8D9 ; Saurashtra + 0xA8DA, # .. 0xA8DF ; Unknown + 0xA8E0, # .. 0xA8FF ; Devanagari + 0xA900, # .. 0xA92D ; Kayah_Li + 0xA92E, # .. 0xA92E ; Common + 0xA92F, # .. 0xA92F ; Kayah_Li + 0xA930, # .. 0xA953 ; Rejang + 0xA954, # .. 0xA95E ; Unknown + 0xA95F, # .. 0xA95F ; Rejang + 0xA960, # .. 0xA97C ; Hangul + 0xA97D, # .. 0xA97F ; Unknown + 0xA980, # .. 0xA9CD ; Javanese + 0xA9CE, # .. 0xA9CE ; Unknown + 0xA9CF, # .. 0xA9CF ; Common + 0xA9D0, # .. 0xA9D9 ; Javanese + 0xA9DA, # .. 0xA9DD ; Unknown + 0xA9DE, # .. 0xA9DF ; Javanese + 0xA9E0, # .. 0xA9FE ; Myanmar + 0xA9FF, # .. 0xA9FF ; Unknown + 0xAA00, # .. 0xAA36 ; Cham + 0xAA37, # .. 0xAA3F ; Unknown + 0xAA40, # .. 0xAA4D ; Cham + 0xAA4E, # .. 0xAA4F ; Unknown + 0xAA50, # .. 0xAA59 ; Cham + 0xAA5A, # .. 0xAA5B ; Unknown + 0xAA5C, # .. 0xAA5F ; Cham + 0xAA60, # .. 0xAA7F ; Myanmar + 0xAA80, # .. 0xAAC2 ; Tai_Viet + 0xAAC3, # .. 0xAADA ; Unknown + 0xAADB, # .. 0xAADF ; Tai_Viet + 0xAAE0, # .. 0xAAF6 ; Meetei_Mayek + 0xAAF7, # .. 0xAB00 ; Unknown + 0xAB01, # .. 0xAB06 ; Ethiopic + 0xAB07, # .. 0xAB08 ; Unknown + 0xAB09, # .. 0xAB0E ; Ethiopic + 0xAB0F, # .. 0xAB10 ; Unknown + 0xAB11, # .. 0xAB16 ; Ethiopic + 0xAB17, # .. 0xAB1F ; Unknown + 0xAB20, # .. 0xAB26 ; Ethiopic + 0xAB27, # .. 0xAB27 ; Unknown + 0xAB28, # .. 0xAB2E ; Ethiopic + 0xAB2F, # .. 0xAB2F ; Unknown + 0xAB30, # .. 0xAB5A ; Latin + 0xAB5B, # .. 0xAB5B ; Common + 0xAB5C, # .. 0xAB64 ; Latin + 0xAB65, # .. 0xAB65 ; Greek + 0xAB66, # .. 0xAB69 ; Latin + 0xAB6A, # .. 0xAB6B ; Common + 0xAB6C, # .. 0xAB6F ; Unknown + 0xAB70, # .. 0xABBF ; Cherokee + 0xABC0, # .. 0xABED ; Meetei_Mayek + 0xABEE, # .. 0xABEF ; Unknown + 0xABF0, # .. 0xABF9 ; Meetei_Mayek + 0xABFA, # .. 0xABFF ; Unknown + 0xAC00, # .. 0xD7A3 ; Hangul + 0xD7A4, # .. 0xD7AF ; Unknown + 0xD7B0, # .. 0xD7C6 ; Hangul + 0xD7C7, # .. 0xD7CA ; Unknown + 0xD7CB, # .. 0xD7FB ; Hangul + 0xD7FC, # .. 0xF8FF ; Unknown + 0xF900, # .. 0xFA6D ; Han + 0xFA6E, # .. 0xFA6F ; Unknown + 0xFA70, # .. 0xFAD9 ; Han + 0xFADA, # .. 0xFAFF ; Unknown + 0xFB00, # .. 0xFB06 ; Latin + 0xFB07, # .. 0xFB12 ; Unknown + 0xFB13, # .. 0xFB17 ; Armenian + 0xFB18, # .. 0xFB1C ; Unknown + 0xFB1D, # .. 0xFB36 ; Hebrew + 0xFB37, # .. 0xFB37 ; Unknown + 0xFB38, # .. 0xFB3C ; Hebrew + 0xFB3D, # .. 0xFB3D ; Unknown + 0xFB3E, # .. 0xFB3E ; Hebrew + 0xFB3F, # .. 0xFB3F ; Unknown + 0xFB40, # .. 0xFB41 ; Hebrew + 0xFB42, # .. 0xFB42 ; Unknown + 0xFB43, # .. 0xFB44 ; Hebrew + 0xFB45, # .. 0xFB45 ; Unknown + 0xFB46, # .. 0xFB4F ; Hebrew + 0xFB50, # .. 0xFD3D ; Arabic + 0xFD3E, # .. 0xFD3F ; Common + 0xFD40, # .. 0xFDCF ; Arabic + 0xFDD0, # .. 0xFDEF ; Unknown + 0xFDF0, # .. 0xFDFF ; Arabic + 0xFE00, # .. 0xFE0F ; Inherited + 0xFE10, # .. 0xFE19 ; Common + 0xFE1A, # .. 0xFE1F ; Unknown + 0xFE20, # .. 0xFE2D ; Inherited + 0xFE2E, # .. 0xFE2F ; Cyrillic + 0xFE30, # .. 0xFE52 ; Common + 0xFE53, # .. 0xFE53 ; Unknown + 0xFE54, # .. 0xFE66 ; Common + 0xFE67, # .. 0xFE67 ; Unknown + 0xFE68, # .. 0xFE6B ; Common + 0xFE6C, # .. 0xFE6F ; Unknown + 0xFE70, # .. 0xFE74 ; Arabic + 0xFE75, # .. 0xFE75 ; Unknown + 0xFE76, # .. 0xFEFC ; Arabic + 0xFEFD, # .. 0xFEFE ; Unknown + 0xFEFF, # .. 0xFEFF ; Common + 0xFF00, # .. 0xFF00 ; Unknown + 0xFF01, # .. 0xFF20 ; Common + 0xFF21, # .. 0xFF3A ; Latin + 0xFF3B, # .. 0xFF40 ; Common + 0xFF41, # .. 0xFF5A ; Latin + 0xFF5B, # .. 0xFF65 ; Common + 0xFF66, # .. 0xFF6F ; Katakana + 0xFF70, # .. 0xFF70 ; Common + 0xFF71, # .. 0xFF9D ; Katakana + 0xFF9E, # .. 0xFF9F ; Common + 0xFFA0, # .. 0xFFBE ; Hangul + 0xFFBF, # .. 0xFFC1 ; Unknown + 0xFFC2, # .. 0xFFC7 ; Hangul + 0xFFC8, # .. 0xFFC9 ; Unknown + 0xFFCA, # .. 0xFFCF ; Hangul + 0xFFD0, # .. 0xFFD1 ; Unknown + 0xFFD2, # .. 0xFFD7 ; Hangul + 0xFFD8, # .. 0xFFD9 ; Unknown + 0xFFDA, # .. 0xFFDC ; Hangul + 0xFFDD, # .. 0xFFDF ; Unknown + 0xFFE0, # .. 0xFFE6 ; Common + 0xFFE7, # .. 0xFFE7 ; Unknown + 0xFFE8, # .. 0xFFEE ; Common + 0xFFEF, # .. 0xFFF8 ; Unknown + 0xFFF9, # .. 0xFFFD ; Common + 0xFFFE, # .. 0xFFFF ; Unknown + 0x10000, # .. 0x1000B ; Linear_B + 0x1000C, # .. 0x1000C ; Unknown + 0x1000D, # .. 0x10026 ; Linear_B + 0x10027, # .. 0x10027 ; Unknown + 0x10028, # .. 0x1003A ; Linear_B + 0x1003B, # .. 0x1003B ; Unknown + 0x1003C, # .. 0x1003D ; Linear_B + 0x1003E, # .. 0x1003E ; Unknown + 0x1003F, # .. 0x1004D ; Linear_B + 0x1004E, # .. 0x1004F ; Unknown + 0x10050, # .. 0x1005D ; Linear_B + 0x1005E, # .. 0x1007F ; Unknown + 0x10080, # .. 0x100FA ; Linear_B + 0x100FB, # .. 0x100FF ; Unknown + 0x10100, # .. 0x10102 ; Common + 0x10103, # .. 0x10106 ; Unknown + 0x10107, # .. 0x10133 ; Common + 0x10134, # .. 0x10136 ; Unknown + 0x10137, # .. 0x1013F ; Common + 0x10140, # .. 0x1018E ; Greek + 0x1018F, # .. 0x1018F ; Unknown + 0x10190, # .. 0x1019C ; Common + 0x1019D, # .. 0x1019F ; Unknown + 0x101A0, # .. 0x101A0 ; Greek + 0x101A1, # .. 0x101CF ; Unknown + 0x101D0, # .. 0x101FC ; Common + 0x101FD, # .. 0x101FD ; Inherited + 0x101FE, # .. 0x1027F ; Unknown + 0x10280, # .. 0x1029C ; Lycian + 0x1029D, # .. 0x1029F ; Unknown + 0x102A0, # .. 0x102D0 ; Carian + 0x102D1, # .. 0x102DF ; Unknown + 0x102E0, # .. 0x102E0 ; Inherited + 0x102E1, # .. 0x102FB ; Common + 0x102FC, # .. 0x102FF ; Unknown + 0x10300, # .. 0x10323 ; Old_Italic + 0x10324, # .. 0x1032C ; Unknown + 0x1032D, # .. 0x1032F ; Old_Italic + 0x10330, # .. 0x1034A ; Gothic + 0x1034B, # .. 0x1034F ; Unknown + 0x10350, # .. 0x1037A ; Old_Permic + 0x1037B, # .. 0x1037F ; Unknown + 0x10380, # .. 0x1039D ; Ugaritic + 0x1039E, # .. 0x1039E ; Unknown + 0x1039F, # .. 0x1039F ; Ugaritic + 0x103A0, # .. 0x103C3 ; Old_Persian + 0x103C4, # .. 0x103C7 ; Unknown + 0x103C8, # .. 0x103D5 ; Old_Persian + 0x103D6, # .. 0x103FF ; Unknown + 0x10400, # .. 0x1044F ; Deseret + 0x10450, # .. 0x1047F ; Shavian + 0x10480, # .. 0x1049D ; Osmanya + 0x1049E, # .. 0x1049F ; Unknown + 0x104A0, # .. 0x104A9 ; Osmanya + 0x104AA, # .. 0x104AF ; Unknown + 0x104B0, # .. 0x104D3 ; Osage + 0x104D4, # .. 0x104D7 ; Unknown + 0x104D8, # .. 0x104FB ; Osage + 0x104FC, # .. 0x104FF ; Unknown + 0x10500, # .. 0x10527 ; Elbasan + 0x10528, # .. 0x1052F ; Unknown + 0x10530, # .. 0x10563 ; Caucasian_Albanian + 0x10564, # .. 0x1056E ; Unknown + 0x1056F, # .. 0x1056F ; Caucasian_Albanian + 0x10570, # .. 0x1057A ; Vithkuqi + 0x1057B, # .. 0x1057B ; Unknown + 0x1057C, # .. 0x1058A ; Vithkuqi + 0x1058B, # .. 0x1058B ; Unknown + 0x1058C, # .. 0x10592 ; Vithkuqi + 0x10593, # .. 0x10593 ; Unknown + 0x10594, # .. 0x10595 ; Vithkuqi + 0x10596, # .. 0x10596 ; Unknown + 0x10597, # .. 0x105A1 ; Vithkuqi + 0x105A2, # .. 0x105A2 ; Unknown + 0x105A3, # .. 0x105B1 ; Vithkuqi + 0x105B2, # .. 0x105B2 ; Unknown + 0x105B3, # .. 0x105B9 ; Vithkuqi + 0x105BA, # .. 0x105BA ; Unknown + 0x105BB, # .. 0x105BC ; Vithkuqi + 0x105BD, # .. 0x105BF ; Unknown + 0x105C0, # .. 0x105F3 ; Todhri + 0x105F4, # .. 0x105FF ; Unknown + 0x10600, # .. 0x10736 ; Linear_A + 0x10737, # .. 0x1073F ; Unknown + 0x10740, # .. 0x10755 ; Linear_A + 0x10756, # .. 0x1075F ; Unknown + 0x10760, # .. 0x10767 ; Linear_A + 0x10768, # .. 0x1077F ; Unknown + 0x10780, # .. 0x10785 ; Latin + 0x10786, # .. 0x10786 ; Unknown + 0x10787, # .. 0x107B0 ; Latin + 0x107B1, # .. 0x107B1 ; Unknown + 0x107B2, # .. 0x107BA ; Latin + 0x107BB, # .. 0x107FF ; Unknown + 0x10800, # .. 0x10805 ; Cypriot + 0x10806, # .. 0x10807 ; Unknown + 0x10808, # .. 0x10808 ; Cypriot + 0x10809, # .. 0x10809 ; Unknown + 0x1080A, # .. 0x10835 ; Cypriot + 0x10836, # .. 0x10836 ; Unknown + 0x10837, # .. 0x10838 ; Cypriot + 0x10839, # .. 0x1083B ; Unknown + 0x1083C, # .. 0x1083C ; Cypriot + 0x1083D, # .. 0x1083E ; Unknown + 0x1083F, # .. 0x1083F ; Cypriot + 0x10840, # .. 0x10855 ; Imperial_Aramaic + 0x10856, # .. 0x10856 ; Unknown + 0x10857, # .. 0x1085F ; Imperial_Aramaic + 0x10860, # .. 0x1087F ; Palmyrene + 0x10880, # .. 0x1089E ; Nabataean + 0x1089F, # .. 0x108A6 ; Unknown + 0x108A7, # .. 0x108AF ; Nabataean + 0x108B0, # .. 0x108DF ; Unknown + 0x108E0, # .. 0x108F2 ; Hatran + 0x108F3, # .. 0x108F3 ; Unknown + 0x108F4, # .. 0x108F5 ; Hatran + 0x108F6, # .. 0x108FA ; Unknown + 0x108FB, # .. 0x108FF ; Hatran + 0x10900, # .. 0x1091B ; Phoenician + 0x1091C, # .. 0x1091E ; Unknown + 0x1091F, # .. 0x1091F ; Phoenician + 0x10920, # .. 0x10939 ; Lydian + 0x1093A, # .. 0x1093E ; Unknown + 0x1093F, # .. 0x1093F ; Lydian + 0x10940, # .. 0x10959 ; Sidetic + 0x1095A, # .. 0x1097F ; Unknown + 0x10980, # .. 0x1099F ; Meroitic_Hieroglyphs + 0x109A0, # .. 0x109B7 ; Meroitic_Cursive + 0x109B8, # .. 0x109BB ; Unknown + 0x109BC, # .. 0x109CF ; Meroitic_Cursive + 0x109D0, # .. 0x109D1 ; Unknown + 0x109D2, # .. 0x109FF ; Meroitic_Cursive + 0x10A00, # .. 0x10A03 ; Kharoshthi + 0x10A04, # .. 0x10A04 ; Unknown + 0x10A05, # .. 0x10A06 ; Kharoshthi + 0x10A07, # .. 0x10A0B ; Unknown + 0x10A0C, # .. 0x10A13 ; Kharoshthi + 0x10A14, # .. 0x10A14 ; Unknown + 0x10A15, # .. 0x10A17 ; Kharoshthi + 0x10A18, # .. 0x10A18 ; Unknown + 0x10A19, # .. 0x10A35 ; Kharoshthi + 0x10A36, # .. 0x10A37 ; Unknown + 0x10A38, # .. 0x10A3A ; Kharoshthi + 0x10A3B, # .. 0x10A3E ; Unknown + 0x10A3F, # .. 0x10A48 ; Kharoshthi + 0x10A49, # .. 0x10A4F ; Unknown + 0x10A50, # .. 0x10A58 ; Kharoshthi + 0x10A59, # .. 0x10A5F ; Unknown + 0x10A60, # .. 0x10A7F ; Old_South_Arabian + 0x10A80, # .. 0x10A9F ; Old_North_Arabian + 0x10AA0, # .. 0x10ABF ; Unknown + 0x10AC0, # .. 0x10AE6 ; Manichaean + 0x10AE7, # .. 0x10AEA ; Unknown + 0x10AEB, # .. 0x10AF6 ; Manichaean + 0x10AF7, # .. 0x10AFF ; Unknown + 0x10B00, # .. 0x10B35 ; Avestan + 0x10B36, # .. 0x10B38 ; Unknown + 0x10B39, # .. 0x10B3F ; Avestan + 0x10B40, # .. 0x10B55 ; Inscriptional_Parthian + 0x10B56, # .. 0x10B57 ; Unknown + 0x10B58, # .. 0x10B5F ; Inscriptional_Parthian + 0x10B60, # .. 0x10B72 ; Inscriptional_Pahlavi + 0x10B73, # .. 0x10B77 ; Unknown + 0x10B78, # .. 0x10B7F ; Inscriptional_Pahlavi + 0x10B80, # .. 0x10B91 ; Psalter_Pahlavi + 0x10B92, # .. 0x10B98 ; Unknown + 0x10B99, # .. 0x10B9C ; Psalter_Pahlavi + 0x10B9D, # .. 0x10BA8 ; Unknown + 0x10BA9, # .. 0x10BAF ; Psalter_Pahlavi + 0x10BB0, # .. 0x10BFF ; Unknown + 0x10C00, # .. 0x10C48 ; Old_Turkic + 0x10C49, # .. 0x10C7F ; Unknown + 0x10C80, # .. 0x10CB2 ; Old_Hungarian + 0x10CB3, # .. 0x10CBF ; Unknown + 0x10CC0, # .. 0x10CF2 ; Old_Hungarian + 0x10CF3, # .. 0x10CF9 ; Unknown + 0x10CFA, # .. 0x10CFF ; Old_Hungarian + 0x10D00, # .. 0x10D27 ; Hanifi_Rohingya + 0x10D28, # .. 0x10D2F ; Unknown + 0x10D30, # .. 0x10D39 ; Hanifi_Rohingya + 0x10D3A, # .. 0x10D3F ; Unknown + 0x10D40, # .. 0x10D65 ; Garay + 0x10D66, # .. 0x10D68 ; Unknown + 0x10D69, # .. 0x10D85 ; Garay + 0x10D86, # .. 0x10D8D ; Unknown + 0x10D8E, # .. 0x10D8F ; Garay + 0x10D90, # .. 0x10E5F ; Unknown + 0x10E60, # .. 0x10E7E ; Arabic + 0x10E7F, # .. 0x10E7F ; Unknown + 0x10E80, # .. 0x10EA9 ; Yezidi + 0x10EAA, # .. 0x10EAA ; Unknown + 0x10EAB, # .. 0x10EAD ; Yezidi + 0x10EAE, # .. 0x10EAF ; Unknown + 0x10EB0, # .. 0x10EB1 ; Yezidi + 0x10EB2, # .. 0x10EC1 ; Unknown + 0x10EC2, # .. 0x10EC7 ; Arabic + 0x10EC8, # .. 0x10ECF ; Unknown + 0x10ED0, # .. 0x10ED8 ; Arabic + 0x10ED9, # .. 0x10EF9 ; Unknown + 0x10EFA, # .. 0x10EFF ; Arabic + 0x10F00, # .. 0x10F27 ; Old_Sogdian + 0x10F28, # .. 0x10F2F ; Unknown + 0x10F30, # .. 0x10F59 ; Sogdian + 0x10F5A, # .. 0x10F6F ; Unknown + 0x10F70, # .. 0x10F89 ; Old_Uyghur + 0x10F8A, # .. 0x10FAF ; Unknown + 0x10FB0, # .. 0x10FCB ; Chorasmian + 0x10FCC, # .. 0x10FDF ; Unknown + 0x10FE0, # .. 0x10FF6 ; Elymaic + 0x10FF7, # .. 0x10FFF ; Unknown + 0x11000, # .. 0x1104D ; Brahmi + 0x1104E, # .. 0x11051 ; Unknown + 0x11052, # .. 0x11075 ; Brahmi + 0x11076, # .. 0x1107E ; Unknown + 0x1107F, # .. 0x1107F ; Brahmi + 0x11080, # .. 0x110C2 ; Kaithi + 0x110C3, # .. 0x110CC ; Unknown + 0x110CD, # .. 0x110CD ; Kaithi + 0x110CE, # .. 0x110CF ; Unknown + 0x110D0, # .. 0x110E8 ; Sora_Sompeng + 0x110E9, # .. 0x110EF ; Unknown + 0x110F0, # .. 0x110F9 ; Sora_Sompeng + 0x110FA, # .. 0x110FF ; Unknown + 0x11100, # .. 0x11134 ; Chakma + 0x11135, # .. 0x11135 ; Unknown + 0x11136, # .. 0x11147 ; Chakma + 0x11148, # .. 0x1114F ; Unknown + 0x11150, # .. 0x11176 ; Mahajani + 0x11177, # .. 0x1117F ; Unknown + 0x11180, # .. 0x111DF ; Sharada + 0x111E0, # .. 0x111E0 ; Unknown + 0x111E1, # .. 0x111F4 ; Sinhala + 0x111F5, # .. 0x111FF ; Unknown + 0x11200, # .. 0x11211 ; Khojki + 0x11212, # .. 0x11212 ; Unknown + 0x11213, # .. 0x11241 ; Khojki + 0x11242, # .. 0x1127F ; Unknown + 0x11280, # .. 0x11286 ; Multani + 0x11287, # .. 0x11287 ; Unknown + 0x11288, # .. 0x11288 ; Multani + 0x11289, # .. 0x11289 ; Unknown + 0x1128A, # .. 0x1128D ; Multani + 0x1128E, # .. 0x1128E ; Unknown + 0x1128F, # .. 0x1129D ; Multani + 0x1129E, # .. 0x1129E ; Unknown + 0x1129F, # .. 0x112A9 ; Multani + 0x112AA, # .. 0x112AF ; Unknown + 0x112B0, # .. 0x112EA ; Khudawadi + 0x112EB, # .. 0x112EF ; Unknown + 0x112F0, # .. 0x112F9 ; Khudawadi + 0x112FA, # .. 0x112FF ; Unknown + 0x11300, # .. 0x11303 ; Grantha + 0x11304, # .. 0x11304 ; Unknown + 0x11305, # .. 0x1130C ; Grantha + 0x1130D, # .. 0x1130E ; Unknown + 0x1130F, # .. 0x11310 ; Grantha + 0x11311, # .. 0x11312 ; Unknown + 0x11313, # .. 0x11328 ; Grantha + 0x11329, # .. 0x11329 ; Unknown + 0x1132A, # .. 0x11330 ; Grantha + 0x11331, # .. 0x11331 ; Unknown + 0x11332, # .. 0x11333 ; Grantha + 0x11334, # .. 0x11334 ; Unknown + 0x11335, # .. 0x11339 ; Grantha + 0x1133A, # .. 0x1133A ; Unknown + 0x1133B, # .. 0x1133B ; Inherited + 0x1133C, # .. 0x11344 ; Grantha + 0x11345, # .. 0x11346 ; Unknown + 0x11347, # .. 0x11348 ; Grantha + 0x11349, # .. 0x1134A ; Unknown + 0x1134B, # .. 0x1134D ; Grantha + 0x1134E, # .. 0x1134F ; Unknown + 0x11350, # .. 0x11350 ; Grantha + 0x11351, # .. 0x11356 ; Unknown + 0x11357, # .. 0x11357 ; Grantha + 0x11358, # .. 0x1135C ; Unknown + 0x1135D, # .. 0x11363 ; Grantha + 0x11364, # .. 0x11365 ; Unknown + 0x11366, # .. 0x1136C ; Grantha + 0x1136D, # .. 0x1136F ; Unknown + 0x11370, # .. 0x11374 ; Grantha + 0x11375, # .. 0x1137F ; Unknown + 0x11380, # .. 0x11389 ; Tulu_Tigalari + 0x1138A, # .. 0x1138A ; Unknown + 0x1138B, # .. 0x1138B ; Tulu_Tigalari + 0x1138C, # .. 0x1138D ; Unknown + 0x1138E, # .. 0x1138E ; Tulu_Tigalari + 0x1138F, # .. 0x1138F ; Unknown + 0x11390, # .. 0x113B5 ; Tulu_Tigalari + 0x113B6, # .. 0x113B6 ; Unknown + 0x113B7, # .. 0x113C0 ; Tulu_Tigalari + 0x113C1, # .. 0x113C1 ; Unknown + 0x113C2, # .. 0x113C2 ; Tulu_Tigalari + 0x113C3, # .. 0x113C4 ; Unknown + 0x113C5, # .. 0x113C5 ; Tulu_Tigalari + 0x113C6, # .. 0x113C6 ; Unknown + 0x113C7, # .. 0x113CA ; Tulu_Tigalari + 0x113CB, # .. 0x113CB ; Unknown + 0x113CC, # .. 0x113D5 ; Tulu_Tigalari + 0x113D6, # .. 0x113D6 ; Unknown + 0x113D7, # .. 0x113D8 ; Tulu_Tigalari + 0x113D9, # .. 0x113E0 ; Unknown + 0x113E1, # .. 0x113E2 ; Tulu_Tigalari + 0x113E3, # .. 0x113FF ; Unknown + 0x11400, # .. 0x1145B ; Newa + 0x1145C, # .. 0x1145C ; Unknown + 0x1145D, # .. 0x11461 ; Newa + 0x11462, # .. 0x1147F ; Unknown + 0x11480, # .. 0x114C7 ; Tirhuta + 0x114C8, # .. 0x114CF ; Unknown + 0x114D0, # .. 0x114D9 ; Tirhuta + 0x114DA, # .. 0x1157F ; Unknown + 0x11580, # .. 0x115B5 ; Siddham + 0x115B6, # .. 0x115B7 ; Unknown + 0x115B8, # .. 0x115DD ; Siddham + 0x115DE, # .. 0x115FF ; Unknown + 0x11600, # .. 0x11644 ; Modi + 0x11645, # .. 0x1164F ; Unknown + 0x11650, # .. 0x11659 ; Modi + 0x1165A, # .. 0x1165F ; Unknown + 0x11660, # .. 0x1166C ; Mongolian + 0x1166D, # .. 0x1167F ; Unknown + 0x11680, # .. 0x116B9 ; Takri + 0x116BA, # .. 0x116BF ; Unknown + 0x116C0, # .. 0x116C9 ; Takri + 0x116CA, # .. 0x116CF ; Unknown + 0x116D0, # .. 0x116E3 ; Myanmar + 0x116E4, # .. 0x116FF ; Unknown + 0x11700, # .. 0x1171A ; Ahom + 0x1171B, # .. 0x1171C ; Unknown + 0x1171D, # .. 0x1172B ; Ahom + 0x1172C, # .. 0x1172F ; Unknown + 0x11730, # .. 0x11746 ; Ahom + 0x11747, # .. 0x117FF ; Unknown + 0x11800, # .. 0x1183B ; Dogra + 0x1183C, # .. 0x1189F ; Unknown + 0x118A0, # .. 0x118F2 ; Warang_Citi + 0x118F3, # .. 0x118FE ; Unknown + 0x118FF, # .. 0x118FF ; Warang_Citi + 0x11900, # .. 0x11906 ; Dives_Akuru + 0x11907, # .. 0x11908 ; Unknown + 0x11909, # .. 0x11909 ; Dives_Akuru + 0x1190A, # .. 0x1190B ; Unknown + 0x1190C, # .. 0x11913 ; Dives_Akuru + 0x11914, # .. 0x11914 ; Unknown + 0x11915, # .. 0x11916 ; Dives_Akuru + 0x11917, # .. 0x11917 ; Unknown + 0x11918, # .. 0x11935 ; Dives_Akuru + 0x11936, # .. 0x11936 ; Unknown + 0x11937, # .. 0x11938 ; Dives_Akuru + 0x11939, # .. 0x1193A ; Unknown + 0x1193B, # .. 0x11946 ; Dives_Akuru + 0x11947, # .. 0x1194F ; Unknown + 0x11950, # .. 0x11959 ; Dives_Akuru + 0x1195A, # .. 0x1199F ; Unknown + 0x119A0, # .. 0x119A7 ; Nandinagari + 0x119A8, # .. 0x119A9 ; Unknown + 0x119AA, # .. 0x119D7 ; Nandinagari + 0x119D8, # .. 0x119D9 ; Unknown + 0x119DA, # .. 0x119E4 ; Nandinagari + 0x119E5, # .. 0x119FF ; Unknown + 0x11A00, # .. 0x11A47 ; Zanabazar_Square + 0x11A48, # .. 0x11A4F ; Unknown + 0x11A50, # .. 0x11AA2 ; Soyombo + 0x11AA3, # .. 0x11AAF ; Unknown + 0x11AB0, # .. 0x11ABF ; Canadian_Aboriginal + 0x11AC0, # .. 0x11AF8 ; Pau_Cin_Hau + 0x11AF9, # .. 0x11AFF ; Unknown + 0x11B00, # .. 0x11B09 ; Devanagari + 0x11B0A, # .. 0x11B5F ; Unknown + 0x11B60, # .. 0x11B67 ; Sharada + 0x11B68, # .. 0x11BBF ; Unknown + 0x11BC0, # .. 0x11BE1 ; Sunuwar + 0x11BE2, # .. 0x11BEF ; Unknown + 0x11BF0, # .. 0x11BF9 ; Sunuwar + 0x11BFA, # .. 0x11BFF ; Unknown + 0x11C00, # .. 0x11C08 ; Bhaiksuki + 0x11C09, # .. 0x11C09 ; Unknown + 0x11C0A, # .. 0x11C36 ; Bhaiksuki + 0x11C37, # .. 0x11C37 ; Unknown + 0x11C38, # .. 0x11C45 ; Bhaiksuki + 0x11C46, # .. 0x11C4F ; Unknown + 0x11C50, # .. 0x11C6C ; Bhaiksuki + 0x11C6D, # .. 0x11C6F ; Unknown + 0x11C70, # .. 0x11C8F ; Marchen + 0x11C90, # .. 0x11C91 ; Unknown + 0x11C92, # .. 0x11CA7 ; Marchen + 0x11CA8, # .. 0x11CA8 ; Unknown + 0x11CA9, # .. 0x11CB6 ; Marchen + 0x11CB7, # .. 0x11CFF ; Unknown + 0x11D00, # .. 0x11D06 ; Masaram_Gondi + 0x11D07, # .. 0x11D07 ; Unknown + 0x11D08, # .. 0x11D09 ; Masaram_Gondi + 0x11D0A, # .. 0x11D0A ; Unknown + 0x11D0B, # .. 0x11D36 ; Masaram_Gondi + 0x11D37, # .. 0x11D39 ; Unknown + 0x11D3A, # .. 0x11D3A ; Masaram_Gondi + 0x11D3B, # .. 0x11D3B ; Unknown + 0x11D3C, # .. 0x11D3D ; Masaram_Gondi + 0x11D3E, # .. 0x11D3E ; Unknown + 0x11D3F, # .. 0x11D47 ; Masaram_Gondi + 0x11D48, # .. 0x11D4F ; Unknown + 0x11D50, # .. 0x11D59 ; Masaram_Gondi + 0x11D5A, # .. 0x11D5F ; Unknown + 0x11D60, # .. 0x11D65 ; Gunjala_Gondi + 0x11D66, # .. 0x11D66 ; Unknown + 0x11D67, # .. 0x11D68 ; Gunjala_Gondi + 0x11D69, # .. 0x11D69 ; Unknown + 0x11D6A, # .. 0x11D8E ; Gunjala_Gondi + 0x11D8F, # .. 0x11D8F ; Unknown + 0x11D90, # .. 0x11D91 ; Gunjala_Gondi + 0x11D92, # .. 0x11D92 ; Unknown + 0x11D93, # .. 0x11D98 ; Gunjala_Gondi + 0x11D99, # .. 0x11D9F ; Unknown + 0x11DA0, # .. 0x11DA9 ; Gunjala_Gondi + 0x11DAA, # .. 0x11DAF ; Unknown + 0x11DB0, # .. 0x11DDB ; Tolong_Siki + 0x11DDC, # .. 0x11DDF ; Unknown + 0x11DE0, # .. 0x11DE9 ; Tolong_Siki + 0x11DEA, # .. 0x11EDF ; Unknown + 0x11EE0, # .. 0x11EF8 ; Makasar + 0x11EF9, # .. 0x11EFF ; Unknown + 0x11F00, # .. 0x11F10 ; Kawi + 0x11F11, # .. 0x11F11 ; Unknown + 0x11F12, # .. 0x11F3A ; Kawi + 0x11F3B, # .. 0x11F3D ; Unknown + 0x11F3E, # .. 0x11F5A ; Kawi + 0x11F5B, # .. 0x11FAF ; Unknown + 0x11FB0, # .. 0x11FB0 ; Lisu + 0x11FB1, # .. 0x11FBF ; Unknown + 0x11FC0, # .. 0x11FF1 ; Tamil + 0x11FF2, # .. 0x11FFE ; Unknown + 0x11FFF, # .. 0x11FFF ; Tamil + 0x12000, # .. 0x12399 ; Cuneiform + 0x1239A, # .. 0x123FF ; Unknown + 0x12400, # .. 0x1246E ; Cuneiform + 0x1246F, # .. 0x1246F ; Unknown + 0x12470, # .. 0x12474 ; Cuneiform + 0x12475, # .. 0x1247F ; Unknown + 0x12480, # .. 0x12543 ; Cuneiform + 0x12544, # .. 0x12F8F ; Unknown + 0x12F90, # .. 0x12FF2 ; Cypro_Minoan + 0x12FF3, # .. 0x12FFF ; Unknown + 0x13000, # .. 0x13455 ; Egyptian_Hieroglyphs + 0x13456, # .. 0x1345F ; Unknown + 0x13460, # .. 0x143FA ; Egyptian_Hieroglyphs + 0x143FB, # .. 0x143FF ; Unknown + 0x14400, # .. 0x14646 ; Anatolian_Hieroglyphs + 0x14647, # .. 0x160FF ; Unknown + 0x16100, # .. 0x16139 ; Gurung_Khema + 0x1613A, # .. 0x167FF ; Unknown + 0x16800, # .. 0x16A38 ; Bamum + 0x16A39, # .. 0x16A3F ; Unknown + 0x16A40, # .. 0x16A5E ; Mro + 0x16A5F, # .. 0x16A5F ; Unknown + 0x16A60, # .. 0x16A69 ; Mro + 0x16A6A, # .. 0x16A6D ; Unknown + 0x16A6E, # .. 0x16A6F ; Mro + 0x16A70, # .. 0x16ABE ; Tangsa + 0x16ABF, # .. 0x16ABF ; Unknown + 0x16AC0, # .. 0x16AC9 ; Tangsa + 0x16ACA, # .. 0x16ACF ; Unknown + 0x16AD0, # .. 0x16AED ; Bassa_Vah + 0x16AEE, # .. 0x16AEF ; Unknown + 0x16AF0, # .. 0x16AF5 ; Bassa_Vah + 0x16AF6, # .. 0x16AFF ; Unknown + 0x16B00, # .. 0x16B45 ; Pahawh_Hmong + 0x16B46, # .. 0x16B4F ; Unknown + 0x16B50, # .. 0x16B59 ; Pahawh_Hmong + 0x16B5A, # .. 0x16B5A ; Unknown + 0x16B5B, # .. 0x16B61 ; Pahawh_Hmong + 0x16B62, # .. 0x16B62 ; Unknown + 0x16B63, # .. 0x16B77 ; Pahawh_Hmong + 0x16B78, # .. 0x16B7C ; Unknown + 0x16B7D, # .. 0x16B8F ; Pahawh_Hmong + 0x16B90, # .. 0x16D3F ; Unknown + 0x16D40, # .. 0x16D79 ; Kirat_Rai + 0x16D7A, # .. 0x16E3F ; Unknown + 0x16E40, # .. 0x16E9A ; Medefaidrin + 0x16E9B, # .. 0x16E9F ; Unknown + 0x16EA0, # .. 0x16EB8 ; Beria_Erfe + 0x16EB9, # .. 0x16EBA ; Unknown + 0x16EBB, # .. 0x16ED3 ; Beria_Erfe + 0x16ED4, # .. 0x16EFF ; Unknown + 0x16F00, # .. 0x16F4A ; Miao + 0x16F4B, # .. 0x16F4E ; Unknown + 0x16F4F, # .. 0x16F87 ; Miao + 0x16F88, # .. 0x16F8E ; Unknown + 0x16F8F, # .. 0x16F9F ; Miao + 0x16FA0, # .. 0x16FDF ; Unknown + 0x16FE0, # .. 0x16FE0 ; Tangut + 0x16FE1, # .. 0x16FE1 ; Nushu + 0x16FE2, # .. 0x16FE3 ; Han + 0x16FE4, # .. 0x16FE4 ; Khitan_Small_Script + 0x16FE5, # .. 0x16FEF ; Unknown + 0x16FF0, # .. 0x16FF6 ; Han + 0x16FF7, # .. 0x16FFF ; Unknown + 0x17000, # .. 0x18AFF ; Tangut + 0x18B00, # .. 0x18CD5 ; Khitan_Small_Script + 0x18CD6, # .. 0x18CFE ; Unknown + 0x18CFF, # .. 0x18CFF ; Khitan_Small_Script + 0x18D00, # .. 0x18D1E ; Tangut + 0x18D1F, # .. 0x18D7F ; Unknown + 0x18D80, # .. 0x18DF2 ; Tangut + 0x18DF3, # .. 0x1AFEF ; Unknown + 0x1AFF0, # .. 0x1AFF3 ; Katakana + 0x1AFF4, # .. 0x1AFF4 ; Unknown + 0x1AFF5, # .. 0x1AFFB ; Katakana + 0x1AFFC, # .. 0x1AFFC ; Unknown + 0x1AFFD, # .. 0x1AFFE ; Katakana + 0x1AFFF, # .. 0x1AFFF ; Unknown + 0x1B000, # .. 0x1B000 ; Katakana + 0x1B001, # .. 0x1B11F ; Hiragana + 0x1B120, # .. 0x1B122 ; Katakana + 0x1B123, # .. 0x1B131 ; Unknown + 0x1B132, # .. 0x1B132 ; Hiragana + 0x1B133, # .. 0x1B14F ; Unknown + 0x1B150, # .. 0x1B152 ; Hiragana + 0x1B153, # .. 0x1B154 ; Unknown + 0x1B155, # .. 0x1B155 ; Katakana + 0x1B156, # .. 0x1B163 ; Unknown + 0x1B164, # .. 0x1B167 ; Katakana + 0x1B168, # .. 0x1B16F ; Unknown + 0x1B170, # .. 0x1B2FB ; Nushu + 0x1B2FC, # .. 0x1BBFF ; Unknown + 0x1BC00, # .. 0x1BC6A ; Duployan + 0x1BC6B, # .. 0x1BC6F ; Unknown + 0x1BC70, # .. 0x1BC7C ; Duployan + 0x1BC7D, # .. 0x1BC7F ; Unknown + 0x1BC80, # .. 0x1BC88 ; Duployan + 0x1BC89, # .. 0x1BC8F ; Unknown + 0x1BC90, # .. 0x1BC99 ; Duployan + 0x1BC9A, # .. 0x1BC9B ; Unknown + 0x1BC9C, # .. 0x1BC9F ; Duployan + 0x1BCA0, # .. 0x1BCA3 ; Common + 0x1BCA4, # .. 0x1CBFF ; Unknown + 0x1CC00, # .. 0x1CCFC ; Common + 0x1CCFD, # .. 0x1CCFF ; Unknown + 0x1CD00, # .. 0x1CEB3 ; Common + 0x1CEB4, # .. 0x1CEB9 ; Unknown + 0x1CEBA, # .. 0x1CED0 ; Common + 0x1CED1, # .. 0x1CEDF ; Unknown + 0x1CEE0, # .. 0x1CEF0 ; Common + 0x1CEF1, # .. 0x1CEFF ; Unknown + 0x1CF00, # .. 0x1CF2D ; Inherited + 0x1CF2E, # .. 0x1CF2F ; Unknown + 0x1CF30, # .. 0x1CF46 ; Inherited + 0x1CF47, # .. 0x1CF4F ; Unknown + 0x1CF50, # .. 0x1CFC3 ; Common + 0x1CFC4, # .. 0x1CFFF ; Unknown + 0x1D000, # .. 0x1D0F5 ; Common + 0x1D0F6, # .. 0x1D0FF ; Unknown + 0x1D100, # .. 0x1D126 ; Common + 0x1D127, # .. 0x1D128 ; Unknown + 0x1D129, # .. 0x1D166 ; Common + 0x1D167, # .. 0x1D169 ; Inherited + 0x1D16A, # .. 0x1D17A ; Common + 0x1D17B, # .. 0x1D182 ; Inherited + 0x1D183, # .. 0x1D184 ; Common + 0x1D185, # .. 0x1D18B ; Inherited + 0x1D18C, # .. 0x1D1A9 ; Common + 0x1D1AA, # .. 0x1D1AD ; Inherited + 0x1D1AE, # .. 0x1D1EA ; Common + 0x1D1EB, # .. 0x1D1FF ; Unknown + 0x1D200, # .. 0x1D245 ; Greek + 0x1D246, # .. 0x1D2BF ; Unknown + 0x1D2C0, # .. 0x1D2D3 ; Common + 0x1D2D4, # .. 0x1D2DF ; Unknown + 0x1D2E0, # .. 0x1D2F3 ; Common + 0x1D2F4, # .. 0x1D2FF ; Unknown + 0x1D300, # .. 0x1D356 ; Common + 0x1D357, # .. 0x1D35F ; Unknown + 0x1D360, # .. 0x1D378 ; Common + 0x1D379, # .. 0x1D3FF ; Unknown + 0x1D400, # .. 0x1D454 ; Common + 0x1D455, # .. 0x1D455 ; Unknown + 0x1D456, # .. 0x1D49C ; Common + 0x1D49D, # .. 0x1D49D ; Unknown + 0x1D49E, # .. 0x1D49F ; Common + 0x1D4A0, # .. 0x1D4A1 ; Unknown + 0x1D4A2, # .. 0x1D4A2 ; Common + 0x1D4A3, # .. 0x1D4A4 ; Unknown + 0x1D4A5, # .. 0x1D4A6 ; Common + 0x1D4A7, # .. 0x1D4A8 ; Unknown + 0x1D4A9, # .. 0x1D4AC ; Common + 0x1D4AD, # .. 0x1D4AD ; Unknown + 0x1D4AE, # .. 0x1D4B9 ; Common + 0x1D4BA, # .. 0x1D4BA ; Unknown + 0x1D4BB, # .. 0x1D4BB ; Common + 0x1D4BC, # .. 0x1D4BC ; Unknown + 0x1D4BD, # .. 0x1D4C3 ; Common + 0x1D4C4, # .. 0x1D4C4 ; Unknown + 0x1D4C5, # .. 0x1D505 ; Common + 0x1D506, # .. 0x1D506 ; Unknown + 0x1D507, # .. 0x1D50A ; Common + 0x1D50B, # .. 0x1D50C ; Unknown + 0x1D50D, # .. 0x1D514 ; Common + 0x1D515, # .. 0x1D515 ; Unknown + 0x1D516, # .. 0x1D51C ; Common + 0x1D51D, # .. 0x1D51D ; Unknown + 0x1D51E, # .. 0x1D539 ; Common + 0x1D53A, # .. 0x1D53A ; Unknown + 0x1D53B, # .. 0x1D53E ; Common + 0x1D53F, # .. 0x1D53F ; Unknown + 0x1D540, # .. 0x1D544 ; Common + 0x1D545, # .. 0x1D545 ; Unknown + 0x1D546, # .. 0x1D546 ; Common + 0x1D547, # .. 0x1D549 ; Unknown + 0x1D54A, # .. 0x1D550 ; Common + 0x1D551, # .. 0x1D551 ; Unknown + 0x1D552, # .. 0x1D6A5 ; Common + 0x1D6A6, # .. 0x1D6A7 ; Unknown + 0x1D6A8, # .. 0x1D7CB ; Common + 0x1D7CC, # .. 0x1D7CD ; Unknown + 0x1D7CE, # .. 0x1D7FF ; Common + 0x1D800, # .. 0x1DA8B ; SignWriting + 0x1DA8C, # .. 0x1DA9A ; Unknown + 0x1DA9B, # .. 0x1DA9F ; SignWriting + 0x1DAA0, # .. 0x1DAA0 ; Unknown + 0x1DAA1, # .. 0x1DAAF ; SignWriting + 0x1DAB0, # .. 0x1DEFF ; Unknown + 0x1DF00, # .. 0x1DF1E ; Latin + 0x1DF1F, # .. 0x1DF24 ; Unknown + 0x1DF25, # .. 0x1DF2A ; Latin + 0x1DF2B, # .. 0x1DFFF ; Unknown + 0x1E000, # .. 0x1E006 ; Glagolitic + 0x1E007, # .. 0x1E007 ; Unknown + 0x1E008, # .. 0x1E018 ; Glagolitic + 0x1E019, # .. 0x1E01A ; Unknown + 0x1E01B, # .. 0x1E021 ; Glagolitic + 0x1E022, # .. 0x1E022 ; Unknown + 0x1E023, # .. 0x1E024 ; Glagolitic + 0x1E025, # .. 0x1E025 ; Unknown + 0x1E026, # .. 0x1E02A ; Glagolitic + 0x1E02B, # .. 0x1E02F ; Unknown + 0x1E030, # .. 0x1E06D ; Cyrillic + 0x1E06E, # .. 0x1E08E ; Unknown + 0x1E08F, # .. 0x1E08F ; Cyrillic + 0x1E090, # .. 0x1E0FF ; Unknown + 0x1E100, # .. 0x1E12C ; Nyiakeng_Puachue_Hmong + 0x1E12D, # .. 0x1E12F ; Unknown + 0x1E130, # .. 0x1E13D ; Nyiakeng_Puachue_Hmong + 0x1E13E, # .. 0x1E13F ; Unknown + 0x1E140, # .. 0x1E149 ; Nyiakeng_Puachue_Hmong + 0x1E14A, # .. 0x1E14D ; Unknown + 0x1E14E, # .. 0x1E14F ; Nyiakeng_Puachue_Hmong + 0x1E150, # .. 0x1E28F ; Unknown + 0x1E290, # .. 0x1E2AE ; Toto + 0x1E2AF, # .. 0x1E2BF ; Unknown + 0x1E2C0, # .. 0x1E2F9 ; Wancho + 0x1E2FA, # .. 0x1E2FE ; Unknown + 0x1E2FF, # .. 0x1E2FF ; Wancho + 0x1E300, # .. 0x1E4CF ; Unknown + 0x1E4D0, # .. 0x1E4F9 ; Nag_Mundari + 0x1E4FA, # .. 0x1E5CF ; Unknown + 0x1E5D0, # .. 0x1E5FA ; Ol_Onal + 0x1E5FB, # .. 0x1E5FE ; Unknown + 0x1E5FF, # .. 0x1E5FF ; Ol_Onal + 0x1E600, # .. 0x1E6BF ; Unknown + 0x1E6C0, # .. 0x1E6DE ; Tai_Yo + 0x1E6DF, # .. 0x1E6DF ; Unknown + 0x1E6E0, # .. 0x1E6F5 ; Tai_Yo + 0x1E6F6, # .. 0x1E6FD ; Unknown + 0x1E6FE, # .. 0x1E6FF ; Tai_Yo + 0x1E700, # .. 0x1E7DF ; Unknown + 0x1E7E0, # .. 0x1E7E6 ; Ethiopic + 0x1E7E7, # .. 0x1E7E7 ; Unknown + 0x1E7E8, # .. 0x1E7EB ; Ethiopic + 0x1E7EC, # .. 0x1E7EC ; Unknown + 0x1E7ED, # .. 0x1E7EE ; Ethiopic + 0x1E7EF, # .. 0x1E7EF ; Unknown + 0x1E7F0, # .. 0x1E7FE ; Ethiopic + 0x1E7FF, # .. 0x1E7FF ; Unknown + 0x1E800, # .. 0x1E8C4 ; Mende_Kikakui + 0x1E8C5, # .. 0x1E8C6 ; Unknown + 0x1E8C7, # .. 0x1E8D6 ; Mende_Kikakui + 0x1E8D7, # .. 0x1E8FF ; Unknown + 0x1E900, # .. 0x1E94B ; Adlam + 0x1E94C, # .. 0x1E94F ; Unknown + 0x1E950, # .. 0x1E959 ; Adlam + 0x1E95A, # .. 0x1E95D ; Unknown + 0x1E95E, # .. 0x1E95F ; Adlam + 0x1E960, # .. 0x1EC70 ; Unknown + 0x1EC71, # .. 0x1ECB4 ; Common + 0x1ECB5, # .. 0x1ED00 ; Unknown + 0x1ED01, # .. 0x1ED3D ; Common + 0x1ED3E, # .. 0x1EDFF ; Unknown + 0x1EE00, # .. 0x1EE03 ; Arabic + 0x1EE04, # .. 0x1EE04 ; Unknown + 0x1EE05, # .. 0x1EE1F ; Arabic + 0x1EE20, # .. 0x1EE20 ; Unknown + 0x1EE21, # .. 0x1EE22 ; Arabic + 0x1EE23, # .. 0x1EE23 ; Unknown + 0x1EE24, # .. 0x1EE24 ; Arabic + 0x1EE25, # .. 0x1EE26 ; Unknown + 0x1EE27, # .. 0x1EE27 ; Arabic + 0x1EE28, # .. 0x1EE28 ; Unknown + 0x1EE29, # .. 0x1EE32 ; Arabic + 0x1EE33, # .. 0x1EE33 ; Unknown + 0x1EE34, # .. 0x1EE37 ; Arabic + 0x1EE38, # .. 0x1EE38 ; Unknown + 0x1EE39, # .. 0x1EE39 ; Arabic + 0x1EE3A, # .. 0x1EE3A ; Unknown + 0x1EE3B, # .. 0x1EE3B ; Arabic + 0x1EE3C, # .. 0x1EE41 ; Unknown + 0x1EE42, # .. 0x1EE42 ; Arabic + 0x1EE43, # .. 0x1EE46 ; Unknown + 0x1EE47, # .. 0x1EE47 ; Arabic + 0x1EE48, # .. 0x1EE48 ; Unknown + 0x1EE49, # .. 0x1EE49 ; Arabic + 0x1EE4A, # .. 0x1EE4A ; Unknown + 0x1EE4B, # .. 0x1EE4B ; Arabic + 0x1EE4C, # .. 0x1EE4C ; Unknown + 0x1EE4D, # .. 0x1EE4F ; Arabic + 0x1EE50, # .. 0x1EE50 ; Unknown + 0x1EE51, # .. 0x1EE52 ; Arabic + 0x1EE53, # .. 0x1EE53 ; Unknown + 0x1EE54, # .. 0x1EE54 ; Arabic + 0x1EE55, # .. 0x1EE56 ; Unknown + 0x1EE57, # .. 0x1EE57 ; Arabic + 0x1EE58, # .. 0x1EE58 ; Unknown + 0x1EE59, # .. 0x1EE59 ; Arabic + 0x1EE5A, # .. 0x1EE5A ; Unknown + 0x1EE5B, # .. 0x1EE5B ; Arabic + 0x1EE5C, # .. 0x1EE5C ; Unknown + 0x1EE5D, # .. 0x1EE5D ; Arabic + 0x1EE5E, # .. 0x1EE5E ; Unknown + 0x1EE5F, # .. 0x1EE5F ; Arabic + 0x1EE60, # .. 0x1EE60 ; Unknown + 0x1EE61, # .. 0x1EE62 ; Arabic + 0x1EE63, # .. 0x1EE63 ; Unknown + 0x1EE64, # .. 0x1EE64 ; Arabic + 0x1EE65, # .. 0x1EE66 ; Unknown + 0x1EE67, # .. 0x1EE6A ; Arabic + 0x1EE6B, # .. 0x1EE6B ; Unknown + 0x1EE6C, # .. 0x1EE72 ; Arabic + 0x1EE73, # .. 0x1EE73 ; Unknown + 0x1EE74, # .. 0x1EE77 ; Arabic + 0x1EE78, # .. 0x1EE78 ; Unknown + 0x1EE79, # .. 0x1EE7C ; Arabic + 0x1EE7D, # .. 0x1EE7D ; Unknown + 0x1EE7E, # .. 0x1EE7E ; Arabic + 0x1EE7F, # .. 0x1EE7F ; Unknown + 0x1EE80, # .. 0x1EE89 ; Arabic + 0x1EE8A, # .. 0x1EE8A ; Unknown + 0x1EE8B, # .. 0x1EE9B ; Arabic + 0x1EE9C, # .. 0x1EEA0 ; Unknown + 0x1EEA1, # .. 0x1EEA3 ; Arabic + 0x1EEA4, # .. 0x1EEA4 ; Unknown + 0x1EEA5, # .. 0x1EEA9 ; Arabic + 0x1EEAA, # .. 0x1EEAA ; Unknown + 0x1EEAB, # .. 0x1EEBB ; Arabic + 0x1EEBC, # .. 0x1EEEF ; Unknown + 0x1EEF0, # .. 0x1EEF1 ; Arabic + 0x1EEF2, # .. 0x1EFFF ; Unknown + 0x1F000, # .. 0x1F02B ; Common + 0x1F02C, # .. 0x1F02F ; Unknown + 0x1F030, # .. 0x1F093 ; Common + 0x1F094, # .. 0x1F09F ; Unknown + 0x1F0A0, # .. 0x1F0AE ; Common + 0x1F0AF, # .. 0x1F0B0 ; Unknown + 0x1F0B1, # .. 0x1F0BF ; Common + 0x1F0C0, # .. 0x1F0C0 ; Unknown + 0x1F0C1, # .. 0x1F0CF ; Common + 0x1F0D0, # .. 0x1F0D0 ; Unknown + 0x1F0D1, # .. 0x1F0F5 ; Common + 0x1F0F6, # .. 0x1F0FF ; Unknown + 0x1F100, # .. 0x1F1AD ; Common + 0x1F1AE, # .. 0x1F1E5 ; Unknown + 0x1F1E6, # .. 0x1F1FF ; Common + 0x1F200, # .. 0x1F200 ; Hiragana + 0x1F201, # .. 0x1F202 ; Common + 0x1F203, # .. 0x1F20F ; Unknown + 0x1F210, # .. 0x1F23B ; Common + 0x1F23C, # .. 0x1F23F ; Unknown + 0x1F240, # .. 0x1F248 ; Common + 0x1F249, # .. 0x1F24F ; Unknown + 0x1F250, # .. 0x1F251 ; Common + 0x1F252, # .. 0x1F25F ; Unknown + 0x1F260, # .. 0x1F265 ; Common + 0x1F266, # .. 0x1F2FF ; Unknown + 0x1F300, # .. 0x1F6D8 ; Common + 0x1F6D9, # .. 0x1F6DB ; Unknown + 0x1F6DC, # .. 0x1F6EC ; Common + 0x1F6ED, # .. 0x1F6EF ; Unknown + 0x1F6F0, # .. 0x1F6FC ; Common + 0x1F6FD, # .. 0x1F6FF ; Unknown + 0x1F700, # .. 0x1F7D9 ; Common + 0x1F7DA, # .. 0x1F7DF ; Unknown + 0x1F7E0, # .. 0x1F7EB ; Common + 0x1F7EC, # .. 0x1F7EF ; Unknown + 0x1F7F0, # .. 0x1F7F0 ; Common + 0x1F7F1, # .. 0x1F7FF ; Unknown + 0x1F800, # .. 0x1F80B ; Common + 0x1F80C, # .. 0x1F80F ; Unknown + 0x1F810, # .. 0x1F847 ; Common + 0x1F848, # .. 0x1F84F ; Unknown + 0x1F850, # .. 0x1F859 ; Common + 0x1F85A, # .. 0x1F85F ; Unknown + 0x1F860, # .. 0x1F887 ; Common + 0x1F888, # .. 0x1F88F ; Unknown + 0x1F890, # .. 0x1F8AD ; Common + 0x1F8AE, # .. 0x1F8AF ; Unknown + 0x1F8B0, # .. 0x1F8BB ; Common + 0x1F8BC, # .. 0x1F8BF ; Unknown + 0x1F8C0, # .. 0x1F8C1 ; Common + 0x1F8C2, # .. 0x1F8CF ; Unknown + 0x1F8D0, # .. 0x1F8D8 ; Common + 0x1F8D9, # .. 0x1F8FF ; Unknown + 0x1F900, # .. 0x1FA57 ; Common + 0x1FA58, # .. 0x1FA5F ; Unknown + 0x1FA60, # .. 0x1FA6D ; Common + 0x1FA6E, # .. 0x1FA6F ; Unknown + 0x1FA70, # .. 0x1FA7C ; Common + 0x1FA7D, # .. 0x1FA7F ; Unknown + 0x1FA80, # .. 0x1FA8A ; Common + 0x1FA8B, # .. 0x1FA8D ; Unknown + 0x1FA8E, # .. 0x1FAC6 ; Common + 0x1FAC7, # .. 0x1FAC7 ; Unknown + 0x1FAC8, # .. 0x1FAC8 ; Common + 0x1FAC9, # .. 0x1FACC ; Unknown + 0x1FACD, # .. 0x1FADC ; Common + 0x1FADD, # .. 0x1FADE ; Unknown + 0x1FADF, # .. 0x1FAEA ; Common + 0x1FAEB, # .. 0x1FAEE ; Unknown + 0x1FAEF, # .. 0x1FAF8 ; Common + 0x1FAF9, # .. 0x1FAFF ; Unknown + 0x1FB00, # .. 0x1FB92 ; Common + 0x1FB93, # .. 0x1FB93 ; Unknown + 0x1FB94, # .. 0x1FBFA ; Common + 0x1FBFB, # .. 0x1FFFF ; Unknown + 0x20000, # .. 0x2A6DF ; Han + 0x2A6E0, # .. 0x2A6FF ; Unknown + 0x2A700, # .. 0x2B81D ; Han + 0x2B81E, # .. 0x2B81F ; Unknown + 0x2B820, # .. 0x2CEAD ; Han + 0x2CEAE, # .. 0x2CEAF ; Unknown + 0x2CEB0, # .. 0x2EBE0 ; Han + 0x2EBE1, # .. 0x2EBEF ; Unknown + 0x2EBF0, # .. 0x2EE5D ; Han + 0x2EE5E, # .. 0x2F7FF ; Unknown + 0x2F800, # .. 0x2FA1D ; Han + 0x2FA1E, # .. 0x2FFFF ; Unknown + 0x30000, # .. 0x3134A ; Han + 0x3134B, # .. 0x3134F ; Unknown + 0x31350, # .. 0x33479 ; Han + 0x3347A, # .. 0xE0000 ; Unknown + 0xE0001, # .. 0xE0001 ; Common + 0xE0002, # .. 0xE001F ; Unknown + 0xE0020, # .. 0xE007F ; Common + 0xE0080, # .. 0xE00FF ; Unknown + 0xE0100, # .. 0xE01EF ; Inherited + 0xE01F0, # .. 0x10FFFF ; Unknown +] + +VALUES = [ + "Zyyy", # 0000..0040 ; Common + "Latn", # 0041..005A ; Latin + "Zyyy", # 005B..0060 ; Common + "Latn", # 0061..007A ; Latin + "Zyyy", # 007B..00A9 ; Common + "Latn", # 00AA..00AA ; Latin + "Zyyy", # 00AB..00B9 ; Common + "Latn", # 00BA..00BA ; Latin + "Zyyy", # 00BB..00BF ; Common + "Latn", # 00C0..00D6 ; Latin + "Zyyy", # 00D7..00D7 ; Common + "Latn", # 00D8..00F6 ; Latin + "Zyyy", # 00F7..00F7 ; Common + "Latn", # 00F8..02B8 ; Latin + "Zyyy", # 02B9..02DF ; Common + "Latn", # 02E0..02E4 ; Latin + "Zyyy", # 02E5..02E9 ; Common + "Bopo", # 02EA..02EB ; Bopomofo + "Zyyy", # 02EC..02FF ; Common + "Zinh", # 0300..036F ; Inherited + "Grek", # 0370..0373 ; Greek + "Zyyy", # 0374..0374 ; Common + "Grek", # 0375..0377 ; Greek + "Zzzz", # 0378..0379 ; Unknown + "Grek", # 037A..037D ; Greek + "Zyyy", # 037E..037E ; Common + "Grek", # 037F..037F ; Greek + "Zzzz", # 0380..0383 ; Unknown + "Grek", # 0384..0384 ; Greek + "Zyyy", # 0385..0385 ; Common + "Grek", # 0386..0386 ; Greek + "Zyyy", # 0387..0387 ; Common + "Grek", # 0388..038A ; Greek + "Zzzz", # 038B..038B ; Unknown + "Grek", # 038C..038C ; Greek + "Zzzz", # 038D..038D ; Unknown + "Grek", # 038E..03A1 ; Greek + "Zzzz", # 03A2..03A2 ; Unknown + "Grek", # 03A3..03E1 ; Greek + "Copt", # 03E2..03EF ; Coptic + "Grek", # 03F0..03FF ; Greek + "Cyrl", # 0400..0484 ; Cyrillic + "Zinh", # 0485..0486 ; Inherited + "Cyrl", # 0487..052F ; Cyrillic + "Zzzz", # 0530..0530 ; Unknown + "Armn", # 0531..0556 ; Armenian + "Zzzz", # 0557..0558 ; Unknown + "Armn", # 0559..058A ; Armenian + "Zzzz", # 058B..058C ; Unknown + "Armn", # 058D..058F ; Armenian + "Zzzz", # 0590..0590 ; Unknown + "Hebr", # 0591..05C7 ; Hebrew + "Zzzz", # 05C8..05CF ; Unknown + "Hebr", # 05D0..05EA ; Hebrew + "Zzzz", # 05EB..05EE ; Unknown + "Hebr", # 05EF..05F4 ; Hebrew + "Zzzz", # 05F5..05FF ; Unknown + "Arab", # 0600..0604 ; Arabic + "Zyyy", # 0605..0605 ; Common + "Arab", # 0606..060B ; Arabic + "Zyyy", # 060C..060C ; Common + "Arab", # 060D..061A ; Arabic + "Zyyy", # 061B..061B ; Common + "Arab", # 061C..061E ; Arabic + "Zyyy", # 061F..061F ; Common + "Arab", # 0620..063F ; Arabic + "Zyyy", # 0640..0640 ; Common + "Arab", # 0641..064A ; Arabic + "Zinh", # 064B..0655 ; Inherited + "Arab", # 0656..066F ; Arabic + "Zinh", # 0670..0670 ; Inherited + "Arab", # 0671..06DC ; Arabic + "Zyyy", # 06DD..06DD ; Common + "Arab", # 06DE..06FF ; Arabic + "Syrc", # 0700..070D ; Syriac + "Zzzz", # 070E..070E ; Unknown + "Syrc", # 070F..074A ; Syriac + "Zzzz", # 074B..074C ; Unknown + "Syrc", # 074D..074F ; Syriac + "Arab", # 0750..077F ; Arabic + "Thaa", # 0780..07B1 ; Thaana + "Zzzz", # 07B2..07BF ; Unknown + "Nkoo", # 07C0..07FA ; Nko + "Zzzz", # 07FB..07FC ; Unknown + "Nkoo", # 07FD..07FF ; Nko + "Samr", # 0800..082D ; Samaritan + "Zzzz", # 082E..082F ; Unknown + "Samr", # 0830..083E ; Samaritan + "Zzzz", # 083F..083F ; Unknown + "Mand", # 0840..085B ; Mandaic + "Zzzz", # 085C..085D ; Unknown + "Mand", # 085E..085E ; Mandaic + "Zzzz", # 085F..085F ; Unknown + "Syrc", # 0860..086A ; Syriac + "Zzzz", # 086B..086F ; Unknown + "Arab", # 0870..0891 ; Arabic + "Zzzz", # 0892..0896 ; Unknown + "Arab", # 0897..08E1 ; Arabic + "Zyyy", # 08E2..08E2 ; Common + "Arab", # 08E3..08FF ; Arabic + "Deva", # 0900..0950 ; Devanagari + "Zinh", # 0951..0954 ; Inherited + "Deva", # 0955..0963 ; Devanagari + "Zyyy", # 0964..0965 ; Common + "Deva", # 0966..097F ; Devanagari + "Beng", # 0980..0983 ; Bengali + "Zzzz", # 0984..0984 ; Unknown + "Beng", # 0985..098C ; Bengali + "Zzzz", # 098D..098E ; Unknown + "Beng", # 098F..0990 ; Bengali + "Zzzz", # 0991..0992 ; Unknown + "Beng", # 0993..09A8 ; Bengali + "Zzzz", # 09A9..09A9 ; Unknown + "Beng", # 09AA..09B0 ; Bengali + "Zzzz", # 09B1..09B1 ; Unknown + "Beng", # 09B2..09B2 ; Bengali + "Zzzz", # 09B3..09B5 ; Unknown + "Beng", # 09B6..09B9 ; Bengali + "Zzzz", # 09BA..09BB ; Unknown + "Beng", # 09BC..09C4 ; Bengali + "Zzzz", # 09C5..09C6 ; Unknown + "Beng", # 09C7..09C8 ; Bengali + "Zzzz", # 09C9..09CA ; Unknown + "Beng", # 09CB..09CE ; Bengali + "Zzzz", # 09CF..09D6 ; Unknown + "Beng", # 09D7..09D7 ; Bengali + "Zzzz", # 09D8..09DB ; Unknown + "Beng", # 09DC..09DD ; Bengali + "Zzzz", # 09DE..09DE ; Unknown + "Beng", # 09DF..09E3 ; Bengali + "Zzzz", # 09E4..09E5 ; Unknown + "Beng", # 09E6..09FE ; Bengali + "Zzzz", # 09FF..0A00 ; Unknown + "Guru", # 0A01..0A03 ; Gurmukhi + "Zzzz", # 0A04..0A04 ; Unknown + "Guru", # 0A05..0A0A ; Gurmukhi + "Zzzz", # 0A0B..0A0E ; Unknown + "Guru", # 0A0F..0A10 ; Gurmukhi + "Zzzz", # 0A11..0A12 ; Unknown + "Guru", # 0A13..0A28 ; Gurmukhi + "Zzzz", # 0A29..0A29 ; Unknown + "Guru", # 0A2A..0A30 ; Gurmukhi + "Zzzz", # 0A31..0A31 ; Unknown + "Guru", # 0A32..0A33 ; Gurmukhi + "Zzzz", # 0A34..0A34 ; Unknown + "Guru", # 0A35..0A36 ; Gurmukhi + "Zzzz", # 0A37..0A37 ; Unknown + "Guru", # 0A38..0A39 ; Gurmukhi + "Zzzz", # 0A3A..0A3B ; Unknown + "Guru", # 0A3C..0A3C ; Gurmukhi + "Zzzz", # 0A3D..0A3D ; Unknown + "Guru", # 0A3E..0A42 ; Gurmukhi + "Zzzz", # 0A43..0A46 ; Unknown + "Guru", # 0A47..0A48 ; Gurmukhi + "Zzzz", # 0A49..0A4A ; Unknown + "Guru", # 0A4B..0A4D ; Gurmukhi + "Zzzz", # 0A4E..0A50 ; Unknown + "Guru", # 0A51..0A51 ; Gurmukhi + "Zzzz", # 0A52..0A58 ; Unknown + "Guru", # 0A59..0A5C ; Gurmukhi + "Zzzz", # 0A5D..0A5D ; Unknown + "Guru", # 0A5E..0A5E ; Gurmukhi + "Zzzz", # 0A5F..0A65 ; Unknown + "Guru", # 0A66..0A76 ; Gurmukhi + "Zzzz", # 0A77..0A80 ; Unknown + "Gujr", # 0A81..0A83 ; Gujarati + "Zzzz", # 0A84..0A84 ; Unknown + "Gujr", # 0A85..0A8D ; Gujarati + "Zzzz", # 0A8E..0A8E ; Unknown + "Gujr", # 0A8F..0A91 ; Gujarati + "Zzzz", # 0A92..0A92 ; Unknown + "Gujr", # 0A93..0AA8 ; Gujarati + "Zzzz", # 0AA9..0AA9 ; Unknown + "Gujr", # 0AAA..0AB0 ; Gujarati + "Zzzz", # 0AB1..0AB1 ; Unknown + "Gujr", # 0AB2..0AB3 ; Gujarati + "Zzzz", # 0AB4..0AB4 ; Unknown + "Gujr", # 0AB5..0AB9 ; Gujarati + "Zzzz", # 0ABA..0ABB ; Unknown + "Gujr", # 0ABC..0AC5 ; Gujarati + "Zzzz", # 0AC6..0AC6 ; Unknown + "Gujr", # 0AC7..0AC9 ; Gujarati + "Zzzz", # 0ACA..0ACA ; Unknown + "Gujr", # 0ACB..0ACD ; Gujarati + "Zzzz", # 0ACE..0ACF ; Unknown + "Gujr", # 0AD0..0AD0 ; Gujarati + "Zzzz", # 0AD1..0ADF ; Unknown + "Gujr", # 0AE0..0AE3 ; Gujarati + "Zzzz", # 0AE4..0AE5 ; Unknown + "Gujr", # 0AE6..0AF1 ; Gujarati + "Zzzz", # 0AF2..0AF8 ; Unknown + "Gujr", # 0AF9..0AFF ; Gujarati + "Zzzz", # 0B00..0B00 ; Unknown + "Orya", # 0B01..0B03 ; Oriya + "Zzzz", # 0B04..0B04 ; Unknown + "Orya", # 0B05..0B0C ; Oriya + "Zzzz", # 0B0D..0B0E ; Unknown + "Orya", # 0B0F..0B10 ; Oriya + "Zzzz", # 0B11..0B12 ; Unknown + "Orya", # 0B13..0B28 ; Oriya + "Zzzz", # 0B29..0B29 ; Unknown + "Orya", # 0B2A..0B30 ; Oriya + "Zzzz", # 0B31..0B31 ; Unknown + "Orya", # 0B32..0B33 ; Oriya + "Zzzz", # 0B34..0B34 ; Unknown + "Orya", # 0B35..0B39 ; Oriya + "Zzzz", # 0B3A..0B3B ; Unknown + "Orya", # 0B3C..0B44 ; Oriya + "Zzzz", # 0B45..0B46 ; Unknown + "Orya", # 0B47..0B48 ; Oriya + "Zzzz", # 0B49..0B4A ; Unknown + "Orya", # 0B4B..0B4D ; Oriya + "Zzzz", # 0B4E..0B54 ; Unknown + "Orya", # 0B55..0B57 ; Oriya + "Zzzz", # 0B58..0B5B ; Unknown + "Orya", # 0B5C..0B5D ; Oriya + "Zzzz", # 0B5E..0B5E ; Unknown + "Orya", # 0B5F..0B63 ; Oriya + "Zzzz", # 0B64..0B65 ; Unknown + "Orya", # 0B66..0B77 ; Oriya + "Zzzz", # 0B78..0B81 ; Unknown + "Taml", # 0B82..0B83 ; Tamil + "Zzzz", # 0B84..0B84 ; Unknown + "Taml", # 0B85..0B8A ; Tamil + "Zzzz", # 0B8B..0B8D ; Unknown + "Taml", # 0B8E..0B90 ; Tamil + "Zzzz", # 0B91..0B91 ; Unknown + "Taml", # 0B92..0B95 ; Tamil + "Zzzz", # 0B96..0B98 ; Unknown + "Taml", # 0B99..0B9A ; Tamil + "Zzzz", # 0B9B..0B9B ; Unknown + "Taml", # 0B9C..0B9C ; Tamil + "Zzzz", # 0B9D..0B9D ; Unknown + "Taml", # 0B9E..0B9F ; Tamil + "Zzzz", # 0BA0..0BA2 ; Unknown + "Taml", # 0BA3..0BA4 ; Tamil + "Zzzz", # 0BA5..0BA7 ; Unknown + "Taml", # 0BA8..0BAA ; Tamil + "Zzzz", # 0BAB..0BAD ; Unknown + "Taml", # 0BAE..0BB9 ; Tamil + "Zzzz", # 0BBA..0BBD ; Unknown + "Taml", # 0BBE..0BC2 ; Tamil + "Zzzz", # 0BC3..0BC5 ; Unknown + "Taml", # 0BC6..0BC8 ; Tamil + "Zzzz", # 0BC9..0BC9 ; Unknown + "Taml", # 0BCA..0BCD ; Tamil + "Zzzz", # 0BCE..0BCF ; Unknown + "Taml", # 0BD0..0BD0 ; Tamil + "Zzzz", # 0BD1..0BD6 ; Unknown + "Taml", # 0BD7..0BD7 ; Tamil + "Zzzz", # 0BD8..0BE5 ; Unknown + "Taml", # 0BE6..0BFA ; Tamil + "Zzzz", # 0BFB..0BFF ; Unknown + "Telu", # 0C00..0C0C ; Telugu + "Zzzz", # 0C0D..0C0D ; Unknown + "Telu", # 0C0E..0C10 ; Telugu + "Zzzz", # 0C11..0C11 ; Unknown + "Telu", # 0C12..0C28 ; Telugu + "Zzzz", # 0C29..0C29 ; Unknown + "Telu", # 0C2A..0C39 ; Telugu + "Zzzz", # 0C3A..0C3B ; Unknown + "Telu", # 0C3C..0C44 ; Telugu + "Zzzz", # 0C45..0C45 ; Unknown + "Telu", # 0C46..0C48 ; Telugu + "Zzzz", # 0C49..0C49 ; Unknown + "Telu", # 0C4A..0C4D ; Telugu + "Zzzz", # 0C4E..0C54 ; Unknown + "Telu", # 0C55..0C56 ; Telugu + "Zzzz", # 0C57..0C57 ; Unknown + "Telu", # 0C58..0C5A ; Telugu + "Zzzz", # 0C5B..0C5B ; Unknown + "Telu", # 0C5C..0C5D ; Telugu + "Zzzz", # 0C5E..0C5F ; Unknown + "Telu", # 0C60..0C63 ; Telugu + "Zzzz", # 0C64..0C65 ; Unknown + "Telu", # 0C66..0C6F ; Telugu + "Zzzz", # 0C70..0C76 ; Unknown + "Telu", # 0C77..0C7F ; Telugu + "Knda", # 0C80..0C8C ; Kannada + "Zzzz", # 0C8D..0C8D ; Unknown + "Knda", # 0C8E..0C90 ; Kannada + "Zzzz", # 0C91..0C91 ; Unknown + "Knda", # 0C92..0CA8 ; Kannada + "Zzzz", # 0CA9..0CA9 ; Unknown + "Knda", # 0CAA..0CB3 ; Kannada + "Zzzz", # 0CB4..0CB4 ; Unknown + "Knda", # 0CB5..0CB9 ; Kannada + "Zzzz", # 0CBA..0CBB ; Unknown + "Knda", # 0CBC..0CC4 ; Kannada + "Zzzz", # 0CC5..0CC5 ; Unknown + "Knda", # 0CC6..0CC8 ; Kannada + "Zzzz", # 0CC9..0CC9 ; Unknown + "Knda", # 0CCA..0CCD ; Kannada + "Zzzz", # 0CCE..0CD4 ; Unknown + "Knda", # 0CD5..0CD6 ; Kannada + "Zzzz", # 0CD7..0CDB ; Unknown + "Knda", # 0CDC..0CDE ; Kannada + "Zzzz", # 0CDF..0CDF ; Unknown + "Knda", # 0CE0..0CE3 ; Kannada + "Zzzz", # 0CE4..0CE5 ; Unknown + "Knda", # 0CE6..0CEF ; Kannada + "Zzzz", # 0CF0..0CF0 ; Unknown + "Knda", # 0CF1..0CF3 ; Kannada + "Zzzz", # 0CF4..0CFF ; Unknown + "Mlym", # 0D00..0D0C ; Malayalam + "Zzzz", # 0D0D..0D0D ; Unknown + "Mlym", # 0D0E..0D10 ; Malayalam + "Zzzz", # 0D11..0D11 ; Unknown + "Mlym", # 0D12..0D44 ; Malayalam + "Zzzz", # 0D45..0D45 ; Unknown + "Mlym", # 0D46..0D48 ; Malayalam + "Zzzz", # 0D49..0D49 ; Unknown + "Mlym", # 0D4A..0D4F ; Malayalam + "Zzzz", # 0D50..0D53 ; Unknown + "Mlym", # 0D54..0D63 ; Malayalam + "Zzzz", # 0D64..0D65 ; Unknown + "Mlym", # 0D66..0D7F ; Malayalam + "Zzzz", # 0D80..0D80 ; Unknown + "Sinh", # 0D81..0D83 ; Sinhala + "Zzzz", # 0D84..0D84 ; Unknown + "Sinh", # 0D85..0D96 ; Sinhala + "Zzzz", # 0D97..0D99 ; Unknown + "Sinh", # 0D9A..0DB1 ; Sinhala + "Zzzz", # 0DB2..0DB2 ; Unknown + "Sinh", # 0DB3..0DBB ; Sinhala + "Zzzz", # 0DBC..0DBC ; Unknown + "Sinh", # 0DBD..0DBD ; Sinhala + "Zzzz", # 0DBE..0DBF ; Unknown + "Sinh", # 0DC0..0DC6 ; Sinhala + "Zzzz", # 0DC7..0DC9 ; Unknown + "Sinh", # 0DCA..0DCA ; Sinhala + "Zzzz", # 0DCB..0DCE ; Unknown + "Sinh", # 0DCF..0DD4 ; Sinhala + "Zzzz", # 0DD5..0DD5 ; Unknown + "Sinh", # 0DD6..0DD6 ; Sinhala + "Zzzz", # 0DD7..0DD7 ; Unknown + "Sinh", # 0DD8..0DDF ; Sinhala + "Zzzz", # 0DE0..0DE5 ; Unknown + "Sinh", # 0DE6..0DEF ; Sinhala + "Zzzz", # 0DF0..0DF1 ; Unknown + "Sinh", # 0DF2..0DF4 ; Sinhala + "Zzzz", # 0DF5..0E00 ; Unknown + "Thai", # 0E01..0E3A ; Thai + "Zzzz", # 0E3B..0E3E ; Unknown + "Zyyy", # 0E3F..0E3F ; Common + "Thai", # 0E40..0E5B ; Thai + "Zzzz", # 0E5C..0E80 ; Unknown + "Laoo", # 0E81..0E82 ; Lao + "Zzzz", # 0E83..0E83 ; Unknown + "Laoo", # 0E84..0E84 ; Lao + "Zzzz", # 0E85..0E85 ; Unknown + "Laoo", # 0E86..0E8A ; Lao + "Zzzz", # 0E8B..0E8B ; Unknown + "Laoo", # 0E8C..0EA3 ; Lao + "Zzzz", # 0EA4..0EA4 ; Unknown + "Laoo", # 0EA5..0EA5 ; Lao + "Zzzz", # 0EA6..0EA6 ; Unknown + "Laoo", # 0EA7..0EBD ; Lao + "Zzzz", # 0EBE..0EBF ; Unknown + "Laoo", # 0EC0..0EC4 ; Lao + "Zzzz", # 0EC5..0EC5 ; Unknown + "Laoo", # 0EC6..0EC6 ; Lao + "Zzzz", # 0EC7..0EC7 ; Unknown + "Laoo", # 0EC8..0ECE ; Lao + "Zzzz", # 0ECF..0ECF ; Unknown + "Laoo", # 0ED0..0ED9 ; Lao + "Zzzz", # 0EDA..0EDB ; Unknown + "Laoo", # 0EDC..0EDF ; Lao + "Zzzz", # 0EE0..0EFF ; Unknown + "Tibt", # 0F00..0F47 ; Tibetan + "Zzzz", # 0F48..0F48 ; Unknown + "Tibt", # 0F49..0F6C ; Tibetan + "Zzzz", # 0F6D..0F70 ; Unknown + "Tibt", # 0F71..0F97 ; Tibetan + "Zzzz", # 0F98..0F98 ; Unknown + "Tibt", # 0F99..0FBC ; Tibetan + "Zzzz", # 0FBD..0FBD ; Unknown + "Tibt", # 0FBE..0FCC ; Tibetan + "Zzzz", # 0FCD..0FCD ; Unknown + "Tibt", # 0FCE..0FD4 ; Tibetan + "Zyyy", # 0FD5..0FD8 ; Common + "Tibt", # 0FD9..0FDA ; Tibetan + "Zzzz", # 0FDB..0FFF ; Unknown + "Mymr", # 1000..109F ; Myanmar + "Geor", # 10A0..10C5 ; Georgian + "Zzzz", # 10C6..10C6 ; Unknown + "Geor", # 10C7..10C7 ; Georgian + "Zzzz", # 10C8..10CC ; Unknown + "Geor", # 10CD..10CD ; Georgian + "Zzzz", # 10CE..10CF ; Unknown + "Geor", # 10D0..10FA ; Georgian + "Zyyy", # 10FB..10FB ; Common + "Geor", # 10FC..10FF ; Georgian + "Hang", # 1100..11FF ; Hangul + "Ethi", # 1200..1248 ; Ethiopic + "Zzzz", # 1249..1249 ; Unknown + "Ethi", # 124A..124D ; Ethiopic + "Zzzz", # 124E..124F ; Unknown + "Ethi", # 1250..1256 ; Ethiopic + "Zzzz", # 1257..1257 ; Unknown + "Ethi", # 1258..1258 ; Ethiopic + "Zzzz", # 1259..1259 ; Unknown + "Ethi", # 125A..125D ; Ethiopic + "Zzzz", # 125E..125F ; Unknown + "Ethi", # 1260..1288 ; Ethiopic + "Zzzz", # 1289..1289 ; Unknown + "Ethi", # 128A..128D ; Ethiopic + "Zzzz", # 128E..128F ; Unknown + "Ethi", # 1290..12B0 ; Ethiopic + "Zzzz", # 12B1..12B1 ; Unknown + "Ethi", # 12B2..12B5 ; Ethiopic + "Zzzz", # 12B6..12B7 ; Unknown + "Ethi", # 12B8..12BE ; Ethiopic + "Zzzz", # 12BF..12BF ; Unknown + "Ethi", # 12C0..12C0 ; Ethiopic + "Zzzz", # 12C1..12C1 ; Unknown + "Ethi", # 12C2..12C5 ; Ethiopic + "Zzzz", # 12C6..12C7 ; Unknown + "Ethi", # 12C8..12D6 ; Ethiopic + "Zzzz", # 12D7..12D7 ; Unknown + "Ethi", # 12D8..1310 ; Ethiopic + "Zzzz", # 1311..1311 ; Unknown + "Ethi", # 1312..1315 ; Ethiopic + "Zzzz", # 1316..1317 ; Unknown + "Ethi", # 1318..135A ; Ethiopic + "Zzzz", # 135B..135C ; Unknown + "Ethi", # 135D..137C ; Ethiopic + "Zzzz", # 137D..137F ; Unknown + "Ethi", # 1380..1399 ; Ethiopic + "Zzzz", # 139A..139F ; Unknown + "Cher", # 13A0..13F5 ; Cherokee + "Zzzz", # 13F6..13F7 ; Unknown + "Cher", # 13F8..13FD ; Cherokee + "Zzzz", # 13FE..13FF ; Unknown + "Cans", # 1400..167F ; Canadian_Aboriginal + "Ogam", # 1680..169C ; Ogham + "Zzzz", # 169D..169F ; Unknown + "Runr", # 16A0..16EA ; Runic + "Zyyy", # 16EB..16ED ; Common + "Runr", # 16EE..16F8 ; Runic + "Zzzz", # 16F9..16FF ; Unknown + "Tglg", # 1700..1715 ; Tagalog + "Zzzz", # 1716..171E ; Unknown + "Tglg", # 171F..171F ; Tagalog + "Hano", # 1720..1734 ; Hanunoo + "Zyyy", # 1735..1736 ; Common + "Zzzz", # 1737..173F ; Unknown + "Buhd", # 1740..1753 ; Buhid + "Zzzz", # 1754..175F ; Unknown + "Tagb", # 1760..176C ; Tagbanwa + "Zzzz", # 176D..176D ; Unknown + "Tagb", # 176E..1770 ; Tagbanwa + "Zzzz", # 1771..1771 ; Unknown + "Tagb", # 1772..1773 ; Tagbanwa + "Zzzz", # 1774..177F ; Unknown + "Khmr", # 1780..17DD ; Khmer + "Zzzz", # 17DE..17DF ; Unknown + "Khmr", # 17E0..17E9 ; Khmer + "Zzzz", # 17EA..17EF ; Unknown + "Khmr", # 17F0..17F9 ; Khmer + "Zzzz", # 17FA..17FF ; Unknown + "Mong", # 1800..1801 ; Mongolian + "Zyyy", # 1802..1803 ; Common + "Mong", # 1804..1804 ; Mongolian + "Zyyy", # 1805..1805 ; Common + "Mong", # 1806..1819 ; Mongolian + "Zzzz", # 181A..181F ; Unknown + "Mong", # 1820..1878 ; Mongolian + "Zzzz", # 1879..187F ; Unknown + "Mong", # 1880..18AA ; Mongolian + "Zzzz", # 18AB..18AF ; Unknown + "Cans", # 18B0..18F5 ; Canadian_Aboriginal + "Zzzz", # 18F6..18FF ; Unknown + "Limb", # 1900..191E ; Limbu + "Zzzz", # 191F..191F ; Unknown + "Limb", # 1920..192B ; Limbu + "Zzzz", # 192C..192F ; Unknown + "Limb", # 1930..193B ; Limbu + "Zzzz", # 193C..193F ; Unknown + "Limb", # 1940..1940 ; Limbu + "Zzzz", # 1941..1943 ; Unknown + "Limb", # 1944..194F ; Limbu + "Tale", # 1950..196D ; Tai_Le + "Zzzz", # 196E..196F ; Unknown + "Tale", # 1970..1974 ; Tai_Le + "Zzzz", # 1975..197F ; Unknown + "Talu", # 1980..19AB ; New_Tai_Lue + "Zzzz", # 19AC..19AF ; Unknown + "Talu", # 19B0..19C9 ; New_Tai_Lue + "Zzzz", # 19CA..19CF ; Unknown + "Talu", # 19D0..19DA ; New_Tai_Lue + "Zzzz", # 19DB..19DD ; Unknown + "Talu", # 19DE..19DF ; New_Tai_Lue + "Khmr", # 19E0..19FF ; Khmer + "Bugi", # 1A00..1A1B ; Buginese + "Zzzz", # 1A1C..1A1D ; Unknown + "Bugi", # 1A1E..1A1F ; Buginese + "Lana", # 1A20..1A5E ; Tai_Tham + "Zzzz", # 1A5F..1A5F ; Unknown + "Lana", # 1A60..1A7C ; Tai_Tham + "Zzzz", # 1A7D..1A7E ; Unknown + "Lana", # 1A7F..1A89 ; Tai_Tham + "Zzzz", # 1A8A..1A8F ; Unknown + "Lana", # 1A90..1A99 ; Tai_Tham + "Zzzz", # 1A9A..1A9F ; Unknown + "Lana", # 1AA0..1AAD ; Tai_Tham + "Zzzz", # 1AAE..1AAF ; Unknown + "Zinh", # 1AB0..1ADD ; Inherited + "Zzzz", # 1ADE..1ADF ; Unknown + "Zinh", # 1AE0..1AEB ; Inherited + "Zzzz", # 1AEC..1AFF ; Unknown + "Bali", # 1B00..1B4C ; Balinese + "Zzzz", # 1B4D..1B4D ; Unknown + "Bali", # 1B4E..1B7F ; Balinese + "Sund", # 1B80..1BBF ; Sundanese + "Batk", # 1BC0..1BF3 ; Batak + "Zzzz", # 1BF4..1BFB ; Unknown + "Batk", # 1BFC..1BFF ; Batak + "Lepc", # 1C00..1C37 ; Lepcha + "Zzzz", # 1C38..1C3A ; Unknown + "Lepc", # 1C3B..1C49 ; Lepcha + "Zzzz", # 1C4A..1C4C ; Unknown + "Lepc", # 1C4D..1C4F ; Lepcha + "Olck", # 1C50..1C7F ; Ol_Chiki + "Cyrl", # 1C80..1C8A ; Cyrillic + "Zzzz", # 1C8B..1C8F ; Unknown + "Geor", # 1C90..1CBA ; Georgian + "Zzzz", # 1CBB..1CBC ; Unknown + "Geor", # 1CBD..1CBF ; Georgian + "Sund", # 1CC0..1CC7 ; Sundanese + "Zzzz", # 1CC8..1CCF ; Unknown + "Zinh", # 1CD0..1CD2 ; Inherited + "Zyyy", # 1CD3..1CD3 ; Common + "Zinh", # 1CD4..1CE0 ; Inherited + "Zyyy", # 1CE1..1CE1 ; Common + "Zinh", # 1CE2..1CE8 ; Inherited + "Zyyy", # 1CE9..1CEC ; Common + "Zinh", # 1CED..1CED ; Inherited + "Zyyy", # 1CEE..1CF3 ; Common + "Zinh", # 1CF4..1CF4 ; Inherited + "Zyyy", # 1CF5..1CF7 ; Common + "Zinh", # 1CF8..1CF9 ; Inherited + "Zyyy", # 1CFA..1CFA ; Common + "Zzzz", # 1CFB..1CFF ; Unknown + "Latn", # 1D00..1D25 ; Latin + "Grek", # 1D26..1D2A ; Greek + "Cyrl", # 1D2B..1D2B ; Cyrillic + "Latn", # 1D2C..1D5C ; Latin + "Grek", # 1D5D..1D61 ; Greek + "Latn", # 1D62..1D65 ; Latin + "Grek", # 1D66..1D6A ; Greek + "Latn", # 1D6B..1D77 ; Latin + "Cyrl", # 1D78..1D78 ; Cyrillic + "Latn", # 1D79..1DBE ; Latin + "Grek", # 1DBF..1DBF ; Greek + "Zinh", # 1DC0..1DFF ; Inherited + "Latn", # 1E00..1EFF ; Latin + "Grek", # 1F00..1F15 ; Greek + "Zzzz", # 1F16..1F17 ; Unknown + "Grek", # 1F18..1F1D ; Greek + "Zzzz", # 1F1E..1F1F ; Unknown + "Grek", # 1F20..1F45 ; Greek + "Zzzz", # 1F46..1F47 ; Unknown + "Grek", # 1F48..1F4D ; Greek + "Zzzz", # 1F4E..1F4F ; Unknown + "Grek", # 1F50..1F57 ; Greek + "Zzzz", # 1F58..1F58 ; Unknown + "Grek", # 1F59..1F59 ; Greek + "Zzzz", # 1F5A..1F5A ; Unknown + "Grek", # 1F5B..1F5B ; Greek + "Zzzz", # 1F5C..1F5C ; Unknown + "Grek", # 1F5D..1F5D ; Greek + "Zzzz", # 1F5E..1F5E ; Unknown + "Grek", # 1F5F..1F7D ; Greek + "Zzzz", # 1F7E..1F7F ; Unknown + "Grek", # 1F80..1FB4 ; Greek + "Zzzz", # 1FB5..1FB5 ; Unknown + "Grek", # 1FB6..1FC4 ; Greek + "Zzzz", # 1FC5..1FC5 ; Unknown + "Grek", # 1FC6..1FD3 ; Greek + "Zzzz", # 1FD4..1FD5 ; Unknown + "Grek", # 1FD6..1FDB ; Greek + "Zzzz", # 1FDC..1FDC ; Unknown + "Grek", # 1FDD..1FEF ; Greek + "Zzzz", # 1FF0..1FF1 ; Unknown + "Grek", # 1FF2..1FF4 ; Greek + "Zzzz", # 1FF5..1FF5 ; Unknown + "Grek", # 1FF6..1FFE ; Greek + "Zzzz", # 1FFF..1FFF ; Unknown + "Zyyy", # 2000..200B ; Common + "Zinh", # 200C..200D ; Inherited + "Zyyy", # 200E..2064 ; Common + "Zzzz", # 2065..2065 ; Unknown + "Zyyy", # 2066..2070 ; Common + "Latn", # 2071..2071 ; Latin + "Zzzz", # 2072..2073 ; Unknown + "Zyyy", # 2074..207E ; Common + "Latn", # 207F..207F ; Latin + "Zyyy", # 2080..208E ; Common + "Zzzz", # 208F..208F ; Unknown + "Latn", # 2090..209C ; Latin + "Zzzz", # 209D..209F ; Unknown + "Zyyy", # 20A0..20C1 ; Common + "Zzzz", # 20C2..20CF ; Unknown + "Zinh", # 20D0..20F0 ; Inherited + "Zzzz", # 20F1..20FF ; Unknown + "Zyyy", # 2100..2125 ; Common + "Grek", # 2126..2126 ; Greek + "Zyyy", # 2127..2129 ; Common + "Latn", # 212A..212B ; Latin + "Zyyy", # 212C..2131 ; Common + "Latn", # 2132..2132 ; Latin + "Zyyy", # 2133..214D ; Common + "Latn", # 214E..214E ; Latin + "Zyyy", # 214F..215F ; Common + "Latn", # 2160..2188 ; Latin + "Zyyy", # 2189..218B ; Common + "Zzzz", # 218C..218F ; Unknown + "Zyyy", # 2190..2429 ; Common + "Zzzz", # 242A..243F ; Unknown + "Zyyy", # 2440..244A ; Common + "Zzzz", # 244B..245F ; Unknown + "Zyyy", # 2460..27FF ; Common + "Brai", # 2800..28FF ; Braille + "Zyyy", # 2900..2B73 ; Common + "Zzzz", # 2B74..2B75 ; Unknown + "Zyyy", # 2B76..2BFF ; Common + "Glag", # 2C00..2C5F ; Glagolitic + "Latn", # 2C60..2C7F ; Latin + "Copt", # 2C80..2CF3 ; Coptic + "Zzzz", # 2CF4..2CF8 ; Unknown + "Copt", # 2CF9..2CFF ; Coptic + "Geor", # 2D00..2D25 ; Georgian + "Zzzz", # 2D26..2D26 ; Unknown + "Geor", # 2D27..2D27 ; Georgian + "Zzzz", # 2D28..2D2C ; Unknown + "Geor", # 2D2D..2D2D ; Georgian + "Zzzz", # 2D2E..2D2F ; Unknown + "Tfng", # 2D30..2D67 ; Tifinagh + "Zzzz", # 2D68..2D6E ; Unknown + "Tfng", # 2D6F..2D70 ; Tifinagh + "Zzzz", # 2D71..2D7E ; Unknown + "Tfng", # 2D7F..2D7F ; Tifinagh + "Ethi", # 2D80..2D96 ; Ethiopic + "Zzzz", # 2D97..2D9F ; Unknown + "Ethi", # 2DA0..2DA6 ; Ethiopic + "Zzzz", # 2DA7..2DA7 ; Unknown + "Ethi", # 2DA8..2DAE ; Ethiopic + "Zzzz", # 2DAF..2DAF ; Unknown + "Ethi", # 2DB0..2DB6 ; Ethiopic + "Zzzz", # 2DB7..2DB7 ; Unknown + "Ethi", # 2DB8..2DBE ; Ethiopic + "Zzzz", # 2DBF..2DBF ; Unknown + "Ethi", # 2DC0..2DC6 ; Ethiopic + "Zzzz", # 2DC7..2DC7 ; Unknown + "Ethi", # 2DC8..2DCE ; Ethiopic + "Zzzz", # 2DCF..2DCF ; Unknown + "Ethi", # 2DD0..2DD6 ; Ethiopic + "Zzzz", # 2DD7..2DD7 ; Unknown + "Ethi", # 2DD8..2DDE ; Ethiopic + "Zzzz", # 2DDF..2DDF ; Unknown + "Cyrl", # 2DE0..2DFF ; Cyrillic + "Zyyy", # 2E00..2E5D ; Common + "Zzzz", # 2E5E..2E7F ; Unknown + "Hani", # 2E80..2E99 ; Han + "Zzzz", # 2E9A..2E9A ; Unknown + "Hani", # 2E9B..2EF3 ; Han + "Zzzz", # 2EF4..2EFF ; Unknown + "Hani", # 2F00..2FD5 ; Han + "Zzzz", # 2FD6..2FEF ; Unknown + "Zyyy", # 2FF0..3004 ; Common + "Hani", # 3005..3005 ; Han + "Zyyy", # 3006..3006 ; Common + "Hani", # 3007..3007 ; Han + "Zyyy", # 3008..3020 ; Common + "Hani", # 3021..3029 ; Han + "Zinh", # 302A..302D ; Inherited + "Hang", # 302E..302F ; Hangul + "Zyyy", # 3030..3037 ; Common + "Hani", # 3038..303B ; Han + "Zyyy", # 303C..303F ; Common + "Zzzz", # 3040..3040 ; Unknown + "Hira", # 3041..3096 ; Hiragana + "Zzzz", # 3097..3098 ; Unknown + "Zinh", # 3099..309A ; Inherited + "Zyyy", # 309B..309C ; Common + "Hira", # 309D..309F ; Hiragana + "Zyyy", # 30A0..30A0 ; Common + "Kana", # 30A1..30FA ; Katakana + "Zyyy", # 30FB..30FC ; Common + "Kana", # 30FD..30FF ; Katakana + "Zzzz", # 3100..3104 ; Unknown + "Bopo", # 3105..312F ; Bopomofo + "Zzzz", # 3130..3130 ; Unknown + "Hang", # 3131..318E ; Hangul + "Zzzz", # 318F..318F ; Unknown + "Zyyy", # 3190..319F ; Common + "Bopo", # 31A0..31BF ; Bopomofo + "Zyyy", # 31C0..31E5 ; Common + "Zzzz", # 31E6..31EE ; Unknown + "Zyyy", # 31EF..31EF ; Common + "Kana", # 31F0..31FF ; Katakana + "Hang", # 3200..321E ; Hangul + "Zzzz", # 321F..321F ; Unknown + "Zyyy", # 3220..325F ; Common + "Hang", # 3260..327E ; Hangul + "Zyyy", # 327F..32CF ; Common + "Kana", # 32D0..32FE ; Katakana + "Zyyy", # 32FF..32FF ; Common + "Kana", # 3300..3357 ; Katakana + "Zyyy", # 3358..33FF ; Common + "Hani", # 3400..4DBF ; Han + "Zyyy", # 4DC0..4DFF ; Common + "Hani", # 4E00..9FFF ; Han + "Yiii", # A000..A48C ; Yi + "Zzzz", # A48D..A48F ; Unknown + "Yiii", # A490..A4C6 ; Yi + "Zzzz", # A4C7..A4CF ; Unknown + "Lisu", # A4D0..A4FF ; Lisu + "Vaii", # A500..A62B ; Vai + "Zzzz", # A62C..A63F ; Unknown + "Cyrl", # A640..A69F ; Cyrillic + "Bamu", # A6A0..A6F7 ; Bamum + "Zzzz", # A6F8..A6FF ; Unknown + "Zyyy", # A700..A721 ; Common + "Latn", # A722..A787 ; Latin + "Zyyy", # A788..A78A ; Common + "Latn", # A78B..A7DC ; Latin + "Zzzz", # A7DD..A7F0 ; Unknown + "Latn", # A7F1..A7FF ; Latin + "Sylo", # A800..A82C ; Syloti_Nagri + "Zzzz", # A82D..A82F ; Unknown + "Zyyy", # A830..A839 ; Common + "Zzzz", # A83A..A83F ; Unknown + "Phag", # A840..A877 ; Phags_Pa + "Zzzz", # A878..A87F ; Unknown + "Saur", # A880..A8C5 ; Saurashtra + "Zzzz", # A8C6..A8CD ; Unknown + "Saur", # A8CE..A8D9 ; Saurashtra + "Zzzz", # A8DA..A8DF ; Unknown + "Deva", # A8E0..A8FF ; Devanagari + "Kali", # A900..A92D ; Kayah_Li + "Zyyy", # A92E..A92E ; Common + "Kali", # A92F..A92F ; Kayah_Li + "Rjng", # A930..A953 ; Rejang + "Zzzz", # A954..A95E ; Unknown + "Rjng", # A95F..A95F ; Rejang + "Hang", # A960..A97C ; Hangul + "Zzzz", # A97D..A97F ; Unknown + "Java", # A980..A9CD ; Javanese + "Zzzz", # A9CE..A9CE ; Unknown + "Zyyy", # A9CF..A9CF ; Common + "Java", # A9D0..A9D9 ; Javanese + "Zzzz", # A9DA..A9DD ; Unknown + "Java", # A9DE..A9DF ; Javanese + "Mymr", # A9E0..A9FE ; Myanmar + "Zzzz", # A9FF..A9FF ; Unknown + "Cham", # AA00..AA36 ; Cham + "Zzzz", # AA37..AA3F ; Unknown + "Cham", # AA40..AA4D ; Cham + "Zzzz", # AA4E..AA4F ; Unknown + "Cham", # AA50..AA59 ; Cham + "Zzzz", # AA5A..AA5B ; Unknown + "Cham", # AA5C..AA5F ; Cham + "Mymr", # AA60..AA7F ; Myanmar + "Tavt", # AA80..AAC2 ; Tai_Viet + "Zzzz", # AAC3..AADA ; Unknown + "Tavt", # AADB..AADF ; Tai_Viet + "Mtei", # AAE0..AAF6 ; Meetei_Mayek + "Zzzz", # AAF7..AB00 ; Unknown + "Ethi", # AB01..AB06 ; Ethiopic + "Zzzz", # AB07..AB08 ; Unknown + "Ethi", # AB09..AB0E ; Ethiopic + "Zzzz", # AB0F..AB10 ; Unknown + "Ethi", # AB11..AB16 ; Ethiopic + "Zzzz", # AB17..AB1F ; Unknown + "Ethi", # AB20..AB26 ; Ethiopic + "Zzzz", # AB27..AB27 ; Unknown + "Ethi", # AB28..AB2E ; Ethiopic + "Zzzz", # AB2F..AB2F ; Unknown + "Latn", # AB30..AB5A ; Latin + "Zyyy", # AB5B..AB5B ; Common + "Latn", # AB5C..AB64 ; Latin + "Grek", # AB65..AB65 ; Greek + "Latn", # AB66..AB69 ; Latin + "Zyyy", # AB6A..AB6B ; Common + "Zzzz", # AB6C..AB6F ; Unknown + "Cher", # AB70..ABBF ; Cherokee + "Mtei", # ABC0..ABED ; Meetei_Mayek + "Zzzz", # ABEE..ABEF ; Unknown + "Mtei", # ABF0..ABF9 ; Meetei_Mayek + "Zzzz", # ABFA..ABFF ; Unknown + "Hang", # AC00..D7A3 ; Hangul + "Zzzz", # D7A4..D7AF ; Unknown + "Hang", # D7B0..D7C6 ; Hangul + "Zzzz", # D7C7..D7CA ; Unknown + "Hang", # D7CB..D7FB ; Hangul + "Zzzz", # D7FC..F8FF ; Unknown + "Hani", # F900..FA6D ; Han + "Zzzz", # FA6E..FA6F ; Unknown + "Hani", # FA70..FAD9 ; Han + "Zzzz", # FADA..FAFF ; Unknown + "Latn", # FB00..FB06 ; Latin + "Zzzz", # FB07..FB12 ; Unknown + "Armn", # FB13..FB17 ; Armenian + "Zzzz", # FB18..FB1C ; Unknown + "Hebr", # FB1D..FB36 ; Hebrew + "Zzzz", # FB37..FB37 ; Unknown + "Hebr", # FB38..FB3C ; Hebrew + "Zzzz", # FB3D..FB3D ; Unknown + "Hebr", # FB3E..FB3E ; Hebrew + "Zzzz", # FB3F..FB3F ; Unknown + "Hebr", # FB40..FB41 ; Hebrew + "Zzzz", # FB42..FB42 ; Unknown + "Hebr", # FB43..FB44 ; Hebrew + "Zzzz", # FB45..FB45 ; Unknown + "Hebr", # FB46..FB4F ; Hebrew + "Arab", # FB50..FD3D ; Arabic + "Zyyy", # FD3E..FD3F ; Common + "Arab", # FD40..FDCF ; Arabic + "Zzzz", # FDD0..FDEF ; Unknown + "Arab", # FDF0..FDFF ; Arabic + "Zinh", # FE00..FE0F ; Inherited + "Zyyy", # FE10..FE19 ; Common + "Zzzz", # FE1A..FE1F ; Unknown + "Zinh", # FE20..FE2D ; Inherited + "Cyrl", # FE2E..FE2F ; Cyrillic + "Zyyy", # FE30..FE52 ; Common + "Zzzz", # FE53..FE53 ; Unknown + "Zyyy", # FE54..FE66 ; Common + "Zzzz", # FE67..FE67 ; Unknown + "Zyyy", # FE68..FE6B ; Common + "Zzzz", # FE6C..FE6F ; Unknown + "Arab", # FE70..FE74 ; Arabic + "Zzzz", # FE75..FE75 ; Unknown + "Arab", # FE76..FEFC ; Arabic + "Zzzz", # FEFD..FEFE ; Unknown + "Zyyy", # FEFF..FEFF ; Common + "Zzzz", # FF00..FF00 ; Unknown + "Zyyy", # FF01..FF20 ; Common + "Latn", # FF21..FF3A ; Latin + "Zyyy", # FF3B..FF40 ; Common + "Latn", # FF41..FF5A ; Latin + "Zyyy", # FF5B..FF65 ; Common + "Kana", # FF66..FF6F ; Katakana + "Zyyy", # FF70..FF70 ; Common + "Kana", # FF71..FF9D ; Katakana + "Zyyy", # FF9E..FF9F ; Common + "Hang", # FFA0..FFBE ; Hangul + "Zzzz", # FFBF..FFC1 ; Unknown + "Hang", # FFC2..FFC7 ; Hangul + "Zzzz", # FFC8..FFC9 ; Unknown + "Hang", # FFCA..FFCF ; Hangul + "Zzzz", # FFD0..FFD1 ; Unknown + "Hang", # FFD2..FFD7 ; Hangul + "Zzzz", # FFD8..FFD9 ; Unknown + "Hang", # FFDA..FFDC ; Hangul + "Zzzz", # FFDD..FFDF ; Unknown + "Zyyy", # FFE0..FFE6 ; Common + "Zzzz", # FFE7..FFE7 ; Unknown + "Zyyy", # FFE8..FFEE ; Common + "Zzzz", # FFEF..FFF8 ; Unknown + "Zyyy", # FFF9..FFFD ; Common + "Zzzz", # FFFE..FFFF ; Unknown + "Linb", # 10000..1000B ; Linear_B + "Zzzz", # 1000C..1000C ; Unknown + "Linb", # 1000D..10026 ; Linear_B + "Zzzz", # 10027..10027 ; Unknown + "Linb", # 10028..1003A ; Linear_B + "Zzzz", # 1003B..1003B ; Unknown + "Linb", # 1003C..1003D ; Linear_B + "Zzzz", # 1003E..1003E ; Unknown + "Linb", # 1003F..1004D ; Linear_B + "Zzzz", # 1004E..1004F ; Unknown + "Linb", # 10050..1005D ; Linear_B + "Zzzz", # 1005E..1007F ; Unknown + "Linb", # 10080..100FA ; Linear_B + "Zzzz", # 100FB..100FF ; Unknown + "Zyyy", # 10100..10102 ; Common + "Zzzz", # 10103..10106 ; Unknown + "Zyyy", # 10107..10133 ; Common + "Zzzz", # 10134..10136 ; Unknown + "Zyyy", # 10137..1013F ; Common + "Grek", # 10140..1018E ; Greek + "Zzzz", # 1018F..1018F ; Unknown + "Zyyy", # 10190..1019C ; Common + "Zzzz", # 1019D..1019F ; Unknown + "Grek", # 101A0..101A0 ; Greek + "Zzzz", # 101A1..101CF ; Unknown + "Zyyy", # 101D0..101FC ; Common + "Zinh", # 101FD..101FD ; Inherited + "Zzzz", # 101FE..1027F ; Unknown + "Lyci", # 10280..1029C ; Lycian + "Zzzz", # 1029D..1029F ; Unknown + "Cari", # 102A0..102D0 ; Carian + "Zzzz", # 102D1..102DF ; Unknown + "Zinh", # 102E0..102E0 ; Inherited + "Zyyy", # 102E1..102FB ; Common + "Zzzz", # 102FC..102FF ; Unknown + "Ital", # 10300..10323 ; Old_Italic + "Zzzz", # 10324..1032C ; Unknown + "Ital", # 1032D..1032F ; Old_Italic + "Goth", # 10330..1034A ; Gothic + "Zzzz", # 1034B..1034F ; Unknown + "Perm", # 10350..1037A ; Old_Permic + "Zzzz", # 1037B..1037F ; Unknown + "Ugar", # 10380..1039D ; Ugaritic + "Zzzz", # 1039E..1039E ; Unknown + "Ugar", # 1039F..1039F ; Ugaritic + "Xpeo", # 103A0..103C3 ; Old_Persian + "Zzzz", # 103C4..103C7 ; Unknown + "Xpeo", # 103C8..103D5 ; Old_Persian + "Zzzz", # 103D6..103FF ; Unknown + "Dsrt", # 10400..1044F ; Deseret + "Shaw", # 10450..1047F ; Shavian + "Osma", # 10480..1049D ; Osmanya + "Zzzz", # 1049E..1049F ; Unknown + "Osma", # 104A0..104A9 ; Osmanya + "Zzzz", # 104AA..104AF ; Unknown + "Osge", # 104B0..104D3 ; Osage + "Zzzz", # 104D4..104D7 ; Unknown + "Osge", # 104D8..104FB ; Osage + "Zzzz", # 104FC..104FF ; Unknown + "Elba", # 10500..10527 ; Elbasan + "Zzzz", # 10528..1052F ; Unknown + "Aghb", # 10530..10563 ; Caucasian_Albanian + "Zzzz", # 10564..1056E ; Unknown + "Aghb", # 1056F..1056F ; Caucasian_Albanian + "Vith", # 10570..1057A ; Vithkuqi + "Zzzz", # 1057B..1057B ; Unknown + "Vith", # 1057C..1058A ; Vithkuqi + "Zzzz", # 1058B..1058B ; Unknown + "Vith", # 1058C..10592 ; Vithkuqi + "Zzzz", # 10593..10593 ; Unknown + "Vith", # 10594..10595 ; Vithkuqi + "Zzzz", # 10596..10596 ; Unknown + "Vith", # 10597..105A1 ; Vithkuqi + "Zzzz", # 105A2..105A2 ; Unknown + "Vith", # 105A3..105B1 ; Vithkuqi + "Zzzz", # 105B2..105B2 ; Unknown + "Vith", # 105B3..105B9 ; Vithkuqi + "Zzzz", # 105BA..105BA ; Unknown + "Vith", # 105BB..105BC ; Vithkuqi + "Zzzz", # 105BD..105BF ; Unknown + "Todr", # 105C0..105F3 ; Todhri + "Zzzz", # 105F4..105FF ; Unknown + "Lina", # 10600..10736 ; Linear_A + "Zzzz", # 10737..1073F ; Unknown + "Lina", # 10740..10755 ; Linear_A + "Zzzz", # 10756..1075F ; Unknown + "Lina", # 10760..10767 ; Linear_A + "Zzzz", # 10768..1077F ; Unknown + "Latn", # 10780..10785 ; Latin + "Zzzz", # 10786..10786 ; Unknown + "Latn", # 10787..107B0 ; Latin + "Zzzz", # 107B1..107B1 ; Unknown + "Latn", # 107B2..107BA ; Latin + "Zzzz", # 107BB..107FF ; Unknown + "Cprt", # 10800..10805 ; Cypriot + "Zzzz", # 10806..10807 ; Unknown + "Cprt", # 10808..10808 ; Cypriot + "Zzzz", # 10809..10809 ; Unknown + "Cprt", # 1080A..10835 ; Cypriot + "Zzzz", # 10836..10836 ; Unknown + "Cprt", # 10837..10838 ; Cypriot + "Zzzz", # 10839..1083B ; Unknown + "Cprt", # 1083C..1083C ; Cypriot + "Zzzz", # 1083D..1083E ; Unknown + "Cprt", # 1083F..1083F ; Cypriot + "Armi", # 10840..10855 ; Imperial_Aramaic + "Zzzz", # 10856..10856 ; Unknown + "Armi", # 10857..1085F ; Imperial_Aramaic + "Palm", # 10860..1087F ; Palmyrene + "Nbat", # 10880..1089E ; Nabataean + "Zzzz", # 1089F..108A6 ; Unknown + "Nbat", # 108A7..108AF ; Nabataean + "Zzzz", # 108B0..108DF ; Unknown + "Hatr", # 108E0..108F2 ; Hatran + "Zzzz", # 108F3..108F3 ; Unknown + "Hatr", # 108F4..108F5 ; Hatran + "Zzzz", # 108F6..108FA ; Unknown + "Hatr", # 108FB..108FF ; Hatran + "Phnx", # 10900..1091B ; Phoenician + "Zzzz", # 1091C..1091E ; Unknown + "Phnx", # 1091F..1091F ; Phoenician + "Lydi", # 10920..10939 ; Lydian + "Zzzz", # 1093A..1093E ; Unknown + "Lydi", # 1093F..1093F ; Lydian + "Sidt", # 10940..10959 ; Sidetic + "Zzzz", # 1095A..1097F ; Unknown + "Mero", # 10980..1099F ; Meroitic_Hieroglyphs + "Merc", # 109A0..109B7 ; Meroitic_Cursive + "Zzzz", # 109B8..109BB ; Unknown + "Merc", # 109BC..109CF ; Meroitic_Cursive + "Zzzz", # 109D0..109D1 ; Unknown + "Merc", # 109D2..109FF ; Meroitic_Cursive + "Khar", # 10A00..10A03 ; Kharoshthi + "Zzzz", # 10A04..10A04 ; Unknown + "Khar", # 10A05..10A06 ; Kharoshthi + "Zzzz", # 10A07..10A0B ; Unknown + "Khar", # 10A0C..10A13 ; Kharoshthi + "Zzzz", # 10A14..10A14 ; Unknown + "Khar", # 10A15..10A17 ; Kharoshthi + "Zzzz", # 10A18..10A18 ; Unknown + "Khar", # 10A19..10A35 ; Kharoshthi + "Zzzz", # 10A36..10A37 ; Unknown + "Khar", # 10A38..10A3A ; Kharoshthi + "Zzzz", # 10A3B..10A3E ; Unknown + "Khar", # 10A3F..10A48 ; Kharoshthi + "Zzzz", # 10A49..10A4F ; Unknown + "Khar", # 10A50..10A58 ; Kharoshthi + "Zzzz", # 10A59..10A5F ; Unknown + "Sarb", # 10A60..10A7F ; Old_South_Arabian + "Narb", # 10A80..10A9F ; Old_North_Arabian + "Zzzz", # 10AA0..10ABF ; Unknown + "Mani", # 10AC0..10AE6 ; Manichaean + "Zzzz", # 10AE7..10AEA ; Unknown + "Mani", # 10AEB..10AF6 ; Manichaean + "Zzzz", # 10AF7..10AFF ; Unknown + "Avst", # 10B00..10B35 ; Avestan + "Zzzz", # 10B36..10B38 ; Unknown + "Avst", # 10B39..10B3F ; Avestan + "Prti", # 10B40..10B55 ; Inscriptional_Parthian + "Zzzz", # 10B56..10B57 ; Unknown + "Prti", # 10B58..10B5F ; Inscriptional_Parthian + "Phli", # 10B60..10B72 ; Inscriptional_Pahlavi + "Zzzz", # 10B73..10B77 ; Unknown + "Phli", # 10B78..10B7F ; Inscriptional_Pahlavi + "Phlp", # 10B80..10B91 ; Psalter_Pahlavi + "Zzzz", # 10B92..10B98 ; Unknown + "Phlp", # 10B99..10B9C ; Psalter_Pahlavi + "Zzzz", # 10B9D..10BA8 ; Unknown + "Phlp", # 10BA9..10BAF ; Psalter_Pahlavi + "Zzzz", # 10BB0..10BFF ; Unknown + "Orkh", # 10C00..10C48 ; Old_Turkic + "Zzzz", # 10C49..10C7F ; Unknown + "Hung", # 10C80..10CB2 ; Old_Hungarian + "Zzzz", # 10CB3..10CBF ; Unknown + "Hung", # 10CC0..10CF2 ; Old_Hungarian + "Zzzz", # 10CF3..10CF9 ; Unknown + "Hung", # 10CFA..10CFF ; Old_Hungarian + "Rohg", # 10D00..10D27 ; Hanifi_Rohingya + "Zzzz", # 10D28..10D2F ; Unknown + "Rohg", # 10D30..10D39 ; Hanifi_Rohingya + "Zzzz", # 10D3A..10D3F ; Unknown + "Gara", # 10D40..10D65 ; Garay + "Zzzz", # 10D66..10D68 ; Unknown + "Gara", # 10D69..10D85 ; Garay + "Zzzz", # 10D86..10D8D ; Unknown + "Gara", # 10D8E..10D8F ; Garay + "Zzzz", # 10D90..10E5F ; Unknown + "Arab", # 10E60..10E7E ; Arabic + "Zzzz", # 10E7F..10E7F ; Unknown + "Yezi", # 10E80..10EA9 ; Yezidi + "Zzzz", # 10EAA..10EAA ; Unknown + "Yezi", # 10EAB..10EAD ; Yezidi + "Zzzz", # 10EAE..10EAF ; Unknown + "Yezi", # 10EB0..10EB1 ; Yezidi + "Zzzz", # 10EB2..10EC1 ; Unknown + "Arab", # 10EC2..10EC7 ; Arabic + "Zzzz", # 10EC8..10ECF ; Unknown + "Arab", # 10ED0..10ED8 ; Arabic + "Zzzz", # 10ED9..10EF9 ; Unknown + "Arab", # 10EFA..10EFF ; Arabic + "Sogo", # 10F00..10F27 ; Old_Sogdian + "Zzzz", # 10F28..10F2F ; Unknown + "Sogd", # 10F30..10F59 ; Sogdian + "Zzzz", # 10F5A..10F6F ; Unknown + "Ougr", # 10F70..10F89 ; Old_Uyghur + "Zzzz", # 10F8A..10FAF ; Unknown + "Chrs", # 10FB0..10FCB ; Chorasmian + "Zzzz", # 10FCC..10FDF ; Unknown + "Elym", # 10FE0..10FF6 ; Elymaic + "Zzzz", # 10FF7..10FFF ; Unknown + "Brah", # 11000..1104D ; Brahmi + "Zzzz", # 1104E..11051 ; Unknown + "Brah", # 11052..11075 ; Brahmi + "Zzzz", # 11076..1107E ; Unknown + "Brah", # 1107F..1107F ; Brahmi + "Kthi", # 11080..110C2 ; Kaithi + "Zzzz", # 110C3..110CC ; Unknown + "Kthi", # 110CD..110CD ; Kaithi + "Zzzz", # 110CE..110CF ; Unknown + "Sora", # 110D0..110E8 ; Sora_Sompeng + "Zzzz", # 110E9..110EF ; Unknown + "Sora", # 110F0..110F9 ; Sora_Sompeng + "Zzzz", # 110FA..110FF ; Unknown + "Cakm", # 11100..11134 ; Chakma + "Zzzz", # 11135..11135 ; Unknown + "Cakm", # 11136..11147 ; Chakma + "Zzzz", # 11148..1114F ; Unknown + "Mahj", # 11150..11176 ; Mahajani + "Zzzz", # 11177..1117F ; Unknown + "Shrd", # 11180..111DF ; Sharada + "Zzzz", # 111E0..111E0 ; Unknown + "Sinh", # 111E1..111F4 ; Sinhala + "Zzzz", # 111F5..111FF ; Unknown + "Khoj", # 11200..11211 ; Khojki + "Zzzz", # 11212..11212 ; Unknown + "Khoj", # 11213..11241 ; Khojki + "Zzzz", # 11242..1127F ; Unknown + "Mult", # 11280..11286 ; Multani + "Zzzz", # 11287..11287 ; Unknown + "Mult", # 11288..11288 ; Multani + "Zzzz", # 11289..11289 ; Unknown + "Mult", # 1128A..1128D ; Multani + "Zzzz", # 1128E..1128E ; Unknown + "Mult", # 1128F..1129D ; Multani + "Zzzz", # 1129E..1129E ; Unknown + "Mult", # 1129F..112A9 ; Multani + "Zzzz", # 112AA..112AF ; Unknown + "Sind", # 112B0..112EA ; Khudawadi + "Zzzz", # 112EB..112EF ; Unknown + "Sind", # 112F0..112F9 ; Khudawadi + "Zzzz", # 112FA..112FF ; Unknown + "Gran", # 11300..11303 ; Grantha + "Zzzz", # 11304..11304 ; Unknown + "Gran", # 11305..1130C ; Grantha + "Zzzz", # 1130D..1130E ; Unknown + "Gran", # 1130F..11310 ; Grantha + "Zzzz", # 11311..11312 ; Unknown + "Gran", # 11313..11328 ; Grantha + "Zzzz", # 11329..11329 ; Unknown + "Gran", # 1132A..11330 ; Grantha + "Zzzz", # 11331..11331 ; Unknown + "Gran", # 11332..11333 ; Grantha + "Zzzz", # 11334..11334 ; Unknown + "Gran", # 11335..11339 ; Grantha + "Zzzz", # 1133A..1133A ; Unknown + "Zinh", # 1133B..1133B ; Inherited + "Gran", # 1133C..11344 ; Grantha + "Zzzz", # 11345..11346 ; Unknown + "Gran", # 11347..11348 ; Grantha + "Zzzz", # 11349..1134A ; Unknown + "Gran", # 1134B..1134D ; Grantha + "Zzzz", # 1134E..1134F ; Unknown + "Gran", # 11350..11350 ; Grantha + "Zzzz", # 11351..11356 ; Unknown + "Gran", # 11357..11357 ; Grantha + "Zzzz", # 11358..1135C ; Unknown + "Gran", # 1135D..11363 ; Grantha + "Zzzz", # 11364..11365 ; Unknown + "Gran", # 11366..1136C ; Grantha + "Zzzz", # 1136D..1136F ; Unknown + "Gran", # 11370..11374 ; Grantha + "Zzzz", # 11375..1137F ; Unknown + "Tutg", # 11380..11389 ; Tulu_Tigalari + "Zzzz", # 1138A..1138A ; Unknown + "Tutg", # 1138B..1138B ; Tulu_Tigalari + "Zzzz", # 1138C..1138D ; Unknown + "Tutg", # 1138E..1138E ; Tulu_Tigalari + "Zzzz", # 1138F..1138F ; Unknown + "Tutg", # 11390..113B5 ; Tulu_Tigalari + "Zzzz", # 113B6..113B6 ; Unknown + "Tutg", # 113B7..113C0 ; Tulu_Tigalari + "Zzzz", # 113C1..113C1 ; Unknown + "Tutg", # 113C2..113C2 ; Tulu_Tigalari + "Zzzz", # 113C3..113C4 ; Unknown + "Tutg", # 113C5..113C5 ; Tulu_Tigalari + "Zzzz", # 113C6..113C6 ; Unknown + "Tutg", # 113C7..113CA ; Tulu_Tigalari + "Zzzz", # 113CB..113CB ; Unknown + "Tutg", # 113CC..113D5 ; Tulu_Tigalari + "Zzzz", # 113D6..113D6 ; Unknown + "Tutg", # 113D7..113D8 ; Tulu_Tigalari + "Zzzz", # 113D9..113E0 ; Unknown + "Tutg", # 113E1..113E2 ; Tulu_Tigalari + "Zzzz", # 113E3..113FF ; Unknown + "Newa", # 11400..1145B ; Newa + "Zzzz", # 1145C..1145C ; Unknown + "Newa", # 1145D..11461 ; Newa + "Zzzz", # 11462..1147F ; Unknown + "Tirh", # 11480..114C7 ; Tirhuta + "Zzzz", # 114C8..114CF ; Unknown + "Tirh", # 114D0..114D9 ; Tirhuta + "Zzzz", # 114DA..1157F ; Unknown + "Sidd", # 11580..115B5 ; Siddham + "Zzzz", # 115B6..115B7 ; Unknown + "Sidd", # 115B8..115DD ; Siddham + "Zzzz", # 115DE..115FF ; Unknown + "Modi", # 11600..11644 ; Modi + "Zzzz", # 11645..1164F ; Unknown + "Modi", # 11650..11659 ; Modi + "Zzzz", # 1165A..1165F ; Unknown + "Mong", # 11660..1166C ; Mongolian + "Zzzz", # 1166D..1167F ; Unknown + "Takr", # 11680..116B9 ; Takri + "Zzzz", # 116BA..116BF ; Unknown + "Takr", # 116C0..116C9 ; Takri + "Zzzz", # 116CA..116CF ; Unknown + "Mymr", # 116D0..116E3 ; Myanmar + "Zzzz", # 116E4..116FF ; Unknown + "Ahom", # 11700..1171A ; Ahom + "Zzzz", # 1171B..1171C ; Unknown + "Ahom", # 1171D..1172B ; Ahom + "Zzzz", # 1172C..1172F ; Unknown + "Ahom", # 11730..11746 ; Ahom + "Zzzz", # 11747..117FF ; Unknown + "Dogr", # 11800..1183B ; Dogra + "Zzzz", # 1183C..1189F ; Unknown + "Wara", # 118A0..118F2 ; Warang_Citi + "Zzzz", # 118F3..118FE ; Unknown + "Wara", # 118FF..118FF ; Warang_Citi + "Diak", # 11900..11906 ; Dives_Akuru + "Zzzz", # 11907..11908 ; Unknown + "Diak", # 11909..11909 ; Dives_Akuru + "Zzzz", # 1190A..1190B ; Unknown + "Diak", # 1190C..11913 ; Dives_Akuru + "Zzzz", # 11914..11914 ; Unknown + "Diak", # 11915..11916 ; Dives_Akuru + "Zzzz", # 11917..11917 ; Unknown + "Diak", # 11918..11935 ; Dives_Akuru + "Zzzz", # 11936..11936 ; Unknown + "Diak", # 11937..11938 ; Dives_Akuru + "Zzzz", # 11939..1193A ; Unknown + "Diak", # 1193B..11946 ; Dives_Akuru + "Zzzz", # 11947..1194F ; Unknown + "Diak", # 11950..11959 ; Dives_Akuru + "Zzzz", # 1195A..1199F ; Unknown + "Nand", # 119A0..119A7 ; Nandinagari + "Zzzz", # 119A8..119A9 ; Unknown + "Nand", # 119AA..119D7 ; Nandinagari + "Zzzz", # 119D8..119D9 ; Unknown + "Nand", # 119DA..119E4 ; Nandinagari + "Zzzz", # 119E5..119FF ; Unknown + "Zanb", # 11A00..11A47 ; Zanabazar_Square + "Zzzz", # 11A48..11A4F ; Unknown + "Soyo", # 11A50..11AA2 ; Soyombo + "Zzzz", # 11AA3..11AAF ; Unknown + "Cans", # 11AB0..11ABF ; Canadian_Aboriginal + "Pauc", # 11AC0..11AF8 ; Pau_Cin_Hau + "Zzzz", # 11AF9..11AFF ; Unknown + "Deva", # 11B00..11B09 ; Devanagari + "Zzzz", # 11B0A..11B5F ; Unknown + "Shrd", # 11B60..11B67 ; Sharada + "Zzzz", # 11B68..11BBF ; Unknown + "Sunu", # 11BC0..11BE1 ; Sunuwar + "Zzzz", # 11BE2..11BEF ; Unknown + "Sunu", # 11BF0..11BF9 ; Sunuwar + "Zzzz", # 11BFA..11BFF ; Unknown + "Bhks", # 11C00..11C08 ; Bhaiksuki + "Zzzz", # 11C09..11C09 ; Unknown + "Bhks", # 11C0A..11C36 ; Bhaiksuki + "Zzzz", # 11C37..11C37 ; Unknown + "Bhks", # 11C38..11C45 ; Bhaiksuki + "Zzzz", # 11C46..11C4F ; Unknown + "Bhks", # 11C50..11C6C ; Bhaiksuki + "Zzzz", # 11C6D..11C6F ; Unknown + "Marc", # 11C70..11C8F ; Marchen + "Zzzz", # 11C90..11C91 ; Unknown + "Marc", # 11C92..11CA7 ; Marchen + "Zzzz", # 11CA8..11CA8 ; Unknown + "Marc", # 11CA9..11CB6 ; Marchen + "Zzzz", # 11CB7..11CFF ; Unknown + "Gonm", # 11D00..11D06 ; Masaram_Gondi + "Zzzz", # 11D07..11D07 ; Unknown + "Gonm", # 11D08..11D09 ; Masaram_Gondi + "Zzzz", # 11D0A..11D0A ; Unknown + "Gonm", # 11D0B..11D36 ; Masaram_Gondi + "Zzzz", # 11D37..11D39 ; Unknown + "Gonm", # 11D3A..11D3A ; Masaram_Gondi + "Zzzz", # 11D3B..11D3B ; Unknown + "Gonm", # 11D3C..11D3D ; Masaram_Gondi + "Zzzz", # 11D3E..11D3E ; Unknown + "Gonm", # 11D3F..11D47 ; Masaram_Gondi + "Zzzz", # 11D48..11D4F ; Unknown + "Gonm", # 11D50..11D59 ; Masaram_Gondi + "Zzzz", # 11D5A..11D5F ; Unknown + "Gong", # 11D60..11D65 ; Gunjala_Gondi + "Zzzz", # 11D66..11D66 ; Unknown + "Gong", # 11D67..11D68 ; Gunjala_Gondi + "Zzzz", # 11D69..11D69 ; Unknown + "Gong", # 11D6A..11D8E ; Gunjala_Gondi + "Zzzz", # 11D8F..11D8F ; Unknown + "Gong", # 11D90..11D91 ; Gunjala_Gondi + "Zzzz", # 11D92..11D92 ; Unknown + "Gong", # 11D93..11D98 ; Gunjala_Gondi + "Zzzz", # 11D99..11D9F ; Unknown + "Gong", # 11DA0..11DA9 ; Gunjala_Gondi + "Zzzz", # 11DAA..11DAF ; Unknown + "Tols", # 11DB0..11DDB ; Tolong_Siki + "Zzzz", # 11DDC..11DDF ; Unknown + "Tols", # 11DE0..11DE9 ; Tolong_Siki + "Zzzz", # 11DEA..11EDF ; Unknown + "Maka", # 11EE0..11EF8 ; Makasar + "Zzzz", # 11EF9..11EFF ; Unknown + "Kawi", # 11F00..11F10 ; Kawi + "Zzzz", # 11F11..11F11 ; Unknown + "Kawi", # 11F12..11F3A ; Kawi + "Zzzz", # 11F3B..11F3D ; Unknown + "Kawi", # 11F3E..11F5A ; Kawi + "Zzzz", # 11F5B..11FAF ; Unknown + "Lisu", # 11FB0..11FB0 ; Lisu + "Zzzz", # 11FB1..11FBF ; Unknown + "Taml", # 11FC0..11FF1 ; Tamil + "Zzzz", # 11FF2..11FFE ; Unknown + "Taml", # 11FFF..11FFF ; Tamil + "Xsux", # 12000..12399 ; Cuneiform + "Zzzz", # 1239A..123FF ; Unknown + "Xsux", # 12400..1246E ; Cuneiform + "Zzzz", # 1246F..1246F ; Unknown + "Xsux", # 12470..12474 ; Cuneiform + "Zzzz", # 12475..1247F ; Unknown + "Xsux", # 12480..12543 ; Cuneiform + "Zzzz", # 12544..12F8F ; Unknown + "Cpmn", # 12F90..12FF2 ; Cypro_Minoan + "Zzzz", # 12FF3..12FFF ; Unknown + "Egyp", # 13000..13455 ; Egyptian_Hieroglyphs + "Zzzz", # 13456..1345F ; Unknown + "Egyp", # 13460..143FA ; Egyptian_Hieroglyphs + "Zzzz", # 143FB..143FF ; Unknown + "Hluw", # 14400..14646 ; Anatolian_Hieroglyphs + "Zzzz", # 14647..160FF ; Unknown + "Gukh", # 16100..16139 ; Gurung_Khema + "Zzzz", # 1613A..167FF ; Unknown + "Bamu", # 16800..16A38 ; Bamum + "Zzzz", # 16A39..16A3F ; Unknown + "Mroo", # 16A40..16A5E ; Mro + "Zzzz", # 16A5F..16A5F ; Unknown + "Mroo", # 16A60..16A69 ; Mro + "Zzzz", # 16A6A..16A6D ; Unknown + "Mroo", # 16A6E..16A6F ; Mro + "Tnsa", # 16A70..16ABE ; Tangsa + "Zzzz", # 16ABF..16ABF ; Unknown + "Tnsa", # 16AC0..16AC9 ; Tangsa + "Zzzz", # 16ACA..16ACF ; Unknown + "Bass", # 16AD0..16AED ; Bassa_Vah + "Zzzz", # 16AEE..16AEF ; Unknown + "Bass", # 16AF0..16AF5 ; Bassa_Vah + "Zzzz", # 16AF6..16AFF ; Unknown + "Hmng", # 16B00..16B45 ; Pahawh_Hmong + "Zzzz", # 16B46..16B4F ; Unknown + "Hmng", # 16B50..16B59 ; Pahawh_Hmong + "Zzzz", # 16B5A..16B5A ; Unknown + "Hmng", # 16B5B..16B61 ; Pahawh_Hmong + "Zzzz", # 16B62..16B62 ; Unknown + "Hmng", # 16B63..16B77 ; Pahawh_Hmong + "Zzzz", # 16B78..16B7C ; Unknown + "Hmng", # 16B7D..16B8F ; Pahawh_Hmong + "Zzzz", # 16B90..16D3F ; Unknown + "Krai", # 16D40..16D79 ; Kirat_Rai + "Zzzz", # 16D7A..16E3F ; Unknown + "Medf", # 16E40..16E9A ; Medefaidrin + "Zzzz", # 16E9B..16E9F ; Unknown + "Berf", # 16EA0..16EB8 ; Beria_Erfe + "Zzzz", # 16EB9..16EBA ; Unknown + "Berf", # 16EBB..16ED3 ; Beria_Erfe + "Zzzz", # 16ED4..16EFF ; Unknown + "Plrd", # 16F00..16F4A ; Miao + "Zzzz", # 16F4B..16F4E ; Unknown + "Plrd", # 16F4F..16F87 ; Miao + "Zzzz", # 16F88..16F8E ; Unknown + "Plrd", # 16F8F..16F9F ; Miao + "Zzzz", # 16FA0..16FDF ; Unknown + "Tang", # 16FE0..16FE0 ; Tangut + "Nshu", # 16FE1..16FE1 ; Nushu + "Hani", # 16FE2..16FE3 ; Han + "Kits", # 16FE4..16FE4 ; Khitan_Small_Script + "Zzzz", # 16FE5..16FEF ; Unknown + "Hani", # 16FF0..16FF6 ; Han + "Zzzz", # 16FF7..16FFF ; Unknown + "Tang", # 17000..18AFF ; Tangut + "Kits", # 18B00..18CD5 ; Khitan_Small_Script + "Zzzz", # 18CD6..18CFE ; Unknown + "Kits", # 18CFF..18CFF ; Khitan_Small_Script + "Tang", # 18D00..18D1E ; Tangut + "Zzzz", # 18D1F..18D7F ; Unknown + "Tang", # 18D80..18DF2 ; Tangut + "Zzzz", # 18DF3..1AFEF ; Unknown + "Kana", # 1AFF0..1AFF3 ; Katakana + "Zzzz", # 1AFF4..1AFF4 ; Unknown + "Kana", # 1AFF5..1AFFB ; Katakana + "Zzzz", # 1AFFC..1AFFC ; Unknown + "Kana", # 1AFFD..1AFFE ; Katakana + "Zzzz", # 1AFFF..1AFFF ; Unknown + "Kana", # 1B000..1B000 ; Katakana + "Hira", # 1B001..1B11F ; Hiragana + "Kana", # 1B120..1B122 ; Katakana + "Zzzz", # 1B123..1B131 ; Unknown + "Hira", # 1B132..1B132 ; Hiragana + "Zzzz", # 1B133..1B14F ; Unknown + "Hira", # 1B150..1B152 ; Hiragana + "Zzzz", # 1B153..1B154 ; Unknown + "Kana", # 1B155..1B155 ; Katakana + "Zzzz", # 1B156..1B163 ; Unknown + "Kana", # 1B164..1B167 ; Katakana + "Zzzz", # 1B168..1B16F ; Unknown + "Nshu", # 1B170..1B2FB ; Nushu + "Zzzz", # 1B2FC..1BBFF ; Unknown + "Dupl", # 1BC00..1BC6A ; Duployan + "Zzzz", # 1BC6B..1BC6F ; Unknown + "Dupl", # 1BC70..1BC7C ; Duployan + "Zzzz", # 1BC7D..1BC7F ; Unknown + "Dupl", # 1BC80..1BC88 ; Duployan + "Zzzz", # 1BC89..1BC8F ; Unknown + "Dupl", # 1BC90..1BC99 ; Duployan + "Zzzz", # 1BC9A..1BC9B ; Unknown + "Dupl", # 1BC9C..1BC9F ; Duployan + "Zyyy", # 1BCA0..1BCA3 ; Common + "Zzzz", # 1BCA4..1CBFF ; Unknown + "Zyyy", # 1CC00..1CCFC ; Common + "Zzzz", # 1CCFD..1CCFF ; Unknown + "Zyyy", # 1CD00..1CEB3 ; Common + "Zzzz", # 1CEB4..1CEB9 ; Unknown + "Zyyy", # 1CEBA..1CED0 ; Common + "Zzzz", # 1CED1..1CEDF ; Unknown + "Zyyy", # 1CEE0..1CEF0 ; Common + "Zzzz", # 1CEF1..1CEFF ; Unknown + "Zinh", # 1CF00..1CF2D ; Inherited + "Zzzz", # 1CF2E..1CF2F ; Unknown + "Zinh", # 1CF30..1CF46 ; Inherited + "Zzzz", # 1CF47..1CF4F ; Unknown + "Zyyy", # 1CF50..1CFC3 ; Common + "Zzzz", # 1CFC4..1CFFF ; Unknown + "Zyyy", # 1D000..1D0F5 ; Common + "Zzzz", # 1D0F6..1D0FF ; Unknown + "Zyyy", # 1D100..1D126 ; Common + "Zzzz", # 1D127..1D128 ; Unknown + "Zyyy", # 1D129..1D166 ; Common + "Zinh", # 1D167..1D169 ; Inherited + "Zyyy", # 1D16A..1D17A ; Common + "Zinh", # 1D17B..1D182 ; Inherited + "Zyyy", # 1D183..1D184 ; Common + "Zinh", # 1D185..1D18B ; Inherited + "Zyyy", # 1D18C..1D1A9 ; Common + "Zinh", # 1D1AA..1D1AD ; Inherited + "Zyyy", # 1D1AE..1D1EA ; Common + "Zzzz", # 1D1EB..1D1FF ; Unknown + "Grek", # 1D200..1D245 ; Greek + "Zzzz", # 1D246..1D2BF ; Unknown + "Zyyy", # 1D2C0..1D2D3 ; Common + "Zzzz", # 1D2D4..1D2DF ; Unknown + "Zyyy", # 1D2E0..1D2F3 ; Common + "Zzzz", # 1D2F4..1D2FF ; Unknown + "Zyyy", # 1D300..1D356 ; Common + "Zzzz", # 1D357..1D35F ; Unknown + "Zyyy", # 1D360..1D378 ; Common + "Zzzz", # 1D379..1D3FF ; Unknown + "Zyyy", # 1D400..1D454 ; Common + "Zzzz", # 1D455..1D455 ; Unknown + "Zyyy", # 1D456..1D49C ; Common + "Zzzz", # 1D49D..1D49D ; Unknown + "Zyyy", # 1D49E..1D49F ; Common + "Zzzz", # 1D4A0..1D4A1 ; Unknown + "Zyyy", # 1D4A2..1D4A2 ; Common + "Zzzz", # 1D4A3..1D4A4 ; Unknown + "Zyyy", # 1D4A5..1D4A6 ; Common + "Zzzz", # 1D4A7..1D4A8 ; Unknown + "Zyyy", # 1D4A9..1D4AC ; Common + "Zzzz", # 1D4AD..1D4AD ; Unknown + "Zyyy", # 1D4AE..1D4B9 ; Common + "Zzzz", # 1D4BA..1D4BA ; Unknown + "Zyyy", # 1D4BB..1D4BB ; Common + "Zzzz", # 1D4BC..1D4BC ; Unknown + "Zyyy", # 1D4BD..1D4C3 ; Common + "Zzzz", # 1D4C4..1D4C4 ; Unknown + "Zyyy", # 1D4C5..1D505 ; Common + "Zzzz", # 1D506..1D506 ; Unknown + "Zyyy", # 1D507..1D50A ; Common + "Zzzz", # 1D50B..1D50C ; Unknown + "Zyyy", # 1D50D..1D514 ; Common + "Zzzz", # 1D515..1D515 ; Unknown + "Zyyy", # 1D516..1D51C ; Common + "Zzzz", # 1D51D..1D51D ; Unknown + "Zyyy", # 1D51E..1D539 ; Common + "Zzzz", # 1D53A..1D53A ; Unknown + "Zyyy", # 1D53B..1D53E ; Common + "Zzzz", # 1D53F..1D53F ; Unknown + "Zyyy", # 1D540..1D544 ; Common + "Zzzz", # 1D545..1D545 ; Unknown + "Zyyy", # 1D546..1D546 ; Common + "Zzzz", # 1D547..1D549 ; Unknown + "Zyyy", # 1D54A..1D550 ; Common + "Zzzz", # 1D551..1D551 ; Unknown + "Zyyy", # 1D552..1D6A5 ; Common + "Zzzz", # 1D6A6..1D6A7 ; Unknown + "Zyyy", # 1D6A8..1D7CB ; Common + "Zzzz", # 1D7CC..1D7CD ; Unknown + "Zyyy", # 1D7CE..1D7FF ; Common + "Sgnw", # 1D800..1DA8B ; SignWriting + "Zzzz", # 1DA8C..1DA9A ; Unknown + "Sgnw", # 1DA9B..1DA9F ; SignWriting + "Zzzz", # 1DAA0..1DAA0 ; Unknown + "Sgnw", # 1DAA1..1DAAF ; SignWriting + "Zzzz", # 1DAB0..1DEFF ; Unknown + "Latn", # 1DF00..1DF1E ; Latin + "Zzzz", # 1DF1F..1DF24 ; Unknown + "Latn", # 1DF25..1DF2A ; Latin + "Zzzz", # 1DF2B..1DFFF ; Unknown + "Glag", # 1E000..1E006 ; Glagolitic + "Zzzz", # 1E007..1E007 ; Unknown + "Glag", # 1E008..1E018 ; Glagolitic + "Zzzz", # 1E019..1E01A ; Unknown + "Glag", # 1E01B..1E021 ; Glagolitic + "Zzzz", # 1E022..1E022 ; Unknown + "Glag", # 1E023..1E024 ; Glagolitic + "Zzzz", # 1E025..1E025 ; Unknown + "Glag", # 1E026..1E02A ; Glagolitic + "Zzzz", # 1E02B..1E02F ; Unknown + "Cyrl", # 1E030..1E06D ; Cyrillic + "Zzzz", # 1E06E..1E08E ; Unknown + "Cyrl", # 1E08F..1E08F ; Cyrillic + "Zzzz", # 1E090..1E0FF ; Unknown + "Hmnp", # 1E100..1E12C ; Nyiakeng_Puachue_Hmong + "Zzzz", # 1E12D..1E12F ; Unknown + "Hmnp", # 1E130..1E13D ; Nyiakeng_Puachue_Hmong + "Zzzz", # 1E13E..1E13F ; Unknown + "Hmnp", # 1E140..1E149 ; Nyiakeng_Puachue_Hmong + "Zzzz", # 1E14A..1E14D ; Unknown + "Hmnp", # 1E14E..1E14F ; Nyiakeng_Puachue_Hmong + "Zzzz", # 1E150..1E28F ; Unknown + "Toto", # 1E290..1E2AE ; Toto + "Zzzz", # 1E2AF..1E2BF ; Unknown + "Wcho", # 1E2C0..1E2F9 ; Wancho + "Zzzz", # 1E2FA..1E2FE ; Unknown + "Wcho", # 1E2FF..1E2FF ; Wancho + "Zzzz", # 1E300..1E4CF ; Unknown + "Nagm", # 1E4D0..1E4F9 ; Nag_Mundari + "Zzzz", # 1E4FA..1E5CF ; Unknown + "Onao", # 1E5D0..1E5FA ; Ol_Onal + "Zzzz", # 1E5FB..1E5FE ; Unknown + "Onao", # 1E5FF..1E5FF ; Ol_Onal + "Zzzz", # 1E600..1E6BF ; Unknown + "Tayo", # 1E6C0..1E6DE ; Tai_Yo + "Zzzz", # 1E6DF..1E6DF ; Unknown + "Tayo", # 1E6E0..1E6F5 ; Tai_Yo + "Zzzz", # 1E6F6..1E6FD ; Unknown + "Tayo", # 1E6FE..1E6FF ; Tai_Yo + "Zzzz", # 1E700..1E7DF ; Unknown + "Ethi", # 1E7E0..1E7E6 ; Ethiopic + "Zzzz", # 1E7E7..1E7E7 ; Unknown + "Ethi", # 1E7E8..1E7EB ; Ethiopic + "Zzzz", # 1E7EC..1E7EC ; Unknown + "Ethi", # 1E7ED..1E7EE ; Ethiopic + "Zzzz", # 1E7EF..1E7EF ; Unknown + "Ethi", # 1E7F0..1E7FE ; Ethiopic + "Zzzz", # 1E7FF..1E7FF ; Unknown + "Mend", # 1E800..1E8C4 ; Mende_Kikakui + "Zzzz", # 1E8C5..1E8C6 ; Unknown + "Mend", # 1E8C7..1E8D6 ; Mende_Kikakui + "Zzzz", # 1E8D7..1E8FF ; Unknown + "Adlm", # 1E900..1E94B ; Adlam + "Zzzz", # 1E94C..1E94F ; Unknown + "Adlm", # 1E950..1E959 ; Adlam + "Zzzz", # 1E95A..1E95D ; Unknown + "Adlm", # 1E95E..1E95F ; Adlam + "Zzzz", # 1E960..1EC70 ; Unknown + "Zyyy", # 1EC71..1ECB4 ; Common + "Zzzz", # 1ECB5..1ED00 ; Unknown + "Zyyy", # 1ED01..1ED3D ; Common + "Zzzz", # 1ED3E..1EDFF ; Unknown + "Arab", # 1EE00..1EE03 ; Arabic + "Zzzz", # 1EE04..1EE04 ; Unknown + "Arab", # 1EE05..1EE1F ; Arabic + "Zzzz", # 1EE20..1EE20 ; Unknown + "Arab", # 1EE21..1EE22 ; Arabic + "Zzzz", # 1EE23..1EE23 ; Unknown + "Arab", # 1EE24..1EE24 ; Arabic + "Zzzz", # 1EE25..1EE26 ; Unknown + "Arab", # 1EE27..1EE27 ; Arabic + "Zzzz", # 1EE28..1EE28 ; Unknown + "Arab", # 1EE29..1EE32 ; Arabic + "Zzzz", # 1EE33..1EE33 ; Unknown + "Arab", # 1EE34..1EE37 ; Arabic + "Zzzz", # 1EE38..1EE38 ; Unknown + "Arab", # 1EE39..1EE39 ; Arabic + "Zzzz", # 1EE3A..1EE3A ; Unknown + "Arab", # 1EE3B..1EE3B ; Arabic + "Zzzz", # 1EE3C..1EE41 ; Unknown + "Arab", # 1EE42..1EE42 ; Arabic + "Zzzz", # 1EE43..1EE46 ; Unknown + "Arab", # 1EE47..1EE47 ; Arabic + "Zzzz", # 1EE48..1EE48 ; Unknown + "Arab", # 1EE49..1EE49 ; Arabic + "Zzzz", # 1EE4A..1EE4A ; Unknown + "Arab", # 1EE4B..1EE4B ; Arabic + "Zzzz", # 1EE4C..1EE4C ; Unknown + "Arab", # 1EE4D..1EE4F ; Arabic + "Zzzz", # 1EE50..1EE50 ; Unknown + "Arab", # 1EE51..1EE52 ; Arabic + "Zzzz", # 1EE53..1EE53 ; Unknown + "Arab", # 1EE54..1EE54 ; Arabic + "Zzzz", # 1EE55..1EE56 ; Unknown + "Arab", # 1EE57..1EE57 ; Arabic + "Zzzz", # 1EE58..1EE58 ; Unknown + "Arab", # 1EE59..1EE59 ; Arabic + "Zzzz", # 1EE5A..1EE5A ; Unknown + "Arab", # 1EE5B..1EE5B ; Arabic + "Zzzz", # 1EE5C..1EE5C ; Unknown + "Arab", # 1EE5D..1EE5D ; Arabic + "Zzzz", # 1EE5E..1EE5E ; Unknown + "Arab", # 1EE5F..1EE5F ; Arabic + "Zzzz", # 1EE60..1EE60 ; Unknown + "Arab", # 1EE61..1EE62 ; Arabic + "Zzzz", # 1EE63..1EE63 ; Unknown + "Arab", # 1EE64..1EE64 ; Arabic + "Zzzz", # 1EE65..1EE66 ; Unknown + "Arab", # 1EE67..1EE6A ; Arabic + "Zzzz", # 1EE6B..1EE6B ; Unknown + "Arab", # 1EE6C..1EE72 ; Arabic + "Zzzz", # 1EE73..1EE73 ; Unknown + "Arab", # 1EE74..1EE77 ; Arabic + "Zzzz", # 1EE78..1EE78 ; Unknown + "Arab", # 1EE79..1EE7C ; Arabic + "Zzzz", # 1EE7D..1EE7D ; Unknown + "Arab", # 1EE7E..1EE7E ; Arabic + "Zzzz", # 1EE7F..1EE7F ; Unknown + "Arab", # 1EE80..1EE89 ; Arabic + "Zzzz", # 1EE8A..1EE8A ; Unknown + "Arab", # 1EE8B..1EE9B ; Arabic + "Zzzz", # 1EE9C..1EEA0 ; Unknown + "Arab", # 1EEA1..1EEA3 ; Arabic + "Zzzz", # 1EEA4..1EEA4 ; Unknown + "Arab", # 1EEA5..1EEA9 ; Arabic + "Zzzz", # 1EEAA..1EEAA ; Unknown + "Arab", # 1EEAB..1EEBB ; Arabic + "Zzzz", # 1EEBC..1EEEF ; Unknown + "Arab", # 1EEF0..1EEF1 ; Arabic + "Zzzz", # 1EEF2..1EFFF ; Unknown + "Zyyy", # 1F000..1F02B ; Common + "Zzzz", # 1F02C..1F02F ; Unknown + "Zyyy", # 1F030..1F093 ; Common + "Zzzz", # 1F094..1F09F ; Unknown + "Zyyy", # 1F0A0..1F0AE ; Common + "Zzzz", # 1F0AF..1F0B0 ; Unknown + "Zyyy", # 1F0B1..1F0BF ; Common + "Zzzz", # 1F0C0..1F0C0 ; Unknown + "Zyyy", # 1F0C1..1F0CF ; Common + "Zzzz", # 1F0D0..1F0D0 ; Unknown + "Zyyy", # 1F0D1..1F0F5 ; Common + "Zzzz", # 1F0F6..1F0FF ; Unknown + "Zyyy", # 1F100..1F1AD ; Common + "Zzzz", # 1F1AE..1F1E5 ; Unknown + "Zyyy", # 1F1E6..1F1FF ; Common + "Hira", # 1F200..1F200 ; Hiragana + "Zyyy", # 1F201..1F202 ; Common + "Zzzz", # 1F203..1F20F ; Unknown + "Zyyy", # 1F210..1F23B ; Common + "Zzzz", # 1F23C..1F23F ; Unknown + "Zyyy", # 1F240..1F248 ; Common + "Zzzz", # 1F249..1F24F ; Unknown + "Zyyy", # 1F250..1F251 ; Common + "Zzzz", # 1F252..1F25F ; Unknown + "Zyyy", # 1F260..1F265 ; Common + "Zzzz", # 1F266..1F2FF ; Unknown + "Zyyy", # 1F300..1F6D8 ; Common + "Zzzz", # 1F6D9..1F6DB ; Unknown + "Zyyy", # 1F6DC..1F6EC ; Common + "Zzzz", # 1F6ED..1F6EF ; Unknown + "Zyyy", # 1F6F0..1F6FC ; Common + "Zzzz", # 1F6FD..1F6FF ; Unknown + "Zyyy", # 1F700..1F7D9 ; Common + "Zzzz", # 1F7DA..1F7DF ; Unknown + "Zyyy", # 1F7E0..1F7EB ; Common + "Zzzz", # 1F7EC..1F7EF ; Unknown + "Zyyy", # 1F7F0..1F7F0 ; Common + "Zzzz", # 1F7F1..1F7FF ; Unknown + "Zyyy", # 1F800..1F80B ; Common + "Zzzz", # 1F80C..1F80F ; Unknown + "Zyyy", # 1F810..1F847 ; Common + "Zzzz", # 1F848..1F84F ; Unknown + "Zyyy", # 1F850..1F859 ; Common + "Zzzz", # 1F85A..1F85F ; Unknown + "Zyyy", # 1F860..1F887 ; Common + "Zzzz", # 1F888..1F88F ; Unknown + "Zyyy", # 1F890..1F8AD ; Common + "Zzzz", # 1F8AE..1F8AF ; Unknown + "Zyyy", # 1F8B0..1F8BB ; Common + "Zzzz", # 1F8BC..1F8BF ; Unknown + "Zyyy", # 1F8C0..1F8C1 ; Common + "Zzzz", # 1F8C2..1F8CF ; Unknown + "Zyyy", # 1F8D0..1F8D8 ; Common + "Zzzz", # 1F8D9..1F8FF ; Unknown + "Zyyy", # 1F900..1FA57 ; Common + "Zzzz", # 1FA58..1FA5F ; Unknown + "Zyyy", # 1FA60..1FA6D ; Common + "Zzzz", # 1FA6E..1FA6F ; Unknown + "Zyyy", # 1FA70..1FA7C ; Common + "Zzzz", # 1FA7D..1FA7F ; Unknown + "Zyyy", # 1FA80..1FA8A ; Common + "Zzzz", # 1FA8B..1FA8D ; Unknown + "Zyyy", # 1FA8E..1FAC6 ; Common + "Zzzz", # 1FAC7..1FAC7 ; Unknown + "Zyyy", # 1FAC8..1FAC8 ; Common + "Zzzz", # 1FAC9..1FACC ; Unknown + "Zyyy", # 1FACD..1FADC ; Common + "Zzzz", # 1FADD..1FADE ; Unknown + "Zyyy", # 1FADF..1FAEA ; Common + "Zzzz", # 1FAEB..1FAEE ; Unknown + "Zyyy", # 1FAEF..1FAF8 ; Common + "Zzzz", # 1FAF9..1FAFF ; Unknown + "Zyyy", # 1FB00..1FB92 ; Common + "Zzzz", # 1FB93..1FB93 ; Unknown + "Zyyy", # 1FB94..1FBFA ; Common + "Zzzz", # 1FBFB..1FFFF ; Unknown + "Hani", # 20000..2A6DF ; Han + "Zzzz", # 2A6E0..2A6FF ; Unknown + "Hani", # 2A700..2B81D ; Han + "Zzzz", # 2B81E..2B81F ; Unknown + "Hani", # 2B820..2CEAD ; Han + "Zzzz", # 2CEAE..2CEAF ; Unknown + "Hani", # 2CEB0..2EBE0 ; Han + "Zzzz", # 2EBE1..2EBEF ; Unknown + "Hani", # 2EBF0..2EE5D ; Han + "Zzzz", # 2EE5E..2F7FF ; Unknown + "Hani", # 2F800..2FA1D ; Han + "Zzzz", # 2FA1E..2FFFF ; Unknown + "Hani", # 30000..3134A ; Han + "Zzzz", # 3134B..3134F ; Unknown + "Hani", # 31350..33479 ; Han + "Zzzz", # 3347A..E0000 ; Unknown + "Zyyy", # E0001..E0001 ; Common + "Zzzz", # E0002..E001F ; Unknown + "Zyyy", # E0020..E007F ; Common + "Zzzz", # E0080..E00FF ; Unknown + "Zinh", # E0100..E01EF ; Inherited + "Zzzz", # E01F0..10FFFF ; Unknown +] + +NAMES = { + "Adlm": "Adlam", + "Aghb": "Caucasian_Albanian", + "Ahom": "Ahom", + "Arab": "Arabic", + "Armi": "Imperial_Aramaic", + "Armn": "Armenian", + "Avst": "Avestan", + "Bali": "Balinese", + "Bamu": "Bamum", + "Bass": "Bassa_Vah", + "Batk": "Batak", + "Beng": "Bengali", + "Berf": "Beria_Erfe", + "Bhks": "Bhaiksuki", + "Bopo": "Bopomofo", + "Brah": "Brahmi", + "Brai": "Braille", + "Bugi": "Buginese", + "Buhd": "Buhid", + "Cakm": "Chakma", + "Cans": "Canadian_Aboriginal", + "Cari": "Carian", + "Cham": "Cham", + "Cher": "Cherokee", + "Chrs": "Chorasmian", + "Copt": "Coptic", + "Cpmn": "Cypro_Minoan", + "Cprt": "Cypriot", + "Cyrl": "Cyrillic", + "Deva": "Devanagari", + "Diak": "Dives_Akuru", + "Dogr": "Dogra", + "Dsrt": "Deseret", + "Dupl": "Duployan", + "Egyp": "Egyptian_Hieroglyphs", + "Elba": "Elbasan", + "Elym": "Elymaic", + "Ethi": "Ethiopic", + "Gara": "Garay", + "Geor": "Georgian", + "Glag": "Glagolitic", + "Gong": "Gunjala_Gondi", + "Gonm": "Masaram_Gondi", + "Goth": "Gothic", + "Gran": "Grantha", + "Grek": "Greek", + "Gujr": "Gujarati", + "Gukh": "Gurung_Khema", + "Guru": "Gurmukhi", + "Hang": "Hangul", + "Hani": "Han", + "Hano": "Hanunoo", + "Hatr": "Hatran", + "Hebr": "Hebrew", + "Hira": "Hiragana", + "Hluw": "Anatolian_Hieroglyphs", + "Hmng": "Pahawh_Hmong", + "Hmnp": "Nyiakeng_Puachue_Hmong", + "Hrkt": "Katakana_Or_Hiragana", + "Hung": "Old_Hungarian", + "Ital": "Old_Italic", + "Java": "Javanese", + "Kali": "Kayah_Li", + "Kana": "Katakana", + "Kawi": "Kawi", + "Khar": "Kharoshthi", + "Khmr": "Khmer", + "Khoj": "Khojki", + "Kits": "Khitan_Small_Script", + "Knda": "Kannada", + "Krai": "Kirat_Rai", + "Kthi": "Kaithi", + "Lana": "Tai_Tham", + "Laoo": "Lao", + "Latn": "Latin", + "Lepc": "Lepcha", + "Limb": "Limbu", + "Lina": "Linear_A", + "Linb": "Linear_B", + "Lisu": "Lisu", + "Lyci": "Lycian", + "Lydi": "Lydian", + "Mahj": "Mahajani", + "Maka": "Makasar", + "Mand": "Mandaic", + "Mani": "Manichaean", + "Marc": "Marchen", + "Medf": "Medefaidrin", + "Mend": "Mende_Kikakui", + "Merc": "Meroitic_Cursive", + "Mero": "Meroitic_Hieroglyphs", + "Mlym": "Malayalam", + "Modi": "Modi", + "Mong": "Mongolian", + "Mroo": "Mro", + "Mtei": "Meetei_Mayek", + "Mult": "Multani", + "Mymr": "Myanmar", + "Nagm": "Nag_Mundari", + "Nand": "Nandinagari", + "Narb": "Old_North_Arabian", + "Nbat": "Nabataean", + "Newa": "Newa", + "Nkoo": "Nko", + "Nshu": "Nushu", + "Ogam": "Ogham", + "Olck": "Ol_Chiki", + "Onao": "Ol_Onal", + "Orkh": "Old_Turkic", + "Orya": "Oriya", + "Osge": "Osage", + "Osma": "Osmanya", + "Ougr": "Old_Uyghur", + "Palm": "Palmyrene", + "Pauc": "Pau_Cin_Hau", + "Perm": "Old_Permic", + "Phag": "Phags_Pa", + "Phli": "Inscriptional_Pahlavi", + "Phlp": "Psalter_Pahlavi", + "Phnx": "Phoenician", + "Plrd": "Miao", + "Prti": "Inscriptional_Parthian", + "Rjng": "Rejang", + "Rohg": "Hanifi_Rohingya", + "Runr": "Runic", + "Samr": "Samaritan", + "Sarb": "Old_South_Arabian", + "Saur": "Saurashtra", + "Sgnw": "SignWriting", + "Shaw": "Shavian", + "Shrd": "Sharada", + "Sidd": "Siddham", + "Sidt": "Sidetic", + "Sind": "Khudawadi", + "Sinh": "Sinhala", + "Sogd": "Sogdian", + "Sogo": "Old_Sogdian", + "Sora": "Sora_Sompeng", + "Soyo": "Soyombo", + "Sund": "Sundanese", + "Sunu": "Sunuwar", + "Sylo": "Syloti_Nagri", + "Syrc": "Syriac", + "Tagb": "Tagbanwa", + "Takr": "Takri", + "Tale": "Tai_Le", + "Talu": "New_Tai_Lue", + "Taml": "Tamil", + "Tang": "Tangut", + "Tavt": "Tai_Viet", + "Tayo": "Tai_Yo", + "Telu": "Telugu", + "Tfng": "Tifinagh", + "Tglg": "Tagalog", + "Thaa": "Thaana", + "Thai": "Thai", + "Tibt": "Tibetan", + "Tirh": "Tirhuta", + "Tnsa": "Tangsa", + "Todr": "Todhri", + "Tols": "Tolong_Siki", + "Toto": "Toto", + "Tutg": "Tulu_Tigalari", + "Ugar": "Ugaritic", + "Vaii": "Vai", + "Vith": "Vithkuqi", + "Wara": "Warang_Citi", + "Wcho": "Wancho", + "Xpeo": "Old_Persian", + "Xsux": "Cuneiform", + "Yezi": "Yezidi", + "Yiii": "Yi", + "Zanb": "Zanabazar_Square", + "Zinh": "Inherited", + "Zyyy": "Common", + "Zzzz": "Unknown", +} diff --git a/.venv/lib/python3.11/site-packages/fontTools/unicodedata/__init__.py b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0919e793ea16654f3830dc3e2013c06de9d5f9fa --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/unicodedata/__init__.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +from fontTools.misc.textTools import byteord, tostr + +import re +from bisect import bisect_right +from typing import Literal, TypeVar, overload + + +try: + # use unicodedata backport compatible with python2: + # https://github.com/fonttools/unicodedata2 + from unicodedata2 import * +except ImportError: # pragma: no cover + # fall back to built-in unicodedata (possibly outdated) + from unicodedata import * + +from . import Blocks, Mirrored, Scripts, ScriptExtensions, OTTags + +__all__ = [ + # names from built-in unicodedata module + "lookup", + "name", + "decimal", + "digit", + "numeric", + "category", + "bidirectional", + "combining", + "east_asian_width", + "mirrored", + "decomposition", + "normalize", + "unidata_version", + "ucd_3_2_0", + # additonal functions + "block", + "script", + "script_extension", + "script_name", + "script_code", + "script_horizontal_direction", + "ot_tags_from_script", + "ot_tag_to_script", +] + + +def mirrored(code): + """If code (unicode codepoint) has a mirrored version returns it, otherwise None.""" + return Mirrored.MIRRORED.get(code) + + +def script(char): + """Return the four-letter script code assigned to the Unicode character + 'char' as string. + + >>> script("a") + 'Latn' + >>> script(",") + 'Zyyy' + >>> script(chr(0x10FFFF)) + 'Zzzz' + """ + code = byteord(char) + # 'bisect_right(a, x, lo=0, hi=len(a))' returns an insertion point which + # comes after (to the right of) any existing entries of x in a, and it + # partitions array a into two halves so that, for the left side + # all(val <= x for val in a[lo:i]), and for the right side + # all(val > x for val in a[i:hi]). + # Our 'SCRIPT_RANGES' is a sorted list of ranges (only their starting + # breakpoints); we want to use `bisect_right` to look up the range that + # contains the given codepoint: i.e. whose start is less than or equal + # to the codepoint. Thus, we subtract -1 from the index returned. + i = bisect_right(Scripts.RANGES, code) + return Scripts.VALUES[i - 1] + + +def script_extension(char): + """Return the script extension property assigned to the Unicode character + 'char' as a set of string. + + >>> script_extension("a") == {'Latn'} + True + >>> script_extension(chr(0x060C)) == {'Nkoo', 'Arab', 'Rohg', 'Thaa', 'Syrc', 'Gara', 'Yezi'} + True + >>> script_extension(chr(0x10FFFF)) == {'Zzzz'} + True + """ + code = byteord(char) + i = bisect_right(ScriptExtensions.RANGES, code) + value = ScriptExtensions.VALUES[i - 1] + if value is None: + # code points not explicitly listed for Script Extensions + # have as their value the corresponding Script property value + return {script(char)} + return value + + +def script_name(code, default=KeyError): + """Return the long, human-readable script name given a four-letter + Unicode script code. + + If no matching name is found, a KeyError is raised by default. + + You can use the 'default' argument to return a fallback value (e.g. + 'Unknown' or None) instead of throwing an error. + """ + try: + return str(Scripts.NAMES[code].replace("_", " ")) + except KeyError: + if isinstance(default, type) and issubclass(default, KeyError): + raise + return default + + +_normalize_re = re.compile(r"[-_ ]+") + + +def _normalize_property_name(string): + """Remove case, strip space, '-' and '_' for loose matching.""" + return _normalize_re.sub("", string).lower() + + +_SCRIPT_CODES = {_normalize_property_name(v): k for k, v in Scripts.NAMES.items()} + + +def script_code(script_name, default=KeyError): + """Returns the four-letter Unicode script code from its long name + + If no matching script code is found, a KeyError is raised by default. + + You can use the 'default' argument to return a fallback string (e.g. + 'Zzzz' or None) instead of throwing an error. + """ + normalized_name = _normalize_property_name(script_name) + try: + return _SCRIPT_CODES[normalized_name] + except KeyError: + if isinstance(default, type) and issubclass(default, KeyError): + raise + return default + + +# The data on script direction is taken from Harfbuzz source code: +# https://github.com/harfbuzz/harfbuzz/blob/3.2.0/src/hb-common.cc#L514-L613 +# This in turn references the following "Script_Metadata" document: +# https://docs.google.com/spreadsheets/d/1Y90M0Ie3MUJ6UVCRDOypOtijlMDLNNyyLk36T6iMu0o +RTL_SCRIPTS = { + # Unicode-1.1 additions + "Arab", # Arabic + "Hebr", # Hebrew + # Unicode-3.0 additions + "Syrc", # Syriac + "Thaa", # Thaana + # Unicode-4.0 additions + "Cprt", # Cypriot + # Unicode-4.1 additions + "Khar", # Kharoshthi + # Unicode-5.0 additions + "Phnx", # Phoenician + "Nkoo", # Nko + # Unicode-5.1 additions + "Lydi", # Lydian + # Unicode-5.2 additions + "Avst", # Avestan + "Armi", # Imperial Aramaic + "Phli", # Inscriptional Pahlavi + "Prti", # Inscriptional Parthian + "Sarb", # Old South Arabian + "Orkh", # Old Turkic + "Samr", # Samaritan + # Unicode-6.0 additions + "Mand", # Mandaic + # Unicode-6.1 additions + "Merc", # Meroitic Cursive + "Mero", # Meroitic Hieroglyphs + # Unicode-7.0 additions + "Mani", # Manichaean + "Mend", # Mende Kikakui + "Nbat", # Nabataean + "Narb", # Old North Arabian + "Palm", # Palmyrene + "Phlp", # Psalter Pahlavi + # Unicode-8.0 additions + "Hatr", # Hatran + "Hung", # Old Hungarian + # Unicode-9.0 additions + "Adlm", # Adlam + # Unicode-11.0 additions + "Rohg", # Hanifi Rohingya + "Sogo", # Old Sogdian + "Sogd", # Sogdian + # Unicode-12.0 additions + "Elym", # Elymaic + # Unicode-13.0 additions + "Chrs", # Chorasmian + "Yezi", # Yezidi + # Unicode-14.0 additions + "Ougr", # Old Uyghur + # Unicode-16.0 additions + "Gara", # Garay + # Unicode-17.0 additions + "Sidt", # Sidetic +} + + +HorizDirection = Literal["RTL", "LTR"] +T = TypeVar("T") + + +@overload +def script_horizontal_direction(script_code: str, default: T) -> HorizDirection | T: ... + + +@overload +def script_horizontal_direction( + script_code: str, default: type[KeyError] = KeyError +) -> HorizDirection: ... + + +def script_horizontal_direction( + script_code: str, default: T | type[KeyError] = KeyError +) -> HorizDirection | T: + """Return "RTL" for scripts that contain right-to-left characters + according to the Bidi_Class property. Otherwise return "LTR". + """ + if script_code not in Scripts.NAMES: + if isinstance(default, type) and issubclass(default, KeyError): + raise default(script_code) + return default + return "RTL" if script_code in RTL_SCRIPTS else "LTR" + + +def block(char): + """Return the block property assigned to the Unicode character 'char' + as a string. + + >>> block("a") + 'Basic Latin' + >>> block(chr(0x060C)) + 'Arabic' + >>> block(chr(0xEFFFF)) + 'No_Block' + """ + code = byteord(char) + i = bisect_right(Blocks.RANGES, code) + return Blocks.VALUES[i - 1] + + +def ot_tags_from_script(script_code): + """Return a list of OpenType script tags associated with a given + Unicode script code. + Return ['DFLT'] script tag for invalid/unknown script codes. + """ + if script_code in OTTags.SCRIPT_EXCEPTIONS: + return [OTTags.SCRIPT_EXCEPTIONS[script_code]] + + if script_code not in Scripts.NAMES: + return [OTTags.DEFAULT_SCRIPT] + + script_tags = [script_code[0].lower() + script_code[1:]] + if script_code in OTTags.NEW_SCRIPT_TAGS: + script_tags.extend(OTTags.NEW_SCRIPT_TAGS[script_code]) + script_tags.reverse() # last in, first out + + return script_tags + + +def ot_tag_to_script(tag): + """Return the Unicode script code for the given OpenType script tag, or + None for "DFLT" tag or if there is no Unicode script associated with it. + Raises ValueError if the tag is invalid. + """ + tag = tostr(tag).strip() + if not tag or " " in tag or len(tag) > 4: + raise ValueError("invalid OpenType tag: %r" % tag) + + if tag in OTTags.SCRIPT_ALIASES: + tag = OTTags.SCRIPT_ALIASES[tag] + + while len(tag) != 4: + tag += str(" ") # pad with spaces + + if tag == OTTags.DEFAULT_SCRIPT: + # it's unclear which Unicode script the "DFLT" OpenType tag maps to, + # so here we return None + return None + + if tag in OTTags.NEW_SCRIPT_TAGS_REVERSED: + return OTTags.NEW_SCRIPT_TAGS_REVERSED[tag] + + if tag in OTTags.SCRIPT_EXCEPTIONS_REVERSED: + return OTTags.SCRIPT_EXCEPTIONS_REVERSED[tag] + + # This side of the conversion is fully algorithmic + + # Any spaces at the end of the tag are replaced by repeating the last + # letter. Eg 'nko ' -> 'Nkoo'. + # Change first char to uppercase + script_code = tag[0].upper() + tag[1] + for i in range(2, 4): + script_code += script_code[i - 1] if tag[i] == " " else tag[i] + + if script_code not in Scripts.NAMES: + return None + return script_code diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/__init__.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..34bf8fc20150a261e881454e47ccb292f27b45c1 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/__init__.py @@ -0,0 +1,1582 @@ +""" +Module for dealing with 'gvar'-style font variations, also known as run-time +interpolation. + +The ideas here are very similar to MutatorMath. There is even code to read +MutatorMath .designspace files in the varLib.designspace module. + +For now, if you run this file on a designspace file, it tries to find +ttf-interpolatable files for the masters and build a variable-font from +them. Such ttf-interpolatable and designspace files can be generated from +a Glyphs source, eg., using noto-source as an example: + + .. code-block:: sh + + $ fontmake -o ttf-interpolatable -g NotoSansArabic-MM.glyphs + +Then you can make a variable-font this way: + + .. code-block:: sh + + $ fonttools varLib master_ufo/NotoSansArabic.designspace + +API *will* change in near future. +""" + +from typing import List +from fontTools.misc.vector import Vector +from fontTools.misc.roundTools import noRound, otRound +from fontTools.misc.fixedTools import floatToFixed as fl2fi +from fontTools.misc.textTools import Tag, tostr +from fontTools.ttLib import TTFont, newTable +from fontTools.ttLib.tables._f_v_a_r import Axis, NamedInstance +from fontTools.ttLib.tables._g_l_y_f import ( + GlyphCoordinates, + dropImpliedOnCurvePoints, + USE_MY_METRICS, +) +from fontTools.ttLib.tables.ttProgram import Program +from fontTools.ttLib.tables.TupleVariation import TupleVariation +from fontTools.ttLib.tables import otTables as ot +from fontTools.ttLib.tables.otBase import OTTableWriter +from fontTools.varLib import builder, models, varStore +from fontTools.varLib.merger import VariationMerger, COLRVariationMerger +from fontTools.varLib.mvar import MVAR_ENTRIES +from fontTools.varLib.iup import iup_delta_optimize +from fontTools.varLib.featureVars import addFeatureVariations +from fontTools.designspaceLib import DesignSpaceDocument, InstanceDescriptor +from fontTools.designspaceLib.split import splitInterpolable, splitVariableFonts +from fontTools.varLib.stat import buildVFStatTable +from fontTools.colorLib.builder import buildColrV1 +from fontTools.colorLib.unbuilder import unbuildColrV1 +from functools import partial +from collections import OrderedDict, defaultdict, namedtuple +import os.path +import logging +from copy import deepcopy +from pprint import pformat +from re import fullmatch +from .errors import VarLibError, VarLibValidationError + +log = logging.getLogger("fontTools.varLib") + +# This is a lib key for the designspace document. The value should be +# a comma-separated list of OpenType feature tag(s), to be used as the +# FeatureVariations feature. +# If present, the DesignSpace <rules processing="..."> flag is ignored. +FEAVAR_FEATURETAG_LIB_KEY = "com.github.fonttools.varLib.featureVarsFeatureTag" + +# +# Creation routines +# + + +def _add_fvar(font, axes, instances: List[InstanceDescriptor]): + """ + Add 'fvar' table to font. + + axes is an ordered dictionary of DesignspaceAxis objects. + + instances is list of dictionary objects with 'location', 'stylename', + and possibly 'postscriptfontname' entries. + """ + + assert axes + assert isinstance(axes, OrderedDict) + + log.info("Generating fvar") + + fvar = newTable("fvar") + nameTable = font["name"] + + # if there are not currently any mac names don't add them here, that's inconsistent + # https://github.com/fonttools/fonttools/issues/683 + macNames = any(nr.platformID == 1 for nr in getattr(nameTable, "names", ())) + + # we have all the best ways to express mac names + platforms = ((3, 1, 0x409),) + if macNames: + platforms = ((1, 0, 0),) + platforms + + for a in axes.values(): + axis = Axis() + axis.axisTag = Tag(a.tag) + # TODO Skip axes that have no variation. + axis.minValue, axis.defaultValue, axis.maxValue = ( + a.minimum, + a.default, + a.maximum, + ) + axis.axisNameID = nameTable.addMultilingualName( + a.labelNames, font, minNameID=256, mac=macNames + ) + axis.flags = int(a.hidden) + fvar.axes.append(axis) + + default_coordinates = {axis.axisTag: axis.defaultValue for axis in fvar.axes} + + for instance in instances: + # Filter out discrete axis locations + coordinates = { + name: value for name, value in instance.location.items() if name in axes + } + + if "en" not in instance.localisedStyleName: + if not instance.styleName: + raise VarLibValidationError( + f"Instance at location '{coordinates}' must have a default English " + "style name ('stylename' attribute on the instance element or a " + "stylename element with an 'xml:lang=\"en\"' attribute)." + ) + localisedStyleName = dict(instance.localisedStyleName) + localisedStyleName["en"] = tostr(instance.styleName) + else: + localisedStyleName = instance.localisedStyleName + + psname = instance.postScriptFontName + + inst = NamedInstance() + inst.coordinates = { + axes[k].tag: axes[k].map_backward(v) for k, v in coordinates.items() + } + + subfamilyNameID = nameTable.findMultilingualName( + localisedStyleName, windows=True, mac=macNames + ) + if subfamilyNameID in {2, 17} and inst.coordinates == default_coordinates: + # Instances can only reuse an existing name ID 2 or 17 if they are at the + # default location across all axes, see: + # https://github.com/fonttools/fonttools/issues/3825. + inst.subfamilyNameID = subfamilyNameID + else: + inst.subfamilyNameID = nameTable.addMultilingualName( + localisedStyleName, windows=True, mac=macNames, minNameID=256 + ) + + if psname is not None: + psname = tostr(psname) + inst.postscriptNameID = nameTable.addName(psname, platforms=platforms) + fvar.instances.append(inst) + + assert "fvar" not in font + font["fvar"] = fvar + + return fvar + + +def _add_avar(font, axes, mappings, axisTags): + """ + Add 'avar' table to font. + + axes is an ordered dictionary of AxisDescriptor objects. + """ + + assert axes + assert isinstance(axes, OrderedDict) + + log.info("Generating avar") + + avar = newTable("avar") + + interesting = False + vals_triples = {} + for axis in axes.values(): + # Currently, some rasterizers require that the default value maps + # (-1 to -1, 0 to 0, and 1 to 1) be present for all the segment + # maps, even when the default normalization mapping for the axis + # was not modified. + # https://github.com/googlei18n/fontmake/issues/295 + # https://github.com/fonttools/fonttools/issues/1011 + # TODO(anthrotype) revert this (and 19c4b37) when issue is fixed + curve = avar.segments[axis.tag] = {-1.0: -1.0, 0.0: 0.0, 1.0: 1.0} + + keys_triple = (axis.minimum, axis.default, axis.maximum) + vals_triple = tuple(axis.map_forward(v) for v in keys_triple) + vals_triples[axis.tag] = vals_triple + + if not axis.map: + continue + + items = sorted(axis.map) + keys = [item[0] for item in items] + vals = [item[1] for item in items] + + # Current avar requirements. We don't have to enforce + # these on the designer and can deduce some ourselves, + # but for now just enforce them. + if axis.minimum != min(keys): + raise VarLibValidationError( + f"Axis '{axis.name}': there must be a mapping for the axis minimum " + f"value {axis.minimum} and it must be the lowest input mapping value." + ) + if axis.maximum != max(keys): + raise VarLibValidationError( + f"Axis '{axis.name}': there must be a mapping for the axis maximum " + f"value {axis.maximum} and it must be the highest input mapping value." + ) + if axis.default not in keys: + raise VarLibValidationError( + f"Axis '{axis.name}': there must be a mapping for the axis default " + f"value {axis.default}." + ) + # No duplicate input values (output values can be >= their preceeding value). + if len(set(keys)) != len(keys): + raise VarLibValidationError( + f"Axis '{axis.name}': All axis mapping input='...' values must be " + "unique, but we found duplicates." + ) + # Ascending values + if sorted(vals) != vals: + raise VarLibValidationError( + f"Axis '{axis.name}': mapping output values must be in ascending order." + ) + + keys = [models.normalizeValue(v, keys_triple) for v in keys] + vals = [models.normalizeValue(v, vals_triple) for v in vals] + + if all(k == v for k, v in zip(keys, vals)): + continue + interesting = True + + curve.update(zip(keys, vals)) + + assert 0.0 in curve and curve[0.0] == 0.0 + assert -1.0 not in curve or curve[-1.0] == -1.0 + assert +1.0 not in curve or curve[+1.0] == +1.0 + # curve.update({-1.0: -1.0, 0.0: 0.0, 1.0: 1.0}) + + if mappings: + interesting = True + + inputLocations = [ + { + axes[name].tag: models.normalizeValue(v, vals_triples[axes[name].tag]) + for name, v in mapping.inputLocation.items() + } + for mapping in mappings + ] + outputLocations = [ + { + axes[name].tag: models.normalizeValue(v, vals_triples[axes[name].tag]) + for name, v in mapping.outputLocation.items() + } + for mapping in mappings + ] + assert len(inputLocations) == len(outputLocations) + + # If base-master is missing, insert it at zero location. + if not any(all(v == 0 for k, v in loc.items()) for loc in inputLocations): + inputLocations.insert(0, {}) + outputLocations.insert(0, {}) + + model = models.VariationModel(inputLocations, axisTags) + storeBuilder = varStore.OnlineVarStoreBuilder(axisTags) + storeBuilder.setModel(model) + varIdxes = {} + for tag in axisTags: + masterValues = [] + for vo, vi in zip(outputLocations, inputLocations): + if tag not in vo: + masterValues.append(0) + continue + v = vo[tag] - vi.get(tag, 0) + masterValues.append(fl2fi(v, 14)) + varIdxes[tag] = storeBuilder.storeMasters(masterValues)[1] + + store = storeBuilder.finish() + optimized = store.optimize() + varIdxes = {axis: optimized[value] for axis, value in varIdxes.items()} + + varIdxMap = builder.buildDeltaSetIndexMap(varIdxes[t] for t in axisTags) + + avar.majorVersion = 2 + avar.table = ot.avar() + avar.table.VarIdxMap = varIdxMap + avar.table.VarStore = store + + assert "avar" not in font + if not interesting: + log.info("No need for avar") + avar = None + else: + font["avar"] = avar + + return avar + + +def _add_stat(font): + # Note: this function only gets called by old code that calls `build()` + # directly. Newer code that wants to benefit from STAT data from the + # designspace should call `build_many()` + + if "STAT" in font: + return + + from ..otlLib.builder import buildStatTable + + fvarTable = font["fvar"] + axes = [dict(tag=a.axisTag, name=a.axisNameID) for a in fvarTable.axes] + buildStatTable(font, axes) + + +_MasterData = namedtuple("_MasterData", ["glyf", "hMetrics", "vMetrics"]) + + +def _add_gvar(font, masterModel, master_ttfs, tolerance=0.5, optimize=True): + if tolerance < 0: + raise ValueError("`tolerance` must be a positive number.") + + log.info("Generating gvar") + assert "gvar" not in font + gvar = font["gvar"] = newTable("gvar") + glyf = font["glyf"] + defaultMasterIndex = masterModel.reverseMapping[0] + + master_datas = [ + _MasterData( + m["glyf"], m["hmtx"].metrics, getattr(m.get("vmtx"), "metrics", None) + ) + for m in master_ttfs + ] + + for glyph in font.getGlyphOrder(): + log.debug("building gvar for glyph '%s'", glyph) + + allData = [ + m.glyf._getCoordinatesAndControls(glyph, m.hMetrics, m.vMetrics) + for m in master_datas + ] + + if allData[defaultMasterIndex][1].numberOfContours != 0: + # If the default master is not empty, interpret empty non-default masters + # as missing glyphs from a sparse master + allData = [ + d if d is not None and d[1].numberOfContours != 0 else None + for d in allData + ] + + model, allData = masterModel.getSubModel(allData) + + allCoords = [d[0] for d in allData] + allControls = [d[1] for d in allData] + control = allControls[0] + if not models.allEqual(allControls): + log.warning("glyph %s has incompatible masters; skipping" % glyph) + continue + del allControls + + # Update gvar + gvar.variations[glyph] = [] + deltas = model.getDeltas( + allCoords, round=partial(GlyphCoordinates.__round__, round=round) + ) + supports = model.supports + assert len(deltas) == len(supports) + + # Prepare for IUP optimization + origCoords = deltas[0] + endPts = control.endPts + + for i, (delta, support) in enumerate(zip(deltas[1:], supports[1:])): + if all(v == 0 for v in delta.array): + continue + var = TupleVariation(support, delta) + if optimize: + var.optimize(origCoords, endPts, tolerance=tolerance) + gvar.variations[glyph].append(var) + + +def _remove_TTHinting(font): + for tag in ("cvar", "cvt ", "fpgm", "prep"): + if tag in font: + del font[tag] + maxp = font["maxp"] + for attr in ( + "maxTwilightPoints", + "maxStorage", + "maxFunctionDefs", + "maxInstructionDefs", + "maxStackElements", + "maxSizeOfInstructions", + ): + setattr(maxp, attr, 0) + maxp.maxZones = 1 + font["glyf"].removeHinting() + # TODO: Modify gasp table to deactivate gridfitting for all ranges? + + +def _merge_TTHinting(font, masterModel, master_ttfs): + log.info("Merging TT hinting") + assert "cvar" not in font + + # Check that the existing hinting is compatible + + # fpgm and prep table + + for tag in ("fpgm", "prep"): + all_pgms = [m[tag].program for m in master_ttfs if tag in m] + if not all_pgms: + continue + font_pgm = getattr(font.get(tag), "program", None) + if any(pgm != font_pgm for pgm in all_pgms): + log.warning( + "Masters have incompatible %s tables, hinting is discarded." % tag + ) + _remove_TTHinting(font) + return + + # glyf table + + font_glyf = font["glyf"] + master_glyfs = [m["glyf"] for m in master_ttfs] + for name, glyph in font_glyf.glyphs.items(): + all_pgms = [getattr(glyf.get(name), "program", None) for glyf in master_glyfs] + if not any(all_pgms): + continue + glyph.expand(font_glyf) + font_pgm = getattr(glyph, "program", None) + if any(pgm != font_pgm for pgm in all_pgms if pgm): + log.warning( + "Masters have incompatible glyph programs in glyph '%s', hinting is discarded." + % name + ) + # TODO Only drop hinting from this glyph. + _remove_TTHinting(font) + return + + # cvt table + + all_cvs = [Vector(m["cvt "].values) if "cvt " in m else None for m in master_ttfs] + + nonNone_cvs = models.nonNone(all_cvs) + if not nonNone_cvs: + # There is no cvt table to make a cvar table from, we're done here. + return + + if not models.allEqual(len(c) for c in nonNone_cvs): + log.warning("Masters have incompatible cvt tables, hinting is discarded.") + _remove_TTHinting(font) + return + + variations = [] + deltas, supports = masterModel.getDeltasAndSupports( + all_cvs, round=round + ) # builtin round calls into Vector.__round__, which uses builtin round as we like + for i, (delta, support) in enumerate(zip(deltas[1:], supports[1:])): + if all(v == 0 for v in delta): + continue + var = TupleVariation(support, delta) + variations.append(var) + + # We can build the cvar table now. + if variations: + cvar = font["cvar"] = newTable("cvar") + cvar.version = 1 + cvar.variations = variations + + +def _has_inconsistent_use_my_metrics_flag( + master_glyf, glyph_name, flagged_components, expected_num_components +) -> bool: + master_glyph = master_glyf.get(glyph_name) + # 'sparse' glyph master doesn't contribute. Besides when components don't match + # the VF build is going to fail anyway, so be lenient here. + if ( + master_glyph is not None + and master_glyph.isComposite() + and len(master_glyph.components) == expected_num_components + ): + for i, base_glyph in flagged_components: + comp = master_glyph.components[i] + if comp.glyphName != base_glyph: + break + if not (comp.flags & USE_MY_METRICS): + return True + return False + + +def _unset_inconsistent_use_my_metrics_flags(vf, master_fonts): + """Clear USE_MY_METRICS on composite components if inconsistent across masters. + + If a composite glyph's component has USE_MY_METRICS set differently among + the masters, the flag is removed from the variable font's glyf table so that + advance widths are not determined by that single component's phantom points. + """ + glyf = vf["glyf"] + master_glyfs = [m["glyf"] for m in master_fonts if "glyf" in m] + if not master_glyfs: + # Should not happen: at least the base master (as copied into vf) has glyf + return + + for glyph_name in glyf.keys(): + glyph = glyf[glyph_name] + if not glyph.isComposite(): + continue + + # collect indices of component(s) that carry the USE_MY_METRICS flag. + # This is supposed to be 1 component per composite, but you never know. + flagged_components = [ + (i, comp.glyphName) + for i, comp in enumerate(glyph.components) + if (comp.flags & USE_MY_METRICS) + ] + if not flagged_components: + # Nothing to fix + continue + + # Verify that for all master glyf tables that contribute this glyph, the + # corresponding component (same glyphName and index) also carries USE_MY_METRICS + # and unset the flag if not. + expected_num_components = len(glyph.components) + if any( + _has_inconsistent_use_my_metrics_flag( + master_glyf, glyph_name, flagged_components, expected_num_components + ) + for master_glyf in master_glyfs + ): + comp_names = [name for _, name in flagged_components] + log.info( + "Composite glyph '%s' has inconsistent USE_MY_METRICS flags across " + "masters; clearing the flag on component%s %s", + glyph_name, + "s" if len(comp_names) > 1 else "", + comp_names if len(comp_names) > 1 else comp_names[0], + ) + for i, _ in flagged_components: + glyph.components[i].flags &= ~USE_MY_METRICS + + +_MetricsFields = namedtuple( + "_MetricsFields", + [ + "tableTag", + "metricsTag", + "sb1", + "sb2", + "advMapping", + "vOrigMapping", + "phantomIndex", + ], +) + +HVAR_FIELDS = _MetricsFields( + tableTag="HVAR", + metricsTag="hmtx", + sb1="LsbMap", + sb2="RsbMap", + advMapping="AdvWidthMap", + vOrigMapping=None, + phantomIndex=0, +) + +VVAR_FIELDS = _MetricsFields( + tableTag="VVAR", + metricsTag="vmtx", + sb1="TsbMap", + sb2="BsbMap", + advMapping="AdvHeightMap", + vOrigMapping="VOrgMap", + phantomIndex=1, +) + + +def _add_HVAR(font, masterModel, master_ttfs, axisTags): + getAdvanceMetrics = partial( + _get_advance_metrics, font, masterModel, master_ttfs, axisTags, HVAR_FIELDS + ) + _add_VHVAR(font, axisTags, HVAR_FIELDS, getAdvanceMetrics) + + +def _add_VVAR(font, masterModel, master_ttfs, axisTags): + getAdvanceMetrics = partial( + _get_advance_metrics, font, masterModel, master_ttfs, axisTags, VVAR_FIELDS + ) + _add_VHVAR(font, axisTags, VVAR_FIELDS, getAdvanceMetrics) + + +def _add_VHVAR(font, axisTags, tableFields, getAdvanceMetrics): + tableTag = tableFields.tableTag + assert tableTag not in font + glyphOrder = font.getGlyphOrder() + log.info("Generating " + tableTag) + VHVAR = newTable(tableTag) + tableClass = getattr(ot, tableTag) + vhvar = VHVAR.table = tableClass() + vhvar.Version = 0x00010000 + + vhAdvanceDeltasAndSupports, vOrigDeltasAndSupports = getAdvanceMetrics() + + if vOrigDeltasAndSupports: + singleModel = False + else: + singleModel = models.allEqual( + id(v[1]) for v in vhAdvanceDeltasAndSupports.values() + ) + + directStore = None + if singleModel: + # Build direct mapping + supports = next(iter(vhAdvanceDeltasAndSupports.values()))[1][1:] + varTupleList = builder.buildVarRegionList(supports, axisTags) + varTupleIndexes = list(range(len(supports))) + varData = builder.buildVarData(varTupleIndexes, [], optimize=False) + for glyphName in glyphOrder: + varData.addItem(vhAdvanceDeltasAndSupports[glyphName][0], round=noRound) + varData.optimize() + directStore = builder.buildVarStore(varTupleList, [varData]) + # remove unused regions from VarRegionList + directStore.prune_regions() + + # Build optimized indirect mapping + storeBuilder = varStore.OnlineVarStoreBuilder(axisTags) + advMapping = {} + for glyphName in glyphOrder: + deltas, supports = vhAdvanceDeltasAndSupports[glyphName] + storeBuilder.setSupports(supports) + advMapping[glyphName] = storeBuilder.storeDeltas(deltas, round=noRound) + + if vOrigDeltasAndSupports: + vOrigMap = {} + for glyphName in glyphOrder: + deltas, supports = vOrigDeltasAndSupports[glyphName] + storeBuilder.setSupports(supports) + vOrigMap[glyphName] = storeBuilder.storeDeltas(deltas, round=noRound) + + indirectStore = storeBuilder.finish() + mapping2 = indirectStore.optimize(use_NO_VARIATION_INDEX=False) + advMapping = [mapping2[advMapping[g]] for g in glyphOrder] + advanceMapping = builder.buildVarIdxMap(advMapping, glyphOrder) + + if vOrigDeltasAndSupports: + vOrigMap = [mapping2[vOrigMap[g]] for g in glyphOrder] + + useDirect = False + vOrigMapping = None + if directStore: + # Compile both, see which is more compact + + writer = OTTableWriter() + directStore.compile(writer, font) + directSize = len(writer.getAllData()) + + writer = OTTableWriter() + indirectStore.compile(writer, font) + advanceMapping.compile(writer, font) + indirectSize = len(writer.getAllData()) + + useDirect = directSize < indirectSize + + if useDirect: + metricsStore = directStore + advanceMapping = None + else: + metricsStore = indirectStore + if vOrigDeltasAndSupports: + vOrigMapping = builder.buildVarIdxMap(vOrigMap, glyphOrder) + + vhvar.VarStore = metricsStore + setattr(vhvar, tableFields.advMapping, advanceMapping) + if vOrigMapping is not None: + setattr(vhvar, tableFields.vOrigMapping, vOrigMapping) + setattr(vhvar, tableFields.sb1, None) + setattr(vhvar, tableFields.sb2, None) + + font[tableTag] = VHVAR + return + + +def _get_advance_metrics(font, masterModel, master_ttfs, axisTags, tableFields): + tableTag = tableFields.tableTag + glyphOrder = font.getGlyphOrder() + + # Build list of source font advance widths for each glyph + metricsTag = tableFields.metricsTag + advMetricses = [m[metricsTag].metrics for m in master_ttfs] + + # Build list of source font vertical origin coords for each glyph + if tableTag == "VVAR" and "VORG" in master_ttfs[0]: + vOrigMetricses = [m["VORG"].VOriginRecords for m in master_ttfs] + defaultYOrigs = [m["VORG"].defaultVertOriginY for m in master_ttfs] + vOrigMetricses = list(zip(vOrigMetricses, defaultYOrigs)) + else: + vOrigMetricses = None + + vhAdvanceDeltasAndSupports = {} + vOrigDeltasAndSupports = {} + # HACK: we treat width 65535 as a sentinel value to signal that a glyph + # from a non-default master should not participate in computing {H,V}VAR, + # as if it were missing. Allows to variate other glyph-related data independently + # from glyph metrics + sparse_advance = 0xFFFF + for glyph in glyphOrder: + vhAdvances = [ + ( + metrics[glyph][0] + if glyph in metrics and metrics[glyph][0] != sparse_advance + else None + ) + for metrics in advMetricses + ] + vhAdvanceDeltasAndSupports[glyph] = masterModel.getDeltasAndSupports( + vhAdvances, round=round + ) + + if vOrigMetricses: + for glyph in glyphOrder: + # We need to supply a vOrigs tuple with non-None default values + # for each glyph. vOrigMetricses contains values only for those + # glyphs which have a non-default vOrig. + vOrigs = [ + metrics[glyph] if glyph in metrics else defaultVOrig + for metrics, defaultVOrig in vOrigMetricses + ] + vOrigDeltasAndSupports[glyph] = masterModel.getDeltasAndSupports( + vOrigs, round=round + ) + + return vhAdvanceDeltasAndSupports, vOrigDeltasAndSupports + + +def _add_MVAR(font, masterModel, master_ttfs, axisTags): + log.info("Generating MVAR") + + store_builder = varStore.OnlineVarStoreBuilder(axisTags) + + records = [] + lastTableTag = None + fontTable = None + tables = None + # HACK: we need to special-case post.underlineThickness and .underlinePosition + # and unilaterally/arbitrarily define a sentinel value to distinguish the case + # when a post table is present in a given master simply because that's where + # the glyph names in TrueType must be stored, but the underline values are not + # meant to be used for building MVAR's deltas. The value of -0x8000 (-32768) + # the minimum FWord (int16) value, was chosen for its unlikelyhood to appear + # in real-world underline position/thickness values. + specialTags = {"unds": -0x8000, "undo": -0x8000} + + for tag, (tableTag, itemName) in sorted(MVAR_ENTRIES.items(), key=lambda kv: kv[1]): + # For each tag, fetch the associated table from all fonts (or not when we are + # still looking at a tag from the same tables) and set up the variation model + # for them. + if tableTag != lastTableTag: + tables = fontTable = None + if tableTag in font: + fontTable = font[tableTag] + tables = [] + for master in master_ttfs: + if tableTag not in master or ( + tag in specialTags + and getattr(master[tableTag], itemName) == specialTags[tag] + ): + tables.append(None) + else: + tables.append(master[tableTag]) + model, tables = masterModel.getSubModel(tables) + store_builder.setModel(model) + lastTableTag = tableTag + + if tables is None: # Tag not applicable to the master font. + continue + + # TODO support gasp entries + + master_values = [getattr(table, itemName) for table in tables] + if models.allEqual(master_values): + base, varIdx = master_values[0], None + else: + base, varIdx = store_builder.storeMasters(master_values) + setattr(fontTable, itemName, base) + + if varIdx is None: + continue + log.info(" %s: %s.%s %s", tag, tableTag, itemName, master_values) + rec = ot.MetricsValueRecord() + rec.ValueTag = tag + rec.VarIdx = varIdx + records.append(rec) + + assert "MVAR" not in font + if records: + store = store_builder.finish() + # Optimize + mapping = store.optimize() + for rec in records: + rec.VarIdx = mapping[rec.VarIdx] + + MVAR = font["MVAR"] = newTable("MVAR") + mvar = MVAR.table = ot.MVAR() + mvar.Version = 0x00010000 + mvar.Reserved = 0 + mvar.VarStore = store + # XXX these should not be hard-coded but computed automatically + mvar.ValueRecordSize = 8 + mvar.ValueRecordCount = len(records) + mvar.ValueRecord = sorted(records, key=lambda r: r.ValueTag) + + +def _add_BASE(font, masterModel, master_ttfs, axisTags): + log.info("Generating BASE") + + merger = VariationMerger(masterModel, axisTags, font) + merger.mergeTables(font, master_ttfs, ["BASE"]) + store = merger.store_builder.finish() + + if not store: + return + base = font["BASE"].table + assert base.Version == 0x00010000 + base.Version = 0x00010001 + base.VarStore = store + + +def _merge_OTL(font, model, master_fonts, axisTags): + otl_tags = ["GSUB", "GDEF", "GPOS"] + if not any(tag in font for tag in otl_tags): + return + + log.info("Merging OpenType Layout tables") + merger = VariationMerger(model, axisTags, font) + + merger.mergeTables(font, master_fonts, otl_tags) + store = merger.store_builder.finish() + if not store: + return + try: + GDEF = font["GDEF"].table + assert GDEF.Version <= 0x00010002 + except KeyError: + GDEFTable = font["GDEF"] = newTable("GDEF") + GDEF = GDEFTable.table = ot.GDEF() + GDEF.GlyphClassDef = None + GDEF.AttachList = None + GDEF.LigCaretList = None + GDEF.MarkAttachClassDef = None + GDEF.MarkGlyphSetsDef = None + + GDEF.Version = 0x00010003 + GDEF.VarStore = store + + # Optimize + varidx_map = store.optimize() + GDEF.remap_device_varidxes(varidx_map) + if "GPOS" in font: + font["GPOS"].table.remap_device_varidxes(varidx_map) + + +def _add_GSUB_feature_variations( + font, axes, internal_axis_supports, rules, featureTags +): + def normalize(name, value): + return models.normalizeLocation({name: value}, internal_axis_supports)[name] + + log.info("Generating GSUB FeatureVariations") + + axis_tags = {name: axis.tag for name, axis in axes.items()} + + conditional_subs = [] + for rule in rules: + region = [] + for conditions in rule.conditionSets: + space = {} + for condition in conditions: + axis_name = condition["name"] + if condition["minimum"] is not None: + minimum = normalize(axis_name, condition["minimum"]) + else: + minimum = -1.0 + if condition["maximum"] is not None: + maximum = normalize(axis_name, condition["maximum"]) + else: + maximum = 1.0 + tag = axis_tags[axis_name] + space[tag] = (minimum, maximum) + region.append(space) + + subs = {k: v for k, v in rule.subs} + + conditional_subs.append((region, subs)) + + addFeatureVariations(font, conditional_subs, featureTags) + + +_DesignSpaceData = namedtuple( + "_DesignSpaceData", + [ + "axes", + "axisMappings", + "internal_axis_supports", + "base_idx", + "normalized_master_locs", + "masters", + "instances", + "rules", + "rulesProcessingLast", + "lib", + ], +) + + +def _add_CFF2(varFont, model, master_fonts): + from .cff import merge_region_fonts + + glyphOrder = varFont.getGlyphOrder() + if "CFF2" not in varFont: + from fontTools.cffLib.CFFToCFF2 import convertCFFToCFF2 + + convertCFFToCFF2(varFont) + + ordered_fonts_list = model.reorderMasters(master_fonts, model.reverseMapping) + # re-ordering the master list simplifies building the CFF2 data item lists. + merge_region_fonts(varFont, model, ordered_fonts_list, glyphOrder) + + +def _add_COLR(font, model, master_fonts, axisTags, colr_layer_reuse=True): + merger = COLRVariationMerger( + model, axisTags, font, allowLayerReuse=colr_layer_reuse + ) + merger.mergeTables(font, master_fonts) + store = merger.store_builder.finish() + + colr = font["COLR"].table + if store: + mapping = store.optimize() + colr.VarStore = store + varIdxes = [mapping[v] for v in merger.varIdxes] + colr.VarIndexMap = builder.buildDeltaSetIndexMap(varIdxes) + + +def load_designspace(designspace, log_enabled=True, *, require_sources=True): + # TODO: remove this and always assume 'designspace' is a DesignSpaceDocument, + # never a file path, as that's already handled by caller + if hasattr(designspace, "sources"): # Assume a DesignspaceDocument + ds = designspace + else: # Assume a file path + ds = DesignSpaceDocument.fromfile(designspace) + + masters = ds.sources + if require_sources and not masters: + raise VarLibValidationError("Designspace must have at least one source.") + instances = ds.instances + + # TODO: Use fontTools.designspaceLib.tagForAxisName instead. + standard_axis_map = OrderedDict( + [ + ("weight", ("wght", {"en": "Weight"})), + ("width", ("wdth", {"en": "Width"})), + ("slant", ("slnt", {"en": "Slant"})), + ("optical", ("opsz", {"en": "Optical Size"})), + ("italic", ("ital", {"en": "Italic"})), + ] + ) + + # Setup axes + if not ds.axes: + raise VarLibValidationError(f"Designspace must have at least one axis.") + + axes = OrderedDict() + for axis_index, axis in enumerate(ds.axes): + axis_name = axis.name + if not axis_name: + if not axis.tag: + raise VarLibValidationError(f"Axis at index {axis_index} needs a tag.") + axis_name = axis.name = axis.tag + + if axis_name in standard_axis_map: + if axis.tag is None: + axis.tag = standard_axis_map[axis_name][0] + if not axis.labelNames: + axis.labelNames.update(standard_axis_map[axis_name][1]) + else: + if not axis.tag: + raise VarLibValidationError(f"Axis at index {axis_index} needs a tag.") + if not axis.labelNames: + axis.labelNames["en"] = tostr(axis_name) + + axes[axis_name] = axis + if log_enabled: + log.info("Axes:\n%s", pformat([axis.asdict() for axis in axes.values()])) + + axisMappings = ds.axisMappings + if axisMappings and log_enabled: + log.info("Mappings:\n%s", pformat(axisMappings)) + + # Check all master and instance locations are valid and fill in defaults + for obj in masters + instances: + obj_name = obj.name or obj.styleName or "" + loc = obj.getFullDesignLocation(ds) + obj.designLocation = loc + if loc is None: + raise VarLibValidationError( + f"Source or instance '{obj_name}' has no location." + ) + for axis_name in loc.keys(): + if axis_name not in axes: + raise VarLibValidationError( + f"Location axis '{axis_name}' unknown for '{obj_name}'." + ) + for axis_name, axis in axes.items(): + v = axis.map_backward(loc[axis_name]) + if not (axis.minimum <= v <= axis.maximum): + raise VarLibValidationError( + f"Source or instance '{obj_name}' has out-of-range location " + f"for axis '{axis_name}': is mapped to {v} but must be in " + f"mapped range [{axis.minimum}..{axis.maximum}] (NOTE: all " + "values are in user-space)." + ) + + # Normalize master locations + + internal_master_locs = [o.getFullDesignLocation(ds) for o in masters] + if log_enabled: + log.info("Internal master locations:\n%s", pformat(internal_master_locs)) + + # TODO This mapping should ideally be moved closer to logic in _add_fvar/avar + internal_axis_supports = {} + for axis in axes.values(): + triple = (axis.minimum, axis.default, axis.maximum) + internal_axis_supports[axis.name] = [axis.map_forward(v) for v in triple] + if log_enabled: + log.info("Internal axis supports:\n%s", pformat(internal_axis_supports)) + + normalized_master_locs = [ + models.normalizeLocation(m, internal_axis_supports) + for m in internal_master_locs + ] + if log_enabled: + log.info("Normalized master locations:\n%s", pformat(normalized_master_locs)) + + # Find base master + base_idx = None + for i, m in enumerate(normalized_master_locs): + if all(v == 0 for v in m.values()): + if base_idx is not None: + raise VarLibValidationError( + "More than one base master found in Designspace." + ) + base_idx = i + if require_sources and base_idx is None: + raise VarLibValidationError( + "Base master not found; no master at default location?" + ) + if log_enabled: + log.info("Index of base master: %s", base_idx) + + return _DesignSpaceData( + axes, + axisMappings, + internal_axis_supports, + base_idx, + normalized_master_locs, + masters, + instances, + ds.rules, + ds.rulesProcessingLast, + ds.lib, + ) + + +# https://docs.microsoft.com/en-us/typography/opentype/spec/os2#uswidthclass +WDTH_VALUE_TO_OS2_WIDTH_CLASS = { + 50: 1, + 62.5: 2, + 75: 3, + 87.5: 4, + 100: 5, + 112.5: 6, + 125: 7, + 150: 8, + 200: 9, +} + + +def set_default_weight_width_slant(font, location): + if "OS/2" in font: + if "wght" in location: + weight_class = otRound(max(1, min(location["wght"], 1000))) + if font["OS/2"].usWeightClass != weight_class: + log.info("Setting OS/2.usWeightClass = %s", weight_class) + font["OS/2"].usWeightClass = weight_class + + if "wdth" in location: + # map 'wdth' axis (50..200) to OS/2.usWidthClass (1..9), rounding to closest + widthValue = min(max(location["wdth"], 50), 200) + widthClass = otRound( + models.piecewiseLinearMap(widthValue, WDTH_VALUE_TO_OS2_WIDTH_CLASS) + ) + if font["OS/2"].usWidthClass != widthClass: + log.info("Setting OS/2.usWidthClass = %s", widthClass) + font["OS/2"].usWidthClass = widthClass + + if "slnt" in location and "post" in font: + italicAngle = max(-90, min(location["slnt"], 90)) + if font["post"].italicAngle != italicAngle: + log.info("Setting post.italicAngle = %s", italicAngle) + font["post"].italicAngle = italicAngle + + +def drop_implied_oncurve_points(*masters: TTFont) -> int: + """Drop impliable on-curve points from all the simple glyphs in masters. + + In TrueType glyf outlines, on-curve points can be implied when they are located + exactly at the midpoint of the line connecting two consecutive off-curve points. + + The input masters' glyf tables are assumed to contain same-named glyphs that are + interpolatable. Oncurve points are only dropped if they can be implied for all + the masters. The fonts are modified in-place. + + Args: + masters: The TTFont(s) to modify + + Returns: + The total number of points that were dropped if any. + + Reference: + https://developer.apple.com/fonts/TrueType-Reference-Manual/RM01/Chap1.html + """ + + glyph_masters = defaultdict(list) + # multiple DS source may point to the same TTFont object and we want to + # avoid processing the same glyph twice as they are modified in-place + for font in {id(m): m for m in masters}.values(): + glyf = font["glyf"] + for glyphName in glyf.keys(): + glyph_masters[glyphName].append(glyf[glyphName]) + count = 0 + for glyphName, glyphs in glyph_masters.items(): + try: + dropped = dropImpliedOnCurvePoints(*glyphs) + except ValueError as e: + # we don't fail for incompatible glyphs in _add_gvar so we shouldn't here + log.warning("Failed to drop implied oncurves for %r: %s", glyphName, e) + else: + count += len(dropped) + return count + + +def build_many( + designspace: DesignSpaceDocument, + master_finder=lambda s: s, + exclude=[], + optimize=True, + skip_vf=lambda vf_name: False, + colr_layer_reuse=True, + drop_implied_oncurves=False, +): + """ + Build variable fonts from a designspace file, version 5 which can define + several VFs, or version 4 which has implicitly one VF covering the whole doc. + + If master_finder is set, it should be a callable that takes master + filename as found in designspace file and map it to master font + binary as to be opened (eg. .ttf or .otf). + + skip_vf can be used to skip building some of the variable fonts defined in + the input designspace. It's a predicate that takes as argument the name + of the variable font and returns `bool`. + + Always returns a Dict[str, TTFont] keyed by VariableFontDescriptor.name + """ + res = {} + # varLib.build (used further below) by default only builds an incomplete 'STAT' + # with an empty AxisValueArray--unless the VF inherited 'STAT' from its base master. + # Designspace version 5 can also be used to define 'STAT' labels or customize + # axes ordering, etc. To avoid overwriting a pre-existing 'STAT' or redoing the + # same work twice, here we check if designspace contains any 'STAT' info before + # proceeding to call buildVFStatTable for each VF. + # https://github.com/fonttools/fonttools/pull/3024 + # https://github.com/fonttools/fonttools/issues/3045 + doBuildStatFromDSv5 = ( + "STAT" not in exclude + and designspace.formatTuple >= (5, 0) + and ( + any(a.axisLabels or a.axisOrdering is not None for a in designspace.axes) + or designspace.locationLabels + ) + ) + for _location, subDoc in splitInterpolable(designspace): + for name, vfDoc in splitVariableFonts(subDoc): + if skip_vf(name): + log.debug(f"Skipping variable TTF font: {name}") + continue + vf = build( + vfDoc, + master_finder, + exclude=exclude, + optimize=optimize, + colr_layer_reuse=colr_layer_reuse, + drop_implied_oncurves=drop_implied_oncurves, + )[0] + if doBuildStatFromDSv5: + buildVFStatTable(vf, designspace, name) + res[name] = vf + return res + + +def build( + designspace, + master_finder=lambda s: s, + exclude=[], + optimize=True, + colr_layer_reuse=True, + drop_implied_oncurves=False, +): + """ + Build variation font from a designspace file. + + If master_finder is set, it should be a callable that takes master + filename as found in designspace file and map it to master font + binary as to be opened (eg. .ttf or .otf). + """ + if hasattr(designspace, "sources"): # Assume a DesignspaceDocument + pass + else: # Assume a file path + designspace = DesignSpaceDocument.fromfile(designspace) + + ds = load_designspace(designspace) + log.info("Building variable font") + + log.info("Loading master fonts") + master_fonts = load_masters(designspace, master_finder) + + # TODO: 'master_ttfs' is unused except for return value, remove later + master_ttfs = [] + for master in master_fonts: + try: + master_ttfs.append(master.reader.file.name) + except AttributeError: + master_ttfs.append(None) # in-memory fonts have no path + + if drop_implied_oncurves and "glyf" in master_fonts[ds.base_idx]: + drop_count = drop_implied_oncurve_points(*master_fonts) + log.info( + "Dropped %s on-curve points from simple glyphs in the 'glyf' table", + drop_count, + ) + + # Copy the base master to work from it + vf = deepcopy(master_fonts[ds.base_idx]) + + if "DSIG" in vf: + del vf["DSIG"] + + # Clear USE_MY_METRICS composite flags if set inconsistently across masters. + if "glyf" in vf: + _unset_inconsistent_use_my_metrics_flags(vf, master_fonts) + + # TODO append masters as named-instances as well; needs .designspace change. + fvar = _add_fvar(vf, ds.axes, ds.instances) + if "STAT" not in exclude: + _add_stat(vf) + + # Map from axis names to axis tags... + normalized_master_locs = [ + {ds.axes[k].tag: v for k, v in loc.items()} for loc in ds.normalized_master_locs + ] + # From here on, we use fvar axes only + axisTags = [axis.axisTag for axis in fvar.axes] + + # Assume single-model for now. + model = models.VariationModel(normalized_master_locs, axisOrder=axisTags) + assert 0 == model.mapping[ds.base_idx] + + log.info("Building variations tables") + if "avar" not in exclude: + _add_avar(vf, ds.axes, ds.axisMappings, axisTags) + if "BASE" not in exclude and "BASE" in vf: + _add_BASE(vf, model, master_fonts, axisTags) + if "MVAR" not in exclude: + _add_MVAR(vf, model, master_fonts, axisTags) + if "HVAR" not in exclude: + _add_HVAR(vf, model, master_fonts, axisTags) + if "VVAR" not in exclude and "vmtx" in vf: + _add_VVAR(vf, model, master_fonts, axisTags) + if "GDEF" not in exclude or "GPOS" not in exclude: + _merge_OTL(vf, model, master_fonts, axisTags) + if "gvar" not in exclude and "glyf" in vf: + _add_gvar(vf, model, master_fonts, optimize=optimize) + if "cvar" not in exclude and "glyf" in vf: + _merge_TTHinting(vf, model, master_fonts) + if "GSUB" not in exclude and ds.rules: + featureTags = _feature_variations_tags(ds) + _add_GSUB_feature_variations( + vf, ds.axes, ds.internal_axis_supports, ds.rules, featureTags + ) + if "CFF2" not in exclude and ("CFF " in vf or "CFF2" in vf): + _add_CFF2(vf, model, master_fonts) + if "post" in vf: + # set 'post' to format 2 to keep the glyph names dropped from CFF2 + post = vf["post"] + if post.formatType != 2.0: + post.formatType = 2.0 + post.extraNames = [] + post.mapping = {} + if "COLR" not in exclude and "COLR" in vf and vf["COLR"].version > 0: + _add_COLR(vf, model, master_fonts, axisTags, colr_layer_reuse) + + set_default_weight_width_slant( + vf, location={axis.axisTag: axis.defaultValue for axis in vf["fvar"].axes} + ) + + for tag in exclude: + if tag in vf: + del vf[tag] + + # TODO: Only return vf for 4.0+, the rest is unused. + return vf, model, master_ttfs + + +def _open_font(path, master_finder=lambda s: s): + # load TTFont masters from given 'path': this can be either a .TTX or an + # OpenType binary font; or if neither of these, try use the 'master_finder' + # callable to resolve the path to a valid .TTX or OpenType font binary. + from fontTools.ttx import guessFileType + + master_path = os.path.normpath(path) + tp = guessFileType(master_path) + if tp is None: + # not an OpenType binary/ttx, fall back to the master finder. + master_path = master_finder(master_path) + tp = guessFileType(master_path) + if tp in ("TTX", "OTX"): + font = TTFont() + font.importXML(master_path) + elif tp in ("TTF", "OTF", "WOFF", "WOFF2"): + font = TTFont(master_path) + else: + raise VarLibValidationError("Invalid master path: %r" % master_path) + return font + + +def load_masters(designspace, master_finder=lambda s: s): + """Ensure that all SourceDescriptor.font attributes have an appropriate TTFont + object loaded, or else open TTFont objects from the SourceDescriptor.path + attributes. + + The paths can point to either an OpenType font, a TTX file, or a UFO. In the + latter case, use the provided master_finder callable to map from UFO paths to + the respective master font binaries (e.g. .ttf, .otf or .ttx). + + Return list of master TTFont objects in the same order they are listed in the + DesignSpaceDocument. + """ + for master in designspace.sources: + # If a SourceDescriptor has a layer name, demand that the compiled TTFont + # be supplied by the caller. This spares us from modifying MasterFinder. + if master.layerName and master.font is None: + raise VarLibValidationError( + f"Designspace source '{master.name or '<Unknown>'}' specified a " + "layer name but lacks the required TTFont object in the 'font' " + "attribute." + ) + + return designspace.loadSourceFonts(_open_font, master_finder=master_finder) + + +class MasterFinder(object): + def __init__(self, template): + self.template = template + + def __call__(self, src_path): + fullname = os.path.abspath(src_path) + dirname, basename = os.path.split(fullname) + stem, ext = os.path.splitext(basename) + path = self.template.format( + fullname=fullname, + dirname=dirname, + basename=basename, + stem=stem, + ext=ext, + ) + return os.path.normpath(path) + + +def _feature_variations_tags(ds): + raw_tags = ds.lib.get( + FEAVAR_FEATURETAG_LIB_KEY, + "rclt" if ds.rulesProcessingLast else "rvrn", + ) + return sorted({t.strip() for t in raw_tags.split(",")}) + + +def addGSUBFeatureVariations(vf, designspace, featureTags=(), *, log_enabled=False): + """Add GSUB FeatureVariations table to variable font, based on DesignSpace rules. + + Args: + vf: A TTFont object representing the variable font. + designspace: A DesignSpaceDocument object. + featureTags: Optional feature tag(s) to use for the FeatureVariations records. + If unset, the key 'com.github.fonttools.varLib.featureVarsFeatureTag' is + looked up in the DS <lib> and used; otherwise the default is 'rclt' if + the <rules processing="last"> attribute is set, else 'rvrn'. + See <https://fonttools.readthedocs.io/en/latest/designspaceLib/xml.html#rules-element> + log_enabled: If True, log info about DS axes and sources. Default is False, as + the same info may have already been logged as part of varLib.build. + """ + ds = load_designspace(designspace, log_enabled=log_enabled) + if not ds.rules: + return + if not featureTags: + featureTags = _feature_variations_tags(ds) + _add_GSUB_feature_variations( + vf, ds.axes, ds.internal_axis_supports, ds.rules, featureTags + ) + + +def main(args=None): + """Build variable fonts from a designspace file and masters""" + from argparse import ArgumentParser + from fontTools import configLogger + + parser = ArgumentParser(prog="varLib", description=main.__doc__) + parser.add_argument("designspace") + output_group = parser.add_mutually_exclusive_group() + output_group.add_argument( + "-o", metavar="OUTPUTFILE", dest="outfile", default=None, help="output file" + ) + output_group.add_argument( + "-d", + "--output-dir", + metavar="OUTPUTDIR", + default=None, + help="output dir (default: same as input designspace file)", + ) + parser.add_argument( + "-x", + metavar="TAG", + dest="exclude", + action="append", + default=[], + help="exclude table", + ) + parser.add_argument( + "--disable-iup", + dest="optimize", + action="store_false", + help="do not perform IUP optimization", + ) + parser.add_argument( + "--no-colr-layer-reuse", + dest="colr_layer_reuse", + action="store_false", + help="do not rebuild variable COLR table to optimize COLR layer reuse", + ) + parser.add_argument( + "--drop-implied-oncurves", + action="store_true", + help=( + "drop on-curve points that can be implied when exactly in the middle of " + "two off-curve points (only applies to TrueType fonts)" + ), + ) + parser.add_argument( + "--master-finder", + default="master_ttf_interpolatable/{stem}.ttf", + help=( + "templated string used for finding binary font " + "files given the source file names defined in the " + "designspace document. The following special strings " + "are defined: {fullname} is the absolute source file " + "name; {basename} is the file name without its " + "directory; {stem} is the basename without the file " + "extension; {ext} is the source file extension; " + "{dirname} is the directory of the absolute file " + 'name. The default value is "%(default)s".' + ), + ) + parser.add_argument( + "--variable-fonts", + default=".*", + metavar="VF_NAME", + help=( + "Filter the list of variable fonts produced from the input " + "Designspace v5 file. By default all listed variable fonts are " + "generated. To generate a specific variable font (or variable fonts) " + 'that match a given "name" attribute, you can pass as argument ' + "the full name or a regular expression. E.g.: --variable-fonts " + '"MyFontVF_WeightOnly"; or --variable-fonts "MyFontVFItalic_.*".' + ), + ) + logging_group = parser.add_mutually_exclusive_group(required=False) + logging_group.add_argument( + "-v", "--verbose", action="store_true", help="Run more verbosely." + ) + logging_group.add_argument( + "-q", "--quiet", action="store_true", help="Turn verbosity off." + ) + options = parser.parse_args(args) + + configLogger( + level=("DEBUG" if options.verbose else "ERROR" if options.quiet else "INFO") + ) + + designspace_filename = options.designspace + designspace = DesignSpaceDocument.fromfile(designspace_filename) + + vf_descriptors = designspace.getVariableFonts() + if not vf_descriptors: + parser.error(f"No variable fonts in given designspace {designspace.path!r}") + + vfs_to_build = [] + for vf in vf_descriptors: + # Skip variable fonts that do not match the user's inclusion regex if given. + if not fullmatch(options.variable_fonts, vf.name): + continue + vfs_to_build.append(vf) + + if not vfs_to_build: + parser.error(f"No variable fonts matching {options.variable_fonts!r}") + + if options.outfile is not None and len(vfs_to_build) > 1: + parser.error( + "can't specify -o because there are multiple VFs to build; " + "use --output-dir, or select a single VF with --variable-fonts" + ) + + output_dir = options.output_dir + if output_dir is None: + output_dir = os.path.dirname(designspace_filename) + + vf_name_to_output_path = {} + if len(vfs_to_build) == 1 and options.outfile is not None: + vf_name_to_output_path[vfs_to_build[0].name] = options.outfile + else: + for vf in vfs_to_build: + if vf.filename is not None: + # Only use basename to prevent path traversal attacks + filename = os.path.basename(vf.filename) + else: + filename = vf.name + ".{ext}" + vf_name_to_output_path[vf.name] = os.path.join(output_dir, filename) + + vf_names_to_build = {vf.name for vf in vfs_to_build} + finder = MasterFinder(options.master_finder) + + vfs = build_many( + designspace, + finder, + exclude=options.exclude, + optimize=options.optimize, + skip_vf=lambda name: name not in vf_names_to_build, + colr_layer_reuse=options.colr_layer_reuse, + drop_implied_oncurves=options.drop_implied_oncurves, + ) + + for vf_name, vf in vfs.items(): + ext = "otf" if vf.sfntVersion == "OTTO" else "ttf" + output_path = vf_name_to_output_path[vf_name].format(ext=ext) + output_dir = os.path.dirname(output_path) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + log.info("Saving variation font %s", output_path) + vf.save(output_path) + + +if __name__ == "__main__": + import sys + + if len(sys.argv) > 1: + sys.exit(main()) + import doctest + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/__main__.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..56fab06e0fe6ac22fce428209c373ecb82d8472a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/__main__.py @@ -0,0 +1,6 @@ +import sys +from fontTools.varLib import main + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/__init__.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/__main__.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..1bb27dbfd53902914177ee7fb24a21402b9ae3d3 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/__main__.py @@ -0,0 +1,72 @@ +import logging + +log = logging.getLogger("fontTools.varLib.avar") + + +def main(args=None): + from fontTools.ttLib import TTFont + from fontTools.misc.cliTools import makeOutputFileName + from fontTools import configLogger + import argparse + import sys + + print( + "WARNING: This script is deprecated. Use `fonttools varLib.avar.build` " + "or `fonttools varLib.avar.unbuild` instead.\n", + file=sys.stderr, + ) + + if args is None: + args = sys.argv[1:] + + parser = argparse.ArgumentParser( + "fonttools varLib.avar", + description="Add `avar` table from designspace file to variable font.", + ) + parser.add_argument("font", metavar="varfont.ttf", help="Variable-font file.") + parser.add_argument( + "designspace", + metavar="family.designspace", + help="Designspace file.", + nargs="?", + default=None, + ) + parser.add_argument( + "-o", + "--output-file", + type=str, + help="Output font file name.", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="Run more verbosely." + ) + + options = parser.parse_args(args) + + configLogger(level=("INFO" if options.verbose else "WARNING")) + + font = TTFont(options.font) + + if options.designspace is None: + from .unbuild import unbuild + + unbuild(font) + return 0 + + from .build import build + + build(font, options.designspace) + + if options.output_file is None: + outfile = makeOutputFileName(options.font, overWrite=True, suffix=".avar") + else: + outfile = options.output_file + if outfile: + log.info("Saving %s", outfile) + font.save(outfile) + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/build.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/build.py new file mode 100644 index 0000000000000000000000000000000000000000..d9706705db819c7f02b16995765679bde47de40d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/build.py @@ -0,0 +1,79 @@ +from fontTools.varLib import _add_fvar, _add_avar, load_designspace +from fontTools.ttLib import newTable +import logging + +log = logging.getLogger("fontTools.varLib.avar") + + +def build(font, designspace_file): + ds = load_designspace(designspace_file, require_sources=False) + + if not "fvar" in font: + if "name" not in font: + font["name"] = newTable("name") + _add_fvar(font, ds.axes, ds.instances) + + axisTags = [a.axisTag for a in font["fvar"].axes] + + if "avar" in font: + log.warning("avar table already present, overwriting.") + del font["avar"] + + _add_avar(font, ds.axes, ds.axisMappings, axisTags) + + +def main(args=None): + """Add `avar` table from designspace file to variable font.""" + + from fontTools.ttLib import TTFont + from fontTools.misc.cliTools import makeOutputFileName + from fontTools import configLogger + import argparse + + if args is None: + import sys + + args = sys.argv[1:] + + parser = argparse.ArgumentParser( + "fonttools varLib.avar.build", + description="Add `avar` table from designspace file to variable font.", + ) + parser.add_argument("font", metavar="varfont.ttf", help="Variable-font file.") + parser.add_argument( + "designspace", + metavar="family.designspace", + help="Designspace file.", + default=None, + ) + parser.add_argument( + "-o", + "--output-file", + type=str, + help="Output font file name.", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="Run more verbosely." + ) + + options = parser.parse_args(args) + + configLogger(level=("INFO" if options.verbose else "WARNING")) + + font = TTFont(options.font) + + build(font, options.designspace) + + if options.output_file is None: + outfile = makeOutputFileName(options.font, overWrite=True, suffix=".avar") + else: + outfile = options.output_file + if outfile: + log.info("Saving %s", outfile) + font.save(outfile) + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/map.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/map.py new file mode 100644 index 0000000000000000000000000000000000000000..a5331f109b8ce9606bb9b99f59819e5dcb7f3402 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/map.py @@ -0,0 +1,124 @@ +from fontTools.varLib.models import normalizeValue + + +def _denormalize(v, triplet): + if v >= 0: + return triplet[1] + v * (triplet[2] - triplet[1]) + else: + return triplet[1] + v * (triplet[1] - triplet[0]) + + +def map( + font, location, *, inputNormalized=False, outputNormalized=False, dropZeroes=False +): + if "fvar" not in font: + return None + + fvar = font["fvar"] + axes = {a.axisTag: (a.minValue, a.defaultValue, a.maxValue) for a in fvar.axes} + unknownAxes = sorted(tag for tag in location if tag not in axes) + if unknownAxes: + raise ValueError(f"Unknown axis tag(s): {', '.join(unknownAxes)}") + + if not inputNormalized: + location = { + tag: normalizeValue(value, axes[tag]) for tag, value in location.items() + } + + if "avar" in font: + location = font["avar"].renormalizeLocation(location, font, dropZeroes) + + if not outputNormalized: + location = { + tag: _denormalize(value, axes[tag]) for tag, value in location.items() + } + + return location + + +def main(args=None): + """Map variation coordinates through the `avar` table.""" + + from fontTools.ttLib import TTFont + import argparse + + if args is None: + import sys + + args = sys.argv[1:] + + parser = argparse.ArgumentParser( + "fonttools varLib.avar.map", + description="Map variation coordinates through the `avar` table.", + ) + parser.add_argument("font", metavar="varfont.ttf", help="Variable-font file.") + parser.add_argument( + "coords", + metavar="[AXIS=value...]", + help="Coordinates to map, e.g. 'wght=700 wdth=75'.", + nargs="*", + default=None, + ) + parser.add_argument( + "-f", action="store_true", help="Do not omit axes at default location." + ) + parser.add_argument( + "-i", action="store_true", help="Input coordinates are normalized (-1..1)." + ) + parser.add_argument( + "-o", action="store_true", help="Output coordinates as normalized (-1..1)." + ) + + options = parser.parse_args(args) + + if not options.coords: + parser.error( + "No coordinates provided. Please specify at least one axis coordinate (e.g., wght=500)" + ) + + if options.font.endswith(".designspace"): + from .build import build + + font = TTFont() + build(font, options.font) + else: + font = TTFont(options.font) + if "fvar" not in font: + parser.error(f"Font '{options.font}' does not contain an 'fvar' table.") + + location = {} + for item in options.coords: + tag, sep, value = item.partition("=") + if not sep or not tag or not value: + parser.error( + f"Invalid coordinate {item!r}. Expected AXIS=value, e.g. wght=500" + ) + try: + location[tag] = float(value) + except ValueError: + parser.error( + f"Invalid coordinate value in {item!r}. Expected a number after '='" + ) + + try: + mapped = map( + font, + location, + inputNormalized=options.i, + outputNormalized=options.o, + dropZeroes=not options.f, + ) + except ValueError as e: + parser.error(str(e)) + assert mapped is not None + + for tag in mapped: + v = mapped[tag] + v = int(v) if v == int(v) else v + print(f"{tag}={v:g}") + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/plan.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/plan.py new file mode 100644 index 0000000000000000000000000000000000000000..c211dd24626560f2a60386391098e427b616aa6f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/plan.py @@ -0,0 +1,1004 @@ +from fontTools.ttLib import newTable +from fontTools.ttLib.tables._f_v_a_r import Axis as fvarAxis +from fontTools.pens.areaPen import AreaPen +from fontTools.pens.basePen import NullPen +from fontTools.pens.statisticsPen import StatisticsPen +from fontTools.varLib.models import piecewiseLinearMap, normalizeValue +from fontTools.misc.cliTools import makeOutputFileName +import math +import logging +from pprint import pformat + +__all__ = [ + "planWeightAxis", + "planWidthAxis", + "planSlantAxis", + "planOpticalSizeAxis", + "planAxis", + "sanitizeWeight", + "sanitizeWidth", + "sanitizeSlant", + "measureWeight", + "measureWidth", + "measureSlant", + "normalizeLinear", + "normalizeLog", + "normalizeDegrees", + "interpolateLinear", + "interpolateLog", + "processAxis", + "makeDesignspaceSnippet", + "addEmptyAvar", + "main", +] + +log = logging.getLogger("fontTools.varLib.avar.plan") + +WEIGHTS = [ + 50, + 100, + 150, + 200, + 250, + 300, + 350, + 400, + 450, + 500, + 550, + 600, + 650, + 700, + 750, + 800, + 850, + 900, + 950, +] + +WIDTHS = [ + 25.0, + 37.5, + 50.0, + 62.5, + 75.0, + 87.5, + 100.0, + 112.5, + 125.0, + 137.5, + 150.0, + 162.5, + 175.0, + 187.5, + 200.0, +] + +SLANTS = list(math.degrees(math.atan(d / 20.0)) for d in range(-20, 21)) + +SIZES = [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 14, + 18, + 24, + 30, + 36, + 48, + 60, + 72, + 96, + 120, + 144, + 192, + 240, + 288, +] + + +SAMPLES = 8 + + +def normalizeLinear(value, rangeMin, rangeMax): + """Linearly normalize value in [rangeMin, rangeMax] to [0, 1], with extrapolation.""" + return (value - rangeMin) / (rangeMax - rangeMin) + + +def interpolateLinear(t, a, b): + """Linear interpolation between a and b, with t typically in [0, 1].""" + return a + t * (b - a) + + +def normalizeLog(value, rangeMin, rangeMax): + """Logarithmically normalize value in [rangeMin, rangeMax] to [0, 1], with extrapolation.""" + logMin = math.log(rangeMin) + logMax = math.log(rangeMax) + return (math.log(value) - logMin) / (logMax - logMin) + + +def interpolateLog(t, a, b): + """Logarithmic interpolation between a and b, with t typically in [0, 1].""" + logA = math.log(a) + logB = math.log(b) + return math.exp(logA + t * (logB - logA)) + + +def normalizeDegrees(value, rangeMin, rangeMax): + """Angularly normalize value in [rangeMin, rangeMax] to [0, 1], with extrapolation.""" + tanMin = math.tan(math.radians(rangeMin)) + tanMax = math.tan(math.radians(rangeMax)) + return (math.tan(math.radians(value)) - tanMin) / (tanMax - tanMin) + + +def measureWeight(glyphset, glyphs=None): + """Measure the perceptual average weight of the given glyphs.""" + if isinstance(glyphs, dict): + frequencies = glyphs + else: + frequencies = {g: 1 for g in glyphs} + + wght_sum = wdth_sum = 0 + for glyph_name in glyphs: + if frequencies is not None: + frequency = frequencies.get(glyph_name, 0) + if frequency == 0: + continue + else: + frequency = 1 + + glyph = glyphset[glyph_name] + + pen = AreaPen(glyphset=glyphset) + glyph.draw(pen) + + mult = glyph.width * frequency + wght_sum += mult * abs(pen.value) + wdth_sum += mult + + return wght_sum / wdth_sum + + +def measureWidth(glyphset, glyphs=None): + """Measure the average width of the given glyphs.""" + if isinstance(glyphs, dict): + frequencies = glyphs + else: + frequencies = {g: 1 for g in glyphs} + + wdth_sum = 0 + freq_sum = 0 + for glyph_name in glyphs: + if frequencies is not None: + frequency = frequencies.get(glyph_name, 0) + if frequency == 0: + continue + else: + frequency = 1 + + glyph = glyphset[glyph_name] + + pen = NullPen() + glyph.draw(pen) + + wdth_sum += glyph.width * frequency + freq_sum += frequency + + return wdth_sum / freq_sum + + +def measureSlant(glyphset, glyphs=None): + """Measure the perceptual average slant angle of the given glyphs.""" + if isinstance(glyphs, dict): + frequencies = glyphs + else: + frequencies = {g: 1 for g in glyphs} + + slnt_sum = 0 + freq_sum = 0 + for glyph_name in glyphs: + if frequencies is not None: + frequency = frequencies.get(glyph_name, 0) + if frequency == 0: + continue + else: + frequency = 1 + + glyph = glyphset[glyph_name] + + pen = StatisticsPen(glyphset=glyphset) + glyph.draw(pen) + + mult = glyph.width * frequency + slnt_sum += mult * pen.slant + freq_sum += mult + + return -math.degrees(math.atan(slnt_sum / freq_sum)) + + +def sanitizeWidth(userTriple, designTriple, pins, measurements): + """Sanitize the width axis limits.""" + + minVal, defaultVal, maxVal = ( + measurements[designTriple[0]], + measurements[designTriple[1]], + measurements[designTriple[2]], + ) + + calculatedMinVal = userTriple[1] * (minVal / defaultVal) + calculatedMaxVal = userTriple[1] * (maxVal / defaultVal) + + log.info("Original width axis limits: %g:%g:%g", *userTriple) + log.info( + "Calculated width axis limits: %g:%g:%g", + calculatedMinVal, + userTriple[1], + calculatedMaxVal, + ) + + if ( + abs(calculatedMinVal - userTriple[0]) / userTriple[1] > 0.05 + or abs(calculatedMaxVal - userTriple[2]) / userTriple[1] > 0.05 + ): + log.warning("Calculated width axis min/max do not match user input.") + log.warning( + " Current width axis limits: %g:%g:%g", + *userTriple, + ) + log.warning( + " Suggested width axis limits: %g:%g:%g", + calculatedMinVal, + userTriple[1], + calculatedMaxVal, + ) + + return False + + return True + + +def sanitizeWeight(userTriple, designTriple, pins, measurements): + """Sanitize the weight axis limits.""" + + if len(set(userTriple)) < 3: + return True + + minVal, defaultVal, maxVal = ( + measurements[designTriple[0]], + measurements[designTriple[1]], + measurements[designTriple[2]], + ) + + logMin = math.log(minVal) + logDefault = math.log(defaultVal) + logMax = math.log(maxVal) + + t = (userTriple[1] - userTriple[0]) / (userTriple[2] - userTriple[0]) + y = math.exp(logMin + t * (logMax - logMin)) + t = (y - minVal) / (maxVal - minVal) + calculatedDefaultVal = userTriple[0] + t * (userTriple[2] - userTriple[0]) + + log.info("Original weight axis limits: %g:%g:%g", *userTriple) + log.info( + "Calculated weight axis limits: %g:%g:%g", + userTriple[0], + calculatedDefaultVal, + userTriple[2], + ) + + if abs(calculatedDefaultVal - userTriple[1]) / userTriple[1] > 0.05: + log.warning("Calculated weight axis default does not match user input.") + + log.warning( + " Current weight axis limits: %g:%g:%g", + *userTriple, + ) + + log.warning( + " Suggested weight axis limits, changing default: %g:%g:%g", + userTriple[0], + calculatedDefaultVal, + userTriple[2], + ) + + t = (userTriple[2] - userTriple[0]) / (userTriple[1] - userTriple[0]) + y = math.exp(logMin + t * (logDefault - logMin)) + t = (y - minVal) / (defaultVal - minVal) + calculatedMaxVal = userTriple[0] + t * (userTriple[1] - userTriple[0]) + log.warning( + " Suggested weight axis limits, changing maximum: %g:%g:%g", + userTriple[0], + userTriple[1], + calculatedMaxVal, + ) + + t = (userTriple[0] - userTriple[2]) / (userTriple[1] - userTriple[2]) + y = math.exp(logMax + t * (logDefault - logMax)) + t = (y - maxVal) / (defaultVal - maxVal) + calculatedMinVal = userTriple[2] + t * (userTriple[1] - userTriple[2]) + log.warning( + " Suggested weight axis limits, changing minimum: %g:%g:%g", + calculatedMinVal, + userTriple[1], + userTriple[2], + ) + + return False + + return True + + +def sanitizeSlant(userTriple, designTriple, pins, measurements): + """Sanitize the slant axis limits.""" + + log.info("Original slant axis limits: %g:%g:%g", *userTriple) + log.info( + "Calculated slant axis limits: %g:%g:%g", + measurements[designTriple[0]], + measurements[designTriple[1]], + measurements[designTriple[2]], + ) + + if ( + abs(measurements[designTriple[0]] - userTriple[0]) > 1 + or abs(measurements[designTriple[1]] - userTriple[1]) > 1 + or abs(measurements[designTriple[2]] - userTriple[2]) > 1 + ): + log.warning("Calculated slant axis min/default/max do not match user input.") + log.warning( + " Current slant axis limits: %g:%g:%g", + *userTriple, + ) + log.warning( + " Suggested slant axis limits: %g:%g:%g", + measurements[designTriple[0]], + measurements[designTriple[1]], + measurements[designTriple[2]], + ) + + return False + + return True + + +def planAxis( + measureFunc, + normalizeFunc, + interpolateFunc, + glyphSetFunc, + axisTag, + axisLimits, + values, + samples=None, + glyphs=None, + designLimits=None, + pins=None, + sanitizeFunc=None, +): + """Plan an axis. + + measureFunc: callable that takes a glyphset and an optional + list of glyphnames, and returns the glyphset-wide measurement + to be used for the axis. + + normalizeFunc: callable that takes a measurement and a minimum + and maximum, and normalizes the measurement into the range 0..1, + possibly extrapolating too. + + interpolateFunc: callable that takes a normalized t value, and a + minimum and maximum, and returns the interpolated value, + possibly extrapolating too. + + glyphSetFunc: callable that takes a variations "location" dictionary, + and returns a glyphset. + + axisTag: the axis tag string. + + axisLimits: a triple of minimum, default, and maximum values for + the axis. Or an `fvar` Axis object. + + values: a list of output values to map for this axis. + + samples: the number of samples to use when sampling. Default 8. + + glyphs: a list of glyph names to use when sampling. Defaults to None, + which will process all glyphs. + + designLimits: an optional triple of minimum, default, and maximum values + represenging the "design" limits for the axis. If not provided, the + axisLimits will be used. + + pins: an optional dictionary of before/after mapping entries to pin in + the output. + + sanitizeFunc: an optional callable to call to sanitize the axis limits. + """ + + if isinstance(axisLimits, fvarAxis): + axisLimits = (axisLimits.minValue, axisLimits.defaultValue, axisLimits.maxValue) + minValue, defaultValue, maxValue = axisLimits + + if samples is None: + samples = SAMPLES + if glyphs is None: + glyphs = glyphSetFunc({}).keys() + if pins is None: + pins = {} + else: + pins = pins.copy() + + log.info( + "Axis limits min %g / default %g / max %g", minValue, defaultValue, maxValue + ) + triple = (minValue, defaultValue, maxValue) + + if designLimits is not None: + log.info("Axis design-limits min %g / default %g / max %g", *designLimits) + else: + designLimits = triple + + if pins: + log.info("Pins %s", sorted(pins.items())) + pins.update( + { + minValue: designLimits[0], + defaultValue: designLimits[1], + maxValue: designLimits[2], + } + ) + + out = {} + outNormalized = {} + + axisMeasurements = {} + for value in sorted({minValue, defaultValue, maxValue} | set(pins.keys())): + glyphset = glyphSetFunc(location={axisTag: value}) + designValue = pins[value] + axisMeasurements[designValue] = measureFunc(glyphset, glyphs) + + if sanitizeFunc is not None: + log.info("Sanitizing axis limit values for the `%s` axis.", axisTag) + sanitizeFunc(triple, designLimits, pins, axisMeasurements) + + log.debug("Calculated average value:\n%s", pformat(axisMeasurements)) + + for (rangeMin, targetMin), (rangeMax, targetMax) in zip( + list(sorted(pins.items()))[:-1], + list(sorted(pins.items()))[1:], + ): + targetValues = {w for w in values if rangeMin < w < rangeMax} + if not targetValues: + continue + + normalizedMin = normalizeValue(rangeMin, triple) + normalizedMax = normalizeValue(rangeMax, triple) + normalizedTargetMin = normalizeValue(targetMin, designLimits) + normalizedTargetMax = normalizeValue(targetMax, designLimits) + + log.info("Planning target values %s.", sorted(targetValues)) + log.info("Sampling %u points in range %g,%g.", samples, rangeMin, rangeMax) + valueMeasurements = axisMeasurements.copy() + for sample in range(1, samples + 1): + value = rangeMin + (rangeMax - rangeMin) * sample / (samples + 1) + log.debug("Sampling value %g.", value) + glyphset = glyphSetFunc(location={axisTag: value}) + designValue = piecewiseLinearMap(value, pins) + valueMeasurements[designValue] = measureFunc(glyphset, glyphs) + log.debug("Sampled average value:\n%s", pformat(valueMeasurements)) + + measurementValue = {} + for value in sorted(valueMeasurements): + measurementValue[valueMeasurements[value]] = value + + out[rangeMin] = targetMin + outNormalized[normalizedMin] = normalizedTargetMin + for value in sorted(targetValues): + t = normalizeFunc(value, rangeMin, rangeMax) + targetMeasurement = interpolateFunc( + t, valueMeasurements[targetMin], valueMeasurements[targetMax] + ) + targetValue = piecewiseLinearMap(targetMeasurement, measurementValue) + log.debug("Planned mapping value %g to %g." % (value, targetValue)) + out[value] = targetValue + valueNormalized = normalizedMin + (value - rangeMin) / ( + rangeMax - rangeMin + ) * (normalizedMax - normalizedMin) + outNormalized[valueNormalized] = normalizedTargetMin + ( + targetValue - targetMin + ) / (targetMax - targetMin) * (normalizedTargetMax - normalizedTargetMin) + out[rangeMax] = targetMax + outNormalized[normalizedMax] = normalizedTargetMax + + log.info("Planned mapping for the `%s` axis:\n%s", axisTag, pformat(out)) + log.info( + "Planned normalized mapping for the `%s` axis:\n%s", + axisTag, + pformat(outNormalized), + ) + + if all(abs(k - v) < 0.01 for k, v in outNormalized.items()): + log.info("Detected identity mapping for the `%s` axis. Dropping.", axisTag) + out = {} + outNormalized = {} + + return out, outNormalized + + +def planWeightAxis( + glyphSetFunc, + axisLimits, + weights=None, + samples=None, + glyphs=None, + designLimits=None, + pins=None, + sanitize=False, +): + """Plan a weight (`wght`) axis. + + weights: A list of weight values to plan for. If None, the default + values are used. + + This function simply calls planAxis with values=weights, and the appropriate + arguments. See documenation for planAxis for more information. + """ + + if weights is None: + weights = WEIGHTS + + return planAxis( + measureWeight, + normalizeLinear, + interpolateLog, + glyphSetFunc, + "wght", + axisLimits, + values=weights, + samples=samples, + glyphs=glyphs, + designLimits=designLimits, + pins=pins, + sanitizeFunc=sanitizeWeight if sanitize else None, + ) + + +def planWidthAxis( + glyphSetFunc, + axisLimits, + widths=None, + samples=None, + glyphs=None, + designLimits=None, + pins=None, + sanitize=False, +): + """Plan a width (`wdth`) axis. + + widths: A list of width values (percentages) to plan for. If None, the default + values are used. + + This function simply calls planAxis with values=widths, and the appropriate + arguments. See documenation for planAxis for more information. + """ + + if widths is None: + widths = WIDTHS + + return planAxis( + measureWidth, + normalizeLinear, + interpolateLinear, + glyphSetFunc, + "wdth", + axisLimits, + values=widths, + samples=samples, + glyphs=glyphs, + designLimits=designLimits, + pins=pins, + sanitizeFunc=sanitizeWidth if sanitize else None, + ) + + +def planSlantAxis( + glyphSetFunc, + axisLimits, + slants=None, + samples=None, + glyphs=None, + designLimits=None, + pins=None, + sanitize=False, +): + """Plan a slant (`slnt`) axis. + + slants: A list slant angles to plan for. If None, the default + values are used. + + This function simply calls planAxis with values=slants, and the appropriate + arguments. See documenation for planAxis for more information. + """ + + if slants is None: + slants = SLANTS + + return planAxis( + measureSlant, + normalizeDegrees, + interpolateLinear, + glyphSetFunc, + "slnt", + axisLimits, + values=slants, + samples=samples, + glyphs=glyphs, + designLimits=designLimits, + pins=pins, + sanitizeFunc=sanitizeSlant if sanitize else None, + ) + + +def planOpticalSizeAxis( + glyphSetFunc, + axisLimits, + sizes=None, + samples=None, + glyphs=None, + designLimits=None, + pins=None, + sanitize=False, +): + """Plan a optical-size (`opsz`) axis. + + sizes: A list of optical size values to plan for. If None, the default + values are used. + + This function simply calls planAxis with values=sizes, and the appropriate + arguments. See documenation for planAxis for more information. + """ + + if sizes is None: + sizes = SIZES + + return planAxis( + measureWeight, + normalizeLog, + interpolateLog, + glyphSetFunc, + "opsz", + axisLimits, + values=sizes, + samples=samples, + glyphs=glyphs, + designLimits=designLimits, + pins=pins, + ) + + +def makeDesignspaceSnippet(axisTag, axisName, axisLimit, mapping): + """Make a designspace snippet for a single axis.""" + + designspaceSnippet = ( + ' <axis tag="%s" name="%s" minimum="%g" default="%g" maximum="%g"' + % ((axisTag, axisName) + axisLimit) + ) + if mapping: + designspaceSnippet += ">\n" + else: + designspaceSnippet += "/>" + + for key, value in mapping.items(): + designspaceSnippet += ' <map input="%g" output="%g"/>\n' % (key, value) + + if mapping: + designspaceSnippet += " </axis>" + + return designspaceSnippet + + +def addEmptyAvar(font): + """Add an empty `avar` table to the font.""" + font["avar"] = avar = newTable("avar") + for axis in font["fvar"].axes: + avar.segments[axis.axisTag] = {} + + +def processAxis( + font, + planFunc, + axisTag, + axisName, + values, + samples=None, + glyphs=None, + designLimits=None, + pins=None, + sanitize=False, + plot=False, +): + """Process a single axis.""" + + axisLimits = None + for axis in font["fvar"].axes: + if axis.axisTag == axisTag: + axisLimits = axis + break + if axisLimits is None: + return "" + axisLimits = (axisLimits.minValue, axisLimits.defaultValue, axisLimits.maxValue) + + log.info("Planning %s axis.", axisName) + + if "avar" in font: + existingMapping = font["avar"].segments[axisTag] + font["avar"].segments[axisTag] = {} + else: + existingMapping = None + + if values is not None and isinstance(values, str): + values = [float(w) for w in values.split()] + + if designLimits is not None and isinstance(designLimits, str): + designLimits = [float(d) for d in designLimits.split(":")] + assert ( + len(designLimits) == 3 + and designLimits[0] <= designLimits[1] <= designLimits[2] + ) + else: + designLimits = None + + if pins is not None and isinstance(pins, str): + newPins = {} + for pin in pins.split(): + before, after = pin.split(":") + newPins[float(before)] = float(after) + pins = newPins + del newPins + + mapping, mappingNormalized = planFunc( + font.getGlyphSet, + axisLimits, + values, + samples=samples, + glyphs=glyphs, + designLimits=designLimits, + pins=pins, + sanitize=sanitize, + ) + + if plot: + from matplotlib import pyplot + + pyplot.plot( + sorted(mappingNormalized), + [mappingNormalized[k] for k in sorted(mappingNormalized)], + ) + pyplot.show() + + if existingMapping is not None: + log.info("Existing %s mapping:\n%s", axisName, pformat(existingMapping)) + + if mapping: + if "avar" not in font: + addEmptyAvar(font) + font["avar"].segments[axisTag] = mappingNormalized + else: + if "avar" in font: + font["avar"].segments[axisTag] = {} + + designspaceSnippet = makeDesignspaceSnippet( + axisTag, + axisName, + axisLimits, + mapping, + ) + return designspaceSnippet + + +def main(args=None): + """Plan the standard axis mappings for a variable font""" + + if args is None: + import sys + + args = sys.argv[1:] + + from fontTools import configLogger + from fontTools.ttLib import TTFont + import argparse + + parser = argparse.ArgumentParser( + "fonttools varLib.avar.plan", + description="Plan `avar` table for variable font", + ) + parser.add_argument("font", metavar="varfont.ttf", help="Variable-font file.") + parser.add_argument( + "-o", + "--output-file", + type=str, + help="Output font file name.", + ) + parser.add_argument( + "--weights", type=str, help="Space-separate list of weights to generate." + ) + parser.add_argument( + "--widths", type=str, help="Space-separate list of widths to generate." + ) + parser.add_argument( + "--slants", type=str, help="Space-separate list of slants to generate." + ) + parser.add_argument( + "--sizes", type=str, help="Space-separate list of optical-sizes to generate." + ) + parser.add_argument("--samples", type=int, help="Number of samples.") + parser.add_argument( + "-s", "--sanitize", action="store_true", help="Sanitize axis limits" + ) + parser.add_argument( + "-g", + "--glyphs", + type=str, + help="Space-separate list of glyphs to use for sampling.", + ) + parser.add_argument( + "--weight-design-limits", + type=str, + help="min:default:max in design units for the `wght` axis.", + ) + parser.add_argument( + "--width-design-limits", + type=str, + help="min:default:max in design units for the `wdth` axis.", + ) + parser.add_argument( + "--slant-design-limits", + type=str, + help="min:default:max in design units for the `slnt` axis.", + ) + parser.add_argument( + "--optical-size-design-limits", + type=str, + help="min:default:max in design units for the `opsz` axis.", + ) + parser.add_argument( + "--weight-pins", + type=str, + help="Space-separate list of before:after pins for the `wght` axis.", + ) + parser.add_argument( + "--width-pins", + type=str, + help="Space-separate list of before:after pins for the `wdth` axis.", + ) + parser.add_argument( + "--slant-pins", + type=str, + help="Space-separate list of before:after pins for the `slnt` axis.", + ) + parser.add_argument( + "--optical-size-pins", + type=str, + help="Space-separate list of before:after pins for the `opsz` axis.", + ) + parser.add_argument( + "-p", "--plot", action="store_true", help="Plot the resulting mapping." + ) + + logging_group = parser.add_mutually_exclusive_group(required=False) + logging_group.add_argument( + "-v", "--verbose", action="store_true", help="Run more verbosely." + ) + logging_group.add_argument( + "-q", "--quiet", action="store_true", help="Turn verbosity off." + ) + + options = parser.parse_args(args) + + configLogger( + level=("DEBUG" if options.verbose else "WARNING" if options.quiet else "INFO") + ) + + font = TTFont(options.font) + if not "fvar" in font: + log.error("Not a variable font.") + return 1 + + if options.glyphs is not None: + glyphs = options.glyphs.split() + if ":" in options.glyphs: + glyphs = {} + for g in options.glyphs.split(): + if ":" in g: + glyph, frequency = g.split(":") + glyphs[glyph] = float(frequency) + else: + glyphs[g] = 1.0 + else: + glyphs = None + + designspaceSnippets = [] + + designspaceSnippets.append( + processAxis( + font, + planWeightAxis, + "wght", + "Weight", + values=options.weights, + samples=options.samples, + glyphs=glyphs, + designLimits=options.weight_design_limits, + pins=options.weight_pins, + sanitize=options.sanitize, + plot=options.plot, + ) + ) + designspaceSnippets.append( + processAxis( + font, + planWidthAxis, + "wdth", + "Width", + values=options.widths, + samples=options.samples, + glyphs=glyphs, + designLimits=options.width_design_limits, + pins=options.width_pins, + sanitize=options.sanitize, + plot=options.plot, + ) + ) + designspaceSnippets.append( + processAxis( + font, + planSlantAxis, + "slnt", + "Slant", + values=options.slants, + samples=options.samples, + glyphs=glyphs, + designLimits=options.slant_design_limits, + pins=options.slant_pins, + sanitize=options.sanitize, + plot=options.plot, + ) + ) + designspaceSnippets.append( + processAxis( + font, + planOpticalSizeAxis, + "opsz", + "OpticalSize", + values=options.sizes, + samples=options.samples, + glyphs=glyphs, + designLimits=options.optical_size_design_limits, + pins=options.optical_size_pins, + sanitize=options.sanitize, + plot=options.plot, + ) + ) + + log.info("Designspace snippet:") + for snippet in designspaceSnippets: + if snippet: + print(snippet) + + if options.output_file is None: + outfile = makeOutputFileName(options.font, overWrite=True, suffix=".avar") + else: + outfile = options.output_file + if outfile: + log.info("Saving %s", outfile) + font.save(outfile) + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/unbuild.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/unbuild.py new file mode 100644 index 0000000000000000000000000000000000000000..ddf677902cea837c5cbf3ba37a6b3f0847e72bbc --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/avar/unbuild.py @@ -0,0 +1,274 @@ +from fontTools.varLib.models import VariationModel +from fontTools.varLib.varStore import VarStoreInstancer +from fontTools.misc.fixedTools import fixedToFloat as fi2fl +from itertools import product +import sys + + +def _denormalize(v, axis): + if v >= 0: + return axis.defaultValue + v * (axis.maxValue - axis.defaultValue) + else: + return axis.defaultValue + v * (axis.defaultValue - axis.minValue) + + +def _pruneLocations(locations, poles, axisTags): + # Now we have all the input locations, find which ones are + # not needed and remove them. + + # Note: This algorithm is heavily tied to how VariationModel + # is implemented. It assumes that input was extracted from + # VariationModel-generated object, like an ItemVariationStore + # created by fontmake using varLib.models.VariationModel. + # Some CoPilot blabbering: + # I *think* I can prove that this algorithm is correct, but + # I'm not 100% sure. It's possible that there are edge cases + # where this algorithm will fail. I'm not sure how to prove + # that it's correct, but I'm also not sure how to prove that + # it's incorrect. I'm not sure how to write a test case that + # would prove that it's incorrect. I'm not sure how to write + # a test case that would prove that it's correct. + + model = VariationModel(locations, axisTags) + modelMapping = model.mapping + modelSupports = model.supports + pins = {tuple(k.items()): None for k in poles} + for location in poles: + i = locations.index(location) + i = modelMapping[i] + support = modelSupports[i] + supportAxes = set(support.keys()) + for axisTag, (minV, _, maxV) in support.items(): + for v in (minV, maxV): + if v in (-1, 0, 1): + continue + for pin in pins.keys(): + pinLocation = dict(pin) + pinAxes = set(pinLocation.keys()) + if pinAxes != supportAxes: + continue + if axisTag not in pinAxes: + continue + if pinLocation[axisTag] == v: + break + else: + # No pin found. Go through the previous masters + # and find a suitable pin. Going backwards is + # better because it can find a pin that is close + # to the pole in more dimensions, and reducing + # the total number of pins needed. + for candidateIdx in range(i - 1, -1, -1): + candidate = modelSupports[candidateIdx] + candidateAxes = set(candidate.keys()) + if candidateAxes != supportAxes: + continue + if axisTag not in candidateAxes: + continue + candidate = { + k: defaultV for k, (_, defaultV, _) in candidate.items() + } + if candidate[axisTag] == v: + pins[tuple(candidate.items())] = None + break + else: + assert False, "No pin found" + return [dict(t) for t in pins.keys()] + + +def mappings_from_avar(font, denormalize=True): + fvarAxes = font["fvar"].axes + axisMap = {a.axisTag: a for a in fvarAxes} + axisTags = [a.axisTag for a in fvarAxes] + axisIndexes = {a.axisTag: i for i, a in enumerate(fvarAxes)} + if "avar" not in font: + return {}, [] + avar = font["avar"] + axisMaps = { + tag: seg + for tag, seg in avar.segments.items() + if seg and seg != {-1: -1, 0: 0, 1: 1} + } + mappings = [] + + if getattr(avar, "majorVersion", 1) == 2: + varStore = avar.table.VarStore + regions = varStore.VarRegionList.Region + + # Find all the input locations; this finds "poles", that are + # locations of the peaks, and "corners", that are locations + # of the corners of the regions. These two sets of locations + # together constitute inputLocations to consider. + + poles = {(): None} # Just using it as an ordered set + inputLocations = set({()}) + for varData in varStore.VarData: + regionIndices = varData.VarRegionIndex + for regionIndex in regionIndices: + peakLocation = [] + corners = [] + region = regions[regionIndex] + for axisIndex, axis in enumerate(region.VarRegionAxis): + if axis.PeakCoord == 0: + continue + axisTag = axisTags[axisIndex] + peakLocation.append((axisTag, axis.PeakCoord)) + corner = [] + if axis.StartCoord != 0: + corner.append((axisTag, axis.StartCoord)) + if axis.EndCoord != 0: + corner.append((axisTag, axis.EndCoord)) + corners.append(corner) + corners = set(product(*corners)) + peakLocation = tuple(peakLocation) + poles[peakLocation] = None + inputLocations.add(peakLocation) + inputLocations.update(corners) + + # Sort them by number of axes, then by axis order + inputLocations = [ + dict(t) + for t in sorted( + inputLocations, + key=lambda t: (len(t), tuple(axisIndexes[tag] for tag, _ in t)), + ) + ] + poles = [dict(t) for t in poles.keys()] + inputLocations = _pruneLocations(inputLocations, list(poles), axisTags) + + # Find the output locations, at input locations + varIdxMap = avar.table.VarIdxMap + instancer = VarStoreInstancer(varStore, fvarAxes) + for location in inputLocations: + instancer.setLocation(location) + outputLocation = {} + for axisIndex, axisTag in enumerate(axisTags): + varIdx = axisIndex + if varIdxMap is not None: + varIdx = varIdxMap[varIdx] + delta = instancer[varIdx] + if delta != 0: + v = location.get(axisTag, 0) + v = v + fi2fl(delta, 14) + # See https://github.com/fonttools/fonttools/pull/3598#issuecomment-2266082009 + # v = max(-1, min(1, v)) + outputLocation[axisTag] = v + mappings.append((location, outputLocation)) + + # Remove base master we added, if it maps to the default location + assert mappings[0][0] == {} + if mappings[0][1] == {}: + mappings.pop(0) + + if denormalize: + for tag, seg in axisMaps.items(): + if tag not in axisMap: + raise ValueError(f"Unknown axis tag {tag}") + denorm = lambda v: _denormalize(v, axisMap[tag]) + axisMaps[tag] = {denorm(k): denorm(v) for k, v in seg.items()} + + for i, (inputLoc, outputLoc) in enumerate(mappings): + inputLoc = { + tag: _denormalize(val, axisMap[tag]) for tag, val in inputLoc.items() + } + outputLoc = { + tag: _denormalize(val, axisMap[tag]) for tag, val in outputLoc.items() + } + mappings[i] = (inputLoc, outputLoc) + + return axisMaps, mappings + + +def unbuild(font, f=sys.stdout): + fvar = font["fvar"] + axes = fvar.axes + segments, mappings = mappings_from_avar(font) + + if "name" in font: + name = font["name"] + axisNames = { + axis.axisTag: name.getDebugName(axis.axisNameID) or axis.axisTag + for axis in axes + } + else: + axisNames = {a.axisTag: a.axisTag for a in axes} + + print("<?xml version='1.0' encoding='UTF-8'?>", file=f) + print('<designspace format="5.1">', file=f) + print(" <axes>", file=f) + for axis in axes: + + axisName = axisNames[axis.axisTag] + + triplet = (axis.minValue, axis.defaultValue, axis.maxValue) + triplet = [int(v) if v == int(v) else v for v in triplet] + + axisMap = segments.get(axis.axisTag) + closing = "/>" if axisMap is None else ">" + + print( + f' <axis tag="{axis.axisTag}" name="{axisName}" minimum="{triplet[0]}" maximum="{triplet[2]}" default="{triplet[1]}"{closing}', + file=f, + ) + if axisMap is not None: + for k in sorted(axisMap.keys()): + v = axisMap[k] + k = int(k) if k == int(k) else k + v = int(v) if v == int(v) else v + print(f' <map input="{k}" output="{v}"/>', file=f) + print(" </axis>", file=f) + if mappings: + print(" <mappings>", file=f) + for inputLoc, outputLoc in mappings: + print(" <mapping>", file=f) + print(" <input>", file=f) + for tag in sorted(inputLoc.keys()): + v = inputLoc[tag] + v = int(v) if v == int(v) else v + print( + f' <dimension name="{axisNames[tag]}" xvalue="{v}"/>', + file=f, + ) + print(" </input>", file=f) + print(" <output>", file=f) + for tag in sorted(outputLoc.keys()): + v = outputLoc[tag] + v = int(v) if v == int(v) else v + print( + f' <dimension name="{axisNames[tag]}" xvalue="{v}"/>', + file=f, + ) + print(" </output>", file=f) + print(" </mapping>", file=f) + print(" </mappings>", file=f) + print(" </axes>", file=f) + print("</designspace>", file=f) + + +def main(args=None): + """Print `avar` table as a designspace snippet.""" + + if args is None: + args = sys.argv[1:] + + from fontTools.ttLib import TTFont + import argparse + + parser = argparse.ArgumentParser( + "fonttools varLib.avar.unbuild", + description="Print `avar` table as a designspace snippet.", + ) + parser.add_argument("font", metavar="varfont.ttf", help="Variable-font file.") + options = parser.parse_args(args) + + font = TTFont(options.font) + if "fvar" not in font: + print("Not a variable font.", file=sys.stderr) + return 1 + + unbuild(font) + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/avarPlanner.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/avarPlanner.py new file mode 100644 index 0000000000000000000000000000000000000000..1fc63b1c1b32cc04151c06708d28cb3d75f0b978 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/avarPlanner.py @@ -0,0 +1,8 @@ +def main(args=None): + from .avar.plan import main + + main(args) + + +if __name__ == "__main__": + main() diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/builder.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..456c34c4dd903d1ba7704c46bfb8e2af212cd0a4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/builder.py @@ -0,0 +1,215 @@ +from fontTools import ttLib +from fontTools.ttLib.tables import otTables as ot + +# VariationStore + + +def buildVarRegionAxis(axisSupport): + self = ot.VarRegionAxis() + self.StartCoord, self.PeakCoord, self.EndCoord = [float(v) for v in axisSupport] + return self + + +def buildSparseVarRegionAxis(axisIndex, axisSupport): + self = ot.SparseVarRegionAxis() + self.AxisIndex = axisIndex + self.StartCoord, self.PeakCoord, self.EndCoord = [float(v) for v in axisSupport] + return self + + +def buildVarRegion(support, axisTags): + assert all(tag in axisTags for tag in support.keys()), ( + "Unknown axis tag found.", + support, + axisTags, + ) + self = ot.VarRegion() + self.VarRegionAxis = [] + for tag in axisTags: + self.VarRegionAxis.append(buildVarRegionAxis(support.get(tag, (0, 0, 0)))) + return self + + +def buildSparseVarRegion(support, axisTags): + assert all(tag in axisTags for tag in support.keys()), ( + "Unknown axis tag found.", + support, + axisTags, + ) + self = ot.SparseVarRegion() + self.SparseVarRegionAxis = [] + for i, tag in enumerate(axisTags): + if tag not in support: + continue + self.SparseVarRegionAxis.append( + buildSparseVarRegionAxis(i, support.get(tag, (0, 0, 0))) + ) + self.SparseRegionCount = len(self.SparseVarRegionAxis) + return self + + +def buildVarRegionList(supports, axisTags): + self = ot.VarRegionList() + self.RegionAxisCount = len(axisTags) + self.Region = [] + for support in supports: + self.Region.append(buildVarRegion(support, axisTags)) + self.RegionCount = len(self.Region) + return self + + +def buildSparseVarRegionList(supports, axisTags): + self = ot.SparseVarRegionList() + self.RegionAxisCount = len(axisTags) + self.Region = [] + for support in supports: + self.Region.append(buildSparseVarRegion(support, axisTags)) + self.RegionCount = len(self.Region) + return self + + +def _reorderItem(lst, mapping): + return [lst[i] for i in mapping] + + +def VarData_calculateNumShorts(self, optimize=False): + count = self.VarRegionCount + items = self.Item + bit_lengths = [0] * count + for item in items: + # The "+ (i < -1)" magic is to handle two's-compliment. + # That is, we want to get back 7 for -128, whereas + # bit_length() returns 8. Similarly for -65536. + # The reason "i < -1" is used instead of "i < 0" is that + # the latter would make it return 0 for "-1" instead of 1. + bl = [(i + (i < -1)).bit_length() for i in item] + bit_lengths = [max(*pair) for pair in zip(bl, bit_lengths)] + # The addition of 8, instead of seven, is to account for the sign bit. + # This "((b + 8) >> 3) if b else 0" when combined with the above + # "(i + (i < -1)).bit_length()" is a faster way to compute byte-lengths + # conforming to: + # + # byte_length = (0 if i == 0 else + # 1 if -128 <= i < 128 else + # 2 if -65536 <= i < 65536 else + # ...) + byte_lengths = [((b + 8) >> 3) if b else 0 for b in bit_lengths] + + # https://github.com/fonttools/fonttools/issues/2279 + longWords = any(b > 2 for b in byte_lengths) + + if optimize: + # Reorder columns such that wider columns come before narrower columns + mapping = [] + mapping.extend(i for i, b in enumerate(byte_lengths) if b > 2) + mapping.extend(i for i, b in enumerate(byte_lengths) if b == 2) + mapping.extend(i for i, b in enumerate(byte_lengths) if b == 1) + + byte_lengths = _reorderItem(byte_lengths, mapping) + self.VarRegionIndex = _reorderItem(self.VarRegionIndex, mapping) + self.VarRegionCount = len(self.VarRegionIndex) + for i in range(len(items)): + items[i] = _reorderItem(items[i], mapping) + + if longWords: + self.NumShorts = ( + max((i for i, b in enumerate(byte_lengths) if b > 2), default=-1) + 1 + ) + self.NumShorts |= 0x8000 + else: + self.NumShorts = ( + max((i for i, b in enumerate(byte_lengths) if b > 1), default=-1) + 1 + ) + + self.VarRegionCount = len(self.VarRegionIndex) + return self + + +ot.VarData.calculateNumShorts = VarData_calculateNumShorts + + +def VarData_CalculateNumShorts(self, optimize=True): + """Deprecated name for VarData_calculateNumShorts() which + defaults to optimize=True. Use varData.calculateNumShorts() + or varData.optimize().""" + return VarData_calculateNumShorts(self, optimize=optimize) + + +def VarData_optimize(self): + return VarData_calculateNumShorts(self, optimize=True) + + +ot.VarData.optimize = VarData_optimize + + +def buildVarData(varRegionIndices, items, optimize=True): + self = ot.VarData() + self.VarRegionIndex = list(varRegionIndices) + regionCount = self.VarRegionCount = len(self.VarRegionIndex) + records = self.Item = [] + if items: + for item in items: + assert len(item) == regionCount + records.append(list(item)) + self.ItemCount = len(self.Item) + self.calculateNumShorts(optimize=optimize) + return self + + +def buildVarStore(varRegionList, varDataList): + self = ot.VarStore() + self.Format = 1 + self.VarRegionList = varRegionList + self.VarData = list(varDataList) + self.VarDataCount = len(self.VarData) + return self + + +def buildMultiVarData(varRegionIndices, items): + self = ot.MultiVarData() + self.Format = 1 + self.VarRegionIndex = list(varRegionIndices) + regionCount = self.VarRegionCount = len(self.VarRegionIndex) + records = self.Item = [] + if items: + for item in items: + assert len(item) == regionCount + records.append(list(item)) + self.ItemCount = len(self.Item) + return self + + +def buildMultiVarStore(varRegionList, multiVarDataList): + self = ot.MultiVarStore() + self.Format = 1 + self.SparseVarRegionList = varRegionList + self.MultiVarData = list(multiVarDataList) + self.MultiVarDataCount = len(self.MultiVarData) + return self + + +# Variation helpers + + +def buildVarIdxMap(varIdxes, glyphOrder): + self = ot.VarIdxMap() + self.mapping = {g: v for g, v in zip(glyphOrder, varIdxes)} + return self + + +def buildDeltaSetIndexMap(varIdxes): + mapping = list(varIdxes) + if all(i == v for i, v in enumerate(mapping)): + return None + self = ot.DeltaSetIndexMap() + self.mapping = mapping + self.Format = 1 if len(mapping) > 0xFFFF else 0 + return self + + +def buildVarDevTable(varIdx): + self = ot.Device() + self.DeltaFormat = 0x8000 + self.StartSize = varIdx >> 16 + self.EndSize = varIdx & 0xFFFF + return self diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/cff.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/cff.py new file mode 100644 index 0000000000000000000000000000000000000000..155429ab512bfb5e7b96d416a6446dd765e4a9ed --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/cff.py @@ -0,0 +1,631 @@ +from collections import namedtuple +from fontTools.cffLib import ( + maxStackLimit, + TopDictIndex, + buildOrder, + topDictOperators, + topDictOperators2, + privateDictOperators, + privateDictOperators2, + FDArrayIndex, + FontDict, + VarStoreData, +) +from io import BytesIO +from fontTools.cffLib.specializer import specializeCommands, commandsToProgram +from fontTools.ttLib import newTable +from fontTools import varLib +from fontTools.varLib.models import allEqual +from fontTools.misc.loggingTools import deprecateFunction +from fontTools.misc.roundTools import roundFunc +from fontTools.misc.psCharStrings import T2CharString, T2OutlineExtractor +from fontTools.pens.t2CharStringPen import T2CharStringPen +from functools import partial + +from .errors import ( + VarLibCFFDictMergeError, + VarLibCFFPointTypeMergeError, + VarLibCFFHintTypeMergeError, + VarLibMergeError, +) + + +# Backwards compatibility +MergeDictError = VarLibCFFDictMergeError +MergeTypeError = VarLibCFFPointTypeMergeError + + +def addCFFVarStore(varFont, varModel, varDataList, masterSupports): + fvarTable = varFont["fvar"] + axisKeys = [axis.axisTag for axis in fvarTable.axes] + varTupleList = varLib.builder.buildVarRegionList(masterSupports, axisKeys) + varStoreCFFV = varLib.builder.buildVarStore(varTupleList, varDataList) + + topDict = varFont["CFF2"].cff.topDictIndex[0] + topDict.VarStore = VarStoreData(otVarStore=varStoreCFFV) + if topDict.FDArray[0].vstore is None: + fdArray = topDict.FDArray + for fontDict in fdArray: + if hasattr(fontDict, "Private"): + fontDict.Private.vstore = topDict.VarStore + + +@deprecateFunction("Use fontTools.cffLib.CFFToCFF2.convertCFFToCFF2 instead.") +def convertCFFtoCFF2(varFont): + from fontTools.cffLib.CFFToCFF2 import convertCFFToCFF2 + + return convertCFFToCFF2(varFont) + + +def conv_to_int(num): + if isinstance(num, float) and num.is_integer(): + return int(num) + return num + + +pd_blend_fields = ( + "BlueValues", + "OtherBlues", + "FamilyBlues", + "FamilyOtherBlues", + "BlueScale", + "BlueShift", + "BlueFuzz", + "StdHW", + "StdVW", + "StemSnapH", + "StemSnapV", +) + + +def get_private(regionFDArrays, fd_index, ri, fd_map): + region_fdArray = regionFDArrays[ri] + region_fd_map = fd_map[fd_index] + if ri in region_fd_map: + region_fdIndex = region_fd_map[ri] + private = region_fdArray[region_fdIndex].Private + else: + private = None + return private + + +def merge_PrivateDicts(top_dicts, vsindex_dict, var_model, fd_map): + """ + I step through the FontDicts in the FDArray of the varfont TopDict. + For each varfont FontDict: + + * step through each key in FontDict.Private. + * For each key, step through each relevant source font Private dict, and + build a list of values to blend. + + The 'relevant' source fonts are selected by first getting the right + submodel using ``vsindex_dict[vsindex]``. The indices of the + ``subModel.locations`` are mapped to source font list indices by + assuming the latter order is the same as the order of the + ``var_model.locations``. I can then get the index of each subModel + location in the list of ``var_model.locations``. + """ + + topDict = top_dicts[0] + region_top_dicts = top_dicts[1:] + if hasattr(region_top_dicts[0], "FDArray"): + regionFDArrays = [fdTopDict.FDArray for fdTopDict in region_top_dicts] + else: + regionFDArrays = [[fdTopDict] for fdTopDict in region_top_dicts] + for fd_index, font_dict in enumerate(topDict.FDArray): + private_dict = font_dict.Private + vsindex = getattr(private_dict, "vsindex", 0) + # At the moment, no PrivateDict has a vsindex key, but let's support + # how it should work. See comment at end of + # merge_charstrings() - still need to optimize use of vsindex. + sub_model, _ = vsindex_dict[vsindex] + master_indices = [] + for loc in sub_model.locations[1:]: + i = var_model.locations.index(loc) - 1 + master_indices.append(i) + pds = [private_dict] + last_pd = private_dict + for ri in master_indices: + pd = get_private(regionFDArrays, fd_index, ri, fd_map) + # If the region font doesn't have this FontDict, just reference + # the last one used. + if pd is None: + pd = last_pd + else: + last_pd = pd + pds.append(pd) + num_masters = len(pds) + for key, value in private_dict.rawDict.items(): + dataList = [] + if key not in pd_blend_fields: + continue + if isinstance(value, list): + try: + values = [pd.rawDict[key] for pd in pds] + except KeyError: + print( + "Warning: {key} in default font Private dict is " + "missing from another font, and was " + "discarded.".format(key=key) + ) + continue + try: + values = zip(*values) + except IndexError: + raise VarLibCFFDictMergeError(key, value, values) + """ + Row 0 contains the first value from each master. + Convert each row from absolute values to relative + values from the previous row. + e.g for three masters, a list of values was: + master 0 OtherBlues = [-217,-205] + master 1 OtherBlues = [-234,-222] + master 1 OtherBlues = [-188,-176] + The call to zip() converts this to: + [(-217, -234, -188), (-205, -222, -176)] + and is converted finally to: + OtherBlues = [[-217, 17.0, 46.0], [-205, 0.0, 0.0]] + """ + prev_val_list = [0] * num_masters + any_points_differ = False + for val_list in values: + rel_list = [ + (val - prev_val_list[i]) for (i, val) in enumerate(val_list) + ] + if (not any_points_differ) and not allEqual(rel_list): + any_points_differ = True + prev_val_list = val_list + deltas = sub_model.getDeltas(rel_list) + # For PrivateDict BlueValues, the default font + # values are absolute, not relative to the prior value. + deltas[0] = val_list[0] + dataList.append(deltas) + # If there are no blend values,then + # we can collapse the blend lists. + if not any_points_differ: + dataList = [data[0] for data in dataList] + else: + values = [pd.rawDict[key] for pd in pds] + if not allEqual(values): + dataList = sub_model.getDeltas(values) + else: + dataList = values[0] + + # Convert numbers with no decimal part to an int + if isinstance(dataList, list): + for i, item in enumerate(dataList): + if isinstance(item, list): + for j, jtem in enumerate(item): + dataList[i][j] = conv_to_int(jtem) + else: + dataList[i] = conv_to_int(item) + else: + dataList = conv_to_int(dataList) + + private_dict.rawDict[key] = dataList + + +def _cff_or_cff2(font): + if "CFF " in font: + return font["CFF "] + return font["CFF2"] + + +def getfd_map(varFont, fonts_list): + """Since a subset source font may have fewer FontDicts in their + FDArray than the default font, we have to match up the FontDicts in + the different fonts . We do this with the FDSelect array, and by + assuming that the same glyph will reference matching FontDicts in + each source font. We return a mapping from fdIndex in the default + font to a dictionary which maps each master list index of each + region font to the equivalent fdIndex in the region font.""" + fd_map = {} + default_font = fonts_list[0] + region_fonts = fonts_list[1:] + num_regions = len(region_fonts) + topDict = _cff_or_cff2(default_font).cff.topDictIndex[0] + if not hasattr(topDict, "FDSelect"): + # All glyphs reference only one FontDict. + # Map the FD index for regions to index 0. + fd_map[0] = {ri: 0 for ri in range(num_regions)} + return fd_map + + gname_mapping = {} + default_fdSelect = topDict.FDSelect + glyphOrder = default_font.getGlyphOrder() + for gid, fdIndex in enumerate(default_fdSelect): + gname_mapping[glyphOrder[gid]] = fdIndex + if fdIndex not in fd_map: + fd_map[fdIndex] = {} + for ri, region_font in enumerate(region_fonts): + region_glyphOrder = region_font.getGlyphOrder() + region_topDict = _cff_or_cff2(region_font).cff.topDictIndex[0] + if not hasattr(region_topDict, "FDSelect"): + # All the glyphs share the same FontDict. Pick any glyph. + default_fdIndex = gname_mapping[region_glyphOrder[0]] + fd_map[default_fdIndex][ri] = 0 + else: + region_fdSelect = region_topDict.FDSelect + for gid, fdIndex in enumerate(region_fdSelect): + default_fdIndex = gname_mapping[region_glyphOrder[gid]] + region_map = fd_map[default_fdIndex] + if ri not in region_map: + region_map[ri] = fdIndex + return fd_map + + +CVarData = namedtuple("CVarData", "varDataList masterSupports vsindex_dict") + + +def merge_region_fonts(varFont, model, ordered_fonts_list, glyphOrder): + topDict = varFont["CFF2"].cff.topDictIndex[0] + top_dicts = [topDict] + [ + _cff_or_cff2(ttFont).cff.topDictIndex[0] for ttFont in ordered_fonts_list[1:] + ] + num_masters = len(model.mapping) + cvData = merge_charstrings(glyphOrder, num_masters, top_dicts, model) + fd_map = getfd_map(varFont, ordered_fonts_list) + merge_PrivateDicts(top_dicts, cvData.vsindex_dict, model, fd_map) + addCFFVarStore(varFont, model, cvData.varDataList, cvData.masterSupports) + + +def _get_cs(charstrings, glyphName, filterEmpty=False): + if glyphName not in charstrings: + return None + cs = charstrings[glyphName] + + if filterEmpty: + cs.decompile() + if cs.program == []: # CFF2 empty charstring + return None + elif ( + len(cs.program) <= 2 + and cs.program[-1] == "endchar" + and (len(cs.program) == 1 or type(cs.program[0]) in (int, float)) + ): # CFF1 empty charstring + return None + + return cs + + +def _add_new_vsindex( + model, key, masterSupports, vsindex_dict, vsindex_by_key, varDataList +): + varTupleIndexes = [] + for support in model.supports[1:]: + if support not in masterSupports: + masterSupports.append(support) + varTupleIndexes.append(masterSupports.index(support)) + var_data = varLib.builder.buildVarData(varTupleIndexes, None, False) + vsindex = len(vsindex_dict) + vsindex_by_key[key] = vsindex + vsindex_dict[vsindex] = (model, [key]) + varDataList.append(var_data) + return vsindex + + +def merge_charstrings(glyphOrder, num_masters, top_dicts, masterModel): + vsindex_dict = {} + vsindex_by_key = {} + varDataList = [] + masterSupports = [] + default_charstrings = top_dicts[0].CharStrings + for gid, gname in enumerate(glyphOrder): + # interpret empty non-default masters as missing glyphs from a sparse master + all_cs = [ + _get_cs(td.CharStrings, gname, i != 0) for i, td in enumerate(top_dicts) + ] + model, model_cs = masterModel.getSubModel(all_cs) + # create the first pass CFF2 charstring, from + # the default charstring. + default_charstring = model_cs[0] + var_pen = CFF2CharStringMergePen([], gname, num_masters, 0) + # We need to override outlineExtractor because these + # charstrings do have widths in the 'program'; we need to drop these + # values rather than post assertion error for them. + default_charstring.outlineExtractor = MergeOutlineExtractor + default_charstring.draw(var_pen) + + # Add the coordinates from all the other regions to the + # blend lists in the CFF2 charstring. + region_cs = model_cs[1:] + for region_idx, region_charstring in enumerate(region_cs, start=1): + var_pen.restart(region_idx) + region_charstring.outlineExtractor = MergeOutlineExtractor + region_charstring.draw(var_pen) + + # Collapse each coordinate list to a blend operator and its args. + new_cs = var_pen.getCharString( + private=default_charstring.private, + globalSubrs=default_charstring.globalSubrs, + var_model=model, + optimize=True, + ) + default_charstrings[gname] = new_cs + + if not region_cs: + continue + + if (not var_pen.seen_moveto) or ("blend" not in new_cs.program): + # If this is not a marking glyph, or if there are no blend + # arguments, then we can use vsindex 0. No need to + # check if we need a new vsindex. + continue + + # If the charstring required a new model, create + # a VarData table to go with, and set vsindex. + key = tuple(v is not None for v in all_cs) + try: + vsindex = vsindex_by_key[key] + except KeyError: + vsindex = _add_new_vsindex( + model, key, masterSupports, vsindex_dict, vsindex_by_key, varDataList + ) + # We do not need to check for an existing new_cs.private.vsindex, + # as we know it doesn't exist yet. + if vsindex != 0: + new_cs.program[:0] = [vsindex, "vsindex"] + + # If there is no variation in any of the charstrings, then vsindex_dict + # never gets built. This could still be needed if there is variation + # in the PrivatDict, so we will build the default data for vsindex = 0. + if not vsindex_dict: + key = (True,) * num_masters + _add_new_vsindex( + masterModel, key, masterSupports, vsindex_dict, vsindex_by_key, varDataList + ) + cvData = CVarData( + varDataList=varDataList, + masterSupports=masterSupports, + vsindex_dict=vsindex_dict, + ) + # XXX To do: optimize use of vsindex between the PrivateDicts and + # charstrings + return cvData + + +class CFFToCFF2OutlineExtractor(T2OutlineExtractor): + """This class is used to remove the initial width from the CFF + charstring without trying to add the width to self.nominalWidthX, + which is None.""" + + def popallWidth(self, evenOdd=0): + args = self.popall() + if not self.gotWidth: + if evenOdd ^ (len(args) % 2): + args = args[1:] + self.width = self.defaultWidthX + self.gotWidth = 1 + return args + + +class MergeOutlineExtractor(CFFToCFF2OutlineExtractor): + """Used to extract the charstring commands - including hints - from a + CFF charstring in order to merge it as another set of region data + into a CFF2 variable font charstring.""" + + def __init__( + self, + pen, + localSubrs, + globalSubrs, + nominalWidthX, + defaultWidthX, + private=None, + blender=None, + ): + super().__init__( + pen, localSubrs, globalSubrs, nominalWidthX, defaultWidthX, private, blender + ) + + def countHints(self): + args = self.popallWidth() + self.hintCount = self.hintCount + len(args) // 2 + return args + + def _hint_op(self, type, args): + self.pen.add_hint(type, args) + + def op_hstem(self, index): + args = self.countHints() + self._hint_op("hstem", args) + + def op_vstem(self, index): + args = self.countHints() + self._hint_op("vstem", args) + + def op_hstemhm(self, index): + args = self.countHints() + self._hint_op("hstemhm", args) + + def op_vstemhm(self, index): + args = self.countHints() + self._hint_op("vstemhm", args) + + def _get_hintmask(self, index): + if not self.hintMaskBytes: + args = self.countHints() + if args: + self._hint_op("vstemhm", args) + self.hintMaskBytes = (self.hintCount + 7) // 8 + hintMaskBytes, index = self.callingStack[-1].getBytes(index, self.hintMaskBytes) + return index, hintMaskBytes + + def op_hintmask(self, index): + index, hintMaskBytes = self._get_hintmask(index) + self.pen.add_hintmask("hintmask", [hintMaskBytes]) + return hintMaskBytes, index + + def op_cntrmask(self, index): + index, hintMaskBytes = self._get_hintmask(index) + self.pen.add_hintmask("cntrmask", [hintMaskBytes]) + return hintMaskBytes, index + + +class CFF2CharStringMergePen(T2CharStringPen): + """Pen to merge Type 2 CharStrings.""" + + def __init__( + self, default_commands, glyphName, num_masters, master_idx, roundTolerance=0.01 + ): + # For roundTolerance see https://github.com/fonttools/fonttools/issues/2838 + super().__init__( + width=None, glyphSet=None, CFF2=True, roundTolerance=roundTolerance + ) + self.pt_index = 0 + self._commands = default_commands + self.m_index = master_idx + self.num_masters = num_masters + self.prev_move_idx = 0 + self.seen_moveto = False + self.glyphName = glyphName + self.round = roundFunc(roundTolerance, round=round) + + def add_point(self, point_type, pt_coords): + if self.m_index == 0: + self._commands.append([point_type, [pt_coords]]) + else: + cmd = self._commands[self.pt_index] + if cmd[0] != point_type: + raise VarLibCFFPointTypeMergeError( + point_type, self.pt_index, len(cmd[1]), cmd[0], self.glyphName + ) + cmd[1].append(pt_coords) + self.pt_index += 1 + + def add_hint(self, hint_type, args): + if self.m_index == 0: + self._commands.append([hint_type, [args]]) + else: + cmd = self._commands[self.pt_index] + if cmd[0] != hint_type: + raise VarLibCFFHintTypeMergeError( + hint_type, self.pt_index, len(cmd[1]), cmd[0], self.glyphName + ) + cmd[1].append(args) + self.pt_index += 1 + + def add_hintmask(self, hint_type, abs_args): + # For hintmask, fonttools.cffLib.specializer.py expects + # each of these to be represented by two sequential commands: + # first holding only the operator name, with an empty arg list, + # second with an empty string as the op name, and the mask arg list. + if self.m_index == 0: + self._commands.append([hint_type, []]) + self._commands.append(["", [abs_args]]) + else: + cmd = self._commands[self.pt_index] + if cmd[0] != hint_type: + raise VarLibCFFHintTypeMergeError( + hint_type, self.pt_index, len(cmd[1]), cmd[0], self.glyphName + ) + self.pt_index += 1 + cmd = self._commands[self.pt_index] + cmd[1].append(abs_args) + self.pt_index += 1 + + def _moveTo(self, pt): + if not self.seen_moveto: + self.seen_moveto = True + pt_coords = self._p(pt) + self.add_point("rmoveto", pt_coords) + # I set prev_move_idx here because add_point() + # can change self.pt_index. + self.prev_move_idx = self.pt_index - 1 + + def _lineTo(self, pt): + pt_coords = self._p(pt) + self.add_point("rlineto", pt_coords) + + def _curveToOne(self, pt1, pt2, pt3): + _p = self._p + pt_coords = _p(pt1) + _p(pt2) + _p(pt3) + self.add_point("rrcurveto", pt_coords) + + def _closePath(self): + pass + + def _endPath(self): + pass + + def restart(self, region_idx): + self.pt_index = 0 + self.m_index = region_idx + self._p0 = (0, 0) + + def getCommands(self): + return self._commands + + def reorder_blend_args(self, commands, get_delta_func): + """ + We first re-order the master coordinate values. + For a moveto to lineto, the args are now arranged as:: + + [ [master_0 x,y], [master_1 x,y], [master_2 x,y] ] + + We re-arrange this to:: + + [ [master_0 x, master_1 x, master_2 x], + [master_0 y, master_1 y, master_2 y] + ] + + If the master values are all the same, we collapse the list to + as single value instead of a list. + + We then convert this to:: + + [ [master_0 x] + [x delta tuple] + [numBlends=1] + [master_0 y] + [y delta tuple] + [numBlends=1] + ] + """ + for cmd in commands: + # arg[i] is the set of arguments for this operator from master i. + args = cmd[1] + m_args = zip(*args) + # m_args[n] is now all num_master args for the i'th argument + # for this operation. + cmd[1] = list(m_args) + lastOp = None + for cmd in commands: + op = cmd[0] + # masks are represented by two cmd's: first has only op names, + # second has only args. + if lastOp in ["hintmask", "cntrmask"]: + coord = list(cmd[1]) + if not allEqual(coord): + raise VarLibMergeError( + "Hintmask values cannot differ between source fonts." + ) + cmd[1] = [coord[0][0]] + else: + coords = cmd[1] + new_coords = [] + for coord in coords: + if allEqual(coord): + new_coords.append(coord[0]) + else: + # convert to deltas + deltas = get_delta_func(coord)[1:] + coord = [coord[0]] + deltas + coord.append(1) + new_coords.append(coord) + cmd[1] = new_coords + lastOp = op + return commands + + def getCharString( + self, private=None, globalSubrs=None, var_model=None, optimize=True + ): + commands = self._commands + commands = self.reorder_blend_args( + commands, partial(var_model.getDeltas, round=self.round) + ) + if optimize: + commands = specializeCommands( + commands, generalizeFirst=False, maxstack=maxStackLimit + ) + program = commandsToProgram(commands) + charString = T2CharString( + program=program, private=private, globalSubrs=globalSubrs + ) + return charString diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/errors.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..4f30f901babed2b985ae5c333420b6a9e7a3baa8 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/errors.py @@ -0,0 +1,219 @@ +import textwrap + + +class VarLibError(Exception): + """Base exception for the varLib module.""" + + +class VarLibValidationError(VarLibError): + """Raised when input data is invalid from varLib's point of view.""" + + +class VarLibMergeError(VarLibError): + """Raised when input data cannot be merged into a variable font.""" + + def __init__(self, merger=None, **kwargs): + self.merger = merger + if not kwargs: + kwargs = {} + if "stack" in kwargs: + self.stack = kwargs["stack"] + del kwargs["stack"] + else: + self.stack = [] + self.cause = kwargs + + @property + def reason(self): + return self.__doc__ + + def _master_name(self, ix): + if self.merger is not None: + ttf = self.merger.ttfs[ix] + if "name" in ttf and ttf["name"].getBestFullName(): + return ttf["name"].getBestFullName() + elif hasattr(ttf.reader, "file") and hasattr(ttf.reader.file, "name"): + return ttf.reader.file.name + return f"master number {ix}" + + @property + def offender(self): + if "expected" in self.cause and "got" in self.cause: + index = [x == self.cause["expected"] for x in self.cause["got"]].index( + False + ) + master_name = self._master_name(index) + if "location" in self.cause: + master_name = f"{master_name} ({self.cause['location']})" + return index, master_name + return None, None + + @property + def details(self): + if "expected" in self.cause and "got" in self.cause: + offender_index, offender = self.offender + got = self.cause["got"][offender_index] + return f"Expected to see {self.stack[0]}=={self.cause['expected']!r}, instead saw {got!r}\n" + return "" + + def __str__(self): + offender_index, offender = self.offender + location = "" + if offender: + location = f"\n\nThe problem is likely to be in {offender}:\n" + context = "".join(reversed(self.stack)) + basic = textwrap.fill( + f"Couldn't merge the fonts, because {self.reason}. " + f"This happened while performing the following operation: {context}", + width=78, + ) + return "\n\n" + basic + location + self.details + + +class ShouldBeConstant(VarLibMergeError): + """some values were different, but should have been the same""" + + @property + def details(self): + basic_message = super().details + + if self.stack[0] != ".FeatureCount" or self.merger is None: + return basic_message + + assert self.stack[0] == ".FeatureCount" + offender_index, _ = self.offender + bad_ttf = self.merger.ttfs[offender_index] + good_ttf = next( + ttf + for ttf in self.merger.ttfs + if self.stack[-1] in ttf + and ttf[self.stack[-1]].table.FeatureList.FeatureCount + == self.cause["expected"] + ) + + good_features = [ + x.FeatureTag + for x in good_ttf[self.stack[-1]].table.FeatureList.FeatureRecord + ] + bad_features = [ + x.FeatureTag + for x in bad_ttf[self.stack[-1]].table.FeatureList.FeatureRecord + ] + return basic_message + ( + "\nIncompatible features between masters.\n" + f"Expected: {', '.join(good_features)}.\n" + f"Got: {', '.join(bad_features)}.\n" + ) + + +class FoundANone(VarLibMergeError): + """one of the values in a list was empty when it shouldn't have been""" + + @property + def offender(self): + index = [x is None for x in self.cause["got"]].index(True) + return index, self._master_name(index) + + @property + def details(self): + cause, stack = self.cause, self.stack + return f"{stack[0]}=={cause['got']}\n" + + +class NotANone(VarLibMergeError): + """one of the values in a list was not empty when it should have been""" + + @property + def offender(self): + index = [x is not None for x in self.cause["got"]].index(True) + return index, self._master_name(index) + + @property + def details(self): + cause, stack = self.cause, self.stack + return f"{stack[0]}=={cause['got']}\n" + + +class MismatchedTypes(VarLibMergeError): + """data had inconsistent types""" + + +class LengthsDiffer(VarLibMergeError): + """a list of objects had inconsistent lengths""" + + +class KeysDiffer(VarLibMergeError): + """a list of objects had different keys""" + + +class InconsistentGlyphOrder(VarLibMergeError): + """the glyph order was inconsistent between masters""" + + +class InconsistentExtensions(VarLibMergeError): + """the masters use extension lookups in inconsistent ways""" + + +class UnsupportedFormat(VarLibMergeError): + """an OpenType subtable (%s) had a format I didn't expect""" + + def __init__(self, merger=None, **kwargs): + super().__init__(merger, **kwargs) + if not self.stack: + self.stack = [".Format"] + + @property + def reason(self): + s = self.__doc__ % self.cause["subtable"] + if "value" in self.cause: + s += f" ({self.cause['value']!r})" + return s + + +class InconsistentFormats(UnsupportedFormat): + """an OpenType subtable (%s) had inconsistent formats between masters""" + + +class VarLibCFFMergeError(VarLibError): + pass + + +class VarLibCFFDictMergeError(VarLibCFFMergeError): + """Raised when a CFF PrivateDict cannot be merged.""" + + def __init__(self, key, value, values): + error_msg = ( + f"For the Private Dict key '{key}', the default font value list:" + f"\n\t{value}\nhad a different number of values than a region font:" + ) + for region_value in values: + error_msg += f"\n\t{region_value}" + self.args = (error_msg,) + + +class VarLibCFFPointTypeMergeError(VarLibCFFMergeError): + """Raised when a CFF glyph cannot be merged because of point type differences.""" + + def __init__(self, point_type, pt_index, m_index, default_type, glyph_name): + error_msg = ( + f"Glyph '{glyph_name}': '{point_type}' at point index {pt_index} in " + f"master index {m_index} differs from the default font point type " + f"'{default_type}'" + ) + self.args = (error_msg,) + + +class VarLibCFFHintTypeMergeError(VarLibCFFMergeError): + """Raised when a CFF glyph cannot be merged because of hint type differences.""" + + def __init__(self, hint_type, cmd_index, m_index, default_type, glyph_name): + error_msg = ( + f"Glyph '{glyph_name}': '{hint_type}' at index {cmd_index} in " + f"master index {m_index} differs from the default font hint type " + f"'{default_type}'" + ) + self.args = (error_msg,) + + +class VariationModelError(VarLibError): + """Raised when a variation model is faulty.""" diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/featureVars.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/featureVars.py new file mode 100644 index 0000000000000000000000000000000000000000..40ad9dfca6c56f26aa767a454c3f5ed0b4cdb28a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/featureVars.py @@ -0,0 +1,703 @@ +"""Module to build FeatureVariation tables: +https://docs.microsoft.com/en-us/typography/opentype/spec/chapter2#featurevariations-table + +NOTE: The API is experimental and subject to change. +""" + +from fontTools.misc.dictTools import hashdict +from fontTools.misc.intTools import bit_count +from fontTools.ttLib import newTable +from fontTools.ttLib.tables import otTables as ot +from fontTools.ttLib.ttVisitor import TTVisitor +from fontTools.otlLib.builder import buildLookup, buildSingleSubstSubtable +from collections import OrderedDict + +from .errors import VarLibError, VarLibValidationError + + +def addFeatureVariations(font, conditionalSubstitutions, featureTag="rvrn"): + """Add conditional substitutions to a Variable Font. + + The `conditionalSubstitutions` argument is a list of (Region, Substitutions) + tuples. + + A Region is a list of Boxes. A Box is a dict mapping axisTags to + (minValue, maxValue) tuples. Irrelevant axes may be omitted and they are + interpretted as extending to end of axis in each direction. A Box represents + an orthogonal 'rectangular' subset of an N-dimensional design space. + A Region represents a more complex subset of an N-dimensional design space, + ie. the union of all the Boxes in the Region. + For efficiency, Boxes within a Region should ideally not overlap, but + functionality is not compromised if they do. + + The minimum and maximum values are expressed in normalized coordinates. + + A Substitution is a dict mapping source glyph names to substitute glyph names. + + Example: + + # >>> f = TTFont(srcPath) + # >>> condSubst = [ + # ... # A list of (Region, Substitution) tuples. + # ... ([{"wdth": (0.5, 1.0)}], {"cent": "cent.rvrn"}), + # ... ([{"wght": (0.5, 1.0)}], {"dollar": "dollar.rvrn"}), + # ... ] + # >>> addFeatureVariations(f, condSubst) + # >>> f.save(dstPath) + + The `featureTag` parameter takes either a str or a iterable of str (the single str + is kept for backwards compatibility), and defines which feature(s) will be + associated with the feature variations. + Note, if this is "rvrn", then the substitution lookup will be inserted at the + beginning of the lookup list so that it is processed before others, otherwise + for any other feature tags it will be appended last. + """ + + # process first when "rvrn" is the only listed tag + featureTags = [featureTag] if isinstance(featureTag, str) else sorted(featureTag) + processLast = "rvrn" not in featureTags or len(featureTags) > 1 + + _checkSubstitutionGlyphsExist( + glyphNames=set(font.getGlyphOrder()), + substitutions=conditionalSubstitutions, + ) + + substitutions = overlayFeatureVariations(conditionalSubstitutions) + + # turn substitution dicts into tuples of tuples, so they are hashable + conditionalSubstitutions, allSubstitutions = makeSubstitutionsHashable( + substitutions + ) + if "GSUB" not in font: + font["GSUB"] = buildGSUB() + else: + existingTags = _existingVariableFeatures(font["GSUB"].table).intersection( + featureTags + ) + if existingTags: + raise VarLibError( + f"FeatureVariations already exist for feature tag(s): {existingTags}" + ) + + # setup lookups + lookupMap = buildSubstitutionLookups( + font["GSUB"].table, allSubstitutions, processLast + ) + + # addFeatureVariationsRaw takes a list of + # ( {condition}, [ lookup indices ] ) + # so rearrange our lookups to match + conditionsAndLookups = [] + for conditionSet, substitutions in conditionalSubstitutions: + conditionsAndLookups.append( + (conditionSet, [lookupMap[s] for s in substitutions]) + ) + + addFeatureVariationsRaw(font, font["GSUB"].table, conditionsAndLookups, featureTags) + + # Update OS/2.usMaxContext in case the font didn't have features before, but + # does now, if the OS/2 table exists. The table may be required, but + # fontTools needs to be able to deal with non-standard fonts. Since feature + # variations are always 1:1 mappings, we can set the value to at least 1 + # instead of recomputing it with `otlLib.maxContextCalc.maxCtxFont()`. + if (os2 := font.get("OS/2")) is not None: + os2.usMaxContext = max(1, os2.usMaxContext) + + +def _existingVariableFeatures(table): + existingFeatureVarsTags = set() + if hasattr(table, "FeatureVariations") and table.FeatureVariations is not None: + features = table.FeatureList.FeatureRecord + for fvr in table.FeatureVariations.FeatureVariationRecord: + for ftsr in fvr.FeatureTableSubstitution.SubstitutionRecord: + existingFeatureVarsTags.add(features[ftsr.FeatureIndex].FeatureTag) + return existingFeatureVarsTags + + +def _checkSubstitutionGlyphsExist(glyphNames, substitutions): + referencedGlyphNames = set() + for _, substitution in substitutions: + referencedGlyphNames |= substitution.keys() + referencedGlyphNames |= set(substitution.values()) + missing = referencedGlyphNames - glyphNames + if missing: + raise VarLibValidationError( + "Missing glyphs are referenced in conditional substitution rules:" + f" {', '.join(missing)}" + ) + + +def overlayFeatureVariations(conditionalSubstitutions): + """Compute overlaps between all conditional substitutions. + + The `conditionalSubstitutions` argument is a list of (Region, Substitutions) + tuples. + + A Region is a list of Boxes. A Box is a dict mapping axisTags to + (minValue, maxValue) tuples. Irrelevant axes may be omitted and they are + interpretted as extending to end of axis in each direction. A Box represents + an orthogonal 'rectangular' subset of an N-dimensional design space. + A Region represents a more complex subset of an N-dimensional design space, + ie. the union of all the Boxes in the Region. + For efficiency, Boxes within a Region should ideally not overlap, but + functionality is not compromised if they do. + + The minimum and maximum values are expressed in normalized coordinates. + + A Substitution is a dict mapping source glyph names to substitute glyph names. + + Returns data is in similar but different format. Overlaps of distinct + substitution Boxes (*not* Regions) are explicitly listed as distinct rules, + and rules with the same Box merged. The more specific rules appear earlier + in the resulting list. Moreover, instead of just a dictionary of substitutions, + a list of dictionaries is returned for substitutions corresponding to each + unique space, with each dictionary being identical to one of the input + substitution dictionaries. These dictionaries are not merged to allow data + sharing when they are converted into font tables. + + Example:: + + >>> condSubst = [ + ... # A list of (Region, Substitution) tuples. + ... ([{"wght": (0.5, 1.0)}], {"dollar": "dollar.rvrn"}), + ... ([{"wght": (0.5, 1.0)}], {"dollar": "dollar.rvrn"}), + ... ([{"wdth": (0.5, 1.0)}], {"cent": "cent.rvrn"}), + ... ([{"wght": (0.5, 1.0), "wdth": (-1, 1.0)}], {"dollar": "dollar.rvrn"}), + ... ] + >>> from pprint import pprint + >>> pprint(overlayFeatureVariations(condSubst)) + [({'wdth': (0.5, 1.0), 'wght': (0.5, 1.0)}, + [{'dollar': 'dollar.rvrn'}, {'cent': 'cent.rvrn'}]), + ({'wdth': (0.5, 1.0)}, [{'cent': 'cent.rvrn'}]), + ({'wght': (0.5, 1.0)}, [{'dollar': 'dollar.rvrn'}])] + + """ + + # Merge same-substitutions rules, as this creates fewer number oflookups. + merged = OrderedDict() + for value, key in conditionalSubstitutions: + key = hashdict(key) + if key in merged: + merged[key].extend(value) + else: + merged[key] = value + conditionalSubstitutions = [(v, dict(k)) for k, v in merged.items()] + del merged + + # Merge same-region rules, as this is cheaper. + # Also convert boxes to hashdict() + # + # Reversing is such that earlier entries win in case of conflicting substitution + # rules for the same region. + merged = OrderedDict() + for key, value in reversed(conditionalSubstitutions): + key = tuple( + sorted( + (hashdict(cleanupBox(k)) for k in key), + key=lambda d: tuple(sorted(d.items())), + ) + ) + if key in merged: + merged[key].update(value) + else: + merged[key] = dict(value) + conditionalSubstitutions = list(reversed(merged.items())) + del merged + + # Overlay + # + # Rank is the bit-set of the index of all contributing layers. + initMapInit = ((hashdict(), 0),) # Initializer representing the entire space + boxMap = OrderedDict(initMapInit) # Map from Box to Rank + for i, (currRegion, _) in enumerate(conditionalSubstitutions): + newMap = OrderedDict(initMapInit) + currRank = 1 << i + for box, rank in boxMap.items(): + for currBox in currRegion: + intersection, remainder = overlayBox(currBox, box) + if intersection is not None: + intersection = hashdict(intersection) + newMap[intersection] = newMap.get(intersection, 0) | rank | currRank + if remainder is not None: + remainder = hashdict(remainder) + newMap[remainder] = newMap.get(remainder, 0) | rank + boxMap = newMap + + # Generate output + items = [] + for box, rank in sorted( + boxMap.items(), key=(lambda BoxAndRank: -bit_count(BoxAndRank[1])) + ): + # Skip any box that doesn't have any substitution. + if rank == 0: + continue + substsList = [] + i = 0 + while rank: + if rank & 1: + substsList.append(conditionalSubstitutions[i][1]) + rank >>= 1 + i += 1 + items.append((dict(box), substsList)) + return items + + +# +# Terminology: +# +# A 'Box' is a dict representing an orthogonal "rectangular" bit of N-dimensional space. +# The keys in the dict are axis tags, the values are (minValue, maxValue) tuples. +# Missing dimensions (keys) are substituted by the default min and max values +# from the corresponding axes. +# + + +def overlayBox(top, bot): + """Overlays ``top`` box on top of ``bot`` box. + + Returns two items: + + * Box for intersection of ``top`` and ``bot``, or None if they don't intersect. + * Box for remainder of ``bot``. Remainder box might not be exact (since the + remainder might not be a simple box), but is inclusive of the exact + remainder. + """ + + # Intersection + intersection = {} + intersection.update(top) + intersection.update(bot) + for axisTag in set(top) & set(bot): + min1, max1 = top[axisTag] + min2, max2 = bot[axisTag] + minimum = max(min1, min2) + maximum = min(max1, max2) + if not minimum < maximum: + return None, bot # Do not intersect + intersection[axisTag] = minimum, maximum + + # Remainder + # + # Remainder is empty if bot's each axis range lies within that of intersection. + # + # Remainder is shrank if bot's each, except for exactly one, axis range lies + # within that of intersection, and that one axis, it extrudes out of the + # intersection only on one side. + # + # Bot is returned in full as remainder otherwise, as true remainder is not + # representable as a single box. + + remainder = dict(bot) + extruding = False + fullyInside = True + for axisTag in top: + if axisTag in bot: + continue + extruding = True + fullyInside = False + break + for axisTag in bot: + if axisTag not in top: + continue # Axis range lies fully within + min1, max1 = intersection[axisTag] + min2, max2 = bot[axisTag] + if min1 <= min2 and max2 <= max1: + continue # Axis range lies fully within + + # Bot's range doesn't fully lie within that of top's for this axis. + # We know they intersect, so it cannot lie fully without either; so they + # overlap. + + # If we have had an overlapping axis before, remainder is not + # representable as a box, so return full bottom and go home. + if extruding: + return intersection, bot + extruding = True + fullyInside = False + + # Otherwise, cut remainder on this axis and continue. + if min1 <= min2: + # Right side survives. + minimum = max(max1, min2) + maximum = max2 + elif max2 <= max1: + # Left side survives. + minimum = min2 + maximum = min(min1, max2) + else: + # Remainder leaks out from both sides. Can't cut either. + return intersection, bot + + remainder[axisTag] = minimum, maximum + + if fullyInside: + # bot is fully within intersection. Remainder is empty. + return intersection, None + + return intersection, remainder + + +def cleanupBox(box): + """Return a sparse copy of `box`, without redundant (default) values. + + >>> cleanupBox({}) + {} + >>> cleanupBox({'wdth': (0.0, 1.0)}) + {'wdth': (0.0, 1.0)} + >>> cleanupBox({'wdth': (-1.0, 1.0)}) + {} + + """ + return {tag: limit for tag, limit in box.items() if limit != (-1.0, 1.0)} + + +# +# Low level implementation +# + + +def addFeatureVariationsRaw(font, table, conditionalSubstitutions, featureTag="rvrn"): + """Low level implementation of addFeatureVariations that directly + models the possibilities of the FeatureVariations table.""" + + featureTags = [featureTag] if isinstance(featureTag, str) else sorted(featureTag) + processLast = "rvrn" not in featureTags or len(featureTags) > 1 + + # + # if a <featureTag> feature is not present: + # make empty <featureTag> feature + # sort features, get <featureTag> feature index + # add <featureTag> feature to all scripts + # if a <featureTag> feature is present: + # reuse <featureTag> feature index + # make lookups + # add feature variations + # + if table.Version < 0x00010001: + table.Version = 0x00010001 # allow table.FeatureVariations + + varFeatureIndices = set() + + existingTags = { + feature.FeatureTag + for feature in table.FeatureList.FeatureRecord + if feature.FeatureTag in featureTags + } + + newTags = set(featureTags) - existingTags + if newTags: + varFeatures = [] + for featureTag in sorted(newTags): + varFeature = buildFeatureRecord(featureTag, []) + table.FeatureList.FeatureRecord.append(varFeature) + varFeatures.append(varFeature) + table.FeatureList.FeatureCount = len(table.FeatureList.FeatureRecord) + + sortFeatureList(table) + + for varFeature in varFeatures: + varFeatureIndex = table.FeatureList.FeatureRecord.index(varFeature) + + for scriptRecord in table.ScriptList.ScriptRecord: + if scriptRecord.Script.DefaultLangSys is None: + # We need to have a default LangSys to attach variations to. + langSys = ot.LangSys() + langSys.LookupOrder = None + langSys.ReqFeatureIndex = 0xFFFF + langSys.FeatureIndex = [] + langSys.FeatureCount = 0 + scriptRecord.Script.DefaultLangSys = langSys + langSystems = [lsr.LangSys for lsr in scriptRecord.Script.LangSysRecord] + for langSys in [scriptRecord.Script.DefaultLangSys] + langSystems: + langSys.FeatureIndex.append(varFeatureIndex) + langSys.FeatureCount = len(langSys.FeatureIndex) + varFeatureIndices.add(varFeatureIndex) + + if existingTags: + # indices may have changed if we inserted new features and sorted feature list + # so we must do this after the above + varFeatureIndices.update( + index + for index, feature in enumerate(table.FeatureList.FeatureRecord) + if feature.FeatureTag in existingTags + ) + + axisIndices = { + axis.axisTag: axisIndex for axisIndex, axis in enumerate(font["fvar"].axes) + } + + hasFeatureVariations = ( + hasattr(table, "FeatureVariations") and table.FeatureVariations is not None + ) + + featureVariationRecords = [] + for conditionSet, lookupIndices in conditionalSubstitutions: + conditionTable = [] + for axisTag, (minValue, maxValue) in sorted(conditionSet.items()): + if minValue > maxValue: + raise VarLibValidationError( + "A condition set has a minimum value above the maximum value." + ) + ct = buildConditionTable(axisIndices[axisTag], minValue, maxValue) + conditionTable.append(ct) + records = [] + for varFeatureIndex in sorted(varFeatureIndices): + existingLookupIndices = table.FeatureList.FeatureRecord[ + varFeatureIndex + ].Feature.LookupListIndex + combinedLookupIndices = ( + existingLookupIndices + lookupIndices + if processLast + else lookupIndices + existingLookupIndices + ) + + records.append( + buildFeatureTableSubstitutionRecord( + varFeatureIndex, combinedLookupIndices + ) + ) + if hasFeatureVariations and ( + fvr := findFeatureVariationRecord(table.FeatureVariations, conditionTable) + ): + fvr.FeatureTableSubstitution.SubstitutionRecord.extend(records) + fvr.FeatureTableSubstitution.SubstitutionCount = len( + fvr.FeatureTableSubstitution.SubstitutionRecord + ) + else: + featureVariationRecords.append( + buildFeatureVariationRecord(conditionTable, records) + ) + + if hasFeatureVariations: + if table.FeatureVariations.Version != 0x00010000: + raise VarLibError( + "Unsupported FeatureVariations table version: " + f"0x{table.FeatureVariations.Version:08x} (expected 0x00010000)." + ) + table.FeatureVariations.FeatureVariationRecord.extend(featureVariationRecords) + table.FeatureVariations.FeatureVariationCount = len( + table.FeatureVariations.FeatureVariationRecord + ) + else: + table.FeatureVariations = buildFeatureVariations(featureVariationRecords) + + +# +# Building GSUB/FeatureVariations internals +# + + +def buildGSUB(): + """Build a GSUB table from scratch.""" + fontTable = newTable("GSUB") + gsub = fontTable.table = ot.GSUB() + gsub.Version = 0x00010001 # allow gsub.FeatureVariations + + gsub.ScriptList = ot.ScriptList() + gsub.ScriptList.ScriptRecord = [] + gsub.FeatureList = ot.FeatureList() + gsub.FeatureList.FeatureRecord = [] + gsub.LookupList = ot.LookupList() + gsub.LookupList.Lookup = [] + + srec = ot.ScriptRecord() + srec.ScriptTag = "DFLT" + srec.Script = ot.Script() + srec.Script.DefaultLangSys = None + srec.Script.LangSysRecord = [] + srec.Script.LangSysCount = 0 + + langrec = ot.LangSysRecord() + langrec.LangSys = ot.LangSys() + langrec.LangSys.ReqFeatureIndex = 0xFFFF + langrec.LangSys.FeatureIndex = [] + srec.Script.DefaultLangSys = langrec.LangSys + + gsub.ScriptList.ScriptRecord.append(srec) + gsub.ScriptList.ScriptCount = 1 + gsub.FeatureVariations = None + + return fontTable + + +def makeSubstitutionsHashable(conditionalSubstitutions): + """Turn all the substitution dictionaries in sorted tuples of tuples so + they are hashable, to detect duplicates so we don't write out redundant + data.""" + allSubstitutions = set() + condSubst = [] + for conditionSet, substitutionMaps in conditionalSubstitutions: + substitutions = [] + for substitutionMap in substitutionMaps: + subst = tuple(sorted(substitutionMap.items())) + substitutions.append(subst) + allSubstitutions.add(subst) + condSubst.append((conditionSet, substitutions)) + return condSubst, sorted(allSubstitutions) + + +class ShifterVisitor(TTVisitor): + def __init__(self, shift): + self.shift = shift + + +@ShifterVisitor.register_attr(ot.Feature, "LookupListIndex") # GSUB/GPOS +def visit(visitor, obj, attr, value): + shift = visitor.shift + value = [l + shift for l in value] + setattr(obj, attr, value) + + +@ShifterVisitor.register_attr( + (ot.SubstLookupRecord, ot.PosLookupRecord), "LookupListIndex" +) +def visit(visitor, obj, attr, value): + setattr(obj, attr, visitor.shift + value) + + +def buildSubstitutionLookups(gsub, allSubstitutions, processLast=False): + """Build the lookups for the glyph substitutions, return a dict mapping + the substitution to lookup indices.""" + + # Insert lookups at the beginning of the lookup vector + # https://github.com/googlefonts/fontmake/issues/950 + + firstIndex = len(gsub.LookupList.Lookup) if processLast else 0 + lookupMap = {} + for i, substitutionMap in enumerate(allSubstitutions): + lookupMap[substitutionMap] = firstIndex + i + + if not processLast: + # Shift all lookup indices in gsub by len(allSubstitutions) + shift = len(allSubstitutions) + visitor = ShifterVisitor(shift) + visitor.visit(gsub.FeatureList.FeatureRecord) + visitor.visit(gsub.LookupList.Lookup) + + for i, subst in enumerate(allSubstitutions): + substMap = dict(subst) + lookup = buildLookup([buildSingleSubstSubtable(substMap)]) + if processLast: + gsub.LookupList.Lookup.append(lookup) + else: + gsub.LookupList.Lookup.insert(i, lookup) + assert gsub.LookupList.Lookup[lookupMap[subst]] is lookup + gsub.LookupList.LookupCount = len(gsub.LookupList.Lookup) + return lookupMap + + +def buildFeatureVariations(featureVariationRecords): + """Build the FeatureVariations subtable.""" + fv = ot.FeatureVariations() + fv.Version = 0x00010000 + fv.FeatureVariationRecord = featureVariationRecords + fv.FeatureVariationCount = len(featureVariationRecords) + return fv + + +def buildFeatureRecord(featureTag, lookupListIndices): + """Build a FeatureRecord.""" + fr = ot.FeatureRecord() + fr.FeatureTag = featureTag + fr.Feature = ot.Feature() + fr.Feature.LookupListIndex = lookupListIndices + fr.Feature.populateDefaults() + return fr + + +def buildFeatureVariationRecord(conditionTable, substitutionRecords): + """Build a FeatureVariationRecord.""" + fvr = ot.FeatureVariationRecord() + if len(conditionTable) != 0: + fvr.ConditionSet = ot.ConditionSet() + fvr.ConditionSet.ConditionTable = conditionTable + fvr.ConditionSet.ConditionCount = len(conditionTable) + else: + fvr.ConditionSet = None + fvr.FeatureTableSubstitution = ot.FeatureTableSubstitution() + fvr.FeatureTableSubstitution.Version = 0x00010000 + fvr.FeatureTableSubstitution.SubstitutionRecord = substitutionRecords + fvr.FeatureTableSubstitution.SubstitutionCount = len(substitutionRecords) + return fvr + + +def buildFeatureTableSubstitutionRecord(featureIndex, lookupListIndices): + """Build a FeatureTableSubstitutionRecord.""" + ftsr = ot.FeatureTableSubstitutionRecord() + ftsr.FeatureIndex = featureIndex + ftsr.Feature = ot.Feature() + ftsr.Feature.LookupListIndex = lookupListIndices + ftsr.Feature.LookupCount = len(lookupListIndices) + return ftsr + + +def buildConditionTable(axisIndex, filterRangeMinValue, filterRangeMaxValue): + """Build a ConditionTable.""" + ct = ot.ConditionTable() + ct.Format = 1 + ct.AxisIndex = axisIndex + ct.FilterRangeMinValue = filterRangeMinValue + ct.FilterRangeMaxValue = filterRangeMaxValue + return ct + + +def findFeatureVariationRecord(featureVariations, conditionTable): + """Find a FeatureVariationRecord that has the same conditionTable.""" + if featureVariations.Version != 0x00010000: + raise VarLibError( + "Unsupported FeatureVariations table version: " + f"0x{featureVariations.Version:08x} (expected 0x00010000)." + ) + + for fvr in featureVariations.FeatureVariationRecord: + if conditionTable == fvr.ConditionSet.ConditionTable: + return fvr + + return None + + +def sortFeatureList(table): + """Sort the feature list by feature tag, and remap the feature indices + elsewhere. This is needed after the feature list has been modified. + """ + # decorate, sort, undecorate, because we need to make an index remapping table + tagIndexFea = [ + (fea.FeatureTag, index, fea) + for index, fea in enumerate(table.FeatureList.FeatureRecord) + ] + tagIndexFea.sort() + table.FeatureList.FeatureRecord = [fea for tag, index, fea in tagIndexFea] + featureRemap = dict( + zip([index for tag, index, fea in tagIndexFea], range(len(tagIndexFea))) + ) + + # Remap the feature indices + remapFeatures(table, featureRemap) + + +def remapFeatures(table, featureRemap): + """Go through the scripts list, and remap feature indices.""" + for scriptIndex, script in enumerate(table.ScriptList.ScriptRecord): + defaultLangSys = script.Script.DefaultLangSys + if defaultLangSys is not None: + _remapLangSys(defaultLangSys, featureRemap) + for langSysRecordIndex, langSysRec in enumerate(script.Script.LangSysRecord): + langSys = langSysRec.LangSys + _remapLangSys(langSys, featureRemap) + + if hasattr(table, "FeatureVariations") and table.FeatureVariations is not None: + for fvr in table.FeatureVariations.FeatureVariationRecord: + for ftsr in fvr.FeatureTableSubstitution.SubstitutionRecord: + ftsr.FeatureIndex = featureRemap[ftsr.FeatureIndex] + + +def _remapLangSys(langSys, featureRemap): + if langSys.ReqFeatureIndex != 0xFFFF: + langSys.ReqFeatureIndex = featureRemap[langSys.ReqFeatureIndex] + langSys.FeatureIndex = [featureRemap[index] for index in langSys.FeatureIndex] + + +if __name__ == "__main__": + import doctest, sys + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/hvar.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/hvar.py new file mode 100644 index 0000000000000000000000000000000000000000..14b662ebf72f11a7981da23c65144080896e642b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/hvar.py @@ -0,0 +1,113 @@ +from fontTools.misc.roundTools import noRound +from fontTools.ttLib import TTFont, newTable +from fontTools.ttLib.tables import otTables as ot +from fontTools.ttLib.tables.otBase import OTTableWriter +from fontTools.varLib import HVAR_FIELDS, VVAR_FIELDS, _add_VHVAR +from fontTools.varLib import builder, models, varStore +from fontTools.misc.fixedTools import fixedToFloat as fi2fl +from fontTools.misc.cliTools import makeOutputFileName +from functools import partial +import logging + +log = logging.getLogger("fontTools.varLib.avar") + + +def _get_advance_metrics(font, axisTags, tableFields): + # There's two ways we can go from here: + # 1. For each glyph, at each master peak, compute the value of the + # advance width at that peak. Then pass these all to a VariationModel + # builder to compute back the deltas. + # 2. For each master peak, pull out the deltas of the advance width directly, + # and feed these to the VarStoreBuilder, forgoing the remodeling step. + # We'll go with the second option, as it's simpler, faster, and more direct. + gvar = font["gvar"] + vhAdvanceDeltasAndSupports = {} + glyphOrder = font.getGlyphOrder() + phantomIndex = tableFields.phantomIndex + for glyphName in glyphOrder: + supports = [] + deltas = [] + variations = gvar.variations.get(glyphName, []) + + for tv in variations: + supports.append(tv.axes) + phantoms = tv.coordinates[-4:] + phantoms = phantoms[phantomIndex * 2 : phantomIndex * 2 + 2] + assert len(phantoms) == 2 + phantoms[0] = phantoms[0][phantomIndex] if phantoms[0] is not None else 0 + phantoms[1] = phantoms[1][phantomIndex] if phantoms[1] is not None else 0 + deltas.append(phantoms[1] - phantoms[0]) + + vhAdvanceDeltasAndSupports[glyphName] = (deltas, supports) + + vOrigDeltasAndSupports = None # TODO + + return vhAdvanceDeltasAndSupports, vOrigDeltasAndSupports + + +def add_HVAR(font): + if "HVAR" in font: + del font["HVAR"] + axisTags = [axis.axisTag for axis in font["fvar"].axes] + getAdvanceMetrics = partial(_get_advance_metrics, font, axisTags, HVAR_FIELDS) + _add_VHVAR(font, axisTags, HVAR_FIELDS, getAdvanceMetrics) + + +def add_VVAR(font): + if "VVAR" in font: + del font["VVAR"] + getAdvanceMetrics = partial(_get_advance_metrics, font, axisTags, VVAR_FIELDS) + axisTags = [axis.axisTag for axis in font["fvar"].axes] + _add_VHVAR(font, axisTags, VVAR_FIELDS, getAdvanceMetrics) + + +def main(args=None): + """Add `HVAR` table to variable font.""" + + if args is None: + import sys + + args = sys.argv[1:] + + from fontTools import configLogger + from fontTools.designspaceLib import DesignSpaceDocument + import argparse + + parser = argparse.ArgumentParser( + "fonttools varLib.hvar", + description="Add `HVAR` table from to variable font.", + ) + parser.add_argument("font", metavar="varfont.ttf", help="Variable-font file.") + parser.add_argument( + "-o", + "--output-file", + type=str, + help="Output font file name.", + ) + + options = parser.parse_args(args) + + configLogger(level="WARNING") + + font = TTFont(options.font) + if not "fvar" in font: + log.error("Not a variable font.") + return 1 + + add_HVAR(font) + if "vmtx" in font: + add_VVAR(font) + + if options.output_file is None: + outfile = makeOutputFileName(options.font, overWrite=True, suffix=".hvar") + else: + outfile = options.output_file + if outfile: + log.info("Saving %s", outfile) + font.save(outfile) + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/__init__.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dc6bf18e586aac5dc85fe040235ddafbc0d514ec --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/__init__.py @@ -0,0 +1,2049 @@ +"""Partially instantiate a variable font. + +The module exports an `instantiateVariableFont` function and CLI that allow to +create full instances (i.e. static fonts) from variable fonts, as well as "partial" +variable fonts that only contain a subset of the original variation space. + +For example, if you wish to pin the width axis to a given location while also +restricting the weight axis to 400..700 range, you can do: + +.. code-block:: sh + + $ fonttools varLib.instancer ./NotoSans-VF.ttf wdth=85 wght=400:700 + +See `fonttools varLib.instancer --help` for more info on the CLI options. + +The module's entry point is the `instantiateVariableFont` function, which takes +a TTFont object and a dict specifying either axis coodinates or (min, max) ranges, +and returns a new TTFont representing either a partial VF, or full instance if all +the VF axes were given an explicit coordinate. + +E.g. here's how to pin the wght axis at a given location in a wght+wdth variable +font, keeping only the deltas associated with the wdth axis: +.. code-block:: pycon + + >>> + >> from fontTools import ttLib + >> from fontTools.varLib import instancer + >> varfont = ttLib.TTFont("path/to/MyVariableFont.ttf") + >> [a.axisTag for a in varfont["fvar"].axes] # the varfont's current axes + ['wght', 'wdth'] + >> partial = instancer.instantiateVariableFont(varfont, {"wght": 300}) + >> [a.axisTag for a in partial["fvar"].axes] # axes left after pinning 'wght' + ['wdth'] + +If the input location specifies all the axes, the resulting instance is no longer +'variable' (same as using fontools varLib.mutator): +.. code-block:: pycon + + >>> + >> instance = instancer.instantiateVariableFont( + ... varfont, {"wght": 700, "wdth": 67.5} + ... ) + >> "fvar" not in instance + True + +If one just want to drop an axis at the default location, without knowing in +advance what the default value for that axis is, one can pass a `None` value: +.. code-block:: pycon + + >>> + >> instance = instancer.instantiateVariableFont(varfont, {"wght": None}) + >> len(varfont["fvar"].axes) + 1 + +From the console script, this is equivalent to passing `wght=drop` as input. + +This module is similar to fontTools.varLib.mutator, which it's intended to supersede. +Note that, unlike varLib.mutator, when an axis is not mentioned in the input +location, by default the varLib.instancer will keep the axis and the corresponding +deltas, whereas mutator implicitly drops the axis at its default coordinate. +To obtain the same behavior as mutator, pass the `static=True` parameter or +the `--static` CLI option. + +The module supports all the following "levels" of instancing, which can of +course be combined: + +L1 + dropping one or more axes while leaving the default tables unmodified; + .. code-block:: pycon + + >>> + >> font = instancer.instantiateVariableFont(varfont, {"wght": None}) + +L2 + dropping one or more axes while pinning them at non-default locations; + .. code-block:: pycon + + >>> + >> font = instancer.instantiateVariableFont(varfont, {"wght": 700}) + +L3 + restricting the range of variation of one or more axes, by setting either + a new minimum or maximum, potentially -- though not necessarily -- dropping + entire regions of variations that fall completely outside this new range. + .. code-block:: pycon + + >>> + >> font = instancer.instantiateVariableFont(varfont, {"wght": (100, 300)}) + +L4 + moving the default location of an axis, by specifying (min,defalt,max) values: + .. code-block:: pycon + + >>> + >> font = instancer.instantiateVariableFont(varfont, {"wght": (100, 300, 700)}) + +Both TrueType-flavored (glyf+gvar) variable and CFF2 variable fonts are supported. +""" + +from fontTools.misc.fixedTools import ( + floatToFixedToFloat, + strToFixedToFloat, + otRound, +) +from fontTools.varLib.models import normalizeValue, piecewiseLinearMap +from fontTools.ttLib import TTFont, newTable +from fontTools.ttLib.tables.TupleVariation import TupleVariation +from fontTools.ttLib.tables import _g_l_y_f +from fontTools import varLib + +# we import the `subset` module because we use the `prune_lookups` method on the GSUB +# table class, and that method is only defined dynamically upon importing `subset` +from fontTools import subset # noqa: F401 +from fontTools.cffLib import privateDictOperators2 +from fontTools.cffLib.specializer import ( + programToCommands, + commandsToProgram, + specializeCommands, + generalizeCommands, +) +from fontTools.cffLib.CFF2ToCFF import convertCFF2ToCFF +from fontTools.varLib import builder +from fontTools.varLib.mvar import MVAR_ENTRIES +from fontTools.varLib.merger import MutatorMerger +from fontTools.varLib.instancer import names +from .featureVars import instantiateFeatureVariations +from fontTools.misc.cliTools import makeOutputFileName +from fontTools.varLib.instancer import solver +from fontTools.ttLib.tables.otTables import VarComponentFlags +import collections +import dataclasses +from contextlib import contextmanager +from copy import deepcopy +from enum import IntEnum +import logging +import os +import re +import io +from typing import Dict, Iterable, Mapping, Optional, Sequence, Tuple, Union +import warnings + + +log = logging.getLogger("fontTools.varLib.instancer") + + +def AxisRange(minimum, maximum): + warnings.warn( + "AxisRange is deprecated; use AxisTriple instead", + DeprecationWarning, + stacklevel=2, + ) + return AxisTriple(minimum, None, maximum) + + +def NormalizedAxisRange(minimum, maximum): + warnings.warn( + "NormalizedAxisRange is deprecated; use AxisTriple instead", + DeprecationWarning, + stacklevel=2, + ) + return NormalizedAxisTriple(minimum, None, maximum) + + +@dataclasses.dataclass(frozen=True, order=True, repr=False) +class AxisTriple(Sequence): + """A triple of (min, default, max) axis values. + + Any of the values can be None, in which case the limitRangeAndPopulateDefaults() + method can be used to fill in the missing values based on the fvar axis values. + """ + + minimum: Optional[float] + default: Optional[float] + maximum: Optional[float] + + def __post_init__(self): + if self.default is None and self.minimum == self.maximum: + object.__setattr__(self, "default", self.minimum) + if ( + ( + self.minimum is not None + and self.default is not None + and self.minimum > self.default + ) + or ( + self.default is not None + and self.maximum is not None + and self.default > self.maximum + ) + or ( + self.minimum is not None + and self.maximum is not None + and self.minimum > self.maximum + ) + ): + raise ValueError( + f"{type(self).__name__} minimum ({self.minimum}), default ({self.default}), maximum ({self.maximum}) must be in sorted order" + ) + + def __getitem__(self, i): + fields = dataclasses.fields(self) + return getattr(self, fields[i].name) + + def __len__(self): + return len(dataclasses.fields(self)) + + def _replace(self, **kwargs): + return dataclasses.replace(self, **kwargs) + + def __repr__(self): + return ( + f"({', '.join(format(v, 'g') if v is not None else 'None' for v in self)})" + ) + + @classmethod + def expand( + cls, + v: Union[ + "AxisTriple", + float, # pin axis at single value, same as min==default==max + Tuple[float, float], # (min, max), restrict axis and keep default + Tuple[float, float, float], # (min, default, max) + ], + ) -> "AxisTriple": + """Convert a single value or a tuple into an AxisTriple. + + If the input is a single value, it is interpreted as a pin at that value. + If the input is a tuple, it is interpreted as (min, max) or (min, default, max). + """ + if isinstance(v, cls): + return v + if isinstance(v, (int, float)): + return cls(v, v, v) + try: + n = len(v) + except TypeError as e: + raise ValueError( + f"expected float, 2- or 3-tuple of floats; got {type(v)}: {v!r}" + ) from e + default = None + if n == 2: + minimum, maximum = v + elif n >= 3: + return cls(*v) + else: + raise ValueError(f"expected sequence of 2 or 3; got {n}: {v!r}") + return cls(minimum, default, maximum) + + def limitRangeAndPopulateDefaults(self, fvarTriple) -> "AxisTriple": + """Return a new AxisTriple with the default value filled in. + + Set default to fvar axis default if the latter is within the min/max range, + otherwise set default to the min or max value, whichever is closer to the + fvar axis default. + If the default value is already set, return self. + """ + minimum = self.minimum + if minimum is None: + minimum = fvarTriple[0] + default = self.default + if default is None: + default = fvarTriple[1] + maximum = self.maximum + if maximum is None: + maximum = fvarTriple[2] + + minimum = max(minimum, fvarTriple[0]) + maximum = max(maximum, fvarTriple[0]) + minimum = min(minimum, fvarTriple[2]) + maximum = min(maximum, fvarTriple[2]) + default = max(minimum, min(maximum, default)) + + return AxisTriple(minimum, default, maximum) + + +@dataclasses.dataclass(frozen=True, order=True, repr=False) +class NormalizedAxisTriple(AxisTriple): + """A triple of (min, default, max) normalized axis values.""" + + minimum: float + default: float + maximum: float + + def __post_init__(self): + if self.default is None: + object.__setattr__(self, "default", max(self.minimum, min(self.maximum, 0))) + if not (-1.0 <= self.minimum <= self.default <= self.maximum <= 1.0): + raise ValueError( + "Normalized axis values not in -1..+1 range; got " + f"minimum={self.minimum:g}, default={self.default:g}, maximum={self.maximum:g})" + ) + + +@dataclasses.dataclass(frozen=True, order=True, repr=False) +class NormalizedAxisTripleAndDistances(AxisTriple): + """A triple of (min, default, max) normalized axis values, + with distances between min and default, and default and max, + in the *pre-normalized* space.""" + + minimum: float + default: float + maximum: float + distanceNegative: Optional[float] = 1 + distancePositive: Optional[float] = 1 + + def __post_init__(self): + if self.default is None: + object.__setattr__(self, "default", max(self.minimum, min(self.maximum, 0))) + if not (-1.0 <= self.minimum <= self.default <= self.maximum <= 1.0): + raise ValueError( + "Normalized axis values not in -1..+1 range; got " + f"minimum={self.minimum:g}, default={self.default:g}, maximum={self.maximum:g})" + ) + + def reverse_negate(self): + v = self + return self.__class__(-v[2], -v[1], -v[0], v[4], v[3]) + + def renormalizeValue(self, v, extrapolate=True): + """Renormalizes a normalized value v to the range of this axis, + considering the pre-normalized distances as well as the new + axis limits.""" + + lower, default, upper, distanceNegative, distancePositive = self + assert lower <= default <= upper + + if not extrapolate: + v = max(lower, min(upper, v)) + + if v == default: + return 0 + + if default < 0: + return -self.reverse_negate().renormalizeValue(-v, extrapolate=extrapolate) + + # default >= 0 and v != default + + if v > default: + return (v - default) / (upper - default) + + # v < default + + if lower >= 0: + return (v - default) / (default - lower) + + # lower < 0 and v < default + + totalDistance = distanceNegative * -lower + distancePositive * default + + if v >= 0: + vDistance = (default - v) * distancePositive + else: + vDistance = -v * distanceNegative + distancePositive * default + + return -vDistance / totalDistance + + +class _BaseAxisLimits(Mapping[str, AxisTriple]): + def __getitem__(self, key: str) -> AxisTriple: + return self._data[key] + + def __iter__(self) -> Iterable[str]: + return iter(self._data) + + def __len__(self) -> int: + return len(self._data) + + def __repr__(self) -> str: + return f"{type(self).__name__}({self._data!r})" + + def __str__(self) -> str: + return str(self._data) + + def defaultLocation(self) -> Dict[str, float]: + """Return a dict of default axis values.""" + return {k: v.default for k, v in self.items()} + + def pinnedLocation(self) -> Dict[str, float]: + """Return a location dict with only the pinned axes.""" + return {k: v.default for k, v in self.items() if v.minimum == v.maximum} + + +class AxisLimits(_BaseAxisLimits): + """Maps axis tags (str) to AxisTriple values.""" + + def __init__(self, *args, **kwargs): + self._data = data = {} + for k, v in dict(*args, **kwargs).items(): + if v is None: + # will be filled in by limitAxesAndPopulateDefaults + data[k] = v + else: + try: + triple = AxisTriple.expand(v) + except ValueError as e: + raise ValueError(f"Invalid axis limits for {k!r}: {v!r}") from e + data[k] = triple + + def limitAxesAndPopulateDefaults(self, varfont) -> "AxisLimits": + """Return a new AxisLimits with defaults filled in from fvar table. + + If all axis limits already have defaults, return self. + """ + fvar = varfont["fvar"] + fvarTriples = { + a.axisTag: (a.minValue, a.defaultValue, a.maxValue) for a in fvar.axes + } + newLimits = {} + for axisTag, triple in self.items(): + fvarTriple = fvarTriples[axisTag] + default = fvarTriple[1] + if triple is None: + newLimits[axisTag] = AxisTriple(default, default, default) + else: + newLimits[axisTag] = triple.limitRangeAndPopulateDefaults(fvarTriple) + return type(self)(newLimits) + + def normalize(self, varfont, usingAvar=True) -> "NormalizedAxisLimits": + """Return a new NormalizedAxisLimits with normalized -1..0..+1 values. + + If usingAvar is True, the avar table is used to warp the default normalization. + """ + fvar = varfont["fvar"] + badLimits = set(self.keys()).difference(a.axisTag for a in fvar.axes) + if badLimits: + raise ValueError("Cannot limit: {} not present in fvar".format(badLimits)) + + axes = { + a.axisTag: (a.minValue, a.defaultValue, a.maxValue) + for a in fvar.axes + if a.axisTag in self + } + + avarSegments = {} + if usingAvar and "avar" in varfont: + avar = varfont["avar"] + avarSegments = avar.segments + + if getattr(avar, "majorVersion", 1) >= 2 and avar.table.VarStore: + pinnedAxes = set(self.pinnedLocation()) + if not pinnedAxes.issuperset(avarSegments): + raise NotImplementedError( + "Partial-instancing avar2 table is not supported" + ) + + # TODO: Merge this with the main codepath. + + # Full instancing of avar2 font. Use avar table to normalize location and return. + location = self.pinnedLocation() + location = { + tag: normalize(value, axes[tag], None) + for tag, value in location.items() + } + return NormalizedAxisLimits( + **avar.renormalizeLocation(location, varfont, dropZeroes=False) + ) + + normalizedLimits = {} + + for axis_tag, triple in axes.items(): + distanceNegative = triple[1] - triple[0] + distancePositive = triple[2] - triple[1] + + if self[axis_tag] is None: + normalizedLimits[axis_tag] = NormalizedAxisTripleAndDistances( + 0, 0, 0, distanceNegative, distancePositive + ) + continue + + minV, defaultV, maxV = self[axis_tag] + + if defaultV is None: + defaultV = triple[1] + + avarMapping = avarSegments.get(axis_tag, None) + normalizedLimits[axis_tag] = NormalizedAxisTripleAndDistances( + *(normalize(v, triple, avarMapping) for v in (minV, defaultV, maxV)), + distanceNegative, + distancePositive, + ) + + return NormalizedAxisLimits(normalizedLimits) + + +class NormalizedAxisLimits(_BaseAxisLimits): + """Maps axis tags (str) to NormalizedAxisTriple values.""" + + def __init__(self, *args, **kwargs): + self._data = data = {} + for k, v in dict(*args, **kwargs).items(): + try: + triple = NormalizedAxisTripleAndDistances.expand(v) + except ValueError as e: + raise ValueError(f"Invalid axis limits for {k!r}: {v!r}") from e + data[k] = triple + + +class OverlapMode(IntEnum): + KEEP_AND_DONT_SET_FLAGS = 0 + KEEP_AND_SET_FLAGS = 1 + REMOVE = 2 + REMOVE_AND_IGNORE_ERRORS = 3 + + +def instantiateVARC(varfont, axisLimits): + log.info("Instantiating VARC tables") + + # TODO(behdad) My confidence in this function is rather low; + # It needs more testing. Specially with partial-instancing, + # I don't think it currently works. + + varc = varfont["VARC"].table + fvarAxes = varfont["fvar"].axes if "fvar" in varfont else [] + + location = axisLimits.pinnedLocation() + axisMap = [i for i, axis in enumerate(fvarAxes) if axis.axisTag not in location] + reverseAxisMap = {i: j for j, i in enumerate(axisMap)} + + if varc.AxisIndicesList: + axisIndicesList = varc.AxisIndicesList.Item + for i, axisIndices in enumerate(axisIndicesList): + if any(fvarAxes[j].axisTag in axisLimits for j in axisIndices): + raise NotImplementedError( + "Instancing across VarComponent axes is not supported." + ) + axisIndicesList[i] = [reverseAxisMap[j] for j in axisIndices] + + store = varc.MultiVarStore + if store: + for region in store.SparseVarRegionList.Region: + newRegionAxis = [] + for regionRecord in region.SparseVarRegionAxis: + tag = fvarAxes[regionRecord.AxisIndex].axisTag + if tag in axisLimits: + raise NotImplementedError( + "Instancing across VarComponent axes is not supported." + ) + regionRecord.AxisIndex = reverseAxisMap[regionRecord.AxisIndex] + + +def instantiateTupleVariationStore( + variations, axisLimits, origCoords=None, endPts=None +): + """Instantiate TupleVariation list at the given location, or limit axes' min/max. + + The 'variations' list of TupleVariation objects is modified in-place. + The 'axisLimits' (dict) maps axis tags (str) to NormalizedAxisTriple namedtuples + specifying (minimum, default, maximum) in the -1,0,+1 normalized space. Pinned axes + have minimum == default == maximum. + + A 'full' instance (i.e. static font) is produced when all the axes are pinned to + single coordinates; a 'partial' instance (i.e. a less variable font) is produced + when some of the axes are omitted, or restricted with a new range. + + Tuples that do not participate are kept as they are. Those that have 0 influence + at the given location are removed from the variation store. + Those that are fully instantiated (i.e. all their axes are being pinned) are also + removed from the variation store, their scaled deltas accummulated and returned, so + that they can be added by the caller to the default instance's coordinates. + Tuples that are only partially instantiated (i.e. not all the axes that they + participate in are being pinned) are kept in the store, and their deltas multiplied + by the scalar support of the axes to be pinned at the desired location. + + Args: + variations: List[TupleVariation] from either 'gvar' or 'cvar'. + axisLimits: NormalizedAxisLimits: map from axis tags to (min, default, max) + normalized coordinates for the full or partial instance. + origCoords: GlyphCoordinates: default instance's coordinates for computing 'gvar' + inferred points (cf. table__g_l_y_f._getCoordinatesAndControls). + endPts: List[int]: indices of contour end points, for inferring 'gvar' deltas. + + Returns: + List[float]: the overall delta adjustment after applicable deltas were summed. + """ + + newVariations = changeTupleVariationsAxisLimits(variations, axisLimits) + + mergedVariations = collections.OrderedDict() + for var in newVariations: + # compute inferred deltas only for gvar ('origCoords' is None for cvar) + if origCoords is not None: + var.calcInferredDeltas(origCoords, endPts) + + # merge TupleVariations with overlapping "tents" + axes = frozenset(var.axes.items()) + if axes in mergedVariations: + mergedVariations[axes] += var + else: + mergedVariations[axes] = var + + # drop TupleVariation if all axes have been pinned (var.axes.items() is empty); + # its deltas will be added to the default instance's coordinates + defaultVar = mergedVariations.pop(frozenset(), None) + + for var in mergedVariations.values(): + var.roundDeltas() + variations[:] = list(mergedVariations.values()) + + return defaultVar.coordinates if defaultVar is not None else [] + + +def changeTupleVariationsAxisLimits(variations, axisLimits): + for axisTag, axisLimit in sorted(axisLimits.items()): + newVariations = [] + for var in variations: + newVariations.extend(changeTupleVariationAxisLimit(var, axisTag, axisLimit)) + variations = newVariations + return variations + + +def changeTupleVariationAxisLimit(var, axisTag, axisLimit): + assert isinstance(axisLimit, NormalizedAxisTripleAndDistances) + + # Skip when current axis is missing or peaks at 0 (i.e. doesn't participate) + lower, peak, upper = var.axes.get(axisTag, (-1, 0, 1)) + if peak == 0: + # explicitly defined, no-op axes can be omitted + # https://github.com/fonttools/fonttools/issues/3453 + if axisTag in var.axes: + del var.axes[axisTag] + return [var] + # Drop if the var 'tent' isn't well-formed + if not (lower <= peak <= upper) or (lower < 0 and upper > 0): + return [] + + if axisTag not in var.axes: + return [var] + + tent = var.axes[axisTag] + + solutions = solver.rebaseTent(tent, axisLimit) + + out = [] + for scalar, tent in solutions: + newVar = ( + TupleVariation(var.axes, var.coordinates) if len(solutions) > 1 else var + ) + if tent is None: + newVar.axes.pop(axisTag) + else: + assert tent[1] != 0, tent + newVar.axes[axisTag] = tent + newVar *= scalar + out.append(newVar) + + return out + + +def instantiateCFF2( + varfont, + axisLimits, + *, + round=round, + specialize=True, + generalize=False, + downgrade=False, +): + # The algorithm here is rather simple: + # + # Take all blend operations and store their deltas in the (otherwise empty) + # CFF2 VarStore. Then, instantiate the VarStore with the given axis limits, + # and read back the new deltas. This is done for both the CharStrings and + # the Private dicts. + # + # Then prune unused things and possibly drop the VarStore if it's empty. + # + # If the downgrade parameter is True, no actual downgrading is done, but + # the function returns True if the VarStore was empty after instantiation, + # and hence a downgrade to CFF is possible. In all other cases it returns + # False. + + log.info("Instantiating CFF2 table") + + fvarAxes = varfont["fvar"].axes + + cff = varfont["CFF2"].cff + topDict = cff.topDictIndex[0] + varStore = topDict.VarStore.otVarStore + if not varStore: + if downgrade: + from fontTools.cffLib.CFF2ToCFF import convertCFF2ToCFF + + convertCFF2ToCFF(varfont) + return + + cff.desubroutinize() + + def getNumRegions(vsindex): + return varStore.VarData[vsindex if vsindex is not None else 0].VarRegionCount + + charStrings = topDict.CharStrings.values() + + # Gather all unique private dicts + uniquePrivateDicts = set() + privateDicts = [] + for fd in topDict.FDArray: + if fd.Private not in uniquePrivateDicts: + uniquePrivateDicts.add(fd.Private) + privateDicts.append(fd.Private) + + allCommands = [] + allCommandPrivates = [] + for cs in charStrings: + assert cs.private.vstore.otVarStore is varStore # Or in many places!! + commands = programToCommands(cs.program, getNumRegions=getNumRegions) + if generalize: + commands = generalizeCommands(commands) + if specialize: + commands = specializeCommands(commands, generalizeFirst=not generalize) + allCommands.append(commands) + allCommandPrivates.append(cs.private) + + def storeBlendsToVarStore(arg): + if not isinstance(arg, list): + return + + if any(isinstance(subarg, list) for subarg in arg[:-1]): + raise NotImplementedError("Nested blend lists not supported (yet)") + + count = arg[-1] + assert (len(arg) - 1) % count == 0 + nRegions = (len(arg) - 1) // count - 1 + assert nRegions == getNumRegions(vsindex) + for i in range(count, len(arg) - 1, nRegions): + deltas = arg[i : i + nRegions] + assert len(deltas) == nRegions + varData = varStore.VarData[vsindex] + varData.Item.append(deltas) + varData.ItemCount += 1 + + def fetchBlendsFromVarStore(arg): + if not isinstance(arg, list): + return [arg] + + if any(isinstance(subarg, list) for subarg in arg[:-1]): + raise NotImplementedError("Nested blend lists not supported (yet)") + + count = arg[-1] + assert (len(arg) - 1) % count == 0 + numRegions = getNumRegions(vsindex) + newDefaults = [] + newDeltas = [] + for i in range(count): + defaultValue = arg[i] + + major = vsindex + minor = varDataCursor[major] + varDataCursor[major] += 1 + + defaultValue += round(defaultDeltas[major][minor]) + newDefaults.append(defaultValue) + + varData = varStore.VarData[major] + deltas = varData.Item[minor] + assert len(deltas) == numRegions + newDeltas.extend(deltas) + + if not numRegions: + return newDefaults # No deltas, just return the defaults + + return [newDefaults + newDeltas + [count]] + + # Check VarData's are empty + for varData in varStore.VarData: + assert varData.Item == [] + assert varData.ItemCount == 0 + + # Add charstring blend lists to VarStore so we can instantiate them + for commands, private in zip(allCommands, allCommandPrivates): + vsindex = getattr(private, "vsindex", 0) + for command in commands: + if command[0] == "vsindex": + vsindex = command[1][0] + continue + for arg in command[1]: + storeBlendsToVarStore(arg) + + # Add private blend lists to VarStore so we can instantiate values + for opcode, name, arg_type, default, converter in privateDictOperators2: + if arg_type not in ("number", "delta", "array"): + continue + + vsindex = 0 + for private in privateDicts: + if not hasattr(private, name): + continue + values = getattr(private, name) + + # This is safe here since "vsindex" is the first in the privateDictOperators2 + if name == "vsindex": + vsindex = values[0] + continue + + if arg_type == "number": + values = [values] + + for value in values: + if not isinstance(value, list): + continue + + assert len(value) % (getNumRegions(vsindex) + 1) == 0 + count = len(value) // (getNumRegions(vsindex) + 1) + storeBlendsToVarStore(value + [count]) + + # Instantiate VarStore + defaultDeltas = instantiateItemVariationStore( + varStore, fvarAxes, axisLimits, hierarchical=True + ) + + # Read back new charstring blends from the instantiated VarStore + varDataCursor = [0] * len(varStore.VarData) + for commands, private in zip(allCommands, allCommandPrivates): + vsindex = getattr(private, "vsindex", 0) + for command in commands: + if command[0] == "vsindex": + vsindex = command[1][0] + continue + newArgs = [] + for arg in command[1]: + newArgs.extend(fetchBlendsFromVarStore(arg)) + command[1][:] = newArgs + + # Read back new private blends from the instantiated VarStore + for opcode, name, arg_type, default, converter in privateDictOperators2: + if arg_type not in ("number", "delta", "array"): + continue + + vsindex = 0 + for private in privateDicts: + if not hasattr(private, name): + continue + + # This is safe here since "vsindex" is the first in the privateDictOperators2 + if name == "vsindex": + vsindex = values[0] + continue + + values = getattr(private, name) + if arg_type == "number": + values = [values] + + newValues = [] + for value in values: + if not isinstance(value, list): + newValues.append(value) + continue + + value.append(1) + value = fetchBlendsFromVarStore(value) + newValues.extend(v[:-1] if isinstance(v, list) else v for v in value) + + if arg_type == "number": + newValues = newValues[0] + + setattr(private, name, newValues) + + # Empty out the VarStore + for i, varData in enumerate(varStore.VarData): + assert varDataCursor[i] == varData.ItemCount, ( + varDataCursor[i], + varData.ItemCount, + ) + varData.Item = [] + varData.ItemCount = 0 + + # Collect surviving vsindexes + usedVsindex = set( + i for i in range(len(varStore.VarData)) if varStore.VarData[i].VarRegionCount + ) + # Remove vsindex commands that are no longer needed + for commands, private in zip(allCommands, allCommandPrivates): + if not any(isinstance(arg, list) for command in commands for arg in command[1]): + commands[:] = [command for command in commands if command[0] != "vsindex"] + + # Remove unused VarData and update vsindex values + vsindexMapping = {v: i for i, v in enumerate(sorted(usedVsindex))} + varStore.VarData = [ + varData for i, varData in enumerate(varStore.VarData) if i in usedVsindex + ] + varStore.VarDataCount = len(varStore.VarData) + for commands in allCommands: + for command in commands: + if command[0] == "vsindex": + command[1][0] = vsindexMapping[command[1][0]] + for private in privateDicts: + if hasattr(private, "vsindex"): + private.vsindex = vsindexMapping[private.vsindex] + + # Remove initial vsindex commands that are implied + for commands, private in zip(allCommands, allCommandPrivates): + vsindex = getattr(private, "vsindex", 0) + if commands and commands[0] == ("vsindex", [vsindex]): + commands.pop(0) + + # Ship the charstrings! + for cs, commands in zip(charStrings, allCommands): + cs.program = commandsToProgram(commands) + + # Remove empty VarStore + if not varStore.VarData: + if "VarStore" in topDict.rawDict: + del topDict.rawDict["VarStore"] + del topDict.VarStore + del topDict.CharStrings.varStore + for private in privateDicts: + del private.vstore + + if downgrade: + return True + + return False + + +def _instantiateGvarGlyph( + glyphname, glyf, gvar, hMetrics, vMetrics, axisLimits, optimize=True +): + coordinates, ctrl = glyf._getCoordinatesAndControls(glyphname, hMetrics, vMetrics) + endPts = ctrl.endPts + + # Not every glyph may have variations + tupleVarStore = gvar.variations.get(glyphname) + + if tupleVarStore: + defaultDeltas = instantiateTupleVariationStore( + tupleVarStore, axisLimits, coordinates, endPts + ) + + if defaultDeltas: + coordinates += _g_l_y_f.GlyphCoordinates(defaultDeltas) + + # _setCoordinates also sets the hmtx/vmtx advance widths and sidebearings from + # the four phantom points and glyph bounding boxes. + # We call it unconditionally even if a glyph has no variations or no deltas are + # applied at this location, in case the glyph's xMin and in turn its sidebearing + # have changed. E.g. a composite glyph has no deltas for the component's (x, y) + # offset nor for the 4 phantom points (e.g. it's monospaced). Thus its entry in + # gvar table is empty; however, the composite's base glyph may have deltas + # applied, hence the composite's bbox and left/top sidebearings may need updating + # in the instanced font. + glyf._setCoordinates(glyphname, coordinates, hMetrics, vMetrics) + + if not tupleVarStore: + if glyphname in gvar.variations: + del gvar.variations[glyphname] + return + + if optimize: + # IUP semantics depend on point equality, and so round prior to + # optimization to ensure that comparisons that happen now will be the + # same as those that happen at render time. This is especially needed + # when floating point deltas have been applied to the default position. + # See https://github.com/fonttools/fonttools/issues/3634 + # Rounding must happen only after calculating glyf metrics above, to + # preserve backwards compatibility. + # See 0010a3cd9aa25f84a3a6250dafb119743d32aa40 + coordinates.toInt() + for var in tupleVarStore: + var.optimize(coordinates, endPts) + + +def instantiateGvarGlyph(varfont, glyphname, axisLimits, optimize=True): + """Remove? + https://github.com/fonttools/fonttools/pull/2266""" + gvar = varfont["gvar"] + glyf = varfont["glyf"] + hMetrics = varfont["hmtx"].metrics + vMetrics = getattr(varfont.get("vmtx"), "metrics", None) + _instantiateGvarGlyph( + glyphname, glyf, gvar, hMetrics, vMetrics, axisLimits, optimize=optimize + ) + + +def instantiateGvar(varfont, axisLimits, optimize=True): + log.info("Instantiating glyf/gvar tables") + + gvar = varfont["gvar"] + glyf = varfont["glyf"] + hMetrics = varfont["hmtx"].metrics + vMetrics = getattr(varfont.get("vmtx"), "metrics", None) + # Get list of glyph names sorted by component depth. + # If a composite glyph is processed before its base glyph, the bounds may + # be calculated incorrectly because deltas haven't been applied to the + # base glyph yet. + glyphnames = sorted( + glyf.glyphOrder, + key=lambda name: ( + ( + glyf[name].getCompositeMaxpValues(glyf).maxComponentDepth + if glyf[name].isComposite() + else 0 + ), + name, + ), + ) + for glyphname in glyphnames: + _instantiateGvarGlyph( + glyphname, glyf, gvar, hMetrics, vMetrics, axisLimits, optimize=optimize + ) + + if not gvar.variations: + del varfont["gvar"] + + +def setCvarDeltas(cvt, deltas): + for i, delta in enumerate(deltas): + if delta: + cvt[i] += otRound(delta) + + +def instantiateCvar(varfont, axisLimits): + log.info("Instantiating cvt/cvar tables") + + cvar = varfont["cvar"] + + defaultDeltas = instantiateTupleVariationStore(cvar.variations, axisLimits) + + if defaultDeltas: + setCvarDeltas(varfont["cvt "], defaultDeltas) + + if not cvar.variations: + del varfont["cvar"] + + +def setMvarDeltas(varfont, deltas): + mvar = varfont["MVAR"].table + records = mvar.ValueRecord + for rec in records: + mvarTag = rec.ValueTag + if mvarTag not in MVAR_ENTRIES: + continue + tableTag, itemName = MVAR_ENTRIES[mvarTag] + delta = deltas[rec.VarIdx] + if delta != 0: + setattr( + varfont[tableTag], + itemName, + getattr(varfont[tableTag], itemName) + otRound(delta), + ) + + +@contextmanager +def verticalMetricsKeptInSync(varfont): + """Ensure hhea vertical metrics stay in sync with OS/2 ones after instancing. + + When applying MVAR deltas to the OS/2 table, if the ascender, descender and + line gap change but they were the same as the respective hhea metrics in the + original font, this context manager ensures that hhea metrcs also get updated + accordingly. + The MVAR spec only has tags for the OS/2 metrics, but it is common in fonts + to have the hhea metrics be equal to those for compat reasons. + + https://learn.microsoft.com/en-us/typography/opentype/spec/mvar + https://googlefonts.github.io/gf-guide/metrics.html#7-hhea-and-typo-metrics-should-be-equal + https://github.com/fonttools/fonttools/issues/3297 + """ + current_os2_vmetrics = [ + getattr(varfont["OS/2"], attr) + for attr in ("sTypoAscender", "sTypoDescender", "sTypoLineGap") + ] + metrics_are_synced = current_os2_vmetrics == [ + getattr(varfont["hhea"], attr) for attr in ("ascender", "descender", "lineGap") + ] + + yield metrics_are_synced + + if metrics_are_synced: + new_os2_vmetrics = [ + getattr(varfont["OS/2"], attr) + for attr in ("sTypoAscender", "sTypoDescender", "sTypoLineGap") + ] + if current_os2_vmetrics != new_os2_vmetrics: + for attr, value in zip( + ("ascender", "descender", "lineGap"), new_os2_vmetrics + ): + setattr(varfont["hhea"], attr, value) + + +def instantiateMVAR(varfont, axisLimits): + log.info("Instantiating MVAR table") + + mvar = varfont["MVAR"].table + fvarAxes = varfont["fvar"].axes + varStore = mvar.VarStore + defaultDeltas = instantiateItemVariationStore(varStore, fvarAxes, axisLimits) + + with verticalMetricsKeptInSync(varfont): + setMvarDeltas(varfont, defaultDeltas) + + if varStore.VarRegionList.Region: + varIndexMapping = varStore.optimize() + for rec in mvar.ValueRecord: + rec.VarIdx = varIndexMapping[rec.VarIdx] + else: + del varfont["MVAR"] + + +def _remapVarIdxMap(table, attrName, varIndexMapping, glyphOrder): + oldMapping = getattr(table, attrName).mapping + newMapping = [varIndexMapping[oldMapping[glyphName]] for glyphName in glyphOrder] + setattr(table, attrName, builder.buildVarIdxMap(newMapping, glyphOrder)) + + +# TODO(anthrotype) Add support for HVAR/VVAR in CFF2 +def _instantiateVHVAR(varfont, axisLimits, tableFields, *, round=round): + location = axisLimits.pinnedLocation() + tableTag = tableFields.tableTag + fvarAxes = varfont["fvar"].axes + + log.info("Instantiating %s table", tableTag) + vhvar = varfont[tableTag].table + varStore = vhvar.VarStore + + if "glyf" in varfont: + # Deltas from gvar table have already been applied to the hmtx/vmtx. For full + # instances (i.e. all axes pinned), we can simply drop HVAR/VVAR and return + if set(location).issuperset(axis.axisTag for axis in fvarAxes): + log.info("Dropping %s table", tableTag) + del varfont[tableTag] + return + + defaultDeltas = instantiateItemVariationStore(varStore, fvarAxes, axisLimits) + + if "glyf" not in varfont: + # CFF2 fonts need hmtx/vmtx updated here. For glyf fonts, the instantiateGvar + # function already updated the hmtx/vmtx from phantom points. Maybe remove + # that and do it here for both CFF2 and glyf fonts? + # + # Specially, if a font has glyf but not gvar, the hmtx/vmtx will not have been + # updated by instantiateGvar. Though one can call that a faulty font. + metricsTag = "vmtx" if tableTag == "VVAR" else "hmtx" + if metricsTag in varfont: + advMapping = getattr(vhvar, tableFields.advMapping) + metricsTable = varfont[metricsTag] + metrics = metricsTable.metrics + for glyphName, (advanceWidth, sb) in metrics.items(): + if advMapping: + varIdx = advMapping.mapping[glyphName] + else: + varIdx = varfont.getGlyphID(glyphName) + delta = round(defaultDeltas[varIdx]) + metrics[glyphName] = (max(0, advanceWidth + delta), sb) + + if ( + tableTag == "VVAR" + and getattr(vhvar, tableFields.vOrigMapping) is not None + ): + log.warning( + "VORG table not yet updated to reflect changes in VVAR table" + ) + + # For full instances (i.e. all axes pinned), we can simply drop HVAR/VVAR and return + if set(location).issuperset(axis.axisTag for axis in fvarAxes): + log.info("Dropping %s table", tableTag) + del varfont[tableTag] + return + + if varStore.VarRegionList.Region: + # Only re-optimize VarStore if the HVAR/VVAR already uses indirect AdvWidthMap + # or AdvHeightMap. If a direct, implicit glyphID->VariationIndex mapping is + # used for advances, skip re-optimizing and maintain original VariationIndex. + if getattr(vhvar, tableFields.advMapping): + varIndexMapping = varStore.optimize(use_NO_VARIATION_INDEX=False) + glyphOrder = varfont.getGlyphOrder() + _remapVarIdxMap(vhvar, tableFields.advMapping, varIndexMapping, glyphOrder) + if getattr(vhvar, tableFields.sb1): # left or top sidebearings + _remapVarIdxMap(vhvar, tableFields.sb1, varIndexMapping, glyphOrder) + if getattr(vhvar, tableFields.sb2): # right or bottom sidebearings + _remapVarIdxMap(vhvar, tableFields.sb2, varIndexMapping, glyphOrder) + if tableTag == "VVAR" and getattr(vhvar, tableFields.vOrigMapping): + _remapVarIdxMap( + vhvar, tableFields.vOrigMapping, varIndexMapping, glyphOrder + ) + + +def instantiateHVAR(varfont, axisLimits): + return _instantiateVHVAR(varfont, axisLimits, varLib.HVAR_FIELDS) + + +def instantiateVVAR(varfont, axisLimits): + return _instantiateVHVAR(varfont, axisLimits, varLib.VVAR_FIELDS) + + +class _TupleVarStoreAdapter(object): + def __init__(self, regions, axisOrder, tupleVarData, itemCounts): + self.regions = regions + self.axisOrder = axisOrder + self.tupleVarData = tupleVarData + self.itemCounts = itemCounts + + @classmethod + def fromItemVarStore(cls, itemVarStore, fvarAxes): + axisOrder = [axis.axisTag for axis in fvarAxes] + regions = [ + region.get_support(fvarAxes) for region in itemVarStore.VarRegionList.Region + ] + tupleVarData = [] + itemCounts = [] + for varData in itemVarStore.VarData: + variations = [] + varDataRegions = (regions[i] for i in varData.VarRegionIndex) + for axes, coordinates in zip(varDataRegions, zip(*varData.Item)): + variations.append(TupleVariation(axes, list(coordinates))) + tupleVarData.append(variations) + itemCounts.append(varData.ItemCount) + return cls(regions, axisOrder, tupleVarData, itemCounts) + + def rebuildRegions(self): + # Collect the set of all unique region axes from the current TupleVariations. + # We use an OrderedDict to de-duplicate regions while keeping the order. + uniqueRegions = collections.OrderedDict.fromkeys( + ( + frozenset(var.axes.items()) + for variations in self.tupleVarData + for var in variations + ) + ) + # Maintain the original order for the regions that pre-existed, appending + # the new regions at the end of the region list. + newRegions = [] + for region in self.regions: + regionAxes = frozenset(region.items()) + if regionAxes in uniqueRegions: + newRegions.append(region) + del uniqueRegions[regionAxes] + if uniqueRegions: + newRegions.extend(dict(region) for region in uniqueRegions) + self.regions = newRegions + + def instantiate(self, axisLimits): + defaultDeltaArray = [] + for variations, itemCount in zip(self.tupleVarData, self.itemCounts): + defaultDeltas = instantiateTupleVariationStore(variations, axisLimits) + if not defaultDeltas: + defaultDeltas = [0] * itemCount + defaultDeltaArray.append(defaultDeltas) + + # rebuild regions whose axes were dropped or limited + self.rebuildRegions() + + pinnedAxes = set(axisLimits.pinnedLocation()) + self.axisOrder = [ + axisTag for axisTag in self.axisOrder if axisTag not in pinnedAxes + ] + + return defaultDeltaArray + + def asItemVarStore(self): + regionOrder = [frozenset(axes.items()) for axes in self.regions] + varDatas = [] + for variations, itemCount in zip(self.tupleVarData, self.itemCounts): + if variations: + assert len(variations[0].coordinates) == itemCount + varRegionIndices = [ + regionOrder.index(frozenset(var.axes.items())) for var in variations + ] + varDataItems = list(zip(*(var.coordinates for var in variations))) + varDatas.append( + builder.buildVarData(varRegionIndices, varDataItems, optimize=False) + ) + else: + varDatas.append( + builder.buildVarData([], [[] for _ in range(itemCount)]) + ) + regionList = builder.buildVarRegionList(self.regions, self.axisOrder) + itemVarStore = builder.buildVarStore(regionList, varDatas) + # remove unused regions from VarRegionList + itemVarStore.prune_regions() + return itemVarStore + + +def instantiateItemVariationStore( + itemVarStore, fvarAxes, axisLimits, hierarchical=False +): + """Compute deltas at partial location, and update varStore in-place. + + Remove regions in which all axes were instanced, or fall outside the new axis + limits. Scale the deltas of the remaining regions where only some of the axes + were instanced. + + The number of VarData subtables, and the number of items within each, are + not modified, in order to keep the existing VariationIndex valid. + One may call VarStore.optimize() method after this to further optimize those. + + Args: + varStore: An otTables.VarStore object (Item Variation Store) + fvarAxes: list of fvar's Axis objects + axisLimits: NormalizedAxisLimits: mapping axis tags to normalized + min/default/max axis coordinates. May not specify coordinates/ranges for + all the fvar axes. + + Returns: + defaultDeltas: to be added to the default instance, of type dict of floats + keyed by VariationIndex compound values: i.e. (outer << 16) + inner. + """ + tupleVarStore = _TupleVarStoreAdapter.fromItemVarStore(itemVarStore, fvarAxes) + defaultDeltaArray = tupleVarStore.instantiate(axisLimits) + newItemVarStore = tupleVarStore.asItemVarStore() + + itemVarStore.VarRegionList = newItemVarStore.VarRegionList + if not hasattr(itemVarStore, "VarDataCount"): # Happens fromXML + itemVarStore.VarDataCount = len(newItemVarStore.VarData) + assert itemVarStore.VarDataCount == newItemVarStore.VarDataCount + itemVarStore.VarData = newItemVarStore.VarData + + if not hierarchical: + defaultDeltas = { + ((major << 16) + minor): delta + for major, deltas in enumerate(defaultDeltaArray) + for minor, delta in enumerate(deltas) + } + defaultDeltas[itemVarStore.NO_VARIATION_INDEX] = 0 + else: + defaultDeltas = {0xFFFF: {0xFFFF: 0}} # NO_VARIATION_INDEX + for major, deltas in enumerate(defaultDeltaArray): + defaultDeltasForMajor = defaultDeltas.setdefault(major, {}) + for minor, delta in enumerate(deltas): + defaultDeltasForMajor[minor] = delta + return defaultDeltas + + +def instantiateOTL(varfont, axisLimits): + # TODO(anthrotype) Support partial instancing of JSTF and BASE tables + + if ( + "GDEF" not in varfont + or varfont["GDEF"].table.Version < 0x00010003 + or not varfont["GDEF"].table.VarStore + ): + return + + if "GPOS" in varfont: + msg = "Instantiating GDEF and GPOS tables" + else: + msg = "Instantiating GDEF table" + log.info(msg) + + gdef = varfont["GDEF"].table + varStore = gdef.VarStore + fvarAxes = varfont["fvar"].axes + + defaultDeltas = instantiateItemVariationStore(varStore, fvarAxes, axisLimits) + + # When VF are built, big lookups may overflow and be broken into multiple + # subtables. MutatorMerger (which inherits from AligningMerger) reattaches + # them upon instancing, in case they can now fit a single subtable (if not, + # they will be split again upon compilation). + # This 'merger' also works as a 'visitor' that traverses the OTL tables and + # calls specific methods when instances of a given type are found. + # Specifically, it adds default deltas to GPOS Anchors/ValueRecords and GDEF + # LigatureCarets, and optionally deletes all VariationIndex tables if the + # VarStore is fully instanced. + merger = MutatorMerger( + varfont, defaultDeltas, deleteVariations=(not varStore.VarRegionList.Region) + ) + merger.mergeTables(varfont, [varfont], ["GDEF", "GPOS"]) + + if varStore.VarRegionList.Region: + varIndexMapping = varStore.optimize() + gdef.remap_device_varidxes(varIndexMapping) + if "GPOS" in varfont: + varfont["GPOS"].table.remap_device_varidxes(varIndexMapping) + else: + # Downgrade GDEF. + del gdef.VarStore + gdef.Version = 0x00010002 + if gdef.MarkGlyphSetsDef is None: + del gdef.MarkGlyphSetsDef + gdef.Version = 0x00010000 + + if not ( + gdef.LigCaretList + or gdef.MarkAttachClassDef + or gdef.GlyphClassDef + or gdef.AttachList + or (gdef.Version >= 0x00010002 and gdef.MarkGlyphSetsDef) + ): + del varfont["GDEF"] + + +def _isValidAvarSegmentMap(axisTag, segmentMap): + if not segmentMap: + return True + if not {(-1.0, -1.0), (0, 0), (1.0, 1.0)}.issubset(segmentMap.items()): + log.warning( + f"Invalid avar SegmentMap record for axis '{axisTag}': does not " + "include all required value maps {-1.0: -1.0, 0: 0, 1.0: 1.0}" + ) + return False + previousValue = None + for fromCoord, toCoord in sorted(segmentMap.items()): + if previousValue is not None and previousValue > toCoord: + log.warning( + f"Invalid avar AxisValueMap({fromCoord}, {toCoord}) record " + f"for axis '{axisTag}': the toCoordinate value must be >= to " + f"the toCoordinate value of the preceding record ({previousValue})." + ) + return False + previousValue = toCoord + return True + + +def downgradeCFF2ToCFF(varfont): + # Save these properties + recalcTimestamp = varfont.recalcTimestamp + recalcBBoxes = varfont.recalcBBoxes + + # Disable them + varfont.recalcTimestamp = False + varfont.recalcBBoxes = False + + # Save to memory, reload, downgrade and save again, reload. + # We do this dance because the convertCFF2ToCFF changes glyph + # names, so following save would fail if any other table was + # loaded and referencing glyph names. + # + # The second save+load is unfortunate but also necessary. + + stream = io.BytesIO() + log.info("Saving CFF2 font to memory for downgrade") + varfont.save(stream) + stream.seek(0) + varfont = TTFont(stream, recalcTimestamp=False, recalcBBoxes=False) + + convertCFF2ToCFF(varfont) + + stream = io.BytesIO() + log.info("Saving downgraded CFF font to memory") + varfont.save(stream) + stream.seek(0) + varfont = TTFont(stream, recalcTimestamp=False, recalcBBoxes=False) + + # Uncomment, to see test all tables can be loaded. This fails without + # the extra save+load above. + """ + for tag in varfont.keys(): + print("Loading", tag) + varfont[tag] + """ + + # Restore them + varfont.recalcTimestamp = recalcTimestamp + varfont.recalcBBoxes = recalcBBoxes + + return varfont + + +def instantiateAvar(varfont, axisLimits): + # 'axisLimits' dict must contain user-space (non-normalized) coordinates. + + avar = varfont["avar"] + + segments = avar.segments + + # drop table if we instantiate all the axes + pinnedAxes = set(axisLimits.pinnedLocation()) + if pinnedAxes.issuperset(segments): + log.info("Dropping avar table") + del varfont["avar"] + return + + if getattr(avar, "majorVersion", 1) >= 2 and avar.table.VarStore: + raise NotImplementedError("avar table with VarStore is not supported") + + log.info("Instantiating avar table") + for axis in pinnedAxes: + if axis in segments: + del segments[axis] + + # First compute the default normalization for axisLimits coordinates: i.e. + # min = -1.0, default = 0, max = +1.0, and in between values interpolated linearly, + # without using the avar table's mappings. + # Then, for each SegmentMap, if we are restricting its axis, compute the new + # mappings by dividing the key/value pairs by the desired new min/max values, + # dropping any mappings that fall outside the restricted range. + # The keys ('fromCoord') are specified in default normalized coordinate space, + # whereas the values ('toCoord') are "mapped forward" using the SegmentMap. + normalizedRanges = axisLimits.normalize(varfont, usingAvar=False) + newSegments = {} + for axisTag, mapping in segments.items(): + if not _isValidAvarSegmentMap(axisTag, mapping): + continue + if mapping and axisTag in normalizedRanges: + axisRange = normalizedRanges[axisTag] + mappedMin = floatToFixedToFloat( + piecewiseLinearMap(axisRange.minimum, mapping), 14 + ) + mappedDef = floatToFixedToFloat( + piecewiseLinearMap(axisRange.default, mapping), 14 + ) + mappedMax = floatToFixedToFloat( + piecewiseLinearMap(axisRange.maximum, mapping), 14 + ) + mappedAxisLimit = NormalizedAxisTripleAndDistances( + mappedMin, + mappedDef, + mappedMax, + axisRange.distanceNegative, + axisRange.distancePositive, + ) + newMapping = {} + for fromCoord, toCoord in mapping.items(): + if fromCoord < axisRange.minimum or fromCoord > axisRange.maximum: + continue + fromCoord = axisRange.renormalizeValue(fromCoord) + + assert mappedMin <= toCoord <= mappedMax + toCoord = mappedAxisLimit.renormalizeValue(toCoord) + + fromCoord = floatToFixedToFloat(fromCoord, 14) + toCoord = floatToFixedToFloat(toCoord, 14) + newMapping[fromCoord] = toCoord + newMapping.update({-1.0: -1.0, 0.0: 0.0, 1.0: 1.0}) + newSegments[axisTag] = newMapping + else: + newSegments[axisTag] = mapping + avar.segments = newSegments + + +def isInstanceWithinAxisRanges(location, axisRanges): + for axisTag, coord in location.items(): + if axisTag in axisRanges: + axisRange = axisRanges[axisTag] + if coord < axisRange.minimum or coord > axisRange.maximum: + return False + return True + + +def instantiateFvar(varfont, axisLimits): + # 'axisLimits' dict must contain user-space (non-normalized) coordinates + + location = axisLimits.pinnedLocation() + + fvar = varfont["fvar"] + + # drop table if we instantiate all the axes + if set(location).issuperset(axis.axisTag for axis in fvar.axes): + log.info("Dropping fvar table") + del varfont["fvar"] + return + + log.info("Instantiating fvar table") + + axes = [] + for axis in fvar.axes: + axisTag = axis.axisTag + if axisTag in location: + continue + if axisTag in axisLimits: + triple = axisLimits[axisTag] + if triple.default is None: + triple = (triple.minimum, axis.defaultValue, triple.maximum) + axis.minValue, axis.defaultValue, axis.maxValue = triple + axes.append(axis) + fvar.axes = axes + + # only keep NamedInstances whose coordinates == pinned axis location + instances = [] + for instance in fvar.instances: + if any(instance.coordinates[axis] != value for axis, value in location.items()): + continue + for axisTag in location: + del instance.coordinates[axisTag] + if not isInstanceWithinAxisRanges(instance.coordinates, axisLimits): + continue + instances.append(instance) + fvar.instances = instances + + +def instantiateSTAT(varfont, axisLimits): + # 'axisLimits' dict must contain user-space (non-normalized) coordinates + + stat = varfont["STAT"].table + if not stat.DesignAxisRecord or not ( + stat.AxisValueArray and stat.AxisValueArray.AxisValue + ): + return # STAT table empty, nothing to do + + log.info("Instantiating STAT table") + newAxisValueTables = axisValuesFromAxisLimits(stat, axisLimits) + stat.AxisValueCount = len(newAxisValueTables) + if stat.AxisValueCount: + stat.AxisValueArray.AxisValue = newAxisValueTables + else: + stat.AxisValueArray = None + + +def axisValuesFromAxisLimits(stat, axisLimits): + def isAxisValueOutsideLimits(axisTag, axisValue): + if axisTag in axisLimits: + triple = axisLimits[axisTag] + if axisValue < triple.minimum or axisValue > triple.maximum: + return True + return False + + # only keep AxisValues whose axis is not pinned nor restricted, or is pinned at the + # exact (nominal) value, or is restricted but the value is within the new range + designAxes = stat.DesignAxisRecord.Axis + newAxisValueTables = [] + for axisValueTable in stat.AxisValueArray.AxisValue: + axisValueFormat = axisValueTable.Format + if axisValueFormat in (1, 2, 3): + axisTag = designAxes[axisValueTable.AxisIndex].AxisTag + if axisValueFormat == 2: + axisValue = axisValueTable.NominalValue + else: + axisValue = axisValueTable.Value + if isAxisValueOutsideLimits(axisTag, axisValue): + continue + elif axisValueFormat == 4: + # drop 'non-analytic' AxisValue if _any_ AxisValueRecord doesn't match + # the pinned location or is outside range + dropAxisValueTable = False + for rec in axisValueTable.AxisValueRecord: + axisTag = designAxes[rec.AxisIndex].AxisTag + axisValue = rec.Value + if isAxisValueOutsideLimits(axisTag, axisValue): + dropAxisValueTable = True + break + if dropAxisValueTable: + continue + else: + log.warning("Unknown AxisValue table format (%s); ignored", axisValueFormat) + newAxisValueTables.append(axisValueTable) + return newAxisValueTables + + +def setMacOverlapFlags(glyfTable): + flagOverlapCompound = _g_l_y_f.OVERLAP_COMPOUND + flagOverlapSimple = _g_l_y_f.flagOverlapSimple + for glyphName in glyfTable.keys(): + glyph = glyfTable[glyphName] + # Set OVERLAP_COMPOUND bit for compound glyphs + if glyph.isComposite(): + glyph.components[0].flags |= flagOverlapCompound + # Set OVERLAP_SIMPLE bit for simple glyphs + elif glyph.numberOfContours > 0: + glyph.flags[0] |= flagOverlapSimple + + +def normalize(value, triple, avarMapping): + value = normalizeValue(value, triple) + if avarMapping: + value = piecewiseLinearMap(value, avarMapping) + # Quantize to F2Dot14, to avoid surprise interpolations. + return floatToFixedToFloat(value, 14) + + +def sanityCheckVariableTables(varfont): + if "fvar" not in varfont: + raise ValueError("Missing required table fvar") + if "gvar" in varfont: + if "glyf" not in varfont: + raise ValueError("Can't have gvar without glyf") + + +def instantiateVariableFont( + varfont, + axisLimits, + inplace=False, + optimize=True, + overlap=OverlapMode.KEEP_AND_SET_FLAGS, + updateFontNames=False, + *, + downgradeCFF2=False, + static=False, +): + """Instantiate variable font, either fully or partially. + + Depending on whether the `axisLimits` dictionary references all or some of the + input varfont's axes, the output font will either be a full instance (static + font) or a variable font with possibly less variation data. + + Args: + varfont: a TTFont instance, which must contain at least an 'fvar' table. + axisLimits: a dict keyed by axis tags (str) containing the coordinates (float) + along one or more axes where the desired instance will be located. + If the value is `None`, the default coordinate as per 'fvar' table for + that axis is used. + The limit values can also be (min, max) tuples for restricting an + axis's variation range. The default axis value must be included in + the new range. + inplace (bool): whether to modify input TTFont object in-place instead of + returning a distinct object. + optimize (bool): if False, do not perform IUP-delta optimization on the + remaining 'gvar' table's deltas. Possibly faster, and might work around + rendering issues in some buggy environments, at the cost of a slightly + larger file size. + overlap (OverlapMode): variable fonts usually contain overlapping contours, and + some font rendering engines on Apple platforms require that the + `OVERLAP_SIMPLE` and `OVERLAP_COMPOUND` flags in the 'glyf' table be set to + force rendering using a non-zero fill rule. Thus we always set these flags + on all glyphs to maximise cross-compatibility of the generated instance. + You can disable this by passing OverlapMode.KEEP_AND_DONT_SET_FLAGS. + If you want to remove the overlaps altogether and merge overlapping + contours and components, you can pass OverlapMode.REMOVE (or + REMOVE_AND_IGNORE_ERRORS to not hard-fail on tricky glyphs). Note that this + requires the skia-pathops package (available to pip install). + The overlap parameter only has effect when generating full static instances. + updateFontNames (bool): if True, update the instantiated font's name table using + the Axis Value Tables from the STAT table. The name table and the style bits + in the head and OS/2 table will be updated so they conform to the R/I/B/BI + model. If the STAT table is missing or an Axis Value table is missing for + a given axis coordinate, a ValueError will be raised. + downgradeCFF2 (bool): if True, downgrade the CFF2 table to CFF table when possible + ie. full instancing of all axes. This is useful for compatibility with older + software that does not support CFF2. Defaults to False. Note that this + operation also removes overlaps within glyph shapes, as CFF does not support + overlaps but CFF2 does. + static (bool): if True, generate a full instance (static font) instead of a partial + instance (variable font). + """ + # 'overlap' used to be bool and is now enum; for backward compat keep accepting bool + overlap = OverlapMode(int(overlap)) + + sanityCheckVariableTables(varfont) + + if static: + unspecified = [] + for axis in varfont["fvar"].axes: + if axis.axisTag not in axisLimits: + axisLimits[axis.axisTag] = None + unspecified.append(axis.axisTag) + if unspecified: + log.info("Pinning unspecified axes to default: %s", unspecified) + + axisLimits = AxisLimits(axisLimits).limitAxesAndPopulateDefaults(varfont) + + log.info("Restricted limits: %s", axisLimits) + + normalizedLimits = axisLimits.normalize(varfont) + + log.info("Normalized limits: %s", normalizedLimits) + + if not inplace: + varfont = deepcopy(varfont) + + if "DSIG" in varfont: + del varfont["DSIG"] + + if updateFontNames: + log.info("Updating name table") + names.updateNameTable(varfont, axisLimits) + + if "VARC" in varfont: + instantiateVARC(varfont, normalizedLimits) + + if "CFF2" in varfont: + downgradeCFF2 = instantiateCFF2( + varfont, normalizedLimits, downgrade=downgradeCFF2 + ) + + if "gvar" in varfont: + instantiateGvar(varfont, normalizedLimits, optimize=optimize) + + if "cvar" in varfont: + instantiateCvar(varfont, normalizedLimits) + + if "MVAR" in varfont: + instantiateMVAR(varfont, normalizedLimits) + + if "HVAR" in varfont: + instantiateHVAR(varfont, normalizedLimits) + + if "VVAR" in varfont: + instantiateVVAR(varfont, normalizedLimits) + + instantiateOTL(varfont, normalizedLimits) + + instantiateFeatureVariations(varfont, normalizedLimits) + + if "avar" in varfont: + instantiateAvar(varfont, axisLimits) + + with names.pruningUnusedNames(varfont): + if "STAT" in varfont: + instantiateSTAT(varfont, axisLimits) + + instantiateFvar(varfont, axisLimits) + + if "OS/2" in varfont: + varfont["OS/2"].recalcAvgCharWidth(varfont) + + varLib.set_default_weight_width_slant( + varfont, location=axisLimits.defaultLocation() + ) + + if updateFontNames: + # Set Regular/Italic/Bold/Bold Italic bits as appropriate, after the + # name table has been updated. + setRibbiBits(varfont) + + if downgradeCFF2: + origVarfont = varfont + varfont = downgradeCFF2ToCFF(varfont) + if inplace: + origVarfont.__dict__ = varfont.__dict__.copy() + + if "fvar" not in varfont: + if overlap == OverlapMode.KEEP_AND_SET_FLAGS: + if "glyf" in varfont: + setMacOverlapFlags(varfont["glyf"]) + elif overlap in (OverlapMode.REMOVE, OverlapMode.REMOVE_AND_IGNORE_ERRORS): + from fontTools.ttLib.removeOverlaps import removeOverlaps + + log.info("Removing glyph outlines overlaps") + removeOverlaps( + varfont, + ignoreErrors=(overlap == OverlapMode.REMOVE_AND_IGNORE_ERRORS), + ) + + return varfont + + +def setRibbiBits(font): + """Set the `head.macStyle` and `OS/2.fsSelection` style bits + appropriately.""" + + english_ribbi_style = font["name"].getName(names.NameID.SUBFAMILY_NAME, 3, 1, 0x409) + if english_ribbi_style is None: + return + + styleMapStyleName = english_ribbi_style.toStr().lower() + if styleMapStyleName not in {"regular", "bold", "italic", "bold italic"}: + return + + if styleMapStyleName == "bold": + font["head"].macStyle = 0b01 + elif styleMapStyleName == "bold italic": + font["head"].macStyle = 0b11 + elif styleMapStyleName == "italic": + font["head"].macStyle = 0b10 + + selection = font["OS/2"].fsSelection + # First clear... + selection &= ~(1 << 0) + selection &= ~(1 << 5) + selection &= ~(1 << 6) + # ...then re-set the bits. + if styleMapStyleName == "regular": + selection |= 1 << 6 + elif styleMapStyleName == "bold": + selection |= 1 << 5 + elif styleMapStyleName == "italic": + selection |= 1 << 0 + elif styleMapStyleName == "bold italic": + selection |= 1 << 0 + selection |= 1 << 5 + font["OS/2"].fsSelection = selection + + +def parseLimits(limits: Iterable[str]) -> Dict[str, Optional[AxisTriple]]: + result = {} + for limitString in limits: + match = re.match( + r"^(\w{1,4})=(?:(drop)|(?:([^:]*)(?:[:]([^:]*))?(?:[:]([^:]*))?))$", + limitString, + ) + if not match: + raise ValueError("invalid location format: %r" % limitString) + tag = match.group(1).ljust(4) + + if match.group(2): # 'drop' + result[tag] = None + continue + + triple = match.group(3, 4, 5) + + if triple[1] is None: # "value" syntax + triple = (triple[0], triple[0], triple[0]) + elif triple[2] is None: # "min:max" syntax + triple = (triple[0], None, triple[1]) + + triple = tuple(float(v) if v else None for v in triple) + + result[tag] = AxisTriple(*triple) + + return result + + +def parseArgs(args): + """Parse argv. + + Returns: + 3-tuple (infile, axisLimits, options) + axisLimits is either a Dict[str, Optional[float]], for pinning variation axes + to specific coordinates along those axes (with `None` as a placeholder for an + axis' default value); or a Dict[str, Tuple(float, float)], meaning limit this + axis to min/max range. + Axes locations are in user-space coordinates, as defined in the "fvar" table. + """ + from fontTools import configLogger + import argparse + + parser = argparse.ArgumentParser( + "fonttools varLib.instancer", + description="Partially instantiate a variable font", + ) + parser.add_argument("input", metavar="INPUT.ttf", help="Input variable TTF file.") + parser.add_argument( + "locargs", + metavar="AXIS=LOC", + nargs="*", + help="List of space separated locations. A location consists of " + "the tag of a variation axis, followed by '=' and the literal, " + "string 'drop', or colon-separated list of one to three values, " + "each of which is the empty string, or a number. " + "E.g.: wdth=100 or wght=75.0:125.0 or wght=100:400:700 or wght=:500: " + "or wght=drop", + ) + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT.ttf", + default=None, + help="Output instance TTF file (default: INPUT-instance.ttf).", + ) + parser.add_argument( + "--static", + dest="static", + action="store_true", + help="Make a static font: pin unspecified axes to their default location.", + ) + parser.add_argument( + "--no-optimize", + dest="optimize", + action="store_false", + help="Don't perform IUP optimization on the remaining gvar TupleVariations", + ) + parser.add_argument( + "--no-overlap-flag", + dest="overlap", + action="store_false", + help="Don't set OVERLAP_SIMPLE/OVERLAP_COMPOUND glyf flags (only applicable " + "when generating a full instance)", + ) + parser.add_argument( + "--remove-overlaps", + dest="remove_overlaps", + action="store_true", + help="Merge overlapping contours and components (only applicable " + "when generating a full instance). Requires skia-pathops", + ) + parser.add_argument( + "--ignore-overlap-errors", + dest="ignore_overlap_errors", + action="store_true", + help="Don't crash if the remove-overlaps operation fails for some glyphs.", + ) + parser.add_argument( + "--update-name-table", + action="store_true", + help="Update the instantiated font's `name` table. Input font must have " + "a STAT table with Axis Value Tables", + ) + parser.add_argument( + "--downgrade-cff2", + action="store_true", + help="If all axes are pinned, downgrade CFF2 to CFF table format", + ) + parser.add_argument( + "--no-recalc-timestamp", + dest="recalc_timestamp", + action="store_false", + help="Don't set the output font's timestamp to the current time.", + ) + parser.add_argument( + "--no-recalc-bounds", + dest="recalc_bounds", + action="store_false", + help="Don't recalculate font bounding boxes", + ) + loggingGroup = parser.add_mutually_exclusive_group(required=False) + loggingGroup.add_argument( + "-v", "--verbose", action="store_true", help="Run more verbosely." + ) + loggingGroup.add_argument( + "-q", "--quiet", action="store_true", help="Turn verbosity off." + ) + options = parser.parse_args(args) + + if options.remove_overlaps: + if options.ignore_overlap_errors: + options.overlap = OverlapMode.REMOVE_AND_IGNORE_ERRORS + else: + options.overlap = OverlapMode.REMOVE + else: + options.overlap = OverlapMode(int(options.overlap)) + + infile = options.input + if not os.path.isfile(infile): + parser.error("No such file '{}'".format(infile)) + + configLogger( + level=("DEBUG" if options.verbose else "ERROR" if options.quiet else "INFO") + ) + + try: + axisLimits = parseLimits(options.locargs) + except ValueError as e: + parser.error(str(e)) + + if len(axisLimits) != len(options.locargs): + parser.error("Specified multiple limits for the same axis") + + return (infile, axisLimits, options) + + +def main(args=None): + """Partially instantiate a variable font""" + infile, axisLimits, options = parseArgs(args) + log.info("Restricting axes: %s", axisLimits) + + log.info("Loading variable font") + varfont = TTFont( + infile, + recalcTimestamp=options.recalc_timestamp, + recalcBBoxes=options.recalc_bounds, + ) + + isFullInstance = options.static or { + axisTag + for axisTag, limit in axisLimits.items() + if limit is None or limit[0] == limit[2] + }.issuperset(axis.axisTag for axis in varfont["fvar"].axes) + + varfont = instantiateVariableFont( + varfont, + axisLimits, + inplace=True, + optimize=options.optimize, + overlap=options.overlap, + updateFontNames=options.update_name_table, + downgradeCFF2=options.downgrade_cff2, + static=options.static, + ) + + suffix = "-instance" if isFullInstance else "-partial" + outfile = ( + makeOutputFileName(infile, overWrite=True, suffix=suffix) + if not options.output + else options.output + ) + + log.info( + "Saving %s font %s", + "instance" if isFullInstance else "partial variable", + outfile, + ) + varfont.save(outfile) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/__main__.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..64ffff2b9fdf58d8a557de7c1ae631b5c6fb4b6f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/__main__.py @@ -0,0 +1,5 @@ +import sys +from fontTools.varLib.instancer import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/featureVars.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/featureVars.py new file mode 100644 index 0000000000000000000000000000000000000000..d9370d9d653aa847a0ada1fae2c3869b66fa27af --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/featureVars.py @@ -0,0 +1,190 @@ +from fontTools.ttLib.tables import otTables as ot +from copy import deepcopy +import logging + + +log = logging.getLogger("fontTools.varLib.instancer") + + +def _featureVariationRecordIsUnique(rec, seen): + conditionSet = [] + conditionSets = ( + rec.ConditionSet.ConditionTable if rec.ConditionSet is not None else [] + ) + for cond in conditionSets: + if cond.Format != 1: + # can't tell whether this is duplicate, assume is unique + return True + conditionSet.append( + (cond.AxisIndex, cond.FilterRangeMinValue, cond.FilterRangeMaxValue) + ) + # besides the set of conditions, we also include the FeatureTableSubstitution + # version to identify unique FeatureVariationRecords, even though only one + # version is currently defined. It's theoretically possible that multiple + # records with same conditions but different substitution table version be + # present in the same font for backward compatibility. + recordKey = frozenset([rec.FeatureTableSubstitution.Version] + conditionSet) + if recordKey in seen: + return False + else: + seen.add(recordKey) # side effect + return True + + +def _limitFeatureVariationConditionRange(condition, axisLimit): + minValue = condition.FilterRangeMinValue + maxValue = condition.FilterRangeMaxValue + + if ( + minValue > maxValue + or minValue > axisLimit.maximum + or maxValue < axisLimit.minimum + ): + # condition invalid or out of range + return + + return tuple( + axisLimit.renormalizeValue(v, extrapolate=False) for v in (minValue, maxValue) + ) + + +def _instantiateFeatureVariationRecord( + record, recIdx, axisLimits, fvarAxes, axisIndexMap +): + applies = True + shouldKeep = False + newConditions = [] + from fontTools.varLib.instancer import NormalizedAxisTripleAndDistances + + default_triple = NormalizedAxisTripleAndDistances(-1, 0, +1) + if record.ConditionSet is None: + record.ConditionSet = ot.ConditionSet() + record.ConditionSet.ConditionTable = [] + record.ConditionSet.ConditionCount = 0 + for i, condition in enumerate(record.ConditionSet.ConditionTable): + if condition.Format == 1: + axisIdx = condition.AxisIndex + axisTag = fvarAxes[axisIdx].axisTag + + minValue = condition.FilterRangeMinValue + maxValue = condition.FilterRangeMaxValue + triple = axisLimits.get(axisTag, default_triple) + + if not (minValue <= triple.default <= maxValue): + applies = False + + # if condition not met, remove entire record + if triple.minimum > maxValue or triple.maximum < minValue: + newConditions = None + break + + if axisTag in axisIndexMap: + # remap axis index + condition.AxisIndex = axisIndexMap[axisTag] + + # remap condition limits + newRange = _limitFeatureVariationConditionRange(condition, triple) + if newRange: + # keep condition with updated limits + minimum, maximum = newRange + condition.FilterRangeMinValue = minimum + condition.FilterRangeMaxValue = maximum + shouldKeep = True + if minimum != -1 or maximum != +1: + newConditions.append(condition) + else: + # condition out of range, remove entire record + newConditions = None + break + + else: + log.warning( + "Condition table {0} of FeatureVariationRecord {1} has " + "unsupported format ({2}); ignored".format(i, recIdx, condition.Format) + ) + applies = False + newConditions.append(condition) + + if newConditions is not None and shouldKeep: + record.ConditionSet.ConditionTable = newConditions + if not newConditions: + record.ConditionSet = None + shouldKeep = True + else: + shouldKeep = False + + # Does this *always* apply? + universal = shouldKeep and not newConditions + + return applies, shouldKeep, universal + + +def _instantiateFeatureVariations(table, fvarAxes, axisLimits): + pinnedAxes = set(axisLimits.pinnedLocation()) + axisOrder = [axis.axisTag for axis in fvarAxes if axis.axisTag not in pinnedAxes] + axisIndexMap = {axisTag: axisOrder.index(axisTag) for axisTag in axisOrder} + + featureVariationApplied = False + uniqueRecords = set() + newRecords = [] + defaultsSubsts = None + + for i, record in enumerate(table.FeatureVariations.FeatureVariationRecord): + applies, shouldKeep, universal = _instantiateFeatureVariationRecord( + record, i, axisLimits, fvarAxes, axisIndexMap + ) + + if shouldKeep and _featureVariationRecordIsUnique(record, uniqueRecords): + newRecords.append(record) + + if applies and not featureVariationApplied: + assert record.FeatureTableSubstitution.Version == 0x00010000 + defaultsSubsts = deepcopy(record.FeatureTableSubstitution) + for default, rec in zip( + defaultsSubsts.SubstitutionRecord, + record.FeatureTableSubstitution.SubstitutionRecord, + ): + default.Feature = deepcopy( + table.FeatureList.FeatureRecord[rec.FeatureIndex].Feature + ) + table.FeatureList.FeatureRecord[rec.FeatureIndex].Feature = deepcopy( + rec.Feature + ) + # Set variations only once + featureVariationApplied = True + + # Further records don't have a chance to apply after a universal record + if universal: + break + + # Insert a catch-all record to reinstate the old features if necessary + if featureVariationApplied and newRecords and not universal: + defaultRecord = ot.FeatureVariationRecord() + defaultRecord.ConditionSet = ot.ConditionSet() + defaultRecord.ConditionSet.ConditionTable = [] + defaultRecord.ConditionSet.ConditionCount = 0 + defaultRecord.FeatureTableSubstitution = defaultsSubsts + + newRecords.append(defaultRecord) + + if newRecords: + table.FeatureVariations.FeatureVariationRecord = newRecords + table.FeatureVariations.FeatureVariationCount = len(newRecords) + else: + del table.FeatureVariations + # downgrade table version if there are no FeatureVariations left + table.Version = 0x00010000 + + +def instantiateFeatureVariations(varfont, axisLimits): + for tableTag in ("GPOS", "GSUB"): + if tableTag not in varfont or not getattr( + varfont[tableTag].table, "FeatureVariations", None + ): + continue + log.info("Instantiating FeatureVariations of %s table", tableTag) + _instantiateFeatureVariations( + varfont[tableTag].table, varfont["fvar"].axes, axisLimits + ) + # remove unreferenced lookups + varfont[tableTag].prune_lookups() diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/names.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/names.py new file mode 100644 index 0000000000000000000000000000000000000000..f9454688e06597b8eb984b658cd0d5d19eece601 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/names.py @@ -0,0 +1,388 @@ +"""Helpers for instantiating name table records.""" + +from contextlib import contextmanager +from copy import deepcopy +from enum import IntEnum +import re + + +class NameID(IntEnum): + FAMILY_NAME = 1 + SUBFAMILY_NAME = 2 + UNIQUE_FONT_IDENTIFIER = 3 + FULL_FONT_NAME = 4 + VERSION_STRING = 5 + POSTSCRIPT_NAME = 6 + TYPOGRAPHIC_FAMILY_NAME = 16 + TYPOGRAPHIC_SUBFAMILY_NAME = 17 + VARIATIONS_POSTSCRIPT_NAME_PREFIX = 25 + + +ELIDABLE_AXIS_VALUE_NAME = 2 + + +def getVariationNameIDs(varfont): + used = [] + if "fvar" in varfont: + fvar = varfont["fvar"] + for axis in fvar.axes: + used.append(axis.axisNameID) + for instance in fvar.instances: + used.append(instance.subfamilyNameID) + if instance.postscriptNameID != 0xFFFF: + used.append(instance.postscriptNameID) + if "STAT" in varfont: + stat = varfont["STAT"].table + for axis in stat.DesignAxisRecord.Axis if stat.DesignAxisRecord else (): + used.append(axis.AxisNameID) + for value in stat.AxisValueArray.AxisValue if stat.AxisValueArray else (): + used.append(value.ValueNameID) + elidedFallbackNameID = getattr(stat, "ElidedFallbackNameID", None) + if elidedFallbackNameID is not None: + used.append(elidedFallbackNameID) + # nameIDs <= 255 are reserved by OT spec so we don't touch them + return {nameID for nameID in used if nameID > 255} + + +@contextmanager +def pruningUnusedNames(varfont): + from . import log + + origNameIDs = getVariationNameIDs(varfont) + + yield + + log.info("Pruning name table") + exclude = origNameIDs - getVariationNameIDs(varfont) + varfont["name"].names[:] = [ + record for record in varfont["name"].names if record.nameID not in exclude + ] + if "ltag" in varfont: + # Drop the whole 'ltag' table if all the language-dependent Unicode name + # records that reference it have been dropped. + # TODO: Only prune unused ltag tags, renumerating langIDs accordingly. + # Note ltag can also be used by feat or morx tables, so check those too. + if not any( + record + for record in varfont["name"].names + if record.platformID == 0 and record.langID != 0xFFFF + ): + del varfont["ltag"] + + +def updateNameTable(varfont, axisLimits): + """Update instatiated variable font's name table using STAT AxisValues. + + Raises ValueError if the STAT table is missing or an Axis Value table is + missing for requested axis locations. + + First, collect all STAT AxisValues that match the new default axis locations + (excluding "elided" ones); concatenate the strings in design axis order, + while giving priority to "synthetic" values (Format 4), to form the + typographic subfamily name associated with the new default instance. + Finally, update all related records in the name table, making sure that + legacy family/sub-family names conform to the the R/I/B/BI (Regular, Italic, + Bold, Bold Italic) naming model. + + Example: Updating a partial variable font: + | >>> ttFont = TTFont("OpenSans[wdth,wght].ttf") + | >>> updateNameTable(ttFont, {"wght": (400, 900), "wdth": 75}) + + The name table records will be updated in the following manner: + NameID 1 familyName: "Open Sans" --> "Open Sans Condensed" + NameID 2 subFamilyName: "Regular" --> "Regular" + NameID 3 Unique font identifier: "3.000;GOOG;OpenSans-Regular" --> \ + "3.000;GOOG;OpenSans-Condensed" + NameID 4 Full font name: "Open Sans Regular" --> "Open Sans Condensed" + NameID 6 PostScript name: "OpenSans-Regular" --> "OpenSans-Condensed" + NameID 16 Typographic Family name: None --> "Open Sans" + NameID 17 Typographic Subfamily name: None --> "Condensed" + + References: + https://docs.microsoft.com/en-us/typography/opentype/spec/stat + https://docs.microsoft.com/en-us/typography/opentype/spec/name#name-ids + """ + from . import AxisLimits, axisValuesFromAxisLimits + + if "STAT" not in varfont: + raise ValueError("Cannot update name table since there is no STAT table.") + stat = varfont["STAT"].table + if not stat.AxisValueArray: + raise ValueError("Cannot update name table since there are no STAT Axis Values") + fvar = varfont["fvar"] + + # The updated name table will reflect the new 'zero origin' of the font. + # If we're instantiating a partial font, we will populate the unpinned + # axes with their default axis values from fvar. + axisLimits = AxisLimits(axisLimits).limitAxesAndPopulateDefaults(varfont) + partialDefaults = axisLimits.defaultLocation() + fvarDefaults = {a.axisTag: a.defaultValue for a in fvar.axes} + defaultAxisCoords = AxisLimits({**fvarDefaults, **partialDefaults}) + assert all(v.minimum == v.maximum for v in defaultAxisCoords.values()) + + axisValueTables = axisValuesFromAxisLimits(stat, defaultAxisCoords) + checkAxisValuesExist(stat, axisValueTables, defaultAxisCoords.pinnedLocation()) + + # ignore "elidable" axis values, should be omitted in application font menus. + axisValueTables = [ + v for v in axisValueTables if not v.Flags & ELIDABLE_AXIS_VALUE_NAME + ] + axisValueTables = _sortAxisValues(axisValueTables) + _updateNameRecords(varfont, axisValueTables) + + +def checkAxisValuesExist(stat, axisValues, axisCoords): + seen = set() + designAxes = stat.DesignAxisRecord.Axis + hasValues = set() + for value in stat.AxisValueArray.AxisValue: + if value.Format in (1, 2, 3): + hasValues.add(designAxes[value.AxisIndex].AxisTag) + elif value.Format == 4: + for rec in value.AxisValueRecord: + hasValues.add(designAxes[rec.AxisIndex].AxisTag) + + for axisValueTable in axisValues: + axisValueFormat = axisValueTable.Format + if axisValueTable.Format in (1, 2, 3): + axisTag = designAxes[axisValueTable.AxisIndex].AxisTag + if axisValueFormat == 2: + axisValue = axisValueTable.NominalValue + else: + axisValue = axisValueTable.Value + if axisTag in axisCoords and axisValue == axisCoords[axisTag]: + seen.add(axisTag) + elif axisValueTable.Format == 4: + for rec in axisValueTable.AxisValueRecord: + axisTag = designAxes[rec.AxisIndex].AxisTag + if axisTag in axisCoords and rec.Value == axisCoords[axisTag]: + seen.add(axisTag) + + missingAxes = (set(axisCoords) - seen) & hasValues + if missingAxes: + missing = ", ".join(f"'{i}': {axisCoords[i]}" for i in missingAxes) + raise ValueError(f"Cannot find Axis Values {{{missing}}}") + + +def _sortAxisValues(axisValues): + # Sort by axis index, remove duplicates and ensure that format 4 AxisValues + # are dominant. + # The MS Spec states: "if a format 1, format 2 or format 3 table has a + # (nominal) value used in a format 4 table that also has values for + # other axes, the format 4 table, being the more specific match, is used", + # https://docs.microsoft.com/en-us/typography/opentype/spec/stat#axis-value-table-format-4 + results = [] + seenAxes = set() + # Sort format 4 axes so the tables with the most AxisValueRecords are first + format4 = sorted( + [v for v in axisValues if v.Format == 4], + key=lambda v: len(v.AxisValueRecord), + reverse=True, + ) + + for val in format4: + axisIndexes = set(r.AxisIndex for r in val.AxisValueRecord) + minIndex = min(axisIndexes) + if not seenAxes & axisIndexes: + seenAxes |= axisIndexes + results.append((minIndex, val)) + + for val in axisValues: + if val in format4: + continue + axisIndex = val.AxisIndex + if axisIndex not in seenAxes: + seenAxes.add(axisIndex) + results.append((axisIndex, val)) + + return [axisValue for _, axisValue in sorted(results)] + + +def _updateNameRecords(varfont, axisValues): + # Update nametable based on the axisValues using the R/I/B/BI model. + nametable = varfont["name"] + stat = varfont["STAT"].table + + axisValueNameIDs = [a.ValueNameID for a in axisValues] + ribbiNameIDs = [n for n in axisValueNameIDs if _isRibbi(nametable, n)] + nonRibbiNameIDs = [n for n in axisValueNameIDs if n not in ribbiNameIDs] + elidedNameID = stat.ElidedFallbackNameID + elidedNameIsRibbi = _isRibbi(nametable, elidedNameID) + + getName = nametable.getName + platforms = set((r.platformID, r.platEncID, r.langID) for r in nametable.names) + for platform in platforms: + if not all(getName(i, *platform) for i in (1, 2, elidedNameID)): + # Since no family name and subfamily name records were found, + # we cannot update this set of name Records. + continue + + subFamilyName = " ".join( + getName(n, *platform).toUnicode() for n in ribbiNameIDs + ) + if nonRibbiNameIDs: + typoSubFamilyName = " ".join( + getName(n, *platform).toUnicode() for n in axisValueNameIDs + ) + else: + typoSubFamilyName = None + + # If neither subFamilyName and typographic SubFamilyName exist, + # we will use the STAT's elidedFallbackName + if not typoSubFamilyName and not subFamilyName: + if elidedNameIsRibbi: + subFamilyName = getName(elidedNameID, *platform).toUnicode() + else: + typoSubFamilyName = getName(elidedNameID, *platform).toUnicode() + + familyNameSuffix = " ".join( + getName(n, *platform).toUnicode() for n in nonRibbiNameIDs + ) + + _updateNameTableStyleRecords( + varfont, + familyNameSuffix, + subFamilyName, + typoSubFamilyName, + *platform, + ) + + +def _isRibbi(nametable, nameID): + englishRecord = nametable.getName(nameID, 3, 1, 0x409) + return ( + True + if englishRecord is not None + and englishRecord.toUnicode() in ("Regular", "Italic", "Bold", "Bold Italic") + else False + ) + + +def _updateNameTableStyleRecords( + varfont, + familyNameSuffix, + subFamilyName, + typoSubFamilyName, + platformID=3, + platEncID=1, + langID=0x409, +): + # TODO (Marc F) It may be nice to make this part a standalone + # font renamer in the future. + nametable = varfont["name"] + platform = (platformID, platEncID, langID) + + currentFamilyName = nametable.getName( + NameID.TYPOGRAPHIC_FAMILY_NAME, *platform + ) or nametable.getName(NameID.FAMILY_NAME, *platform) + + currentStyleName = nametable.getName( + NameID.TYPOGRAPHIC_SUBFAMILY_NAME, *platform + ) or nametable.getName(NameID.SUBFAMILY_NAME, *platform) + + if not all([currentFamilyName, currentStyleName]): + raise ValueError(f"Missing required NameIDs 1 and 2 for platform {platform}") + + currentFamilyName = currentFamilyName.toUnicode() + currentStyleName = currentStyleName.toUnicode() + + nameIDs = { + NameID.FAMILY_NAME: currentFamilyName, + NameID.SUBFAMILY_NAME: subFamilyName or "Regular", + } + if typoSubFamilyName: + nameIDs[NameID.FAMILY_NAME] = f"{currentFamilyName} {familyNameSuffix}".strip() + nameIDs[NameID.TYPOGRAPHIC_FAMILY_NAME] = currentFamilyName + nameIDs[NameID.TYPOGRAPHIC_SUBFAMILY_NAME] = typoSubFamilyName + else: + # Remove previous Typographic Family and SubFamily names since they're + # no longer required + for nameID in ( + NameID.TYPOGRAPHIC_FAMILY_NAME, + NameID.TYPOGRAPHIC_SUBFAMILY_NAME, + ): + nametable.removeNames(nameID=nameID) + + newFamilyName = ( + nameIDs.get(NameID.TYPOGRAPHIC_FAMILY_NAME) or nameIDs[NameID.FAMILY_NAME] + ) + newStyleName = ( + nameIDs.get(NameID.TYPOGRAPHIC_SUBFAMILY_NAME) or nameIDs[NameID.SUBFAMILY_NAME] + ) + + nameIDs[NameID.FULL_FONT_NAME] = f"{newFamilyName} {newStyleName}" + nameIDs[NameID.POSTSCRIPT_NAME] = _updatePSNameRecord( + varfont, newFamilyName, newStyleName, platform + ) + + uniqueID = _updateUniqueIdNameRecord(varfont, nameIDs, platform) + if uniqueID: + nameIDs[NameID.UNIQUE_FONT_IDENTIFIER] = uniqueID + + for nameID, string in nameIDs.items(): + assert string, nameID + nametable.setName(string, nameID, *platform) + + if "fvar" not in varfont: + nametable.removeNames(NameID.VARIATIONS_POSTSCRIPT_NAME_PREFIX) + + +def _updatePSNameRecord(varfont, familyName, styleName, platform): + # Implementation based on Adobe Technical Note #5902 : + # https://wwwimages2.adobe.com/content/dam/acom/en/devnet/font/pdfs/5902.AdobePSNameGeneration.pdf + nametable = varfont["name"] + + family_prefix = nametable.getName( + NameID.VARIATIONS_POSTSCRIPT_NAME_PREFIX, *platform + ) + if family_prefix: + family_prefix = family_prefix.toUnicode() + else: + family_prefix = familyName + + psName = f"{family_prefix}-{styleName}" + # Remove any characters other than uppercase Latin letters, lowercase + # Latin letters, digits and hyphens. + psName = re.sub(r"[^A-Za-z0-9-]", r"", psName) + + if len(psName) > 127: + # Abbreviating the stylename so it fits within 127 characters whilst + # conforming to every vendor's specification is too complex. Instead + # we simply truncate the psname and add the required "..." + return f"{psName[:124]}..." + return psName + + +def _updateUniqueIdNameRecord(varfont, nameIDs, platform): + nametable = varfont["name"] + currentRecord = nametable.getName(NameID.UNIQUE_FONT_IDENTIFIER, *platform) + if not currentRecord: + return None + + # Check if full name and postscript name are a substring of currentRecord + for nameID in (NameID.FULL_FONT_NAME, NameID.POSTSCRIPT_NAME): + nameRecord = nametable.getName(nameID, *platform) + if not nameRecord: + continue + if nameRecord.toUnicode() in currentRecord.toUnicode(): + return currentRecord.toUnicode().replace( + nameRecord.toUnicode(), nameIDs[nameRecord.nameID] + ) + + # Create a new string since we couldn't find any substrings. + fontVersion = _fontVersion(varfont, platform) + achVendID = varfont["OS/2"].achVendID + # Remove non-ASCII characers and trailing spaces + vendor = re.sub(r"[^\x00-\x7F]", "", achVendID).strip() + psName = nameIDs[NameID.POSTSCRIPT_NAME] + return f"{fontVersion};{vendor};{psName}" + + +def _fontVersion(font, platform=(3, 1, 0x409)): + nameRecord = font["name"].getName(NameID.VERSION_STRING, *platform) + if nameRecord is None: + return f'{font["head"].fontRevision:.3f}' + # "Version 1.101; ttfautohint (v1.8.1.43-b0c9)" --> "1.101" + # Also works fine with inputs "Version 1.101" or "1.101" etc + versionNumber = nameRecord.toUnicode().split(";")[0] + return versionNumber.lstrip("Version ").strip() diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/solver.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/solver.py new file mode 100644 index 0000000000000000000000000000000000000000..ba5231b796795259f9b7a471242e142d48623b0e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/instancer/solver.py @@ -0,0 +1,309 @@ +from fontTools.varLib.models import supportScalar +from fontTools.misc.fixedTools import MAX_F2DOT14 +from functools import lru_cache + +__all__ = ["rebaseTent"] + +EPSILON = 1 / (1 << 14) + + +def _reverse_negate(v): + return (-v[2], -v[1], -v[0]) + + +def _solve(tent, axisLimit, negative=False): + axisMin, axisDef, axisMax, _distanceNegative, _distancePositive = axisLimit + lower, peak, upper = tent + + # Mirror the problem such that axisDef <= peak + if axisDef > peak: + return [ + (scalar, _reverse_negate(t) if t is not None else None) + for scalar, t in _solve( + _reverse_negate(tent), + axisLimit.reverse_negate(), + not negative, + ) + ] + # axisDef <= peak + + # case 1: The whole deltaset falls outside the new limit; we can drop it + # + # peak + # 1.........................................o.......... + # / \ + # / \ + # / \ + # / \ + # 0---|-----------|----------|-------- o o----1 + # axisMin axisDef axisMax lower upper + # + if axisMax <= lower and axisMax < peak: + return [] # No overlap + + # case 2: Only the peak and outermost bound fall outside the new limit; + # we keep the deltaset, update peak and outermost bound and and scale deltas + # by the scalar value for the restricted axis at the new limit, and solve + # recursively. + # + # |peak + # 1...............................|.o.......... + # |/ \ + # / \ + # /| \ + # / | \ + # 0--------------------------- o | o----1 + # lower | upper + # | + # axisMax + # + # Convert to: + # + # 1............................................ + # | + # o peak + # /| + # /x| + # 0--------------------------- o o upper ----1 + # lower | + # | + # axisMax + if axisMax < peak: + mult = supportScalar({"tag": axisMax}, {"tag": tent}) + tent = (lower, axisMax, axisMax) + return [(scalar * mult, t) for scalar, t in _solve(tent, axisLimit)] + + # lower <= axisDef <= peak <= axisMax + + gain = supportScalar({"tag": axisDef}, {"tag": tent}) + out = [(gain, None)] + + # First, the positive side + + # outGain is the scalar of axisMax at the tent. + outGain = supportScalar({"tag": axisMax}, {"tag": tent}) + + # Case 3a: Gain is more than outGain. The tent down-slope crosses + # the axis into negative. We have to split it into multiples. + # + # | peak | + # 1...................|.o.....|.............. + # |/x\_ | + # gain................+....+_.|.............. + # /| |y\| + # ................../.|....|..+_......outGain + # / | | | \ + # 0---|-----------o | | | o----------1 + # axisMin lower | | | upper + # | | | + # axisDef | axisMax + # | + # crossing + if gain >= outGain: + # Note that this is the branch taken if both gain and outGain are 0. + + # Crossing point on the axis. + crossing = peak + (1 - gain) * (upper - peak) + + loc = (max(lower, axisDef), peak, crossing) + scalar = 1 + + # The part before the crossing point. + out.append((scalar - gain, loc)) + + # The part after the crossing point may use one or two tents, + # depending on whether upper is before axisMax or not, in one + # case we need to keep it down to eternity. + + # Case 3a1, similar to case 1neg; just one tent needed, as in + # the drawing above. + if upper >= axisMax: + loc = (crossing, axisMax, axisMax) + scalar = outGain + + out.append((scalar - gain, loc)) + + # Case 3a2: Similar to case 2neg; two tents needed, to keep + # down to eternity. + # + # | peak | + # 1...................|.o................|... + # |/ \_ | + # gain................+....+_............|... + # /| | \xxxxxxxxxxy| + # / | | \_xxxxxyyyy| + # / | | \xxyyyyyy| + # 0---|-----------o | | o-------|--1 + # axisMin lower | | upper | + # | | | + # axisDef | axisMax + # | + # crossing + else: + # A tent's peak cannot fall on axis default. Nudge it. + if upper == axisDef: + upper += EPSILON + + # Downslope. + loc1 = (crossing, upper, axisMax) + scalar1 = 0 + + # Eternity justify. + loc2 = (upper, axisMax, axisMax) + scalar2 = 0 + + out.append((scalar1 - gain, loc1)) + out.append((scalar2 - gain, loc2)) + + else: + # Special-case if peak is at axisMax. + if axisMax == peak: + upper = peak + + # Case 3: + # We keep delta as is and only scale the axis upper to achieve + # the desired new tent if feasible. + # + # peak + # 1.....................o.................... + # / \_| + # ..................../....+_.........outGain + # / | \ + # gain..............+......|..+_............. + # /| | | \ + # 0---|-----------o | | | o----------1 + # axisMin lower| | | upper + # | | newUpper + # axisDef axisMax + # + newUpper = peak + (1 - gain) * (upper - peak) + assert axisMax <= newUpper # Because outGain > gain + # Disabled because ots doesn't like us: + # https://github.com/fonttools/fonttools/issues/3350 + if False and newUpper <= axisDef + (axisMax - axisDef) * 2: + upper = newUpper + if not negative and axisDef + (axisMax - axisDef) * MAX_F2DOT14 < upper: + # we clamp +2.0 to the max F2Dot14 (~1.99994) for convenience + upper = axisDef + (axisMax - axisDef) * MAX_F2DOT14 + assert peak < upper + + loc = (max(axisDef, lower), peak, upper) + scalar = 1 + + out.append((scalar - gain, loc)) + + # Case 4: New limit doesn't fit; we need to chop into two tents, + # because the shape of a triangle with part of one side cut off + # cannot be represented as a triangle itself. + # + # | peak | + # 1.........|......o.|.................... + # ..........|...../x\|.............outGain + # | |xxy|\_ + # | /xxxy| \_ + # | |xxxxy| \_ + # | /xxxxy| \_ + # 0---|-----|-oxxxxxx| o----------1 + # axisMin | lower | upper + # | | + # axisDef axisMax + # + else: + loc1 = (max(axisDef, lower), peak, axisMax) + scalar1 = 1 + + loc2 = (peak, axisMax, axisMax) + scalar2 = outGain + + out.append((scalar1 - gain, loc1)) + # Don't add a dirac delta! + if peak < axisMax: + out.append((scalar2 - gain, loc2)) + + # Now, the negative side + + # Case 1neg: Lower extends beyond axisMin: we chop. Simple. + # + # | |peak + # 1..................|...|.o................. + # | |/ \ + # gain...............|...+...\............... + # |x_/| \ + # |/ | \ + # _/| | \ + # 0---------------o | | o----------1 + # lower | | upper + # | | + # axisMin axisDef + # + if lower <= axisMin: + loc = (axisMin, axisMin, axisDef) + scalar = supportScalar({"tag": axisMin}, {"tag": tent}) + + out.append((scalar - gain, loc)) + + # Case 2neg: Lower is betwen axisMin and axisDef: we add two + # tents to keep it down all the way to eternity. + # + # | |peak + # 1...|...............|.o................. + # | |/ \ + # gain|...............+...\............... + # |yxxxxxxxxxxxxx/| \ + # |yyyyyyxxxxxxx/ | \ + # |yyyyyyyyyyyx/ | \ + # 0---|-----------o | o----------1 + # axisMin lower | upper + # | + # axisDef + # + else: + # A tent's peak cannot fall on axis default. Nudge it. + if lower == axisDef: + lower -= EPSILON + + # Downslope. + loc1 = (axisMin, lower, axisDef) + scalar1 = 0 + + # Eternity justify. + loc2 = (axisMin, axisMin, lower) + scalar2 = 0 + + out.append((scalar1 - gain, loc1)) + out.append((scalar2 - gain, loc2)) + + return out + + +@lru_cache(128) +def rebaseTent(tent, axisLimit): + """Given a tuple (lower,peak,upper) "tent" and new axis limits + (axisMin,axisDefault,axisMax), solves how to represent the tent + under the new axis configuration. All values are in normalized + -1,0,+1 coordinate system. Tent values can be outside this range. + + Return value is a list of tuples. Each tuple is of the form + (scalar,tent), where scalar is a multipler to multiply any + delta-sets by, and tent is a new tent for that output delta-set. + If tent value is None, that is a special deltaset that should + be always-enabled (called "gain").""" + + axisMin, axisDef, axisMax, _distanceNegative, _distancePositive = axisLimit + assert -1 <= axisMin <= axisDef <= axisMax <= +1 + + lower, peak, upper = tent + assert -2 <= lower <= peak <= upper <= +2 + + assert peak != 0 + + sols = _solve(tent, axisLimit) + + n = lambda v: axisLimit.renormalizeValue(v) + sols = [ + (scalar, (n(v[0]), n(v[1]), n(v[2])) if v is not None else None) + for scalar, v in sols + if scalar + ] + + return sols diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatable.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatable.py new file mode 100644 index 0000000000000000000000000000000000000000..64b3a6d196810c1e75db7b29ee8fc3cc061742cd --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatable.py @@ -0,0 +1,1209 @@ +""" +Tool to find wrong contour order between different masters, and +other interpolatability (or lack thereof) issues. + +Call as: +$ fonttools varLib.interpolatable font1 font2 ... +""" + +from .interpolatableHelpers import * +from .interpolatableTestContourOrder import test_contour_order +from .interpolatableTestStartingPoint import test_starting_point +from fontTools.pens.recordingPen import ( + RecordingPen, + DecomposingRecordingPen, + lerpRecordings, +) +from fontTools.pens.transformPen import TransformPen +from fontTools.pens.statisticsPen import StatisticsPen, StatisticsControlPen +from fontTools.pens.momentsPen import OpenContourError +from fontTools.varLib.models import piecewiseLinearMap, normalizeLocation +from fontTools.misc.fixedTools import floatToFixedToStr +from fontTools.misc.transform import Transform +from collections import defaultdict +from types import SimpleNamespace +from functools import wraps +from pprint import pformat +from math import sqrt, atan2, pi +import logging +import os + +log = logging.getLogger("fontTools.varLib.interpolatable") + +DEFAULT_TOLERANCE = 0.95 +DEFAULT_KINKINESS = 0.5 +DEFAULT_KINKINESS_LENGTH = 0.002 # ratio of UPEM +DEFAULT_UPEM = 1000 + + +class Glyph: + ITEMS = ( + "recordings", + "greenStats", + "controlStats", + "greenVectors", + "controlVectors", + "nodeTypes", + "isomorphisms", + "points", + "openContours", + ) + + def __init__(self, glyphname, glyphset): + self.name = glyphname + for item in self.ITEMS: + setattr(self, item, []) + self._populate(glyphset) + + def _fill_in(self, ix): + for item in self.ITEMS: + if len(getattr(self, item)) == ix: + getattr(self, item).append(None) + + def _populate(self, glyphset): + glyph = glyphset[self.name] + self.doesnt_exist = glyph is None + if self.doesnt_exist: + return + + perContourPen = PerContourOrComponentPen(RecordingPen, glyphset=glyphset) + try: + glyph.draw(perContourPen, outputImpliedClosingLine=True) + except TypeError: + glyph.draw(perContourPen) + self.recordings = perContourPen.value + del perContourPen + + for ix, contour in enumerate(self.recordings): + nodeTypes = [op for op, arg in contour.value] + self.nodeTypes.append(nodeTypes) + + greenStats = StatisticsPen(glyphset=glyphset) + controlStats = StatisticsControlPen(glyphset=glyphset) + try: + contour.replay(greenStats) + contour.replay(controlStats) + self.openContours.append(False) + except OpenContourError as e: + self.openContours.append(True) + self._fill_in(ix) + continue + self.greenStats.append(greenStats) + self.controlStats.append(controlStats) + self.greenVectors.append(contour_vector_from_stats(greenStats)) + self.controlVectors.append(contour_vector_from_stats(controlStats)) + + # Check starting point + if nodeTypes[0] == "addComponent": + self._fill_in(ix) + continue + + assert nodeTypes[0] == "moveTo" + assert nodeTypes[-1] in ("closePath", "endPath") + points = SimpleRecordingPointPen() + converter = SegmentToPointPen(points, False) + contour.replay(converter) + # points.value is a list of pt,bool where bool is true if on-curve and false if off-curve; + # now check all rotations and mirror-rotations of the contour and build list of isomorphic + # possible starting points. + self.points.append(points.value) + + isomorphisms = [] + self.isomorphisms.append(isomorphisms) + + # Add rotations + add_isomorphisms(points.value, isomorphisms, False) + # Add mirrored rotations + add_isomorphisms(points.value, isomorphisms, True) + + def draw(self, pen, contour_idx=None): + if contour_idx is None: + for contour in self.recordings: + contour.draw(pen) + else: + self.recordings[contour_idx].draw(pen) + + +def test_gen( + glyphsets, + glyphs=None, + names=None, + ignore_missing=False, + *, + locations=None, + tolerance=DEFAULT_TOLERANCE, + kinkiness=DEFAULT_KINKINESS, + upem=DEFAULT_UPEM, + show_all=False, + discrete_axes=[], +): + if tolerance >= 10: + tolerance *= 0.01 + assert 0 <= tolerance <= 1 + if kinkiness >= 10: + kinkiness *= 0.01 + assert 0 <= kinkiness + + names = names or [repr(g) for g in glyphsets] + + if glyphs is None: + # `glyphs = glyphsets[0].keys()` is faster, certainly, but doesn't allow for sparse TTFs/OTFs given out of order + # ... risks the sparse master being the first one, and only processing a subset of the glyphs + glyphs = {g for glyphset in glyphsets for g in glyphset.keys()} + + parents, order = find_parents_and_order( + glyphsets, locations, discrete_axes=discrete_axes + ) + + def grand_parent(i, glyphname): + if i is None: + return None + i = parents[i] + if i is None: + return None + while parents[i] is not None and glyphsets[i][glyphname] is None: + i = parents[i] + return i + + for glyph_name in glyphs: + log.info("Testing glyph %s", glyph_name) + allGlyphs = [Glyph(glyph_name, glyphset) for glyphset in glyphsets] + if len([1 for glyph in allGlyphs if not glyph.doesnt_exist]) <= 1: + continue + for master_idx, (glyph, glyphset, name) in enumerate( + zip(allGlyphs, glyphsets, names) + ): + if glyph.doesnt_exist: + if not ignore_missing: + yield ( + glyph_name, + { + "type": InterpolatableProblem.MISSING, + "master": name, + "master_idx": master_idx, + }, + ) + continue + + has_open = False + for ix, open in enumerate(glyph.openContours): + if not open: + continue + has_open = True + yield ( + glyph_name, + { + "type": InterpolatableProblem.OPEN_PATH, + "master": name, + "master_idx": master_idx, + "contour": ix, + }, + ) + if has_open: + continue + + matchings = [None] * len(glyphsets) + + for m1idx in order: + glyph1 = allGlyphs[m1idx] + if glyph1 is None or not glyph1.nodeTypes: + continue + m0idx = grand_parent(m1idx, glyph_name) + if m0idx is None: + continue + glyph0 = allGlyphs[m0idx] + if glyph0 is None or not glyph0.nodeTypes: + continue + + # + # Basic compatibility checks + # + + m0 = glyph0.nodeTypes + m1 = glyph1.nodeTypes + if len(m0) != len(m1): + yield ( + glyph_name, + { + "type": InterpolatableProblem.PATH_COUNT, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "value_1": len(m0), + "value_2": len(m1), + }, + ) + continue + + if m0 != m1: + for pathIx, (nodes1, nodes2) in enumerate(zip(m0, m1)): + if nodes1 == nodes2: + continue + if len(nodes1) != len(nodes2): + yield ( + glyph_name, + { + "type": InterpolatableProblem.NODE_COUNT, + "path": pathIx, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "value_1": len(nodes1), + "value_2": len(nodes2), + }, + ) + continue + for nodeIx, (n1, n2) in enumerate(zip(nodes1, nodes2)): + if n1 != n2: + yield ( + glyph_name, + { + "type": InterpolatableProblem.NODE_INCOMPATIBILITY, + "path": pathIx, + "node": nodeIx, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "value_1": n1, + "value_2": n2, + }, + ) + continue + + # + # InterpolatableProblem.CONTOUR_ORDER check + # + + this_tolerance, matching = test_contour_order(glyph0, glyph1) + if this_tolerance < tolerance: + yield ( + glyph_name, + { + "type": InterpolatableProblem.CONTOUR_ORDER, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "value_1": list(range(len(matching))), + "value_2": matching, + "tolerance": this_tolerance, + }, + ) + matchings[m1idx] = matching + + # + # wrong-start-point / weight check + # + + m0Isomorphisms = glyph0.isomorphisms + m1Isomorphisms = glyph1.isomorphisms + m0Vectors = glyph0.greenVectors + m1Vectors = glyph1.greenVectors + recording0 = glyph0.recordings + recording1 = glyph1.recordings + + # If contour-order is wrong, adjust it + matching = matchings[m1idx] + if ( + matching is not None and m1Isomorphisms + ): # m1 is empty for composite glyphs + m1Isomorphisms = [m1Isomorphisms[i] for i in matching] + m1Vectors = [m1Vectors[i] for i in matching] + recording1 = [recording1[i] for i in matching] + + midRecording = [] + for c0, c1 in zip(recording0, recording1): + try: + r = RecordingPen() + r.value = list(lerpRecordings(c0.value, c1.value)) + midRecording.append(r) + except ValueError: + # Mismatch because of the reordering above + midRecording.append(None) + + for ix, (contour0, contour1) in enumerate( + zip(m0Isomorphisms, m1Isomorphisms) + ): + if ( + contour0 is None + or contour1 is None + or len(contour0) == 0 + or len(contour0) != len(contour1) + ): + # We already reported this; or nothing to do; or not compatible + # after reordering above. + continue + + this_tolerance, proposed_point, reverse = test_starting_point( + glyph0, glyph1, ix, tolerance, matching + ) + + if this_tolerance < tolerance: + yield ( + glyph_name, + { + "type": InterpolatableProblem.WRONG_START_POINT, + "contour": ix, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "value_1": 0, + "value_2": proposed_point, + "reversed": reverse, + "tolerance": this_tolerance, + }, + ) + + # Weight check. + # + # If contour could be mid-interpolated, and the two + # contours have the same area sign, proceeed. + # + # The sign difference can happen if it's a weirdo + # self-intersecting contour; ignore it. + contour = midRecording[ix] + + if contour and (m0Vectors[ix][0] < 0) == (m1Vectors[ix][0] < 0): + midStats = StatisticsPen(glyphset=None) + contour.replay(midStats) + + midVector = contour_vector_from_stats(midStats) + + m0Vec = m0Vectors[ix] + m1Vec = m1Vectors[ix] + size0 = m0Vec[0] * m0Vec[0] + size1 = m1Vec[0] * m1Vec[0] + midSize = midVector[0] * midVector[0] + + for overweight, problem_type in enumerate( + ( + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, + ) + ): + if overweight: + expectedSize = max(size0, size1) + continue # disabled + else: + expectedSize = sqrt(size0 * size1) + + log.debug( + "%s: actual size %g; threshold size %g, master sizes: %g, %g", + problem_type, + midSize, + expectedSize, + size0, + size1, + ) + + if ( + not overweight and expectedSize * tolerance > midSize + 1e-5 + ) or (overweight and 1e-5 + expectedSize / tolerance < midSize): + try: + if overweight: + this_tolerance = expectedSize / midSize + else: + this_tolerance = midSize / expectedSize + except ZeroDivisionError: + this_tolerance = 0 + log.debug("tolerance %g", this_tolerance) + yield ( + glyph_name, + { + "type": problem_type, + "contour": ix, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "tolerance": this_tolerance, + }, + ) + + # + # "kink" detector + # + m0 = glyph0.points + m1 = glyph1.points + + # If contour-order is wrong, adjust it + if matchings[m1idx] is not None and m1: # m1 is empty for composite glyphs + m1 = [m1[i] for i in matchings[m1idx]] + + t = 0.1 # ~sin(radian(6)) for tolerance 0.95 + deviation_threshold = ( + upem * DEFAULT_KINKINESS_LENGTH * DEFAULT_KINKINESS / kinkiness + ) + + for ix, (contour0, contour1) in enumerate(zip(m0, m1)): + if ( + contour0 is None + or contour1 is None + or len(contour0) == 0 + or len(contour0) != len(contour1) + ): + # We already reported this; or nothing to do; or not compatible + # after reordering above. + continue + + # Walk the contour, keeping track of three consecutive points, with + # middle one being an on-curve. If the three are co-linear then + # check for kinky-ness. + for i in range(len(contour0)): + pt0 = contour0[i] + pt1 = contour1[i] + if not pt0[1] or not pt1[1]: + # Skip off-curves + continue + pt0_prev = contour0[i - 1] + pt1_prev = contour1[i - 1] + pt0_next = contour0[(i + 1) % len(contour0)] + pt1_next = contour1[(i + 1) % len(contour1)] + + if pt0_prev[1] and pt1_prev[1]: + # At least one off-curve is required + continue + if pt0_next[1] and pt1_next[1]: + # At least one off-curve is required + continue + + pt0 = complex(*pt0[0]) + pt1 = complex(*pt1[0]) + pt0_prev = complex(*pt0_prev[0]) + pt1_prev = complex(*pt1_prev[0]) + pt0_next = complex(*pt0_next[0]) + pt1_next = complex(*pt1_next[0]) + + # We have three consecutive points. Check whether + # they are colinear. + d0_prev = pt0 - pt0_prev + d0_next = pt0_next - pt0 + d1_prev = pt1 - pt1_prev + d1_next = pt1_next - pt1 + + sin0 = d0_prev.real * d0_next.imag - d0_prev.imag * d0_next.real + sin1 = d1_prev.real * d1_next.imag - d1_prev.imag * d1_next.real + try: + sin0 /= abs(d0_prev) * abs(d0_next) + sin1 /= abs(d1_prev) * abs(d1_next) + except ZeroDivisionError: + continue + + if abs(sin0) > t or abs(sin1) > t: + # Not colinear / not smooth. + continue + + # Check the mid-point is actually, well, in the middle. + dot0 = d0_prev.real * d0_next.real + d0_prev.imag * d0_next.imag + dot1 = d1_prev.real * d1_next.real + d1_prev.imag * d1_next.imag + if dot0 < 0 or dot1 < 0: + # Sharp corner. + continue + + # Fine, if handle ratios are similar... + r0 = abs(d0_prev) / (abs(d0_prev) + abs(d0_next)) + r1 = abs(d1_prev) / (abs(d1_prev) + abs(d1_next)) + r_diff = abs(r0 - r1) + if abs(r_diff) < t: + # Smooth enough. + continue + + mid = (pt0 + pt1) / 2 + mid_prev = (pt0_prev + pt1_prev) / 2 + mid_next = (pt0_next + pt1_next) / 2 + + mid_d0 = mid - mid_prev + mid_d1 = mid_next - mid + + sin_mid = mid_d0.real * mid_d1.imag - mid_d0.imag * mid_d1.real + try: + sin_mid /= abs(mid_d0) * abs(mid_d1) + except ZeroDivisionError: + continue + + # ...or if the angles are similar. + if abs(sin_mid) * (tolerance * kinkiness) <= t: + # Smooth enough. + continue + + # How visible is the kink? + + cross = sin_mid * abs(mid_d0) * abs(mid_d1) + arc_len = abs(mid_d0 + mid_d1) + deviation = abs(cross / arc_len) + if deviation < deviation_threshold: + continue + deviation_ratio = deviation / arc_len + if deviation_ratio > t: + continue + + this_tolerance = t / (abs(sin_mid) * kinkiness) + + log.debug( + "kink: deviation %g; deviation_ratio %g; sin_mid %g; r_diff %g", + deviation, + deviation_ratio, + sin_mid, + r_diff, + ) + log.debug("tolerance %g", this_tolerance) + yield ( + glyph_name, + { + "type": InterpolatableProblem.KINK, + "contour": ix, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + "value": i, + "tolerance": this_tolerance, + }, + ) + + # + # --show-all + # + + if show_all: + yield ( + glyph_name, + { + "type": InterpolatableProblem.NOTHING, + "master_1": names[m0idx], + "master_2": names[m1idx], + "master_1_idx": m0idx, + "master_2_idx": m1idx, + }, + ) + + +@wraps(test_gen) +def test(*args, **kwargs): + problems = defaultdict(list) + for glyphname, problem in test_gen(*args, **kwargs): + problems[glyphname].append(problem) + return problems + + +def recursivelyAddGlyph(glyphname, glyphset, ttGlyphSet, glyf): + if glyphname in glyphset: + return + glyphset[glyphname] = ttGlyphSet[glyphname] + + for component in getattr(glyf[glyphname], "components", []): + recursivelyAddGlyph(component.glyphName, glyphset, ttGlyphSet, glyf) + + +def ensure_parent_dir(path): + dirname = os.path.dirname(path) + if dirname: + os.makedirs(dirname, exist_ok=True) + return path + + +def main(args=None): + """Test for interpolatability issues between fonts""" + import argparse + import sys + + parser = argparse.ArgumentParser( + "fonttools varLib.interpolatable", + description=main.__doc__, + ) + parser.add_argument( + "--glyphs", + action="store", + help="Space-separate name of glyphs to check", + ) + parser.add_argument( + "--show-all", + action="store_true", + help="Show all glyph pairs, even if no problems are found", + ) + parser.add_argument( + "--tolerance", + action="store", + type=float, + help="Error tolerance. Between 0 and 1. Default %s" % DEFAULT_TOLERANCE, + ) + parser.add_argument( + "--kinkiness", + action="store", + type=float, + help="How aggressively report kinks. Default %s" % DEFAULT_KINKINESS, + ) + parser.add_argument( + "--json", + action="store_true", + help="Output report in JSON format", + ) + parser.add_argument( + "--pdf", + action="store", + help="Output report in PDF format", + ) + parser.add_argument( + "--ps", + action="store", + help="Output report in PostScript format", + ) + parser.add_argument( + "--html", + action="store", + help="Output report in HTML format", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="Only exit with code 1 or 0, no output", + ) + parser.add_argument( + "--output", + action="store", + help="Output file for the problem report; Default: stdout", + ) + parser.add_argument( + "--ignore-missing", + action="store_true", + help="Will not report glyphs missing from sparse masters as errors", + ) + parser.add_argument( + "inputs", + metavar="FILE", + type=str, + nargs="+", + help="Input a single variable font / DesignSpace / Glyphs file, or multiple TTF/UFO files", + ) + parser.add_argument( + "--name", + metavar="NAME", + type=str, + action="append", + help="Name of the master to use in the report. If not provided, all are used.", + ) + parser.add_argument("-v", "--verbose", action="store_true", help="Run verbosely.") + parser.add_argument("--debug", action="store_true", help="Run with debug output.") + + args = parser.parse_args(args) + + from fontTools import configLogger + + configLogger(level=("INFO" if args.verbose else "WARNING")) + if args.debug: + configLogger(level="DEBUG") + + glyphs = args.glyphs.split() if args.glyphs else None + + from os.path import basename + + fonts = [] + names = [] + locations = [] + discrete_axes = set() + upem = DEFAULT_UPEM + + original_args_inputs = tuple(args.inputs) + + if len(args.inputs) == 1: + designspace = None + if args.inputs[0].endswith(".designspace"): + from fontTools.designspaceLib import DesignSpaceDocument + + designspace = DesignSpaceDocument.fromfile(args.inputs[0]) + args.inputs = [master.path for master in designspace.sources] + locations = [master.location for master in designspace.sources] + discrete_axes = { + a.name for a in designspace.axes if not hasattr(a, "minimum") + } + axis_triples = { + a.name: (a.minimum, a.default, a.maximum) + for a in designspace.axes + if a.name not in discrete_axes + } + axis_mappings = {a.name: a.map for a in designspace.axes} + axis_triples = { + k: tuple(piecewiseLinearMap(v, dict(axis_mappings[k])) for v in vv) + for k, vv in axis_triples.items() + } + + elif args.inputs[0].endswith((".glyphs", ".glyphspackage")): + from glyphsLib import GSFont, to_designspace + + gsfont = GSFont(args.inputs[0]) + upem = gsfont.upm + designspace = to_designspace(gsfont) + fonts = [source.font for source in designspace.sources] + names = ["%s-%s" % (f.info.familyName, f.info.styleName) for f in fonts] + args.inputs = [] + locations = [master.location for master in designspace.sources] + axis_triples = { + a.name: (a.minimum, a.default, a.maximum) for a in designspace.axes + } + axis_mappings = {a.name: a.map for a in designspace.axes} + axis_triples = { + k: tuple(piecewiseLinearMap(v, dict(axis_mappings[k])) for v in vv) + for k, vv in axis_triples.items() + } + + elif args.inputs[0].endswith(".ttf") or args.inputs[0].endswith(".otf"): + from fontTools.ttLib import TTFont + + # Is variable font? + + font = TTFont(args.inputs[0]) + upem = font["head"].unitsPerEm + + fvar = font["fvar"] + axisMapping = {} + for axis in fvar.axes: + axisMapping[axis.axisTag] = { + -1: axis.minValue, + 0: axis.defaultValue, + 1: axis.maxValue, + } + normalized = False + if "avar" in font: + avar = font["avar"] + if getattr(avar.table, "VarStore", None): + axisMapping = {tag: {-1: -1, 0: 0, 1: 1} for tag in axisMapping} + normalized = True + else: + for axisTag, segments in avar.segments.items(): + fvarMapping = axisMapping[axisTag].copy() + for location, value in segments.items(): + axisMapping[axisTag][value] = piecewiseLinearMap( + location, fvarMapping + ) + + # Gather all glyphs at their "master" locations + ttGlyphSets = {} + glyphsets = defaultdict(dict) + + if "gvar" in font: + gvar = font["gvar"] + glyf = font["glyf"] + + if glyphs is None: + glyphs = sorted(gvar.variations.keys()) + for glyphname in glyphs: + for var in gvar.variations[glyphname]: + locDict = {} + loc = [] + for tag, val in sorted(var.axes.items()): + locDict[tag] = val[1] + loc.append((tag, val[1])) + + locTuple = tuple(loc) + if locTuple not in ttGlyphSets: + ttGlyphSets[locTuple] = font.getGlyphSet( + location=locDict, normalized=True, recalcBounds=False + ) + + recursivelyAddGlyph( + glyphname, glyphsets[locTuple], ttGlyphSets[locTuple], glyf + ) + + elif "CFF2" in font: + fvarAxes = font["fvar"].axes + cff2 = font["CFF2"].cff.topDictIndex[0] + charstrings = cff2.CharStrings + + if glyphs is None: + glyphs = sorted(charstrings.keys()) + for glyphname in glyphs: + cs = charstrings[glyphname] + private = cs.private + + # Extract vsindex for the glyph + vsindices = {getattr(private, "vsindex", 0)} + vsindex = getattr(private, "vsindex", 0) + last_op = 0 + # The spec says vsindex can only appear once and must be the first + # operator in the charstring, but we support multiple. + # https://github.com/harfbuzz/boring-expansion-spec/issues/158 + for op in cs.program: + if op == "blend": + vsindices.add(vsindex) + elif op == "vsindex": + assert isinstance(last_op, int) + vsindex = last_op + last_op = op + + if not hasattr(private, "vstore"): + continue + + varStore = private.vstore.otVarStore + for vsindex in vsindices: + varData = varStore.VarData[vsindex] + for regionIndex in varData.VarRegionIndex: + region = varStore.VarRegionList.Region[regionIndex] + + locDict = {} + loc = [] + for axisIndex, axis in enumerate(region.VarRegionAxis): + tag = fvarAxes[axisIndex].axisTag + val = axis.PeakCoord + locDict[tag] = val + loc.append((tag, val)) + + locTuple = tuple(loc) + if locTuple not in ttGlyphSets: + ttGlyphSets[locTuple] = font.getGlyphSet( + location=locDict, + normalized=True, + recalcBounds=False, + ) + + glyphset = glyphsets[locTuple] + glyphset[glyphname] = ttGlyphSets[locTuple][glyphname] + + names = ["''"] + fonts = [font.getGlyphSet()] + locations = [{}] + axis_triples = {a: (-1, 0, +1) for a in sorted(axisMapping.keys())} + for locTuple in sorted(glyphsets.keys(), key=lambda v: (len(v), v)): + name = ( + "'" + + " ".join( + "%s=%s" + % ( + k, + floatToFixedToStr( + piecewiseLinearMap(v, axisMapping[k]), 14 + ), + ) + for k, v in locTuple + ) + + "'" + ) + if normalized: + name += " (normalized)" + names.append(name) + fonts.append(glyphsets[locTuple]) + locations.append(dict(locTuple)) + + args.ignore_missing = True + args.inputs = [] + + if not locations: + locations = [{} for _ in fonts] + + for filename in args.inputs: + if filename.endswith(".ufo"): + from fontTools.ufoLib import UFOReader + + font = UFOReader(filename) + info = SimpleNamespace() + font.readInfo(info) + upem = info.unitsPerEm + fonts.append(font) + else: + from fontTools.ttLib import TTFont + + font = TTFont(filename) + upem = font["head"].unitsPerEm + fonts.append(font) + + names.append(basename(filename).rsplit(".", 1)[0]) + + if len(fonts) < 2: + log.warning("Font file does not seem to be variable. Nothing to check.") + return + + glyphsets = [] + for font in fonts: + if hasattr(font, "getGlyphSet"): + glyphset = font.getGlyphSet() + else: + glyphset = font + glyphsets.append({k: glyphset[k] for k in glyphset.keys()}) + + if args.name: + accepted_names = set(args.name) + glyphsets = [ + glyphset + for name, glyphset in zip(names, glyphsets) + if name in accepted_names + ] + locations = [ + location + for name, location in zip(names, locations) + if name in accepted_names + ] + names = [name for name in names if name in accepted_names] + + if not glyphs: + glyphs = sorted(set([gn for glyphset in glyphsets for gn in glyphset.keys()])) + + glyphsSet = set(glyphs) + for glyphset in glyphsets: + glyphSetGlyphNames = set(glyphset.keys()) + diff = glyphsSet - glyphSetGlyphNames + if diff: + for gn in diff: + glyphset[gn] = None + + # Normalize locations + locations = [ + { + **normalizeLocation(loc, axis_triples), + **{k: v for k, v in loc.items() if k in discrete_axes}, + } + for loc in locations + ] + tolerance = args.tolerance or DEFAULT_TOLERANCE + kinkiness = args.kinkiness if args.kinkiness is not None else DEFAULT_KINKINESS + + try: + log.info("Running on %d glyphsets", len(glyphsets)) + log.info("Locations: %s", pformat(locations)) + problems_gen = test_gen( + glyphsets, + glyphs=glyphs, + names=names, + locations=locations, + upem=upem, + ignore_missing=args.ignore_missing, + tolerance=tolerance, + kinkiness=kinkiness, + show_all=args.show_all, + discrete_axes=discrete_axes, + ) + problems = defaultdict(list) + + f = ( + sys.stdout + if args.output is None + else open(ensure_parent_dir(args.output), "w") + ) + + if not args.quiet: + if args.json: + import json + + for glyphname, problem in problems_gen: + problems[glyphname].append(problem) + + print(json.dumps(problems), file=f) + else: + last_glyphname = None + for glyphname, p in problems_gen: + problems[glyphname].append(p) + + if glyphname != last_glyphname: + print(f"Glyph {glyphname} was not compatible:", file=f) + last_glyphname = glyphname + last_master_idxs = None + + master_idxs = ( + (p["master_idx"],) + if "master_idx" in p + else (p["master_1_idx"], p["master_2_idx"]) + ) + if master_idxs != last_master_idxs: + master_names = ( + (p["master"],) + if "master" in p + else (p["master_1"], p["master_2"]) + ) + print(f" Masters: %s:" % ", ".join(master_names), file=f) + last_master_idxs = master_idxs + + if p["type"] == InterpolatableProblem.MISSING: + print( + " Glyph was missing in master %s" % p["master"], file=f + ) + elif p["type"] == InterpolatableProblem.OPEN_PATH: + print( + " Glyph has an open path in master %s" % p["master"], + file=f, + ) + elif p["type"] == InterpolatableProblem.PATH_COUNT: + print( + " Path count differs: %i in %s, %i in %s" + % ( + p["value_1"], + p["master_1"], + p["value_2"], + p["master_2"], + ), + file=f, + ) + elif p["type"] == InterpolatableProblem.NODE_COUNT: + print( + " Node count differs in path %i: %i in %s, %i in %s" + % ( + p["path"], + p["value_1"], + p["master_1"], + p["value_2"], + p["master_2"], + ), + file=f, + ) + elif p["type"] == InterpolatableProblem.NODE_INCOMPATIBILITY: + print( + " Node %d incompatible in path %i: %s in %s, %s in %s" + % ( + p["node"], + p["path"], + p["value_1"], + p["master_1"], + p["value_2"], + p["master_2"], + ), + file=f, + ) + elif p["type"] == InterpolatableProblem.CONTOUR_ORDER: + print( + " Contour order differs: %s in %s, %s in %s" + % ( + p["value_1"], + p["master_1"], + p["value_2"], + p["master_2"], + ), + file=f, + ) + elif p["type"] == InterpolatableProblem.WRONG_START_POINT: + print( + " Contour %d start point differs: %s in %s, %s in %s; reversed: %s" + % ( + p["contour"], + p["value_1"], + p["master_1"], + p["value_2"], + p["master_2"], + p["reversed"], + ), + file=f, + ) + elif p["type"] == InterpolatableProblem.UNDERWEIGHT: + print( + " Contour %d interpolation is underweight: %s, %s" + % ( + p["contour"], + p["master_1"], + p["master_2"], + ), + file=f, + ) + elif p["type"] == InterpolatableProblem.OVERWEIGHT: + print( + " Contour %d interpolation is overweight: %s, %s" + % ( + p["contour"], + p["master_1"], + p["master_2"], + ), + file=f, + ) + elif p["type"] == InterpolatableProblem.KINK: + print( + " Contour %d has a kink at %s: %s, %s" + % ( + p["contour"], + p["value"], + p["master_1"], + p["master_2"], + ), + file=f, + ) + elif p["type"] == InterpolatableProblem.NOTHING: + print( + " Showing %s and %s" + % ( + p["master_1"], + p["master_2"], + ), + file=f, + ) + else: + for glyphname, problem in problems_gen: + problems[glyphname].append(problem) + + problems = sort_problems(problems) + + for p in "ps", "pdf": + arg = getattr(args, p) + if arg is None: + continue + log.info("Writing %s to %s", p.upper(), arg) + from .interpolatablePlot import InterpolatablePS, InterpolatablePDF + + PlotterClass = InterpolatablePS if p == "ps" else InterpolatablePDF + + with PlotterClass( + ensure_parent_dir(arg), glyphsets=glyphsets, names=names + ) as doc: + doc.add_title_page( + original_args_inputs, tolerance=tolerance, kinkiness=kinkiness + ) + if problems: + doc.add_summary(problems) + doc.add_problems(problems) + if not problems and not args.quiet: + doc.draw_cupcake() + if problems: + doc.add_index() + doc.add_table_of_contents() + + if args.html: + log.info("Writing HTML to %s", args.html) + from .interpolatablePlot import InterpolatableSVG + + svgs = [] + glyph_starts = {} + with InterpolatableSVG(svgs, glyphsets=glyphsets, names=names) as svg: + svg.add_title_page( + original_args_inputs, + show_tolerance=False, + tolerance=tolerance, + kinkiness=kinkiness, + ) + for glyph, glyph_problems in problems.items(): + glyph_starts[len(svgs)] = glyph + svg.add_problems( + {glyph: glyph_problems}, + show_tolerance=False, + show_page_number=False, + ) + if not problems and not args.quiet: + svg.draw_cupcake() + + import base64 + + with open(ensure_parent_dir(args.html), "wb") as f: + f.write(b"<!DOCTYPE html>\n") + f.write( + b'<html><body align="center" style="font-family: sans-serif; text-color: #222">\n' + ) + f.write(b"<title>fonttools varLib.interpolatable report</title>\n") + for i, svg in enumerate(svgs): + if i in glyph_starts: + f.write(f"<h1>Glyph {glyph_starts[i]}</h1>\n".encode("utf-8")) + f.write("<img src='data:image/svg+xml;base64,".encode("utf-8")) + f.write(base64.b64encode(svg)) + f.write(b"' />\n") + f.write(b"<hr>\n") + f.write(b"</body></html>\n") + + except Exception as e: + e.args += original_args_inputs + log.error(e) + raise + + if problems: + return problems + + +if __name__ == "__main__": + import sys + + problems = main() + sys.exit(int(bool(problems))) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatableHelpers.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatableHelpers.py new file mode 100644 index 0000000000000000000000000000000000000000..bbe072c83cf65a6ecf503e6b4ba9ad58453b3808 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatableHelpers.py @@ -0,0 +1,360 @@ +from fontTools.ttLib.ttGlyphSet import LerpGlyphSet +from fontTools.pens.basePen import AbstractPen, BasePen, DecomposingPen +from fontTools.pens.pointPen import AbstractPointPen, SegmentToPointPen +from fontTools.pens.recordingPen import RecordingPen, DecomposingRecordingPen +from fontTools.misc.transform import Transform +from collections import defaultdict, deque +from math import sqrt, copysign, atan2, pi +import itertools + +import logging + +log = logging.getLogger("fontTools.varLib.interpolatable") + + +class InterpolatableProblem: + NOTHING = "nothing" + MISSING = "missing" + OPEN_PATH = "open_path" + PATH_COUNT = "path_count" + NODE_COUNT = "node_count" + NODE_INCOMPATIBILITY = "node_incompatibility" + CONTOUR_ORDER = "contour_order" + WRONG_START_POINT = "wrong_start_point" + KINK = "kink" + UNDERWEIGHT = "underweight" + OVERWEIGHT = "overweight" + + severity = { + MISSING: 1, + OPEN_PATH: 2, + PATH_COUNT: 3, + NODE_COUNT: 4, + NODE_INCOMPATIBILITY: 5, + CONTOUR_ORDER: 6, + WRONG_START_POINT: 7, + KINK: 8, + UNDERWEIGHT: 9, + OVERWEIGHT: 10, + NOTHING: 11, + } + + +def sort_problems(problems): + """Sort problem groups by their most severe problem type.""" + return dict( + sorted( + problems.items(), + key=lambda _: -min( + ( + (InterpolatableProblem.severity[p["type"]] + p.get("tolerance", 0)) + for p in _[1] + ), + ), + reverse=True, + ) + ) + + +def rot_list(l, k): + """Rotate list by k items forward. Ie. item at position 0 will be + at position k in returned list. Negative k is allowed.""" + return l[-k:] + l[:-k] + + +class PerContourPen(BasePen): + def __init__(self, Pen, glyphset=None): + BasePen.__init__(self, glyphset) + self._Pen = Pen + self._pen = None + self.value = [] + + def _moveTo(self, p0): + self._newItem() + self._pen.moveTo(p0) + + def _lineTo(self, p1): + self._pen.lineTo(p1) + + def _qCurveToOne(self, p1, p2): + self._pen.qCurveTo(p1, p2) + + def _curveToOne(self, p1, p2, p3): + self._pen.curveTo(p1, p2, p3) + + def _closePath(self): + self._pen.closePath() + self._pen = None + + def _endPath(self): + self._pen.endPath() + self._pen = None + + def _newItem(self): + self._pen = pen = self._Pen() + self.value.append(pen) + + +class PerContourOrComponentPen(PerContourPen): + def addComponent(self, glyphName, transformation): + self._newItem() + self.value[-1].addComponent(glyphName, transformation) + + +class SimpleRecordingPointPen(AbstractPointPen): + def __init__(self): + self.value = [] + + def beginPath(self, identifier=None, **kwargs): + pass + + def endPath(self) -> None: + pass + + def addPoint(self, pt, segmentType=None): + self.value.append((pt, False if segmentType is None else True)) + + +def vdiff_hypot2(v0, v1): + s = 0 + for x0, x1 in zip(v0, v1): + d = x1 - x0 + s += d * d + return s + + +def vdiff_hypot2_complex(v0, v1): + s = 0 + for x0, x1 in zip(v0, v1): + d = x1 - x0 + s += d.real * d.real + d.imag * d.imag + # This does the same but seems to be slower: + # s += (d * d.conjugate()).real + return s + + +def matching_cost(G, matching): + return sum(G[i][j] for i, j in enumerate(matching)) + + +def min_cost_perfect_bipartite_matching_scipy(G): + n = len(G) + rows, cols = linear_sum_assignment(G) + assert (rows == list(range(n))).all() + # Convert numpy array and integer to Python types, + # to ensure that this is JSON-serializable. + cols = list(int(e) for e in cols) + return list(cols), matching_cost(G, cols) + + +def min_cost_perfect_bipartite_matching_munkres(G): + n = len(G) + cols = [None] * n + for row, col in Munkres().compute(G): + cols[row] = col + return cols, matching_cost(G, cols) + + +def min_cost_perfect_bipartite_matching_bruteforce(G): + n = len(G) + + if n > 6: + raise Exception("Install Python module 'munkres' or 'scipy >= 0.17.0'") + + # Otherwise just brute-force + permutations = itertools.permutations(range(n)) + best = list(next(permutations)) + best_cost = matching_cost(G, best) + for p in permutations: + cost = matching_cost(G, p) + if cost < best_cost: + best, best_cost = list(p), cost + return best, best_cost + + +# Prefer `scipy.optimize.linear_sum_assignment` for performance. +# `Munkres` is also supported as a fallback for minimalistic systems +# where installing SciPy is not feasible. +try: + from scipy.optimize import linear_sum_assignment + + min_cost_perfect_bipartite_matching = min_cost_perfect_bipartite_matching_scipy +except ImportError: + try: + from munkres import Munkres + + min_cost_perfect_bipartite_matching = ( + min_cost_perfect_bipartite_matching_munkres + ) + except ImportError: + min_cost_perfect_bipartite_matching = ( + min_cost_perfect_bipartite_matching_bruteforce + ) + + +def contour_vector_from_stats(stats): + # Don't change the order of items here. + # It's okay to add to the end, but otherwise, other + # code depends on it. Search for "covariance". + size = sqrt(abs(stats.area)) + return ( + copysign((size), stats.area), + stats.meanX, + stats.meanY, + stats.stddevX * 2, + stats.stddevY * 2, + stats.correlation * size, + ) + + +def matching_for_vectors(m0, m1): + n = len(m0) + + costs = [[vdiff_hypot2(v0, v1) for v1 in m1] for v0 in m0] + ( + matching, + matching_cost, + ) = min_cost_perfect_bipartite_matching(costs) + identity_cost = sum(costs[i][i] for i in range(n)) + return matching, matching_cost, identity_cost + + +def points_characteristic_bits(points): + bits = 0 + for pt, b in reversed(points): + bits = (bits << 1) | b + return bits + + +_NUM_ITEMS_PER_POINTS_COMPLEX_VECTOR = 4 + + +def points_complex_vector(points): + vector = [] + if not points: + return vector + points = [complex(*pt) for pt, _ in points] + n = len(points) + assert _NUM_ITEMS_PER_POINTS_COMPLEX_VECTOR == 4 + points.extend(points[: _NUM_ITEMS_PER_POINTS_COMPLEX_VECTOR - 1]) + while len(points) < _NUM_ITEMS_PER_POINTS_COMPLEX_VECTOR: + points.extend(points[: _NUM_ITEMS_PER_POINTS_COMPLEX_VECTOR - 1]) + for i in range(n): + # The weights are magic numbers. + + # The point itself + p0 = points[i] + vector.append(p0) + + # The vector to the next point + p1 = points[i + 1] + d0 = p1 - p0 + vector.append(d0 * 3) + + # The turn vector + p2 = points[i + 2] + d1 = p2 - p1 + vector.append(d1 - d0) + + # The angle to the next point, as a cross product; + # Square root of, to match dimentionality of distance. + cross = d0.real * d1.imag - d0.imag * d1.real + cross = copysign(sqrt(abs(cross)), cross) + vector.append(cross * 4) + + return vector + + +def add_isomorphisms(points, isomorphisms, reverse): + reference_bits = points_characteristic_bits(points) + n = len(points) + + # if points[0][0] == points[-1][0]: + # abort + + if reverse: + points = points[::-1] + bits = points_characteristic_bits(points) + else: + bits = reference_bits + + vector = points_complex_vector(points) + + assert len(vector) % n == 0 + mult = len(vector) // n + mask = (1 << n) - 1 + + for i in range(n): + b = ((bits << (n - i)) & mask) | (bits >> i) + if b == reference_bits: + isomorphisms.append( + (rot_list(vector, -i * mult), n - 1 - i if reverse else i, reverse) + ) + + +def find_parents_and_order(glyphsets, locations, *, discrete_axes=set()): + parents = [None] + list(range(len(glyphsets) - 1)) + order = list(range(len(glyphsets))) + if locations: + # Order base master first + bases = [ + i + for i, l in enumerate(locations) + if all(v == 0 for k, v in l.items() if k not in discrete_axes) + ] + if bases: + log.info("Found %s base masters: %s", len(bases), bases) + else: + log.warning("No base master location found") + + # Form a minimum spanning tree of the locations + try: + from scipy.sparse.csgraph import minimum_spanning_tree + + graph = [[0] * len(locations) for _ in range(len(locations))] + axes = set() + for l in locations: + axes.update(l.keys()) + axes = sorted(axes) + vectors = [tuple(l.get(k, 0) for k in axes) for l in locations] + for i, j in itertools.combinations(range(len(locations)), 2): + i_discrete_location = { + k: v for k, v in zip(axes, vectors[i]) if k in discrete_axes + } + j_discrete_location = { + k: v for k, v in zip(axes, vectors[j]) if k in discrete_axes + } + if i_discrete_location != j_discrete_location: + continue + graph[i][j] = vdiff_hypot2(vectors[i], vectors[j]) + + tree = minimum_spanning_tree(graph, overwrite=True) + rows, cols = tree.nonzero() + graph = defaultdict(set) + for row, col in zip(rows, cols): + graph[row].add(col) + graph[col].add(row) + + # Traverse graph from the base and assign parents + parents = [None] * len(locations) + order = [] + visited = set() + queue = deque(bases) + while queue: + i = queue.popleft() + visited.add(i) + order.append(i) + for j in sorted(graph[i]): + if j not in visited: + parents[j] = i + queue.append(j) + assert len(order) == len( + parents + ), "Not all masters are reachable; report an issue" + + except ImportError: + pass + + log.info("Parents: %s", parents) + log.info("Order: %s", order) + return parents, order diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatablePlot.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatablePlot.py new file mode 100644 index 0000000000000000000000000000000000000000..6685cd763a0c126c544145cea46193142a56b68d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatablePlot.py @@ -0,0 +1,1264 @@ +from .interpolatableHelpers import * +from fontTools.ttLib import TTFont +from fontTools.ttLib.ttGlyphSet import LerpGlyphSet +from fontTools.pens.recordingPen import ( + RecordingPen, + DecomposingRecordingPen, + RecordingPointPen, +) +from fontTools.pens.boundsPen import ControlBoundsPen +from fontTools.pens.cairoPen import CairoPen +from fontTools.pens.pointPen import ( + SegmentToPointPen, + PointToSegmentPen, + ReverseContourPointPen, +) +from itertools import cycle +from io import BytesIO +import cairo +import math +import os +import logging + +log = logging.getLogger("fontTools.varLib.interpolatable") + + +class OverridingDict(dict): + def __init__(self, parent_dict): + self.parent_dict = parent_dict + + def __missing__(self, key): + return self.parent_dict[key] + + +class InterpolatablePlot: + width = 8.5 * 72 + height = 11 * 72 + pad = 0.1 * 72 + title_font_size = 24 + font_size = 16 + page_number = 1 + head_color = (0.3, 0.3, 0.3) + label_color = (0.2, 0.2, 0.2) + border_color = (0.9, 0.9, 0.9) + border_width = 0.5 + fill_color = (0.8, 0.8, 0.8) + stroke_color = (0.1, 0.1, 0.1) + stroke_width = 1 + oncurve_node_color = (0, 0.8, 0, 0.7) + oncurve_node_diameter = 6 + offcurve_node_color = (0, 0.5, 0, 0.7) + offcurve_node_diameter = 4 + handle_color = (0, 0.5, 0, 0.7) + handle_width = 0.5 + corrected_start_point_color = (0, 0.9, 0, 0.7) + corrected_start_point_size = 7 + wrong_start_point_color = (1, 0, 0, 0.7) + start_point_color = (0, 0, 1, 0.7) + start_arrow_length = 9 + kink_point_size = 7 + kink_point_color = (1, 0, 1, 0.7) + kink_circle_size = 15 + kink_circle_stroke_width = 1 + kink_circle_color = (1, 0, 1, 0.7) + contour_colors = ((1, 0, 0), (0, 0, 1), (0, 1, 0), (1, 1, 0), (1, 0, 1), (0, 1, 1)) + contour_alpha = 0.5 + weight_issue_contour_color = (0, 0, 0, 0.4) + no_issues_label = "Your font's good! Have a cupcake..." + no_issues_label_color = (0, 0.5, 0) + cupcake_color = (0.3, 0, 0.3) + cupcake = r""" + ,@. + ,@.@@,. + ,@@,.@@@. @.@@@,. + ,@@. @@@. @@. @@,. + ,@@@.@,.@. @. @@@@,.@.@@,. + ,@@.@. @@.@@. @,. .@' @' @@, + ,@@. @. .@@.@@@. @@' @, +,@. @@. @, +@. @,@@,. , .@@, +@,. .@,@@,. .@@,. , .@@, @, @, +@. .@. @ @@,. , @ + @,.@@. @,. @@,. @. @,. @' + @@||@,. @'@,. @@,. @@ @,. @'@@, @' + \\@@@@' @,. @'@@@@' @@,. @@@' //@@@' + |||||||| @@,. @@' ||||||| |@@@|@|| || + \\\\\\\ ||@@@|| ||||||| ||||||| // + ||||||| |||||| |||||| |||||| || + \\\\\\ |||||| |||||| |||||| // + |||||| ||||| ||||| ||||| || + \\\\\ ||||| ||||| ||||| // + ||||| |||| ||||| |||| || + \\\\ |||| |||| |||| // + |||||||||||||||||||||||| +""" + emoticon_color = (0, 0.3, 0.3) + shrug = r"""\_(")_/""" + underweight = r""" + o +/|\ +/ \ +""" + overweight = r""" + o +/O\ +/ \ +""" + yay = r""" \o/ """ + + def __init__(self, out, glyphsets, names=None, **kwargs): + self.out = out + self.glyphsets = glyphsets + self.names = names or [repr(g) for g in glyphsets] + self.toc = {} + + for k, v in kwargs.items(): + if not hasattr(self, k): + raise TypeError("Unknown keyword argument: %s" % k) + setattr(self, k, v) + + self.panel_width = self.width / 2 - self.pad * 3 + self.panel_height = ( + self.height / 2 - self.pad * 6 - self.font_size * 2 - self.title_font_size + ) + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + pass + + def show_page(self): + self.page_number += 1 + + def add_title_page( + self, files, *, show_tolerance=True, tolerance=None, kinkiness=None + ): + pad = self.pad + width = self.width - 3 * self.pad + x = y = pad + + self.draw_label( + "Problem report for:", + x=x, + y=y, + bold=True, + width=width, + font_size=self.title_font_size, + ) + y += self.title_font_size + + import hashlib + + for file in files: + base_file = os.path.basename(file) + y += self.font_size + self.pad + self.draw_label(base_file, x=x, y=y, bold=True, width=width) + y += self.font_size + self.pad + + try: + with open(file, "rb") as f: + h = hashlib.sha1(f.read()).hexdigest() + self.draw_label("sha1: %s" % h, x=x + pad, y=y, width=width) + y += self.font_size + except IsADirectoryError: + pass + + if file.endswith(".ttf"): + ttFont = TTFont(file) + name = ttFont["name"] if "name" in ttFont else None + if name: + for what, nameIDs in ( + ("Family name", (21, 16, 1)), + ("Version", (5,)), + ): + n = name.getFirstDebugName(nameIDs) + if n is None: + continue + self.draw_label( + "%s: %s" % (what, n), x=x + pad, y=y, width=width + ) + y += self.font_size + self.pad + elif file.endswith((".glyphs", ".glyphspackage")): + from glyphsLib import GSFont + + f = GSFont(file) + for what, field in ( + ("Family name", "familyName"), + ("VersionMajor", "versionMajor"), + ("VersionMinor", "_versionMinor"), + ): + self.draw_label( + "%s: %s" % (what, getattr(f, field)), + x=x + pad, + y=y, + width=width, + ) + y += self.font_size + self.pad + + self.draw_legend( + show_tolerance=show_tolerance, tolerance=tolerance, kinkiness=kinkiness + ) + self.show_page() + + def draw_legend(self, *, show_tolerance=True, tolerance=None, kinkiness=None): + cr = cairo.Context(self.surface) + + x = self.pad + y = self.height - self.pad - self.font_size * 2 + width = self.width - 2 * self.pad + + xx = x + self.pad * 2 + xxx = x + self.pad * 4 + + if show_tolerance: + self.draw_label( + "Tolerance: badness; closer to zero the worse", x=xxx, y=y, width=width + ) + y -= self.pad + self.font_size + + self.draw_label("Underweight contours", x=xxx, y=y, width=width) + cr.rectangle(xx - self.pad * 0.7, y, 1.5 * self.pad, self.font_size) + cr.set_source_rgb(*self.fill_color) + cr.fill_preserve() + if self.stroke_color: + cr.set_source_rgb(*self.stroke_color) + cr.set_line_width(self.stroke_width) + cr.stroke_preserve() + cr.set_source_rgba(*self.weight_issue_contour_color) + cr.fill() + y -= self.pad + self.font_size + + self.draw_label( + "Colored contours: contours with the wrong order", x=xxx, y=y, width=width + ) + cr.rectangle(xx - self.pad * 0.7, y, 1.5 * self.pad, self.font_size) + if self.fill_color: + cr.set_source_rgb(*self.fill_color) + cr.fill_preserve() + if self.stroke_color: + cr.set_source_rgb(*self.stroke_color) + cr.set_line_width(self.stroke_width) + cr.stroke_preserve() + cr.set_source_rgba(*self.contour_colors[0], self.contour_alpha) + cr.fill() + y -= self.pad + self.font_size + + self.draw_label("Kink artifact", x=xxx, y=y, width=width) + self.draw_circle( + cr, + x=xx, + y=y + self.font_size * 0.5, + diameter=self.kink_circle_size, + stroke_width=self.kink_circle_stroke_width, + color=self.kink_circle_color, + ) + y -= self.pad + self.font_size + + self.draw_label("Point causing kink in the contour", x=xxx, y=y, width=width) + self.draw_dot( + cr, + x=xx, + y=y + self.font_size * 0.5, + diameter=self.kink_point_size, + color=self.kink_point_color, + ) + y -= self.pad + self.font_size + + self.draw_label("Suggested new contour start point", x=xxx, y=y, width=width) + self.draw_dot( + cr, + x=xx, + y=y + self.font_size * 0.5, + diameter=self.corrected_start_point_size, + color=self.corrected_start_point_color, + ) + y -= self.pad + self.font_size + + self.draw_label( + "Contour start point in contours with wrong direction", + x=xxx, + y=y, + width=width, + ) + self.draw_arrow( + cr, + x=xx - self.start_arrow_length * 0.3, + y=y + self.font_size * 0.5, + color=self.wrong_start_point_color, + ) + y -= self.pad + self.font_size + + self.draw_label( + "Contour start point when the first two points overlap", + x=xxx, + y=y, + width=width, + ) + self.draw_dot( + cr, + x=xx, + y=y + self.font_size * 0.5, + diameter=self.corrected_start_point_size, + color=self.start_point_color, + ) + y -= self.pad + self.font_size + + self.draw_label("Contour start point and direction", x=xxx, y=y, width=width) + self.draw_arrow( + cr, + x=xx - self.start_arrow_length * 0.3, + y=y + self.font_size * 0.5, + color=self.start_point_color, + ) + y -= self.pad + self.font_size + + self.draw_label("Legend:", x=x, y=y, width=width, bold=True) + y -= self.pad + self.font_size + + if kinkiness is not None: + self.draw_label( + "Kink-reporting aggressiveness: %g" % kinkiness, + x=xxx, + y=y, + width=width, + ) + y -= self.pad + self.font_size + + if tolerance is not None: + self.draw_label( + "Error tolerance: %g" % tolerance, + x=xxx, + y=y, + width=width, + ) + y -= self.pad + self.font_size + + self.draw_label("Parameters:", x=x, y=y, width=width, bold=True) + y -= self.pad + self.font_size + + def add_summary(self, problems): + pad = self.pad + width = self.width - 3 * self.pad + height = self.height - 2 * self.pad + x = y = pad + + self.draw_label( + "Summary of problems", + x=x, + y=y, + bold=True, + width=width, + font_size=self.title_font_size, + ) + y += self.title_font_size + + glyphs_per_problem = defaultdict(set) + for glyphname, glyph_problems in sorted(problems.items()): + for problem in glyph_problems: + glyphs_per_problem[problem["type"]].add(glyphname) + + if InterpolatableProblem.NOTHING in glyphs_per_problem: + del glyphs_per_problem[InterpolatableProblem.NOTHING] + + for problem_type in sorted( + glyphs_per_problem, key=lambda x: InterpolatableProblem.severity[x] + ): + y += self.font_size + self.draw_label( + "%s: %d" % (problem_type, len(glyphs_per_problem[problem_type])), + x=x, + y=y, + width=width, + bold=True, + ) + y += self.font_size + + for glyphname in sorted(glyphs_per_problem[problem_type]): + if y + self.font_size > height: + self.show_page() + y = self.font_size + pad + self.draw_label(glyphname, x=x + 2 * pad, y=y, width=width - 2 * pad) + y += self.font_size + + self.show_page() + + def _add_listing(self, title, items): + pad = self.pad + width = self.width - 2 * self.pad + height = self.height - 2 * self.pad + x = y = pad + + self.draw_label( + title, x=x, y=y, bold=True, width=width, font_size=self.title_font_size + ) + y += self.title_font_size + self.pad + + last_glyphname = None + for page_no, (glyphname, problems) in items: + if glyphname == last_glyphname: + continue + last_glyphname = glyphname + if y + self.font_size > height: + self.show_page() + y = self.font_size + pad + self.draw_label(glyphname, x=x + 5 * pad, y=y, width=width - 2 * pad) + self.draw_label(str(page_no), x=x, y=y, width=4 * pad, align=1) + y += self.font_size + + self.show_page() + + def add_table_of_contents(self): + self._add_listing("Table of contents", sorted(self.toc.items())) + + def add_index(self): + self._add_listing("Index", sorted(self.toc.items(), key=lambda x: x[1][0])) + + def add_problems(self, problems, *, show_tolerance=True, show_page_number=True): + for glyph, glyph_problems in problems.items(): + last_masters = None + current_glyph_problems = [] + for p in glyph_problems: + masters = ( + p["master_idx"] + if "master_idx" in p + else (p["master_1_idx"], p["master_2_idx"]) + ) + if masters == last_masters: + current_glyph_problems.append(p) + continue + # Flush + if current_glyph_problems: + self.add_problem( + glyph, + current_glyph_problems, + show_tolerance=show_tolerance, + show_page_number=show_page_number, + ) + self.show_page() + current_glyph_problems = [] + last_masters = masters + current_glyph_problems.append(p) + if current_glyph_problems: + self.add_problem( + glyph, + current_glyph_problems, + show_tolerance=show_tolerance, + show_page_number=show_page_number, + ) + self.show_page() + + def add_problem( + self, glyphname, problems, *, show_tolerance=True, show_page_number=True + ): + if type(problems) not in (list, tuple): + problems = [problems] + + self.toc[self.page_number] = (glyphname, problems) + + problem_type = problems[0]["type"] + problem_types = set(problem["type"] for problem in problems) + if not all(pt == problem_type for pt in problem_types): + problem_type = ", ".join(sorted({problem["type"] for problem in problems})) + + log.info("Drawing %s: %s", glyphname, problem_type) + + master_keys = ( + ("master_idx",) + if "master_idx" in problems[0] + else ("master_1_idx", "master_2_idx") + ) + master_indices = [problems[0][k] for k in master_keys] + + if problem_type == InterpolatableProblem.MISSING: + sample_glyph = next( + i for i, m in enumerate(self.glyphsets) if m[glyphname] is not None + ) + master_indices.insert(0, sample_glyph) + + x = self.pad + y = self.pad + + self.draw_label( + "Glyph name: " + glyphname, + x=x, + y=y, + color=self.head_color, + align=0, + bold=True, + font_size=self.title_font_size, + ) + tolerance = min(p.get("tolerance", 1) for p in problems) + if tolerance < 1 and show_tolerance: + self.draw_label( + "tolerance: %.2f" % tolerance, + x=x, + y=y, + width=self.width - 2 * self.pad, + align=1, + bold=True, + ) + y += self.title_font_size + self.pad + self.draw_label( + "Problems: " + problem_type, + x=x, + y=y, + width=self.width - 2 * self.pad, + color=self.head_color, + bold=True, + ) + y += self.font_size + self.pad * 2 + + scales = [] + for which, master_idx in enumerate(master_indices): + glyphset = self.glyphsets[master_idx] + name = self.names[master_idx] + + self.draw_label( + name, + x=x, + y=y, + color=self.label_color, + width=self.panel_width, + align=0.5, + ) + y += self.font_size + self.pad + + if glyphset[glyphname] is not None: + scales.append( + self.draw_glyph(glyphset, glyphname, problems, which, x=x, y=y) + ) + else: + self.draw_emoticon(self.shrug, x=x, y=y) + y += self.panel_height + self.font_size + self.pad + + if any( + pt + in ( + InterpolatableProblem.NOTHING, + InterpolatableProblem.WRONG_START_POINT, + InterpolatableProblem.CONTOUR_ORDER, + InterpolatableProblem.KINK, + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, + ) + for pt in problem_types + ): + x = self.pad + self.panel_width + self.pad + y = self.pad + y += self.title_font_size + self.pad * 2 + y += self.font_size + self.pad + + glyphset1 = self.glyphsets[master_indices[0]] + glyphset2 = self.glyphsets[master_indices[1]] + + # Draw the mid-way of the two masters + + self.draw_label( + "midway interpolation", + x=x, + y=y, + color=self.head_color, + width=self.panel_width, + align=0.5, + ) + y += self.font_size + self.pad + + midway_glyphset = LerpGlyphSet(glyphset1, glyphset2) + self.draw_glyph( + midway_glyphset, + glyphname, + [{"type": "midway"}] + + [ + p + for p in problems + if p["type"] + in ( + InterpolatableProblem.KINK, + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, + ) + ], + None, + x=x, + y=y, + scale=min(scales), + ) + + y += self.panel_height + self.font_size + self.pad + + if any( + pt + in ( + InterpolatableProblem.WRONG_START_POINT, + InterpolatableProblem.CONTOUR_ORDER, + InterpolatableProblem.KINK, + ) + for pt in problem_types + ): + # Draw the proposed fix + + self.draw_label( + "proposed fix", + x=x, + y=y, + color=self.head_color, + width=self.panel_width, + align=0.5, + ) + y += self.font_size + self.pad + + overriding1 = OverridingDict(glyphset1) + overriding2 = OverridingDict(glyphset2) + perContourPen1 = PerContourOrComponentPen( + RecordingPen, glyphset=overriding1 + ) + perContourPen2 = PerContourOrComponentPen( + RecordingPen, glyphset=overriding2 + ) + glyphset1[glyphname].draw(perContourPen1) + glyphset2[glyphname].draw(perContourPen2) + + for problem in problems: + if problem["type"] == InterpolatableProblem.CONTOUR_ORDER: + fixed_contours = [ + perContourPen2.value[i] for i in problems[0]["value_2"] + ] + perContourPen2.value = fixed_contours + + for problem in problems: + if problem["type"] == InterpolatableProblem.WRONG_START_POINT: + # Save the wrong contours + wrongContour1 = perContourPen1.value[problem["contour"]] + wrongContour2 = perContourPen2.value[problem["contour"]] + + # Convert the wrong contours to point pens + points1 = RecordingPointPen() + converter = SegmentToPointPen(points1, False) + wrongContour1.replay(converter) + points2 = RecordingPointPen() + converter = SegmentToPointPen(points2, False) + wrongContour2.replay(converter) + + proposed_start = problem["value_2"] + + # See if we need reversing; fragile but worth a try + if problem["reversed"]: + new_points2 = RecordingPointPen() + reversedPen = ReverseContourPointPen(new_points2) + points2.replay(reversedPen) + points2 = new_points2 + proposed_start = len(points2.value) - 2 - proposed_start + + # Rotate points2 so that the first point is the same as in points1 + beginPath = points2.value[:1] + endPath = points2.value[-1:] + pts = points2.value[1:-1] + pts = pts[proposed_start:] + pts[:proposed_start] + points2.value = beginPath + pts + endPath + + # Convert the point pens back to segment pens + segment1 = RecordingPen() + converter = PointToSegmentPen(segment1, True) + points1.replay(converter) + segment2 = RecordingPen() + converter = PointToSegmentPen(segment2, True) + points2.replay(converter) + + # Replace the wrong contours + wrongContour1.value = segment1.value + wrongContour2.value = segment2.value + perContourPen1.value[problem["contour"]] = wrongContour1 + perContourPen2.value[problem["contour"]] = wrongContour2 + + for problem in problems: + # If we have a kink, try to fix it. + if problem["type"] == InterpolatableProblem.KINK: + # Save the wrong contours + wrongContour1 = perContourPen1.value[problem["contour"]] + wrongContour2 = perContourPen2.value[problem["contour"]] + + # Convert the wrong contours to point pens + points1 = RecordingPointPen() + converter = SegmentToPointPen(points1, False) + wrongContour1.replay(converter) + points2 = RecordingPointPen() + converter = SegmentToPointPen(points2, False) + wrongContour2.replay(converter) + + i = problem["value"] + + # Position points to be around the same ratio + # beginPath / endPath dance + j = i + 1 + pt0 = points1.value[j][1][0] + pt1 = points2.value[j][1][0] + j_prev = (i - 1) % (len(points1.value) - 2) + 1 + pt0_prev = points1.value[j_prev][1][0] + pt1_prev = points2.value[j_prev][1][0] + j_next = (i + 1) % (len(points1.value) - 2) + 1 + pt0_next = points1.value[j_next][1][0] + pt1_next = points2.value[j_next][1][0] + + pt0 = complex(*pt0) + pt1 = complex(*pt1) + pt0_prev = complex(*pt0_prev) + pt1_prev = complex(*pt1_prev) + pt0_next = complex(*pt0_next) + pt1_next = complex(*pt1_next) + + # Find the ratio of the distance between the points + r0 = abs(pt0 - pt0_prev) / abs(pt0_next - pt0_prev) + r1 = abs(pt1 - pt1_prev) / abs(pt1_next - pt1_prev) + r_mid = (r0 + r1) / 2 + + pt0 = pt0_prev + r_mid * (pt0_next - pt0_prev) + pt1 = pt1_prev + r_mid * (pt1_next - pt1_prev) + + points1.value[j] = ( + points1.value[j][0], + (((pt0.real, pt0.imag),) + points1.value[j][1][1:]), + points1.value[j][2], + ) + points2.value[j] = ( + points2.value[j][0], + (((pt1.real, pt1.imag),) + points2.value[j][1][1:]), + points2.value[j][2], + ) + + # Convert the point pens back to segment pens + segment1 = RecordingPen() + converter = PointToSegmentPen(segment1, True) + points1.replay(converter) + segment2 = RecordingPen() + converter = PointToSegmentPen(segment2, True) + points2.replay(converter) + + # Replace the wrong contours + wrongContour1.value = segment1.value + wrongContour2.value = segment2.value + + # Assemble + fixed1 = RecordingPen() + fixed2 = RecordingPen() + for contour in perContourPen1.value: + fixed1.value.extend(contour.value) + for contour in perContourPen2.value: + fixed2.value.extend(contour.value) + fixed1.draw = fixed1.replay + fixed2.draw = fixed2.replay + + overriding1[glyphname] = fixed1 + overriding2[glyphname] = fixed2 + + try: + midway_glyphset = LerpGlyphSet(overriding1, overriding2) + self.draw_glyph( + midway_glyphset, + glyphname, + {"type": "fixed"}, + None, + x=x, + y=y, + scale=min(scales), + ) + except ValueError: + self.draw_emoticon(self.shrug, x=x, y=y) + y += self.panel_height + self.pad + + else: + emoticon = self.shrug + if InterpolatableProblem.UNDERWEIGHT in problem_types: + emoticon = self.underweight + elif InterpolatableProblem.OVERWEIGHT in problem_types: + emoticon = self.overweight + elif InterpolatableProblem.NOTHING in problem_types: + emoticon = self.yay + self.draw_emoticon(emoticon, x=x, y=y) + + if show_page_number: + self.draw_label( + str(self.page_number), + x=0, + y=self.height - self.font_size - self.pad, + width=self.width, + color=self.head_color, + align=0.5, + ) + + def draw_label( + self, + label, + *, + x=0, + y=0, + color=(0, 0, 0), + align=0, + bold=False, + width=None, + height=None, + font_size=None, + ): + if width is None: + width = self.width + if height is None: + height = self.height + if font_size is None: + font_size = self.font_size + cr = cairo.Context(self.surface) + cr.select_font_face( + "@cairo:", + cairo.FONT_SLANT_NORMAL, + cairo.FONT_WEIGHT_BOLD if bold else cairo.FONT_WEIGHT_NORMAL, + ) + cr.set_font_size(font_size) + font_extents = cr.font_extents() + font_size = font_size * font_size / font_extents[2] + cr.set_font_size(font_size) + font_extents = cr.font_extents() + + cr.set_source_rgb(*color) + + extents = cr.text_extents(label) + if extents.width > width: + # Shrink + font_size *= width / extents.width + cr.set_font_size(font_size) + font_extents = cr.font_extents() + extents = cr.text_extents(label) + + # Center + label_x = x + (width - extents.width) * align + label_y = y + font_extents[0] + cr.move_to(label_x, label_y) + cr.show_text(label) + + def draw_glyph(self, glyphset, glyphname, problems, which, *, x=0, y=0, scale=None): + if type(problems) not in (list, tuple): + problems = [problems] + + midway = any(problem["type"] == "midway" for problem in problems) + problem_type = problems[0]["type"] + problem_types = set(problem["type"] for problem in problems) + if not all(pt == problem_type for pt in problem_types): + problem_type = "mixed" + glyph = glyphset[glyphname] + + recording = RecordingPen() + glyph.draw(recording) + decomposedRecording = DecomposingRecordingPen(glyphset) + glyph.draw(decomposedRecording) + + boundsPen = ControlBoundsPen(glyphset) + decomposedRecording.replay(boundsPen) + bounds = boundsPen.bounds + if bounds is None: + bounds = (0, 0, 0, 0) + + glyph_width = bounds[2] - bounds[0] + glyph_height = bounds[3] - bounds[1] + + if glyph_width: + if scale is None: + scale = self.panel_width / glyph_width + else: + scale = min(scale, self.panel_width / glyph_width) + if glyph_height: + if scale is None: + scale = self.panel_height / glyph_height + else: + scale = min(scale, self.panel_height / glyph_height) + if scale is None: + scale = 1 + + cr = cairo.Context(self.surface) + cr.translate(x, y) + # Center + cr.translate( + (self.panel_width - glyph_width * scale) / 2, + (self.panel_height - glyph_height * scale) / 2, + ) + cr.scale(scale, -scale) + cr.translate(-bounds[0], -bounds[3]) + + if self.border_color: + cr.set_source_rgb(*self.border_color) + cr.rectangle(bounds[0], bounds[1], glyph_width, glyph_height) + cr.set_line_width(self.border_width / scale) + cr.stroke() + + if self.fill_color or self.stroke_color: + pen = CairoPen(glyphset, cr) + decomposedRecording.replay(pen) + + if self.fill_color and problem_type != InterpolatableProblem.OPEN_PATH: + cr.set_source_rgb(*self.fill_color) + cr.fill_preserve() + + if self.stroke_color: + cr.set_source_rgb(*self.stroke_color) + cr.set_line_width(self.stroke_width / scale) + cr.stroke_preserve() + + cr.new_path() + + if ( + InterpolatableProblem.UNDERWEIGHT in problem_types + or InterpolatableProblem.OVERWEIGHT in problem_types + ): + perContourPen = PerContourOrComponentPen(RecordingPen, glyphset=glyphset) + recording.replay(perContourPen) + for problem in problems: + if problem["type"] in ( + InterpolatableProblem.UNDERWEIGHT, + InterpolatableProblem.OVERWEIGHT, + ): + contour = perContourPen.value[problem["contour"]] + contour.replay(CairoPen(glyphset, cr)) + cr.set_source_rgba(*self.weight_issue_contour_color) + cr.fill() + + if any( + t in problem_types + for t in { + InterpolatableProblem.NOTHING, + InterpolatableProblem.NODE_COUNT, + InterpolatableProblem.NODE_INCOMPATIBILITY, + } + ): + cr.set_line_cap(cairo.LINE_CAP_ROUND) + + # Oncurve nodes + for segment, args in decomposedRecording.value: + if not args: + continue + x, y = args[-1] + cr.move_to(x, y) + cr.line_to(x, y) + cr.set_source_rgba(*self.oncurve_node_color) + cr.set_line_width(self.oncurve_node_diameter / scale) + cr.stroke() + + # Offcurve nodes + for segment, args in decomposedRecording.value: + if not args: + continue + for x, y in args[:-1]: + cr.move_to(x, y) + cr.line_to(x, y) + cr.set_source_rgba(*self.offcurve_node_color) + cr.set_line_width(self.offcurve_node_diameter / scale) + cr.stroke() + + # Handles + for segment, args in decomposedRecording.value: + if not args: + pass + elif segment in ("moveTo", "lineTo"): + cr.move_to(*args[0]) + elif segment == "qCurveTo": + for x, y in args: + cr.line_to(x, y) + cr.new_sub_path() + cr.move_to(*args[-1]) + elif segment == "curveTo": + cr.line_to(*args[0]) + cr.new_sub_path() + cr.move_to(*args[1]) + cr.line_to(*args[2]) + cr.new_sub_path() + cr.move_to(*args[-1]) + else: + continue + + cr.set_source_rgba(*self.handle_color) + cr.set_line_width(self.handle_width / scale) + cr.stroke() + + matching = None + for problem in problems: + if problem["type"] == InterpolatableProblem.CONTOUR_ORDER: + matching = problem["value_2"] + colors = cycle(self.contour_colors) + perContourPen = PerContourOrComponentPen( + RecordingPen, glyphset=glyphset + ) + recording.replay(perContourPen) + for i, contour in enumerate(perContourPen.value): + if matching[i] == i: + continue + color = next(colors) + contour.replay(CairoPen(glyphset, cr)) + cr.set_source_rgba(*color, self.contour_alpha) + cr.fill() + + for problem in problems: + if problem["type"] in ( + InterpolatableProblem.NOTHING, + InterpolatableProblem.WRONG_START_POINT, + ): + idx = problem.get("contour") + + # Draw suggested point + if idx is not None and which == 1 and "value_2" in problem: + perContourPen = PerContourOrComponentPen( + RecordingPen, glyphset=glyphset + ) + decomposedRecording.replay(perContourPen) + points = SimpleRecordingPointPen() + converter = SegmentToPointPen(points, False) + perContourPen.value[ + idx if matching is None else matching[idx] + ].replay(converter) + targetPoint = points.value[problem["value_2"]][0] + cr.save() + cr.translate(*targetPoint) + cr.scale(1 / scale, 1 / scale) + self.draw_dot( + cr, + diameter=self.corrected_start_point_size, + color=self.corrected_start_point_color, + ) + cr.restore() + + # Draw start-point arrow + if which == 0 or not problem.get("reversed"): + color = self.start_point_color + else: + color = self.wrong_start_point_color + first_pt = None + i = 0 + cr.save() + for segment, args in decomposedRecording.value: + if segment == "moveTo": + first_pt = args[0] + continue + if first_pt is None: + continue + if segment == "closePath": + second_pt = first_pt + else: + second_pt = args[0] + + if idx is None or i == idx: + cr.save() + first_pt = complex(*first_pt) + second_pt = complex(*second_pt) + length = abs(second_pt - first_pt) + cr.translate(first_pt.real, first_pt.imag) + if length: + # Draw arrowhead + cr.rotate( + math.atan2( + second_pt.imag - first_pt.imag, + second_pt.real - first_pt.real, + ) + ) + cr.scale(1 / scale, 1 / scale) + self.draw_arrow(cr, color=color) + else: + # Draw circle + cr.scale(1 / scale, 1 / scale) + self.draw_dot( + cr, + diameter=self.corrected_start_point_size, + color=color, + ) + cr.restore() + + if idx is not None: + break + + first_pt = None + i += 1 + + cr.restore() + + if problem["type"] == InterpolatableProblem.KINK: + idx = problem.get("contour") + perContourPen = PerContourOrComponentPen( + RecordingPen, glyphset=glyphset + ) + decomposedRecording.replay(perContourPen) + points = SimpleRecordingPointPen() + converter = SegmentToPointPen(points, False) + perContourPen.value[idx if matching is None else matching[idx]].replay( + converter + ) + + targetPoint = points.value[problem["value"]][0] + cr.save() + cr.translate(*targetPoint) + cr.scale(1 / scale, 1 / scale) + if midway: + self.draw_circle( + cr, + diameter=self.kink_circle_size, + stroke_width=self.kink_circle_stroke_width, + color=self.kink_circle_color, + ) + else: + self.draw_dot( + cr, + diameter=self.kink_point_size, + color=self.kink_point_color, + ) + cr.restore() + + return scale + + def draw_dot(self, cr, *, x=0, y=0, color=(0, 0, 0), diameter=10): + cr.save() + cr.set_line_width(diameter) + cr.set_line_cap(cairo.LINE_CAP_ROUND) + cr.move_to(x, y) + cr.line_to(x, y) + if len(color) == 3: + color = color + (1,) + cr.set_source_rgba(*color) + cr.stroke() + cr.restore() + + def draw_circle( + self, cr, *, x=0, y=0, color=(0, 0, 0), diameter=10, stroke_width=1 + ): + cr.save() + cr.set_line_width(stroke_width) + cr.set_line_cap(cairo.LINE_CAP_SQUARE) + cr.arc(x, y, diameter / 2, 0, 2 * math.pi) + if len(color) == 3: + color = color + (1,) + cr.set_source_rgba(*color) + cr.stroke() + cr.restore() + + def draw_arrow(self, cr, *, x=0, y=0, color=(0, 0, 0)): + cr.save() + if len(color) == 3: + color = color + (1,) + cr.set_source_rgba(*color) + cr.translate(self.start_arrow_length + x, y) + cr.move_to(0, 0) + cr.line_to( + -self.start_arrow_length, + -self.start_arrow_length * 0.4, + ) + cr.line_to( + -self.start_arrow_length, + self.start_arrow_length * 0.4, + ) + cr.close_path() + cr.fill() + cr.restore() + + def draw_text(self, text, *, x=0, y=0, color=(0, 0, 0), width=None, height=None): + if width is None: + width = self.width + if height is None: + height = self.height + + text = text.splitlines() + cr = cairo.Context(self.surface) + cr.set_source_rgb(*color) + cr.set_font_size(self.font_size) + cr.select_font_face( + "@cairo:monospace", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL + ) + text_width = 0 + text_height = 0 + font_extents = cr.font_extents() + font_font_size = font_extents[2] + font_ascent = font_extents[0] + for line in text: + extents = cr.text_extents(line) + text_width = max(text_width, extents.x_advance) + text_height += font_font_size + if not text_width: + return + cr.translate(x, y) + scale = min(width / text_width, height / text_height) + # center + cr.translate( + (width - text_width * scale) / 2, (height - text_height * scale) / 2 + ) + cr.scale(scale, scale) + + cr.translate(0, font_ascent) + for line in text: + cr.move_to(0, 0) + cr.show_text(line) + cr.translate(0, font_font_size) + + def draw_cupcake(self): + self.draw_label( + self.no_issues_label, + x=self.pad, + y=self.pad, + color=self.no_issues_label_color, + width=self.width - 2 * self.pad, + align=0.5, + bold=True, + font_size=self.title_font_size, + ) + + self.draw_text( + self.cupcake, + x=self.pad, + y=self.pad + self.font_size, + width=self.width - 2 * self.pad, + height=self.height - 2 * self.pad - self.font_size, + color=self.cupcake_color, + ) + + def draw_emoticon(self, emoticon, x=0, y=0): + self.draw_text( + emoticon, + x=x, + y=y, + color=self.emoticon_color, + width=self.panel_width, + height=self.panel_height, + ) + + +class InterpolatablePostscriptLike(InterpolatablePlot): + def __exit__(self, type, value, traceback): + self.surface.finish() + + def show_page(self): + super().show_page() + self.surface.show_page() + + +class InterpolatablePS(InterpolatablePostscriptLike): + def __enter__(self): + self.surface = cairo.PSSurface(self.out, self.width, self.height) + return self + + +class InterpolatablePDF(InterpolatablePostscriptLike): + def __enter__(self): + self.surface = cairo.PDFSurface(self.out, self.width, self.height) + self.surface.set_metadata( + cairo.PDF_METADATA_CREATOR, "fonttools varLib.interpolatable" + ) + self.surface.set_metadata(cairo.PDF_METADATA_CREATE_DATE, "") + return self + + +class InterpolatableSVG(InterpolatablePlot): + def __enter__(self): + self.sink = BytesIO() + self.surface = cairo.SVGSurface(self.sink, self.width, self.height) + return self + + def __exit__(self, type, value, traceback): + if self.surface is not None: + self.show_page() + + def show_page(self): + super().show_page() + self.surface.finish() + self.out.append(self.sink.getvalue()) + self.sink = BytesIO() + self.surface = cairo.SVGSurface(self.sink, self.width, self.height) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatableTestContourOrder.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatableTestContourOrder.py new file mode 100644 index 0000000000000000000000000000000000000000..f593e2bd4dbcd08dd6f8a9aad88d541a33c8009f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatableTestContourOrder.py @@ -0,0 +1,81 @@ +from .interpolatableHelpers import * +import logging + +log = logging.getLogger("fontTools.varLib.interpolatable") + + +def test_contour_order(glyph0, glyph1): + # We try matching both the StatisticsControlPen vector + # and the StatisticsPen vector. + # + # If either method found an identity matching, accept it. + # This is crucial for fonts like Kablammo[MORF].ttf and + # Nabla[EDPT,EHLT].ttf, since they really confuse the + # StatisticsPen vector because of their area=0 contours. + + n = len(glyph0.controlVectors) + matching = None + matching_cost = 0 + identity_cost = 0 + done = n <= 1 + if not done: + m0Control = glyph0.controlVectors + m1Control = glyph1.controlVectors + ( + matching_control, + matching_cost_control, + identity_cost_control, + ) = matching_for_vectors(m0Control, m1Control) + done = matching_cost_control == identity_cost_control + if not done: + m0Green = glyph0.greenVectors + m1Green = glyph1.greenVectors + ( + matching_green, + matching_cost_green, + identity_cost_green, + ) = matching_for_vectors(m0Green, m1Green) + done = matching_cost_green == identity_cost_green + + if not done: + # See if reversing contours in one master helps. + # That's a common problem. Then the wrong_start_point + # test will fix them. + # + # Reverse the sign of the area (0); the rest stay the same. + m1ControlReversed = [(-m[0],) + m[1:] for m in m1Control] + ( + matching_control_reversed, + matching_cost_control_reversed, + identity_cost_control_reversed, + ) = matching_for_vectors(m0Control, m1ControlReversed) + done = matching_cost_control_reversed == identity_cost_control_reversed + if not done: + m1GreenReversed = [(-m[0],) + m[1:] for m in m1Green] + ( + matching_green_reversed, + matching_cost_green_reversed, + identity_cost_green_reversed, + ) = matching_for_vectors(m0Green, m1GreenReversed) + done = matching_cost_green_reversed == identity_cost_green_reversed + + if not done: + # Otherwise, use the best of the two matchings. + if ( + matching_cost_control * identity_cost_green + < matching_cost_green * identity_cost_control + ): + matching = matching_control + matching_cost = matching_cost_control + identity_cost = identity_cost_control + else: + matching = matching_green + matching_cost = matching_cost_green + identity_cost = identity_cost_green + + this_tolerance = matching_cost / identity_cost if identity_cost else 1 + log.debug( + "test-contour-order: tolerance %g", + this_tolerance, + ) + return this_tolerance, matching diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatableTestStartingPoint.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatableTestStartingPoint.py new file mode 100644 index 0000000000000000000000000000000000000000..6c619ebe689ba3982a3eb7ea1b14d670c8f5536c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolatableTestStartingPoint.py @@ -0,0 +1,109 @@ +from .interpolatableHelpers import * + + +def test_starting_point(glyph0, glyph1, ix, tolerance, matching): + if matching is None: + matching = list(range(len(glyph0.isomorphisms))) + contour0 = glyph0.isomorphisms[ix] + contour1 = glyph1.isomorphisms[matching[ix]] + m0Vectors = glyph0.greenVectors + m1Vectors = [glyph1.greenVectors[i] for i in matching] + + c0 = contour0[0] + # Next few lines duplicated below. + costs = [vdiff_hypot2_complex(c0[0], c1[0]) for c1 in contour1] + min_cost_idx, min_cost = min(enumerate(costs), key=lambda x: x[1]) + first_cost = costs[0] + proposed_point = contour1[min_cost_idx][1] + reverse = contour1[min_cost_idx][2] + + if min_cost < first_cost * tolerance: + # c0 is the first isomorphism of the m0 master + # contour1 is list of all isomorphisms of the m1 master + # + # If the two shapes are both circle-ish and slightly + # rotated, we detect wrong start point. This is for + # example the case hundreds of times in + # RobotoSerif-Italic[GRAD,opsz,wdth,wght].ttf + # + # If the proposed point is only one off from the first + # point (and not reversed), try harder: + # + # Find the major eigenvector of the covariance matrix, + # and rotate the contours by that angle. Then find the + # closest point again. If it matches this time, let it + # pass. + + num_points = len(glyph1.points[ix]) + leeway = 3 + if not reverse and ( + proposed_point <= leeway or proposed_point >= num_points - leeway + ): + # Try harder + + # Recover the covariance matrix from the GreenVectors. + # This is a 2x2 matrix. + transforms = [] + for vector in (m0Vectors[ix], m1Vectors[ix]): + # meanX = vector[1] + # meanY = vector[2] + stddevX = vector[3] * 0.5 + stddevY = vector[4] * 0.5 + correlation = vector[5] + if correlation: + correlation /= abs(vector[0]) + + # https://cookierobotics.com/007/ + a = stddevX * stddevX # VarianceX + c = stddevY * stddevY # VarianceY + b = correlation * stddevX * stddevY # Covariance + + delta = (((a - c) * 0.5) ** 2 + b * b) ** 0.5 + lambda1 = (a + c) * 0.5 + delta # Major eigenvalue + lambda2 = (a + c) * 0.5 - delta # Minor eigenvalue + theta = atan2(lambda1 - a, b) if b != 0 else (pi * 0.5 if a < c else 0) + trans = Transform() + # Don't translate here. We are working on the complex-vector + # that includes more than just the points. It's horrible what + # we are doing anyway... + # trans = trans.translate(meanX, meanY) + trans = trans.rotate(theta) + if lambda2 < 0: + lambda2 = 0 + trans = trans.scale(sqrt(lambda1), sqrt(lambda2)) + transforms.append(trans) + + trans = transforms[0] + new_c0 = ( + [complex(*trans.transformPoint((pt.real, pt.imag))) for pt in c0[0]], + ) + c0[1:] + trans = transforms[1] + new_contour1 = [] + for c1 in contour1: + new_c1 = ( + [ + complex(*trans.transformPoint((pt.real, pt.imag))) + for pt in c1[0] + ], + ) + c1[1:] + new_contour1.append(new_c1) + + # Next few lines duplicate from above. + costs = [ + vdiff_hypot2_complex(new_c0[0], new_c1[0]) for new_c1 in new_contour1 + ] + min_cost_idx, min_cost = min(enumerate(costs), key=lambda x: x[1]) + first_cost = costs[0] + if min_cost < first_cost * tolerance: + # Don't report this + # min_cost = first_cost + # reverse = False + # proposed_point = 0 # new_contour1[min_cost_idx][1] + pass + + this_tolerance = min_cost / first_cost if first_cost else 1 + log.debug( + "test-starting-point: tolerance %g", + this_tolerance, + ) + return this_tolerance, proposed_point, reverse diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolate_layout.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolate_layout.py new file mode 100644 index 0000000000000000000000000000000000000000..798b29590d31ded570e4974e227e0973d1546e72 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/interpolate_layout.py @@ -0,0 +1,124 @@ +""" +Interpolate OpenType Layout tables (GDEF / GPOS / GSUB). +""" + +from fontTools.ttLib import TTFont +from fontTools.varLib import models, VarLibError, load_designspace, load_masters +from fontTools.varLib.merger import InstancerMerger +import os.path +import logging +from copy import deepcopy +from pprint import pformat + +log = logging.getLogger("fontTools.varLib.interpolate_layout") + + +def interpolate_layout(designspace, loc, master_finder=lambda s: s, mapped=False): + """ + Interpolate GPOS from a designspace file and location. + + If master_finder is set, it should be a callable that takes master + filename as found in designspace file and map it to master font + binary as to be opened (eg. .ttf or .otf). + + If mapped is False (default), then location is mapped using the + map element of the axes in designspace file. If mapped is True, + it is assumed that location is in designspace's internal space and + no mapping is performed. + """ + if hasattr(designspace, "sources"): # Assume a DesignspaceDocument + pass + else: # Assume a file path + from fontTools.designspaceLib import DesignSpaceDocument + + designspace = DesignSpaceDocument.fromfile(designspace) + + ds = load_designspace(designspace) + log.info("Building interpolated font") + + log.info("Loading master fonts") + master_fonts = load_masters(designspace, master_finder) + font = deepcopy(master_fonts[ds.base_idx]) + + log.info("Location: %s", pformat(loc)) + if not mapped: + loc = {name: ds.axes[name].map_forward(v) for name, v in loc.items()} + log.info("Internal location: %s", pformat(loc)) + loc = models.normalizeLocation(loc, ds.internal_axis_supports) + log.info("Normalized location: %s", pformat(loc)) + + # Assume single-model for now. + model = models.VariationModel(ds.normalized_master_locs) + assert 0 == model.mapping[ds.base_idx] + + merger = InstancerMerger(font, model, loc) + + log.info("Building interpolated tables") + # TODO GSUB/GDEF + merger.mergeTables(font, master_fonts, ["GPOS"]) + return font + + +def main(args=None): + """Interpolate GDEF/GPOS/GSUB tables for a point on a designspace""" + from fontTools import configLogger + import argparse + import sys + + parser = argparse.ArgumentParser( + "fonttools varLib.interpolate_layout", + description=main.__doc__, + ) + parser.add_argument( + "designspace_filename", metavar="DESIGNSPACE", help="Input TTF files" + ) + parser.add_argument( + "locations", + metavar="LOCATION", + type=str, + nargs="+", + help="Axis locations (e.g. wdth=120", + ) + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT", + help="Output font file (defaults to <designspacename>-instance.ttf)", + ) + parser.add_argument( + "-l", + "--loglevel", + metavar="LEVEL", + default="INFO", + help="Logging level (defaults to INFO)", + ) + + args = parser.parse_args(args) + + if not args.output: + args.output = os.path.splitext(args.designspace_filename)[0] + "-instance.ttf" + + configLogger(level=args.loglevel) + + finder = lambda s: s.replace("master_ufo", "master_ttf_interpolatable").replace( + ".ufo", ".ttf" + ) + + loc = {} + for arg in args.locations: + tag, val = arg.split("=") + loc[tag] = float(val) + + font = interpolate_layout(args.designspace_filename, loc, finder) + log.info("Saving font %s", args.output) + font.save(args.output) + + +if __name__ == "__main__": + import sys + + if len(sys.argv) > 1: + sys.exit(main()) + import doctest + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/iup.c b/.venv/lib/python3.11/site-packages/fontTools/varLib/iup.c new file mode 100644 index 0000000000000000000000000000000000000000..3df4d8e48b1c4d273e81b3d2f4b2905a29a5850b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/iup.c @@ -0,0 +1,19845 @@ +/* Generated by Cython 3.2.4 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "name": "fontTools.varLib.iup", + "sources": [ + "Lib/fontTools/varLib/iup.py" + ] + }, + "module_name": "fontTools.varLib.iup" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_2_4" +#define CYTHON_HEX_VERSION 0x030204F0 +#define CYTHON_FUTURE_DIVISION 1 +/* CModulePreamble */ +#include <stddef.h> +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 1 + #endif + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif + #if defined(CYTHON_IMMORTAL_CONSTANTS) && PY_VERSION_HEX < 0x030C0000 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 // definitely won't work + #elif !defined(CYTHON_IMMORTAL_CONSTANTS) + #define CYTHON_IMMORTAL_CONSTANTS (PY_VERSION_HEX >= 0x030C0000 && !CYTHON_USE_MODULE_STATE && CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_COMPRESS_STRINGS + #define CYTHON_COMPRESS_STRINGS 1 +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL) +#endif +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template<class T> void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#include <stdint.h> +typedef uintptr_t __pyx_uintptr_t; +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template <typename T> + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL<type>::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CInitCode */ +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #elif defined(__GNUC__) + #define CYTHON_INLINE __inline__ + #elif defined(_MSC_VER) + #define CYTHON_INLINE __inline + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_INLINE inline + #else + #define CYTHON_INLINE + #endif +#endif + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef Py_TPFLAGS_IMMUTABLETYPE + #define Py_TPFLAGS_IMMUTABLETYPE (1UL << 8) +#endif +#ifndef Py_TPFLAGS_DISALLOW_INSTANTIATION + #define Py_TPFLAGS_DISALLOW_INSTANTIATION (1UL << 7) +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if PY_VERSION_HEX < 0x03090000 || (CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) GraalPyFrame_SetLineNumber((frame), (lineno)) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && PyUnstable_Object_IsUniquelyReferenced(obj)) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#elif CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +enum __Pyx_ReferenceSharing { + __Pyx_ReferenceSharing_DefinitelyUnique, // We created it so we know it's unshared - no need to check + __Pyx_ReferenceSharing_OwnStrongReference, + __Pyx_ReferenceSharing_FunctionArgument, + __Pyx_ReferenceSharing_SharedReference, // Never trust it to be unshared because it's a global or similar +}; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && PY_VERSION_HEX >= 0x030E0000 +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing)\ + (sharing == __Pyx_ReferenceSharing_DefinitelyUnique ? 1 :\ + (sharing == __Pyx_ReferenceSharing_FunctionArgument ? PyUnstable_Object_IsUniqueReferencedTemporary(o) :\ + (sharing == __Pyx_ReferenceSharing_OwnStrongReference ? PyUnstable_Object_IsUniquelyReferenced(o) : 0))) +#elif (CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)sharing), Py_REFCNT(o) == 1) +#else +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)o), ((void)sharing), 0) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !CYTHON_COMPILING_IN_LIMITED_API && CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) (__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared) ?\ + __Pyx_NewRef(PyList_GET_ITEM(o, i)) : __Pyx_PyList_GetItemRef(o, i)) +#else + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) __Pyx_PyList_GetItemRef(o, i) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include <math.h> +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#define __PYX_HAVE__fontTools__varLib__iup +#define __PYX_HAVE_API__fontTools__varLib__iup +/* Early includes */ +#ifdef _OPENMP +#include <omp.h> +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#ifdef CYTHON_FREETHREADING_COMPATIBLE +#if CYTHON_FREETHREADING_COMPATIBLE +#define __Pyx_FREETHREADING_COMPATIBLE Py_MOD_GIL_NOT_USED +#else +#define __Pyx_FREETHREADING_COMPATIBLE Py_MOD_GIL_USED +#endif +#else +#define __Pyx_FREETHREADING_COMPATIBLE Py_MOD_GIL_USED +#endif +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include <cstdlib> + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include <type_traits> +#endif +template <typename T> +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible<T>::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "Lib/fontTools/varLib/iup.py", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* Atomics.proto (used by UnpackUnboundCMethod) */ +#include <pythread.h> +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include <stdatomic.h> +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include <atomic> +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_nonatomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_nonatomic_ptr_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include <intrin.h> + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #define __pyx_nonatomic_ptr_type void* + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_atomic_ptr_type old = _InterlockedCompareExchangePointer(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif + +/* CriticalSectionsDefinition.proto (used by CriticalSections) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_End(cs) +#define __Pyx_PyCriticalSection2_End(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_End PyCriticalSection_End +#define __Pyx_PyCriticalSection2_End PyCriticalSection2_End +#endif + +/* CriticalSections.proto (used by ParseKeywordsImpl) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection_Begin(cs, arg) (void)(cs) +#define __Pyx_PyCriticalSection2_Begin(cs, arg1, arg2) (void)(cs) +#else +#define __Pyx_PyCriticalSection_Begin PyCriticalSection_Begin +#define __Pyx_PyCriticalSection2_Begin PyCriticalSection2_Begin +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* IncludeStructmemberH.proto (used by FixUpExtensionType) */ +#include <structmember.h> + +/* #### Code section: numeric_typedefs ### */ +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +struct __pyx_defaults; +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between; +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr; +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize; +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr; +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr; + +/* "fontTools/varLib/iup.py":293 + * + * + * @cython.locals( # <<<<<<<<<<<<<< + * i=cython.int, + * j=cython.int, +*/ +struct __pyx_defaults { + PyObject_HEAD + PyObject *arg0; +}; + + +/* "fontTools/varLib/iup.py":169 + * + * + * @cython.cfunc # <<<<<<<<<<<<<< + * @cython.inline + * @cython.locals( +*/ +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between { + PyObject_HEAD + PyObject *__pyx_v_tolerance; +}; + + +/* "fontTools/varLib/iup.py":197 + * + * return all( + * abs(complex(x - p, y - q)) <= tolerance # <<<<<<<<<<<<<< + * for (x, y), (p, q) in zip(deltas, interp) + * ) +*/ +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr { + PyObject_HEAD + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *__pyx_outer_scope; + PyObject *__pyx_genexpr_arg_0; + int __pyx_v_i; + int __pyx_v_j; + double __pyx_v_p; + double __pyx_v_q; + double __pyx_v_x; + double __pyx_v_y; +}; + + +/* "fontTools/varLib/iup.py":363 + * + * + * def iup_contour_optimize( # <<<<<<<<<<<<<< + * deltas: _DeltaSegment, coords: _PointSegment, tolerance: Real = 0.0 + * ) -> _DeltaOrNoneSegment: +*/ +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize { + PyObject_HEAD + PyObject *__pyx_v_d0; + PyObject *__pyx_v_tolerance; +}; + + +/* "fontTools/varLib/iup.py":378 + * + * # If all are within tolerance distance of 0, encode nothing: + * if all(abs(complex(*p)) <= tolerance for p in deltas): # <<<<<<<<<<<<<< + * return [None] * n + * +*/ +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr { + PyObject_HEAD + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *__pyx_outer_scope; + PyObject *__pyx_genexpr_arg_0; + PyObject *__pyx_v_p; +}; + + +/* "fontTools/varLib/iup.py":387 + * # If all deltas are exactly the same, return just one (the first one): + * d0 = deltas[0] + * if all(d0 == d for d in deltas): # <<<<<<<<<<<<<< + * return [d0] + [None] * (n - 1) + * +*/ +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr { + PyObject_HEAD + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *__pyx_outer_scope; + PyObject *__pyx_genexpr_arg_0; + PyObject *__pyx_v_d; +}; + +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyErrExceptionMatches.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto (used by PyErrFetchRestore) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto (used by GetBuiltinName) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* SetItemInt.proto */ +#define __Pyx_SetItemInt(o, i, v, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_SetItemInt_Fast(o, (Py_ssize_t)i, v, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list assignment index out of range"), -1) :\ + __Pyx_SetItemInt_Generic(o, to_py_func(i), v))) +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v); +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* GetItemInt.proto */ +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* ListExtend.proto */ +static CYTHON_INLINE int __Pyx_PyList_Extend(PyObject* L, PyObject* v) { +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00a2 + return PyList_Extend(L, v); +#elif CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030d0000 + PyObject* none = _PyList_Extend((PyListObject*)L, v); + if (unlikely(!none)) + return -1; + Py_DECREF(none); + return 0; +#else + return PyList_SetSlice(L, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, v); +#endif +} + +/* ListAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_PyList_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len) & likely(len > (L->allocated >> 1))) { + Py_INCREF(x); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 + L->ob_item[len] = x; + #else + PyList_SET_ITEM(list, len, x); + #endif + __Pyx_SET_SIZE(list, len + 1); + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_PyList_Append(L,x) PyList_Append(L,x) +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* TupleAndListFromArray.proto (used by fastcall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* IncludeStringH.proto (used by BytesEquals) */ +#include <string.h> + +/* BytesEquals.proto (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* py_dict_items.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d); + +/* CallCFunction.proto (used by CallUnboundCMethod0) */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* PyObjectCallMethO.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto (used by PyObjectCallOneArg) */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectCallOneArg.proto (used by CallUnboundCMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* UnpackUnboundCMethod.proto (used by CallUnboundCMethod0) */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod0.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#else +#define __Pyx_CallUnboundCMethod0(cfunc, self) __Pyx__CallUnboundCMethod0(cfunc, self) +#endif + +/* py_dict_values.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d); + +/* OwnedDictNext.proto (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue); +#else +CYTHON_INLINE +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue); +#endif + +/* RaiseDoubleKeywords.proto (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywordsImpl.export */ +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name +); +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* AssertionsEnabled.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000 + static int __pyx_assertions_enabled_flag; + #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag) + #if __clang__ || __GNUC__ + __attribute__((no_sanitize("thread"))) + #endif + static int __Pyx_init_assertions_enabled(void) { + PyObject *builtins, *debug, *debug_str; + int flag; + builtins = PyEval_GetBuiltins(); + if (!builtins) goto bad; + debug_str = PyUnicode_FromStringAndSize("__debug__", 9); + if (!debug_str) goto bad; + debug = PyObject_GetItem(builtins, debug_str); + Py_DECREF(debug_str); + if (!debug) goto bad; + flag = PyObject_IsTrue(debug); + Py_DECREF(debug); + if (flag == -1) goto bad; + __pyx_assertions_enabled_flag = flag; + return 0; + bad: + __pyx_assertions_enabled_flag = 1; + return -1; + } +#else + #define __Pyx_init_assertions_enabled() (0) + #define __pyx_assertions_enabled() (!Py_OptimizeFlag) +#endif + +/* PyAssertionError_Check.proto */ +#define __Pyx_PyExc_AssertionError_Check(obj) __Pyx_TypeCheck(obj, PyExc_AssertionError) + +/* RaiseException.export */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* PySequenceContains.proto */ +static CYTHON_INLINE int __Pyx_PySequence_ContainsTF(PyObject* item, PyObject* seq, int eq) { + int result = PySequence_Contains(seq, item); + return unlikely(result < 0) ? result : (result == (eq == Py_EQ)); +} + +/* PyLongBinop.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static CYTHON_INLINE PyObject* __Pyx_PyLong_AddObjC(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check); +#else +#define __Pyx_PyLong_AddObjC(op1, op2, intval, inplace, zerodivision_check)\ + (inplace ? PyNumber_InPlaceAdd(op1, op2) : PyNumber_Add(op1, op2)) +#endif + +/* ListCompAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len)) { + Py_INCREF(x); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 + L->ob_item[len] = x; + #else + PyList_SET_ITEM(list, len, x); + #endif + __Pyx_SET_SIZE(list, len + 1); + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) +#endif + +/* IterNextPlain.proto (used by IterNext) */ +static CYTHON_INLINE PyObject *__Pyx_PyIter_Next_Plain(PyObject *iterator); +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject *__Pyx_GetBuiltinNext_LimitedAPI(void); +#endif + +/* IterNext.proto */ +#define __Pyx_PyIter_Next(obj) __Pyx_PyIter_Next2(obj, NULL) +static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject *, PyObject *); + +/* PyLongCompare.proto */ +static CYTHON_INLINE int __Pyx_PyLong_BoolNeObjC(PyObject *op1, PyObject *op2, long intval, long inplace); + +/* SliceObject.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetSlice( + PyObject* obj, Py_ssize_t cstart, Py_ssize_t cstop, + PyObject** py_start, PyObject** py_stop, PyObject** py_slice, + int has_cstart, int has_cstop, int wraparound); + +/* ObjectGetItem.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key); +#else +#define __Pyx_PyObject_GetItem(obj, key) PyObject_GetItem(obj, key) +#endif + +/* PyLongBinop.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static CYTHON_INLINE PyObject* __Pyx_PyLong_SubtractObjC(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check); +#else +#define __Pyx_PyLong_SubtractObjC(op1, op2, intval, inplace, zerodivision_check)\ + (inplace ? PyNumber_InPlaceSubtract(op1, op2) : PyNumber_Subtract(op1, op2)) +#endif + +/* PyDictVersioning.proto (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __Pyx_XNewRef(__pyx_dict_cached_value);\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* RaiseUnboundLocalError.proto */ +static void __Pyx_RaiseUnboundLocalError(const char *varname); + +/* RaiseTooManyValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); + +/* RaiseNeedMoreValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +/* IterFinish.proto */ +static CYTHON_INLINE int __Pyx_IterFinish(void); + +/* UnpackItemEndCheck.proto */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); + +/* py_abs.proto */ +#if CYTHON_USE_PYLONG_INTERNALS +static PyObject *__Pyx_PyLong_AbsNeg(PyObject *num); +#define __Pyx_PyNumber_Absolute(x)\ + ((likely(PyLong_CheckExact(x))) ?\ + (likely(__Pyx_PyLong_IsNonNeg(x)) ? __Pyx_NewRef(x) : __Pyx_PyLong_AbsNeg(x)) :\ + PyNumber_Absolute(x)) +#else +#define __Pyx_PyNumber_Absolute(x) PyNumber_Absolute(x) +#endif + +/* RaiseClosureNameError.proto */ +static void __Pyx_RaiseClosureNameError(const char *varname); + +/* GetException.proto (used by pep479) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* pep479.proto */ +static void __Pyx_Generator_Replace_StopIteration(int in_async_gen); + +/* PyRange_Check.proto */ +#if CYTHON_COMPILING_IN_PYPY && !defined(PyRange_Check) + #define PyRange_Check(obj) __Pyx_TypeCheck((obj), &PyRange_Type) +#endif + +/* IncludeStdlibH.proto */ +#include <stdlib.h> + +/* ArgTypeTestFunc.export */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) + +/* DictGetItem.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); +#define __Pyx_PyObject_Dict_GetItem(obj, name)\ + (likely(PyDict_CheckExact(obj)) ?\ + __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name)) +#else +#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) +#define __Pyx_PyObject_Dict_GetItem(obj, name) PyObject_GetItem(obj, name) +#endif + +/* PyObjectCallNoArg.proto (used by pyfrozenset_new) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); + +/* pyfrozenset_new.proto (used by PySetContains) */ +static CYTHON_INLINE PyObject* __Pyx_PyFrozenSet_New(PyObject* it); + +/* PySetContains.proto */ +static CYTHON_INLINE int __Pyx_PySet_ContainsTF(PyObject* key, PyObject* set, int eq); + +/* pyint_simplify.proto */ +static CYTHON_INLINE int __Pyx_PyInt_FromNumber(PyObject **number_var, const char *argname, int accept_none); + +/* SliceTupleAndList.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_GetSlice(PyObject* src, Py_ssize_t start, Py_ssize_t stop); +static CYTHON_INLINE PyObject* __Pyx_PyTuple_GetSlice(PyObject* src, Py_ssize_t start, Py_ssize_t stop); +#else +#define __Pyx_PyList_GetSlice(seq, start, stop) PySequence_GetSlice(seq, start, stop) +#define __Pyx_PyTuple_GetSlice(seq, start, stop) PySequence_GetSlice(seq, start, stop) +#endif + +/* set_iter.proto */ +static CYTHON_INLINE PyObject* __Pyx_set_iterator(PyObject* iterable, int is_set, + Py_ssize_t* p_orig_length, int* p_source_is_set); +static CYTHON_INLINE int __Pyx_set_iter_next( + PyObject* iter_obj, Py_ssize_t orig_length, + Py_ssize_t* ppos, PyObject **value, + int source_is_set); + +/* PyLongCompare.proto */ +static CYTHON_INLINE int __Pyx_PyLong_BoolEqObjC(PyObject *op1, PyObject *op2, long intval, long inplace); + +/* py_set_remove.proto */ +static CYTHON_INLINE int __Pyx_PySet_Remove(PyObject *set, PyObject *key); + +/* AllocateExtensionType.proto */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final); + +/* CallTypeTraverse.proto */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* CheckTypeForFreelists.proto */ +#if CYTHON_USE_FREELISTS +#if CYTHON_USE_TYPE_SPECS +#define __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, expected_tp, expected_size) ((int) ((t) == (expected_tp))) +#define __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS Py_TPFLAGS_IS_ABSTRACT +#else +#define __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, expected_tp, expected_size) ((int) ((t)->tp_basicsize == (expected_size))) +#define __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE) +#endif +#define __PYX_CHECK_TYPE_FOR_FREELISTS(t, expected_tp, expected_size)\ + (__PYX_CHECK_FINAL_TYPE_FOR_FREELISTS((t), (expected_tp), (expected_size)) &\ + (int) (!__Pyx_PyType_HasFeature((t), __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS))) +#endif + +/* LimitedApiGetTypeDict.proto (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* PyObjectGetMethod.proto (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +#endif + +/* PyObjectCallMethod0.proto (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); + +/* ValidateBasesTuple.proto (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases); +#endif + +/* PyType_Ready.proto */ +CYTHON_UNUSED static int __Pyx_PyType_Ready(PyTypeObject *t); + +/* HasAttr.proto (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_HasAttr(o, n) PyObject_HasAttrWithError(o, n) +#else +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *); +#endif + +/* ImportImpl.export */ +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level); + +/* Import.proto */ +static CYTHON_INLINE PyObject *__Pyx_Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, int level); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* dict_setdefault.proto (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value); + +/* AddModuleRef.proto (used by FetchSharedCythonModule) */ +#if ((CYTHON_COMPILING_IN_CPYTHON_FREETHREADING ) ||\ + __PYX_LIMITED_VERSION_HEX < 0x030d0000) + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name); +#else + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#endif + +/* FetchSharedCythonModule.proto (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* FetchCommonType.proto (used by CommonTypesMetaclass) */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto (used by CythonFunctionShared) */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* PyMethodNew.proto (used by CythonFunctionShared) */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto (used by CythonFunction) */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_dict; +#endif + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* CLineInTraceback.proto (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto (used by AddTraceback) */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* PyObjectVectorCallKwBuilder.proto (used by CIntToPy) */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +#if __PYX_LIMITED_VERSION_HEX < 0x030b0000 +static unsigned long __Pyx_cached_runtime_version = 0; +static void __Pyx_init_runtime_version(void); +#else +#define __Pyx_init_runtime_version() +#endif +static unsigned long __Pyx_get_runtime_version(void); + +/* GetTopmostException.proto (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto (used by CoroutineBase) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* SwapException.proto (used by CoroutineBase) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* PyObjectCall2Args.proto (used by PyObjectCallMethod1) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); + +/* PyObjectCallMethod1.proto (used by CoroutineBase) */ +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); + +/* ReturnWithStopIteration.proto (used by CoroutineBase) */ +static CYTHON_INLINE void __Pyx_ReturnWithStopIteration(PyObject* value, int async, int iternext); + +/* CoroutineBase.proto (used by Generator) */ +struct __pyx_CoroutineObject; +typedef PyObject *(*__pyx_coroutine_body_t)(struct __pyx_CoroutineObject *, PyThreadState *, PyObject *); +#if CYTHON_USE_EXC_INFO_STACK +#define __Pyx_ExcInfoStruct _PyErr_StackItem +#else +typedef struct { + PyObject *exc_type; + PyObject *exc_value; + PyObject *exc_traceback; +} __Pyx_ExcInfoStruct; +#endif +typedef struct __pyx_CoroutineObject { + PyObject_HEAD + __pyx_coroutine_body_t body; + PyObject *closure; + __Pyx_ExcInfoStruct gi_exc_state; +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *gi_weakreflist; +#endif + PyObject *classobj; + PyObject *yieldfrom; + __Pyx_pyiter_sendfunc yieldfrom_am_send; + PyObject *gi_name; + PyObject *gi_qualname; + PyObject *gi_modulename; + PyObject *gi_code; + PyObject *gi_frame; +#if CYTHON_USE_SYS_MONITORING && (CYTHON_PROFILE || CYTHON_TRACE) + PyMonitoringState __pyx_pymonitoring_state[__Pyx_MonitoringEventTypes_CyGen_count]; + uint64_t __pyx_pymonitoring_version; +#endif + int resume_label; + char is_running; +} __pyx_CoroutineObject; +static __pyx_CoroutineObject *__Pyx__Coroutine_New( + PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name); +static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( + __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name); +static CYTHON_INLINE void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *self); +static int __Pyx_Coroutine_clear(PyObject *self); +static __Pyx_PySendResult __Pyx_Coroutine_AmSend(PyObject *self, PyObject *value, PyObject **retval); +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); +static __Pyx_PySendResult __Pyx_Coroutine_Close(PyObject *self, PyObject **retval); +static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); +#if CYTHON_USE_EXC_INFO_STACK +#define __Pyx_Coroutine_SwapException(self) +#define __Pyx_Coroutine_ResetAndClearException(self) __Pyx_Coroutine_ExceptionClear(&(self)->gi_exc_state) +#else +#define __Pyx_Coroutine_SwapException(self) {\ + __Pyx_ExceptionSwap(&(self)->gi_exc_state.exc_type, &(self)->gi_exc_state.exc_value, &(self)->gi_exc_state.exc_traceback);\ + __Pyx_Coroutine_ResetFrameBackpointer(&(self)->gi_exc_state);\ + } +#define __Pyx_Coroutine_ResetAndClearException(self) {\ + __Pyx_ExceptionReset((self)->gi_exc_state.exc_type, (self)->gi_exc_state.exc_value, (self)->gi_exc_state.exc_traceback);\ + (self)->gi_exc_state.exc_type = (self)->gi_exc_state.exc_value = (self)->gi_exc_state.exc_traceback = NULL;\ + } +#endif +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyGen_FetchStopIterationValue(pvalue)\ + __Pyx_PyGen__FetchStopIterationValue(__pyx_tstate, pvalue) +#else +#define __Pyx_PyGen_FetchStopIterationValue(pvalue)\ + __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, pvalue) +#endif +static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *tstate, PyObject **pvalue); +static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStruct *exc_state); +static char __Pyx_Coroutine_test_and_set_is_running(__pyx_CoroutineObject *gen); +static void __Pyx_Coroutine_unset_is_running(__pyx_CoroutineObject *gen); +static char __Pyx_Coroutine_get_is_running(__pyx_CoroutineObject *gen); +static PyObject *__Pyx_Coroutine_get_is_running_getter(PyObject *gen, void *closure); +#if __PYX_HAS_PY_AM_SEND == 2 +static void __Pyx_SetBackportTypeAmSend(PyTypeObject *type, __Pyx_PyAsyncMethodsStruct *static_amsend_methods, __Pyx_pyiter_sendfunc am_send); +#endif +static PyObject *__Pyx_Coroutine_fail_reduce_ex(PyObject *self, PyObject *arg); + +/* Generator.proto */ +#define __Pyx_Generator_USED +#define __Pyx_Generator_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_GeneratorType) +#define __Pyx_Generator_New(body, code, closure, name, qualname, module_name)\ + __Pyx__Coroutine_New(__pyx_mstate_global->__pyx_GeneratorType, body, code, closure, name, qualname, module_name) +static PyObject *__Pyx_Generator_Next(PyObject *self); +static int __pyx_Generator_init(PyObject *module); +static CYTHON_INLINE PyObject *__Pyx_Generator_GetInlinedResult(PyObject *self); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* DecompressString.proto */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + + +/* Module declarations from "cython" */ + +/* Module declarations from "fontTools.varLib.iup" */ +static PyObject *__pyx_f_9fontTools_6varLib_3iup_iup_segment(PyObject *, PyObject *, PyObject *, PyObject *, PyObject *); /*proto*/ +static CYTHON_INLINE int __pyx_f_9fontTools_6varLib_3iup_can_iup_in_between(PyObject *, PyObject *, int, int, PyObject *); /*proto*/ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "fontTools.varLib.iup" +extern int __pyx_module_is_main_fontTools__varLib__iup; +int __pyx_module_is_main_fontTools__varLib__iup = 0; + +/* Implementation of "fontTools.varLib.iup" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin_zip; +static PyObject *__pyx_builtin_enumerate; +static PyObject *__pyx_builtin_max; +/* #### Code section: string_decls ### */ +/* #### Code section: decls ### */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_iup_contour(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_2iup_delta(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, PyObject *__pyx_v_ends); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_18can_iup_in_between_genexpr(PyObject *__pyx_self, PyObject *__pyx_genexpr_arg_0); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_4_iup_contour_bound_forced_set(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, PyObject *__pyx_v_tolerance); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_16__defaults__(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_6_iup_contour_optimize_dp(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, PyObject *__pyx_v_forced, double __pyx_v_tolerance, PyObject *__pyx_v_lookback); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_8_rot_list(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_l, PyObject *__pyx_v_k); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_10_rot_set(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_s, PyObject *__pyx_v_k, PyObject *__pyx_v_n); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_20iup_contour_optimize_genexpr(PyObject *__pyx_self, PyObject *__pyx_genexpr_arg_0); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_20iup_contour_optimize_3genexpr(PyObject *__pyx_self, PyObject *__pyx_genexpr_arg_0); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_12iup_contour_optimize(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, PyObject *__pyx_v_tolerance); /* proto */ +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_14iup_delta_optimize(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, PyObject *__pyx_v_ends, PyObject *__pyx_v_tolerance); /* proto */ +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_defaults(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + PyObject *__pyx_type_9fontTools_6varLib_3iup___pyx_defaults; + PyObject *__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between; + PyObject *__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr; + PyObject *__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize; + PyObject *__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr; + PyObject *__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr; + PyTypeObject *__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults; + PyTypeObject *__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between; + PyTypeObject *__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr; + PyTypeObject *__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize; + PyTypeObject *__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr; + PyTypeObject *__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_items; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_values; + PyObject *__pyx_tuple[4]; + PyObject *__pyx_codeobj_tab[11]; + PyObject *__pyx_string_tab[128]; + PyObject *__pyx_number_tab[8]; +/* #### Code section: module_state_contents ### */ +/* IterNextPlain.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +PyObject *__Pyx_GetBuiltinNext_LimitedAPI_cache; +#endif + + +#if CYTHON_USE_FREELISTS +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between[8]; +int __pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between; +#endif + +#if CYTHON_USE_FREELISTS +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr[8]; +int __pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr; +#endif + +#if CYTHON_USE_FREELISTS +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize[8]; +int __pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize; +#endif + +#if CYTHON_USE_FREELISTS +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr[8]; +int __pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr; +#endif + +#if CYTHON_USE_FREELISTS +struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr[8]; +int __pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr; +#endif +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CythonFunctionShared.module_state_decls */ +PyTypeObject *__pyx_CyFunctionType; + +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* Generator.module_state_decls */ +PyTypeObject *__pyx_GeneratorType; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_kp_u_Lib_fontTools_varLib_iup_py __pyx_string_tab[1] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[2] +#define __pyx_kp_u__2 __pyx_string_tab[3] +#define __pyx_kp_u_add_note __pyx_string_tab[4] +#define __pyx_kp_u_disable __pyx_string_tab[5] +#define __pyx_kp_u_enable __pyx_string_tab[6] +#define __pyx_kp_u_gc __pyx_string_tab[7] +#define __pyx_kp_u_isenabled __pyx_string_tab[8] +#define __pyx_n_u_COMPILED __pyx_string_tab[9] +#define __pyx_n_u_Delta __pyx_string_tab[10] +#define __pyx_n_u_DeltaOrNone __pyx_string_tab[11] +#define __pyx_n_u_DeltaOrNoneSegment __pyx_string_tab[12] +#define __pyx_n_u_DeltaSegment __pyx_string_tab[13] +#define __pyx_n_u_Endpoints __pyx_string_tab[14] +#define __pyx_n_u_Integral __pyx_string_tab[15] +#define __pyx_n_u_MAX_LOOKBACK __pyx_string_tab[16] +#define __pyx_n_u_Point __pyx_string_tab[17] +#define __pyx_n_u_PointSegment __pyx_string_tab[18] +#define __pyx_n_u_Pyx_PyDict_NextRef __pyx_string_tab[19] +#define __pyx_n_u_Real __pyx_string_tab[20] +#define __pyx_n_u_Sequence __pyx_string_tab[21] +#define __pyx_n_u_Tuple __pyx_string_tab[22] +#define __pyx_n_u_Union __pyx_string_tab[23] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[24] +#define __pyx_n_u_best_cost __pyx_string_tab[25] +#define __pyx_n_u_best_j __pyx_string_tab[26] +#define __pyx_n_u_best_sol __pyx_string_tab[27] +#define __pyx_n_u_c __pyx_string_tab[28] +#define __pyx_n_u_c1 __pyx_string_tab[29] +#define __pyx_n_u_c2 __pyx_string_tab[30] +#define __pyx_n_u_can_iup_in_between_locals_genexp __pyx_string_tab[31] +#define __pyx_n_u_chain __pyx_string_tab[32] +#define __pyx_n_u_cj __pyx_string_tab[33] +#define __pyx_n_u_class_getitem __pyx_string_tab[34] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[35] +#define __pyx_n_u_close __pyx_string_tab[36] +#define __pyx_n_u_contour __pyx_string_tab[37] +#define __pyx_n_u_coords __pyx_string_tab[38] +#define __pyx_n_u_cost __pyx_string_tab[39] +#define __pyx_n_u_costs __pyx_string_tab[40] +#define __pyx_n_u_d __pyx_string_tab[41] +#define __pyx_n_u_d0 __pyx_string_tab[42] +#define __pyx_n_u_d1 __pyx_string_tab[43] +#define __pyx_n_u_d2 __pyx_string_tab[44] +#define __pyx_n_u_deltas __pyx_string_tab[45] +#define __pyx_n_u_dj __pyx_string_tab[46] +#define __pyx_n_u_end __pyx_string_tab[47] +#define __pyx_n_u_ends __pyx_string_tab[48] +#define __pyx_n_u_enumerate __pyx_string_tab[49] +#define __pyx_n_u_fontTools_varLib_iup __pyx_string_tab[50] +#define __pyx_n_u_force __pyx_string_tab[51] +#define __pyx_n_u_forced __pyx_string_tab[52] +#define __pyx_n_u_func __pyx_string_tab[53] +#define __pyx_n_u_genexpr __pyx_string_tab[54] +#define __pyx_n_u_i __pyx_string_tab[55] +#define __pyx_n_u_i1 __pyx_string_tab[56] +#define __pyx_n_u_i2 __pyx_string_tab[57] +#define __pyx_n_u_indices __pyx_string_tab[58] +#define __pyx_n_u_int __pyx_string_tab[59] +#define __pyx_n_u_is_coroutine __pyx_string_tab[60] +#define __pyx_n_u_it __pyx_string_tab[61] +#define __pyx_n_u_items __pyx_string_tab[62] +#define __pyx_n_u_iup_contour __pyx_string_tab[63] +#define __pyx_n_u_iup_contour_bound_forced_set __pyx_string_tab[64] +#define __pyx_n_u_iup_contour_optimize __pyx_string_tab[65] +#define __pyx_n_u_iup_contour_optimize_dp __pyx_string_tab[66] +#define __pyx_n_u_iup_contour_optimize_locals_gene __pyx_string_tab[67] +#define __pyx_n_u_iup_delta __pyx_string_tab[68] +#define __pyx_n_u_iup_delta_optimize __pyx_string_tab[69] +#define __pyx_n_u_j __pyx_string_tab[70] +#define __pyx_n_u_k __pyx_string_tab[71] +#define __pyx_n_u_l __pyx_string_tab[72] +#define __pyx_n_u_lc __pyx_string_tab[73] +#define __pyx_n_u_lcj __pyx_string_tab[74] +#define __pyx_n_u_ld __pyx_string_tab[75] +#define __pyx_n_u_ldj __pyx_string_tab[76] +#define __pyx_n_u_list __pyx_string_tab[77] +#define __pyx_n_u_lookback __pyx_string_tab[78] +#define __pyx_n_u_main __pyx_string_tab[79] +#define __pyx_n_u_max __pyx_string_tab[80] +#define __pyx_n_u_module __pyx_string_tab[81] +#define __pyx_n_u_n __pyx_string_tab[82] +#define __pyx_n_u_name __pyx_string_tab[83] +#define __pyx_n_u_nc __pyx_string_tab[84] +#define __pyx_n_u_ncj __pyx_string_tab[85] +#define __pyx_n_u_nd __pyx_string_tab[86] +#define __pyx_n_u_ndj __pyx_string_tab[87] +#define __pyx_n_u_next __pyx_string_tab[88] +#define __pyx_n_u_numbers __pyx_string_tab[89] +#define __pyx_n_u_out __pyx_string_tab[90] +#define __pyx_n_u_p __pyx_string_tab[91] +#define __pyx_n_u_pop __pyx_string_tab[92] +#define __pyx_n_u_q __pyx_string_tab[93] +#define __pyx_n_u_qualname __pyx_string_tab[94] +#define __pyx_n_u_return __pyx_string_tab[95] +#define __pyx_n_u_ri1 __pyx_string_tab[96] +#define __pyx_n_u_ri2 __pyx_string_tab[97] +#define __pyx_n_u_rot_list __pyx_string_tab[98] +#define __pyx_n_u_rot_set __pyx_string_tab[99] +#define __pyx_n_u_s __pyx_string_tab[100] +#define __pyx_n_u_send __pyx_string_tab[101] +#define __pyx_n_u_set __pyx_string_tab[102] +#define __pyx_n_u_set_name __pyx_string_tab[103] +#define __pyx_n_u_setdefault __pyx_string_tab[104] +#define __pyx_n_u_solution __pyx_string_tab[105] +#define __pyx_n_u_start __pyx_string_tab[106] +#define __pyx_n_u_test __pyx_string_tab[107] +#define __pyx_n_u_throw __pyx_string_tab[108] +#define __pyx_n_u_tolerance __pyx_string_tab[109] +#define __pyx_n_u_typing __pyx_string_tab[110] +#define __pyx_n_u_v __pyx_string_tab[111] +#define __pyx_n_u_value __pyx_string_tab[112] +#define __pyx_n_u_values __pyx_string_tab[113] +#define __pyx_n_u_x __pyx_string_tab[114] +#define __pyx_n_u_y __pyx_string_tab[115] +#define __pyx_n_u_zip __pyx_string_tab[116] +#define __pyx_kp_b_iso88591_1 __pyx_string_tab[117] +#define __pyx_kp_b_iso88591_1A_t1_q_1ARr_b_Bb __pyx_string_tab[118] +#define __pyx_kp_b_iso88591_55Gq_3axs_Qa_uG1_q_1A_ar_S_Yaxs __pyx_string_tab[119] +#define __pyx_kp_b_iso88591_A __pyx_string_tab[120] +#define __pyx_kp_b_iso88591_Q_1A_A_q_r_A_q_q_a_q_Bay_A_881 __pyx_string_tab[121] +#define __pyx_kp_b_iso88591_Q_3axs_Qa_1A_U_3axr_T_E_q_D_ar __pyx_string_tab[122] +#define __pyx_kp_b_iso88591_a __pyx_string_tab[123] +#define __pyx_kp_b_iso88591_a_5_Cq_D_RuJcQSST_1A_5_2Rs_Bc_2 __pyx_string_tab[124] +#define __pyx_kp_b_iso88591_q_5_Cq_D_RuJcQSST_1A_5_2Rs_Bc_2 __pyx_string_tab[125] +#define __pyx_kp_b_iso88591_q_a_1A_y_1_1_Q_c_c_U_3a_E_Bc_1 __pyx_string_tab[126] +#define __pyx_kp_b_iso88591_t1_q_2Rr_Bb_E __pyx_string_tab[127] +#define __pyx_float_0_0 __pyx_number_tab[0] +#define __pyx_int_0 __pyx_number_tab[1] +#define __pyx_int_neg_1 __pyx_number_tab[2] +#define __pyx_int_1 __pyx_number_tab[3] +#define __pyx_int_2 __pyx_number_tab[4] +#define __pyx_int_3 __pyx_number_tab[5] +#define __pyx_int_4 __pyx_number_tab[6] +#define __pyx_int_8 __pyx_number_tab[7] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + Py_CLEAR(clear_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults); + Py_CLEAR(clear_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_defaults); + Py_CLEAR(clear_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between); + Py_CLEAR(clear_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between); + Py_CLEAR(clear_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr); + Py_CLEAR(clear_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr); + Py_CLEAR(clear_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize); + Py_CLEAR(clear_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize); + Py_CLEAR(clear_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr); + Py_CLEAR(clear_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr); + Py_CLEAR(clear_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr); + Py_CLEAR(clear_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr); + for (int i=0; i<4; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<11; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<128; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + for (int i=0; i<8; ++i) { Py_CLEAR(clear_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_clear_contents ### */ +/* CommonTypesMetaclass.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + +/* Generator.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_GeneratorType); + +/* #### Code section: module_state_clear_end ### */ +return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + Py_VISIT(traverse_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults); + Py_VISIT(traverse_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_defaults); + Py_VISIT(traverse_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between); + Py_VISIT(traverse_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between); + Py_VISIT(traverse_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr); + Py_VISIT(traverse_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr); + Py_VISIT(traverse_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize); + Py_VISIT(traverse_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize); + Py_VISIT(traverse_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr); + Py_VISIT(traverse_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr); + Py_VISIT(traverse_module_state->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr); + Py_VISIT(traverse_module_state->__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr); + for (int i=0; i<4; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<11; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<128; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + for (int i=0; i<8; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_traverse_contents ### */ +/* CommonTypesMetaclass.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + +/* Generator.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_GeneratorType); + +/* #### Code section: module_state_traverse_end ### */ +return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "fontTools/varLib/iup.py":28 + * + * + * @cython.cfunc # <<<<<<<<<<<<<< + * @cython.locals( + * j=cython.int, +*/ + +static PyObject *__pyx_f_9fontTools_6varLib_3iup_iup_segment(PyObject *__pyx_v_coords, PyObject *__pyx_v_rc1, PyObject *__pyx_v_rd1, PyObject *__pyx_v_rc2, PyObject *__pyx_v_rd2) { + int __pyx_v_j; + int __pyx_v_n; + double __pyx_v_x1; + double __pyx_v_x2; + double __pyx_v_d1; + double __pyx_v_d2; + double __pyx_v_scale; + double __pyx_v_x; + double __pyx_v_d; + PyObject *__pyx_v_out_arrays = NULL; + PyObject *__pyx_v_out = NULL; + PyObject *__pyx_v_pair = NULL; + double __pyx_v_nudge; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + Py_ssize_t __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + double __pyx_t_5; + double __pyx_t_6; + double __pyx_t_7; + double __pyx_t_8; + int __pyx_t_9; + Py_ssize_t __pyx_t_10; + PyObject *__pyx_t_11 = NULL; + int __pyx_t_12; + PyObject *(*__pyx_t_13)(PyObject *); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("iup_segment", 0); + + /* "fontTools/varLib/iup.py":49 + * # rc1 = reference coord 1 + * # rd1 = reference delta 1 + * out_arrays = [None, None] # <<<<<<<<<<<<<< + * for j in 0, 1: + * out_arrays[j] = out = [] +*/ + __pyx_t_1 = PyList_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 49, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyList_SET_ITEM(__pyx_t_1, 0, Py_None) != (0)) __PYX_ERR(0, 49, __pyx_L1_error); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyList_SET_ITEM(__pyx_t_1, 1, Py_None) != (0)) __PYX_ERR(0, 49, __pyx_L1_error); + __pyx_v_out_arrays = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "fontTools/varLib/iup.py":50 + * # rd1 = reference delta 1 + * out_arrays = [None, None] + * for j in 0, 1: # <<<<<<<<<<<<<< + * out_arrays[j] = out = [] + * x1, x2, d1, d2 = rc1[j], rc2[j], rd1[j], rd2[j] +*/ + __pyx_t_1 = __pyx_mstate_global->__pyx_tuple[0]; __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = 0; + for (;;) { + if (__pyx_t_2 >= 2) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_2)); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(__pyx_t_1, __pyx_t_2); + #endif + ++__pyx_t_2; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 50, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyLong_As_int(__pyx_t_3); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 50, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_j = __pyx_t_4; + + /* "fontTools/varLib/iup.py":51 + * out_arrays = [None, None] + * for j in 0, 1: + * out_arrays[j] = out = [] # <<<<<<<<<<<<<< + * x1, x2, d1, d2 = rc1[j], rc2[j], rd1[j], rd2[j] + * +*/ + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 51, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((__Pyx_SetItemInt(__pyx_v_out_arrays, __pyx_v_j, __pyx_t_3, int, 1, __Pyx_PyLong_From_int, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference) < 0))) __PYX_ERR(0, 51, __pyx_L1_error) + __Pyx_INCREF(__pyx_t_3); + __Pyx_XDECREF_SET(__pyx_v_out, __pyx_t_3); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":52 + * for j in 0, 1: + * out_arrays[j] = out = [] + * x1, x2, d1, d2 = rc1[j], rc2[j], rd1[j], rd2[j] # <<<<<<<<<<<<<< + * + * if x1 == x2: +*/ + __pyx_t_3 = __Pyx_GetItemInt(__pyx_v_rc1, __pyx_v_j, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyFloat_AsDouble(__pyx_t_3); if (unlikely((__pyx_t_5 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_GetItemInt(__pyx_v_rc2, __pyx_v_j, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = __Pyx_PyFloat_AsDouble(__pyx_t_3); if (unlikely((__pyx_t_6 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_GetItemInt(__pyx_v_rd1, __pyx_v_j, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_7 = __Pyx_PyFloat_AsDouble(__pyx_t_3); if (unlikely((__pyx_t_7 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_GetItemInt(__pyx_v_rd2, __pyx_v_j, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_8 = __Pyx_PyFloat_AsDouble(__pyx_t_3); if (unlikely((__pyx_t_8 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_x1 = __pyx_t_5; + __pyx_v_x2 = __pyx_t_6; + __pyx_v_d1 = __pyx_t_7; + __pyx_v_d2 = __pyx_t_8; + + /* "fontTools/varLib/iup.py":54 + * x1, x2, d1, d2 = rc1[j], rc2[j], rd1[j], rd2[j] + * + * if x1 == x2: # <<<<<<<<<<<<<< + * n = len(coords) + * if d1 == d2: +*/ + __pyx_t_9 = (__pyx_v_x1 == __pyx_v_x2); + if (__pyx_t_9) { + + /* "fontTools/varLib/iup.py":55 + * + * if x1 == x2: + * n = len(coords) # <<<<<<<<<<<<<< + * if d1 == d2: + * out.extend([d1] * n) +*/ + __pyx_t_10 = PyObject_Length(__pyx_v_coords); if (unlikely(__pyx_t_10 == ((Py_ssize_t)-1))) __PYX_ERR(0, 55, __pyx_L1_error) + __pyx_v_n = __pyx_t_10; + + /* "fontTools/varLib/iup.py":56 + * if x1 == x2: + * n = len(coords) + * if d1 == d2: # <<<<<<<<<<<<<< + * out.extend([d1] * n) + * else: +*/ + __pyx_t_9 = (__pyx_v_d1 == __pyx_v_d2); + if (__pyx_t_9) { + + /* "fontTools/varLib/iup.py":57 + * n = len(coords) + * if d1 == d2: + * out.extend([d1] * n) # <<<<<<<<<<<<<< + * else: + * out.extend([0] * n) +*/ + __pyx_t_3 = PyFloat_FromDouble(__pyx_v_d1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 57, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_11 = PyList_New(1 * ((__pyx_v_n<0) ? 0:__pyx_v_n)); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 57, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + { Py_ssize_t __pyx_temp; + for (__pyx_temp=0; __pyx_temp < __pyx_v_n; __pyx_temp++) { + __Pyx_INCREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyList_SET_ITEM(__pyx_t_11, __pyx_temp, __pyx_t_3) != (0)) __PYX_ERR(0, 57, __pyx_L1_error); + } + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_12 = __Pyx_PyList_Extend(__pyx_v_out, __pyx_t_11); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 57, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "fontTools/varLib/iup.py":56 + * if x1 == x2: + * n = len(coords) + * if d1 == d2: # <<<<<<<<<<<<<< + * out.extend([d1] * n) + * else: +*/ + goto __pyx_L6; + } + + /* "fontTools/varLib/iup.py":59 + * out.extend([d1] * n) + * else: + * out.extend([0] * n) # <<<<<<<<<<<<<< + * continue + * +*/ + /*else*/ { + __pyx_t_11 = PyList_New(1 * ((__pyx_v_n<0) ? 0:__pyx_v_n)); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 59, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + { Py_ssize_t __pyx_temp; + for (__pyx_temp=0; __pyx_temp < __pyx_v_n; __pyx_temp++) { + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_0); + if (__Pyx_PyList_SET_ITEM(__pyx_t_11, __pyx_temp, __pyx_mstate_global->__pyx_int_0) != (0)) __PYX_ERR(0, 59, __pyx_L1_error); + } + } + __pyx_t_12 = __Pyx_PyList_Extend(__pyx_v_out, __pyx_t_11); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 59, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + } + __pyx_L6:; + + /* "fontTools/varLib/iup.py":60 + * else: + * out.extend([0] * n) + * continue # <<<<<<<<<<<<<< + * + * if x1 > x2: +*/ + goto __pyx_L3_continue; + + /* "fontTools/varLib/iup.py":54 + * x1, x2, d1, d2 = rc1[j], rc2[j], rd1[j], rd2[j] + * + * if x1 == x2: # <<<<<<<<<<<<<< + * n = len(coords) + * if d1 == d2: +*/ + } + + /* "fontTools/varLib/iup.py":62 + * continue + * + * if x1 > x2: # <<<<<<<<<<<<<< + * x1, x2 = x2, x1 + * d1, d2 = d2, d1 +*/ + __pyx_t_9 = (__pyx_v_x1 > __pyx_v_x2); + if (__pyx_t_9) { + + /* "fontTools/varLib/iup.py":63 + * + * if x1 > x2: + * x1, x2 = x2, x1 # <<<<<<<<<<<<<< + * d1, d2 = d2, d1 + * +*/ + __pyx_t_8 = __pyx_v_x2; + __pyx_t_7 = __pyx_v_x1; + __pyx_v_x1 = __pyx_t_8; + __pyx_v_x2 = __pyx_t_7; + + /* "fontTools/varLib/iup.py":64 + * if x1 > x2: + * x1, x2 = x2, x1 + * d1, d2 = d2, d1 # <<<<<<<<<<<<<< + * + * # x1 < x2 +*/ + __pyx_t_7 = __pyx_v_d2; + __pyx_t_8 = __pyx_v_d1; + __pyx_v_d1 = __pyx_t_7; + __pyx_v_d2 = __pyx_t_8; + + /* "fontTools/varLib/iup.py":62 + * continue + * + * if x1 > x2: # <<<<<<<<<<<<<< + * x1, x2 = x2, x1 + * d1, d2 = d2, d1 +*/ + } + + /* "fontTools/varLib/iup.py":67 + * + * # x1 < x2 + * scale = (d2 - d1) / (x2 - x1) # <<<<<<<<<<<<<< + * for pair in coords: + * x = pair[j] +*/ + __pyx_t_8 = (__pyx_v_d2 - __pyx_v_d1); + __pyx_t_7 = (__pyx_v_x2 - __pyx_v_x1); + if (unlikely(__pyx_t_7 == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "float division"); + __PYX_ERR(0, 67, __pyx_L1_error) + } + __pyx_v_scale = (__pyx_t_8 / __pyx_t_7); + + /* "fontTools/varLib/iup.py":68 + * # x1 < x2 + * scale = (d2 - d1) / (x2 - x1) + * for pair in coords: # <<<<<<<<<<<<<< + * x = pair[j] + * +*/ + if (likely(PyList_CheckExact(__pyx_v_coords)) || PyTuple_CheckExact(__pyx_v_coords)) { + __pyx_t_11 = __pyx_v_coords; __Pyx_INCREF(__pyx_t_11); + __pyx_t_10 = 0; + __pyx_t_13 = NULL; + } else { + __pyx_t_10 = -1; __pyx_t_11 = PyObject_GetIter(__pyx_v_coords); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 68, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_13 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_11); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 68, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_13)) { + if (likely(PyList_CheckExact(__pyx_t_11))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_11); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 68, __pyx_L1_error) + #endif + if (__pyx_t_10 >= __pyx_temp) break; + } + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(__pyx_t_11, __pyx_t_10, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_10; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_11); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 68, __pyx_L1_error) + #endif + if (__pyx_t_10 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_11, __pyx_t_10)); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(__pyx_t_11, __pyx_t_10); + #endif + ++__pyx_t_10; + } + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 68, __pyx_L1_error) + } else { + __pyx_t_3 = __pyx_t_13(__pyx_t_11); + if (unlikely(!__pyx_t_3)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 68, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_3); + __Pyx_XDECREF_SET(__pyx_v_pair, __pyx_t_3); + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":69 + * scale = (d2 - d1) / (x2 - x1) + * for pair in coords: + * x = pair[j] # <<<<<<<<<<<<<< + * + * if x <= x1: +*/ + __pyx_t_3 = __Pyx_GetItemInt(__pyx_v_pair, __pyx_v_j, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 69, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_7 = __Pyx_PyFloat_AsDouble(__pyx_t_3); if (unlikely((__pyx_t_7 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 69, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_x = __pyx_t_7; + + /* "fontTools/varLib/iup.py":71 + * x = pair[j] + * + * if x <= x1: # <<<<<<<<<<<<<< + * d = d1 + * elif x >= x2: +*/ + __pyx_t_9 = (__pyx_v_x <= __pyx_v_x1); + if (__pyx_t_9) { + + /* "fontTools/varLib/iup.py":72 + * + * if x <= x1: + * d = d1 # <<<<<<<<<<<<<< + * elif x >= x2: + * d = d2 +*/ + __pyx_v_d = __pyx_v_d1; + + /* "fontTools/varLib/iup.py":71 + * x = pair[j] + * + * if x <= x1: # <<<<<<<<<<<<<< + * d = d1 + * elif x >= x2: +*/ + goto __pyx_L10; + } + + /* "fontTools/varLib/iup.py":73 + * if x <= x1: + * d = d1 + * elif x >= x2: # <<<<<<<<<<<<<< + * d = d2 + * else: +*/ + __pyx_t_9 = (__pyx_v_x >= __pyx_v_x2); + if (__pyx_t_9) { + + /* "fontTools/varLib/iup.py":74 + * d = d1 + * elif x >= x2: + * d = d2 # <<<<<<<<<<<<<< + * else: + * # Interpolate +*/ + __pyx_v_d = __pyx_v_d2; + + /* "fontTools/varLib/iup.py":73 + * if x <= x1: + * d = d1 + * elif x >= x2: # <<<<<<<<<<<<<< + * d = d2 + * else: +*/ + goto __pyx_L10; + } + + /* "fontTools/varLib/iup.py":83 + * # - https://godbolt.org/z/YsP4T3TqK, + * # - https://github.com/fonttools/fonttools/issues/3703 + * nudge = (x - x1) * scale # <<<<<<<<<<<<<< + * d = d1 + nudge + * +*/ + /*else*/ { + __pyx_v_nudge = ((__pyx_v_x - __pyx_v_x1) * __pyx_v_scale); + + /* "fontTools/varLib/iup.py":84 + * # - https://github.com/fonttools/fonttools/issues/3703 + * nudge = (x - x1) * scale + * d = d1 + nudge # <<<<<<<<<<<<<< + * + * out.append(d) +*/ + __pyx_v_d = (__pyx_v_d1 + __pyx_v_nudge); + } + __pyx_L10:; + + /* "fontTools/varLib/iup.py":86 + * d = d1 + nudge + * + * out.append(d) # <<<<<<<<<<<<<< + * + * return zip(*out_arrays) +*/ + __pyx_t_3 = PyFloat_FromDouble(__pyx_v_d); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_12 = __Pyx_PyList_Append(__pyx_v_out, __pyx_t_3); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":68 + * # x1 < x2 + * scale = (d2 - d1) / (x2 - x1) + * for pair in coords: # <<<<<<<<<<<<<< + * x = pair[j] + * +*/ + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "fontTools/varLib/iup.py":50 + * # rd1 = reference delta 1 + * out_arrays = [None, None] + * for j in 0, 1: # <<<<<<<<<<<<<< + * out_arrays[j] = out = [] + * x1, x2, d1, d2 = rc1[j], rc2[j], rd1[j], rd2[j] +*/ + __pyx_L3_continue:; + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "fontTools/varLib/iup.py":88 + * out.append(d) + * + * return zip(*out_arrays) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PySequence_Tuple(__pyx_v_out_arrays); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 88, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_builtin_zip, __pyx_t_1, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 88, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_11; + __pyx_t_11 = 0; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":28 + * + * + * @cython.cfunc # <<<<<<<<<<<<<< + * @cython.locals( + * j=cython.int, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_AddTraceback("fontTools.varLib.iup.iup_segment", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_out_arrays); + __Pyx_XDECREF(__pyx_v_out); + __Pyx_XDECREF(__pyx_v_pair); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "fontTools/varLib/iup.py":91 + * + * + * def iup_contour(deltas: _DeltaOrNoneSegment, coords: _PointSegment) -> _DeltaSegment: # <<<<<<<<<<<<<< + * """For the contour given in `coords`, interpolate any missing + * delta values in delta vector `deltas`. +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_1iup_contour(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_9fontTools_6varLib_3iup_iup_contour, "iup_contour(deltas: _DeltaOrNoneSegment, coords: _PointSegment) -> _DeltaSegment\n\nFor the contour given in `coords`, interpolate any missing\ndelta values in delta vector `deltas`.\n\nReturns fully filled-out delta vector."); +static PyMethodDef __pyx_mdef_9fontTools_6varLib_3iup_1iup_contour = {"iup_contour", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_6varLib_3iup_1iup_contour, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_6varLib_3iup_iup_contour}; +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_1iup_contour(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_deltas = 0; + PyObject *__pyx_v_coords = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("iup_contour (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_deltas,&__pyx_mstate_global->__pyx_n_u_coords,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 91, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 91, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 91, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "iup_contour", 0) < (0)) __PYX_ERR(0, 91, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("iup_contour", 1, 2, 2, i); __PYX_ERR(0, 91, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 91, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 91, __pyx_L3_error) + } + __pyx_v_deltas = values[0]; + __pyx_v_coords = values[1]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("iup_contour", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 91, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("fontTools.varLib.iup.iup_contour", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_9fontTools_6varLib_3iup_iup_contour(__pyx_self, __pyx_v_deltas, __pyx_v_coords); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_iup_contour(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords) { + PyObject *__pyx_v_n = NULL; + PyObject *__pyx_v_indices = NULL; + PyObject *__pyx_v_out = NULL; + PyObject *__pyx_v_it = NULL; + PyObject *__pyx_v_start = NULL; + PyObject *__pyx_v_i1 = NULL; + PyObject *__pyx_v_i2 = NULL; + PyObject *__pyx_v_ri1 = NULL; + PyObject *__pyx_v_ri2 = NULL; + PyObject *__pyx_v_end = NULL; + PyObject *__pyx_7genexpr__pyx_v_i = NULL; + PyObject *__pyx_7genexpr__pyx_v_v = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + Py_ssize_t __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + int __pyx_t_12; + PyObject *__pyx_t_13 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("iup_contour", 0); + + /* "fontTools/varLib/iup.py":97 + * Returns fully filled-out delta vector.""" + * + * assert len(deltas) == len(coords) # <<<<<<<<<<<<<< + * if None not in deltas: + * return deltas +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = PyObject_Length(__pyx_v_deltas); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_2 = PyObject_Length(__pyx_v_coords); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_1 == __pyx_t_2); + if (unlikely(!__pyx_t_3)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 97, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 97, __pyx_L1_error) + #endif + + /* "fontTools/varLib/iup.py":98 + * + * assert len(deltas) == len(coords) + * if None not in deltas: # <<<<<<<<<<<<<< + * return deltas + * +*/ + __pyx_t_3 = (__Pyx_PySequence_ContainsTF(Py_None, __pyx_v_deltas, Py_NE)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 98, __pyx_L1_error) + if (__pyx_t_3) { + + /* "fontTools/varLib/iup.py":99 + * assert len(deltas) == len(coords) + * if None not in deltas: + * return deltas # <<<<<<<<<<<<<< + * + * n = len(deltas) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_deltas); + __pyx_r = __pyx_v_deltas; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":98 + * + * assert len(deltas) == len(coords) + * if None not in deltas: # <<<<<<<<<<<<<< + * return deltas + * +*/ + } + + /* "fontTools/varLib/iup.py":101 + * return deltas + * + * n = len(deltas) # <<<<<<<<<<<<<< + * # indices of points with explicit deltas + * indices = [i for i, v in enumerate(deltas) if v is not None] +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_deltas); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_4 = PyLong_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 101, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_n = __pyx_t_4; + __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":103 + * n = len(deltas) + * # indices of points with explicit deltas + * indices = [i for i, v in enumerate(deltas) if v is not None] # <<<<<<<<<<<<<< + * if not indices: + * # All deltas are None. Return 0,0 for all. +*/ + { /* enter inner scope */ + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 103, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __pyx_t_5 = __pyx_mstate_global->__pyx_int_0; + if (likely(PyList_CheckExact(__pyx_v_deltas)) || PyTuple_CheckExact(__pyx_v_deltas)) { + __pyx_t_6 = __pyx_v_deltas; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_deltas); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 103, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 103, __pyx_L6_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 103, __pyx_L6_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRefFast(__pyx_t_6, __pyx_t_2, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 103, __pyx_L6_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 103, __pyx_L6_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 103, __pyx_L6_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_7genexpr__pyx_v_v, __pyx_t_8); + __pyx_t_8 = 0; + __Pyx_INCREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_7genexpr__pyx_v_i, __pyx_t_5); + __pyx_t_8 = __Pyx_PyLong_AddObjC(__pyx_t_5, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 103, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_5); + __pyx_t_5 = __pyx_t_8; + __pyx_t_8 = 0; + __pyx_t_3 = (__pyx_7genexpr__pyx_v_v != Py_None); + if (__pyx_t_3) { + if (unlikely(__Pyx_ListComp_Append(__pyx_t_4, (PyObject*)__pyx_7genexpr__pyx_v_i))) __PYX_ERR(0, 103, __pyx_L6_error) + } + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_7genexpr__pyx_v_i); __pyx_7genexpr__pyx_v_i = 0; + __Pyx_XDECREF(__pyx_7genexpr__pyx_v_v); __pyx_7genexpr__pyx_v_v = 0; + goto __pyx_L11_exit_scope; + __pyx_L6_error:; + __Pyx_XDECREF(__pyx_7genexpr__pyx_v_i); __pyx_7genexpr__pyx_v_i = 0; + __Pyx_XDECREF(__pyx_7genexpr__pyx_v_v); __pyx_7genexpr__pyx_v_v = 0; + goto __pyx_L1_error; + __pyx_L11_exit_scope:; + } /* exit inner scope */ + __pyx_v_indices = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":104 + * # indices of points with explicit deltas + * indices = [i for i, v in enumerate(deltas) if v is not None] + * if not indices: # <<<<<<<<<<<<<< + * # All deltas are None. Return 0,0 for all. + * return [(0, 0)] * n +*/ + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_v_indices); + if (unlikely(((!CYTHON_ASSUME_SAFE_SIZE) && __pyx_temp < 0))) __PYX_ERR(0, 104, __pyx_L1_error) + __pyx_t_3 = (__pyx_temp != 0); + } + + __pyx_t_9 = (!__pyx_t_3); + if (__pyx_t_9) { + + /* "fontTools/varLib/iup.py":106 + * if not indices: + * # All deltas are None. Return 0,0 for all. + * return [(0, 0)] * n # <<<<<<<<<<<<<< + * + * out = [] +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = PyList_New(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_mstate_global->__pyx_tuple[1]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[1]); + if (__Pyx_PyList_SET_ITEM(__pyx_t_4, 0, __pyx_mstate_global->__pyx_tuple[1]) != (0)) __PYX_ERR(0, 106, __pyx_L1_error); + { PyObject* __pyx_temp = PyNumber_InPlaceMultiply(__pyx_t_4, __pyx_v_n); if (unlikely(!__pyx_temp)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_temp); + __Pyx_DECREF(__pyx_t_4); + __pyx_t_4 = __pyx_temp; + } + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":104 + * # indices of points with explicit deltas + * indices = [i for i, v in enumerate(deltas) if v is not None] + * if not indices: # <<<<<<<<<<<<<< + * # All deltas are None. Return 0,0 for all. + * return [(0, 0)] * n +*/ + } + + /* "fontTools/varLib/iup.py":108 + * return [(0, 0)] * n + * + * out = [] # <<<<<<<<<<<<<< + * it = iter(indices) + * start = next(it) +*/ + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 108, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_out = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":109 + * + * out = [] + * it = iter(indices) # <<<<<<<<<<<<<< + * start = next(it) + * if start != 0: +*/ + __pyx_t_4 = PyObject_GetIter(__pyx_v_indices); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_it = __pyx_t_4; + __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":110 + * out = [] + * it = iter(indices) + * start = next(it) # <<<<<<<<<<<<<< + * if start != 0: + * # Initial segment that wraps around +*/ + __pyx_t_4 = __Pyx_PyIter_Next(__pyx_v_it); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_start = __pyx_t_4; + __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":111 + * it = iter(indices) + * start = next(it) + * if start != 0: # <<<<<<<<<<<<<< + * # Initial segment that wraps around + * i1, i2, ri1, ri2 = 0, start, start, indices[-1] +*/ + __pyx_t_9 = (__Pyx_PyLong_BoolNeObjC(__pyx_v_start, __pyx_mstate_global->__pyx_int_0, 0, 0)); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 111, __pyx_L1_error) + if (__pyx_t_9) { + + /* "fontTools/varLib/iup.py":113 + * if start != 0: + * # Initial segment that wraps around + * i1, i2, ri1, ri2 = 0, start, start, indices[-1] # <<<<<<<<<<<<<< + * out.extend( + * iup_segment( +*/ + __pyx_t_4 = __pyx_mstate_global->__pyx_int_0; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = __pyx_v_start; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = __pyx_v_start; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_8 = __Pyx_GetItemInt_List(__pyx_v_indices, -1L, long, 1, __Pyx_PyLong_From_long, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_v_i1 = __pyx_t_4; + __pyx_t_4 = 0; + __pyx_v_i2 = __pyx_t_5; + __pyx_t_5 = 0; + __pyx_v_ri1 = __pyx_t_6; + __pyx_t_6 = 0; + __pyx_v_ri2 = __pyx_t_8; + __pyx_t_8 = 0; + + /* "fontTools/varLib/iup.py":116 + * out.extend( + * iup_segment( + * coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] # <<<<<<<<<<<<<< + * ) + * ) +*/ + __pyx_t_8 = __Pyx_PyObject_GetSlice(__pyx_v_coords, 0, 0, &__pyx_v_i1, &__pyx_v_i2, NULL, 0, 0, 1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 116, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_6 = __Pyx_PyObject_GetItem(__pyx_v_coords, __pyx_v_ri1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 116, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_5 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_v_ri1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 116, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4 = __Pyx_PyObject_GetItem(__pyx_v_coords, __pyx_v_ri2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 116, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_10 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_v_ri2); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 116, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + + /* "fontTools/varLib/iup.py":115 + * i1, i2, ri1, ri2 = 0, start, start, indices[-1] + * out.extend( + * iup_segment( # <<<<<<<<<<<<<< + * coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] + * ) +*/ + __pyx_t_11 = __pyx_f_9fontTools_6varLib_3iup_iup_segment(__pyx_t_8, __pyx_t_6, __pyx_t_5, __pyx_t_4, __pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 115, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + + /* "fontTools/varLib/iup.py":114 + * # Initial segment that wraps around + * i1, i2, ri1, ri2 = 0, start, start, indices[-1] + * out.extend( # <<<<<<<<<<<<<< + * iup_segment( + * coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] +*/ + __pyx_t_12 = __Pyx_PyList_Extend(__pyx_v_out, __pyx_t_11); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "fontTools/varLib/iup.py":111 + * it = iter(indices) + * start = next(it) + * if start != 0: # <<<<<<<<<<<<<< + * # Initial segment that wraps around + * i1, i2, ri1, ri2 = 0, start, start, indices[-1] +*/ + } + + /* "fontTools/varLib/iup.py":119 + * ) + * ) + * out.append(deltas[start]) # <<<<<<<<<<<<<< + * for end in it: + * if end - start > 1: +*/ + __pyx_t_11 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_v_start); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 119, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_12 = __Pyx_PyList_Append(__pyx_v_out, __pyx_t_11); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 119, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "fontTools/varLib/iup.py":120 + * ) + * out.append(deltas[start]) + * for end in it: # <<<<<<<<<<<<<< + * if end - start > 1: + * i1, i2, ri1, ri2 = start + 1, end, start, end +*/ + if (likely(PyList_CheckExact(__pyx_v_it)) || PyTuple_CheckExact(__pyx_v_it)) { + __pyx_t_11 = __pyx_v_it; __Pyx_INCREF(__pyx_t_11); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_11 = PyObject_GetIter(__pyx_v_it); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 120, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_11); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 120, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_11))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_11); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 120, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_10 = __Pyx_PyList_GetItemRefFast(__pyx_t_11, __pyx_t_2, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_11); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 120, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_10 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_11, __pyx_t_2)); + #else + __pyx_t_10 = __Pyx_PySequence_ITEM(__pyx_t_11, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 120, __pyx_L1_error) + } else { + __pyx_t_10 = __pyx_t_7(__pyx_t_11); + if (unlikely(!__pyx_t_10)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 120, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_10); + __Pyx_XDECREF_SET(__pyx_v_end, __pyx_t_10); + __pyx_t_10 = 0; + + /* "fontTools/varLib/iup.py":121 + * out.append(deltas[start]) + * for end in it: + * if end - start > 1: # <<<<<<<<<<<<<< + * i1, i2, ri1, ri2 = start + 1, end, start, end + * out.extend( +*/ + __pyx_t_10 = PyNumber_Subtract(__pyx_v_end, __pyx_v_start); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 121, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_4 = PyObject_RichCompare(__pyx_t_10, __pyx_mstate_global->__pyx_int_1, Py_GT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 121, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 121, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (__pyx_t_9) { + + /* "fontTools/varLib/iup.py":122 + * for end in it: + * if end - start > 1: + * i1, i2, ri1, ri2 = start + 1, end, start, end # <<<<<<<<<<<<<< + * out.extend( + * iup_segment( +*/ + __pyx_t_4 = __Pyx_PyLong_AddObjC(__pyx_v_start, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 122, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_10 = __pyx_v_end; + __Pyx_INCREF(__pyx_t_10); + __pyx_t_5 = __pyx_v_start; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = __pyx_v_end; + __Pyx_INCREF(__pyx_t_6); + __Pyx_XDECREF_SET(__pyx_v_i1, __pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF_SET(__pyx_v_i2, __pyx_t_10); + __pyx_t_10 = 0; + __Pyx_XDECREF_SET(__pyx_v_ri1, __pyx_t_5); + __pyx_t_5 = 0; + __Pyx_XDECREF_SET(__pyx_v_ri2, __pyx_t_6); + __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":125 + * out.extend( + * iup_segment( + * coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] # <<<<<<<<<<<<<< + * ) + * ) +*/ + __pyx_t_6 = __Pyx_PyObject_GetSlice(__pyx_v_coords, 0, 0, &__pyx_v_i1, &__pyx_v_i2, NULL, 0, 0, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 125, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_5 = __Pyx_PyObject_GetItem(__pyx_v_coords, __pyx_v_ri1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 125, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_10 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_v_ri1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 125, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_4 = __Pyx_PyObject_GetItem(__pyx_v_coords, __pyx_v_ri2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 125, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_8 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_v_ri2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 125, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + + /* "fontTools/varLib/iup.py":124 + * i1, i2, ri1, ri2 = start + 1, end, start, end + * out.extend( + * iup_segment( # <<<<<<<<<<<<<< + * coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] + * ) +*/ + __pyx_t_13 = __pyx_f_9fontTools_6varLib_3iup_iup_segment(__pyx_t_6, __pyx_t_5, __pyx_t_10, __pyx_t_4, __pyx_t_8); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 124, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + + /* "fontTools/varLib/iup.py":123 + * if end - start > 1: + * i1, i2, ri1, ri2 = start + 1, end, start, end + * out.extend( # <<<<<<<<<<<<<< + * iup_segment( + * coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] +*/ + __pyx_t_12 = __Pyx_PyList_Extend(__pyx_v_out, __pyx_t_13); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 123, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + + /* "fontTools/varLib/iup.py":121 + * out.append(deltas[start]) + * for end in it: + * if end - start > 1: # <<<<<<<<<<<<<< + * i1, i2, ri1, ri2 = start + 1, end, start, end + * out.extend( +*/ + } + + /* "fontTools/varLib/iup.py":128 + * ) + * ) + * out.append(deltas[end]) # <<<<<<<<<<<<<< + * start = end + * if start != n - 1: +*/ + __pyx_t_13 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_v_end); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_12 = __Pyx_PyList_Append(__pyx_v_out, __pyx_t_13); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + + /* "fontTools/varLib/iup.py":129 + * ) + * out.append(deltas[end]) + * start = end # <<<<<<<<<<<<<< + * if start != n - 1: + * # Final segment that wraps around +*/ + __Pyx_INCREF(__pyx_v_end); + __Pyx_DECREF_SET(__pyx_v_start, __pyx_v_end); + + /* "fontTools/varLib/iup.py":120 + * ) + * out.append(deltas[start]) + * for end in it: # <<<<<<<<<<<<<< + * if end - start > 1: + * i1, i2, ri1, ri2 = start + 1, end, start, end +*/ + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "fontTools/varLib/iup.py":130 + * out.append(deltas[end]) + * start = end + * if start != n - 1: # <<<<<<<<<<<<<< + * # Final segment that wraps around + * i1, i2, ri1, ri2 = start + 1, n, start, indices[0] +*/ + __pyx_t_11 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 130, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_13 = PyObject_RichCompare(__pyx_v_start, __pyx_t_11, Py_NE); __Pyx_XGOTREF(__pyx_t_13); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 130, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_13); if (unlikely((__pyx_t_9 < 0))) __PYX_ERR(0, 130, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + if (__pyx_t_9) { + + /* "fontTools/varLib/iup.py":132 + * if start != n - 1: + * # Final segment that wraps around + * i1, i2, ri1, ri2 = start + 1, n, start, indices[0] # <<<<<<<<<<<<<< + * out.extend( + * iup_segment( +*/ + __pyx_t_13 = __Pyx_PyLong_AddObjC(__pyx_v_start, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 132, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_11 = __pyx_v_n; + __Pyx_INCREF(__pyx_t_11); + __pyx_t_8 = __pyx_v_start; + __Pyx_INCREF(__pyx_t_8); + __pyx_t_4 = __Pyx_GetItemInt_List(__pyx_v_indices, 0, long, 1, __Pyx_PyLong_From_long, 1, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 132, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_XDECREF_SET(__pyx_v_i1, __pyx_t_13); + __pyx_t_13 = 0; + __Pyx_XDECREF_SET(__pyx_v_i2, __pyx_t_11); + __pyx_t_11 = 0; + __Pyx_XDECREF_SET(__pyx_v_ri1, __pyx_t_8); + __pyx_t_8 = 0; + __Pyx_XDECREF_SET(__pyx_v_ri2, __pyx_t_4); + __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":135 + * out.extend( + * iup_segment( + * coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] # <<<<<<<<<<<<<< + * ) + * ) +*/ + __pyx_t_4 = __Pyx_PyObject_GetSlice(__pyx_v_coords, 0, 0, &__pyx_v_i1, &__pyx_v_i2, NULL, 0, 0, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 135, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_8 = __Pyx_PyObject_GetItem(__pyx_v_coords, __pyx_v_ri1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 135, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_v_ri1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 135, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_13 = __Pyx_PyObject_GetItem(__pyx_v_coords, __pyx_v_ri2); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 135, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_10 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_v_ri2); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 135, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + + /* "fontTools/varLib/iup.py":134 + * i1, i2, ri1, ri2 = start + 1, n, start, indices[0] + * out.extend( + * iup_segment( # <<<<<<<<<<<<<< + * coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] + * ) +*/ + __pyx_t_5 = __pyx_f_9fontTools_6varLib_3iup_iup_segment(__pyx_t_4, __pyx_t_8, __pyx_t_11, __pyx_t_13, __pyx_t_10); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 134, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + + /* "fontTools/varLib/iup.py":133 + * # Final segment that wraps around + * i1, i2, ri1, ri2 = start + 1, n, start, indices[0] + * out.extend( # <<<<<<<<<<<<<< + * iup_segment( + * coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] +*/ + __pyx_t_12 = __Pyx_PyList_Extend(__pyx_v_out, __pyx_t_5); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 133, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "fontTools/varLib/iup.py":130 + * out.append(deltas[end]) + * start = end + * if start != n - 1: # <<<<<<<<<<<<<< + * # Final segment that wraps around + * i1, i2, ri1, ri2 = start + 1, n, start, indices[0] +*/ + } + + /* "fontTools/varLib/iup.py":139 + * ) + * + * assert len(deltas) == len(out), (len(deltas), len(out)) # <<<<<<<<<<<<<< + * return out + * +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_2 = PyObject_Length(__pyx_v_deltas); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 139, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyList_GET_SIZE(__pyx_v_out); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(0, 139, __pyx_L1_error) + __pyx_t_9 = (__pyx_t_2 == __pyx_t_1); + if (unlikely(!__pyx_t_9)) { + __pyx_t_1 = PyObject_Length(__pyx_v_deltas); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(0, 139, __pyx_L1_error) + __pyx_t_5 = PyLong_FromSsize_t(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_1 = __Pyx_PyList_GET_SIZE(__pyx_v_out); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(0, 139, __pyx_L1_error) + __pyx_t_10 = PyLong_FromSsize_t(__pyx_t_1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_13 = PyTuple_New(2); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_13, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 139, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_10); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_13, 1, __pyx_t_10) != (0)) __PYX_ERR(0, 139, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_10 = 0; + __pyx_t_10 = PyTuple_Pack(1, __pyx_t_13); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), __pyx_t_10, 0, 0); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __PYX_ERR(0, 139, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 139, __pyx_L1_error) + #endif + + /* "fontTools/varLib/iup.py":140 + * + * assert len(deltas) == len(out), (len(deltas), len(out)) + * return out # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_out); + __pyx_r = __pyx_v_out; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":91 + * + * + * def iup_contour(deltas: _DeltaOrNoneSegment, coords: _PointSegment) -> _DeltaSegment: # <<<<<<<<<<<<<< + * """For the contour given in `coords`, interpolate any missing + * delta values in delta vector `deltas`. +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_AddTraceback("fontTools.varLib.iup.iup_contour", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_n); + __Pyx_XDECREF(__pyx_v_indices); + __Pyx_XDECREF(__pyx_v_out); + __Pyx_XDECREF(__pyx_v_it); + __Pyx_XDECREF(__pyx_v_start); + __Pyx_XDECREF(__pyx_v_i1); + __Pyx_XDECREF(__pyx_v_i2); + __Pyx_XDECREF(__pyx_v_ri1); + __Pyx_XDECREF(__pyx_v_ri2); + __Pyx_XDECREF(__pyx_v_end); + __Pyx_XDECREF(__pyx_7genexpr__pyx_v_i); + __Pyx_XDECREF(__pyx_7genexpr__pyx_v_v); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "fontTools/varLib/iup.py":143 + * + * + * def iup_delta( # <<<<<<<<<<<<<< + * deltas: _DeltaOrNoneSegment, coords: _PointSegment, ends: _Endpoints + * ) -> _DeltaSegment: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_3iup_delta(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_9fontTools_6varLib_3iup_2iup_delta, "iup_delta(deltas: _DeltaOrNoneSegment, coords: _PointSegment, ends: _Endpoints) -> _DeltaSegment\n\nFor the outline given in `coords`, with contour endpoints given\nin sorted increasing order in `ends`, interpolate any missing\ndelta values in delta vector `deltas`.\n\nReturns fully filled-out delta vector."); +static PyMethodDef __pyx_mdef_9fontTools_6varLib_3iup_3iup_delta = {"iup_delta", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_6varLib_3iup_3iup_delta, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_6varLib_3iup_2iup_delta}; +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_3iup_delta(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_deltas = 0; + PyObject *__pyx_v_coords = 0; + PyObject *__pyx_v_ends = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("iup_delta (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_deltas,&__pyx_mstate_global->__pyx_n_u_coords,&__pyx_mstate_global->__pyx_n_u_ends,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 143, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 143, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 143, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 143, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "iup_delta", 0) < (0)) __PYX_ERR(0, 143, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("iup_delta", 1, 3, 3, i); __PYX_ERR(0, 143, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 143, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 143, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 143, __pyx_L3_error) + } + __pyx_v_deltas = values[0]; + __pyx_v_coords = values[1]; + __pyx_v_ends = values[2]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("iup_delta", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 143, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("fontTools.varLib.iup.iup_delta", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_9fontTools_6varLib_3iup_2iup_delta(__pyx_self, __pyx_v_deltas, __pyx_v_coords, __pyx_v_ends); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_2iup_delta(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, PyObject *__pyx_v_ends) { + PyObject *__pyx_v_n = NULL; + PyObject *__pyx_v_out = NULL; + PyObject *__pyx_v_start = NULL; + PyObject *__pyx_v_end = NULL; + PyObject *__pyx_v_contour = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *(*__pyx_t_9)(PyObject *); + PyObject *__pyx_t_10 = NULL; + size_t __pyx_t_11; + int __pyx_t_12; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("iup_delta", 0); + __Pyx_INCREF(__pyx_v_ends); + + /* "fontTools/varLib/iup.py":152 + * Returns fully filled-out delta vector.""" + * + * assert sorted(ends) == ends and len(coords) == (ends[-1] + 1 if ends else 0) + 4 # <<<<<<<<<<<<<< + * n = len(coords) + * ends = ends + [n - 4, n - 3, n - 2, n - 1] +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_2 = PySequence_List(__pyx_v_ends); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (unlikely((PyList_Sort(__pyx_t_2) < 0))) __PYX_ERR(0, 152, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_2, __pyx_v_ends, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (__pyx_t_4) { + } else { + __pyx_t_1 = __pyx_t_4; + goto __pyx_L3_bool_binop_done; + } + __pyx_t_5 = PyObject_Length(__pyx_v_coords); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 152, __pyx_L1_error) + __pyx_t_3 = PyLong_FromSsize_t(__pyx_t_5); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_ends); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 152, __pyx_L1_error) + if (__pyx_t_4) { + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_ends, -1L, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyLong_AddObjC(__pyx_t_6, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_2 = __pyx_t_7; + __pyx_t_7 = 0; + } else { + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __pyx_t_2 = __pyx_mstate_global->__pyx_int_0; + } + __pyx_t_7 = __Pyx_PyLong_AddObjC(__pyx_t_2, __pyx_mstate_global->__pyx_int_4, 4, 0, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = PyObject_RichCompare(__pyx_t_3, __pyx_t_7, Py_EQ); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 152, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_1 = __pyx_t_4; + __pyx_L3_bool_binop_done:; + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 152, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 152, __pyx_L1_error) + #endif + + /* "fontTools/varLib/iup.py":153 + * + * assert sorted(ends) == ends and len(coords) == (ends[-1] + 1 if ends else 0) + 4 + * n = len(coords) # <<<<<<<<<<<<<< + * ends = ends + [n - 4, n - 3, n - 2, n - 1] + * out = [] +*/ + __pyx_t_5 = PyObject_Length(__pyx_v_coords); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 153, __pyx_L1_error) + __pyx_t_2 = PyLong_FromSsize_t(__pyx_t_5); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 153, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_n = __pyx_t_2; + __pyx_t_2 = 0; + + /* "fontTools/varLib/iup.py":154 + * assert sorted(ends) == ends and len(coords) == (ends[-1] + 1 if ends else 0) + 4 + * n = len(coords) + * ends = ends + [n - 4, n - 3, n - 2, n - 1] # <<<<<<<<<<<<<< + * out = [] + * start = 0 +*/ + __pyx_t_2 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_4, 4, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 154, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_3, 3, 0, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 154, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_3 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_2, 2, 0, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 154, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 154, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = PyList_New(4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 154, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyList_SET_ITEM(__pyx_t_8, 0, __pyx_t_2) != (0)) __PYX_ERR(0, 154, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyList_SET_ITEM(__pyx_t_8, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 154, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyList_SET_ITEM(__pyx_t_8, 2, __pyx_t_3) != (0)) __PYX_ERR(0, 154, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyList_SET_ITEM(__pyx_t_8, 3, __pyx_t_6) != (0)) __PYX_ERR(0, 154, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_t_7 = 0; + __pyx_t_3 = 0; + __pyx_t_6 = 0; + __pyx_t_6 = PyNumber_Add(__pyx_v_ends, __pyx_t_8); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 154, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF_SET(__pyx_v_ends, __pyx_t_6); + __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":155 + * n = len(coords) + * ends = ends + [n - 4, n - 3, n - 2, n - 1] + * out = [] # <<<<<<<<<<<<<< + * start = 0 + * for end in ends: +*/ + __pyx_t_6 = PyList_New(0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 155, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_v_out = ((PyObject*)__pyx_t_6); + __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":156 + * ends = ends + [n - 4, n - 3, n - 2, n - 1] + * out = [] + * start = 0 # <<<<<<<<<<<<<< + * for end in ends: + * end += 1 +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __pyx_v_start = __pyx_mstate_global->__pyx_int_0; + + /* "fontTools/varLib/iup.py":157 + * out = [] + * start = 0 + * for end in ends: # <<<<<<<<<<<<<< + * end += 1 + * contour = iup_contour(deltas[start:end], coords[start:end]) +*/ + if (likely(PyList_CheckExact(__pyx_v_ends)) || PyTuple_CheckExact(__pyx_v_ends)) { + __pyx_t_6 = __pyx_v_ends; __Pyx_INCREF(__pyx_t_6); + __pyx_t_5 = 0; + __pyx_t_9 = NULL; + } else { + __pyx_t_5 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_ends); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 157, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_9 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 157, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_9)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 157, __pyx_L1_error) + #endif + if (__pyx_t_5 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRefFast(__pyx_t_6, __pyx_t_5, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_5; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 157, __pyx_L1_error) + #endif + if (__pyx_t_5 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_5)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_5); + #endif + ++__pyx_t_5; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 157, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_9(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 157, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_end, __pyx_t_8); + __pyx_t_8 = 0; + + /* "fontTools/varLib/iup.py":158 + * start = 0 + * for end in ends: + * end += 1 # <<<<<<<<<<<<<< + * contour = iup_contour(deltas[start:end], coords[start:end]) + * out.extend(contour) +*/ + __pyx_t_8 = __Pyx_PyLong_AddObjC(__pyx_v_end, __pyx_mstate_global->__pyx_int_1, 1, 1, 0); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 158, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF_SET(__pyx_v_end, __pyx_t_8); + __pyx_t_8 = 0; + + /* "fontTools/varLib/iup.py":159 + * for end in ends: + * end += 1 + * contour = iup_contour(deltas[start:end], coords[start:end]) # <<<<<<<<<<<<<< + * out.extend(contour) + * start = end +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_iup_contour); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_2 = __Pyx_PyObject_GetSlice(__pyx_v_deltas, 0, 0, &__pyx_v_start, &__pyx_v_end, NULL, 0, 0, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_10 = __Pyx_PyObject_GetSlice(__pyx_v_coords, 0, 0, &__pyx_v_start, &__pyx_v_end, NULL, 0, 0, 1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_7))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_7); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_7); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_7, __pyx__function); + __pyx_t_11 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_t_2, __pyx_t_10}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_7, __pyx_callargs+__pyx_t_11, (3-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_XDECREF_SET(__pyx_v_contour, __pyx_t_8); + __pyx_t_8 = 0; + + /* "fontTools/varLib/iup.py":160 + * end += 1 + * contour = iup_contour(deltas[start:end], coords[start:end]) + * out.extend(contour) # <<<<<<<<<<<<<< + * start = end + * +*/ + __pyx_t_12 = __Pyx_PyList_Extend(__pyx_v_out, __pyx_v_contour); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 160, __pyx_L1_error) + + /* "fontTools/varLib/iup.py":161 + * contour = iup_contour(deltas[start:end], coords[start:end]) + * out.extend(contour) + * start = end # <<<<<<<<<<<<<< + * + * return out +*/ + __Pyx_INCREF(__pyx_v_end); + __Pyx_DECREF_SET(__pyx_v_start, __pyx_v_end); + + /* "fontTools/varLib/iup.py":157 + * out = [] + * start = 0 + * for end in ends: # <<<<<<<<<<<<<< + * end += 1 + * contour = iup_contour(deltas[start:end], coords[start:end]) +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":163 + * start = end + * + * return out # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_out); + __pyx_r = __pyx_v_out; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":143 + * + * + * def iup_delta( # <<<<<<<<<<<<<< + * deltas: _DeltaOrNoneSegment, coords: _PointSegment, ends: _Endpoints + * ) -> _DeltaSegment: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("fontTools.varLib.iup.iup_delta", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_n); + __Pyx_XDECREF(__pyx_v_out); + __Pyx_XDECREF(__pyx_v_start); + __Pyx_XDECREF(__pyx_v_end); + __Pyx_XDECREF(__pyx_v_contour); + __Pyx_XDECREF(__pyx_v_ends); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_9fontTools_6varLib_3iup_18can_iup_in_between_2generator(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value); /* proto */ + +/* "fontTools/varLib/iup.py":197 + * + * return all( + * abs(complex(x - p, y - q)) <= tolerance # <<<<<<<<<<<<<< + * for (x, y), (p, q) in zip(deltas, interp) + * ) +*/ + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_18can_iup_in_between_genexpr(PyObject *__pyx_self, PyObject *__pyx_genexpr_arg_0) { + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("genexpr", 0); + __pyx_cur_scope = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr(__pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 197, __pyx_L1_error) + } else { + __Pyx_GOTREF((PyObject *)__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_outer_scope = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *) __pyx_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_outer_scope); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_outer_scope); + __pyx_cur_scope->__pyx_genexpr_arg_0 = __pyx_genexpr_arg_0; + __Pyx_INCREF(__pyx_cur_scope->__pyx_genexpr_arg_0); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_genexpr_arg_0); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_9fontTools_6varLib_3iup_18can_iup_in_between_2generator, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0]), (PyObject *) __pyx_cur_scope, __pyx_mstate_global->__pyx_n_u_genexpr, __pyx_mstate_global->__pyx_n_u_can_iup_in_between_locals_genexp, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup); if (unlikely(!gen)) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("fontTools.varLib.iup.can_iup_in_between.genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF((PyObject *)__pyx_cur_scope); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_9fontTools_6varLib_3iup_18can_iup_in_between_2generator(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *__pyx_cur_scope = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + Py_ssize_t __pyx_t_2; + PyObject *(*__pyx_t_3)(PyObject *); + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *(*__pyx_t_8)(PyObject *); + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + double __pyx_t_11; + double __pyx_t_12; + size_t __pyx_t_13; + int __pyx_t_14; + int __pyx_t_15; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("genexpr", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 197, __pyx_L1_error) + + /* "fontTools/varLib/iup.py":198 + * return all( + * abs(complex(x - p, y - q)) <= tolerance + * for (x, y), (p, q) in zip(deltas, interp) # <<<<<<<<<<<<<< + * ) + * +*/ + if (unlikely(!__pyx_cur_scope->__pyx_genexpr_arg_0)) { __Pyx_RaiseUnboundLocalError(".0"); __PYX_ERR(0, 198, __pyx_L1_error) } + if (likely(PyList_CheckExact(__pyx_cur_scope->__pyx_genexpr_arg_0)) || PyTuple_CheckExact(__pyx_cur_scope->__pyx_genexpr_arg_0)) { + __pyx_t_1 = __pyx_cur_scope->__pyx_genexpr_arg_0; __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = 0; + __pyx_t_3 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_cur_scope->__pyx_genexpr_arg_0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 198, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_3)) { + if (likely(PyList_CheckExact(__pyx_t_1))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 198, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(__pyx_t_1, __pyx_t_2, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 198, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_4 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_2)); + #else + __pyx_t_4 = __Pyx_PySequence_ITEM(__pyx_t_1, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 198, __pyx_L1_error) + } else { + __pyx_t_4 = __pyx_t_3(__pyx_t_1); + if (unlikely(!__pyx_t_4)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 198, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_4); + if ((likely(PyTuple_CheckExact(__pyx_t_4))) || (PyList_CheckExact(__pyx_t_4))) { + PyObject* sequence = __pyx_t_4; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 198, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_5 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_6); + } else { + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_6); + } + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + #endif + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_7 = PyObject_GetIter(__pyx_t_4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_8 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_7); + index = 0; __pyx_t_5 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_5)) goto __pyx_L6_unpacking_failed; + __Pyx_GOTREF(__pyx_t_5); + index = 1; __pyx_t_6 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_6)) goto __pyx_L6_unpacking_failed; + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_7), 2) < (0)) __PYX_ERR(0, 198, __pyx_L1_error) + __pyx_t_8 = NULL; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L7_unpacking_done; + __pyx_L6_unpacking_failed:; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_8 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 198, __pyx_L1_error) + __pyx_L7_unpacking_done:; + } + if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { + PyObject* sequence = __pyx_t_5; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 198, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_7 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_7); + __pyx_t_9 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_9); + } else { + __pyx_t_7 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_7); + __pyx_t_9 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_9); + } + #else + __pyx_t_7 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_9 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + #endif + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_10 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_8 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_10); + index = 0; __pyx_t_7 = __pyx_t_8(__pyx_t_10); if (unlikely(!__pyx_t_7)) goto __pyx_L8_unpacking_failed; + __Pyx_GOTREF(__pyx_t_7); + index = 1; __pyx_t_9 = __pyx_t_8(__pyx_t_10); if (unlikely(!__pyx_t_9)) goto __pyx_L8_unpacking_failed; + __Pyx_GOTREF(__pyx_t_9); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_10), 2) < (0)) __PYX_ERR(0, 198, __pyx_L1_error) + __pyx_t_8 = NULL; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + goto __pyx_L9_unpacking_done; + __pyx_L8_unpacking_failed:; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_8 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 198, __pyx_L1_error) + __pyx_L9_unpacking_done:; + } + __pyx_t_11 = __Pyx_PyFloat_AsDouble(__pyx_t_7); if (unlikely((__pyx_t_11 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_12 = __Pyx_PyFloat_AsDouble(__pyx_t_9); if (unlikely((__pyx_t_12 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_cur_scope->__pyx_v_x = __pyx_t_11; + __pyx_cur_scope->__pyx_v_y = __pyx_t_12; + if ((likely(PyTuple_CheckExact(__pyx_t_6))) || (PyList_CheckExact(__pyx_t_6))) { + PyObject* sequence = __pyx_t_6; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 198, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_9 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_9); + __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_7); + } else { + __pyx_t_9 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_9); + __pyx_t_7 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_7); + } + #else + __pyx_t_9 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_7 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + #endif + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_10 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_8 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_10); + index = 0; __pyx_t_9 = __pyx_t_8(__pyx_t_10); if (unlikely(!__pyx_t_9)) goto __pyx_L10_unpacking_failed; + __Pyx_GOTREF(__pyx_t_9); + index = 1; __pyx_t_7 = __pyx_t_8(__pyx_t_10); if (unlikely(!__pyx_t_7)) goto __pyx_L10_unpacking_failed; + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_10), 2) < (0)) __PYX_ERR(0, 198, __pyx_L1_error) + __pyx_t_8 = NULL; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + goto __pyx_L11_unpacking_done; + __pyx_L10_unpacking_failed:; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_8 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 198, __pyx_L1_error) + __pyx_L11_unpacking_done:; + } + __pyx_t_12 = __Pyx_PyFloat_AsDouble(__pyx_t_9); if (unlikely((__pyx_t_12 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_11 = __Pyx_PyFloat_AsDouble(__pyx_t_7); if (unlikely((__pyx_t_11 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_cur_scope->__pyx_v_p = __pyx_t_12; + __pyx_cur_scope->__pyx_v_q = __pyx_t_11; + + /* "fontTools/varLib/iup.py":197 + * + * return all( + * abs(complex(x - p, y - q)) <= tolerance # <<<<<<<<<<<<<< + * for (x, y), (p, q) in zip(deltas, interp) + * ) +*/ + __pyx_t_6 = NULL; + __pyx_t_5 = PyFloat_FromDouble((__pyx_cur_scope->__pyx_v_x - __pyx_cur_scope->__pyx_v_p)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = PyFloat_FromDouble((__pyx_cur_scope->__pyx_v_y - __pyx_cur_scope->__pyx_v_q)); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_13 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_6, __pyx_t_5, __pyx_t_7}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(&PyComplex_Type), __pyx_callargs+__pyx_t_13, (3-__pyx_t_13) | (__pyx_t_13*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_7 = __Pyx_PyNumber_Absolute(__pyx_t_4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_tolerance)) { __Pyx_RaiseClosureNameError("tolerance"); __PYX_ERR(0, 197, __pyx_L1_error) } + __pyx_t_4 = PyObject_RichCompare(__pyx_t_7, __pyx_cur_scope->__pyx_outer_scope->__pyx_v_tolerance, Py_LE); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely((__pyx_t_14 < 0))) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_15 = (!__pyx_t_14); + if (__pyx_t_15) { + + /* "fontTools/varLib/iup.py":196 + * deltas = deltas[i + 1 : j] + * + * return all( # <<<<<<<<<<<<<< + * abs(complex(x - p, y - q)) <= tolerance + * for (x, y), (p, q) in zip(deltas, interp) +*/ + __Pyx_XDECREF(__pyx_r); + + /* "fontTools/varLib/iup.py":197 + * + * return all( + * abs(complex(x - p, y - q)) <= tolerance # <<<<<<<<<<<<<< + * for (x, y), (p, q) in zip(deltas, interp) + * ) +*/ + __Pyx_INCREF(Py_False); + __pyx_r = Py_False; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L0; + } + + /* "fontTools/varLib/iup.py":198 + * return all( + * abs(complex(x - p, y - q)) <= tolerance + * for (x, y), (p, q) in zip(deltas, interp) # <<<<<<<<<<<<<< + * ) + * +*/ + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + /*else*/ { + + /* "fontTools/varLib/iup.py":196 + * deltas = deltas[i + 1 : j] + * + * return all( # <<<<<<<<<<<<<< + * abs(complex(x - p, y - q)) <= tolerance + * for (x, y), (p, q) in zip(deltas, interp) +*/ + __Pyx_XDECREF(__pyx_r); + + /* "fontTools/varLib/iup.py":197 + * + * return all( + * abs(complex(x - p, y - q)) <= tolerance # <<<<<<<<<<<<<< + * for (x, y), (p, q) in zip(deltas, interp) + * ) +*/ + __Pyx_INCREF(Py_True); + __pyx_r = Py_True; + goto __pyx_L0; + } + CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + if (__Pyx_PyErr_Occurred()) { + __Pyx_Generator_Replace_StopIteration(0); + __Pyx_AddTraceback("genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + #if !CYTHON_USE_EXC_INFO_STACK + __Pyx_Coroutine_ResetAndClearException(__pyx_generator); + #endif + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "fontTools/varLib/iup.py":169 + * + * + * @cython.cfunc # <<<<<<<<<<<<<< + * @cython.inline + * @cython.locals( +*/ + +static CYTHON_INLINE int __pyx_f_9fontTools_6varLib_3iup_can_iup_in_between(PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, int __pyx_v_i, int __pyx_v_j, PyObject *__pyx_v_tolerance) { + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *__pyx_cur_scope; + PyObject *__pyx_v_interp = NULL; + PyObject *__pyx_gb_9fontTools_6varLib_3iup_18can_iup_in_between_2generator = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + size_t __pyx_t_8; + int __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("can_iup_in_between", 0); + __pyx_cur_scope = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between(__pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 169, __pyx_L1_error) + } else { + __Pyx_GOTREF((PyObject *)__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_tolerance = __pyx_v_tolerance; + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_tolerance); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_tolerance); + __Pyx_INCREF(__pyx_v_deltas); + + /* "fontTools/varLib/iup.py":192 + * provided error tolerance.""" + * + * assert j - i >= 2 # <<<<<<<<<<<<<< + * interp = iup_segment(coords[i + 1 : j], coords[i], deltas[i], coords[j], deltas[j]) + * deltas = deltas[i + 1 : j] +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = ((__pyx_v_j - __pyx_v_i) >= 2); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 192, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 192, __pyx_L1_error) + #endif + + /* "fontTools/varLib/iup.py":193 + * + * assert j - i >= 2 + * interp = iup_segment(coords[i + 1 : j], coords[i], deltas[i], coords[j], deltas[j]) # <<<<<<<<<<<<<< + * deltas = deltas[i + 1 : j] + * +*/ + __pyx_t_2 = __Pyx_PyObject_GetSlice(__pyx_v_coords, (__pyx_v_i + 1), __pyx_v_j, NULL, NULL, NULL, 1, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_GetItemInt(__pyx_v_coords, __pyx_v_i, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_deltas, __pyx_v_i, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_GetItemInt(__pyx_v_coords, __pyx_v_j, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_deltas, __pyx_v_j, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __pyx_f_9fontTools_6varLib_3iup_iup_segment(__pyx_t_2, __pyx_t_3, __pyx_t_4, __pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_v_interp = __pyx_t_7; + __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":194 + * assert j - i >= 2 + * interp = iup_segment(coords[i + 1 : j], coords[i], deltas[i], coords[j], deltas[j]) + * deltas = deltas[i + 1 : j] # <<<<<<<<<<<<<< + * + * return all( +*/ + __pyx_t_7 = __Pyx_PyObject_GetSlice(__pyx_v_deltas, (__pyx_v_i + 1), __pyx_v_j, NULL, NULL, NULL, 1, 1, 1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 194, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF_SET(__pyx_v_deltas, __pyx_t_7); + __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":198 + * return all( + * abs(complex(x - p, y - q)) <= tolerance + * for (x, y), (p, q) in zip(deltas, interp) # <<<<<<<<<<<<<< + * ) + * +*/ + __pyx_t_6 = NULL; + __pyx_t_8 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_6, __pyx_v_deltas, __pyx_v_interp}; + __pyx_t_7 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_zip, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + } + + /* "fontTools/varLib/iup.py":197 + * + * return all( + * abs(complex(x - p, y - q)) <= tolerance # <<<<<<<<<<<<<< + * for (x, y), (p, q) in zip(deltas, interp) + * ) +*/ + __pyx_t_6 = __pyx_pf_9fontTools_6varLib_3iup_18can_iup_in_between_genexpr(((PyObject*)__pyx_cur_scope), __pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = __Pyx_Generator_GetInlinedResult(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = __Pyx_PyLong_As_int(__pyx_t_7); if (unlikely((__pyx_t_9 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_r = __pyx_t_9; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":169 + * + * + * @cython.cfunc # <<<<<<<<<<<<<< + * @cython.inline + * @cython.locals( +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("fontTools.varLib.iup.can_iup_in_between", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_interp); + __Pyx_XDECREF(__pyx_gb_9fontTools_6varLib_3iup_18can_iup_in_between_2generator); + __Pyx_XDECREF(__pyx_v_deltas); + __Pyx_DECREF((PyObject *)__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "fontTools/varLib/iup.py":202 + * + * + * @cython.locals( # <<<<<<<<<<<<<< + * cj=cython.double, + * dj=cython.double, +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_5_iup_contour_bound_forced_set(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_9fontTools_6varLib_3iup_4_iup_contour_bound_forced_set, "_iup_contour_bound_forced_set(deltas: _DeltaSegment, coords: _PointSegment, tolerance: Real = 0) -> set\n\nThe forced set is a conservative set of points on the contour that must be encoded\nexplicitly (ie. cannot be interpolated). Calculating this set allows for significantly\nspeeding up the dynamic-programming, as well as resolve circularity in DP.\n\nThe set is precise; that is, if an index is in the returned set, then there is no way\nthat IUP can generate delta for that point, given `coords` and `deltas`."); +static PyMethodDef __pyx_mdef_9fontTools_6varLib_3iup_5_iup_contour_bound_forced_set = {"_iup_contour_bound_forced_set", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_6varLib_3iup_5_iup_contour_bound_forced_set, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_6varLib_3iup_4_iup_contour_bound_forced_set}; +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_5_iup_contour_bound_forced_set(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_deltas = 0; + PyObject *__pyx_v_coords = 0; + PyObject *__pyx_v_tolerance = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_iup_contour_bound_forced_set (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_deltas,&__pyx_mstate_global->__pyx_n_u_coords,&__pyx_mstate_global->__pyx_n_u_tolerance,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 202, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 202, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 202, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 202, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_iup_contour_bound_forced_set", 0) < (0)) __PYX_ERR(0, 202, __pyx_L3_error) + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_int_0))); + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_iup_contour_bound_forced_set", 0, 2, 3, i); __PYX_ERR(0, 202, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 202, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 202, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 202, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_int_0))); + } + __pyx_v_deltas = values[0]; + __pyx_v_coords = values[1]; + __pyx_v_tolerance = values[2]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_iup_contour_bound_forced_set", 0, 2, 3, __pyx_nargs); __PYX_ERR(0, 202, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("fontTools.varLib.iup._iup_contour_bound_forced_set", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_9fontTools_6varLib_3iup_4_iup_contour_bound_forced_set(__pyx_self, __pyx_v_deltas, __pyx_v_coords, __pyx_v_tolerance); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_4_iup_contour_bound_forced_set(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, PyObject *__pyx_v_tolerance) { + double __pyx_v_cj; + double __pyx_v_dj; + double __pyx_v_lcj; + double __pyx_v_ldj; + double __pyx_v_ncj; + double __pyx_v_ndj; + int __pyx_v_force; + PyObject *__pyx_v_forced = 0; + PyObject *__pyx_v_n = NULL; + PyObject *__pyx_v_i = NULL; + PyObject *__pyx_v_ld = NULL; + PyObject *__pyx_v_lc = NULL; + PyObject *__pyx_v_d = NULL; + PyObject *__pyx_v_c = NULL; + PyObject *__pyx_v_nd = NULL; + PyObject *__pyx_v_nc = NULL; + long __pyx_v_j; + double __pyx_v_c1; + double __pyx_v_c2; + double __pyx_v_d1; + double __pyx_v_d2; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + Py_ssize_t __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + PyObject *(*__pyx_t_8)(PyObject *); + PyObject *__pyx_t_9 = NULL; + long __pyx_t_10; + double __pyx_t_11; + double __pyx_t_12; + int __pyx_t_13; + double __pyx_t_14; + PyObject *__pyx_t_15 = NULL; + PyObject *__pyx_t_16 = NULL; + PyObject *__pyx_t_17 = NULL; + int __pyx_t_18; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_iup_contour_bound_forced_set", 0); + + /* "fontTools/varLib/iup.py":222 + * that IUP can generate delta for that point, given `coords` and `deltas`. + * """ + * assert len(deltas) == len(coords) # <<<<<<<<<<<<<< + * + * n = len(deltas) +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = PyObject_Length(__pyx_v_deltas); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(0, 222, __pyx_L1_error) + __pyx_t_2 = PyObject_Length(__pyx_v_coords); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 222, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_1 == __pyx_t_2); + if (unlikely(!__pyx_t_3)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 222, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 222, __pyx_L1_error) + #endif + + /* "fontTools/varLib/iup.py":224 + * assert len(deltas) == len(coords) + * + * n = len(deltas) # <<<<<<<<<<<<<< + * forced = set() + * # Track "last" and "next" points on the contour as we sweep. +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_deltas); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 224, __pyx_L1_error) + __pyx_t_4 = PyLong_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 224, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_n = __pyx_t_4; + __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":225 + * + * n = len(deltas) + * forced = set() # <<<<<<<<<<<<<< + * # Track "last" and "next" points on the contour as we sweep. + * for i in range(len(deltas) - 1, -1, -1): +*/ + __pyx_t_4 = PySet_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 225, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_forced = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":227 + * forced = set() + * # Track "last" and "next" points on the contour as we sweep. + * for i in range(len(deltas) - 1, -1, -1): # <<<<<<<<<<<<<< + * ld, lc = deltas[i - 1], coords[i - 1] + * d, c = deltas[i], coords[i] +*/ + __pyx_t_5 = NULL; + __pyx_t_2 = PyObject_Length(__pyx_v_deltas); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 227, __pyx_L1_error) + __pyx_t_6 = PyLong_FromSsize_t((__pyx_t_2 - 1)); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 227, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_5, __pyx_t_6, __pyx_mstate_global->__pyx_int_neg_1, __pyx_mstate_global->__pyx_int_neg_1}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(&PyRange_Type), __pyx_callargs+__pyx_t_7, (4-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 227, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_6 = PyObject_GetIter(__pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 227, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 227, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + for (;;) { + { + __pyx_t_4 = __pyx_t_8(__pyx_t_6); + if (unlikely(!__pyx_t_4)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 227, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_4); + __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_4); + __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":228 + * # Track "last" and "next" points on the contour as we sweep. + * for i in range(len(deltas) - 1, -1, -1): + * ld, lc = deltas[i - 1], coords[i - 1] # <<<<<<<<<<<<<< + * d, c = deltas[i], coords[i] + * nd, nc = deltas[i - n + 1], coords[i - n + 1] +*/ + __pyx_t_4 = __Pyx_PyLong_SubtractObjC(__pyx_v_i, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyLong_SubtractObjC(__pyx_v_i, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_9 = __Pyx_PyObject_GetItem(__pyx_v_coords, __pyx_t_4); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF_SET(__pyx_v_ld, __pyx_t_5); + __pyx_t_5 = 0; + __Pyx_XDECREF_SET(__pyx_v_lc, __pyx_t_9); + __pyx_t_9 = 0; + + /* "fontTools/varLib/iup.py":229 + * for i in range(len(deltas) - 1, -1, -1): + * ld, lc = deltas[i - 1], coords[i - 1] + * d, c = deltas[i], coords[i] # <<<<<<<<<<<<<< + * nd, nc = deltas[i - n + 1], coords[i - n + 1] + * +*/ + __pyx_t_9 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_v_i); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 229, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_5 = __Pyx_PyObject_GetItem(__pyx_v_coords, __pyx_v_i); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 229, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_d, __pyx_t_9); + __pyx_t_9 = 0; + __Pyx_XDECREF_SET(__pyx_v_c, __pyx_t_5); + __pyx_t_5 = 0; + + /* "fontTools/varLib/iup.py":230 + * ld, lc = deltas[i - 1], coords[i - 1] + * d, c = deltas[i], coords[i] + * nd, nc = deltas[i - n + 1], coords[i - n + 1] # <<<<<<<<<<<<<< + * + * for j in (0, 1): # For X and for Y +*/ + __pyx_t_5 = PyNumber_Subtract(__pyx_v_i, __pyx_v_n); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_9 = __Pyx_PyLong_AddObjC(__pyx_t_5, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_t_9); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = PyNumber_Subtract(__pyx_v_i, __pyx_v_n); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_4 = __Pyx_PyLong_AddObjC(__pyx_t_9, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = __Pyx_PyObject_GetItem(__pyx_v_coords, __pyx_t_4); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF_SET(__pyx_v_nd, __pyx_t_5); + __pyx_t_5 = 0; + __Pyx_XDECREF_SET(__pyx_v_nc, __pyx_t_9); + __pyx_t_9 = 0; + + /* "fontTools/varLib/iup.py":232 + * nd, nc = deltas[i - n + 1], coords[i - n + 1] + * + * for j in (0, 1): # For X and for Y # <<<<<<<<<<<<<< + * cj = c[j] + * dj = d[j] +*/ + __pyx_t_9 = __pyx_mstate_global->__pyx_tuple[0]; __Pyx_INCREF(__pyx_t_9); + __pyx_t_2 = 0; + for (;;) { + if (__pyx_t_2 >= 2) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_9, __pyx_t_2)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_9, __pyx_t_2); + #endif + ++__pyx_t_2; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 232, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_10 = __Pyx_PyLong_As_long(__pyx_t_5); if (unlikely((__pyx_t_10 == (long)-1) && PyErr_Occurred())) __PYX_ERR(0, 232, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_j = __pyx_t_10; + + /* "fontTools/varLib/iup.py":233 + * + * for j in (0, 1): # For X and for Y + * cj = c[j] # <<<<<<<<<<<<<< + * dj = d[j] + * lcj = lc[j] +*/ + __pyx_t_5 = __Pyx_GetItemInt(__pyx_v_c, __pyx_v_j, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 233, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_11 = __Pyx_PyFloat_AsDouble(__pyx_t_5); if (unlikely((__pyx_t_11 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 233, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_cj = __pyx_t_11; + + /* "fontTools/varLib/iup.py":234 + * for j in (0, 1): # For X and for Y + * cj = c[j] + * dj = d[j] # <<<<<<<<<<<<<< + * lcj = lc[j] + * ldj = ld[j] +*/ + __pyx_t_5 = __Pyx_GetItemInt(__pyx_v_d, __pyx_v_j, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 234, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_11 = __Pyx_PyFloat_AsDouble(__pyx_t_5); if (unlikely((__pyx_t_11 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 234, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_dj = __pyx_t_11; + + /* "fontTools/varLib/iup.py":235 + * cj = c[j] + * dj = d[j] + * lcj = lc[j] # <<<<<<<<<<<<<< + * ldj = ld[j] + * ncj = nc[j] +*/ + __pyx_t_5 = __Pyx_GetItemInt(__pyx_v_lc, __pyx_v_j, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 235, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_11 = __Pyx_PyFloat_AsDouble(__pyx_t_5); if (unlikely((__pyx_t_11 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 235, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_lcj = __pyx_t_11; + + /* "fontTools/varLib/iup.py":236 + * dj = d[j] + * lcj = lc[j] + * ldj = ld[j] # <<<<<<<<<<<<<< + * ncj = nc[j] + * ndj = nd[j] +*/ + __pyx_t_5 = __Pyx_GetItemInt(__pyx_v_ld, __pyx_v_j, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 236, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_11 = __Pyx_PyFloat_AsDouble(__pyx_t_5); if (unlikely((__pyx_t_11 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 236, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_ldj = __pyx_t_11; + + /* "fontTools/varLib/iup.py":237 + * lcj = lc[j] + * ldj = ld[j] + * ncj = nc[j] # <<<<<<<<<<<<<< + * ndj = nd[j] + * +*/ + __pyx_t_5 = __Pyx_GetItemInt(__pyx_v_nc, __pyx_v_j, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 237, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_11 = __Pyx_PyFloat_AsDouble(__pyx_t_5); if (unlikely((__pyx_t_11 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 237, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_ncj = __pyx_t_11; + + /* "fontTools/varLib/iup.py":238 + * ldj = ld[j] + * ncj = nc[j] + * ndj = nd[j] # <<<<<<<<<<<<<< + * + * if lcj <= ncj: +*/ + __pyx_t_5 = __Pyx_GetItemInt(__pyx_v_nd, __pyx_v_j, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 238, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_11 = __Pyx_PyFloat_AsDouble(__pyx_t_5); if (unlikely((__pyx_t_11 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 238, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_ndj = __pyx_t_11; + + /* "fontTools/varLib/iup.py":240 + * ndj = nd[j] + * + * if lcj <= ncj: # <<<<<<<<<<<<<< + * c1, c2 = lcj, ncj + * d1, d2 = ldj, ndj +*/ + __pyx_t_3 = (__pyx_v_lcj <= __pyx_v_ncj); + if (__pyx_t_3) { + + /* "fontTools/varLib/iup.py":241 + * + * if lcj <= ncj: + * c1, c2 = lcj, ncj # <<<<<<<<<<<<<< + * d1, d2 = ldj, ndj + * else: +*/ + __pyx_t_11 = __pyx_v_lcj; + __pyx_t_12 = __pyx_v_ncj; + __pyx_v_c1 = __pyx_t_11; + __pyx_v_c2 = __pyx_t_12; + + /* "fontTools/varLib/iup.py":242 + * if lcj <= ncj: + * c1, c2 = lcj, ncj + * d1, d2 = ldj, ndj # <<<<<<<<<<<<<< + * else: + * c1, c2 = ncj, lcj +*/ + __pyx_t_12 = __pyx_v_ldj; + __pyx_t_11 = __pyx_v_ndj; + __pyx_v_d1 = __pyx_t_12; + __pyx_v_d2 = __pyx_t_11; + + /* "fontTools/varLib/iup.py":240 + * ndj = nd[j] + * + * if lcj <= ncj: # <<<<<<<<<<<<<< + * c1, c2 = lcj, ncj + * d1, d2 = ldj, ndj +*/ + goto __pyx_L7; + } + + /* "fontTools/varLib/iup.py":244 + * d1, d2 = ldj, ndj + * else: + * c1, c2 = ncj, lcj # <<<<<<<<<<<<<< + * d1, d2 = ndj, ldj + * +*/ + /*else*/ { + __pyx_t_11 = __pyx_v_ncj; + __pyx_t_12 = __pyx_v_lcj; + __pyx_v_c1 = __pyx_t_11; + __pyx_v_c2 = __pyx_t_12; + + /* "fontTools/varLib/iup.py":245 + * else: + * c1, c2 = ncj, lcj + * d1, d2 = ndj, ldj # <<<<<<<<<<<<<< + * + * force = False +*/ + __pyx_t_12 = __pyx_v_ndj; + __pyx_t_11 = __pyx_v_ldj; + __pyx_v_d1 = __pyx_t_12; + __pyx_v_d2 = __pyx_t_11; + } + __pyx_L7:; + + /* "fontTools/varLib/iup.py":247 + * d1, d2 = ndj, ldj + * + * force = False # <<<<<<<<<<<<<< + * + * # If the two coordinates are the same, then the interpolation +*/ + __pyx_v_force = 0; + + /* "fontTools/varLib/iup.py":254 + * # + * # This test has to be before the next one. + * if c1 == c2: # <<<<<<<<<<<<<< + * if abs(d1 - d2) > tolerance and abs(dj) > tolerance: + * force = True +*/ + __pyx_t_3 = (__pyx_v_c1 == __pyx_v_c2); + if (__pyx_t_3) { + + /* "fontTools/varLib/iup.py":255 + * # This test has to be before the next one. + * if c1 == c2: + * if abs(d1 - d2) > tolerance and abs(dj) > tolerance: # <<<<<<<<<<<<<< + * force = True + * +*/ + __pyx_t_11 = fabs((__pyx_v_d1 - __pyx_v_d2)); + __pyx_t_5 = PyFloat_FromDouble(__pyx_t_11); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4 = PyObject_RichCompare(__pyx_t_5, __pyx_v_tolerance, Py_GT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely((__pyx_t_13 < 0))) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (__pyx_t_13) { + } else { + __pyx_t_3 = __pyx_t_13; + goto __pyx_L10_bool_binop_done; + } + __pyx_t_11 = fabs(__pyx_v_dj); + __pyx_t_4 = PyFloat_FromDouble(__pyx_t_11); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyObject_RichCompare(__pyx_t_4, __pyx_v_tolerance, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_13 < 0))) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_3 = __pyx_t_13; + __pyx_L10_bool_binop_done:; + if (__pyx_t_3) { + + /* "fontTools/varLib/iup.py":256 + * if c1 == c2: + * if abs(d1 - d2) > tolerance and abs(dj) > tolerance: + * force = True # <<<<<<<<<<<<<< + * + * # If coordinate for current point is between coordinate of adjacent +*/ + __pyx_v_force = 1; + + /* "fontTools/varLib/iup.py":255 + * # This test has to be before the next one. + * if c1 == c2: + * if abs(d1 - d2) > tolerance and abs(dj) > tolerance: # <<<<<<<<<<<<<< + * force = True + * +*/ + } + + /* "fontTools/varLib/iup.py":254 + * # + * # This test has to be before the next one. + * if c1 == c2: # <<<<<<<<<<<<<< + * if abs(d1 - d2) > tolerance and abs(dj) > tolerance: + * force = True +*/ + goto __pyx_L8; + } + + /* "fontTools/varLib/iup.py":263 + * # allowance), then there is no way that current point can be IUP-ed. + * # Mark it forced. + * elif c1 <= cj <= c2: # and c1 != c2 # <<<<<<<<<<<<<< + * if not (min(d1, d2) - tolerance <= dj <= max(d1, d2) + tolerance): + * force = True +*/ + __pyx_t_3 = (__pyx_v_c1 <= __pyx_v_cj); + if (__pyx_t_3) { + __pyx_t_3 = (__pyx_v_cj <= __pyx_v_c2); + } + if (__pyx_t_3) { + + /* "fontTools/varLib/iup.py":264 + * # Mark it forced. + * elif c1 <= cj <= c2: # and c1 != c2 + * if not (min(d1, d2) - tolerance <= dj <= max(d1, d2) + tolerance): # <<<<<<<<<<<<<< + * force = True + * +*/ + __pyx_t_11 = __pyx_v_d2; + __pyx_t_12 = __pyx_v_d1; + __pyx_t_3 = (__pyx_t_11 < __pyx_t_12); + if (__pyx_t_3) { + __pyx_t_14 = __pyx_t_11; + } else { + __pyx_t_14 = __pyx_t_12; + } + __pyx_t_5 = PyFloat_FromDouble(__pyx_t_14); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 264, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4 = PyNumber_Subtract(__pyx_t_5, __pyx_v_tolerance); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 264, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = PyFloat_FromDouble(__pyx_v_dj); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 264, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_15 = PyObject_RichCompare(__pyx_t_4, __pyx_t_5, Py_LE); __Pyx_XGOTREF(__pyx_t_15); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 264, __pyx_L1_error) + if (__Pyx_PyObject_IsTrue(__pyx_t_15)) { + __Pyx_DECREF(__pyx_t_15); + __pyx_t_14 = __pyx_v_d2; + __pyx_t_11 = __pyx_v_d1; + __pyx_t_3 = (__pyx_t_14 > __pyx_t_11); + if (__pyx_t_3) { + __pyx_t_12 = __pyx_t_14; + } else { + __pyx_t_12 = __pyx_t_11; + } + __pyx_t_16 = PyFloat_FromDouble(__pyx_t_12); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 264, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_17 = PyNumber_Add(__pyx_t_16, __pyx_v_tolerance); if (unlikely(!__pyx_t_17)) __PYX_ERR(0, 264, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_17); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __pyx_t_15 = PyObject_RichCompare(__pyx_t_5, __pyx_t_17, Py_LE); __Pyx_XGOTREF(__pyx_t_15); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 264, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_15); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 264, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_13 = (!__pyx_t_3); + if (__pyx_t_13) { + + /* "fontTools/varLib/iup.py":265 + * elif c1 <= cj <= c2: # and c1 != c2 + * if not (min(d1, d2) - tolerance <= dj <= max(d1, d2) + tolerance): + * force = True # <<<<<<<<<<<<<< + * + * # Otherwise, the delta should either match the closest, or have the +*/ + __pyx_v_force = 1; + + /* "fontTools/varLib/iup.py":264 + * # Mark it forced. + * elif c1 <= cj <= c2: # and c1 != c2 + * if not (min(d1, d2) - tolerance <= dj <= max(d1, d2) + tolerance): # <<<<<<<<<<<<<< + * force = True + * +*/ + } + + /* "fontTools/varLib/iup.py":263 + * # allowance), then there is no way that current point can be IUP-ed. + * # Mark it forced. + * elif c1 <= cj <= c2: # and c1 != c2 # <<<<<<<<<<<<<< + * if not (min(d1, d2) - tolerance <= dj <= max(d1, d2) + tolerance): + * force = True +*/ + goto __pyx_L8; + } + + /* "fontTools/varLib/iup.py":270 + * # same sign as the interpolation of the two deltas. + * else: # cj < c1 or c2 < cj + * if d1 != d2: # <<<<<<<<<<<<<< + * if cj < c1: + * if ( +*/ + /*else*/ { + __pyx_t_13 = (__pyx_v_d1 != __pyx_v_d2); + if (__pyx_t_13) { + + /* "fontTools/varLib/iup.py":271 + * else: # cj < c1 or c2 < cj + * if d1 != d2: + * if cj < c1: # <<<<<<<<<<<<<< + * if ( + * abs(dj) > tolerance +*/ + __pyx_t_13 = (__pyx_v_cj < __pyx_v_c1); + if (__pyx_t_13) { + + /* "fontTools/varLib/iup.py":273 + * if cj < c1: + * if ( + * abs(dj) > tolerance # <<<<<<<<<<<<<< + * and abs(dj - d1) > tolerance + * and ((dj - tolerance < d1) != (d1 < d2)) +*/ + __pyx_t_12 = fabs(__pyx_v_dj); + __pyx_t_15 = PyFloat_FromDouble(__pyx_t_12); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_5 = PyObject_RichCompare(__pyx_t_15, __pyx_v_tolerance, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 273, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 273, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_13 = __pyx_t_3; + goto __pyx_L16_bool_binop_done; + } + + /* "fontTools/varLib/iup.py":274 + * if ( + * abs(dj) > tolerance + * and abs(dj - d1) > tolerance # <<<<<<<<<<<<<< + * and ((dj - tolerance < d1) != (d1 < d2)) + * ): +*/ + __pyx_t_12 = fabs((__pyx_v_dj - __pyx_v_d1)); + __pyx_t_5 = PyFloat_FromDouble(__pyx_t_12); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 274, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_15 = PyObject_RichCompare(__pyx_t_5, __pyx_v_tolerance, Py_GT); __Pyx_XGOTREF(__pyx_t_15); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 274, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_15); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 274, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_13 = __pyx_t_3; + goto __pyx_L16_bool_binop_done; + } + + /* "fontTools/varLib/iup.py":275 + * abs(dj) > tolerance + * and abs(dj - d1) > tolerance + * and ((dj - tolerance < d1) != (d1 < d2)) # <<<<<<<<<<<<<< + * ): + * force = True +*/ + __pyx_t_15 = PyFloat_FromDouble(__pyx_v_dj); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 275, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_5 = PyNumber_Subtract(__pyx_t_15, __pyx_v_tolerance); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 275, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = PyFloat_FromDouble(__pyx_v_d1); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 275, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_4 = PyObject_RichCompare(__pyx_t_5, __pyx_t_15, Py_LT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 275, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = __Pyx_PyBool_FromLong((__pyx_v_d1 < __pyx_v_d2)); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 275, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_5 = PyObject_RichCompare(__pyx_t_4, __pyx_t_15, Py_NE); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 275, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 275, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_13 = __pyx_t_3; + __pyx_L16_bool_binop_done:; + + /* "fontTools/varLib/iup.py":272 + * if d1 != d2: + * if cj < c1: + * if ( # <<<<<<<<<<<<<< + * abs(dj) > tolerance + * and abs(dj - d1) > tolerance +*/ + if (__pyx_t_13) { + + /* "fontTools/varLib/iup.py":277 + * and ((dj - tolerance < d1) != (d1 < d2)) + * ): + * force = True # <<<<<<<<<<<<<< + * else: # c2 < cj + * if ( +*/ + __pyx_v_force = 1; + + /* "fontTools/varLib/iup.py":272 + * if d1 != d2: + * if cj < c1: + * if ( # <<<<<<<<<<<<<< + * abs(dj) > tolerance + * and abs(dj - d1) > tolerance +*/ + } + + /* "fontTools/varLib/iup.py":271 + * else: # cj < c1 or c2 < cj + * if d1 != d2: + * if cj < c1: # <<<<<<<<<<<<<< + * if ( + * abs(dj) > tolerance +*/ + goto __pyx_L14; + } + + /* "fontTools/varLib/iup.py":279 + * force = True + * else: # c2 < cj + * if ( # <<<<<<<<<<<<<< + * abs(dj) > tolerance + * and abs(dj - d2) > tolerance +*/ + /*else*/ { + + /* "fontTools/varLib/iup.py":280 + * else: # c2 < cj + * if ( + * abs(dj) > tolerance # <<<<<<<<<<<<<< + * and abs(dj - d2) > tolerance + * and ((d2 < dj + tolerance) != (d1 < d2)) +*/ + __pyx_t_12 = fabs(__pyx_v_dj); + __pyx_t_5 = PyFloat_FromDouble(__pyx_t_12); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 280, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_15 = PyObject_RichCompare(__pyx_t_5, __pyx_v_tolerance, Py_GT); __Pyx_XGOTREF(__pyx_t_15); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 280, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_15); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 280, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_13 = __pyx_t_3; + goto __pyx_L20_bool_binop_done; + } + + /* "fontTools/varLib/iup.py":281 + * if ( + * abs(dj) > tolerance + * and abs(dj - d2) > tolerance # <<<<<<<<<<<<<< + * and ((d2 < dj + tolerance) != (d1 < d2)) + * ): +*/ + __pyx_t_12 = fabs((__pyx_v_dj - __pyx_v_d2)); + __pyx_t_15 = PyFloat_FromDouble(__pyx_t_12); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 281, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_5 = PyObject_RichCompare(__pyx_t_15, __pyx_v_tolerance, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 281, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 281, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_13 = __pyx_t_3; + goto __pyx_L20_bool_binop_done; + } + + /* "fontTools/varLib/iup.py":282 + * abs(dj) > tolerance + * and abs(dj - d2) > tolerance + * and ((d2 < dj + tolerance) != (d1 < d2)) # <<<<<<<<<<<<<< + * ): + * force = True +*/ + __pyx_t_5 = PyFloat_FromDouble(__pyx_v_d2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 282, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_15 = PyFloat_FromDouble(__pyx_v_dj); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 282, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_4 = PyNumber_Add(__pyx_t_15, __pyx_v_tolerance); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 282, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = PyObject_RichCompare(__pyx_t_5, __pyx_t_4, Py_LT); __Pyx_XGOTREF(__pyx_t_15); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 282, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyBool_FromLong((__pyx_v_d1 < __pyx_v_d2)); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 282, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyObject_RichCompare(__pyx_t_15, __pyx_t_4, Py_NE); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 282, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 282, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_13 = __pyx_t_3; + __pyx_L20_bool_binop_done:; + + /* "fontTools/varLib/iup.py":279 + * force = True + * else: # c2 < cj + * if ( # <<<<<<<<<<<<<< + * abs(dj) > tolerance + * and abs(dj - d2) > tolerance +*/ + if (__pyx_t_13) { + + /* "fontTools/varLib/iup.py":284 + * and ((d2 < dj + tolerance) != (d1 < d2)) + * ): + * force = True # <<<<<<<<<<<<<< + * + * if force: +*/ + __pyx_v_force = 1; + + /* "fontTools/varLib/iup.py":279 + * force = True + * else: # c2 < cj + * if ( # <<<<<<<<<<<<<< + * abs(dj) > tolerance + * and abs(dj - d2) > tolerance +*/ + } + } + __pyx_L14:; + + /* "fontTools/varLib/iup.py":270 + * # same sign as the interpolation of the two deltas. + * else: # cj < c1 or c2 < cj + * if d1 != d2: # <<<<<<<<<<<<<< + * if cj < c1: + * if ( +*/ + } + } + __pyx_L8:; + + /* "fontTools/varLib/iup.py":286 + * force = True + * + * if force: # <<<<<<<<<<<<<< + * forced.add(i) + * break +*/ + __pyx_t_13 = (__pyx_v_force != 0); + if (__pyx_t_13) { + + /* "fontTools/varLib/iup.py":287 + * + * if force: + * forced.add(i) # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_t_18 = PySet_Add(__pyx_v_forced, __pyx_v_i); if (unlikely(__pyx_t_18 == ((int)-1))) __PYX_ERR(0, 287, __pyx_L1_error) + + /* "fontTools/varLib/iup.py":288 + * if force: + * forced.add(i) + * break # <<<<<<<<<<<<<< + * + * return forced +*/ + goto __pyx_L6_break; + + /* "fontTools/varLib/iup.py":286 + * force = True + * + * if force: # <<<<<<<<<<<<<< + * forced.add(i) + * break +*/ + } + + /* "fontTools/varLib/iup.py":232 + * nd, nc = deltas[i - n + 1], coords[i - n + 1] + * + * for j in (0, 1): # For X and for Y # <<<<<<<<<<<<<< + * cj = c[j] + * dj = d[j] +*/ + } + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + goto __pyx_L24_for_end; + __pyx_L6_break:; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + goto __pyx_L24_for_end; + __pyx_L24_for_end:; + + /* "fontTools/varLib/iup.py":227 + * forced = set() + * # Track "last" and "next" points on the contour as we sweep. + * for i in range(len(deltas) - 1, -1, -1): # <<<<<<<<<<<<<< + * ld, lc = deltas[i - 1], coords[i - 1] + * d, c = deltas[i], coords[i] +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":290 + * break + * + * return forced # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_forced); + __pyx_r = __pyx_v_forced; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":202 + * + * + * @cython.locals( # <<<<<<<<<<<<<< + * cj=cython.double, + * dj=cython.double, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_15); + __Pyx_XDECREF(__pyx_t_16); + __Pyx_XDECREF(__pyx_t_17); + __Pyx_AddTraceback("fontTools.varLib.iup._iup_contour_bound_forced_set", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_forced); + __Pyx_XDECREF(__pyx_v_n); + __Pyx_XDECREF(__pyx_v_i); + __Pyx_XDECREF(__pyx_v_ld); + __Pyx_XDECREF(__pyx_v_lc); + __Pyx_XDECREF(__pyx_v_d); + __Pyx_XDECREF(__pyx_v_c); + __Pyx_XDECREF(__pyx_v_nd); + __Pyx_XDECREF(__pyx_v_nc); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "fontTools/varLib/iup.py":293 + * + * + * @cython.locals( # <<<<<<<<<<<<<< + * i=cython.int, + * j=cython.int, +*/ + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_16__defaults__(CYTHON_UNUSED PyObject *__pyx_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__defaults__", 0); + __Pyx_XDECREF(__pyx_r); + + /* "fontTools/varLib/iup.py":306 + * coords: _PointSegment, + * forced=set(), + * tolerance: Real = 0, # <<<<<<<<<<<<<< + * lookback: Integral = None, + * ): +*/ + __pyx_t_1 = PyFloat_FromDouble(((double)0.0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 306, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + + /* "fontTools/varLib/iup.py":293 + * + * + * @cython.locals( # <<<<<<<<<<<<<< + * i=cython.int, + * j=cython.int, +*/ + __pyx_t_2 = PyTuple_New(3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 293, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self)->arg0); + __Pyx_GIVEREF(__Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self)->arg0); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self)->arg0) != (0)) __PYX_ERR(0, 293, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_1) != (0)) __PYX_ERR(0, 293, __pyx_L1_error); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 2, Py_None) != (0)) __PYX_ERR(0, 293, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 293, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_2) != (0)) __PYX_ERR(0, 293, __pyx_L1_error); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, Py_None) != (0)) __PYX_ERR(0, 293, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("fontTools.varLib.iup.__defaults__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_7_iup_contour_optimize_dp(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_9fontTools_6varLib_3iup_6_iup_contour_optimize_dp, "_iup_contour_optimize_dp(deltas: _DeltaSegment, coords: _PointSegment, set forced=set(), double tolerance: Real = 0, lookback: Integral = None)\n\nStraightforward Dynamic-Programming. For each index i, find least-costly encoding of\npoints 0 to i where i is explicitly encoded. We find this by considering all previous\nexplicit points j and check whether interpolation can fill points between j and i.\n\nNote that solution always encodes last point explicitly. Higher-level is responsible\nfor removing that restriction.\n\nAs major speedup, we stop looking further whenever we see a \"forced\" point."); +static PyMethodDef __pyx_mdef_9fontTools_6varLib_3iup_7_iup_contour_optimize_dp = {"_iup_contour_optimize_dp", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_6varLib_3iup_7_iup_contour_optimize_dp, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_6varLib_3iup_6_iup_contour_optimize_dp}; +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_7_iup_contour_optimize_dp(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_deltas = 0; + PyObject *__pyx_v_coords = 0; + PyObject *__pyx_v_forced = 0; + double __pyx_v_tolerance; + PyObject *__pyx_v_lookback = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_iup_contour_optimize_dp (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_deltas,&__pyx_mstate_global->__pyx_n_u_coords,&__pyx_mstate_global->__pyx_n_u_forced,&__pyx_mstate_global->__pyx_n_u_tolerance,&__pyx_mstate_global->__pyx_n_u_lookback,0}; + struct __pyx_defaults *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 293, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 293, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 293, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 293, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 293, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 293, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_iup_contour_optimize_dp", 0) < (0)) __PYX_ERR(0, 293, __pyx_L3_error) + if (!values[2]) values[2] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + + /* "fontTools/varLib/iup.py":307 + * forced=set(), + * tolerance: Real = 0, + * lookback: Integral = None, # <<<<<<<<<<<<<< + * ): + * """Straightforward Dynamic-Programming. For each index i, find least-costly encoding of +*/ + if (!values[4]) values[4] = __Pyx_NewRef(((PyObject *)Py_None)); + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_iup_contour_optimize_dp", 0, 2, 5, i); __PYX_ERR(0, 293, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 293, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 293, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 293, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 293, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 293, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[2]) values[2] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + if (!values[4]) values[4] = __Pyx_NewRef(((PyObject *)Py_None)); + } + __pyx_v_deltas = values[0]; + __pyx_v_coords = values[1]; + __pyx_v_forced = ((PyObject*)values[2]); + if (values[3]) { + __pyx_v_tolerance = __Pyx_PyFloat_AsDouble(values[3]); if (unlikely((__pyx_v_tolerance == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 306, __pyx_L3_error) + } else { + __pyx_v_tolerance = ((double)((double)0.0)); + } + __pyx_v_lookback = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_iup_contour_optimize_dp", 0, 2, 5, __pyx_nargs); __PYX_ERR(0, 293, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("fontTools.varLib.iup._iup_contour_optimize_dp", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_forced), (&PySet_Type), 1, "forced", 1))) __PYX_ERR(0, 305, __pyx_L1_error) + __pyx_r = __pyx_pf_9fontTools_6varLib_3iup_6_iup_contour_optimize_dp(__pyx_self, __pyx_v_deltas, __pyx_v_coords, __pyx_v_forced, __pyx_v_tolerance, __pyx_v_lookback); + + /* "fontTools/varLib/iup.py":293 + * + * + * @cython.locals( # <<<<<<<<<<<<<< + * i=cython.int, + * j=cython.int, +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_6_iup_contour_optimize_dp(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, PyObject *__pyx_v_forced, double __pyx_v_tolerance, PyObject *__pyx_v_lookback) { + int __pyx_v_i; + int __pyx_v_j; + double __pyx_v_best_cost; + double __pyx_v_cost; + Py_ssize_t __pyx_v_n; + PyObject *__pyx_v_costs = NULL; + PyObject *__pyx_v_chain = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + Py_ssize_t __pyx_t_7; + int __pyx_t_8; + double __pyx_t_9; + long __pyx_t_10; + Py_ssize_t __pyx_t_11; + Py_ssize_t __pyx_t_12; + int __pyx_t_13; + int __pyx_t_14; + int __pyx_t_15; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_iup_contour_optimize_dp", 0); + __Pyx_INCREF(__pyx_v_lookback); + + /* "fontTools/varLib/iup.py":318 + * As major speedup, we stop looking further whenever we see a "forced" point.""" + * + * n = len(deltas) # <<<<<<<<<<<<<< + * if lookback is None: + * lookback = n +*/ + __pyx_t_1 = PyObject_Length(__pyx_v_deltas); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(0, 318, __pyx_L1_error) + __pyx_v_n = __pyx_t_1; + + /* "fontTools/varLib/iup.py":319 + * + * n = len(deltas) + * if lookback is None: # <<<<<<<<<<<<<< + * lookback = n + * lookback = min(lookback, MAX_LOOKBACK) +*/ + __pyx_t_2 = (__pyx_v_lookback == Py_None); + if (__pyx_t_2) { + + /* "fontTools/varLib/iup.py":320 + * n = len(deltas) + * if lookback is None: + * lookback = n # <<<<<<<<<<<<<< + * lookback = min(lookback, MAX_LOOKBACK) + * costs = {-1: 0} +*/ + __pyx_t_3 = PyLong_FromSsize_t(__pyx_v_n); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 320, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF_SET(__pyx_v_lookback, __pyx_t_3); + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":319 + * + * n = len(deltas) + * if lookback is None: # <<<<<<<<<<<<<< + * lookback = n + * lookback = min(lookback, MAX_LOOKBACK) +*/ + } + + /* "fontTools/varLib/iup.py":321 + * if lookback is None: + * lookback = n + * lookback = min(lookback, MAX_LOOKBACK) # <<<<<<<<<<<<<< + * costs = {-1: 0} + * chain = {-1: None} +*/ + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_MAX_LOOKBACK); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_lookback); + __pyx_t_4 = __pyx_v_lookback; + __pyx_t_6 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_LT); __Pyx_XGOTREF(__pyx_t_6); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 321, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_6); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (__pyx_t_2) { + __Pyx_INCREF(__pyx_t_3); + __pyx_t_5 = __pyx_t_3; + } else { + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = __pyx_t_4; + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __pyx_t_5; + __Pyx_INCREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF_SET(__pyx_v_lookback, __pyx_t_3); + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":322 + * lookback = n + * lookback = min(lookback, MAX_LOOKBACK) + * costs = {-1: 0} # <<<<<<<<<<<<<< + * chain = {-1: None} + * for i in range(0, n): +*/ + __pyx_t_3 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 322, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (PyDict_SetItem(__pyx_t_3, __pyx_mstate_global->__pyx_int_neg_1, __pyx_mstate_global->__pyx_int_0) < (0)) __PYX_ERR(0, 322, __pyx_L1_error) + __pyx_v_costs = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":323 + * lookback = min(lookback, MAX_LOOKBACK) + * costs = {-1: 0} + * chain = {-1: None} # <<<<<<<<<<<<<< + * for i in range(0, n): + * best_cost = costs[i - 1] + 1 +*/ + __pyx_t_3 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 323, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (PyDict_SetItem(__pyx_t_3, __pyx_mstate_global->__pyx_int_neg_1, Py_None) < (0)) __PYX_ERR(0, 323, __pyx_L1_error) + __pyx_v_chain = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":324 + * costs = {-1: 0} + * chain = {-1: None} + * for i in range(0, n): # <<<<<<<<<<<<<< + * best_cost = costs[i - 1] + 1 + * +*/ + __pyx_t_1 = __pyx_v_n; + __pyx_t_7 = __pyx_t_1; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_i = __pyx_t_8; + + /* "fontTools/varLib/iup.py":325 + * chain = {-1: None} + * for i in range(0, n): + * best_cost = costs[i - 1] + 1 # <<<<<<<<<<<<<< + * + * costs[i] = best_cost +*/ + __pyx_t_3 = __Pyx_PyLong_From_long((__pyx_v_i - 1)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 325, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyDict_GetItem(__pyx_v_costs, __pyx_t_3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 325, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyLong_AddObjC(__pyx_t_5, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 325, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_9 = __Pyx_PyFloat_AsDouble(__pyx_t_3); if (unlikely((__pyx_t_9 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 325, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_best_cost = __pyx_t_9; + + /* "fontTools/varLib/iup.py":327 + * best_cost = costs[i - 1] + 1 + * + * costs[i] = best_cost # <<<<<<<<<<<<<< + * chain[i] = i - 1 + * +*/ + __pyx_t_3 = PyFloat_FromDouble(__pyx_v_best_cost); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 327, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 327, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (unlikely((PyDict_SetItem(__pyx_v_costs, __pyx_t_5, __pyx_t_3) < 0))) __PYX_ERR(0, 327, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":328 + * + * costs[i] = best_cost + * chain[i] = i - 1 # <<<<<<<<<<<<<< + * + * if i - 1 in forced: +*/ + __pyx_t_3 = __Pyx_PyLong_From_long((__pyx_v_i - 1)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 328, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 328, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (unlikely((PyDict_SetItem(__pyx_v_chain, __pyx_t_5, __pyx_t_3) < 0))) __PYX_ERR(0, 328, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":330 + * chain[i] = i - 1 + * + * if i - 1 in forced: # <<<<<<<<<<<<<< + * continue + * +*/ + __pyx_t_3 = __Pyx_PyLong_From_long((__pyx_v_i - 1)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 330, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely(__pyx_v_forced == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 330, __pyx_L1_error) + } + __pyx_t_2 = (__Pyx_PySet_ContainsTF(__pyx_t_3, __pyx_v_forced, Py_EQ)); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 330, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (__pyx_t_2) { + + /* "fontTools/varLib/iup.py":331 + * + * if i - 1 in forced: + * continue # <<<<<<<<<<<<<< + * + * for j in range(i - 2, max(i - lookback, -2), -1): +*/ + goto __pyx_L4_continue; + + /* "fontTools/varLib/iup.py":330 + * chain[i] = i - 1 + * + * if i - 1 in forced: # <<<<<<<<<<<<<< + * continue + * +*/ + } + + /* "fontTools/varLib/iup.py":333 + * continue + * + * for j in range(i - 2, max(i - lookback, -2), -1): # <<<<<<<<<<<<<< + * cost = costs[j] + 1 + * +*/ + __pyx_t_10 = -2L; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 333, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = PyNumber_Subtract(__pyx_t_3, __pyx_v_lookback); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 333, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyLong_From_long(__pyx_t_10); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 333, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = PyObject_RichCompare(__pyx_t_4, __pyx_t_5, Py_GT); __Pyx_XGOTREF(__pyx_t_6); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 333, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_6); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 333, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (__pyx_t_2) { + __pyx_t_6 = __Pyx_PyLong_From_long(__pyx_t_10); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 333, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_3 = __pyx_t_6; + __pyx_t_6 = 0; + } else { + __Pyx_INCREF(__pyx_t_5); + __pyx_t_3 = __pyx_t_5; + } + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_3); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 333, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_12 = __pyx_t_11; + for (__pyx_t_13 = (__pyx_v_i - 2); __pyx_t_13 > __pyx_t_12; __pyx_t_13-=1) { + __pyx_v_j = __pyx_t_13; + + /* "fontTools/varLib/iup.py":334 + * + * for j in range(i - 2, max(i - lookback, -2), -1): + * cost = costs[j] + 1 # <<<<<<<<<<<<<< + * + * if cost < best_cost and can_iup_in_between(deltas, coords, j, i, tolerance): +*/ + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_j); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 334, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyDict_GetItem(__pyx_v_costs, __pyx_t_3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 334, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyLong_AddObjC(__pyx_t_5, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 334, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_9 = __Pyx_PyFloat_AsDouble(__pyx_t_3); if (unlikely((__pyx_t_9 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 334, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_cost = __pyx_t_9; + + /* "fontTools/varLib/iup.py":336 + * cost = costs[j] + 1 + * + * if cost < best_cost and can_iup_in_between(deltas, coords, j, i, tolerance): # <<<<<<<<<<<<<< + * costs[i] = best_cost = cost + * chain[i] = j +*/ + __pyx_t_14 = (__pyx_v_cost < __pyx_v_best_cost); + if (__pyx_t_14) { + } else { + __pyx_t_2 = __pyx_t_14; + goto __pyx_L10_bool_binop_done; + } + __pyx_t_3 = PyFloat_FromDouble(__pyx_v_tolerance); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 336, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_15 = __pyx_f_9fontTools_6varLib_3iup_can_iup_in_between(__pyx_v_deltas, __pyx_v_coords, __pyx_v_j, __pyx_v_i, __pyx_t_3); if (unlikely(__pyx_t_15 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 336, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_14 = (__pyx_t_15 != 0); + __pyx_t_2 = __pyx_t_14; + __pyx_L10_bool_binop_done:; + if (__pyx_t_2) { + + /* "fontTools/varLib/iup.py":337 + * + * if cost < best_cost and can_iup_in_between(deltas, coords, j, i, tolerance): + * costs[i] = best_cost = cost # <<<<<<<<<<<<<< + * chain[i] = j + * +*/ + __pyx_t_3 = PyFloat_FromDouble(__pyx_v_cost); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 337, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 337, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (unlikely((PyDict_SetItem(__pyx_v_costs, __pyx_t_5, __pyx_t_3) < 0))) __PYX_ERR(0, 337, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_best_cost = __pyx_v_cost; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":338 + * if cost < best_cost and can_iup_in_between(deltas, coords, j, i, tolerance): + * costs[i] = best_cost = cost + * chain[i] = j # <<<<<<<<<<<<<< + * + * if j in forced: +*/ + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_j); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 338, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 338, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (unlikely((PyDict_SetItem(__pyx_v_chain, __pyx_t_5, __pyx_t_3) < 0))) __PYX_ERR(0, 338, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":336 + * cost = costs[j] + 1 + * + * if cost < best_cost and can_iup_in_between(deltas, coords, j, i, tolerance): # <<<<<<<<<<<<<< + * costs[i] = best_cost = cost + * chain[i] = j +*/ + } + + /* "fontTools/varLib/iup.py":340 + * chain[i] = j + * + * if j in forced: # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_j); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 340, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely(__pyx_v_forced == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 340, __pyx_L1_error) + } + __pyx_t_2 = (__Pyx_PySet_ContainsTF(__pyx_t_3, __pyx_v_forced, Py_EQ)); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 340, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (__pyx_t_2) { + + /* "fontTools/varLib/iup.py":341 + * + * if j in forced: + * break # <<<<<<<<<<<<<< + * + * return chain, costs +*/ + goto __pyx_L8_break; + + /* "fontTools/varLib/iup.py":340 + * chain[i] = j + * + * if j in forced: # <<<<<<<<<<<<<< + * break + * +*/ + } + } + __pyx_L8_break:; + __pyx_L4_continue:; + } + + /* "fontTools/varLib/iup.py":343 + * break + * + * return chain, costs # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 343, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_chain); + __Pyx_GIVEREF(__pyx_v_chain); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_chain) != (0)) __PYX_ERR(0, 343, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_costs); + __Pyx_GIVEREF(__pyx_v_costs); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_v_costs) != (0)) __PYX_ERR(0, 343, __pyx_L1_error); + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":293 + * + * + * @cython.locals( # <<<<<<<<<<<<<< + * i=cython.int, + * j=cython.int, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("fontTools.varLib.iup._iup_contour_optimize_dp", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_costs); + __Pyx_XDECREF(__pyx_v_chain); + __Pyx_XDECREF(__pyx_v_lookback); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "fontTools/varLib/iup.py":346 + * + * + * def _rot_list(l: list, k: int): # <<<<<<<<<<<<<< + * """Rotate list by k items forward. Ie. item at position 0 will be + * at position k in returned list. Negative k is allowed.""" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_9_rot_list(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_9fontTools_6varLib_3iup_8_rot_list, "_rot_list(list l: list, int k: int)\n\nRotate list by k items forward. Ie. item at position 0 will be\nat position k in returned list. Negative k is allowed."); +static PyMethodDef __pyx_mdef_9fontTools_6varLib_3iup_9_rot_list = {"_rot_list", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_6varLib_3iup_9_rot_list, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_6varLib_3iup_8_rot_list}; +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_9_rot_list(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_l = 0; + PyObject *__pyx_v_k = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_rot_list (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_l,&__pyx_mstate_global->__pyx_n_u_k,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 346, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 346, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 346, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_rot_list", 0) < (0)) __PYX_ERR(0, 346, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_rot_list", 1, 2, 2, i); __PYX_ERR(0, 346, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 346, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 346, __pyx_L3_error) + } + __pyx_v_l = ((PyObject*)values[0]); + if (__Pyx_PyInt_FromNumber(&values[1], "k", 0) < (0)) __PYX_ERR(0, 346, __pyx_L3_error) + __pyx_v_k = ((PyObject*)values[1]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_rot_list", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 346, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("fontTools.varLib.iup._rot_list", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_l), (&PyList_Type), 0, "l", 2))) __PYX_ERR(0, 346, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_k), (&PyLong_Type), 0, "k", 2))) __PYX_ERR(0, 346, __pyx_L1_error) + __pyx_r = __pyx_pf_9fontTools_6varLib_3iup_8_rot_list(__pyx_self, __pyx_v_l, __pyx_v_k); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_8_rot_list(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_l, PyObject *__pyx_v_k) { + PyObject *__pyx_v_n = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_rot_list", 0); + __Pyx_INCREF(__pyx_v_k); + + /* "fontTools/varLib/iup.py":349 + * """Rotate list by k items forward. Ie. item at position 0 will be + * at position k in returned list. Negative k is allowed.""" + * n = len(l) # <<<<<<<<<<<<<< + * k %= n + * if not k: +*/ + __pyx_t_1 = __Pyx_PyList_GET_SIZE(__pyx_v_l); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(0, 349, __pyx_L1_error) + __pyx_t_2 = PyLong_FromSsize_t(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 349, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_n = __pyx_t_2; + __pyx_t_2 = 0; + + /* "fontTools/varLib/iup.py":350 + * at position k in returned list. Negative k is allowed.""" + * n = len(l) + * k %= n # <<<<<<<<<<<<<< + * if not k: + * return l +*/ + __pyx_t_2 = PyNumber_InPlaceRemainder(__pyx_v_k, __pyx_v_n); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 350, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_PyInt_FromNumber(&__pyx_t_2, NULL, 1) < (0)) __PYX_ERR(0, 350, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_k, ((PyObject*)__pyx_t_2)); + __pyx_t_2 = 0; + + /* "fontTools/varLib/iup.py":351 + * n = len(l) + * k %= n + * if not k: # <<<<<<<<<<<<<< + * return l + * return l[n - k :] + l[: n - k] +*/ + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v_k); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 351, __pyx_L1_error) + __pyx_t_4 = (!__pyx_t_3); + if (__pyx_t_4) { + + /* "fontTools/varLib/iup.py":352 + * k %= n + * if not k: + * return l # <<<<<<<<<<<<<< + * return l[n - k :] + l[: n - k] + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_l); + __pyx_r = __pyx_v_l; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":351 + * n = len(l) + * k %= n + * if not k: # <<<<<<<<<<<<<< + * return l + * return l[n - k :] + l[: n - k] +*/ + } + + /* "fontTools/varLib/iup.py":353 + * if not k: + * return l + * return l[n - k :] + l[: n - k] # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = PyNumber_Subtract(__pyx_v_n, __pyx_v_k); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (__pyx_t_2 == Py_None); + if (__pyx_t_4) { + __pyx_t_1 = 0; + } else { + __pyx_t_5 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_5 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 353, __pyx_L1_error) + __pyx_t_1 = __pyx_t_5; + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyList_GetSlice(__pyx_v_l, __pyx_t_1, PY_SSIZE_T_MAX); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = PyNumber_Subtract(__pyx_v_n, __pyx_v_k); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_4 = (__pyx_t_6 == Py_None); + if (__pyx_t_4) { + __pyx_t_1 = PY_SSIZE_T_MAX; + } else { + __pyx_t_5 = __Pyx_PyIndex_AsSsize_t(__pyx_t_6); if (unlikely((__pyx_t_5 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 353, __pyx_L1_error) + __pyx_t_1 = __pyx_t_5; + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyList_GetSlice(__pyx_v_l, 0, __pyx_t_1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyNumber_Add(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_r = __pyx_t_7; + __pyx_t_7 = 0; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":346 + * + * + * def _rot_list(l: list, k: int): # <<<<<<<<<<<<<< + * """Rotate list by k items forward. Ie. item at position 0 will be + * at position k in returned list. Negative k is allowed.""" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("fontTools.varLib.iup._rot_list", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_n); + __Pyx_XDECREF(__pyx_v_k); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "fontTools/varLib/iup.py":356 + * + * + * def _rot_set(s: set, k: int, n: int): # <<<<<<<<<<<<<< + * k %= n + * if not k: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_11_rot_set(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_9fontTools_6varLib_3iup_10_rot_set, "_rot_set(set s: set, int k: int, int n: int)"); +static PyMethodDef __pyx_mdef_9fontTools_6varLib_3iup_11_rot_set = {"_rot_set", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_6varLib_3iup_11_rot_set, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_6varLib_3iup_10_rot_set}; +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_11_rot_set(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_s = 0; + PyObject *__pyx_v_k = 0; + PyObject *__pyx_v_n = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_rot_set (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_s,&__pyx_mstate_global->__pyx_n_u_k,&__pyx_mstate_global->__pyx_n_u_n,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 356, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 356, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 356, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 356, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_rot_set", 0) < (0)) __PYX_ERR(0, 356, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_rot_set", 1, 3, 3, i); __PYX_ERR(0, 356, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 356, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 356, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 356, __pyx_L3_error) + } + __pyx_v_s = ((PyObject*)values[0]); + if (__Pyx_PyInt_FromNumber(&values[1], "k", 0) < (0)) __PYX_ERR(0, 356, __pyx_L3_error) + __pyx_v_k = ((PyObject*)values[1]); + if (__Pyx_PyInt_FromNumber(&values[2], "n", 0) < (0)) __PYX_ERR(0, 356, __pyx_L3_error) + __pyx_v_n = ((PyObject*)values[2]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_rot_set", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 356, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("fontTools.varLib.iup._rot_set", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_s), (&PySet_Type), 0, "s", 2))) __PYX_ERR(0, 356, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_k), (&PyLong_Type), 0, "k", 2))) __PYX_ERR(0, 356, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_n), (&PyLong_Type), 0, "n", 2))) __PYX_ERR(0, 356, __pyx_L1_error) + __pyx_r = __pyx_pf_9fontTools_6varLib_3iup_10_rot_set(__pyx_self, __pyx_v_s, __pyx_v_k, __pyx_v_n); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_10_rot_set(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_s, PyObject *__pyx_v_k, PyObject *__pyx_v_n) { + PyObject *__pyx_8genexpr2__pyx_v_v = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + Py_ssize_t __pyx_t_6; + int __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_rot_set", 0); + __Pyx_INCREF(__pyx_v_k); + + /* "fontTools/varLib/iup.py":357 + * + * def _rot_set(s: set, k: int, n: int): + * k %= n # <<<<<<<<<<<<<< + * if not k: + * return s +*/ + __pyx_t_1 = PyNumber_InPlaceRemainder(__pyx_v_k, __pyx_v_n); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 357, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF_SET(__pyx_v_k, ((PyObject*)__pyx_t_1)); + __pyx_t_1 = 0; + + /* "fontTools/varLib/iup.py":358 + * def _rot_set(s: set, k: int, n: int): + * k %= n + * if not k: # <<<<<<<<<<<<<< + * return s + * return {(v + k) % n for v in s} +*/ + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_k); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 358, __pyx_L1_error) + __pyx_t_3 = (!__pyx_t_2); + if (__pyx_t_3) { + + /* "fontTools/varLib/iup.py":359 + * k %= n + * if not k: + * return s # <<<<<<<<<<<<<< + * return {(v + k) % n for v in s} + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_s); + __pyx_r = __pyx_v_s; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":358 + * def _rot_set(s: set, k: int, n: int): + * k %= n + * if not k: # <<<<<<<<<<<<<< + * return s + * return {(v + k) % n for v in s} +*/ + } + + /* "fontTools/varLib/iup.py":360 + * if not k: + * return s + * return {(v + k) % n for v in s} # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_1 = PySet_New(NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 360, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_5 = 0; + __pyx_t_8 = __Pyx_set_iterator(__pyx_v_s, 1, (&__pyx_t_6), (&__pyx_t_7)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 360, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_4); + __pyx_t_4 = __pyx_t_8; + __pyx_t_8 = 0; + while (1) { + __pyx_t_9 = __Pyx_set_iter_next(__pyx_t_4, __pyx_t_6, &__pyx_t_5, &__pyx_t_8, __pyx_t_7); + if (unlikely(__pyx_t_9 == 0)) break; + if (unlikely(__pyx_t_9 == -1)) __PYX_ERR(0, 360, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_8genexpr2__pyx_v_v, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_t_8 = PyNumber_Add(__pyx_8genexpr2__pyx_v_v, __pyx_v_k); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 360, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_10 = PyNumber_Remainder(__pyx_t_8, __pyx_v_n); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 360, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(PySet_Add(__pyx_t_1, (PyObject*)__pyx_t_10))) __PYX_ERR(0, 360, __pyx_L6_error) + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_8genexpr2__pyx_v_v); __pyx_8genexpr2__pyx_v_v = 0; + goto __pyx_L9_exit_scope; + __pyx_L6_error:; + __Pyx_XDECREF(__pyx_8genexpr2__pyx_v_v); __pyx_8genexpr2__pyx_v_v = 0; + goto __pyx_L1_error; + __pyx_L9_exit_scope:; + } /* exit inner scope */ + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":356 + * + * + * def _rot_set(s: set, k: int, n: int): # <<<<<<<<<<<<<< + * k %= n + * if not k: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("fontTools.varLib.iup._rot_set", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_8genexpr2__pyx_v_v); + __Pyx_XDECREF(__pyx_v_k); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "fontTools/varLib/iup.py":363 + * + * + * def iup_contour_optimize( # <<<<<<<<<<<<<< + * deltas: _DeltaSegment, coords: _PointSegment, tolerance: Real = 0.0 + * ) -> _DeltaOrNoneSegment: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_13iup_contour_optimize(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_9fontTools_6varLib_3iup_12iup_contour_optimize, "iup_contour_optimize(deltas: _DeltaSegment, coords: _PointSegment, tolerance: Real = 0.0) -> _DeltaOrNoneSegment\n\nFor contour with coordinates `coords`, optimize a set of delta\nvalues `deltas` within error `tolerance`.\n\nReturns delta vector that has most number of None items instead of\nthe input delta."); +static PyMethodDef __pyx_mdef_9fontTools_6varLib_3iup_13iup_contour_optimize = {"iup_contour_optimize", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_6varLib_3iup_13iup_contour_optimize, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_6varLib_3iup_12iup_contour_optimize}; +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_13iup_contour_optimize(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_deltas = 0; + PyObject *__pyx_v_coords = 0; + PyObject *__pyx_v_tolerance = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("iup_contour_optimize (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_deltas,&__pyx_mstate_global->__pyx_n_u_coords,&__pyx_mstate_global->__pyx_n_u_tolerance,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 363, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 363, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 363, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 363, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "iup_contour_optimize", 0) < (0)) __PYX_ERR(0, 363, __pyx_L3_error) + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_float_0_0))); + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("iup_contour_optimize", 0, 2, 3, i); __PYX_ERR(0, 363, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 363, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 363, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 363, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_float_0_0))); + } + __pyx_v_deltas = values[0]; + __pyx_v_coords = values[1]; + __pyx_v_tolerance = values[2]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("iup_contour_optimize", 0, 2, 3, __pyx_nargs); __PYX_ERR(0, 363, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("fontTools.varLib.iup.iup_contour_optimize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_9fontTools_6varLib_3iup_12iup_contour_optimize(__pyx_self, __pyx_v_deltas, __pyx_v_coords, __pyx_v_tolerance); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_9fontTools_6varLib_3iup_20iup_contour_optimize_2generator1(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value); /* proto */ + +/* "fontTools/varLib/iup.py":378 + * + * # If all are within tolerance distance of 0, encode nothing: + * if all(abs(complex(*p)) <= tolerance for p in deltas): # <<<<<<<<<<<<<< + * return [None] * n + * +*/ + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_20iup_contour_optimize_genexpr(PyObject *__pyx_self, PyObject *__pyx_genexpr_arg_0) { + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("genexpr", 0); + __pyx_cur_scope = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr(__pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 378, __pyx_L1_error) + } else { + __Pyx_GOTREF((PyObject *)__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_outer_scope = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *) __pyx_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_outer_scope); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_outer_scope); + __pyx_cur_scope->__pyx_genexpr_arg_0 = __pyx_genexpr_arg_0; + __Pyx_INCREF(__pyx_cur_scope->__pyx_genexpr_arg_0); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_genexpr_arg_0); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_9fontTools_6varLib_3iup_20iup_contour_optimize_2generator1, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1]), (PyObject *) __pyx_cur_scope, __pyx_mstate_global->__pyx_n_u_genexpr, __pyx_mstate_global->__pyx_n_u_iup_contour_optimize_locals_gene, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup); if (unlikely(!gen)) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("fontTools.varLib.iup.iup_contour_optimize.genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF((PyObject *)__pyx_cur_scope); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_9fontTools_6varLib_3iup_20iup_contour_optimize_2generator1(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *__pyx_cur_scope = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + Py_ssize_t __pyx_t_2; + PyObject *(*__pyx_t_3)(PyObject *); + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + int __pyx_t_6; + int __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("genexpr", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 378, __pyx_L1_error) + if (unlikely(!__pyx_cur_scope->__pyx_genexpr_arg_0)) { __Pyx_RaiseUnboundLocalError(".0"); __PYX_ERR(0, 378, __pyx_L1_error) } + if (likely(PyList_CheckExact(__pyx_cur_scope->__pyx_genexpr_arg_0)) || PyTuple_CheckExact(__pyx_cur_scope->__pyx_genexpr_arg_0)) { + __pyx_t_1 = __pyx_cur_scope->__pyx_genexpr_arg_0; __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = 0; + __pyx_t_3 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_cur_scope->__pyx_genexpr_arg_0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 378, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_3)) { + if (likely(PyList_CheckExact(__pyx_t_1))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 378, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(__pyx_t_1, __pyx_t_2, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 378, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_4 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_2)); + #else + __pyx_t_4 = __Pyx_PySequence_ITEM(__pyx_t_1, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 378, __pyx_L1_error) + } else { + __pyx_t_4 = __pyx_t_3(__pyx_t_1); + if (unlikely(!__pyx_t_4)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 378, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_p); + __Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_p, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PySequence_Tuple(__pyx_cur_scope->__pyx_v_p); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_Call(((PyObject *)(&PyComplex_Type)), __pyx_t_4, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyNumber_Absolute(__pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_tolerance)) { __Pyx_RaiseClosureNameError("tolerance"); __PYX_ERR(0, 378, __pyx_L1_error) } + __pyx_t_5 = PyObject_RichCompare(__pyx_t_4, __pyx_cur_scope->__pyx_outer_scope->__pyx_v_tolerance, Py_LE); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_7 = (!__pyx_t_6); + if (__pyx_t_7) { + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(Py_False); + __pyx_r = Py_False; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L0; + } + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(Py_True); + __pyx_r = Py_True; + goto __pyx_L0; + } + CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + if (__Pyx_PyErr_Occurred()) { + __Pyx_Generator_Replace_StopIteration(0); + __Pyx_AddTraceback("genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + #if !CYTHON_USE_EXC_INFO_STACK + __Pyx_Coroutine_ResetAndClearException(__pyx_generator); + #endif + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_9fontTools_6varLib_3iup_20iup_contour_optimize_5generator2(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value); /* proto */ + +/* "fontTools/varLib/iup.py":387 + * # If all deltas are exactly the same, return just one (the first one): + * d0 = deltas[0] + * if all(d0 == d for d in deltas): # <<<<<<<<<<<<<< + * return [d0] + [None] * (n - 1) + * +*/ + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_20iup_contour_optimize_3genexpr(PyObject *__pyx_self, PyObject *__pyx_genexpr_arg_0) { + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("genexpr", 0); + __pyx_cur_scope = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr(__pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 387, __pyx_L1_error) + } else { + __Pyx_GOTREF((PyObject *)__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_outer_scope = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *) __pyx_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_outer_scope); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_outer_scope); + __pyx_cur_scope->__pyx_genexpr_arg_0 = __pyx_genexpr_arg_0; + __Pyx_INCREF(__pyx_cur_scope->__pyx_genexpr_arg_0); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_genexpr_arg_0); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_9fontTools_6varLib_3iup_20iup_contour_optimize_5generator2, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[2]), (PyObject *) __pyx_cur_scope, __pyx_mstate_global->__pyx_n_u_genexpr, __pyx_mstate_global->__pyx_n_u_iup_contour_optimize_locals_gene, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup); if (unlikely(!gen)) __PYX_ERR(0, 387, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("fontTools.varLib.iup.iup_contour_optimize.genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF((PyObject *)__pyx_cur_scope); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_9fontTools_6varLib_3iup_20iup_contour_optimize_5generator2(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *__pyx_cur_scope = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + Py_ssize_t __pyx_t_2; + PyObject *(*__pyx_t_3)(PyObject *); + PyObject *__pyx_t_4 = NULL; + int __pyx_t_5; + int __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("genexpr", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 387, __pyx_L1_error) + if (unlikely(!__pyx_cur_scope->__pyx_genexpr_arg_0)) { __Pyx_RaiseUnboundLocalError(".0"); __PYX_ERR(0, 387, __pyx_L1_error) } + if (likely(PyList_CheckExact(__pyx_cur_scope->__pyx_genexpr_arg_0)) || PyTuple_CheckExact(__pyx_cur_scope->__pyx_genexpr_arg_0)) { + __pyx_t_1 = __pyx_cur_scope->__pyx_genexpr_arg_0; __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = 0; + __pyx_t_3 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_cur_scope->__pyx_genexpr_arg_0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 387, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 387, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_3)) { + if (likely(PyList_CheckExact(__pyx_t_1))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 387, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(__pyx_t_1, __pyx_t_2, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 387, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_4 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_2)); + #else + __pyx_t_4 = __Pyx_PySequence_ITEM(__pyx_t_1, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 387, __pyx_L1_error) + } else { + __pyx_t_4 = __pyx_t_3(__pyx_t_1); + if (unlikely(!__pyx_t_4)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 387, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_d); + __Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_d, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_d0)) { __Pyx_RaiseClosureNameError("d0"); __PYX_ERR(0, 387, __pyx_L1_error) } + __pyx_t_4 = PyObject_RichCompare(__pyx_cur_scope->__pyx_outer_scope->__pyx_v_d0, __pyx_cur_scope->__pyx_v_d, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 387, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely((__pyx_t_5 < 0))) __PYX_ERR(0, 387, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_6 = (!__pyx_t_5); + if (__pyx_t_6) { + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(Py_False); + __pyx_r = Py_False; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L0; + } + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(Py_True); + __pyx_r = Py_True; + goto __pyx_L0; + } + CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + if (__Pyx_PyErr_Occurred()) { + __Pyx_Generator_Replace_StopIteration(0); + __Pyx_AddTraceback("genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + #if !CYTHON_USE_EXC_INFO_STACK + __Pyx_Coroutine_ResetAndClearException(__pyx_generator); + #endif + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "fontTools/varLib/iup.py":363 + * + * + * def iup_contour_optimize( # <<<<<<<<<<<<<< + * deltas: _DeltaSegment, coords: _PointSegment, tolerance: Real = 0.0 + * ) -> _DeltaOrNoneSegment: +*/ + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_12iup_contour_optimize(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, PyObject *__pyx_v_tolerance) { + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *__pyx_cur_scope; + PyObject *__pyx_v_n = NULL; + PyObject *__pyx_v_forced = NULL; + PyObject *__pyx_v_k = NULL; + PyObject *__pyx_v_chain = NULL; + PyObject *__pyx_v_costs = NULL; + PyObject *__pyx_v_solution = NULL; + PyObject *__pyx_v_i = NULL; + PyObject *__pyx_v_best_sol = NULL; + PyObject *__pyx_v_best_cost = NULL; + PyObject *__pyx_v_start = NULL; + PyObject *__pyx_v_cost = NULL; + PyObject *__pyx_gb_9fontTools_6varLib_3iup_20iup_contour_optimize_2generator1 = 0; + PyObject *__pyx_gb_9fontTools_6varLib_3iup_20iup_contour_optimize_5generator2 = 0; + PyObject *__pyx_8genexpr5__pyx_v_i = NULL; + PyObject *__pyx_8genexpr6__pyx_v_i = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + PyObject *__pyx_t_7 = NULL; + PyObject *(*__pyx_t_8)(PyObject *); + int __pyx_t_9; + PyObject *(*__pyx_t_10)(PyObject *); + PyObject *__pyx_t_11 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("iup_contour_optimize", 0); + __pyx_cur_scope = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize(__pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 363, __pyx_L1_error) + } else { + __Pyx_GOTREF((PyObject *)__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_tolerance = __pyx_v_tolerance; + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_tolerance); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_tolerance); + __Pyx_INCREF(__pyx_v_deltas); + __Pyx_INCREF(__pyx_v_coords); + + /* "fontTools/varLib/iup.py":373 + * """ + * + * n = len(deltas) # <<<<<<<<<<<<<< + * + * # Get the easy cases out of the way: +*/ + __pyx_t_1 = PyObject_Length(__pyx_v_deltas); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(0, 373, __pyx_L1_error) + __pyx_t_2 = PyLong_FromSsize_t(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 373, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_n = __pyx_t_2; + __pyx_t_2 = 0; + + /* "fontTools/varLib/iup.py":378 + * + * # If all are within tolerance distance of 0, encode nothing: + * if all(abs(complex(*p)) <= tolerance for p in deltas): # <<<<<<<<<<<<<< + * return [None] * n + * +*/ + __pyx_t_2 = __pyx_pf_9fontTools_6varLib_3iup_20iup_contour_optimize_genexpr(((PyObject*)__pyx_cur_scope), __pyx_v_deltas); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_Generator_GetInlinedResult(__pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (__pyx_t_4) { + + /* "fontTools/varLib/iup.py":379 + * # If all are within tolerance distance of 0, encode nothing: + * if all(abs(complex(*p)) <= tolerance for p in deltas): + * return [None] * n # <<<<<<<<<<<<<< + * + * # If there's exactly one point, return it: +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = PyList_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 379, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyList_SET_ITEM(__pyx_t_3, 0, Py_None) != (0)) __PYX_ERR(0, 379, __pyx_L1_error); + { PyObject* __pyx_temp = PyNumber_InPlaceMultiply(__pyx_t_3, __pyx_v_n); if (unlikely(!__pyx_temp)) __PYX_ERR(0, 379, __pyx_L1_error) + __Pyx_GOTREF(__pyx_temp); + __Pyx_DECREF(__pyx_t_3); + __pyx_t_3 = __pyx_temp; + } + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":378 + * + * # If all are within tolerance distance of 0, encode nothing: + * if all(abs(complex(*p)) <= tolerance for p in deltas): # <<<<<<<<<<<<<< + * return [None] * n + * +*/ + } + + /* "fontTools/varLib/iup.py":382 + * + * # If there's exactly one point, return it: + * if n == 1: # <<<<<<<<<<<<<< + * return deltas + * +*/ + __pyx_t_4 = (__Pyx_PyLong_BoolEqObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_1, 1, 0)); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 382, __pyx_L1_error) + if (__pyx_t_4) { + + /* "fontTools/varLib/iup.py":383 + * # If there's exactly one point, return it: + * if n == 1: + * return deltas # <<<<<<<<<<<<<< + * + * # If all deltas are exactly the same, return just one (the first one): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_deltas); + __pyx_r = __pyx_v_deltas; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":382 + * + * # If there's exactly one point, return it: + * if n == 1: # <<<<<<<<<<<<<< + * return deltas + * +*/ + } + + /* "fontTools/varLib/iup.py":386 + * + * # If all deltas are exactly the same, return just one (the first one): + * d0 = deltas[0] # <<<<<<<<<<<<<< + * if all(d0 == d for d in deltas): + * return [d0] + [None] * (n - 1) +*/ + __pyx_t_3 = __Pyx_GetItemInt(__pyx_v_deltas, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 386, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_cur_scope->__pyx_v_d0 = __pyx_t_3; + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":387 + * # If all deltas are exactly the same, return just one (the first one): + * d0 = deltas[0] + * if all(d0 == d for d in deltas): # <<<<<<<<<<<<<< + * return [d0] + [None] * (n - 1) + * +*/ + __pyx_t_3 = __pyx_pf_9fontTools_6varLib_3iup_20iup_contour_optimize_3genexpr(((PyObject*)__pyx_cur_scope), __pyx_v_deltas); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 387, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_Generator_GetInlinedResult(__pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 387, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 387, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_4) { + + /* "fontTools/varLib/iup.py":388 + * d0 = deltas[0] + * if all(d0 == d for d in deltas): + * return [d0] + [None] * (n - 1) # <<<<<<<<<<<<<< + * + * # Else, solve the general problem using Dynamic Programming. +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 388, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_d0); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_d0); + if (__Pyx_PyList_SET_ITEM(__pyx_t_2, 0, __pyx_cur_scope->__pyx_v_d0) != (0)) __PYX_ERR(0, 388, __pyx_L1_error); + __pyx_t_3 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 388, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = PyList_New(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 388, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyList_SET_ITEM(__pyx_t_5, 0, Py_None) != (0)) __PYX_ERR(0, 388, __pyx_L1_error); + { PyObject* __pyx_temp = PyNumber_InPlaceMultiply(__pyx_t_5, __pyx_t_3); if (unlikely(!__pyx_temp)) __PYX_ERR(0, 388, __pyx_L1_error) + __Pyx_GOTREF(__pyx_temp); + __Pyx_DECREF(__pyx_t_5); + __pyx_t_5 = __pyx_temp; + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyNumber_Add(__pyx_t_2, __pyx_t_5); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 388, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":387 + * # If all deltas are exactly the same, return just one (the first one): + * d0 = deltas[0] + * if all(d0 == d for d in deltas): # <<<<<<<<<<<<<< + * return [d0] + [None] * (n - 1) + * +*/ + } + + /* "fontTools/varLib/iup.py":392 + * # Else, solve the general problem using Dynamic Programming. + * + * forced = _iup_contour_bound_forced_set(deltas, coords, tolerance) # <<<<<<<<<<<<<< + * # The _iup_contour_optimize_dp() routine returns the optimal encoding + * # solution given the constraint that the last point is always encoded. +*/ + __pyx_t_5 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_iup_contour_bound_forced_set); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 392, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_5); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[4] = {__pyx_t_5, __pyx_v_deltas, __pyx_v_coords, __pyx_cur_scope->__pyx_v_tolerance}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_6, (4-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 392, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_v_forced = __pyx_t_3; + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":401 + * # if the font size changes (reduced); that would mean the forced-set + * # has members it should not have. + * if forced: # <<<<<<<<<<<<<< + * # Forced set is non-empty: rotate the contour start point + * # such that the last point in the list is a forced point. +*/ + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_forced); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 401, __pyx_L1_error) + if (__pyx_t_4) { + + /* "fontTools/varLib/iup.py":404 + * # Forced set is non-empty: rotate the contour start point + * # such that the last point in the list is a forced point. + * k = (n - 1) - max(forced) # <<<<<<<<<<<<<< + * assert k >= 0 + * +*/ + __pyx_t_3 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 404, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_v_forced}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_max, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 404, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_5 = PyNumber_Subtract(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 404, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_k = __pyx_t_5; + __pyx_t_5 = 0; + + /* "fontTools/varLib/iup.py":405 + * # such that the last point in the list is a forced point. + * k = (n - 1) - max(forced) + * assert k >= 0 # <<<<<<<<<<<<<< + * + * deltas = _rot_list(deltas, k) +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_5 = PyObject_RichCompare(__pyx_v_k, __pyx_mstate_global->__pyx_int_0, Py_GE); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 405, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 405, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 405, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 405, __pyx_L1_error) + #endif + + /* "fontTools/varLib/iup.py":407 + * assert k >= 0 + * + * deltas = _rot_list(deltas, k) # <<<<<<<<<<<<<< + * coords = _rot_list(coords, k) + * forced = _rot_set(forced, k, n) +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_rot_list); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 407, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_v_deltas, __pyx_v_k}; + __pyx_t_5 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 407, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + __Pyx_DECREF_SET(__pyx_v_deltas, __pyx_t_5); + __pyx_t_5 = 0; + + /* "fontTools/varLib/iup.py":408 + * + * deltas = _rot_list(deltas, k) + * coords = _rot_list(coords, k) # <<<<<<<<<<<<<< + * forced = _rot_set(forced, k, n) + * +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_rot_list); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_v_coords, __pyx_v_k}; + __pyx_t_5 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + __Pyx_DECREF_SET(__pyx_v_coords, __pyx_t_5); + __pyx_t_5 = 0; + + /* "fontTools/varLib/iup.py":409 + * deltas = _rot_list(deltas, k) + * coords = _rot_list(coords, k) + * forced = _rot_set(forced, k, n) # <<<<<<<<<<<<<< + * + * # Debugging: Pass a set() instead of forced variable to the next call +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_rot_set); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 409, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[4] = {__pyx_t_2, __pyx_v_forced, __pyx_v_k, __pyx_v_n}; + __pyx_t_5 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_6, (4-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 409, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + __Pyx_DECREF_SET(__pyx_v_forced, __pyx_t_5); + __pyx_t_5 = 0; + + /* "fontTools/varLib/iup.py":413 + * # Debugging: Pass a set() instead of forced variable to the next call + * # to exercise forced-set computation for under-counting. + * chain, costs = _iup_contour_optimize_dp(deltas, coords, forced, tolerance) # <<<<<<<<<<<<<< + * + * # Assemble solution. +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_iup_contour_optimize_dp); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 413, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[5] = {__pyx_t_3, __pyx_v_deltas, __pyx_v_coords, __pyx_v_forced, __pyx_cur_scope->__pyx_v_tolerance}; + __pyx_t_5 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_6, (5-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 413, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { + PyObject* sequence = __pyx_t_5; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 413, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_3); + } else { + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 413, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 413, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + } + #else + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 413, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 413, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_7 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 413, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_8 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_7); + index = 0; __pyx_t_2 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_2)) goto __pyx_L7_unpacking_failed; + __Pyx_GOTREF(__pyx_t_2); + index = 1; __pyx_t_3 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_3)) goto __pyx_L7_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_7), 2) < (0)) __PYX_ERR(0, 413, __pyx_L1_error) + __pyx_t_8 = NULL; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L8_unpacking_done; + __pyx_L7_unpacking_failed:; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_8 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 413, __pyx_L1_error) + __pyx_L8_unpacking_done:; + } + __pyx_v_chain = __pyx_t_2; + __pyx_t_2 = 0; + __pyx_v_costs = __pyx_t_3; + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":416 + * + * # Assemble solution. + * solution = set() # <<<<<<<<<<<<<< + * i = n - 1 + * while i is not None: +*/ + __pyx_t_5 = PySet_New(0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 416, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_v_solution = ((PyObject*)__pyx_t_5); + __pyx_t_5 = 0; + + /* "fontTools/varLib/iup.py":417 + * # Assemble solution. + * solution = set() + * i = n - 1 # <<<<<<<<<<<<<< + * while i is not None: + * solution.add(i) +*/ + __pyx_t_5 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 417, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = 0; + + /* "fontTools/varLib/iup.py":418 + * solution = set() + * i = n - 1 + * while i is not None: # <<<<<<<<<<<<<< + * solution.add(i) + * i = chain[i] +*/ + while (1) { + __pyx_t_4 = (__pyx_v_i != Py_None); + if (!__pyx_t_4) break; + + /* "fontTools/varLib/iup.py":419 + * i = n - 1 + * while i is not None: + * solution.add(i) # <<<<<<<<<<<<<< + * i = chain[i] + * solution.remove(-1) +*/ + __pyx_t_9 = PySet_Add(__pyx_v_solution, __pyx_v_i); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(0, 419, __pyx_L1_error) + + /* "fontTools/varLib/iup.py":420 + * while i is not None: + * solution.add(i) + * i = chain[i] # <<<<<<<<<<<<<< + * solution.remove(-1) + * +*/ + __pyx_t_5 = __Pyx_PyObject_GetItem(__pyx_v_chain, __pyx_v_i); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 420, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF_SET(__pyx_v_i, __pyx_t_5); + __pyx_t_5 = 0; + } + + /* "fontTools/varLib/iup.py":421 + * solution.add(i) + * i = chain[i] + * solution.remove(-1) # <<<<<<<<<<<<<< + * + * # if not forced <= solution: +*/ + __pyx_t_9 = __Pyx_PySet_Remove(__pyx_v_solution, __pyx_mstate_global->__pyx_int_neg_1); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(0, 421, __pyx_L1_error) + + /* "fontTools/varLib/iup.py":427 + * # print("deltas", deltas) + * # print("len", len(deltas)) + * assert forced <= solution, (forced, solution) # <<<<<<<<<<<<<< + * + * deltas = [deltas[i] if i in solution else None for i in range(n)] +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_5 = PyObject_RichCompare(__pyx_v_forced, __pyx_v_solution, Py_LE); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 427, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 427, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) { + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 427, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_INCREF(__pyx_v_forced); + __Pyx_GIVEREF(__pyx_v_forced); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_v_forced) != (0)) __PYX_ERR(0, 427, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_solution); + __Pyx_GIVEREF(__pyx_v_solution); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_solution) != (0)) __PYX_ERR(0, 427, __pyx_L1_error); + __pyx_t_3 = PyTuple_Pack(1, __pyx_t_5); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 427, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), __pyx_t_3, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 427, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 427, __pyx_L1_error) + #endif + + /* "fontTools/varLib/iup.py":429 + * assert forced <= solution, (forced, solution) + * + * deltas = [deltas[i] if i in solution else None for i in range(n)] # <<<<<<<<<<<<<< + * + * deltas = _rot_list(deltas, -k) +*/ + { /* enter inner scope */ + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 429, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_n}; + __pyx_t_5 = __Pyx_PyObject_FastCall((PyObject*)(&PyRange_Type), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 429, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); + } + __pyx_t_2 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 429, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_10 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 429, __pyx_L13_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + for (;;) { + { + __pyx_t_5 = __pyx_t_10(__pyx_t_2); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 429, __pyx_L13_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_8genexpr5__pyx_v_i, __pyx_t_5); + __pyx_t_5 = 0; + __pyx_t_4 = (__Pyx_PySet_ContainsTF(__pyx_8genexpr5__pyx_v_i, __pyx_v_solution, Py_EQ)); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 429, __pyx_L13_error) + if (__pyx_t_4) { + __pyx_t_7 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_8genexpr5__pyx_v_i); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 429, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_5 = __pyx_t_7; + __pyx_t_7 = 0; + } else { + __Pyx_INCREF(Py_None); + __pyx_t_5 = Py_None; + } + if (unlikely(__Pyx_ListComp_Append(__pyx_t_3, (PyObject*)__pyx_t_5))) __PYX_ERR(0, 429, __pyx_L13_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_8genexpr5__pyx_v_i); __pyx_8genexpr5__pyx_v_i = 0; + goto __pyx_L17_exit_scope; + __pyx_L13_error:; + __Pyx_XDECREF(__pyx_8genexpr5__pyx_v_i); __pyx_8genexpr5__pyx_v_i = 0; + goto __pyx_L1_error; + __pyx_L17_exit_scope:; + } /* exit inner scope */ + __Pyx_DECREF_SET(__pyx_v_deltas, __pyx_t_3); + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":431 + * deltas = [deltas[i] if i in solution else None for i in range(n)] + * + * deltas = _rot_list(deltas, -k) # <<<<<<<<<<<<<< + * else: + * # Repeat the contour an extra time, solve the new case, then look for solutions of the +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_rot_list); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 431, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = PyNumber_Negative(__pyx_v_k); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 431, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_v_deltas, __pyx_t_7}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 431, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_DECREF_SET(__pyx_v_deltas, __pyx_t_3); + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":401 + * # if the font size changes (reduced); that would mean the forced-set + * # has members it should not have. + * if forced: # <<<<<<<<<<<<<< + * # Forced set is non-empty: rotate the contour start point + * # such that the last point in the list is a forced point. +*/ + goto __pyx_L6; + } + + /* "fontTools/varLib/iup.py":436 + * # circular n-length problem in the solution for new linear case. I cannot prove that + * # this always produces the optimal solution... + * chain, costs = _iup_contour_optimize_dp( # <<<<<<<<<<<<<< + * deltas + deltas, coords + coords, forced, tolerance, n + * ) +*/ + /*else*/ { + __pyx_t_5 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_iup_contour_optimize_dp); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 436, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + + /* "fontTools/varLib/iup.py":437 + * # this always produces the optimal solution... + * chain, costs = _iup_contour_optimize_dp( + * deltas + deltas, coords + coords, forced, tolerance, n # <<<<<<<<<<<<<< + * ) + * best_sol, best_cost = None, n + 1 +*/ + __pyx_t_2 = PyNumber_Add(__pyx_v_deltas, __pyx_v_deltas); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 437, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_11 = PyNumber_Add(__pyx_v_coords, __pyx_v_coords); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 437, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_7))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_7); + assert(__pyx_t_5); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_7); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_7, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[6] = {__pyx_t_5, __pyx_t_2, __pyx_t_11, __pyx_v_forced, __pyx_cur_scope->__pyx_v_tolerance, __pyx_v_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_7, __pyx_callargs+__pyx_t_6, (6-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 436, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + if ((likely(PyTuple_CheckExact(__pyx_t_3))) || (PyList_CheckExact(__pyx_t_3))) { + PyObject* sequence = __pyx_t_3; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 436, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_7 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_7); + __pyx_t_11 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_11); + } else { + __pyx_t_7 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 436, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_7); + __pyx_t_11 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 436, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_11); + } + #else + __pyx_t_7 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 436, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_11 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 436, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + #endif + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_2 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 436, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_8 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); + index = 0; __pyx_t_7 = __pyx_t_8(__pyx_t_2); if (unlikely(!__pyx_t_7)) goto __pyx_L18_unpacking_failed; + __Pyx_GOTREF(__pyx_t_7); + index = 1; __pyx_t_11 = __pyx_t_8(__pyx_t_2); if (unlikely(!__pyx_t_11)) goto __pyx_L18_unpacking_failed; + __Pyx_GOTREF(__pyx_t_11); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_2), 2) < (0)) __PYX_ERR(0, 436, __pyx_L1_error) + __pyx_t_8 = NULL; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + goto __pyx_L19_unpacking_done; + __pyx_L18_unpacking_failed:; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_8 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 436, __pyx_L1_error) + __pyx_L19_unpacking_done:; + } + + /* "fontTools/varLib/iup.py":436 + * # circular n-length problem in the solution for new linear case. I cannot prove that + * # this always produces the optimal solution... + * chain, costs = _iup_contour_optimize_dp( # <<<<<<<<<<<<<< + * deltas + deltas, coords + coords, forced, tolerance, n + * ) +*/ + __pyx_v_chain = __pyx_t_7; + __pyx_t_7 = 0; + __pyx_v_costs = __pyx_t_11; + __pyx_t_11 = 0; + + /* "fontTools/varLib/iup.py":439 + * deltas + deltas, coords + coords, forced, tolerance, n + * ) + * best_sol, best_cost = None, n + 1 # <<<<<<<<<<<<<< + * + * for start in range(n - 1, len(costs) - 1): +*/ + __pyx_t_3 = Py_None; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_11 = __Pyx_PyLong_AddObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 439, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_v_best_sol = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + __pyx_v_best_cost = __pyx_t_11; + __pyx_t_11 = 0; + + /* "fontTools/varLib/iup.py":441 + * best_sol, best_cost = None, n + 1 + * + * for start in range(n - 1, len(costs) - 1): # <<<<<<<<<<<<<< + * # Assemble solution. + * solution = set() +*/ + __pyx_t_3 = NULL; + __pyx_t_7 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 441, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = PyObject_Length(__pyx_v_costs); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(0, 441, __pyx_L1_error) + __pyx_t_2 = PyLong_FromSsize_t((__pyx_t_1 - 1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 441, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_t_7, __pyx_t_2}; + __pyx_t_11 = __Pyx_PyObject_FastCall((PyObject*)(&PyRange_Type), __pyx_callargs+__pyx_t_6, (3-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 441, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + __pyx_t_2 = PyObject_GetIter(__pyx_t_11); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 441, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_10 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 441, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + for (;;) { + { + __pyx_t_11 = __pyx_t_10(__pyx_t_2); + if (unlikely(!__pyx_t_11)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 441, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_11); + __Pyx_XDECREF_SET(__pyx_v_start, __pyx_t_11); + __pyx_t_11 = 0; + + /* "fontTools/varLib/iup.py":443 + * for start in range(n - 1, len(costs) - 1): + * # Assemble solution. + * solution = set() # <<<<<<<<<<<<<< + * i = start + * while i > start - n: +*/ + __pyx_t_11 = PySet_New(0); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 443, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_XDECREF_SET(__pyx_v_solution, ((PyObject*)__pyx_t_11)); + __pyx_t_11 = 0; + + /* "fontTools/varLib/iup.py":444 + * # Assemble solution. + * solution = set() + * i = start # <<<<<<<<<<<<<< + * while i > start - n: + * solution.add(i % n) +*/ + __Pyx_INCREF(__pyx_v_start); + __Pyx_XDECREF_SET(__pyx_v_i, __pyx_v_start); + + /* "fontTools/varLib/iup.py":445 + * solution = set() + * i = start + * while i > start - n: # <<<<<<<<<<<<<< + * solution.add(i % n) + * i = chain[i] +*/ + while (1) { + __pyx_t_11 = PyNumber_Subtract(__pyx_v_start, __pyx_v_n); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 445, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_7 = PyObject_RichCompare(__pyx_v_i, __pyx_t_11, Py_GT); __Pyx_XGOTREF(__pyx_t_7); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 445, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_7); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 445, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (!__pyx_t_4) break; + + /* "fontTools/varLib/iup.py":446 + * i = start + * while i > start - n: + * solution.add(i % n) # <<<<<<<<<<<<<< + * i = chain[i] + * if i == start - n: +*/ + __pyx_t_7 = PyNumber_Remainder(__pyx_v_i, __pyx_v_n); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 446, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_9 = PySet_Add(__pyx_v_solution, __pyx_t_7); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(0, 446, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":447 + * while i > start - n: + * solution.add(i % n) + * i = chain[i] # <<<<<<<<<<<<<< + * if i == start - n: + * cost = costs[start] - costs[start - n] +*/ + __pyx_t_7 = __Pyx_PyObject_GetItem(__pyx_v_chain, __pyx_v_i); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 447, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF_SET(__pyx_v_i, __pyx_t_7); + __pyx_t_7 = 0; + } + + /* "fontTools/varLib/iup.py":448 + * solution.add(i % n) + * i = chain[i] + * if i == start - n: # <<<<<<<<<<<<<< + * cost = costs[start] - costs[start - n] + * if cost <= best_cost: +*/ + __pyx_t_7 = PyNumber_Subtract(__pyx_v_start, __pyx_v_n); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 448, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_11 = PyObject_RichCompare(__pyx_v_i, __pyx_t_7, Py_EQ); __Pyx_XGOTREF(__pyx_t_11); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 448, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_11); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 448, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (__pyx_t_4) { + + /* "fontTools/varLib/iup.py":449 + * i = chain[i] + * if i == start - n: + * cost = costs[start] - costs[start - n] # <<<<<<<<<<<<<< + * if cost <= best_cost: + * best_sol, best_cost = solution, cost +*/ + __pyx_t_11 = __Pyx_PyObject_GetItem(__pyx_v_costs, __pyx_v_start); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 449, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_7 = PyNumber_Subtract(__pyx_v_start, __pyx_v_n); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 449, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_3 = __Pyx_PyObject_GetItem(__pyx_v_costs, __pyx_t_7); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 449, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = PyNumber_Subtract(__pyx_t_11, __pyx_t_3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 449, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF_SET(__pyx_v_cost, __pyx_t_7); + __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":450 + * if i == start - n: + * cost = costs[start] - costs[start - n] + * if cost <= best_cost: # <<<<<<<<<<<<<< + * best_sol, best_cost = solution, cost + * +*/ + __pyx_t_7 = PyObject_RichCompare(__pyx_v_cost, __pyx_v_best_cost, Py_LE); __Pyx_XGOTREF(__pyx_t_7); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 450, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_7); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 450, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (__pyx_t_4) { + + /* "fontTools/varLib/iup.py":451 + * cost = costs[start] - costs[start - n] + * if cost <= best_cost: + * best_sol, best_cost = solution, cost # <<<<<<<<<<<<<< + * + * # if not forced <= best_sol: +*/ + __pyx_t_7 = __pyx_v_solution; + __Pyx_INCREF(__pyx_t_7); + __pyx_t_3 = __pyx_v_cost; + __Pyx_INCREF(__pyx_t_3); + __Pyx_DECREF_SET(__pyx_v_best_sol, ((PyObject*)__pyx_t_7)); + __pyx_t_7 = 0; + __Pyx_DECREF_SET(__pyx_v_best_cost, __pyx_t_3); + __pyx_t_3 = 0; + + /* "fontTools/varLib/iup.py":450 + * if i == start - n: + * cost = costs[start] - costs[start - n] + * if cost <= best_cost: # <<<<<<<<<<<<<< + * best_sol, best_cost = solution, cost + * +*/ + } + + /* "fontTools/varLib/iup.py":448 + * solution.add(i % n) + * i = chain[i] + * if i == start - n: # <<<<<<<<<<<<<< + * cost = costs[start] - costs[start - n] + * if cost <= best_cost: +*/ + } + + /* "fontTools/varLib/iup.py":441 + * best_sol, best_cost = None, n + 1 + * + * for start in range(n - 1, len(costs) - 1): # <<<<<<<<<<<<<< + * # Assemble solution. + * solution = set() +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "fontTools/varLib/iup.py":457 + * # print("deltas", deltas) + * # print("len", len(deltas)) + * assert forced <= best_sol, (forced, best_sol) # <<<<<<<<<<<<<< + * + * deltas = [deltas[i] if i in best_sol else None for i in range(n)] +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_2 = PyObject_RichCompare(__pyx_v_forced, __pyx_v_best_sol, Py_LE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 457, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 457, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_4)) { + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 457, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_v_forced); + __Pyx_GIVEREF(__pyx_v_forced); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_forced) != (0)) __PYX_ERR(0, 457, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_best_sol); + __Pyx_GIVEREF(__pyx_v_best_sol); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_best_sol) != (0)) __PYX_ERR(0, 457, __pyx_L1_error); + __pyx_t_3 = PyTuple_Pack(1, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 457, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), __pyx_t_3, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 457, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 457, __pyx_L1_error) + #endif + + /* "fontTools/varLib/iup.py":459 + * assert forced <= best_sol, (forced, best_sol) + * + * deltas = [deltas[i] if i in best_sol else None for i in range(n)] # <<<<<<<<<<<<<< + * + * return deltas +*/ + { /* enter inner scope */ + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 459, __pyx_L29_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_7 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_7, __pyx_v_n}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)(&PyRange_Type), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 459, __pyx_L29_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_7 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 459, __pyx_L29_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_10 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_7); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 459, __pyx_L29_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + for (;;) { + { + __pyx_t_2 = __pyx_t_10(__pyx_t_7); + if (unlikely(!__pyx_t_2)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 459, __pyx_L29_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XDECREF_SET(__pyx_8genexpr6__pyx_v_i, __pyx_t_2); + __pyx_t_2 = 0; + if (unlikely(__pyx_v_best_sol == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 459, __pyx_L29_error) + } + __pyx_t_4 = (__Pyx_PySet_ContainsTF(__pyx_8genexpr6__pyx_v_i, __pyx_v_best_sol, Py_EQ)); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 459, __pyx_L29_error) + if (__pyx_t_4) { + __pyx_t_11 = __Pyx_PyObject_GetItem(__pyx_v_deltas, __pyx_8genexpr6__pyx_v_i); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 459, __pyx_L29_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_2 = __pyx_t_11; + __pyx_t_11 = 0; + } else { + __Pyx_INCREF(Py_None); + __pyx_t_2 = Py_None; + } + if (unlikely(__Pyx_ListComp_Append(__pyx_t_3, (PyObject*)__pyx_t_2))) __PYX_ERR(0, 459, __pyx_L29_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_8genexpr6__pyx_v_i); __pyx_8genexpr6__pyx_v_i = 0; + goto __pyx_L33_exit_scope; + __pyx_L29_error:; + __Pyx_XDECREF(__pyx_8genexpr6__pyx_v_i); __pyx_8genexpr6__pyx_v_i = 0; + goto __pyx_L1_error; + __pyx_L33_exit_scope:; + } /* exit inner scope */ + __Pyx_DECREF_SET(__pyx_v_deltas, __pyx_t_3); + __pyx_t_3 = 0; + } + __pyx_L6:; + + /* "fontTools/varLib/iup.py":461 + * deltas = [deltas[i] if i in best_sol else None for i in range(n)] + * + * return deltas # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_deltas); + __pyx_r = __pyx_v_deltas; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":363 + * + * + * def iup_contour_optimize( # <<<<<<<<<<<<<< + * deltas: _DeltaSegment, coords: _PointSegment, tolerance: Real = 0.0 + * ) -> _DeltaOrNoneSegment: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_AddTraceback("fontTools.varLib.iup.iup_contour_optimize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_n); + __Pyx_XDECREF(__pyx_v_forced); + __Pyx_XDECREF(__pyx_v_k); + __Pyx_XDECREF(__pyx_v_chain); + __Pyx_XDECREF(__pyx_v_costs); + __Pyx_XDECREF(__pyx_v_solution); + __Pyx_XDECREF(__pyx_v_i); + __Pyx_XDECREF(__pyx_v_best_sol); + __Pyx_XDECREF(__pyx_v_best_cost); + __Pyx_XDECREF(__pyx_v_start); + __Pyx_XDECREF(__pyx_v_cost); + __Pyx_XDECREF(__pyx_gb_9fontTools_6varLib_3iup_20iup_contour_optimize_2generator1); + __Pyx_XDECREF(__pyx_gb_9fontTools_6varLib_3iup_20iup_contour_optimize_5generator2); + __Pyx_XDECREF(__pyx_8genexpr5__pyx_v_i); + __Pyx_XDECREF(__pyx_8genexpr6__pyx_v_i); + __Pyx_XDECREF(__pyx_v_deltas); + __Pyx_XDECREF(__pyx_v_coords); + __Pyx_DECREF((PyObject *)__pyx_cur_scope); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "fontTools/varLib/iup.py":464 + * + * + * def iup_delta_optimize( # <<<<<<<<<<<<<< + * deltas: _DeltaSegment, + * coords: _PointSegment, +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_15iup_delta_optimize(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_9fontTools_6varLib_3iup_14iup_delta_optimize, "iup_delta_optimize(deltas: _DeltaSegment, coords: _PointSegment, ends: _Endpoints, tolerance: Real = 0.0) -> _DeltaOrNoneSegment\n\nFor the outline given in `coords`, with contour endpoints given\nin sorted increasing order in `ends`, optimize a set of delta\nvalues `deltas` within error `tolerance`.\n\nReturns delta vector that has most number of None items instead of\nthe input delta."); +static PyMethodDef __pyx_mdef_9fontTools_6varLib_3iup_15iup_delta_optimize = {"iup_delta_optimize", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_6varLib_3iup_15iup_delta_optimize, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_6varLib_3iup_14iup_delta_optimize}; +static PyObject *__pyx_pw_9fontTools_6varLib_3iup_15iup_delta_optimize(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_deltas = 0; + PyObject *__pyx_v_coords = 0; + PyObject *__pyx_v_ends = 0; + PyObject *__pyx_v_tolerance = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("iup_delta_optimize (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_deltas,&__pyx_mstate_global->__pyx_n_u_coords,&__pyx_mstate_global->__pyx_n_u_ends,&__pyx_mstate_global->__pyx_n_u_tolerance,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 464, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 464, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 464, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 464, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 464, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "iup_delta_optimize", 0) < (0)) __PYX_ERR(0, 464, __pyx_L3_error) + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_float_0_0))); + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("iup_delta_optimize", 0, 3, 4, i); __PYX_ERR(0, 464, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 464, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 464, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 464, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 464, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_float_0_0))); + } + __pyx_v_deltas = values[0]; + __pyx_v_coords = values[1]; + __pyx_v_ends = values[2]; + __pyx_v_tolerance = values[3]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("iup_delta_optimize", 0, 3, 4, __pyx_nargs); __PYX_ERR(0, 464, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("fontTools.varLib.iup.iup_delta_optimize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_9fontTools_6varLib_3iup_14iup_delta_optimize(__pyx_self, __pyx_v_deltas, __pyx_v_coords, __pyx_v_ends, __pyx_v_tolerance); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_9fontTools_6varLib_3iup_14iup_delta_optimize(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_deltas, PyObject *__pyx_v_coords, PyObject *__pyx_v_ends, PyObject *__pyx_v_tolerance) { + PyObject *__pyx_v_n = NULL; + PyObject *__pyx_v_out = NULL; + PyObject *__pyx_v_start = NULL; + PyObject *__pyx_v_end = NULL; + PyObject *__pyx_v_contour = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *(*__pyx_t_9)(PyObject *); + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + size_t __pyx_t_12; + Py_ssize_t __pyx_t_13; + int __pyx_t_14; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("iup_delta_optimize", 0); + __Pyx_INCREF(__pyx_v_ends); + + /* "fontTools/varLib/iup.py":477 + * the input delta. + * """ + * assert sorted(ends) == ends and len(coords) == (ends[-1] + 1 if ends else 0) + 4 # <<<<<<<<<<<<<< + * n = len(coords) + * ends = ends + [n - 4, n - 3, n - 2, n - 1] +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_2 = PySequence_List(__pyx_v_ends); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (unlikely((PyList_Sort(__pyx_t_2) < 0))) __PYX_ERR(0, 477, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_2, __pyx_v_ends, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (__pyx_t_4) { + } else { + __pyx_t_1 = __pyx_t_4; + goto __pyx_L3_bool_binop_done; + } + __pyx_t_5 = PyObject_Length(__pyx_v_coords); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 477, __pyx_L1_error) + __pyx_t_3 = PyLong_FromSsize_t(__pyx_t_5); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_ends); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 477, __pyx_L1_error) + if (__pyx_t_4) { + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_ends, -1L, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyLong_AddObjC(__pyx_t_6, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_2 = __pyx_t_7; + __pyx_t_7 = 0; + } else { + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __pyx_t_2 = __pyx_mstate_global->__pyx_int_0; + } + __pyx_t_7 = __Pyx_PyLong_AddObjC(__pyx_t_2, __pyx_mstate_global->__pyx_int_4, 4, 0, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = PyObject_RichCompare(__pyx_t_3, __pyx_t_7, Py_EQ); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_1 = __pyx_t_4; + __pyx_L3_bool_binop_done:; + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 477, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 477, __pyx_L1_error) + #endif + + /* "fontTools/varLib/iup.py":478 + * """ + * assert sorted(ends) == ends and len(coords) == (ends[-1] + 1 if ends else 0) + 4 + * n = len(coords) # <<<<<<<<<<<<<< + * ends = ends + [n - 4, n - 3, n - 2, n - 1] + * out = [] +*/ + __pyx_t_5 = PyObject_Length(__pyx_v_coords); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 478, __pyx_L1_error) + __pyx_t_2 = PyLong_FromSsize_t(__pyx_t_5); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 478, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_n = __pyx_t_2; + __pyx_t_2 = 0; + + /* "fontTools/varLib/iup.py":479 + * assert sorted(ends) == ends and len(coords) == (ends[-1] + 1 if ends else 0) + 4 + * n = len(coords) + * ends = ends + [n - 4, n - 3, n - 2, n - 1] # <<<<<<<<<<<<<< + * out = [] + * start = 0 +*/ + __pyx_t_2 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_4, 4, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 479, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_3, 3, 0, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 479, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_3 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_2, 2, 0, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 479, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = __Pyx_PyLong_SubtractObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 479, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = PyList_New(4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 479, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyList_SET_ITEM(__pyx_t_8, 0, __pyx_t_2) != (0)) __PYX_ERR(0, 479, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyList_SET_ITEM(__pyx_t_8, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 479, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyList_SET_ITEM(__pyx_t_8, 2, __pyx_t_3) != (0)) __PYX_ERR(0, 479, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyList_SET_ITEM(__pyx_t_8, 3, __pyx_t_6) != (0)) __PYX_ERR(0, 479, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_t_7 = 0; + __pyx_t_3 = 0; + __pyx_t_6 = 0; + __pyx_t_6 = PyNumber_Add(__pyx_v_ends, __pyx_t_8); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 479, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF_SET(__pyx_v_ends, __pyx_t_6); + __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":480 + * n = len(coords) + * ends = ends + [n - 4, n - 3, n - 2, n - 1] + * out = [] # <<<<<<<<<<<<<< + * start = 0 + * for end in ends: +*/ + __pyx_t_6 = PyList_New(0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 480, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_v_out = ((PyObject*)__pyx_t_6); + __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":481 + * ends = ends + [n - 4, n - 3, n - 2, n - 1] + * out = [] + * start = 0 # <<<<<<<<<<<<<< + * for end in ends: + * contour = iup_contour_optimize( +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __pyx_v_start = __pyx_mstate_global->__pyx_int_0; + + /* "fontTools/varLib/iup.py":482 + * out = [] + * start = 0 + * for end in ends: # <<<<<<<<<<<<<< + * contour = iup_contour_optimize( + * deltas[start : end + 1], coords[start : end + 1], tolerance +*/ + if (likely(PyList_CheckExact(__pyx_v_ends)) || PyTuple_CheckExact(__pyx_v_ends)) { + __pyx_t_6 = __pyx_v_ends; __Pyx_INCREF(__pyx_t_6); + __pyx_t_5 = 0; + __pyx_t_9 = NULL; + } else { + __pyx_t_5 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_ends); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 482, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_9 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 482, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_9)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 482, __pyx_L1_error) + #endif + if (__pyx_t_5 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRefFast(__pyx_t_6, __pyx_t_5, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_5; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 482, __pyx_L1_error) + #endif + if (__pyx_t_5 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_5)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_5); + #endif + ++__pyx_t_5; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 482, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_9(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 482, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_end, __pyx_t_8); + __pyx_t_8 = 0; + + /* "fontTools/varLib/iup.py":483 + * start = 0 + * for end in ends: + * contour = iup_contour_optimize( # <<<<<<<<<<<<<< + * deltas[start : end + 1], coords[start : end + 1], tolerance + * ) +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_iup_contour_optimize); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 483, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + + /* "fontTools/varLib/iup.py":484 + * for end in ends: + * contour = iup_contour_optimize( + * deltas[start : end + 1], coords[start : end + 1], tolerance # <<<<<<<<<<<<<< + * ) + * assert len(contour) == end - start + 1 +*/ + __pyx_t_2 = __Pyx_PyLong_AddObjC(__pyx_v_end, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 484, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_10 = __Pyx_PyObject_GetSlice(__pyx_v_deltas, 0, 0, &__pyx_v_start, &__pyx_t_2, NULL, 0, 0, 1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 484, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyLong_AddObjC(__pyx_v_end, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 484, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_11 = __Pyx_PyObject_GetSlice(__pyx_v_coords, 0, 0, &__pyx_v_start, &__pyx_t_2, NULL, 0, 0, 1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 484, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_7))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_7); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_7); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_7, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[4] = {__pyx_t_3, __pyx_t_10, __pyx_t_11, __pyx_v_tolerance}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_7, __pyx_callargs+__pyx_t_12, (4-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 483, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_XDECREF_SET(__pyx_v_contour, __pyx_t_8); + __pyx_t_8 = 0; + + /* "fontTools/varLib/iup.py":486 + * deltas[start : end + 1], coords[start : end + 1], tolerance + * ) + * assert len(contour) == end - start + 1 # <<<<<<<<<<<<<< + * out.extend(contour) + * start = end + 1 +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_13 = PyObject_Length(__pyx_v_contour); if (unlikely(__pyx_t_13 == ((Py_ssize_t)-1))) __PYX_ERR(0, 486, __pyx_L1_error) + __pyx_t_8 = PyLong_FromSsize_t(__pyx_t_13); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 486, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = PyNumber_Subtract(__pyx_v_end, __pyx_v_start); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 486, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_11 = __Pyx_PyLong_AddObjC(__pyx_t_7, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 486, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = PyObject_RichCompare(__pyx_t_8, __pyx_t_11, Py_EQ); __Pyx_XGOTREF(__pyx_t_7); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 486, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_7); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 486, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 486, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 486, __pyx_L1_error) + #endif + + /* "fontTools/varLib/iup.py":487 + * ) + * assert len(contour) == end - start + 1 + * out.extend(contour) # <<<<<<<<<<<<<< + * start = end + 1 + * +*/ + __pyx_t_14 = __Pyx_PyList_Extend(__pyx_v_out, __pyx_v_contour); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 487, __pyx_L1_error) + + /* "fontTools/varLib/iup.py":488 + * assert len(contour) == end - start + 1 + * out.extend(contour) + * start = end + 1 # <<<<<<<<<<<<<< + * + * return out +*/ + __pyx_t_7 = __Pyx_PyLong_AddObjC(__pyx_v_end, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 488, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF_SET(__pyx_v_start, __pyx_t_7); + __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":482 + * out = [] + * start = 0 + * for end in ends: # <<<<<<<<<<<<<< + * contour = iup_contour_optimize( + * deltas[start : end + 1], coords[start : end + 1], tolerance +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":490 + * start = end + 1 + * + * return out # <<<<<<<<<<<<<< +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_out); + __pyx_r = __pyx_v_out; + goto __pyx_L0; + + /* "fontTools/varLib/iup.py":464 + * + * + * def iup_delta_optimize( # <<<<<<<<<<<<<< + * deltas: _DeltaSegment, + * coords: _PointSegment, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_AddTraceback("fontTools.varLib.iup.iup_delta_optimize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_n); + __Pyx_XDECREF(__pyx_v_out); + __Pyx_XDECREF(__pyx_v_start); + __Pyx_XDECREF(__pyx_v_end); + __Pyx_XDECREF(__pyx_v_contour); + __Pyx_XDECREF(__pyx_v_ends); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_defaults(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + return o; +} + +static void __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_defaults(PyObject *o) { + struct __pyx_defaults *p = (struct __pyx_defaults *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_defaults) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->arg0); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_defaults(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_defaults *p = (struct __pyx_defaults *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->arg0) { + e = (*v)(p->arg0, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_9fontTools_6varLib_3iup___pyx_defaults(PyObject *o) { + PyObject* tmp; + struct __pyx_defaults *p = (struct __pyx_defaults *)o; + tmp = ((PyObject*)p->arg0); + p->arg0 = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_9fontTools_6varLib_3iup___pyx_defaults_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_defaults}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_defaults}, + {Py_tp_clear, (void *)__pyx_tp_clear_9fontTools_6varLib_3iup___pyx_defaults}, + {Py_tp_new, (void *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_defaults}, + {0, 0}, +}; +static PyType_Spec __pyx_type_9fontTools_6varLib_3iup___pyx_defaults_spec = { + "fontTools.varLib.iup.__pyx_defaults", + sizeof(struct __pyx_defaults), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, + __pyx_type_9fontTools_6varLib_3iup___pyx_defaults_slots, +}; +#else + +static PyTypeObject __pyx_type_9fontTools_6varLib_3iup___pyx_defaults = { + PyVarObject_HEAD_INIT(0, 0) + "fontTools.varLib.iup.""__pyx_defaults", /*tp_name*/ + sizeof(struct __pyx_defaults), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_defaults, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_defaults, /*tp_traverse*/ + __pyx_tp_clear_9fontTools_6varLib_3iup___pyx_defaults, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_9fontTools_6varLib_3iup___pyx_defaults, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between > 0) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between)))) + { + o = (PyObject*)__pyx_mstate_global->__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between[--__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between]; + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(Py_TYPE(o)); + #endif + memset(o, 0, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between)); + #if CYTHON_COMPILING_IN_LIMITED_API + (void) PyObject_Init(o, t); + #else + (void) PyObject_INIT(o, t); + #endif + PyObject_GC_Track(o); + } else + #endif + { + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between(PyObject *o) { + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->__pyx_v_tolerance); + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between < 8) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(Py_TYPE(o), __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between)))) + { + __pyx_mstate_global->__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between[__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between++] = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *)o); + } else + #endif + { + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif + } +} + +static int __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->__pyx_v_tolerance) { + e = (*v)(p->__pyx_v_tolerance, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between *)o; + tmp = ((PyObject*)p->__pyx_v_tolerance); + p->__pyx_v_tolerance = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between}, + {Py_tp_clear, (void *)__pyx_tp_clear_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between}, + {Py_tp_new, (void *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between}, + {0, 0}, +}; +static PyType_Spec __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between_spec = { + "fontTools.varLib.iup.__pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between", + sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, + __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between_slots, +}; +#else + +static PyTypeObject __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between = { + PyVarObject_HEAD_INIT(0, 0) + "fontTools.varLib.iup.""__pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between", /*tp_name*/ + sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between, /*tp_traverse*/ + __pyx_tp_clear_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr > 0) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr)))) + { + o = (PyObject*)__pyx_mstate_global->__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr[--__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr]; + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(Py_TYPE(o)); + #endif + memset(o, 0, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr)); + #if CYTHON_COMPILING_IN_LIMITED_API + (void) PyObject_Init(o, t); + #else + (void) PyObject_INIT(o, t); + #endif + PyObject_GC_Track(o); + } else + #endif + { + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr(PyObject *o) { + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->__pyx_outer_scope); + Py_CLEAR(p->__pyx_genexpr_arg_0); + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr < 8) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(Py_TYPE(o), __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr)))) + { + __pyx_mstate_global->__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr[__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr++] = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *)o); + } else + #endif + { + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif + } +} + +static int __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->__pyx_outer_scope) { + e = (*v)(((PyObject *)p->__pyx_outer_scope), a); if (e) return e; + } + if (p->__pyx_genexpr_arg_0) { + e = (*v)(p->__pyx_genexpr_arg_0, a); if (e) return e; + } + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr}, + {Py_tp_new, (void *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr}, + {0, 0}, +}; +static PyType_Spec __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr_spec = { + "fontTools.varLib.iup.__pyx_scope_struct_1_genexpr", + sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, + __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr_slots, +}; +#else + +static PyTypeObject __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr = { + PyVarObject_HEAD_INIT(0, 0) + "fontTools.varLib.iup.""__pyx_scope_struct_1_genexpr", /*tp_name*/ + sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize > 0) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize)))) + { + o = (PyObject*)__pyx_mstate_global->__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize[--__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize]; + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(Py_TYPE(o)); + #endif + memset(o, 0, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize)); + #if CYTHON_COMPILING_IN_LIMITED_API + (void) PyObject_Init(o, t); + #else + (void) PyObject_INIT(o, t); + #endif + PyObject_GC_Track(o); + } else + #endif + { + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize(PyObject *o) { + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->__pyx_v_d0); + Py_CLEAR(p->__pyx_v_tolerance); + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize < 8) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(Py_TYPE(o), __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize)))) + { + __pyx_mstate_global->__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize[__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize++] = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *)o); + } else + #endif + { + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif + } +} + +static int __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->__pyx_v_d0) { + e = (*v)(p->__pyx_v_d0, a); if (e) return e; + } + if (p->__pyx_v_tolerance) { + e = (*v)(p->__pyx_v_tolerance, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize *)o; + tmp = ((PyObject*)p->__pyx_v_d0); + p->__pyx_v_d0 = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->__pyx_v_tolerance); + p->__pyx_v_tolerance = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize}, + {Py_tp_clear, (void *)__pyx_tp_clear_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize}, + {Py_tp_new, (void *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize}, + {0, 0}, +}; +static PyType_Spec __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize_spec = { + "fontTools.varLib.iup.__pyx_scope_struct_2_iup_contour_optimize", + sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, + __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize_slots, +}; +#else + +static PyTypeObject __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize = { + PyVarObject_HEAD_INIT(0, 0) + "fontTools.varLib.iup.""__pyx_scope_struct_2_iup_contour_optimize", /*tp_name*/ + sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize, /*tp_traverse*/ + __pyx_tp_clear_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr > 0) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr)))) + { + o = (PyObject*)__pyx_mstate_global->__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr[--__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr]; + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(Py_TYPE(o)); + #endif + memset(o, 0, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr)); + #if CYTHON_COMPILING_IN_LIMITED_API + (void) PyObject_Init(o, t); + #else + (void) PyObject_INIT(o, t); + #endif + PyObject_GC_Track(o); + } else + #endif + { + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr(PyObject *o) { + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->__pyx_outer_scope); + Py_CLEAR(p->__pyx_genexpr_arg_0); + Py_CLEAR(p->__pyx_v_p); + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr < 8) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(Py_TYPE(o), __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr)))) + { + __pyx_mstate_global->__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr[__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr++] = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *)o); + } else + #endif + { + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif + } +} + +static int __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->__pyx_outer_scope) { + e = (*v)(((PyObject *)p->__pyx_outer_scope), a); if (e) return e; + } + if (p->__pyx_genexpr_arg_0) { + e = (*v)(p->__pyx_genexpr_arg_0, a); if (e) return e; + } + if (p->__pyx_v_p) { + e = (*v)(p->__pyx_v_p, a); if (e) return e; + } + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr}, + {Py_tp_new, (void *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr}, + {0, 0}, +}; +static PyType_Spec __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr_spec = { + "fontTools.varLib.iup.__pyx_scope_struct_3_genexpr", + sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, + __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr_slots, +}; +#else + +static PyTypeObject __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr = { + PyVarObject_HEAD_INIT(0, 0) + "fontTools.varLib.iup.""__pyx_scope_struct_3_genexpr", /*tp_name*/ + sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr > 0) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr)))) + { + o = (PyObject*)__pyx_mstate_global->__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr[--__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr]; + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(Py_TYPE(o)); + #endif + memset(o, 0, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr)); + #if CYTHON_COMPILING_IN_LIMITED_API + (void) PyObject_Init(o, t); + #else + (void) PyObject_INIT(o, t); + #endif + PyObject_GC_Track(o); + } else + #endif + { + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr(PyObject *o) { + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->__pyx_outer_scope); + Py_CLEAR(p->__pyx_genexpr_arg_0); + Py_CLEAR(p->__pyx_v_d); + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr < 8) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(Py_TYPE(o), __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr, sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr)))) + { + __pyx_mstate_global->__pyx_freelist_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr[__pyx_mstate_global->__pyx_freecount_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr++] = ((struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *)o); + } else + #endif + { + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif + } +} + +static int __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *p = (struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->__pyx_outer_scope) { + e = (*v)(((PyObject *)p->__pyx_outer_scope), a); if (e) return e; + } + if (p->__pyx_genexpr_arg_0) { + e = (*v)(p->__pyx_genexpr_arg_0, a); if (e) return e; + } + if (p->__pyx_v_d) { + e = (*v)(p->__pyx_v_d, a); if (e) return e; + } + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr}, + {Py_tp_new, (void *)__pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr}, + {0, 0}, +}; +static PyType_Spec __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr_spec = { + "fontTools.varLib.iup.__pyx_scope_struct_4_genexpr", + sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, + __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr_slots, +}; +#else + +static PyTypeObject __pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr = { + PyVarObject_HEAD_INIT(0, 0) + "fontTools.varLib.iup.""__pyx_scope_struct_4_genexpr", /*tp_name*/ + sizeof(struct __pyx_obj_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_9fontTools_6varLib_3iup___pyx_defaults_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults)) __PYX_ERR(0, 293, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_9fontTools_6varLib_3iup___pyx_defaults_spec, __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults) < (0)) __PYX_ERR(0, 293, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults = &__pyx_type_9fontTools_6varLib_3iup___pyx_defaults; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults) < (0)) __PYX_ERR(0, 293, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults->tp_dictoffset && __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults->tp_getattro = PyObject_GenericGetAttr; + } + #endif + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between)) __PYX_ERR(0, 169, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between_spec, __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between) < (0)) __PYX_ERR(0, 169, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between = &__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between) < (0)) __PYX_ERR(0, 169, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between->tp_dictoffset && __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct____pyx_f_9fontTools_6varLib_3iup_can_iup_in_between->tp_getattro = PyObject_GenericGetAttr; + } + #endif + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr)) __PYX_ERR(0, 197, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr_spec, __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr) < (0)) __PYX_ERR(0, 197, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr = &__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr) < (0)) __PYX_ERR(0, 197, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr->tp_dictoffset && __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_1_genexpr->tp_getattro = PyObject_GenericGetAttr; + } + #endif + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize)) __PYX_ERR(0, 363, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize_spec, __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize) < (0)) __PYX_ERR(0, 363, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize = &__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize) < (0)) __PYX_ERR(0, 363, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize->tp_dictoffset && __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_2_iup_contour_optimize->tp_getattro = PyObject_GenericGetAttr; + } + #endif + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr)) __PYX_ERR(0, 378, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr_spec, __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr) < (0)) __PYX_ERR(0, 378, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr = &__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr) < (0)) __PYX_ERR(0, 378, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr->tp_dictoffset && __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_3_genexpr->tp_getattro = PyObject_GenericGetAttr; + } + #endif + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr)) __PYX_ERR(0, 387, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr_spec, __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr) < (0)) __PYX_ERR(0, 387, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr = &__pyx_type_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr) < (0)) __PYX_ERR(0, 387, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr->tp_dictoffset && __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_9fontTools_6varLib_3iup___pyx_scope_struct_4_genexpr->tp_getattro = PyObject_GenericGetAttr; + } + #endif + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_iup(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_iup}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, __Pyx_FREETHREADING_COMPATIBLE}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "iup", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_iup(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_iup(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + PY_INT64_T current_id = GraalPyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_iup(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'iup' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "iup" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + +__Pyx_RefNannySetupContext("PyInit_iup", 0); + __Pyx_init_runtime_version(); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Library function declarations ---*/ + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__pyx_module_is_main_fontTools__varLib__iup) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "fontTools.varLib.iup")) { + if (unlikely((PyDict_SetItemString(modules, "fontTools.varLib.iup", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + (void)__Pyx_modinit_function_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_type_init_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_type_import_code(__pyx_mstate); + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "fontTools/varLib/iup.py":1 + * try: # <<<<<<<<<<<<<< + * import cython + * except (AttributeError, ImportError): +*/ + { + (void)__pyx_t_1; (void)__pyx_t_2; (void)__pyx_t_3; /* mark used */ + /*try:*/ { + + /* "fontTools/varLib/iup.py":2 + * try: + * import cython # <<<<<<<<<<<<<< + * except (AttributeError, ImportError): + * # if cython not installed, use mock module with no-op decorators and types +*/ + } + } + + /* "fontTools/varLib/iup.py":6 + * # if cython not installed, use mock module with no-op decorators and types + * from fontTools.misc import cython + * COMPILED = cython.compiled # <<<<<<<<<<<<<< + * + * from typing import ( +*/ + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_COMPILED, Py_True) < (0)) __PYX_ERR(0, 6, __pyx_L1_error) + + /* "fontTools/varLib/iup.py":8 + * COMPILED = cython.compiled + * + * from typing import ( # <<<<<<<<<<<<<< + * Sequence, + * Tuple, +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Sequence,__pyx_mstate_global->__pyx_n_u_Tuple,__pyx_mstate_global->__pyx_n_u_Union}; + __pyx_t_3 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_typing, __pyx_imported_names, 3, NULL, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 8, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_3; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Sequence,__pyx_mstate_global->__pyx_n_u_Tuple,__pyx_mstate_global->__pyx_n_u_Union}; + for (__pyx_t_5=0; __pyx_t_5 < 3; __pyx_t_5++) { + __pyx_t_6 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_5]); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_5], __pyx_t_6) < (0)) __PYX_ERR(0, 8, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":13 + * Union, + * ) + * from numbers import Integral, Real # <<<<<<<<<<<<<< + * + * +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Integral,__pyx_mstate_global->__pyx_n_u_Real}; + __pyx_t_3 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_numbers, __pyx_imported_names, 2, NULL, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 13, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_3; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Integral,__pyx_mstate_global->__pyx_n_u_Real}; + for (__pyx_t_5=0; __pyx_t_5 < 2; __pyx_t_5++) { + __pyx_t_6 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_5]); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_5], __pyx_t_6) < (0)) __PYX_ERR(0, 13, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":16 + * + * + * _Point = Tuple[Real, Real] # <<<<<<<<<<<<<< + * _Delta = Tuple[Real, Real] + * _PointSegment = Sequence[_Point] +*/ + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Tuple); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 16, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_Real); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 16, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_Real); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 16, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyTuple_Pack(2, __pyx_t_6, __pyx_t_7); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 16, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = __Pyx_PyObject_GetItem(__pyx_t_4, __pyx_t_8); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 16, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_Point, __pyx_t_7) < (0)) __PYX_ERR(0, 16, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":17 + * + * _Point = Tuple[Real, Real] + * _Delta = Tuple[Real, Real] # <<<<<<<<<<<<<< + * _PointSegment = Sequence[_Point] + * _DeltaSegment = Sequence[_Delta] +*/ + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_Tuple); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 17, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GetModuleGlobalName(__pyx_t_8, __pyx_mstate_global->__pyx_n_u_Real); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 17, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Real); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 17, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = PyTuple_Pack(2, __pyx_t_8, __pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 17, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_GetItem(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 17, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_Delta, __pyx_t_4) < (0)) __PYX_ERR(0, 17, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":18 + * _Point = Tuple[Real, Real] + * _Delta = Tuple[Real, Real] + * _PointSegment = Sequence[_Point] # <<<<<<<<<<<<<< + * _DeltaSegment = Sequence[_Delta] + * _DeltaOrNone = Union[_Delta, None] +*/ + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Sequence); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_Point); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyObject_GetItem(__pyx_t_4, __pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_PointSegment, __pyx_t_7) < (0)) __PYX_ERR(0, 18, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":19 + * _Delta = Tuple[Real, Real] + * _PointSegment = Sequence[_Point] + * _DeltaSegment = Sequence[_Delta] # <<<<<<<<<<<<<< + * _DeltaOrNone = Union[_Delta, None] + * _DeltaOrNoneSegment = Sequence[_DeltaOrNone] +*/ + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_Sequence); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_Delta); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_4 = __Pyx_PyObject_GetItem(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_DeltaSegment, __pyx_t_4) < (0)) __PYX_ERR(0, 19, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":20 + * _PointSegment = Sequence[_Point] + * _DeltaSegment = Sequence[_Delta] + * _DeltaOrNone = Union[_Delta, None] # <<<<<<<<<<<<<< + * _DeltaOrNoneSegment = Sequence[_DeltaOrNone] + * _Endpoints = Sequence[Integral] +*/ + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Union); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_Delta); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_Pack(2, __pyx_t_6, Py_None); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyObject_GetItem(__pyx_t_4, __pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_DeltaOrNone, __pyx_t_6) < (0)) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":21 + * _DeltaSegment = Sequence[_Delta] + * _DeltaOrNone = Union[_Delta, None] + * _DeltaOrNoneSegment = Sequence[_DeltaOrNone] # <<<<<<<<<<<<<< + * _Endpoints = Sequence[Integral] + * +*/ + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_Sequence); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_DeltaOrNone); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_4 = __Pyx_PyObject_GetItem(__pyx_t_6, __pyx_t_7); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_DeltaOrNoneSegment, __pyx_t_4) < (0)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "fontTools/varLib/iup.py":22 + * _DeltaOrNone = Union[_Delta, None] + * _DeltaOrNoneSegment = Sequence[_DeltaOrNone] + * _Endpoints = Sequence[Integral] # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Sequence); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 22, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_Integral); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 22, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = __Pyx_PyObject_GetItem(__pyx_t_4, __pyx_t_7); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 22, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_Endpoints, __pyx_t_6) < (0)) __PYX_ERR(0, 22, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":25 + * + * + * MAX_LOOKBACK = 8 # <<<<<<<<<<<<<< + * + * +*/ + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_MAX_LOOKBACK, __pyx_mstate_global->__pyx_int_8) < (0)) __PYX_ERR(0, 25, __pyx_L1_error) + + /* "fontTools/varLib/iup.py":91 + * + * + * def iup_contour(deltas: _DeltaOrNoneSegment, coords: _PointSegment) -> _DeltaSegment: # <<<<<<<<<<<<<< + * """For the contour given in `coords`, interpolate any missing + * delta values in delta vector `deltas`. +*/ + __pyx_t_6 = __Pyx_PyDict_NewPresized(3); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 91, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_deltas, __pyx_mstate_global->__pyx_n_u_DeltaOrNoneSegment) < (0)) __PYX_ERR(0, 91, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_coords, __pyx_mstate_global->__pyx_n_u_PointSegment) < (0)) __PYX_ERR(0, 91, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_n_u_DeltaSegment) < (0)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_7 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_6varLib_3iup_1iup_contour, 0, __pyx_mstate_global->__pyx_n_u_iup_contour, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[3])); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 91, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_7); + #endif + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_7, __pyx_t_6); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_iup_contour, __pyx_t_7) < (0)) __PYX_ERR(0, 91, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":143 + * + * + * def iup_delta( # <<<<<<<<<<<<<< + * deltas: _DeltaOrNoneSegment, coords: _PointSegment, ends: _Endpoints + * ) -> _DeltaSegment: +*/ + __pyx_t_7 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 143, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_deltas, __pyx_mstate_global->__pyx_n_u_DeltaOrNoneSegment) < (0)) __PYX_ERR(0, 143, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_coords, __pyx_mstate_global->__pyx_n_u_PointSegment) < (0)) __PYX_ERR(0, 143, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_ends, __pyx_mstate_global->__pyx_n_u_Endpoints) < (0)) __PYX_ERR(0, 143, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_n_u_DeltaSegment) < (0)) __PYX_ERR(0, 143, __pyx_L1_error) + __pyx_t_6 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_6varLib_3iup_3iup_delta, 0, __pyx_mstate_global->__pyx_n_u_iup_delta, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[4])); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 143, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_6); + #endif + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_6, __pyx_t_7); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_iup_delta, __pyx_t_6) < (0)) __PYX_ERR(0, 143, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":202 + * + * + * @cython.locals( # <<<<<<<<<<<<<< + * cj=cython.double, + * dj=cython.double, +*/ + __pyx_t_6 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 202, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_deltas, __pyx_mstate_global->__pyx_n_u_DeltaSegment) < (0)) __PYX_ERR(0, 202, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_coords, __pyx_mstate_global->__pyx_n_u_PointSegment) < (0)) __PYX_ERR(0, 202, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_tolerance, __pyx_mstate_global->__pyx_n_u_Real) < (0)) __PYX_ERR(0, 202, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_n_u_set) < (0)) __PYX_ERR(0, 202, __pyx_L1_error) + __pyx_t_7 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_6varLib_3iup_5_iup_contour_bound_forced_set, 0, __pyx_mstate_global->__pyx_n_u_iup_contour_bound_forced_set, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[5])); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 202, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_7); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_7, __pyx_mstate_global->__pyx_tuple[2]); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_7, __pyx_t_6); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_iup_contour_bound_forced_set, __pyx_t_7) < (0)) __PYX_ERR(0, 202, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":293 + * + * + * @cython.locals( # <<<<<<<<<<<<<< + * i=cython.int, + * j=cython.int, +*/ + __pyx_t_7 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 293, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_deltas, __pyx_mstate_global->__pyx_n_u_DeltaSegment) < (0)) __PYX_ERR(0, 293, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_coords, __pyx_mstate_global->__pyx_n_u_PointSegment) < (0)) __PYX_ERR(0, 293, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_tolerance, __pyx_mstate_global->__pyx_n_u_Real) < (0)) __PYX_ERR(0, 293, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_lookback, __pyx_mstate_global->__pyx_n_u_Integral) < (0)) __PYX_ERR(0, 293, __pyx_L1_error) + __pyx_t_6 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_6varLib_3iup_7_iup_contour_optimize_dp, 0, __pyx_mstate_global->__pyx_n_u_iup_contour_optimize_dp, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[6])); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 293, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_6); + #endif + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_6, __pyx_mstate_global->__pyx_ptype_9fontTools_6varLib_3iup___pyx_defaults)) __PYX_ERR(0, 293, __pyx_L1_error) + __pyx_t_4 = PySet_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 305, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_t_6)->arg0 = ((PyObject*)__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + __Pyx_CyFunction_SetDefaultsGetter(__pyx_t_6, __pyx_pf_9fontTools_6varLib_3iup_16__defaults__); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_6, __pyx_t_7); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_iup_contour_optimize_dp, __pyx_t_6) < (0)) __PYX_ERR(0, 293, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":346 + * + * + * def _rot_list(l: list, k: int): # <<<<<<<<<<<<<< + * """Rotate list by k items forward. Ie. item at position 0 will be + * at position k in returned list. Negative k is allowed.""" +*/ + __pyx_t_6 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 346, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_l, __pyx_mstate_global->__pyx_n_u_list) < (0)) __PYX_ERR(0, 346, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_k, __pyx_mstate_global->__pyx_n_u_int) < (0)) __PYX_ERR(0, 346, __pyx_L1_error) + __pyx_t_7 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_6varLib_3iup_9_rot_list, 0, __pyx_mstate_global->__pyx_n_u_rot_list, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[7])); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 346, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_7); + #endif + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_7, __pyx_t_6); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_rot_list, __pyx_t_7) < (0)) __PYX_ERR(0, 346, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":356 + * + * + * def _rot_set(s: set, k: int, n: int): # <<<<<<<<<<<<<< + * k %= n + * if not k: +*/ + __pyx_t_7 = __Pyx_PyDict_NewPresized(3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_s, __pyx_mstate_global->__pyx_n_u_set) < (0)) __PYX_ERR(0, 356, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_k, __pyx_mstate_global->__pyx_n_u_int) < (0)) __PYX_ERR(0, 356, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_n, __pyx_mstate_global->__pyx_n_u_int) < (0)) __PYX_ERR(0, 356, __pyx_L1_error) + __pyx_t_6 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_6varLib_3iup_11_rot_set, 0, __pyx_mstate_global->__pyx_n_u_rot_set, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[8])); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_6); + #endif + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_6, __pyx_t_7); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_rot_set, __pyx_t_6) < (0)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":363 + * + * + * def iup_contour_optimize( # <<<<<<<<<<<<<< + * deltas: _DeltaSegment, coords: _PointSegment, tolerance: Real = 0.0 + * ) -> _DeltaOrNoneSegment: +*/ + __pyx_t_6 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 363, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_deltas, __pyx_mstate_global->__pyx_n_u_DeltaSegment) < (0)) __PYX_ERR(0, 363, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_coords, __pyx_mstate_global->__pyx_n_u_PointSegment) < (0)) __PYX_ERR(0, 363, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_tolerance, __pyx_mstate_global->__pyx_n_u_Real) < (0)) __PYX_ERR(0, 363, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_n_u_DeltaOrNoneSegment) < (0)) __PYX_ERR(0, 363, __pyx_L1_error) + __pyx_t_7 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_6varLib_3iup_13iup_contour_optimize, 0, __pyx_mstate_global->__pyx_n_u_iup_contour_optimize, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[9])); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 363, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_7); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_7, __pyx_mstate_global->__pyx_tuple[3]); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_7, __pyx_t_6); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_iup_contour_optimize, __pyx_t_7) < (0)) __PYX_ERR(0, 363, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + + /* "fontTools/varLib/iup.py":464 + * + * + * def iup_delta_optimize( # <<<<<<<<<<<<<< + * deltas: _DeltaSegment, + * coords: _PointSegment, +*/ + __pyx_t_7 = __Pyx_PyDict_NewPresized(5); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 464, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_deltas, __pyx_mstate_global->__pyx_n_u_DeltaSegment) < (0)) __PYX_ERR(0, 464, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_coords, __pyx_mstate_global->__pyx_n_u_PointSegment) < (0)) __PYX_ERR(0, 464, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_ends, __pyx_mstate_global->__pyx_n_u_Endpoints) < (0)) __PYX_ERR(0, 464, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_tolerance, __pyx_mstate_global->__pyx_n_u_Real) < (0)) __PYX_ERR(0, 464, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_n_u_DeltaOrNoneSegment) < (0)) __PYX_ERR(0, 464, __pyx_L1_error) + __pyx_t_6 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_6varLib_3iup_15iup_delta_optimize, 0, __pyx_mstate_global->__pyx_n_u_iup_delta_optimize, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_varLib_iup, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[10])); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 464, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_6); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_6, __pyx_mstate_global->__pyx_tuple[3]); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_6, __pyx_t_7); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_iup_delta_optimize, __pyx_t_6) < (0)) __PYX_ERR(0, 464, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "fontTools/varLib/iup.py":1 + * try: # <<<<<<<<<<<<<< + * import cython + * except (AttributeError, ImportError): +*/ + __pyx_t_6 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_6) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init fontTools.varLib.iup", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init fontTools.varLib.iup"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin_zip = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_zip); if (!__pyx_builtin_zip) __PYX_ERR(0, 88, __pyx_L1_error) + __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_enumerate); if (!__pyx_builtin_enumerate) __PYX_ERR(0, 103, __pyx_L1_error) + __pyx_builtin_max = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_max); if (!__pyx_builtin_max) __PYX_ERR(0, 404, __pyx_L1_error) + + /* Cached unbound methods */ + __pyx_mstate->__pyx_umethod_PyDict_Type_items.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_items.method_name = &__pyx_mstate->__pyx_n_u_items; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.method_name = &__pyx_mstate->__pyx_n_u_values; + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "fontTools/varLib/iup.py":50 + * # rd1 = reference delta 1 + * out_arrays = [None, None] + * for j in 0, 1: # <<<<<<<<<<<<<< + * out_arrays[j] = out = [] + * x1, x2, d1, d2 = rc1[j], rc2[j], rd1[j], rd2[j] +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(2, __pyx_mstate_global->__pyx_int_0, __pyx_mstate_global->__pyx_int_1); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(0, 50, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + + /* "fontTools/varLib/iup.py":106 + * if not indices: + * # All deltas are None. Return 0,0 for all. + * return [(0, 0)] * n # <<<<<<<<<<<<<< + * + * out = [] +*/ + __pyx_mstate_global->__pyx_tuple[1] = PyTuple_Pack(2, __pyx_mstate_global->__pyx_int_0, __pyx_mstate_global->__pyx_int_0); if (unlikely(!__pyx_mstate_global->__pyx_tuple[1])) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[1]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[1]); + + /* "fontTools/varLib/iup.py":202 + * + * + * @cython.locals( # <<<<<<<<<<<<<< + * cj=cython.double, + * dj=cython.double, +*/ + __pyx_mstate_global->__pyx_tuple[2] = PyTuple_Pack(1, ((PyObject*)__pyx_mstate_global->__pyx_int_0)); if (unlikely(!__pyx_mstate_global->__pyx_tuple[2])) __PYX_ERR(0, 202, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[2]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[2]); + + /* "fontTools/varLib/iup.py":363 + * + * + * def iup_contour_optimize( # <<<<<<<<<<<<<< + * deltas: _DeltaSegment, coords: _PointSegment, tolerance: Real = 0.0 + * ) -> _DeltaOrNoneSegment: +*/ + __pyx_mstate_global->__pyx_tuple[3] = PyTuple_Pack(1, ((PyObject*)__pyx_mstate_global->__pyx_float_0_0)); if (unlikely(!__pyx_mstate_global->__pyx_tuple[3])) __PYX_ERR(0, 363, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[3]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[3]); + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_tuple; + for (Py_ssize_t i=0; i<4; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #if PY_VERSION_HEX < 0x030E0000 + if (_Py_IsOwnedByCurrentThread(table[i]) && Py_REFCNT(table[i]) == 1) + #else + if (PyUnstable_Object_IsUniquelyReferenced(table[i])) + #endif + { + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + } + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + { + const struct { const unsigned int length: 10; } index[] = {{1},{27},{179},{1},{8},{7},{6},{2},{9},{8},{6},{12},{19},{13},{10},{8},{12},{6},{13},{20},{4},{8},{5},{5},{18},{9},{6},{8},{1},{2},{2},{35},{5},{2},{17},{18},{5},{7},{6},{4},{5},{1},{2},{2},{2},{6},{2},{3},{4},{9},{20},{5},{6},{8},{7},{1},{2},{2},{7},{3},{13},{2},{5},{11},{29},{20},{24},{37},{9},{18},{1},{1},{1},{2},{3},{2},{3},{4},{8},{8},{3},{10},{1},{8},{2},{3},{2},{3},{4},{7},{3},{1},{3},{1},{12},{6},{3},{3},{9},{8},{1},{4},{3},{12},{10},{8},{5},{8},{5},{9},{6},{1},{5},{6},{1},{1},{3},{2},{60},{438},{2},{522},{504},{2},{162},{200},{242},{47}}; + #if (CYTHON_COMPRESS_STRINGS) == 2 /* compression: bz2 (1925 bytes) */ +const char* const cstring = "BZh91AY&SY\226\356C\005\000\001m\177\377\377\377\377\377\377\377\377\365\277\377\377\340\277\377\377\360@@@@@@@@@@@@@\000@\000`\006\375\360p<Z\316\300h\007:\273\300\020%M\tDi\210\323M\017S&z\243\311\246)\372L\023\311\223Q\032\036\322\236PmA\246\200\310\000d=CA\232\004\022\210\004h& \230\232\031\004L\230\215\032\032hh\000\000\000\000\000\321\223A\221\350L\203@\021\022\206M\000\000\321\352i\342@\000\000\000\006@\000\000\304\000\000h\006\232\232\232H\014\215\006\215\001\246\206\232h\000\032\000\000\000h\000\000\000\000\r\000\203\000\004\300\000L\000\000\000\000\000\000\000\000\000\000\000\000\022)\032d\247\242\207\251\352?R\017Ph\r\000\032\000h\000\000\000\320\320d\320\000\000\r\003\367\202|\277/\2310&\330\371\241\241\252\263\316\274\373iS\353\354\203\337\340\2512O\301 \243\237B\254\330\221T\022*$\037\264\243\025\0234\376 \376\246\223cM\260m6\004TH\212#\244-\022\221R\252\341\376JS\302\244\300\030\022\255VU5r\342L\030\306!\214\030\333m\265%$\025\026]'e\227R\232\232\261aQB\013[\030\306\313\361Q\013\243c\022m6T\264\362$\332SW\340h\202\2659IE\223\324\320\024\016Llm\253f\246+\032D\253\tJL\030,\314m\032#pD\301\0141\215H\265\303\027A\203-\225\367\225V\250VF\220\236A\326,+U,D\221\"\273\220\226\311\022\241B\211\0231m\274yPWYS\321\350\364~b\343\013\r2\257-\361t\362(C\210\225\020\254\240JG\231W\224u8\373\035>\307\243\326\343\362]\332lz\356\356\270\355\325SY\010SY\230\023\0013\033cu#\201\206\246\014\021@\201\254\212Dx\364\320\360tN\373s\212\221\322)\211H7c\250\376\274\207\316\272\253\2253\027\361\307~3\223\237\341?\236\324X/\216h\376\327\262iA.K\213\271\215\323\203\326\002\243\205L\311\025e\212Yb8\240\021%QW' \341\352;]H\017:U\024\014$`U\255\234\332\212\002}\214\241C\251\0360lA\321[\007\203.-\250\010\037\0240\026\203&7\026\250U`pJl\354\231N\240\224\211\310\274Z-{\273\201\345\372\001\217R\210+\227\000\314\273I\010\242Fl'BnA\224\374\357\033L\262\306G\353\367\307\364\263\237\235x\342\346\230X:\214;&LZ1,\006\257+Q\206\027\026\036\200\266\234B\330\311\024\340\203'\215\272\260\"\252""\"\246\025\223\353Vs\260\236z\212\246\222\036E\264\365\306\304\212=\037X\315\3476\023\243\250\275\2057\373\271\233\307\333\273\264\375\357)\234C\315:\231\205\273h\335\020\274\350\"\327\016\314\360\222\242\010\031\313\226\032\214\307@\350\020@B\007S\235\330\370af\313\334\343g\307\313\370-*4\336\344\300\260bb\225k@\361\036\324\004\246\t\010E\020\225\216C\n\302\025\245\215B\241\214\210\351Tv\361)\340@\013/\005\354-\275\324\\\363-+1b4ust\315z\033^t\301\254_9^\2571k\034\315\247\261\r\2513\240\271\263T@\310d\014\211\223\232U\016\016\256V\216s*\302\274\366\231d61\306\345X.\nf\270\336\236\316\3651\3236.\342=\253\306\370U\273m\017'Gy\324c|\035\316F\306\232\243\315\005G\246*\254]\373`\355\211r2\272,v\301\347#\025\321zU\000\245K\235\007\305;K\254\314k\327\243\245\221?\205\325z\312D\216I8\361EU\200\330\020\267\030\214\350r1\234=\024\321\267\272\363j\2248\261\013\305'\031\r\243mnv\013\262\316*\231\nH\031tBu1\366bn\323\267\202n\246\245k\025t\347m\353\261\274\272\307#\326\351F\334\272\3225J\270\362\247\376\027BC\222H\232\246\321\316\335Y\217zp\022H\014\357(\306lll\301\260\215a\243\2056KD>\036{\200\302\300\372s\nN\207r\215\311\351\242\nHb\257\013Df\016Q\227\032k\351\031\345\271&\014\345Pw\302\370&a*\356\014\246\340zR\342\n\227\027\322\271\367-\021\345\253\241\002\3701M\014[c_\212\212\033\370>\374\335\372p\033\307\001\313\341<\"\0006\017Q\274\307\014k\031\363:\215\207\226cC\016C\323\004\271\354\2776\032y\324\310\223\311\033)\256\334B\027\246T1\246;\245f\333\r\340k\234\256\322\352\241d#\0051\263\020Q\300\312w&1G\245\013\226cib\216}\305\310\027\\]\236\021\325R\032\324m\270\r[\034\306.1\025X\301\254$\365\304J\373\203u\231\302y\311,\350Z*\262g>\261%T\"\346N\204\224\031\2352\344\335\013\3724\213I\263o\013\016\tc\020\341`\232\224\241m\267V\371\177\207\303\315\253\206\202\202\214\251\226\312\341\004\327\273\216\3246\236\223\331;\334a\311\327-X\227\026:\253\327t!{;\257\307\277\003 \312\023y\004\001\2643\025\344\206Xe\244\314\372ga16\253 0\314d\000\331\276\347h\341\036+\330\247t\256\235\317""\325t(:\215@\312\345\t@\t)\220A\004Ad\205\337!\305b\205B2\014x\354L\251\000\303k\210\350u\224>\371\314fI\231\3065\300E\261\305\037q\236c\236Q<g\004\302\227\032X@\252\217'\274F!Hp\021\032\261B\241>\0000\022\275\001\201k\272\232\210\326\210\261KG\215\3623\300T\310\220\261\177\030\035\373\336sQ\216\320aTC\223\004\030$`U\274\314\023|H25\223\336\241\266Wxk\017s\224\260\036\334\242`Q\317qb\306\202\010\303R\225\353\312\317R\375LJu\034Z\226\2120X\367\342\2356\034\033-\027\240\3259\035J\277\255\003\354x\340\030\027\321\023\006\003u7\242\212\004\331\205\370H\026\213\021D,\336E\377\267\010l\021Yd[\243\220@\375\233\036\0021q\230\037\006W\206(\225\032\302\t+\337\2345\367\315U\217\252\322>\366\335T0\240\2507\204[Ph\226\007\312\247\035H\233\025\213\202h3{\213:\262\252%U\"\250\233V\326S\026\226\350 \3617\337\234\251\323\344\340\234P$\327\233jrZD\232p\230S\311\323\321\311s\tk\332\370\231A\245\243\005x\311\327zY\227\257KuY\232\227\217\331\033\276Dt\215\271N\345\304t\333\006c\374\272\332\327%\353Z\376[\230@;\232\t3\020\250\0312\272\341t\310.\351\337\347Y\3239\205\"\226\221p\3071\336\322\2077T\357\220\3141\214Y\\\234:u,\022\2448\212|\255=z\330g\236\255\374\235\034\213$\204\206%\025\342\343e\214k\024\334e\216\265\0260cZ\346\026\307\222\226Q\304\205U\024J\006U\205\020iP\201\21404\361\210\320\314\271\320V\214hSV8\n+\024\223:P\250\246L\241\235\250\217\304\020\034\031\326\rD\210vI\252\240\206\006pkV\022m\004\324}Q\2639\231)\323V\200\312\261S\377\213\271\"\234(HKw!\202\200"; + PyObject *data = __Pyx_DecompressString(cstring, 1925, 2); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #elif (CYTHON_COMPRESS_STRINGS) != 0 /* compression: zlib (1891 bytes) */ +const char* const cstring = "x\332\275VKs\333F\022&%R\242dJ&)\352\341\215v\027\224le\343\354\312K\352\021%\225J\212z9\216\035[\244d%{B\201\300P\002\005\001$f \221>\351\210\343\034\3478G\034q\344\221G\035y\304\221?A?a{\000\352a\307[\265\271\244\212\0346\006=3_\177\375u\017W\337\350\265\027u\313$G\226e\340\027\027\212-&t\247\271\332\354\274\265\010\222\310\251B\244\235\0169\265LI\307\222\206\014\275\206l\205 \243#ab\353*A\266p2\245\203\275\203\177\255o\255K\212\251I6j \225`\t;5\325P0FX\262\352R\315\321\r\242\233\022\3514\021^\225^\325\245\216\345H&B\232D,\251\t~\017\027\220SdJ\030\021aH_*\246i\021\205\350\226)\303r\335<\371R\322t\033\016\321/\220X\275\257\030\030\255\376\250h\232\014\216H\323\261R3\0202\305x\242\3528\262\264\235w\277\034\274z\263\267+\357\"\203(\321\370\316~k\231\350\241}\210N\316\221I\242\251\333\207=SkZ\272I\360+\223\240\023[1~)\377&\277y\367\356\365vy\347\265| ^E\343\355\002\371\240\323\206\357.p$\277EmRE\365*R\214C\324r\220\251\242#\247i\240\367&D\244\340\216\251\352\326\252j\331\226\003\004!\\C\230\310\252\205Ih4\302\021[\206\252\026\325\222\252\2302$H\326M\271\206\310%\304\270\372\275a\251\020\377\017\253'\310D\355\246\255\236*\272\2516d9\344R>AD'\350\\<\302\346b!\261\025\025\325\024\365L5,\214T\310\277\345\330\252e\331\032\026\247\212/\326\264\177kE\255\244\t\016\260\326@\246\006\037\340\3219\017\363\177'\232\325H4\253\200\251n\331*\n\007M\226\353\216\251\312\362\020\221\256\027\365\222nj\272\212\260 J\307\362]\264z\210\016\213\230\206H\344\207v\315rLM\036\356\nzx\370\316j\022\375\\\377\200\344\317Nj\315\317M\377\216-\341\024Fyg\334\3716\316\014C5\324\206\241\031Z\303\32011,\353L\360&\313\347@\261\030\3330X\232c Y\206gS9\027\206\n\354\233\232\2515\340\000\002\214A\311`\010\266\331\264\232-Yn9\212\0219\332\2108\266i\353E[/\311\266EdqFh@\240P\005\202p\020\222\030\242\005`i\250\2568\006\00198\242\0340Qlp!B\"29\265\255Kb\031\220!PXT(\027\027\212\341\240p\300\355\316\007\275\311\213\375X\226>a\205\233\261Xr\302\035u\213n9HL\270\331 1~E\334b\220z\354\266\202\304#1\357V]\233\316""\322\032\213\263\031\266\315j<\336\217-\364\0276\372\033/{\255\233t,\231v\327\\\305mS\314\226Y\205)b\013\307}I\243M\006\211T\264\375 1\r^6\315\323C\226d\377\341\no{\330_\362_v\213\303C\007\340o\323\031\272\317\n\203\304\244[\020\200\362n\305\205\035\323\356.\215\323\020\335\205{H\343\203\224\230I\322c\266\306N\370\257\336K\277\344W\202\324#\367\033Z\010\322\363\264\025d\346\250\002\210\236\261\013^\341uo\337/\370\233\335\271n\2537\326\253_\227\257+\"\360\324\325%@\272\240\025\nG\244\334q\010\031\266X\007\020s\324f\331 \235\241\t\272\307\346`\217*\307\3363\357\322W\202\364c\367\022\3743_\260b\220\377\013+\000\002\3047\275\254\267\342\217\371\212\177\321=\356\025{\373\327+\375\203J\277R\205c&\322\002\025\335dYV\010R\031\032\277\215b\204.\321\362}$\233<\307\327\270\352\315\373\343~\253\033\377c\321\244>M\303\t?\364\342\336\023\177\331\257\000h\221IV\276\212\007\351e\376\243\377u\267\022$\307n2w\251\277\231\214%\377\312\312a\302h\212\305#rlw\306\215\346\304\363\244;\007/\001<\2441rL\320m@\325\341#|\211\227\005\202\351\376\324s\257\340m\371[\335\342MNl!VNL\207\n\232\001o5\342 J3\204\036J\220mq\220\312G\346\002\370\375\203/s\021\327D\306m\367\347_x-?\325M\365R\327\002\334\304<d\000\210\333vk\200(5\r?\343\300R:OwA\245a\342\222aV\201\356q\221J\220)\354s\t\000\276\340?y\341\311Y\272\002\276s\314\3463\374\255\377\314'\335\215^\262W\351)\017`q\301\314\035\200 =CK\3647\366+\337\346\247\336\251\177\326\313\016R9\372OV\207\211\232\027\212\362\025}\017KK\254\312\260\300\317O\274\252\327\032\244A'\002\024\360\237\203\244o\323:\024\022hh\201%X\031|m\236\r2y\220\232\240'}[\005K\220\221\314<u\300E\034\211\274\"$~\311\207\311\031\272\301F\205\376\376\306\014\357\271_\376C\341\t-\\\305o\346bSO?\322\302'e|W\270A\002\200\213\247\244\373\036t\274\0069o\303\266y~\344eE\022\366\240ZZl\004\340\356\3621(k\333\313y\305\250\220Diil\205\307\371'\236\242\215$\370>\210\245\004\004\331~\316/\016B\207,]\246\225 -\360\027\240\203\334\033\202\267\262x\365?\215\001\020G\004\274\210\312Yvo\014~?\003Z)""\337L\307\2462.\016\217\004N\327\302\032\023a\214\360\347\336\256?\352\027\375\335\356H\267\020\344\027\231p\316\315\322\035z\301B\347o\301-\001Yi\200\333w\335\243^\276W\355\265\"\307\311X6\017\233\201\330\203\374|\270c9Xx\022,\376\235\265\302%J\260(\361QP\327\016p\225\027\002\t\026\227@-U\376\001\210X\207T\255uk\275\370`\361)/\017\376\277\205\330[\362^\177\2740-:8\364\r\r\364X\0142\3310\357\256\"z\300W\375\257~\350*\"\355\323\261d.\224\312(\333\200#vx\313K\371\2110\352\345n\265\353\364~\276V\373\225\303\376\341Qp/\207G\356\006\364\256\002\324B5\214n\233\251\320\272J\320pF\000\0068\204\275;-\\\207MU@\201J\371\032tU\347e~\354m\372s\320\343\306\000D\324\345\304e\021\252,\324f\220\236\022\237\311GAB\254\214\344\371g\340\314\365s+\3002(\252@\267\330z\250\362c(\273\237\302\223\236v\025qEax\371-\333a\004\266:\006\375V>\212!l\212\2675\226\217MM\213H\246\037G\221\210\336\t\205w\177\343\216_u\350\250X8\003CB\034\373\035\310\013\334Tq\331\r\177\036T^\220\212\272\304m0P5SpC\"\261Ed\210`[\003@T\032v\3352\000\210jk\026\364m\263\034\373\006\372\3656?\363g\305m&ZIt\261+B2\016\300o\360D\377\351&\334e\355n\273\207\257\227\257A\360\263P\311\210}\317[\267fq0\354S\345\241\266\200\201~,\003\267G\212\307?\373W\242tw\r\324\240\355\355\361\354\177\0010\004\330#"; + PyObject *data = __Pyx_DecompressString(cstring, 1891, 1); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #else /* compression: none (3193 bytes) */ +const char* const bytes = ".Lib/fontTools/varLib/iup.pyNote that Cython is deliberately stricter than PEP-484 and rejects subclasses of builtin types. If you need to pass subclasses then set the 'annotation_typing' directive to False.?add_notedisableenablegcisenabledCOMPILED_Delta_DeltaOrNone_DeltaOrNoneSegment_DeltaSegment_EndpointsIntegralMAX_LOOKBACK_Point_PointSegment__Pyx_PyDict_NextRefRealSequenceTupleUnionasyncio.coroutinesbest_costbest_jbest_solcc1c2can_iup_in_between.<locals>.genexprchaincj__class_getitem__cline_in_tracebackclosecontourcoordscostcostsdd0d1d2deltasdjendendsenumeratefontTools.varLib.iupforceforced__func__genexprii1i2indicesint_is_coroutineititemsiup_contour_iup_contour_bound_forced_setiup_contour_optimize_iup_contour_optimize_dpiup_contour_optimize.<locals>.genexpriup_deltaiup_delta_optimizejkllclcjldldjlistlookback__main__max__module__n__name__ncncjndndjnextnumbersoutppopq__qualname__returnri1ri2_rot_list_rot_setssendset__set_name__setdefaultsolutionstart__test__throwtolerancetypingvvaluevaluesxyzip\2401\320\000\021\220\031\230!\360\006\000\005\t\210\003\2101\210A\330\004\t\210\021\330\004\007\200t\2101\330\010\017\210q\330\004\013\2101\210A\210R\210r\220\025\220b\230\001\230\023\230B\230b\240\001\320\000\030\320\0305\3205G\300q\360\014\000\005\014\2103\210a\210x\220s\230#\230Q\230a\330\004\007\200u\210G\2201\330\010\017\210q\340\004\010\210\003\2101\210A\340\004\016\210a\210r\220\024\220S\230\005\230Y\240a\240x\250s\260\"\260G\2701\330\004\007\200t\2101\340\010\017\210r\220\023\220F\230!\340\004\n\210!\330\004\t\210\024\210Q\210a\330\004\014\210D\220\001\220\021\330\004\007\200v\210S\220\001\340\010\014\210D\220\005\220V\2303\230g\240W\250G\2602\260Q\330\010\013\2107\220!\330\014\027\220q\330\020\026\220a\220s\230%\230v\240Q\240f\250F\260!\2606\270\026\270q\300\006\300f\310A\310Q\360\006\000\005\010\200w\210a\210v\220Q\220a\330\004\010\210\007\210q\330\010\013\2104\210r\220\026\220r\230\021\330\014\020\220\004\220E\230\026\230v\240R\240s\250%\250w\260a\330\014\017""\210w\220a\330\020\033\2301\330\024\032\230!\2303\230e\2406\250\021\250&\260\006\260a\260v\270V\3001\300F\310&\320PQ\320QR\360\006\000\t\014\2107\220!\2206\230\021\230!\330\010\020\220\001\330\004\007\200v\210S\220\002\220\"\220A\340\010\014\210D\220\005\220V\2306\240\022\2403\240c\250\027\260\007\260q\270\001\330\010\013\2107\220!\330\014\027\220q\330\020\026\220a\220s\230%\230v\240Q\240f\250F\260!\2606\270\026\270q\300\006\300f\310A\310Q\360\010\000\005\014\2103\210a\210x\220s\230#\230Q\230g\240S\250\001\250\031\260#\260Q\260a\330\004\013\2101\230A\200\001\330\014#\240?\260+\270Q\330\005\006\360\020\000\005\t\210\003\2101\210A\360\n\000\005\035\230A\330\010\017\210q\220\010\230\001\360\006\000\005\010\200r\210\023\210A\330\010\017\210q\360\006\000\005\n\210\026\210q\220\001\330\004\016\210a\330\010\017\210q\220\004\220B\220a\220y\240\002\240\"\240A\360\010\000\005\016\320\r*\250!\2508\2608\2701\360\022\000\005\010\200q\360\006\000\t\016\210R\210r\220\023\220B\220c\230\021\230!\330\010\017\210r\220\023\220A\340\010\021\220\031\230!\2308\2401\330\010\021\220\031\230!\2308\2401\330\010\021\220\030\230\021\230(\240#\240Q\360\010\000\t\020\210x\320\027/\250q\260\010\270\010\300\010\310\001\360\006\000\t\027\220a\330\010\014\210B\210b\220\001\330\010\016\210b\220\007\220q\330\014\024\220D\230\001\230\021\330\014\020\220\005\220Q\220a\330\010\020\220\007\220r\230\021\360\014\000\t\020\210w\220c\230\033\240H\250A\340\010\021\220\021\220&\230\001\230\026\230r\240\023\240N\260%\260t\2705\300\005\300Q\300a\340\010\021\220\031\230!\2308\2401\240A\360\n\000\t\020\210x\320\027/\250q\330\014\023\2202\220X\230W\240B\240h\250h\260k\300\021\340\010\022\220,\230f\240B\240b\250\001\340\010\014\210I\220U\230!\2302\230R\230s\240#\240Q\240g\250R\250q\340\014\032\230!\330\014\020\220\001\330\014\022\220\"\220B\220f\230B\230a\330\020\030\230\004\230A\230R\230r\240\021\330\020\024\220E\230\021\230!\330\014\017\210r\220\023\220F\230\"\230A\330\020\027\220u\230A\230W\240B\240e\2501\250F\260""\"\260A\330\020\023\2205\230\003\2301\330\024\036\230l\250*\260A\360\014\000\t\020\210w\220c\230\033\240H\250A\340\010\021\220\021\220&\230\001\230\026\230r\240\023\240N\260%\260t\2705\300\005\300Q\300a\340\004\013\2101\200\001\360\026\000\r$\240?\260+\270Q\330\005\006\360\020\000\005\014\2103\210a\210x\220s\230#\230Q\230a\340\004\010\210\003\2101\210A\330\004\020\220\001\340\004\010\210\005\210U\220!\2203\220a\220x\230r\240\024\240T\250\021\330\010\014\210E\220\026\220q\230\002\230\"\230D\240\006\240a\240r\250\022\2501\330\010\013\2104\210v\220Q\220d\230&\240\001\240\021\330\010\014\210E\220\026\220q\230\002\230\"\230B\230b\240\004\240F\250!\2502\250R\250r\260\022\2601\340\010\014\210E\220\021\220#\220Q\330\014\021\220\021\220!\2201\330\014\021\220\021\220!\2201\330\014\022\220\"\220A\220Q\330\014\022\220\"\220A\220Q\330\014\022\220\"\220A\220Q\330\014\022\220\"\220A\220Q\340\014\017\210t\2203\220a\330\020\024\220E\230\025\230a\330\020\024\220E\230\025\230a\340\020\024\220E\230\025\230a\330\020\024\220E\230\025\230a\340\014\024\220A\360\016\000\r\020\210s\220#\220Q\330\020\023\2203\220a\220s\230\"\230D\240\002\240*\250D\260\003\2601\260D\270\002\270!\330\024\034\230A\360\016\000\022\025\220C\220v\230Q\330\020\023\2209\230D\240\004\240B\240j\260\003\260:\270T\300\024\300R\300q\330\024\034\230A\360\n\000\021\024\2203\220c\230\021\330\024\027\220s\230\"\230A\330\030\031\330\034\037\230q\240\004\240B\240a\330\034 \240\003\2401\240C\240r\250\024\250R\250q\330\034\"\240#\240R\240z\260\022\2604\260t\2703\270b\300\001\340\034$\240A\340\030\031\330\034\037\230q\240\004\240B\240a\330\034 \240\003\2401\240C\240r\250\024\250R\250q\330\034\"\240#\240R\240s\250\"\250K\260t\2703\270b\300\001\340\034$\240A\340\014\017\210q\330\020\026\220d\230!\2301\330\020\021\340\004\013\2101\210a\200\001\330\014)\320)>\270a\330\005\006\360\016\000\005\022\220\021\220&\230\003\2305\240\004\240C\240q\250\010\260\004\260D\270\002\270#\270R\270u\300J\310c\320QS\320ST\330\004\010\210\003\2101\210A""\330\004\013\2105\220\002\220!\2202\220R\220s\230\"\230B\230c\240\022\2402\240S\250\002\250\"\250A\330\004\n\210!\330\004\014\210A\330\004\010\210\007\210q\330\010\017\210q\330\010\022\220+\230Q\230f\240A\240V\2506\260\026\260q\270\006\270a\330\010\013\2107\220!\2201\330\010\020\220\001\340\004\013\2101\200\001\330\014\r\330\014\r\330\n\013\330\004\017\210q\330\005\006\360\020\000\005\022\220\021\220&\230\003\2305\240\004\240C\240q\250\010\260\004\260D\270\002\270#\270R\270u\300J\310c\320QS\320ST\330\004\010\210\003\2101\210A\330\004\013\2105\220\002\220!\2202\220R\220s\230\"\230B\230c\240\022\2402\240S\250\002\250\"\250A\330\004\n\210!\330\004\014\210A\330\004\010\210\007\210q\330\010\022\320\022&\240a\330\014\022\220!\2208\2304\230r\240\024\240V\2501\250H\260D\270\002\270$\270a\340\010\017\210s\220!\2209\230C\230t\2402\240V\2502\250Q\330\010\013\2107\220!\2201\330\010\020\220\004\220B\220a\340\004\013\2101\200\001\360\024\000\r\016\330\014\r\330\016\017\330\004\017\210q\330\004\016\210a\360\026\000\005\t\210\003\2101\210A\330\004\007\200y\220\003\2201\330\010\023\2201\330\004\022\220!\220:\230Q\330\004\016\210c\220\021\330\004\016\210c\220\021\330\004\010\210\005\210U\220!\2203\220a\330\010\024\220E\230\021\230\"\230B\230c\240\022\2401\340\010\r\210Q\210e\2201\330\010\r\210Q\210e\2202\220R\220q\340\010\013\2102\210R\210r\220\023\220A\330\014\r\340\010\014\210E\220\025\220a\220r\230\022\2307\240\"\240B\240k\260\025\260a\330\014\023\2205\230\001\230\023\230B\230a\340\014\017\210u\220B\220j\240\004\320$6\260a\260x\270x\300s\310#\310Q\330\020\025\220Q\220e\230<\240q\330\020\025\220Q\220e\2301\340\014\017\210r\220\023\220A\330\020\021\340\004\013\2107\220!\320\000\020\220\010\230\010\240\001\330\004\t\210\021\330\004\007\200t\2101\330\010\017\210q\330\004\013\2102\210R\210r\220\023\220B\220b\230\004\230E\240\021"; + PyObject *data = NULL; + CYTHON_UNUSED_VAR(__Pyx_DecompressString); + #endif + PyObject **stringtab = __pyx_mstate->__pyx_string_tab; + Py_ssize_t pos = 0; + for (int i = 0; i < 117; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyUnicode_DecodeUTF8(bytes + pos, bytes_length, NULL); + if (likely(string) && i >= 9) PyUnicode_InternInPlace(&string); + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + stringtab[i] = string; + pos += bytes_length; + } + for (int i = 117; i < 128; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyBytes_FromStringAndSize(bytes + pos, bytes_length); + stringtab[i] = string; + pos += bytes_length; + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + Py_XDECREF(data); + for (Py_ssize_t i = 0; i < 128; i++) { + if (unlikely(PyObject_Hash(stringtab[i]) == -1)) { + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = stringtab + 117; + for (Py_ssize_t i=0; i<11; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #if PY_VERSION_HEX < 0x030E0000 + if (_Py_IsOwnedByCurrentThread(table[i]) && Py_REFCNT(table[i]) == 1) + #else + if (PyUnstable_Object_IsUniquelyReferenced(table[i])) + #endif + { + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + } + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + } + { + PyObject **numbertab = __pyx_mstate->__pyx_number_tab; + double const c_constants[] = {0.0}; + for (int i = 0; i < 1; i++) { + numbertab[i] = PyFloat_FromDouble(c_constants[i]); + if (unlikely(!numbertab[i])) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + { + PyObject **numbertab = __pyx_mstate->__pyx_number_tab + 1; + int8_t const cint_constants_1[] = {0,-1,1,2,3,4,8}; + for (int i = 0; i < 7; i++) { + numbertab[i] = PyLong_FromLong(cint_constants_1[i - 0]); + if (unlikely(!numbertab[i])) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_number_tab; + for (Py_ssize_t i=0; i<8; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #if PY_VERSION_HEX < 0x030E0000 + if (_Py_IsOwnedByCurrentThread(table[i]) && Py_REFCNT(table[i]) == 1) + #else + if (PyUnstable_Object_IsUniquelyReferenced(table[i])) + #endif + { + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + } + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +typedef struct { + unsigned int argcount : 3; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 1; + unsigned int nlocals : 5; + unsigned int flags : 10; + unsigned int first_line : 9; +} __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {0, 0, 0, 6, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS|CO_GENERATOR), 197}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_j, __pyx_mstate->__pyx_n_u_x, __pyx_mstate->__pyx_n_u_y, __pyx_mstate->__pyx_n_u_p, __pyx_mstate->__pyx_n_u_q}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_genexpr, __pyx_mstate->__pyx_kp_b_iso88591_1, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {0, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS|CO_GENERATOR), 378}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_p}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_genexpr, __pyx_mstate->__pyx_kp_b_iso88591_A, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {0, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS|CO_GENERATOR), 387}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_d}; + __pyx_mstate_global->__pyx_codeobj_tab[2] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_genexpr, __pyx_mstate->__pyx_kp_b_iso88591_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[2])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 14, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 91}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_deltas, __pyx_mstate->__pyx_n_u_coords, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_indices, __pyx_mstate->__pyx_n_u_out, __pyx_mstate->__pyx_n_u_it, __pyx_mstate->__pyx_n_u_start, __pyx_mstate->__pyx_n_u_i1, __pyx_mstate->__pyx_n_u_i2, __pyx_mstate->__pyx_n_u_ri1, __pyx_mstate->__pyx_n_u_ri2, __pyx_mstate->__pyx_n_u_end, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_v}; + __pyx_mstate_global->__pyx_codeobj_tab[3] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_iup_contour, __pyx_mstate->__pyx_kp_b_iso88591_55Gq_3axs_Qa_uG1_q_1A_ar_S_Yaxs, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[3])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 8, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 143}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_deltas, __pyx_mstate->__pyx_n_u_coords, __pyx_mstate->__pyx_n_u_ends, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_out, __pyx_mstate->__pyx_n_u_start, __pyx_mstate->__pyx_n_u_end, __pyx_mstate->__pyx_n_u_contour}; + __pyx_mstate_global->__pyx_codeobj_tab[4] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_iup_delta, __pyx_mstate->__pyx_kp_b_iso88591_a_5_Cq_D_RuJcQSST_1A_5_2Rs_Bc_2, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[4])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 24, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 202}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_deltas, __pyx_mstate->__pyx_n_u_coords, __pyx_mstate->__pyx_n_u_tolerance, __pyx_mstate->__pyx_n_u_cj, __pyx_mstate->__pyx_n_u_dj, __pyx_mstate->__pyx_n_u_lcj, __pyx_mstate->__pyx_n_u_ldj, __pyx_mstate->__pyx_n_u_ncj, __pyx_mstate->__pyx_n_u_ndj, __pyx_mstate->__pyx_n_u_force, __pyx_mstate->__pyx_n_u_forced, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_ld, __pyx_mstate->__pyx_n_u_lc, __pyx_mstate->__pyx_n_u_d, __pyx_mstate->__pyx_n_u_c, __pyx_mstate->__pyx_n_u_nd, __pyx_mstate->__pyx_n_u_nc, __pyx_mstate->__pyx_n_u_j, __pyx_mstate->__pyx_n_u_c1, __pyx_mstate->__pyx_n_u_c2, __pyx_mstate->__pyx_n_u_d1, __pyx_mstate->__pyx_n_u_d2}; + __pyx_mstate_global->__pyx_codeobj_tab[5] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_iup_contour_bound_forced_set, __pyx_mstate->__pyx_kp_b_iso88591_Q_3axs_Qa_1A_U_3axr_T_E_q_D_ar, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[5])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {5, 0, 0, 13, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 293}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_deltas, __pyx_mstate->__pyx_n_u_coords, __pyx_mstate->__pyx_n_u_forced, __pyx_mstate->__pyx_n_u_tolerance, __pyx_mstate->__pyx_n_u_lookback, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_j, __pyx_mstate->__pyx_n_u_best_cost, __pyx_mstate->__pyx_n_u_best_j, __pyx_mstate->__pyx_n_u_cost, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_costs, __pyx_mstate->__pyx_n_u_chain}; + __pyx_mstate_global->__pyx_codeobj_tab[6] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_iup_contour_optimize_dp, __pyx_mstate->__pyx_kp_b_iso88591_q_a_1A_y_1_1_Q_c_c_U_3a_E_Bc_1, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[6])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 346}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_l, __pyx_mstate->__pyx_n_u_k, __pyx_mstate->__pyx_n_u_n}; + __pyx_mstate_global->__pyx_codeobj_tab[7] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_rot_list, __pyx_mstate->__pyx_kp_b_iso88591_1A_t1_q_1ARr_b_Bb, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[7])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 356}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_s, __pyx_mstate->__pyx_n_u_k, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_v}; + __pyx_mstate_global->__pyx_codeobj_tab[8] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_rot_set, __pyx_mstate->__pyx_kp_b_iso88591_t1_q_2Rr_Bb_E, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[8])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 20, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 363}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_deltas, __pyx_mstate->__pyx_n_u_coords, __pyx_mstate->__pyx_n_u_tolerance, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_d0, __pyx_mstate->__pyx_n_u_forced, __pyx_mstate->__pyx_n_u_k, __pyx_mstate->__pyx_n_u_chain, __pyx_mstate->__pyx_n_u_costs, __pyx_mstate->__pyx_n_u_solution, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_best_sol, __pyx_mstate->__pyx_n_u_best_cost, __pyx_mstate->__pyx_n_u_start, __pyx_mstate->__pyx_n_u_cost, __pyx_mstate->__pyx_n_u_genexpr, __pyx_mstate->__pyx_n_u_genexpr, __pyx_mstate->__pyx_n_u_genexpr, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_i}; + __pyx_mstate_global->__pyx_codeobj_tab[9] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_iup_contour_optimize, __pyx_mstate->__pyx_kp_b_iso88591_Q_1A_A_q_r_A_q_q_a_q_Bay_A_881, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[9])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 9, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 464}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_deltas, __pyx_mstate->__pyx_n_u_coords, __pyx_mstate->__pyx_n_u_ends, __pyx_mstate->__pyx_n_u_tolerance, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_out, __pyx_mstate->__pyx_n_u_start, __pyx_mstate->__pyx_n_u_end, __pyx_mstate->__pyx_n_u_contour}; + __pyx_mstate_global->__pyx_codeobj_tab[10] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_varLib_iup_py, __pyx_mstate->__pyx_n_u_iup_delta_optimize, __pyx_mstate->__pyx_kp_b_iso88591_q_5_Cq_D_RuJcQSST_1A_5_2Rs_Bc_2, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[10])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* AssertionsEnabled.init */ + if (likely(__Pyx_init_assertions_enabled() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CommonTypesMetaclass.init */ + if (likely(__pyx_CommonTypesMetaclass_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API + { + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } + } // error handling follows + #endif + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CythonFunctionShared.init */ + if (likely(__pyx_CyFunction_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* Generator.init */ + if (likely(__pyx_Generator_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyErrExceptionMatches (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i<n; i++) { + if (exc_type == PyTuple_GET_ITEM(tuple, i)) return 1; + } + for (i=0; i<n; i++) { + if (__Pyx_PyErr_GivenExceptionMatches(exc_type, PyTuple_GET_ITEM(tuple, i))) return 1; + } + return 0; +} +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err) { + int result; + PyObject *exc_type; +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError (used by GetBuiltinName) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* SetItemInt */ +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { + int r; + if (unlikely(!j)) return -1; + r = PyObject_SetItem(o, j, v); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = (!wraparound) ? i : ((likely(i >= 0)) ? i : i + PyList_GET_SIZE(o)); + if ((CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared))) { + Py_INCREF(v); + return PyList_SetItem(o, n, v); + } else if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o)))) { + PyObject* old; + Py_INCREF(v); + old = PyList_GET_ITEM(o, n); + PyList_SET_ITEM(o, n, v); + Py_DECREF(old); + return 0; + } + } else +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_ass_subscript) { + int r; + PyObject *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return -1; + r = mm->mp_ass_subscript(o, key, v); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return -1; + PyErr_Clear(); + } + } + return sm->sq_ass_item(o, i, v); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_SetItem(o, i, v); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_SetItemInt_Generic(o, PyLong_FromSsize_t(i), v); +} + +/* GetItemInt */ +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (unlikely(!j)) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); + } + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS)) { + return __Pyx_PyList_GetItemRefFast(o, wrapped_i, unsafe_shared); + } else + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS)) { + return __Pyx_PyList_GetItemRefFast(o, n, unsafe_shared); + } else if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, n)); + } + } else + #if !CYTHON_AVOID_BORROWED_REFS + if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, n)); + } + } else + #endif +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_subscript) { + PyObject *r, *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return sm->sq_item(o, i); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +} + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = Py_TYPE(func)->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* TupleAndListFromArray (used by fastcall) */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; i<nkwargs; i++) { +#if !CYTHON_ASSUME_SAFE_MACROS + PyObject *key = PyTuple_GetItem(kwnames, i); + if (!key) goto bad; +#else + PyObject *key = PyTuple_GET_ITEM(kwnames, i); +#endif + if (unlikely(PyDict_SetItem(dict, key, kwvalues[i]) < 0)) + goto bad; + } + return dict; +bad: + Py_DECREF(dict); + return NULL; +} +#endif +#endif + +/* PyObjectCallMethO (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall (used by PyObjectCallOneArg) */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL + #if CYTHON_COMPILING_IN_LIMITED_API + return PyObject_Vectorcall(func, args, _nargs, NULL); + #else + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #endif + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectCallOneArg (used by CallUnboundCMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* UnpackUnboundCMethod (used by CallUnboundCMethod0) */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *args, PyObject *kwargs) { + PyObject *result; + PyObject *selfless_args = PyTuple_GetSlice(args, 1, PyTuple_Size(args)); + if (unlikely(!selfless_args)) return NULL; + result = PyObject_Call(method, selfless_args, kwargs); + Py_DECREF(selfless_args); + return result; +} +#elif CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { + return _PyObject_Vectorcall + (method, args ? args+1 : NULL, nargs ? nargs-1 : 0, kwnames); +} +#else +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + return +#if PY_VERSION_HEX < 0x03090000 + _PyObject_Vectorcall +#else + PyObject_Vectorcall +#endif + (method, args ? args+1 : NULL, nargs ? (size_t) nargs-1 : 0, kwnames); +} +#endif +static PyMethodDef __Pyx_UnboundCMethod_Def = { + "CythonUnboundCMethod", + __PYX_REINTERPRET_FUNCION(PyCFunction, __Pyx_SelflessCall), +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 + METH_VARARGS | METH_KEYWORDS, +#else + METH_FASTCALL | METH_KEYWORDS, +#endif + NULL +}; +static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { + PyObject *method, *result=NULL; + method = __Pyx_PyObject_GetAttrStr(target->type, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod0 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + if (likely(cfunc->flag == METH_NOARGS)) + return __Pyx_CallCFunction(cfunc, self, NULL); + if (likely(cfunc->flag == METH_FASTCALL)) + return __Pyx_CallCFunctionFast(cfunc, self, NULL, 0); + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, NULL, 0, NULL); + if (likely(cfunc->flag == (METH_VARARGS | METH_KEYWORDS))) + return __Pyx_CallCFunctionWithKeywords(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (cfunc->flag == METH_VARARGS) + return __Pyx_CallCFunction(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple); + return __Pyx__CallUnboundCMethod0(cfunc, self); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod0(&tmp_cfunc, self); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod0(cfunc, self); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + PyObject *result; + if (unlikely(!cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; + result = __Pyx_PyObject_CallOneArg(cfunc->method, self); + return result; +} + +/* py_dict_items (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_items, d); +} + +/* py_dict_values (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_values, d); +} + +/* OwnedDictNext (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue) { + PyObject *next = NULL; + if (!*ppos) { + if (pvalue) { + PyObject *dictview = pkey ? __Pyx_PyDict_Items(p) : __Pyx_PyDict_Values(p); + if (unlikely(!dictview)) goto bad; + *ppos = PyObject_GetIter(dictview); + Py_DECREF(dictview); + } else { + *ppos = PyObject_GetIter(p); + } + if (unlikely(!*ppos)) goto bad; + } + next = PyIter_Next(*ppos); + if (!next) { + if (PyErr_Occurred()) goto bad; + return 0; + } + if (pkey && pvalue) { + *pkey = __Pyx_PySequence_ITEM(next, 0); + if (unlikely(*pkey)) goto bad; + *pvalue = __Pyx_PySequence_ITEM(next, 1); + if (unlikely(*pvalue)) goto bad; + Py_DECREF(next); + } else if (pkey) { + *pkey = next; + } else { + assert(pvalue); + *pvalue = next; + } + return 1; + bad: + Py_XDECREF(next); +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 + PyErr_FormatUnraisable("Exception ignored in __Pyx_PyDict_NextRef"); +#else + PyErr_WriteUnraisable(__pyx_mstate_global->__pyx_n_u_Pyx_PyDict_NextRef); +#endif + if (pkey) *pkey = NULL; + if (pvalue) *pvalue = NULL; + return 0; +} +#else // !CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { + int result = PyDict_Next(p, ppos, pkey, pvalue); + if (likely(result == 1)) { + if (pkey) Py_INCREF(*pkey); + if (pvalue) Py_INCREF(*pvalue); + } + return result; +} +#endif + +/* RaiseDoubleKeywords (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* ParseKeywordsImpl (used by ParseKeywords) */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; + #else + Py_ssize_t pos = 0; + #endif + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while ( + #if CYTHON_AVOID_BORROWED_REFS + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL) + #else + PyDict_Next(kwds, &pos, &key, NULL) + #endif + ) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + __Pyx_END_CRITICAL_SECTION(); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); + #endif + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} + +/* ParseKeywords */ +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* PyLongBinop */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_Fallback___Pyx_PyLong_AddObjC(PyObject *op1, PyObject *op2, int inplace) { + return (inplace ? PyNumber_InPlaceAdd : PyNumber_Add)(op1, op2); +} +#if CYTHON_USE_PYLONG_INTERNALS +static PyObject* __Pyx_Unpacked___Pyx_PyLong_AddObjC(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check) { + CYTHON_MAYBE_UNUSED_VAR(inplace); + CYTHON_UNUSED_VAR(zerodivision_check); + const long b = intval; + long a; + const PY_LONG_LONG llb = intval; + PY_LONG_LONG lla; + if (unlikely(__Pyx_PyLong_IsZero(op1))) { + return __Pyx_NewRef(op2); + } + const int is_positive = __Pyx_PyLong_IsPos(op1); + const digit* digits = __Pyx_PyLong_Digits(op1); + const Py_ssize_t size = __Pyx_PyLong_DigitCount(op1); + if (likely(size == 1)) { + a = (long) digits[0]; + if (!is_positive) a *= -1; + } else { + switch (size) { + case 2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + if (!is_positive) a *= -1; + goto calculate_long; + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { + lla = (PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + if (!is_positive) lla *= -1; + goto calculate_long_long; + } + break; + case 3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + if (!is_positive) a *= -1; + goto calculate_long; + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { + lla = (PY_LONG_LONG) (((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + if (!is_positive) lla *= -1; + goto calculate_long_long; + } + break; + case 4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + if (!is_positive) a *= -1; + goto calculate_long; + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 4 * PyLong_SHIFT) { + lla = (PY_LONG_LONG) (((((((((unsigned PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + if (!is_positive) lla *= -1; + goto calculate_long_long; + } + break; + } + return PyLong_Type.tp_as_number->nb_add(op1, op2); + } + calculate_long: + { + long x; + x = a + b; + return PyLong_FromLong(x); + } + calculate_long_long: + { + PY_LONG_LONG llx; + llx = lla + llb; + return PyLong_FromLongLong(llx); + } + +} +#endif +static PyObject* __Pyx_Float___Pyx_PyLong_AddObjC(PyObject *float_val, long intval, int zerodivision_check) { + CYTHON_UNUSED_VAR(zerodivision_check); + const long b = intval; + double a = __Pyx_PyFloat_AS_DOUBLE(float_val); + double result; + + result = ((double)a) + (double)b; + return PyFloat_FromDouble(result); +} +static CYTHON_INLINE PyObject* __Pyx_PyLong_AddObjC(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_UNUSED_VAR(zerodivision_check); + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + return __Pyx_Unpacked___Pyx_PyLong_AddObjC(op1, op2, intval, inplace, zerodivision_check); + } + #endif + if (PyFloat_CheckExact(op1)) { + return __Pyx_Float___Pyx_PyLong_AddObjC(op1, intval, zerodivision_check); + } + return __Pyx_Fallback___Pyx_PyLong_AddObjC(op1, op2, inplace); +} +#endif + +/* IterNextPlain (used by IterNext) */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject *__Pyx_GetBuiltinNext_LimitedAPI(void) { + if (unlikely(!__pyx_mstate_global->__Pyx_GetBuiltinNext_LimitedAPI_cache)) + __pyx_mstate_global->__Pyx_GetBuiltinNext_LimitedAPI_cache = __Pyx_GetBuiltinName(__pyx_mstate_global->__pyx_n_u_next); + return __pyx_mstate_global->__Pyx_GetBuiltinNext_LimitedAPI_cache; +} +#endif +static CYTHON_INLINE PyObject *__Pyx_PyIter_Next_Plain(PyObject *iterator) { +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + PyObject *result; + PyObject *next = __Pyx_GetBuiltinNext_LimitedAPI(); + if (unlikely(!next)) return NULL; + result = PyObject_CallFunctionObjArgs(next, iterator, NULL); + return result; +#else + (void)__Pyx_GetBuiltinName; // only for early limited API + iternextfunc iternext = __Pyx_PyObject_GetIterNextFunc(iterator); + assert(iternext); + return iternext(iterator); +#endif +} + +/* IterNext */ +static PyObject *__Pyx_PyIter_Next2Default(PyObject* defval) { + PyObject* exc_type; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + exc_type = __Pyx_PyErr_CurrentExceptionType(); + if (unlikely(exc_type)) { + if (!defval || unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) + return NULL; + __Pyx_PyErr_Clear(); + Py_INCREF(defval); + return defval; + } + if (defval) { + Py_INCREF(defval); + return defval; + } + __Pyx_PyErr_SetNone(PyExc_StopIteration); + return NULL; +} +static void __Pyx_PyIter_Next_ErrorNoIterator(PyObject *iterator) { + __Pyx_TypeName iterator_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(iterator)); + PyErr_Format(PyExc_TypeError, + __Pyx_FMT_TYPENAME " object is not an iterator", iterator_type_name); + __Pyx_DECREF_TypeName(iterator_type_name); +} +static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject* iterator, PyObject* defval) { + PyObject* next; +#if !CYTHON_COMPILING_IN_LIMITED_API + iternextfunc iternext = __Pyx_PyObject_TryGetSlot(iterator, tp_iternext, iternextfunc); + if (likely(iternext)) { + next = iternext(iterator); + if (likely(next)) + return next; + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030d0000 + if (unlikely(iternext == &_PyObject_NextNotImplemented)) + return NULL; + #endif + } else if (CYTHON_USE_TYPE_SLOTS) { + __Pyx_PyIter_Next_ErrorNoIterator(iterator); + return NULL; + } else +#endif + if (unlikely(!PyIter_Check(iterator))) { + __Pyx_PyIter_Next_ErrorNoIterator(iterator); + return NULL; + } else { + next = defval ? PyIter_Next(iterator) : __Pyx_PyIter_Next_Plain(iterator); + if (likely(next)) + return next; + } + return __Pyx_PyIter_Next2Default(defval); +} + +/* PyLongCompare */ +static CYTHON_INLINE int __Pyx_PyLong_BoolNeObjC(PyObject *op1, PyObject *op2, long intval, long inplace) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_UNUSED_VAR(inplace); + if (op1 == op2) { + return 0; + } + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + int unequal; + unsigned long uintval; + Py_ssize_t size = __Pyx_PyLong_DigitCount(op1); + const digit* digits = __Pyx_PyLong_Digits(op1); + if (intval == 0) { + return (__Pyx_PyLong_IsZero(op1) != 1); + } else if (intval < 0) { + if (__Pyx_PyLong_IsNonNeg(op1)) + return 1; + intval = -intval; + } else { + if (__Pyx_PyLong_IsNeg(op1)) + return 1; + } + uintval = (unsigned long) intval; +#if PyLong_SHIFT * 4 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 4)) { + unequal = (size != 5) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[4] != ((uintval >> (4 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 3 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 3)) { + unequal = (size != 4) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 2 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 2)) { + unequal = (size != 3) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 1 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 1)) { + unequal = (size != 2) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif + unequal = (size != 1) || (((unsigned long) digits[0]) != (uintval & (unsigned long) PyLong_MASK)); + return (unequal != 0); + } + #endif + if (PyFloat_CheckExact(op1)) { + const long b = intval; + double a = __Pyx_PyFloat_AS_DOUBLE(op1); + return ((double)a != (double)b); + } + return __Pyx_PyObject_IsTrueAndDecref( + PyObject_RichCompare(op1, op2, Py_NE)); +} + +/* SliceObject */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetSlice(PyObject* obj, + Py_ssize_t cstart, Py_ssize_t cstop, + PyObject** _py_start, PyObject** _py_stop, PyObject** _py_slice, + int has_cstart, int has_cstop, CYTHON_UNUSED int wraparound) { + __Pyx_TypeName obj_type_name; +#if CYTHON_USE_TYPE_SLOTS + PyMappingMethods* mp = Py_TYPE(obj)->tp_as_mapping; + if (likely(mp && mp->mp_subscript)) +#endif + { + PyObject* result; + PyObject *py_slice, *py_start, *py_stop; + if (_py_slice) { + py_slice = *_py_slice; + } else { + PyObject* owned_start = NULL; + PyObject* owned_stop = NULL; + if (_py_start) { + py_start = *_py_start; + } else { + if (has_cstart) { + owned_start = py_start = PyLong_FromSsize_t(cstart); + if (unlikely(!py_start)) goto bad; + } else + py_start = Py_None; + } + if (_py_stop) { + py_stop = *_py_stop; + } else { + if (has_cstop) { + owned_stop = py_stop = PyLong_FromSsize_t(cstop); + if (unlikely(!py_stop)) { + Py_XDECREF(owned_start); + goto bad; + } + } else + py_stop = Py_None; + } + py_slice = PySlice_New(py_start, py_stop, Py_None); + Py_XDECREF(owned_start); + Py_XDECREF(owned_stop); + if (unlikely(!py_slice)) goto bad; + } +#if CYTHON_USE_TYPE_SLOTS + result = mp->mp_subscript(obj, py_slice); +#else + result = PyObject_GetItem(obj, py_slice); +#endif + if (!_py_slice) { + Py_DECREF(py_slice); + } + return result; + } + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is unsliceable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); +bad: + return NULL; +} + +/* ObjectGetItem */ +#if CYTHON_USE_TYPE_SLOTS +static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject *index) { + PyObject *runerr = NULL; + Py_ssize_t key_value; + key_value = __Pyx_PyIndex_AsSsize_t(index); + if (likely(key_value != -1 || !(runerr = PyErr_Occurred()))) { + return __Pyx_GetItemInt_Fast(obj, key_value, 0, 1, 1, 1); + } + if (PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) { + __Pyx_TypeName index_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(index)); + PyErr_Clear(); + PyErr_Format(PyExc_IndexError, + "cannot fit '" __Pyx_FMT_TYPENAME "' into an index-sized integer", index_type_name); + __Pyx_DECREF_TypeName(index_type_name); + } + return NULL; +} +static PyObject *__Pyx_PyObject_GetItem_Slow(PyObject *obj, PyObject *key) { + __Pyx_TypeName obj_type_name; + if (likely(PyType_Check(obj))) { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(obj, __pyx_mstate_global->__pyx_n_u_class_getitem); + if (!meth) { + PyErr_Clear(); + } else { + PyObject *result = __Pyx_PyObject_CallOneArg(meth, key); + Py_DECREF(meth); + return result; + } + } + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not subscriptable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return NULL; +} +static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key) { + PyTypeObject *tp = Py_TYPE(obj); + PyMappingMethods *mm = tp->tp_as_mapping; + PySequenceMethods *sm = tp->tp_as_sequence; + if (likely(mm && mm->mp_subscript)) { + return mm->mp_subscript(obj, key); + } + if (likely(sm && sm->sq_item)) { + return __Pyx_PyObject_GetIndex(obj, key); + } + return __Pyx_PyObject_GetItem_Slow(obj, key); +} +#endif + +/* PyLongBinop */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_Fallback___Pyx_PyLong_SubtractObjC(PyObject *op1, PyObject *op2, int inplace) { + return (inplace ? PyNumber_InPlaceSubtract : PyNumber_Subtract)(op1, op2); +} +#if CYTHON_USE_PYLONG_INTERNALS +static PyObject* __Pyx_Unpacked___Pyx_PyLong_SubtractObjC(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check) { + CYTHON_MAYBE_UNUSED_VAR(inplace); + CYTHON_UNUSED_VAR(zerodivision_check); + const long b = intval; + long a; + const PY_LONG_LONG llb = intval; + PY_LONG_LONG lla; + if (unlikely(__Pyx_PyLong_IsZero(op1))) { + return PyLong_FromLong(-intval); + } + const int is_positive = __Pyx_PyLong_IsPos(op1); + const digit* digits = __Pyx_PyLong_Digits(op1); + const Py_ssize_t size = __Pyx_PyLong_DigitCount(op1); + if (likely(size == 1)) { + a = (long) digits[0]; + if (!is_positive) a *= -1; + } else { + switch (size) { + case 2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + if (!is_positive) a *= -1; + goto calculate_long; + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { + lla = (PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + if (!is_positive) lla *= -1; + goto calculate_long_long; + } + break; + case 3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + if (!is_positive) a *= -1; + goto calculate_long; + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { + lla = (PY_LONG_LONG) (((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + if (!is_positive) lla *= -1; + goto calculate_long_long; + } + break; + case 4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + if (!is_positive) a *= -1; + goto calculate_long; + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 4 * PyLong_SHIFT) { + lla = (PY_LONG_LONG) (((((((((unsigned PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + if (!is_positive) lla *= -1; + goto calculate_long_long; + } + break; + } + return PyLong_Type.tp_as_number->nb_subtract(op1, op2); + } + calculate_long: + { + long x; + x = a - b; + return PyLong_FromLong(x); + } + calculate_long_long: + { + PY_LONG_LONG llx; + llx = lla - llb; + return PyLong_FromLongLong(llx); + } + +} +#endif +static PyObject* __Pyx_Float___Pyx_PyLong_SubtractObjC(PyObject *float_val, long intval, int zerodivision_check) { + CYTHON_UNUSED_VAR(zerodivision_check); + const long b = intval; + double a = __Pyx_PyFloat_AS_DOUBLE(float_val); + double result; + + result = ((double)a) - (double)b; + return PyFloat_FromDouble(result); +} +static CYTHON_INLINE PyObject* __Pyx_PyLong_SubtractObjC(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_UNUSED_VAR(zerodivision_check); + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + return __Pyx_Unpacked___Pyx_PyLong_SubtractObjC(op1, op2, intval, inplace, zerodivision_check); + } + #endif + if (PyFloat_CheckExact(op1)) { + return __Pyx_Float___Pyx_PyLong_SubtractObjC(op1, intval, zerodivision_check); + } + return __Pyx_Fallback___Pyx_PyLong_SubtractObjC(op1, op2, inplace); +} +#endif + +/* PyDictVersioning (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* RaiseUnboundLocalError */ +static void __Pyx_RaiseUnboundLocalError(const char *varname) { + PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); +} + +/* RaiseTooManyValuesToUnpack */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { + PyErr_Format(PyExc_ValueError, + "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); +} + +/* RaiseNeedMoreValuesToUnpack */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", + index, (index == 1) ? "" : "s"); +} + +/* IterFinish */ +static CYTHON_INLINE int __Pyx_IterFinish(void) { + PyObject* exc_type; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + exc_type = __Pyx_PyErr_CurrentExceptionType(); + if (unlikely(exc_type)) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) + return -1; + __Pyx_PyErr_Clear(); + return 0; + } + return 0; +} + +/* UnpackItemEndCheck */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { + if (unlikely(retval)) { + Py_DECREF(retval); + __Pyx_RaiseTooManyValuesError(expected); + return -1; + } + return __Pyx_IterFinish(); +} + +/* py_abs */ +#if CYTHON_USE_PYLONG_INTERNALS +static PyObject *__Pyx_PyLong_AbsNeg(PyObject *n) { +#if PY_VERSION_HEX >= 0x030C00A7 + if (likely(__Pyx_PyLong_IsCompact(n))) { + return PyLong_FromSize_t(__Pyx_PyLong_CompactValueUnsigned(n)); + } +#else + if (likely(Py_SIZE(n) == -1)) { + return PyLong_FromUnsignedLong(__Pyx_PyLong_Digits(n)[0]); + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + { + PyObject *copy = _PyLong_Copy((PyLongObject*)n); + if (likely(copy)) { + #if PY_VERSION_HEX >= 0x030C00A7 + ((PyLongObject*)copy)->long_value.lv_tag ^= ((PyLongObject*)copy)->long_value.lv_tag & _PyLong_SIGN_MASK; + #else + __Pyx_SET_SIZE(copy, -Py_SIZE(copy)); + #endif + } + return copy; + } +#else + return PyNumber_Negative(n); +#endif +} +#endif + +/* RaiseClosureNameError */ +static void __Pyx_RaiseClosureNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); +} + +/* GetException (used by pep479) */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* pep479 */ +static void __Pyx_Generator_Replace_StopIteration(int in_async_gen) { + PyObject *exc, *val, *tb, *cur_exc, *new_exc; + __Pyx_PyThreadState_declare + int is_async_stopiteration = 0; + CYTHON_MAYBE_UNUSED_VAR(in_async_gen); + __Pyx_PyThreadState_assign + cur_exc = __Pyx_PyErr_CurrentExceptionType(); + if (likely(!__Pyx_PyErr_GivenExceptionMatches(cur_exc, PyExc_StopIteration))) { + if (in_async_gen && unlikely(__Pyx_PyErr_GivenExceptionMatches(cur_exc, PyExc_StopAsyncIteration))) { + is_async_stopiteration = 1; + } else { + return; + } + } + __Pyx_GetException(&exc, &val, &tb); + Py_XDECREF(exc); + Py_XDECREF(tb); + new_exc = PyObject_CallFunction(PyExc_RuntimeError, "s", + is_async_stopiteration ? "async generator raised StopAsyncIteration" : + in_async_gen ? "async generator raised StopIteration" : + "generator raised StopIteration"); + if (!new_exc) { + Py_XDECREF(val); + return; + } + PyException_SetCause(new_exc, val); // steals ref to val + PyErr_SetObject(PyExc_RuntimeError, new_exc); +} + +/* ArgTypeTestFunc (used by ArgTypeTest) */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* DictGetItem */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) { + PyObject *value; + if (unlikely(__Pyx_PyDict_GetItemRef(d, key, &value) == 0)) { // no value, no error + if (unlikely(PyTuple_Check(key))) { + PyObject* args = PyTuple_Pack(1, key); + if (likely(args)) { + PyErr_SetObject(PyExc_KeyError, args); + Py_DECREF(args); + } + } else { + PyErr_SetObject(PyExc_KeyError, key); + } + } + return value; +} +#endif + +/* PyObjectCallNoArg (used by pyfrozenset_new) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { + PyObject *arg[2] = {NULL, NULL}; + return __Pyx_PyObject_FastCall(func, arg + 1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* pyfrozenset_new (used by PySetContains) */ +static CYTHON_INLINE PyObject* __Pyx_PyFrozenSet_New(PyObject* it) { + if (it) { + PyObject* result; +#if CYTHON_COMPILING_IN_PYPY + PyObject* args; + args = PyTuple_Pack(1, it); + if (unlikely(!args)) + return NULL; + result = PyObject_Call((PyObject*)&PyFrozenSet_Type, args, NULL); + Py_DECREF(args); + return result; +#else + if (PyFrozenSet_CheckExact(it)) { + Py_INCREF(it); + return it; + } + result = PyFrozenSet_New(it); + if (unlikely(!result)) + return NULL; + if ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000) +#if CYTHON_COMPILING_IN_LIMITED_API + || __Pyx_get_runtime_version() >= 0x030A0000 +#endif + ) + return result; + { + Py_ssize_t size = __Pyx_PySet_GET_SIZE(result); + if (likely(size > 0)) + return result; +#if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(size < 0)) { + Py_DECREF(result); + return NULL; + } +#endif + } + Py_DECREF(result); +#endif + } + return __Pyx_PyObject_CallNoArg((PyObject*) &PyFrozenSet_Type); +} + +/* PySetContains */ +static int __Pyx_PySet_ContainsUnhashable(PyObject *set, PyObject *key) { + int result = -1; + if (PySet_Check(key) && PyErr_ExceptionMatches(PyExc_TypeError)) { + PyObject *tmpkey; + PyErr_Clear(); + tmpkey = __Pyx_PyFrozenSet_New(key); + if (tmpkey != NULL) { + result = PySet_Contains(set, tmpkey); + Py_DECREF(tmpkey); + } + } + return result; +} +static CYTHON_INLINE int __Pyx_PySet_ContainsTF(PyObject* key, PyObject* set, int eq) { + int result = PySet_Contains(set, key); + if (unlikely(result < 0)) { + result = __Pyx_PySet_ContainsUnhashable(set, key); + } + return unlikely(result < 0) ? result : (result == (eq == Py_EQ)); +} + +/* pybuiltin_invalid (used by pyint_simplify) */ +static void __Pyx_PyBuiltin_Invalid(PyObject *obj, const char *type_name, const char *argname) { + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + if (argname) { + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got " __Pyx_FMT_TYPENAME ")", + argname, type_name, obj_type_name + ); + } else { + PyErr_Format(PyExc_TypeError, + "Expected %.200s, got " __Pyx_FMT_TYPENAME, + type_name, obj_type_name + ); + } + __Pyx_DECREF_TypeName(obj_type_name); +} + +/* pyint_simplify */ +static CYTHON_INLINE int __Pyx_PyInt_FromNumber(PyObject **number_var, const char *argname, int accept_none) { + PyObject *number = *number_var; + if (likely((accept_none && number == Py_None) || PyLong_CheckExact(number))) { + return 0; + } + PyObject *int_object; + if (likely(PyNumber_Check(number))) { + int_object = PyNumber_Long(number); + if (unlikely(!int_object)) goto bad; + } else { + __Pyx_PyBuiltin_Invalid(number, "int", argname); + goto bad; + } + *number_var = int_object; + Py_DECREF(number); + return 0; +bad: + *number_var = NULL; + Py_DECREF(number); + return -1; +} + +/* SliceTupleAndList */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_crop_slice(Py_ssize_t* _start, Py_ssize_t* _stop, Py_ssize_t* _length) { + Py_ssize_t start = *_start, stop = *_stop, length = *_length; + if (start < 0) { + start += length; + if (start < 0) + start = 0; + } + if (stop < 0) + stop += length; + else if (stop > length) + stop = length; + *_length = stop - start; + *_start = start; + *_stop = stop; +} +static CYTHON_INLINE PyObject* __Pyx_PyTuple_GetSlice( + PyObject* src, Py_ssize_t start, Py_ssize_t stop) { + Py_ssize_t length = PyTuple_GET_SIZE(src); + __Pyx_crop_slice(&start, &stop, &length); + return __Pyx_PyTuple_FromArray(((PyTupleObject*)src)->ob_item + start, length); +} +static CYTHON_INLINE PyObject* __Pyx_PyList_GetSlice_locked( + PyObject* src, Py_ssize_t start, Py_ssize_t stop) { + Py_ssize_t length = PyList_GET_SIZE(src); + __Pyx_crop_slice(&start, &stop, &length); + if (length <= 0) { + return PyList_New(0); + } + return __Pyx_PyList_FromArray(((PyListObject*)src)->ob_item + start, length); +} +static CYTHON_INLINE PyObject* __Pyx_PyList_GetSlice( + PyObject* src, Py_ssize_t start, Py_ssize_t stop) { + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(src); + result = __Pyx_PyList_GetSlice_locked(src, start, stop); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +#endif // CYTHON_COMPILING_IN_CPYTHON + +/* set_iter */ +static CYTHON_INLINE PyObject* __Pyx_set_iterator(PyObject* iterable, int is_set, + Py_ssize_t* p_orig_length, int* p_source_is_set) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030d0000 + is_set = is_set || likely(PySet_CheckExact(iterable) || PyFrozenSet_CheckExact(iterable)); + *p_source_is_set = is_set; + if (likely(is_set)) { + *p_orig_length = PySet_Size(iterable); + Py_INCREF(iterable); + return iterable; + } +#else + CYTHON_UNUSED_VAR(is_set); + *p_source_is_set = 0; +#endif + *p_orig_length = 0; + return PyObject_GetIter(iterable); +} +static CYTHON_INLINE int __Pyx_set_iter_next( + PyObject* iter_obj, Py_ssize_t orig_length, + Py_ssize_t* ppos, PyObject **value, + int source_is_set) { + if (!CYTHON_COMPILING_IN_CPYTHON || PY_VERSION_HEX >= 0x030d0000 || unlikely(!source_is_set)) { + *value = PyIter_Next(iter_obj); + if (unlikely(!*value)) { + return __Pyx_IterFinish(); + } + CYTHON_UNUSED_VAR(orig_length); + CYTHON_UNUSED_VAR(ppos); + return 1; + } +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030d0000 + if (unlikely(PySet_GET_SIZE(iter_obj) != orig_length)) { + PyErr_SetString( + PyExc_RuntimeError, + "set changed size during iteration"); + return -1; + } + { + Py_hash_t hash; + int ret = _PySet_NextEntry(iter_obj, ppos, value, &hash); + assert (ret != -1); + if (likely(ret)) { + Py_INCREF(*value); + return 1; + } + } +#endif + return 0; +} + +/* PyLongCompare */ +static CYTHON_INLINE int __Pyx_PyLong_BoolEqObjC(PyObject *op1, PyObject *op2, long intval, long inplace) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_UNUSED_VAR(inplace); + if (op1 == op2) { + return 1; + } + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + int unequal; + unsigned long uintval; + Py_ssize_t size = __Pyx_PyLong_DigitCount(op1); + const digit* digits = __Pyx_PyLong_Digits(op1); + if (intval == 0) { + return (__Pyx_PyLong_IsZero(op1) == 1); + } else if (intval < 0) { + if (__Pyx_PyLong_IsNonNeg(op1)) + return 0; + intval = -intval; + } else { + if (__Pyx_PyLong_IsNeg(op1)) + return 0; + } + uintval = (unsigned long) intval; +#if PyLong_SHIFT * 4 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 4)) { + unequal = (size != 5) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[4] != ((uintval >> (4 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 3 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 3)) { + unequal = (size != 4) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 2 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 2)) { + unequal = (size != 3) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 1 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 1)) { + unequal = (size != 2) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif + unequal = (size != 1) || (((unsigned long) digits[0]) != (uintval & (unsigned long) PyLong_MASK)); + return (unequal == 0); + } + #endif + if (PyFloat_CheckExact(op1)) { + const long b = intval; + double a = __Pyx_PyFloat_AS_DOUBLE(op1); + return ((double)a == (double)b); + } + return __Pyx_PyObject_IsTrueAndDecref( + PyObject_RichCompare(op1, op2, Py_EQ)); +} + +/* py_set_discard_unhashable (used by py_set_remove) */ +static int __Pyx_PySet_DiscardUnhashable(PyObject *set, PyObject *key) { + PyObject *tmpkey; + int rv; + if (likely(!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError))) + return -1; + PyErr_Clear(); + tmpkey = __Pyx_PyFrozenSet_New(key); + if (tmpkey == NULL) + return -1; + rv = PySet_Discard(set, tmpkey); + Py_DECREF(tmpkey); + return rv; +} + +/* py_set_remove */ +static int __Pyx_PySet_RemoveNotFound(PyObject *set, PyObject *key, int found) { + if (unlikely(found < 0)) { + found = __Pyx_PySet_DiscardUnhashable(set, key); + } + if (likely(found == 0)) { + PyObject *tup; + tup = PyTuple_Pack(1, key); + if (!tup) + return -1; + PyErr_SetObject(PyExc_KeyError, tup); + Py_DECREF(tup); + return -1; + } + return found; +} +static CYTHON_INLINE int __Pyx_PySet_Remove(PyObject *set, PyObject *key) { + int found = PySet_Discard(set, key); + if (unlikely(found != 1)) { + return __Pyx_PySet_RemoveNotFound(set, key, found); + } + return 0; +} + +/* AllocateExtensionType */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final) { + if (is_final || likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) { + allocfunc alloc_func = __Pyx_PyType_GetSlot(t, tp_alloc, allocfunc); + return alloc_func(t, 0); + } else { + newfunc tp_new = __Pyx_PyType_TryGetSlot(&PyBaseObject_Type, tp_new, newfunc); + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (!tp_new) { + PyObject *new_str = PyUnicode_FromString("__new__"); + if (likely(new_str)) { + PyObject *o = PyObject_CallMethodObjArgs((PyObject *)&PyBaseObject_Type, new_str, t, NULL); + Py_DECREF(new_str); + return o; + } else + return NULL; + } else + #endif + return tp_new(t, __pyx_mstate_global->__pyx_empty_tuple, 0); + } +} + +/* CallTypeTraverse */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* LimitedApiGetTypeDict (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_vectorcall_offset = memb->offset; + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* PyObjectGetMethod (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { + PyObject *attr; +#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; + PyTypeObject *tp = Py_TYPE(obj); + PyObject *descr; + descrgetfunc f = NULL; + PyObject **dictptr, *dict; + int meth_found = 0; + assert (*method == NULL); + if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) { + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; + } + if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) { + return 0; + } + descr = _PyType_Lookup(tp, name); + if (likely(descr != NULL)) { + Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#else + #ifdef __Pyx_CyFunction_USED + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + #else + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) + #endif +#endif + { + meth_found = 1; + } else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + } + } + dictptr = _PyObject_GetDictPtr(obj); + if (dictptr != NULL && (dict = *dictptr) != NULL) { + Py_INCREF(dict); + attr = __Pyx_PyDict_GetItemStr(dict, name); + if (attr != NULL) { + Py_INCREF(attr); + Py_DECREF(dict); + Py_XDECREF(descr); + goto try_unpack; + } + Py_DECREF(dict); + } + if (meth_found) { + *method = descr; + return 1; + } + if (f != NULL) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + if (likely(descr != NULL)) { + *method = descr; + return 0; + } + type_name = __Pyx_PyType_GetFullyQualifiedName(tp); + PyErr_Format(PyExc_AttributeError, + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); + __Pyx_DECREF_TypeName(type_name); + return 0; +#else + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; +#endif +try_unpack: +#if CYTHON_UNPACK_METHODS + if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) { + PyObject *function = PyMethod_GET_FUNCTION(attr); + Py_INCREF(function); + Py_DECREF(attr); + *method = function; + return 1; + } +#endif + *method = attr; + return 0; +} +#endif + +/* PyObjectCallMethod0 (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) { +#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[1] = {obj}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_CallNoArg; + return PyObject_VectorcallMethod(method_name, args, 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#else + PyObject *method = NULL, *result = NULL; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_CallOneArg(method, obj); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) goto bad; + result = __Pyx_PyObject_CallNoArg(method); + Py_DECREF(method); +bad: + return result; +#endif +} + +/* ValidateBasesTuple (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) { + Py_ssize_t i, n; +#if CYTHON_ASSUME_SAFE_SIZE + n = PyTuple_GET_SIZE(bases); +#else + n = PyTuple_Size(bases); + if (unlikely(n < 0)) return -1; +#endif + for (i = 1; i < n; i++) + { + PyTypeObject *b; +#if CYTHON_AVOID_BORROWED_REFS + PyObject *b0 = PySequence_GetItem(bases, i); + if (!b0) return -1; +#elif CYTHON_ASSUME_SAFE_MACROS + PyObject *b0 = PyTuple_GET_ITEM(bases, i); +#else + PyObject *b0 = PyTuple_GetItem(bases, i); + if (!b0) return -1; +#endif + b = (PyTypeObject*) b0; + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + if (dictoffset == 0) + { + Py_ssize_t b_dictoffset = 0; +#if CYTHON_USE_TYPE_SLOTS + b_dictoffset = b->tp_dictoffset; +#else + PyObject *py_b_dictoffset = PyObject_GetAttrString((PyObject*)b, "__dictoffset__"); + if (!py_b_dictoffset) goto dictoffset_return; + b_dictoffset = PyLong_AsSsize_t(py_b_dictoffset); + Py_DECREF(py_b_dictoffset); + if (b_dictoffset == -1 && PyErr_Occurred()) goto dictoffset_return; +#endif + if (b_dictoffset) { + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "extension type '%.200s' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " + "either add 'cdef dict __dict__' to the extension type " + "or add '__slots__ = [...]' to the base type", + type_name, b_name); + __Pyx_DECREF_TypeName(b_name); + } +#if !CYTHON_USE_TYPE_SLOTS + dictoffset_return: +#endif +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + } +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + } + return 0; +} +#endif + +/* PyType_Ready */ +CYTHON_UNUSED static int __Pyx_PyType_HasMultipleInheritance(PyTypeObject *t) { + while (t) { + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases) { + return 1; + } + t = __Pyx_PyType_GetSlot(t, tp_base, PyTypeObject*); + } + return 0; +} +static int __Pyx_PyType_Ready(PyTypeObject *t) { +#if CYTHON_USE_TYPE_SPECS || !CYTHON_COMPILING_IN_CPYTHON || defined(PYSTON_MAJOR_VERSION) + (void)__Pyx_PyObject_CallMethod0; +#if CYTHON_USE_TYPE_SPECS + (void)__Pyx_validate_bases_tuple; +#endif + return PyType_Ready(t); +#else + int r; + if (!__Pyx_PyType_HasMultipleInheritance(t)) { + return PyType_Ready(t); + } + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1)) + return -1; +#if !defined(PYSTON_MAJOR_VERSION) + { + int gc_was_enabled; + #if PY_VERSION_HEX >= 0x030A00b1 + gc_was_enabled = PyGC_Disable(); + (void)__Pyx_PyObject_CallMethod0; + #else + PyObject *ret, *py_status; + PyObject *gc = NULL; + #if (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) &&\ + !CYTHON_COMPILING_IN_GRAAL + gc = PyImport_GetModule(__pyx_mstate_global->__pyx_kp_u_gc); + #endif + if (unlikely(!gc)) gc = PyImport_Import(__pyx_mstate_global->__pyx_kp_u_gc); + if (unlikely(!gc)) return -1; + py_status = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_isenabled); + if (unlikely(!py_status)) { + Py_DECREF(gc); + return -1; + } + gc_was_enabled = __Pyx_PyObject_IsTrue(py_status); + Py_DECREF(py_status); + if (gc_was_enabled > 0) { + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_disable); + if (unlikely(!ret)) { + Py_DECREF(gc); + return -1; + } + Py_DECREF(ret); + } else if (unlikely(gc_was_enabled == -1)) { + Py_DECREF(gc); + return -1; + } + #endif + t->tp_flags |= Py_TPFLAGS_HEAPTYPE; +#if PY_VERSION_HEX >= 0x030A0000 + t->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; +#endif +#else + (void)__Pyx_PyObject_CallMethod0; +#endif + r = PyType_Ready(t); +#if !defined(PYSTON_MAJOR_VERSION) + t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE; + #if PY_VERSION_HEX >= 0x030A00b1 + if (gc_was_enabled) + PyGC_Enable(); + #else + if (gc_was_enabled) { + PyObject *tp, *v, *tb; + PyErr_Fetch(&tp, &v, &tb); + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_enable); + if (likely(ret || r == -1)) { + Py_XDECREF(ret); + PyErr_Restore(tp, v, tb); + } else { + Py_XDECREF(tp); + Py_XDECREF(v); + Py_XDECREF(tb); + r = -1; + } + } + Py_DECREF(gc); + #endif + } +#endif + return r; +#endif +} + +/* HasAttr (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { + PyObject *r; + if (unlikely(!PyUnicode_Check(n))) { + PyErr_SetString(PyExc_TypeError, + "hasattr(): attribute name must be string"); + return -1; + } + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (!r) { + return (unlikely(PyErr_Occurred())) ? -1 : 0; + } else { + Py_DECREF(r); + return 1; + } +} +#endif + +/* ImportImpl (used by Import) */ +static int __Pyx__Import_GetModule(PyObject *qualname, PyObject **module) { + PyObject *imported_module = PyImport_GetModule(qualname); + if (unlikely(!imported_module)) { + *module = NULL; + if (PyErr_Occurred()) { + return -1; + } + return 0; + } + *module = imported_module; + return 1; +} +static int __Pyx__Import_Lookup(PyObject *qualname, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject **module) { + PyObject *imported_module; + PyObject *top_level_package_name; + Py_ssize_t i; + int status, module_found; + Py_ssize_t dot_index; + module_found = __Pyx__Import_GetModule(qualname, &imported_module); + if (unlikely(!module_found || module_found == -1)) { + *module = NULL; + return module_found; + } + if (imported_names) { + for (i = 0; i < len_imported_names; i++) { + PyObject *imported_name = imported_names[i]; +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 + int has_imported_attribute = PyObject_HasAttr(imported_module, imported_name); +#else + int has_imported_attribute = PyObject_HasAttrWithError(imported_module, imported_name); + if (unlikely(has_imported_attribute == -1)) goto error; +#endif + if (!has_imported_attribute) { + goto not_found; + } + } + *module = imported_module; + return 1; + } + dot_index = PyUnicode_FindChar(qualname, '.', 0, PY_SSIZE_T_MAX, 1); + if (dot_index == -1) { + *module = imported_module; + return 1; + } + if (unlikely(dot_index == -2)) goto error; + top_level_package_name = PyUnicode_Substring(qualname, 0, dot_index); + if (unlikely(!top_level_package_name)) goto error; + Py_DECREF(imported_module); + status = __Pyx__Import_GetModule(top_level_package_name, module); + Py_DECREF(top_level_package_name); + return status; +error: + Py_DECREF(imported_module); + *module = NULL; + return -1; +not_found: + Py_DECREF(imported_module); + *module = NULL; + return 0; +} +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *from_list = 0; + int module_found; + if (!qualname) { + qualname = name; + } + module_found = __Pyx__Import_Lookup(qualname, imported_names, len_imported_names, &module); + if (likely(module_found == 1)) { + return module; + } else if (unlikely(module_found == -1)) { + return NULL; + } + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (imported_names) { +#if CYTHON_COMPILING_IN_CPYTHON + from_list = __Pyx_PyList_FromArray(imported_names, len_imported_names); + if (unlikely(!from_list)) + goto bad; +#else + from_list = PyList_New(len_imported_names); + if (unlikely(!from_list)) goto bad; + for (Py_ssize_t i=0; i<len_imported_names; ++i) { + if (PyList_SetItem(from_list, i, __Pyx_NewRef(imported_names[i])) < 0) goto bad; + } +#endif + } + if (level == -1) { + const char* package_sep = strchr(__Pyx_MODULE_NAME, '.'); + if (package_sep != (0)) { + module = PyImport_ImportModuleLevelObject( + name, moddict, empty_dict, from_list, 1); + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + if (!module) { + module = PyImport_ImportModuleLevelObject( + name, moddict, empty_dict, from_list, level); + } +bad: + Py_XDECREF(from_list); + Py_XDECREF(empty_dict); + return module; +} + +/* Import */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, int level) { + return __Pyx__Import(name, imported_names, len_imported_names, qualname, __pyx_mstate_global->__pyx_d, level); +} + +/* ImportFrom */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + const char* module_name_str = 0; + PyObject* module_name = 0; + PyObject* module_dot = 0; + PyObject* full_name = 0; + PyErr_Clear(); + module_name_str = PyModule_GetName(module); + if (unlikely(!module_name_str)) { goto modbad; } + module_name = PyUnicode_FromString(module_name_str); + if (unlikely(!module_name)) { goto modbad; } + module_dot = PyUnicode_Concat(module_name, __pyx_mstate_global->__pyx_kp_u_); + if (unlikely(!module_dot)) { goto modbad; } + full_name = PyUnicode_Concat(module_dot, name); + if (unlikely(!full_name)) { goto modbad; } + #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + { + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + goto modbad; + value = PyObject_GetItem(modules, full_name); + } + #else + value = PyImport_GetModule(full_name); + #endif + modbad: + Py_XDECREF(full_name); + Py_XDECREF(module_dot); + Py_XDECREF(module_name); + } + if (unlikely(!value)) { + PyErr_Format(PyExc_ImportError, "cannot import name %S", name); + } + return value; +} + +/* dict_setdefault (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value) { + PyObject* value; +#if __PYX_LIMITED_VERSION_HEX >= 0x030F0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4) + PyDict_SetDefaultRef(d, key, default_value, &value); +#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + PyObject *args[] = {d, key, default_value}; + value = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_n_u_setdefault, args, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#elif CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethodObjArgs(d, __pyx_mstate_global->__pyx_n_u_setdefault, key, default_value, NULL); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* AddModuleRef (used by FetchSharedCythonModule) */ +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + static PyObject *__Pyx_PyImport_AddModuleObjectRef(PyObject *name) { + PyObject *module_dict = PyImport_GetModuleDict(); + PyObject *m; + if (PyMapping_GetOptionalItem(module_dict, name, &m) < 0) { + return NULL; + } + if (m != NULL && PyModule_Check(m)) { + return m; + } + Py_XDECREF(m); + m = PyModule_NewObject(name); + if (m == NULL) + return NULL; + if (PyDict_CheckExact(module_dict)) { + PyObject *new_m; + (void)PyDict_SetDefaultRef(module_dict, name, m, &new_m); + Py_DECREF(m); + return new_m; + } else { + if (PyObject_SetItem(module_dict, name, m) != 0) { + Py_DECREF(m); + return NULL; + } + return m; + } + } + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *py_name = PyUnicode_FromString(name); + if (!py_name) return NULL; + PyObject *module = __Pyx_PyImport_AddModuleObjectRef(py_name); + Py_DECREF(py_name); + return module; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif + +/* FetchSharedCythonModule (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); +} + +/* FetchCommonType (used by CommonTypesMetaclass) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); +#if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); +#else + result->ob_type = metaclass; +#endif + Py_DECREF(old_tp); + } + return result; +} +#else +#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) +#endif +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; +#else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; +#endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + cached_type = __Pyx_PyType_FromMetaclass( + metaclass, + CYTHON_USE_MODULE_STATE ? module : abi_module, + spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } +done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CommonTypesMetaclass (used by CythonFunctionShared) */ +static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); +} +#if __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject* __pyx_CommonTypesMetaclass_call(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *args, CYTHON_UNUSED PyObject *kwds) { + PyErr_SetString(PyExc_TypeError, "Cannot instantiate Cython internal types"); + return NULL; +} +static int __pyx_CommonTypesMetaclass_setattr(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *attr, CYTHON_UNUSED PyObject *value) { + PyErr_SetString(PyExc_TypeError, "Cython internal types are immutable"); + return -1; +} +#endif +static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {Py_tp_call, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_new, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_setattro, (void*)__pyx_CommonTypesMetaclass_setattr}, + #endif + {0, 0} +}; +static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots +}; +static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; +} + +/* PyMethodNew (used by CythonFunctionShared) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; +} +#else +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#endif + +/* PyVectorcallFastCallDict (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i; + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos; + #else + Py_ssize_t pos; + #endif + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = 0; + i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (__Pyx_PyDict_NextRef(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); +cleanup: + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(pos); + #endif + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : +#if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); +#else + PyDict_GET_SIZE(kw); +#endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } +#endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); +} +#endif + +/* CythonFunctionShared (used by CythonFunction) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; +} +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); +} +#else +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +} +#endif +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} +static PyObject * +__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_doc == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; +#else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +#endif + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); +#else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +#endif +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); +#if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); +#else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } +#endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } +ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); +#endif +} +static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); +#endif +} +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject * +__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); +} +static int +__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); +} +#endif +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, +#else + {"func_dict", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, +#endif + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { +#if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#endif +#if CYTHON_METH_FASTCALL +#if CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { +#if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; +#endif + if (unlikely(op == NULL)) + return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; +#endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; +#if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; +#endif + Py_XINCREF(closure); + op->func_closure = closure; +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + op->func_dict = NULL; +#endif + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); +#else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func_dict); +#elif PY_VERSION_HEX < 0x030d0000 + _PyObject_ClearManagedDict((PyObject*)m); +#else + PyObject_ClearManagedDict((PyObject*)m); +#endif + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif +#endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); +#else + Py_VISIT(((PyCFunctionObject*)m)->m_module); +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func_dict); +#else + { + int e = +#if PY_VERSION_HEX < 0x030d0000 + _PyObject_VisitManagedDict +#else + PyObject_VisitManagedDict +#endif + ((PyObject*)m, visit, arg); + if (e != 0) return e; + } +#endif + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); +#endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("<cyfunction %U at %p>", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; +#else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; +#endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)func)->m_self; +#endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; +#if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); +#else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; +#endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; +} +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); +} +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + #if PY_VERSION_HEX < 0x030e00A6 + size_t nargs_value = (size_t) nargs; + #else + Py_ssize_t nargs_value = nargs; + #endif + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, nargs_value, kwnames); +} +#endif +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if CYTHON_METH_FASTCALL +#if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | +#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif +#endif // CYTHON_METH_FASTCALL +#if PY_VERSION_HEX >= 0x030C0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_TPFLAGS_MANAGED_DICT | +#endif + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots +}; +static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CythonFunction */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + +/* CLineInTraceback (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +#define __Pyx_PyProbablyModule_GetDict(o) __Pyx_XNewRef(PyModule_GetDict(o)) +#elif !CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyProbablyModule_GetDict(o) PyObject_GenericGetDict(o, NULL); +#else +PyObject* __Pyx_PyProbablyModule_GetDict(PyObject *o) { + PyObject **dict_ptr = _PyObject_GetDictPtr(o); + return dict_ptr ? __Pyx_XNewRef(*dict_ptr) : NULL; +} +#endif +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline = NULL; + PyObject *ptype, *pvalue, *ptraceback; + PyObject *cython_runtime_dict; + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + cython_runtime_dict = __Pyx_PyProbablyModule_GetDict(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, cython_runtime_dict, + __Pyx_PyDict_SetDefault(cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False)) + } + if (use_cline == NULL || use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + Py_XDECREF(cython_runtime_dict); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache (used by AddTraceback) */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + } else { + py_funcname = PyUnicode_FromString(funcname); + } + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* PyObjectVectorCallKwBuilder (used by CIntToPy) */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__2); + } + goto done; +} +#endif + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i<n; i++) { + if (exc_type == PyTuple_GET_ITEM(tuple, i)) return 1; + } + for (i=0; i<n; i++) { + PyObject *t = PyTuple_GET_ITEM(tuple, i); + if (likely(PyExceptionClass_Check(t))) { + if (__Pyx_inner_PyErr_GivenExceptionMatches2(exc_type, NULL, t)) return 1; + } else { + } + } + return 0; +} +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject* exc_type) { + if (likely(err == exc_type)) return 1; + if (likely(PyExceptionClass_Check(err))) { + if (likely(PyExceptionClass_Check(exc_type))) { + return __Pyx_inner_PyErr_GivenExceptionMatches2(err, NULL, exc_type); + } else if (likely(PyTuple_Check(exc_type))) { + return __Pyx_PyErr_GivenExceptionMatchesTuple(err, exc_type); + } else { + } + } + return PyErr_GivenExceptionMatches(err, exc_type); +} +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *exc_type1, PyObject *exc_type2) { + assert(PyExceptionClass_Check(exc_type1)); + assert(PyExceptionClass_Check(exc_type2)); + if (likely(err == exc_type1 || err == exc_type2)) return 1; + if (likely(PyExceptionClass_Check(err))) { + return __Pyx_inner_PyErr_GivenExceptionMatches2(err, exc_type1, exc_type2); + } + return (PyErr_GivenExceptionMatches(err, exc_type1) || PyErr_GivenExceptionMatches(err, exc_type2)); +} +#endif + +/* GetRuntimeVersion */ +#if __PYX_LIMITED_VERSION_HEX < 0x030b0000 +void __Pyx_init_runtime_version(void) { + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } +} +#endif +static unsigned long __Pyx_get_runtime_version(void) { +#if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + return __Pyx_cached_runtime_version; +#endif +} + +/* GetTopmostException (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException (used by CoroutineBase) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* SwapException (used by CoroutineBase) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_value = exc_info->exc_value; + exc_info->exc_value = *value; + if (tmp_value == NULL || tmp_value == Py_None) { + Py_XDECREF(tmp_value); + tmp_value = NULL; + tmp_type = NULL; + tmp_tb = NULL; + } else { + tmp_type = (PyObject*) Py_TYPE(tmp_value); + Py_INCREF(tmp_type); + #if CYTHON_COMPILING_IN_CPYTHON + tmp_tb = ((PyBaseExceptionObject*) tmp_value)->traceback; + Py_XINCREF(tmp_tb); + #else + tmp_tb = PyException_GetTraceback(tmp_value); + #endif + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = *type; + exc_info->exc_value = *value; + exc_info->exc_traceback = *tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + #endif + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* PyObjectCall2Args (used by PyObjectCallMethod1) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectCallMethod1 (used by CoroutineBase) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg) { + PyObject *result = __Pyx_PyObject_CallOneArg(method, arg); + Py_DECREF(method); + return result; +} +#endif +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { +#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[2] = {obj, arg}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_Call2Args; + return PyObject_VectorcallMethod(method_name, args, 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#else + PyObject *method = NULL, *result; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_Call2Args(method, obj, arg); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) return NULL; + return __Pyx__PyObject_CallMethod1(method, arg); +#endif +} + +/* ReturnWithStopIteration (used by CoroutineBase) */ +static void __Pyx__ReturnWithStopIteration(PyObject* value, int async); +static CYTHON_INLINE void __Pyx_ReturnWithStopIteration(PyObject* value, int async, int iternext) { + if (value == Py_None) { + if (async || !iternext) + PyErr_SetNone(async ? PyExc_StopAsyncIteration : PyExc_StopIteration); + return; + } + __Pyx__ReturnWithStopIteration(value, async); +} +static void __Pyx__ReturnWithStopIteration(PyObject* value, int async) { +#if CYTHON_COMPILING_IN_CPYTHON + __Pyx_PyThreadState_declare +#endif + PyObject *exc; + PyObject *exc_type = async ? PyExc_StopAsyncIteration : PyExc_StopIteration; +#if CYTHON_COMPILING_IN_CPYTHON + if ((PY_VERSION_HEX >= (0x030C00A6)) || unlikely(PyTuple_Check(value) || PyExceptionInstance_Check(value))) { + if (PY_VERSION_HEX >= (0x030e00A1)) { + exc = __Pyx_PyObject_CallOneArg(exc_type, value); + } else { + PyObject *args_tuple = PyTuple_New(1); + if (unlikely(!args_tuple)) return; + Py_INCREF(value); + PyTuple_SET_ITEM(args_tuple, 0, value); + exc = PyObject_Call(exc_type, args_tuple, NULL); + Py_DECREF(args_tuple); + } + if (unlikely(!exc)) return; + } else { + Py_INCREF(value); + exc = value; + } + #if CYTHON_FAST_THREAD_STATE + __Pyx_PyThreadState_assign + #if CYTHON_USE_EXC_INFO_STACK + if (!__pyx_tstate->exc_info->exc_value) + #else + if (!__pyx_tstate->exc_type) + #endif + { + Py_INCREF(exc_type); + __Pyx_ErrRestore(exc_type, exc, NULL); + return; + } + #endif +#else + exc = __Pyx_PyObject_CallOneArg(exc_type, value); + if (unlikely(!exc)) return; +#endif + PyErr_SetObject(exc_type, exc); + Py_DECREF(exc); +} + +/* CoroutineBase (used by Generator) */ +#if !CYTHON_COMPILING_IN_LIMITED_API +#include <frameobject.h> +#if PY_VERSION_HEX >= 0x030b00a6 && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#endif // CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE void +__Pyx_Coroutine_Undelegate(__pyx_CoroutineObject *gen) { +#if CYTHON_USE_AM_SEND + gen->yieldfrom_am_send = NULL; +#endif + Py_CLEAR(gen->yieldfrom); +} +static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *__pyx_tstate, PyObject **pvalue) { + PyObject *et, *ev, *tb; + PyObject *value = NULL; + CYTHON_UNUSED_VAR(__pyx_tstate); + __Pyx_ErrFetch(&et, &ev, &tb); + if (!et) { + Py_XDECREF(tb); + Py_XDECREF(ev); + Py_INCREF(Py_None); + *pvalue = Py_None; + return 0; + } + if (likely(et == PyExc_StopIteration)) { + if (!ev) { + Py_INCREF(Py_None); + value = Py_None; + } + else if (likely(__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration))) { + #if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + value = PyObject_GetAttr(ev, __pyx_mstate_global->__pyx_n_u_value); + if (unlikely(!value)) goto limited_api_failure; + #else + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); + #endif + Py_DECREF(ev); + } + else if (unlikely(PyTuple_Check(ev))) { + Py_ssize_t tuple_size = __Pyx_PyTuple_GET_SIZE(ev); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(tuple_size < 0)) { + Py_XDECREF(tb); + Py_DECREF(ev); + Py_DECREF(et); + return -1; + } + #endif + if (tuple_size >= 1) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + value = PyTuple_GET_ITEM(ev, 0); + Py_INCREF(value); +#elif CYTHON_ASSUME_SAFE_MACROS + value = PySequence_ITEM(ev, 0); +#else + value = PySequence_GetItem(ev, 0); + if (!value) goto limited_api_failure; +#endif + } else { + Py_INCREF(Py_None); + value = Py_None; + } + Py_DECREF(ev); + } + else if (!__Pyx_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { + value = ev; + } + if (likely(value)) { + Py_XDECREF(tb); + Py_DECREF(et); + *pvalue = value; + return 0; + } + } else if (!__Pyx_PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { + __Pyx_ErrRestore(et, ev, tb); + return -1; + } + PyErr_NormalizeException(&et, &ev, &tb); + if (unlikely(!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration))) { + __Pyx_ErrRestore(et, ev, tb); + return -1; + } + Py_XDECREF(tb); + Py_DECREF(et); +#if CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_GetAttr(ev, __pyx_mstate_global->__pyx_n_u_value); +#else + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); +#endif + Py_DECREF(ev); +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!value)) return -1; +#endif + *pvalue = value; + return 0; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL || !CYTHON_ASSUME_SAFE_MACROS + limited_api_failure: + Py_XDECREF(et); + Py_XDECREF(tb); + Py_XDECREF(ev); + return -1; +#endif +} +static CYTHON_INLINE +__Pyx_PySendResult __Pyx_Coroutine_status_from_result(PyObject **retval) { + if (*retval) { + return PYGEN_NEXT; + } else if (likely(__Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, retval) == 0)) { + return PYGEN_RETURN; + } else { + return PYGEN_ERROR; + } +} +static CYTHON_INLINE +void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *exc_state) { +#if PY_VERSION_HEX >= 0x030B00a4 + Py_CLEAR(exc_state->exc_value); +#else + PyObject *t, *v, *tb; + t = exc_state->exc_type; + v = exc_state->exc_value; + tb = exc_state->exc_traceback; + exc_state->exc_type = NULL; + exc_state->exc_value = NULL; + exc_state->exc_traceback = NULL; + Py_XDECREF(t); + Py_XDECREF(v); + Py_XDECREF(tb); +#endif +} +#define __Pyx_Coroutine_AlreadyRunningError(gen) (__Pyx__Coroutine_AlreadyRunningError(gen), (PyObject*)NULL) +static void __Pyx__Coroutine_AlreadyRunningError(__pyx_CoroutineObject *gen) { + const char *msg; + CYTHON_MAYBE_UNUSED_VAR(gen); + if ((0)) { + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_Coroutine_Check((PyObject*)gen)) { + msg = "coroutine already executing"; + #endif + #ifdef __Pyx_AsyncGen_USED + } else if (__Pyx_AsyncGen_CheckExact((PyObject*)gen)) { + msg = "async generator already executing"; + #endif + } else { + msg = "generator already executing"; + } + PyErr_SetString(PyExc_ValueError, msg); +} +static void __Pyx_Coroutine_AlreadyTerminatedError(PyObject *gen, PyObject *value, int closing) { + CYTHON_MAYBE_UNUSED_VAR(gen); + CYTHON_MAYBE_UNUSED_VAR(closing); + #ifdef __Pyx_Coroutine_USED + if (!closing && __Pyx_Coroutine_Check(gen)) { + PyErr_SetString(PyExc_RuntimeError, "cannot reuse already awaited coroutine"); + } else + #endif + if (value) { + #ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(gen)) + PyErr_SetNone(PyExc_StopAsyncIteration); + else + #endif + PyErr_SetNone(PyExc_StopIteration); + } +} +static +__Pyx_PySendResult __Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, PyObject **result, int closing) { + __Pyx_PyThreadState_declare + PyThreadState *tstate; + __Pyx_ExcInfoStruct *exc_state; + PyObject *retval; + assert(__Pyx_Coroutine_get_is_running(self)); // Callers should ensure is_running + if (unlikely(self->resume_label == -1)) { + __Pyx_Coroutine_AlreadyTerminatedError((PyObject*)self, value, closing); + return PYGEN_ERROR; + } +#if CYTHON_FAST_THREAD_STATE + __Pyx_PyThreadState_assign + tstate = __pyx_tstate; +#else + tstate = __Pyx_PyThreadState_Current; +#endif + exc_state = &self->gi_exc_state; + if (exc_state->exc_value) { + #if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + #else + PyObject *exc_tb; + #if PY_VERSION_HEX >= 0x030B00a4 && !CYTHON_COMPILING_IN_CPYTHON + exc_tb = PyException_GetTraceback(exc_state->exc_value); + #elif PY_VERSION_HEX >= 0x030B00a4 + exc_tb = ((PyBaseExceptionObject*) exc_state->exc_value)->traceback; + #else + exc_tb = exc_state->exc_traceback; + #endif + if (exc_tb) { + PyTracebackObject *tb = (PyTracebackObject *) exc_tb; + PyFrameObject *f = tb->tb_frame; + assert(f->f_back == NULL); + #if PY_VERSION_HEX >= 0x030B00A1 + f->f_back = PyThreadState_GetFrame(tstate); + #else + Py_XINCREF(tstate->frame); + f->f_back = tstate->frame; + #endif + #if PY_VERSION_HEX >= 0x030B00a4 && !CYTHON_COMPILING_IN_CPYTHON + Py_DECREF(exc_tb); + #endif + } + #endif + } +#if CYTHON_USE_EXC_INFO_STACK + exc_state->previous_item = tstate->exc_info; + tstate->exc_info = exc_state; +#else + if (exc_state->exc_type) { + __Pyx_ExceptionSwap(&exc_state->exc_type, &exc_state->exc_value, &exc_state->exc_traceback); + } else { + __Pyx_Coroutine_ExceptionClear(exc_state); + __Pyx_ExceptionSave(&exc_state->exc_type, &exc_state->exc_value, &exc_state->exc_traceback); + } +#endif + retval = self->body(self, tstate, value); +#if CYTHON_USE_EXC_INFO_STACK + exc_state = &self->gi_exc_state; + tstate->exc_info = exc_state->previous_item; + exc_state->previous_item = NULL; + __Pyx_Coroutine_ResetFrameBackpointer(exc_state); +#endif + *result = retval; + if (self->resume_label == -1) { + return likely(retval) ? PYGEN_RETURN : PYGEN_ERROR; + } + return PYGEN_NEXT; +} +static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStruct *exc_state) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED_VAR(exc_state); +#else + PyObject *exc_tb; + #if PY_VERSION_HEX >= 0x030B00a4 + if (!exc_state->exc_value) return; + exc_tb = PyException_GetTraceback(exc_state->exc_value); + #else + exc_tb = exc_state->exc_traceback; + #endif + if (likely(exc_tb)) { + PyTracebackObject *tb = (PyTracebackObject *) exc_tb; + PyFrameObject *f = tb->tb_frame; + Py_CLEAR(f->f_back); + #if PY_VERSION_HEX >= 0x030B00a4 + Py_DECREF(exc_tb); + #endif + } +#endif +} +#define __Pyx_Coroutine_MethodReturnFromResult(gen, result, retval, iternext)\ + ((result) == PYGEN_NEXT ? (retval) : __Pyx__Coroutine_MethodReturnFromResult(gen, result, retval, iternext)) +static PyObject * +__Pyx__Coroutine_MethodReturnFromResult(PyObject* gen, __Pyx_PySendResult result, PyObject *retval, int iternext) { + CYTHON_MAYBE_UNUSED_VAR(gen); + if (likely(result == PYGEN_RETURN)) { + int is_async = 0; + #ifdef __Pyx_AsyncGen_USED + is_async = __Pyx_AsyncGen_CheckExact(gen); + #endif + __Pyx_ReturnWithStopIteration(retval, is_async, iternext); + Py_XDECREF(retval); + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE +PyObject *__Pyx_PyGen_Send(PyGenObject *gen, PyObject *arg) { +#if PY_VERSION_HEX <= 0x030A00A1 + return _PyGen_Send(gen, arg); +#else + PyObject *result; + if (PyIter_Send((PyObject*)gen, arg ? arg : Py_None, &result) == PYGEN_RETURN) { + if (PyAsyncGen_CheckExact(gen)) { + assert(result == Py_None); + PyErr_SetNone(PyExc_StopAsyncIteration); + } + else if (result == Py_None) { + PyErr_SetNone(PyExc_StopIteration); + } + else { +#if PY_VERSION_HEX < 0x030d00A1 + _PyGen_SetStopIterationValue(result); +#else + if (!PyTuple_Check(result) && !PyExceptionInstance_Check(result)) { + PyErr_SetObject(PyExc_StopIteration, result); + } else { + PyObject *exc = __Pyx_PyObject_CallOneArg(PyExc_StopIteration, result); + if (likely(exc != NULL)) { + PyErr_SetObject(PyExc_StopIteration, exc); + Py_DECREF(exc); + } + } +#endif + } + Py_DECREF(result); + result = NULL; + } + return result; +#endif +} +#endif +static CYTHON_INLINE __Pyx_PySendResult +__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen, PyObject** retval) { + __Pyx_PySendResult result; + PyObject *val = NULL; + assert(__Pyx_Coroutine_get_is_running(gen)); + __Pyx_Coroutine_Undelegate(gen); + __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, &val); + result = __Pyx_Coroutine_SendEx(gen, val, retval, 0); + Py_XDECREF(val); + return result; +} +#if CYTHON_USE_AM_SEND +static __Pyx_PySendResult +__Pyx_Coroutine_SendToDelegate(__pyx_CoroutineObject *gen, __Pyx_pyiter_sendfunc gen_am_send, PyObject *value, PyObject **retval) { + PyObject *ret = NULL; + __Pyx_PySendResult delegate_result, result; + assert(__Pyx_Coroutine_get_is_running(gen)); + delegate_result = gen_am_send(gen->yieldfrom, value, &ret); + if (delegate_result == PYGEN_NEXT) { + assert (ret != NULL); + *retval = ret; + return PYGEN_NEXT; + } + assert (delegate_result != PYGEN_ERROR || ret == NULL); + __Pyx_Coroutine_Undelegate(gen); + result = __Pyx_Coroutine_SendEx(gen, ret, retval, 0); + Py_XDECREF(ret); + return result; +} +#endif +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { + PyObject *retval = NULL; + __Pyx_PySendResult result = __Pyx_Coroutine_AmSend(self, value, &retval); + return __Pyx_Coroutine_MethodReturnFromResult(self, result, retval, 0); +} +static __Pyx_PySendResult +__Pyx_Coroutine_AmSend(PyObject *self, PyObject *value, PyObject **retval) { + __Pyx_PySendResult result; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) { + *retval = __Pyx_Coroutine_AlreadyRunningError(gen); + return PYGEN_ERROR; + } + #if CYTHON_USE_AM_SEND + if (gen->yieldfrom_am_send) { + result = __Pyx_Coroutine_SendToDelegate(gen, gen->yieldfrom_am_send, value, retval); + } else + #endif + if (gen->yieldfrom) { + PyObject *yf = gen->yieldfrom; + PyObject *ret; + #if !CYTHON_USE_AM_SEND + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Coroutine_Send(yf, value); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_Check(yf)) { + ret = __Pyx_Coroutine_Send(yf, value); + } else + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_PyAsyncGenASend_CheckExact(yf)) { + ret = __Pyx_async_gen_asend_send(yf, value); + } else + #endif + #if CYTHON_COMPILING_IN_CPYTHON + if (PyGen_CheckExact(yf)) { + ret = __Pyx_PyGen_Send((PyGenObject*)yf, value == Py_None ? NULL : value); + } else + if (PyCoro_CheckExact(yf)) { + ret = __Pyx_PyGen_Send((PyGenObject*)yf, value == Py_None ? NULL : value); + } else + #endif + #endif + { + #if !CYTHON_COMPILING_IN_LIMITED_API || __PYX_LIMITED_VERSION_HEX >= 0x03080000 + if (value == Py_None && PyIter_Check(yf)) + ret = __Pyx_PyIter_Next_Plain(yf); + else + #endif + ret = __Pyx_PyObject_CallMethod1(yf, __pyx_mstate_global->__pyx_n_u_send, value); + } + if (likely(ret)) { + __Pyx_Coroutine_unset_is_running(gen); + *retval = ret; + return PYGEN_NEXT; + } + result = __Pyx_Coroutine_FinishDelegation(gen, retval); + } else { + result = __Pyx_Coroutine_SendEx(gen, value, retval, 0); + } + __Pyx_Coroutine_unset_is_running(gen); + return result; +} +static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { + __Pyx_PySendResult result; + PyObject *retval = NULL; + CYTHON_UNUSED_VAR(gen); + assert(__Pyx_Coroutine_get_is_running(gen)); + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + result = __Pyx_Coroutine_Close(yf, &retval); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_Check(yf)) { + result = __Pyx_Coroutine_Close(yf, &retval); + } else + if (__Pyx_CoroutineAwait_CheckExact(yf)) { + result = __Pyx_CoroutineAwait_Close((__pyx_CoroutineAwaitObject*)yf); + } else + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_PyAsyncGenASend_CheckExact(yf)) { + retval = __Pyx_async_gen_asend_close(yf, NULL); + result = PYGEN_RETURN; + } else + if (__pyx_PyAsyncGenAThrow_CheckExact(yf)) { + retval = __Pyx_async_gen_athrow_close(yf, NULL); + result = PYGEN_RETURN; + } else + #endif + { + PyObject *meth; + result = PYGEN_RETURN; + meth = __Pyx_PyObject_GetAttrStrNoError(yf, __pyx_mstate_global->__pyx_n_u_close); + if (unlikely(!meth)) { + if (unlikely(PyErr_Occurred())) { + PyErr_WriteUnraisable(yf); + } + } else { + retval = __Pyx_PyObject_CallNoArg(meth); + Py_DECREF(meth); + if (unlikely(!retval)) { + result = PYGEN_ERROR; + } + } + } + Py_XDECREF(retval); + return result == PYGEN_ERROR ? -1 : 0; +} +static PyObject *__Pyx_Generator_Next(PyObject *self) { + __Pyx_PySendResult result; + PyObject *retval = NULL; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) { + return __Pyx_Coroutine_AlreadyRunningError(gen); + } + #if CYTHON_USE_AM_SEND + if (gen->yieldfrom_am_send) { + result = __Pyx_Coroutine_SendToDelegate(gen, gen->yieldfrom_am_send, Py_None, &retval); + } else + #endif + if (gen->yieldfrom) { + PyObject *yf = gen->yieldfrom; + PyObject *ret; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Generator_Next(yf); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + ret = __Pyx_Coroutine_Send(yf, Py_None); + } else + #endif + #if CYTHON_COMPILING_IN_CPYTHON && (PY_VERSION_HEX < 0x030A00A3 || !CYTHON_USE_AM_SEND) + if (PyGen_CheckExact(yf)) { + ret = __Pyx_PyGen_Send((PyGenObject*)yf, NULL); + } else + #endif + ret = __Pyx_PyIter_Next_Plain(yf); + if (likely(ret)) { + __Pyx_Coroutine_unset_is_running(gen); + return ret; + } + result = __Pyx_Coroutine_FinishDelegation(gen, &retval); + } else { + result = __Pyx_Coroutine_SendEx(gen, Py_None, &retval, 0); + } + __Pyx_Coroutine_unset_is_running(gen); + return __Pyx_Coroutine_MethodReturnFromResult(self, result, retval, 1); +} +static PyObject *__Pyx_Coroutine_Close_Method(PyObject *self, PyObject *arg) { + PyObject *retval = NULL; + __Pyx_PySendResult result; + CYTHON_UNUSED_VAR(arg); + result = __Pyx_Coroutine_Close(self, &retval); + if (unlikely(result == PYGEN_ERROR)) + return NULL; + Py_XDECREF(retval); + Py_RETURN_NONE; +} +static __Pyx_PySendResult +__Pyx_Coroutine_Close(PyObject *self, PyObject **retval) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + __Pyx_PySendResult result; + PyObject *yf; + int err = 0; + if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) { + *retval = __Pyx_Coroutine_AlreadyRunningError(gen); + return PYGEN_ERROR; + } + yf = gen->yieldfrom; + if (yf) { + Py_INCREF(yf); + err = __Pyx_Coroutine_CloseIter(gen, yf); + __Pyx_Coroutine_Undelegate(gen); + Py_DECREF(yf); + } + if (err == 0) + PyErr_SetNone(PyExc_GeneratorExit); + result = __Pyx_Coroutine_SendEx(gen, NULL, retval, 1); + if (result == PYGEN_ERROR) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_Coroutine_unset_is_running(gen); + if (!__Pyx_PyErr_Occurred()) { + return PYGEN_RETURN; + } else if (likely(__Pyx_PyErr_ExceptionMatches2(PyExc_GeneratorExit, PyExc_StopIteration))) { + __Pyx_PyErr_Clear(); + return PYGEN_RETURN; + } + return PYGEN_ERROR; + } else if (likely(result == PYGEN_RETURN && *retval == Py_None)) { + __Pyx_Coroutine_unset_is_running(gen); + return PYGEN_RETURN; + } else { + const char *msg; + Py_DECREF(*retval); + *retval = NULL; + if ((0)) { + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_Coroutine_Check(self)) { + msg = "coroutine ignored GeneratorExit"; + #endif + #ifdef __Pyx_AsyncGen_USED + } else if (__Pyx_AsyncGen_CheckExact(self)) { + msg = "async generator ignored GeneratorExit"; + #endif + } else { + msg = "generator ignored GeneratorExit"; + } + PyErr_SetString(PyExc_RuntimeError, msg); + __Pyx_Coroutine_unset_is_running(gen); + return PYGEN_ERROR; + } +} +static PyObject *__Pyx__Coroutine_Throw(PyObject *self, PyObject *typ, PyObject *val, PyObject *tb, + PyObject *args, int close_on_genexit) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject *yf; + if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) + return __Pyx_Coroutine_AlreadyRunningError(gen); + yf = gen->yieldfrom; + if (yf) { + __Pyx_PySendResult result; + PyObject *ret; + Py_INCREF(yf); + if (__Pyx_PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && close_on_genexit) { + int err = __Pyx_Coroutine_CloseIter(gen, yf); + Py_DECREF(yf); + __Pyx_Coroutine_Undelegate(gen); + if (err < 0) + goto propagate_exception; + goto throw_here; + } + if (0 + #ifdef __Pyx_Generator_USED + || __Pyx_Generator_CheckExact(yf) + #endif + #ifdef __Pyx_Coroutine_USED + || __Pyx_Coroutine_Check(yf) + #endif + ) { + ret = __Pyx__Coroutine_Throw(yf, typ, val, tb, args, close_on_genexit); + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_CoroutineAwait_CheckExact(yf)) { + ret = __Pyx__Coroutine_Throw(((__pyx_CoroutineAwaitObject*)yf)->coroutine, typ, val, tb, args, close_on_genexit); + #endif + } else { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(yf, __pyx_mstate_global->__pyx_n_u_throw); + if (unlikely(!meth)) { + Py_DECREF(yf); + if (unlikely(PyErr_Occurred())) { + __Pyx_Coroutine_unset_is_running(gen); + return NULL; + } + __Pyx_Coroutine_Undelegate(gen); + goto throw_here; + } + if (likely(args)) { + ret = __Pyx_PyObject_Call(meth, args, NULL); + } else { + PyObject *cargs[4] = {NULL, typ, val, tb}; + ret = __Pyx_PyObject_FastCall(meth, cargs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } + Py_DECREF(meth); + } + Py_DECREF(yf); + if (ret) { + __Pyx_Coroutine_unset_is_running(gen); + return ret; + } + result = __Pyx_Coroutine_FinishDelegation(gen, &ret); + __Pyx_Coroutine_unset_is_running(gen); + return __Pyx_Coroutine_MethodReturnFromResult(self, result, ret, 0); + } +throw_here: + __Pyx_Raise(typ, val, tb, NULL); +propagate_exception: + { + PyObject *retval = NULL; + __Pyx_PySendResult result = __Pyx_Coroutine_SendEx(gen, NULL, &retval, 0); + __Pyx_Coroutine_unset_is_running(gen); + return __Pyx_Coroutine_MethodReturnFromResult(self, result, retval, 0); + } +} +static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { + PyObject *typ; + PyObject *val = NULL; + PyObject *tb = NULL; + if (unlikely(!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb))) + return NULL; + return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1); +} +static CYTHON_INLINE int __Pyx_Coroutine_traverse_excstate(__Pyx_ExcInfoStruct *exc_state, visitproc visit, void *arg) { +#if PY_VERSION_HEX >= 0x030B00a4 + Py_VISIT(exc_state->exc_value); +#else + Py_VISIT(exc_state->exc_type); + Py_VISIT(exc_state->exc_value); + Py_VISIT(exc_state->exc_traceback); +#endif + return 0; +} +static int __Pyx_Coroutine_traverse(__pyx_CoroutineObject *gen, visitproc visit, void *arg) { + { + int e = __Pyx_call_type_traverse((PyObject*)gen, 1, visit, arg); + if (e) return e; + } + Py_VISIT(gen->closure); + Py_VISIT(gen->classobj); + Py_VISIT(gen->yieldfrom); + return __Pyx_Coroutine_traverse_excstate(&gen->gi_exc_state, visit, arg); +} +static int __Pyx_Coroutine_clear(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + Py_CLEAR(gen->closure); + Py_CLEAR(gen->classobj); + __Pyx_Coroutine_Undelegate(gen); + __Pyx_Coroutine_ExceptionClear(&gen->gi_exc_state); +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + Py_CLEAR(((__pyx_PyAsyncGenObject*)gen)->ag_finalizer); + } +#endif + Py_CLEAR(gen->gi_code); + Py_CLEAR(gen->gi_frame); + Py_CLEAR(gen->gi_name); + Py_CLEAR(gen->gi_qualname); + Py_CLEAR(gen->gi_modulename); + return 0; +} +static void __Pyx_Coroutine_dealloc(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject_GC_UnTrack(gen); + #if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + if (gen->gi_weakreflist != NULL) + #endif + PyObject_ClearWeakRefs(self); + if (gen->resume_label >= 0) { + PyObject_GC_Track(self); +#if CYTHON_USE_TP_FINALIZE + if (unlikely(PyObject_CallFinalizerFromDealloc(self))) +#else + { + destructor del = __Pyx_PyObject_GetSlot(gen, tp_del, destructor); + if (del) del(self); + } + if (unlikely(Py_REFCNT(self) > 0)) +#endif + { + return; + } + PyObject_GC_UnTrack(self); + } +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + /* We have to handle this case for asynchronous generators + right here, because this code has to be between UNTRACK + and GC_Del. */ + Py_CLEAR(((__pyx_PyAsyncGenObject*)self)->ag_finalizer); + } +#endif + __Pyx_Coroutine_clear(self); + __Pyx_PyHeapTypeObject_GC_Del(gen); +} +#if CYTHON_USE_TP_FINALIZE +static void __Pyx_Coroutine_del(PyObject *self) { + PyObject *error_type, *error_value, *error_traceback; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + __Pyx_PyThreadState_declare + if (gen->resume_label < 0) { + return; + } + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + __pyx_PyAsyncGenObject *agen = (__pyx_PyAsyncGenObject*)self; + PyObject *finalizer = agen->ag_finalizer; + if (finalizer && !agen->ag_closed) { + PyObject *res = __Pyx_PyObject_CallOneArg(finalizer, self); + if (unlikely(!res)) { + PyErr_WriteUnraisable(self); + } else { + Py_DECREF(res); + } + __Pyx_ErrRestore(error_type, error_value, error_traceback); + return; + } + } +#endif + if (unlikely(gen->resume_label == 0 && !error_value)) { +#ifdef __Pyx_Coroutine_USED +#ifdef __Pyx_Generator_USED + if (!__Pyx_Generator_CheckExact(self)) +#endif + { + PyObject_GC_UnTrack(self); + if (unlikely(PyErr_WarnFormat(PyExc_RuntimeWarning, 1, "coroutine '%.50S' was never awaited", gen->gi_qualname) < 0)) + PyErr_WriteUnraisable(self); + PyObject_GC_Track(self); + } +#endif + } else { + PyObject *retval = NULL; + __Pyx_PySendResult result = __Pyx_Coroutine_Close(self, &retval); + if (result == PYGEN_ERROR) { + PyErr_WriteUnraisable(self); + } else { + Py_XDECREF(retval); + } + } + __Pyx_ErrRestore(error_type, error_value, error_traceback); +} +#endif +static PyObject * +__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, void *context) +{ + PyObject *name = self->gi_name; + CYTHON_UNUSED_VAR(context); + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; +} +static int +__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_Py_XDECREF_SET(self->gi_name, value); + return 0; +} +static PyObject * +__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, void *context) +{ + PyObject *name = self->gi_qualname; + CYTHON_UNUSED_VAR(context); + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; +} +static int +__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_Py_XDECREF_SET(self->gi_qualname, value); + return 0; +} +static PyObject * +__Pyx__Coroutine_get_frame(__pyx_CoroutineObject *self) +{ +#if !CYTHON_COMPILING_IN_LIMITED_API + PyObject *frame; + #if PY_VERSION_HEX >= 0x030d0000 + Py_BEGIN_CRITICAL_SECTION(self); + #endif + frame = self->gi_frame; + if (!frame) { + if (unlikely(!self->gi_code)) { + Py_RETURN_NONE; + } + PyObject *globals = PyDict_New(); + if (unlikely(!globals)) return NULL; + frame = (PyObject *) PyFrame_New( + PyThreadState_Get(), /*PyThreadState *tstate,*/ + (PyCodeObject*) self->gi_code, /*PyCodeObject *code,*/ + globals, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + Py_DECREF(globals); + if (unlikely(!frame)) + return NULL; + if (unlikely(self->gi_frame)) { + Py_DECREF(frame); + frame = self->gi_frame; + } else { + self->gi_frame = frame; + } + } + Py_INCREF(frame); + #if PY_VERSION_HEX >= 0x030d0000 + Py_END_CRITICAL_SECTION(); + #endif + return frame; +#else + CYTHON_UNUSED_VAR(self); + Py_RETURN_NONE; +#endif +} +static PyObject * +__Pyx_Coroutine_get_frame(__pyx_CoroutineObject *self, void *context) { + CYTHON_UNUSED_VAR(context); + PyObject *frame = self->gi_frame; + if (frame) + return __Pyx_NewRef(frame); + return __Pyx__Coroutine_get_frame(self); +} +static __pyx_CoroutineObject *__Pyx__Coroutine_New( + PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name) { + __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type); + if (unlikely(!gen)) + return NULL; + return __Pyx__Coroutine_NewInit(gen, body, code, closure, name, qualname, module_name); +} +static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( + __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name) { + gen->body = body; + gen->closure = closure; + Py_XINCREF(closure); + gen->is_running = 0; + gen->resume_label = 0; + gen->classobj = NULL; + gen->yieldfrom = NULL; + gen->yieldfrom_am_send = NULL; + #if PY_VERSION_HEX >= 0x030B00a4 && !CYTHON_COMPILING_IN_LIMITED_API + gen->gi_exc_state.exc_value = NULL; + #else + gen->gi_exc_state.exc_type = NULL; + gen->gi_exc_state.exc_value = NULL; + gen->gi_exc_state.exc_traceback = NULL; + #endif +#if CYTHON_USE_EXC_INFO_STACK + gen->gi_exc_state.previous_item = NULL; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + gen->gi_weakreflist = NULL; +#endif + Py_XINCREF(qualname); + gen->gi_qualname = qualname; + Py_XINCREF(name); + gen->gi_name = name; + Py_XINCREF(module_name); + gen->gi_modulename = module_name; + Py_XINCREF(code); + gen->gi_code = code; + gen->gi_frame = NULL; + PyObject_GC_Track(gen); + return gen; +} +static char __Pyx_Coroutine_test_and_set_is_running(__pyx_CoroutineObject *gen) { + char result; + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_BEGIN_CRITICAL_SECTION(gen); + #endif + result = gen->is_running; + gen->is_running = 1; + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_END_CRITICAL_SECTION(); + #endif + return result; +} +static void __Pyx_Coroutine_unset_is_running(__pyx_CoroutineObject *gen) { + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_BEGIN_CRITICAL_SECTION(gen); + #endif + assert(gen->is_running); + gen->is_running = 0; + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_END_CRITICAL_SECTION(); + #endif +} +static char __Pyx_Coroutine_get_is_running(__pyx_CoroutineObject *gen) { + char result; + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_BEGIN_CRITICAL_SECTION(gen); + #endif + result = gen->is_running; + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_END_CRITICAL_SECTION(); + #endif + return result; +} +static PyObject *__Pyx_Coroutine_get_is_running_getter(PyObject *gen, void *closure) { + CYTHON_UNUSED_VAR(closure); + char result = __Pyx_Coroutine_get_is_running((__pyx_CoroutineObject*)gen); + if (result) Py_RETURN_TRUE; + else Py_RETURN_FALSE; +} +#if __PYX_HAS_PY_AM_SEND == 2 +static void __Pyx_SetBackportTypeAmSend(PyTypeObject *type, __Pyx_PyAsyncMethodsStruct *static_amsend_methods, __Pyx_pyiter_sendfunc am_send) { + Py_ssize_t ptr_offset = (char*)(type->tp_as_async) - (char*)type; + if (ptr_offset < 0 || ptr_offset > type->tp_basicsize) { + return; + } + memcpy((void*)static_amsend_methods, (void*)(type->tp_as_async), sizeof(*type->tp_as_async)); + static_amsend_methods->am_send = am_send; + type->tp_as_async = __Pyx_SlotTpAsAsync(static_amsend_methods); +} +#endif +static PyObject *__Pyx_Coroutine_fail_reduce_ex(PyObject *self, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); + __Pyx_TypeName self_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE((PyObject*)self)); + PyErr_Format(PyExc_TypeError, "cannot pickle '" __Pyx_FMT_TYPENAME "' object", + self_type_name); + __Pyx_DECREF_TypeName(self_type_name); + return NULL; +} + +/* Generator */ +static PyMethodDef __pyx_Generator_methods[] = { + {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O, + PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")}, + {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, + PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in generator,\nreturn next yielded value or raise StopIteration.")}, + {"close", (PyCFunction) __Pyx_Coroutine_Close_Method, METH_NOARGS, + PyDoc_STR("close() -> raise GeneratorExit inside generator.")}, + {"__reduce_ex__", (PyCFunction) __Pyx_Coroutine_fail_reduce_ex, METH_O, 0}, + {"__reduce__", (PyCFunction) __Pyx_Coroutine_fail_reduce_ex, METH_NOARGS, 0}, + {0, 0, 0, 0} +}; +static PyMemberDef __pyx_Generator_memberlist[] = { + {"gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, + PyDoc_STR("object being iterated by 'yield from', or None")}, + {"gi_code", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_code), READONLY, NULL}, + {"__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyGetSetDef __pyx_Generator_getsets[] = { + {"__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, + PyDoc_STR("name of the generator"), 0}, + {"__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, + PyDoc_STR("qualified name of the generator"), 0}, + {"gi_frame", (getter)__Pyx_Coroutine_get_frame, NULL, + PyDoc_STR("Frame of the generator"), 0}, + {"gi_running", __Pyx_Coroutine_get_is_running_getter, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_GeneratorType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_Generator_Next}, + {Py_tp_methods, (void *)__pyx_Generator_methods}, + {Py_tp_members, (void *)__pyx_Generator_memberlist}, + {Py_tp_getset, (void *)__pyx_Generator_getsets}, + {Py_tp_getattro, (void *) PyObject_GenericGetAttr}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + {Py_am_send, (void *)__Pyx_Coroutine_AmSend}, +#endif + {0, 0}, +}; +static PyType_Spec __pyx_GeneratorType_spec = { + __PYX_TYPE_MODULE_PREFIX "generator", + sizeof(__pyx_CoroutineObject), + 0, +#if PY_VERSION_HEX >= 0x030C0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_TPFLAGS_MANAGED_WEAKREF | +#endif + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | __Pyx_TPFLAGS_HAVE_AM_SEND, + __pyx_GeneratorType_slots +}; +#if __PYX_HAS_PY_AM_SEND == 2 +static __Pyx_PyAsyncMethodsStruct __pyx_Generator_as_async; +#endif +static int __pyx_Generator_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_GeneratorType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_GeneratorType_spec, NULL); + if (unlikely(!mstate->__pyx_GeneratorType)) { + return -1; + } +#if __PYX_HAS_PY_AM_SEND == 2 + __Pyx_SetBackportTypeAmSend(mstate->__pyx_GeneratorType, &__pyx_Generator_as_async, &__Pyx_Coroutine_AmSend); +#endif + return 0; +} +static PyObject *__Pyx_Generator_GetInlinedResult(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + PyObject *retval = NULL; + if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) { + return __Pyx_Coroutine_AlreadyRunningError(gen); + } + __Pyx_PySendResult result = __Pyx_Coroutine_SendEx(gen, Py_None, &retval, 0); + __Pyx_Coroutine_unset_is_running(gen); + (void) result; + assert (result == PYGEN_RETURN || result == PYGEN_ERROR); + assert ((result == PYGEN_RETURN && retval != NULL) || (result == PYGEN_ERROR && retval == NULL)); + return retval; +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* NewCodeObj */ +#if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } +#elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } +#elif !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#endif +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map +) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL && !CYTHON_COMPILING_IN_GRAAL) { + Py_ssize_t line_table_length = __Pyx_PyBytes_GET_SIZE(line_table); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(line_table_length == -1)) goto done; + #endif + Py_ssize_t code_len = (line_table_length * 2 + 4) & ~3LL; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table) ? line_table : __pyx_mstate_global->__pyx_empty_bytes + ); +done: + Py_XDECREF(code_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; +} + +/* DecompressString */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo) { + PyObject *module = NULL, *decompress, *compressed_bytes, *decompressed; + const char* module_name = algo == 3 ? "compression.zstd" : algo == 2 ? "bz2" : "zlib"; + PyObject *methodname = PyUnicode_FromString("decompress"); + if (unlikely(!methodname)) return NULL; + #if __PYX_LIMITED_VERSION_HEX >= 0x030e0000 + if (algo == 3) { + PyObject *fromlist = Py_BuildValue("[O]", methodname); + if (unlikely(!fromlist)) goto bad; + module = PyImport_ImportModuleLevel("compression.zstd", NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + } else + #endif + module = PyImport_ImportModule(module_name); + if (unlikely(!module)) goto import_failed; + decompress = PyObject_GetAttr(module, methodname); + if (unlikely(!decompress)) goto import_failed; + { + #ifdef __cplusplus + char *memview_bytes = const_cast<char*>(s); + #else + #if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wcast-qual" + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wcast-qual" + #endif + char *memview_bytes = (char*) s; + #if defined(__clang__) + #pragma clang diagnostic pop + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic pop + #endif + #endif + #if CYTHON_COMPILING_IN_LIMITED_API && !defined(PyBUF_READ) + int memview_flags = 0x100; + #else + int memview_flags = PyBUF_READ; + #endif + compressed_bytes = PyMemoryView_FromMemory(memview_bytes, length, memview_flags); + } + if (unlikely(!compressed_bytes)) { + Py_DECREF(decompress); + goto bad; + } + decompressed = PyObject_CallFunctionObjArgs(decompress, compressed_bytes, NULL); + Py_DECREF(compressed_bytes); + Py_DECREF(decompress); + Py_DECREF(module); + Py_DECREF(methodname); + return decompressed; +import_failed: + PyErr_Format(PyExc_ImportError, + "Failed to import '%.20s.decompress' - cannot initialise module strings. " + "String compression was configured with the C macro 'CYTHON_COMPRESS_STRINGS=%d'.", + module_name, algo); +bad: + Py_XDECREF(module); + Py_DECREF(methodname); + return NULL; +} + +#include <string.h> +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return __Pyx_NewRef(b ? Py_True: Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include <mutex> +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include <threads.h> +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include <pthread.h> +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include <Windows.h> // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; read<end; ++read) { + if (read->module) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; write<end; ++write) { + write->id = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; i<new_data->count; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_bound<end-1; ++lower_bound) { + lower_bound->id = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/iup.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/fontTools/varLib/iup.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..e9278cc03bce1c75c7ff5d528be317234d6d1915 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/iup.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8094a14da8fa1d8d28f40f8bbc27aaee7acb4fc9562089e3157e323bb9e579c8 +size 1685576 diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/iup.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/iup.py new file mode 100644 index 0000000000000000000000000000000000000000..9ff6c0eca39dc8770561a6a18fe44e75c0a486c7 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/iup.py @@ -0,0 +1,490 @@ +try: + import cython +except (AttributeError, ImportError): + # if cython not installed, use mock module with no-op decorators and types + from fontTools.misc import cython +COMPILED = cython.compiled + +from typing import ( + Sequence, + Tuple, + Union, +) +from numbers import Integral, Real + + +_Point = Tuple[Real, Real] +_Delta = Tuple[Real, Real] +_PointSegment = Sequence[_Point] +_DeltaSegment = Sequence[_Delta] +_DeltaOrNone = Union[_Delta, None] +_DeltaOrNoneSegment = Sequence[_DeltaOrNone] +_Endpoints = Sequence[Integral] + + +MAX_LOOKBACK = 8 + + +@cython.cfunc +@cython.locals( + j=cython.int, + n=cython.int, + x1=cython.double, + x2=cython.double, + d1=cython.double, + d2=cython.double, + scale=cython.double, + x=cython.double, + d=cython.double, +) +def iup_segment( + coords: _PointSegment, rc1: _Point, rd1: _Delta, rc2: _Point, rd2: _Delta +): # -> _DeltaSegment: + """Given two reference coordinates `rc1` & `rc2` and their respective + delta vectors `rd1` & `rd2`, returns interpolated deltas for the set of + coordinates `coords`.""" + + # rc1 = reference coord 1 + # rd1 = reference delta 1 + out_arrays = [None, None] + for j in 0, 1: + out_arrays[j] = out = [] + x1, x2, d1, d2 = rc1[j], rc2[j], rd1[j], rd2[j] + + if x1 == x2: + n = len(coords) + if d1 == d2: + out.extend([d1] * n) + else: + out.extend([0] * n) + continue + + if x1 > x2: + x1, x2 = x2, x1 + d1, d2 = d2, d1 + + # x1 < x2 + scale = (d2 - d1) / (x2 - x1) + for pair in coords: + x = pair[j] + + if x <= x1: + d = d1 + elif x >= x2: + d = d2 + else: + # Interpolate + # + # NOTE: we assign an explicit intermediate variable here in + # order to disable a fused mul-add optimization. See: + # + # - https://godbolt.org/z/YsP4T3TqK, + # - https://github.com/fonttools/fonttools/issues/3703 + nudge = (x - x1) * scale + d = d1 + nudge + + out.append(d) + + return zip(*out_arrays) + + +def iup_contour(deltas: _DeltaOrNoneSegment, coords: _PointSegment) -> _DeltaSegment: + """For the contour given in `coords`, interpolate any missing + delta values in delta vector `deltas`. + + Returns fully filled-out delta vector.""" + + assert len(deltas) == len(coords) + if None not in deltas: + return deltas + + n = len(deltas) + # indices of points with explicit deltas + indices = [i for i, v in enumerate(deltas) if v is not None] + if not indices: + # All deltas are None. Return 0,0 for all. + return [(0, 0)] * n + + out = [] + it = iter(indices) + start = next(it) + if start != 0: + # Initial segment that wraps around + i1, i2, ri1, ri2 = 0, start, start, indices[-1] + out.extend( + iup_segment( + coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] + ) + ) + out.append(deltas[start]) + for end in it: + if end - start > 1: + i1, i2, ri1, ri2 = start + 1, end, start, end + out.extend( + iup_segment( + coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] + ) + ) + out.append(deltas[end]) + start = end + if start != n - 1: + # Final segment that wraps around + i1, i2, ri1, ri2 = start + 1, n, start, indices[0] + out.extend( + iup_segment( + coords[i1:i2], coords[ri1], deltas[ri1], coords[ri2], deltas[ri2] + ) + ) + + assert len(deltas) == len(out), (len(deltas), len(out)) + return out + + +def iup_delta( + deltas: _DeltaOrNoneSegment, coords: _PointSegment, ends: _Endpoints +) -> _DeltaSegment: + """For the outline given in `coords`, with contour endpoints given + in sorted increasing order in `ends`, interpolate any missing + delta values in delta vector `deltas`. + + Returns fully filled-out delta vector.""" + + assert sorted(ends) == ends and len(coords) == (ends[-1] + 1 if ends else 0) + 4 + n = len(coords) + ends = ends + [n - 4, n - 3, n - 2, n - 1] + out = [] + start = 0 + for end in ends: + end += 1 + contour = iup_contour(deltas[start:end], coords[start:end]) + out.extend(contour) + start = end + + return out + + +# Optimizer + + +@cython.cfunc +@cython.inline +@cython.locals( + i=cython.int, + j=cython.int, + # tolerance=cython.double, # https://github.com/fonttools/fonttools/issues/3282 + x=cython.double, + y=cython.double, + p=cython.double, + q=cython.double, +) +@cython.returns(int) +def can_iup_in_between( + deltas: _DeltaSegment, + coords: _PointSegment, + i: Integral, + j: Integral, + tolerance: Real, +): # -> bool: + """Return true if the deltas for points at `i` and `j` (`i < j`) can be + successfully used to interpolate deltas for points in between them within + provided error tolerance.""" + + assert j - i >= 2 + interp = iup_segment(coords[i + 1 : j], coords[i], deltas[i], coords[j], deltas[j]) + deltas = deltas[i + 1 : j] + + return all( + abs(complex(x - p, y - q)) <= tolerance + for (x, y), (p, q) in zip(deltas, interp) + ) + + +@cython.locals( + cj=cython.double, + dj=cython.double, + lcj=cython.double, + ldj=cython.double, + ncj=cython.double, + ndj=cython.double, + force=cython.int, + forced=set, +) +def _iup_contour_bound_forced_set( + deltas: _DeltaSegment, coords: _PointSegment, tolerance: Real = 0 +) -> set: + """The forced set is a conservative set of points on the contour that must be encoded + explicitly (ie. cannot be interpolated). Calculating this set allows for significantly + speeding up the dynamic-programming, as well as resolve circularity in DP. + + The set is precise; that is, if an index is in the returned set, then there is no way + that IUP can generate delta for that point, given `coords` and `deltas`. + """ + assert len(deltas) == len(coords) + + n = len(deltas) + forced = set() + # Track "last" and "next" points on the contour as we sweep. + for i in range(len(deltas) - 1, -1, -1): + ld, lc = deltas[i - 1], coords[i - 1] + d, c = deltas[i], coords[i] + nd, nc = deltas[i - n + 1], coords[i - n + 1] + + for j in (0, 1): # For X and for Y + cj = c[j] + dj = d[j] + lcj = lc[j] + ldj = ld[j] + ncj = nc[j] + ndj = nd[j] + + if lcj <= ncj: + c1, c2 = lcj, ncj + d1, d2 = ldj, ndj + else: + c1, c2 = ncj, lcj + d1, d2 = ndj, ldj + + force = False + + # If the two coordinates are the same, then the interpolation + # algorithm produces the same delta if both deltas are equal, + # and zero if they differ. + # + # This test has to be before the next one. + if c1 == c2: + if abs(d1 - d2) > tolerance and abs(dj) > tolerance: + force = True + + # If coordinate for current point is between coordinate of adjacent + # points on the two sides, but the delta for current point is NOT + # between delta for those adjacent points (considering tolerance + # allowance), then there is no way that current point can be IUP-ed. + # Mark it forced. + elif c1 <= cj <= c2: # and c1 != c2 + if not (min(d1, d2) - tolerance <= dj <= max(d1, d2) + tolerance): + force = True + + # Otherwise, the delta should either match the closest, or have the + # same sign as the interpolation of the two deltas. + else: # cj < c1 or c2 < cj + if d1 != d2: + if cj < c1: + if ( + abs(dj) > tolerance + and abs(dj - d1) > tolerance + and ((dj - tolerance < d1) != (d1 < d2)) + ): + force = True + else: # c2 < cj + if ( + abs(dj) > tolerance + and abs(dj - d2) > tolerance + and ((d2 < dj + tolerance) != (d1 < d2)) + ): + force = True + + if force: + forced.add(i) + break + + return forced + + +@cython.locals( + i=cython.int, + j=cython.int, + best_cost=cython.double, + best_j=cython.int, + cost=cython.double, + forced=set, + tolerance=cython.double, +) +def _iup_contour_optimize_dp( + deltas: _DeltaSegment, + coords: _PointSegment, + forced=set(), + tolerance: Real = 0, + lookback: Integral = None, +): + """Straightforward Dynamic-Programming. For each index i, find least-costly encoding of + points 0 to i where i is explicitly encoded. We find this by considering all previous + explicit points j and check whether interpolation can fill points between j and i. + + Note that solution always encodes last point explicitly. Higher-level is responsible + for removing that restriction. + + As major speedup, we stop looking further whenever we see a "forced" point.""" + + n = len(deltas) + if lookback is None: + lookback = n + lookback = min(lookback, MAX_LOOKBACK) + costs = {-1: 0} + chain = {-1: None} + for i in range(0, n): + best_cost = costs[i - 1] + 1 + + costs[i] = best_cost + chain[i] = i - 1 + + if i - 1 in forced: + continue + + for j in range(i - 2, max(i - lookback, -2), -1): + cost = costs[j] + 1 + + if cost < best_cost and can_iup_in_between(deltas, coords, j, i, tolerance): + costs[i] = best_cost = cost + chain[i] = j + + if j in forced: + break + + return chain, costs + + +def _rot_list(l: list, k: int): + """Rotate list by k items forward. Ie. item at position 0 will be + at position k in returned list. Negative k is allowed.""" + n = len(l) + k %= n + if not k: + return l + return l[n - k :] + l[: n - k] + + +def _rot_set(s: set, k: int, n: int): + k %= n + if not k: + return s + return {(v + k) % n for v in s} + + +def iup_contour_optimize( + deltas: _DeltaSegment, coords: _PointSegment, tolerance: Real = 0.0 +) -> _DeltaOrNoneSegment: + """For contour with coordinates `coords`, optimize a set of delta + values `deltas` within error `tolerance`. + + Returns delta vector that has most number of None items instead of + the input delta. + """ + + n = len(deltas) + + # Get the easy cases out of the way: + + # If all are within tolerance distance of 0, encode nothing: + if all(abs(complex(*p)) <= tolerance for p in deltas): + return [None] * n + + # If there's exactly one point, return it: + if n == 1: + return deltas + + # If all deltas are exactly the same, return just one (the first one): + d0 = deltas[0] + if all(d0 == d for d in deltas): + return [d0] + [None] * (n - 1) + + # Else, solve the general problem using Dynamic Programming. + + forced = _iup_contour_bound_forced_set(deltas, coords, tolerance) + # The _iup_contour_optimize_dp() routine returns the optimal encoding + # solution given the constraint that the last point is always encoded. + # To remove this constraint, we use two different methods, depending on + # whether forced set is non-empty or not: + + # Debugging: Make the next if always take the second branch and observe + # if the font size changes (reduced); that would mean the forced-set + # has members it should not have. + if forced: + # Forced set is non-empty: rotate the contour start point + # such that the last point in the list is a forced point. + k = (n - 1) - max(forced) + assert k >= 0 + + deltas = _rot_list(deltas, k) + coords = _rot_list(coords, k) + forced = _rot_set(forced, k, n) + + # Debugging: Pass a set() instead of forced variable to the next call + # to exercise forced-set computation for under-counting. + chain, costs = _iup_contour_optimize_dp(deltas, coords, forced, tolerance) + + # Assemble solution. + solution = set() + i = n - 1 + while i is not None: + solution.add(i) + i = chain[i] + solution.remove(-1) + + # if not forced <= solution: + # print("coord", coords) + # print("deltas", deltas) + # print("len", len(deltas)) + assert forced <= solution, (forced, solution) + + deltas = [deltas[i] if i in solution else None for i in range(n)] + + deltas = _rot_list(deltas, -k) + else: + # Repeat the contour an extra time, solve the new case, then look for solutions of the + # circular n-length problem in the solution for new linear case. I cannot prove that + # this always produces the optimal solution... + chain, costs = _iup_contour_optimize_dp( + deltas + deltas, coords + coords, forced, tolerance, n + ) + best_sol, best_cost = None, n + 1 + + for start in range(n - 1, len(costs) - 1): + # Assemble solution. + solution = set() + i = start + while i > start - n: + solution.add(i % n) + i = chain[i] + if i == start - n: + cost = costs[start] - costs[start - n] + if cost <= best_cost: + best_sol, best_cost = solution, cost + + # if not forced <= best_sol: + # print("coord", coords) + # print("deltas", deltas) + # print("len", len(deltas)) + assert forced <= best_sol, (forced, best_sol) + + deltas = [deltas[i] if i in best_sol else None for i in range(n)] + + return deltas + + +def iup_delta_optimize( + deltas: _DeltaSegment, + coords: _PointSegment, + ends: _Endpoints, + tolerance: Real = 0.0, +) -> _DeltaOrNoneSegment: + """For the outline given in `coords`, with contour endpoints given + in sorted increasing order in `ends`, optimize a set of delta + values `deltas` within error `tolerance`. + + Returns delta vector that has most number of None items instead of + the input delta. + """ + assert sorted(ends) == ends and len(coords) == (ends[-1] + 1 if ends else 0) + 4 + n = len(coords) + ends = ends + [n - 4, n - 3, n - 2, n - 1] + out = [] + start = 0 + for end in ends: + contour = iup_contour_optimize( + deltas[start : end + 1], coords[start : end + 1], tolerance + ) + assert len(contour) == end - start + 1 + out.extend(contour) + start = end + 1 + + return out diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/merger.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/merger.py new file mode 100644 index 0000000000000000000000000000000000000000..61122f4c67aeb38254d05902d8704cb7ede05564 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/merger.py @@ -0,0 +1,1717 @@ +""" +Merge OpenType Layout tables (GDEF / GPOS / GSUB). +""" + +import os +import copy +import enum +from operator import ior +import logging +from fontTools.colorLib.builder import MAX_PAINT_COLR_LAYER_COUNT, LayerReuseCache +from fontTools.misc import classifyTools +from fontTools.misc.roundTools import otRound +from fontTools.misc.treeTools import build_n_ary_tree +from fontTools.ttLib.tables import otTables as ot +from fontTools.ttLib.tables import otBase as otBase +from fontTools.ttLib.tables.otConverters import BaseFixedValue +from fontTools.ttLib.tables.otTraverse import dfs_base_table +from fontTools.ttLib.tables.DefaultTable import DefaultTable +from fontTools.varLib import builder, models, varStore +from fontTools.varLib.models import nonNone, allNone, allEqual, allEqualTo, subList +from fontTools.varLib.varStore import VarStoreInstancer +from functools import reduce +from fontTools.otlLib.builder import buildSinglePos +from fontTools.otlLib.optimize.gpos import ( + _compression_level_from_env, + compact_pair_pos, +) + +log = logging.getLogger("fontTools.varLib.merger") + +from .errors import ( + ShouldBeConstant, + FoundANone, + MismatchedTypes, + NotANone, + LengthsDiffer, + KeysDiffer, + InconsistentGlyphOrder, + InconsistentExtensions, + InconsistentFormats, + UnsupportedFormat, + VarLibMergeError, +) + + +class Merger(object): + def __init__(self, font=None): + self.font = font + # mergeTables populates this from the parent's master ttfs + self.ttfs = None + + @classmethod + def merger(celf, clazzes, attrs=(None,)): + assert celf != Merger, "Subclass Merger instead." + if "mergers" not in celf.__dict__: + celf.mergers = {} + if type(clazzes) in (type, enum.EnumMeta): + clazzes = (clazzes,) + if type(attrs) == str: + attrs = (attrs,) + + def wrapper(method): + assert method.__name__ == "merge" + done = [] + for clazz in clazzes: + if clazz in done: + continue # Support multiple names of a clazz + done.append(clazz) + mergers = celf.mergers.setdefault(clazz, {}) + for attr in attrs: + assert attr not in mergers, ( + "Oops, class '%s' has merge function for '%s' defined already." + % (clazz.__name__, attr) + ) + mergers[attr] = method + return None + + return wrapper + + @classmethod + def mergersFor(celf, thing, _default={}): + typ = type(thing) + + for celf in celf.mro(): + mergers = getattr(celf, "mergers", None) + if mergers is None: + break + + m = celf.mergers.get(typ, None) + if m is not None: + return m + + return _default + + def mergeObjects(self, out, lst, exclude=()): + if hasattr(out, "ensureDecompiled"): + out.ensureDecompiled(recurse=False) + for item in lst: + if hasattr(item, "ensureDecompiled"): + item.ensureDecompiled(recurse=False) + keys = sorted(vars(out).keys()) + if not all(keys == sorted(vars(v).keys()) for v in lst): + raise KeysDiffer( + self, expected=keys, got=[sorted(vars(v).keys()) for v in lst] + ) + mergers = self.mergersFor(out) + defaultMerger = mergers.get("*", self.__class__.mergeThings) + try: + for key in keys: + if key in exclude: + continue + value = getattr(out, key) + values = [getattr(table, key) for table in lst] + mergerFunc = mergers.get(key, defaultMerger) + mergerFunc(self, value, values) + except VarLibMergeError as e: + e.stack.append("." + key) + raise + + def mergeLists(self, out, lst): + if not allEqualTo(out, lst, len): + raise LengthsDiffer(self, expected=len(out), got=[len(x) for x in lst]) + for i, (value, values) in enumerate(zip(out, zip(*lst))): + try: + self.mergeThings(value, values) + except VarLibMergeError as e: + e.stack.append("[%d]" % i) + raise + + def mergeThings(self, out, lst): + if not allEqualTo(out, lst, type): + raise MismatchedTypes( + self, expected=type(out).__name__, got=[type(x).__name__ for x in lst] + ) + mergerFunc = self.mergersFor(out).get(None, None) + if mergerFunc is not None: + mergerFunc(self, out, lst) + elif isinstance(out, enum.Enum): + # need to special-case Enums as have __dict__ but are not regular 'objects', + # otherwise mergeObjects/mergeThings get trapped in a RecursionError + if not allEqualTo(out, lst): + raise ShouldBeConstant(self, expected=out, got=lst) + elif hasattr(out, "__dict__"): + self.mergeObjects(out, lst) + elif isinstance(out, list): + self.mergeLists(out, lst) + else: + if not allEqualTo(out, lst): + raise ShouldBeConstant(self, expected=out, got=lst) + + def mergeTables(self, font, master_ttfs, tableTags): + for tag in tableTags: + if tag not in font: + continue + try: + self.ttfs = master_ttfs + self.mergeThings(font[tag], [m.get(tag) for m in master_ttfs]) + except VarLibMergeError as e: + e.stack.append(tag) + raise + + +# +# Aligning merger +# +class AligningMerger(Merger): + pass + + +@AligningMerger.merger(ot.GDEF, "GlyphClassDef") +def merge(merger, self, lst): + if self is None: + if not allNone(lst): + raise NotANone(merger, expected=None, got=lst) + return + + lst = [l.classDefs for l in lst] + self.classDefs = {} + # We only care about the .classDefs + self = self.classDefs + + allKeys = set() + allKeys.update(*[l.keys() for l in lst]) + for k in allKeys: + allValues = nonNone(l.get(k) for l in lst) + if not allEqual(allValues): + raise ShouldBeConstant( + merger, expected=allValues[0], got=lst, stack=["." + k] + ) + if not allValues: + self[k] = None + else: + self[k] = allValues[0] + + +def _SinglePosUpgradeToFormat2(self): + if self.Format == 2: + return self + + ret = ot.SinglePos() + ret.Format = 2 + ret.Coverage = self.Coverage + ret.ValueFormat = self.ValueFormat + ret.Value = [self.Value for _ in ret.Coverage.glyphs] + ret.ValueCount = len(ret.Value) + + return ret + + +def _merge_GlyphOrders(font, lst, values_lst=None, default=None): + """Takes font and list of glyph lists (must be sorted by glyph id), and returns + two things: + - Combined glyph list, + - If values_lst is None, return input glyph lists, but padded with None when a glyph + was missing in a list. Otherwise, return values_lst list-of-list, padded with None + to match combined glyph lists. + """ + if values_lst is None: + dict_sets = [set(l) for l in lst] + else: + dict_sets = [{g: v for g, v in zip(l, vs)} for l, vs in zip(lst, values_lst)] + combined = set() + combined.update(*dict_sets) + + sortKey = font.getReverseGlyphMap().__getitem__ + order = sorted(combined, key=sortKey) + # Make sure all input glyphsets were in proper order + if not all(sorted(vs, key=sortKey) == vs for vs in lst): + raise InconsistentGlyphOrder() + del combined + + paddedValues = None + if values_lst is None: + padded = [ + [glyph if glyph in dict_set else default for glyph in order] + for dict_set in dict_sets + ] + else: + assert len(lst) == len(values_lst) + padded = [ + [dict_set[glyph] if glyph in dict_set else default for glyph in order] + for dict_set in dict_sets + ] + return order, padded + + +@AligningMerger.merger(otBase.ValueRecord) +def merge(merger, self, lst): + # Code below sometimes calls us with self being + # a new object. Copy it from lst and recurse. + self.__dict__ = lst[0].__dict__.copy() + merger.mergeObjects(self, lst) + + +@AligningMerger.merger(ot.Anchor) +def merge(merger, self, lst): + # Code below sometimes calls us with self being + # a new object. Copy it from lst and recurse. + self.__dict__ = lst[0].__dict__.copy() + merger.mergeObjects(self, lst) + + +def _Lookup_SinglePos_get_effective_value(merger, subtables, glyph): + for self in subtables: + if ( + self is None + or type(self) != ot.SinglePos + or self.Coverage is None + or glyph not in self.Coverage.glyphs + ): + continue + if self.Format == 1: + return self.Value + elif self.Format == 2: + return self.Value[self.Coverage.glyphs.index(glyph)] + else: + raise UnsupportedFormat(merger, subtable="single positioning lookup") + return None + + +def _Lookup_PairPos_get_effective_value_pair( + merger, subtables, firstGlyph, secondGlyph +): + for self in subtables: + if ( + self is None + or type(self) != ot.PairPos + or self.Coverage is None + or firstGlyph not in self.Coverage.glyphs + ): + continue + if self.Format == 1: + ps = self.PairSet[self.Coverage.glyphs.index(firstGlyph)] + pvr = ps.PairValueRecord + for rec in pvr: # TODO Speed up + if rec.SecondGlyph == secondGlyph: + return rec + continue + elif self.Format == 2: + klass1 = self.ClassDef1.classDefs.get(firstGlyph, 0) + klass2 = self.ClassDef2.classDefs.get(secondGlyph, 0) + return self.Class1Record[klass1].Class2Record[klass2] + else: + raise UnsupportedFormat(merger, subtable="pair positioning lookup") + return None + + +@AligningMerger.merger(ot.SinglePos) +def merge(merger, self, lst): + self.ValueFormat = valueFormat = reduce(int.__or__, [l.ValueFormat for l in lst], 0) + if not (len(lst) == 1 or (valueFormat & ~0xF == 0)): + raise UnsupportedFormat(merger, subtable="single positioning lookup") + + # If all have same coverage table and all are format 1, + coverageGlyphs = self.Coverage.glyphs + if all(v.Format == 1 for v in lst) and all( + coverageGlyphs == v.Coverage.glyphs for v in lst + ): + self.Value = otBase.ValueRecord(valueFormat, self.Value) + if valueFormat != 0: + # If v.Value is None, it means a kerning of 0; we want + # it to participate in the model still. + # https://github.com/fonttools/fonttools/issues/3111 + merger.mergeThings( + self.Value, + [v.Value if v.Value is not None else otBase.ValueRecord() for v in lst], + ) + self.ValueFormat = self.Value.getFormat() + return + + # Upgrade everything to Format=2 + self.Format = 2 + lst = [_SinglePosUpgradeToFormat2(v) for v in lst] + + # Align them + glyphs, padded = _merge_GlyphOrders( + merger.font, [v.Coverage.glyphs for v in lst], [v.Value for v in lst] + ) + + self.Coverage.glyphs = glyphs + self.Value = [otBase.ValueRecord(valueFormat) for _ in glyphs] + self.ValueCount = len(self.Value) + + for i, values in enumerate(padded): + for j, glyph in enumerate(glyphs): + if values[j] is not None: + continue + # Fill in value from other subtables + # Note!!! This *might* result in behavior change if ValueFormat2-zeroedness + # is different between used subtable and current subtable! + # TODO(behdad) Check and warn if that happens? + v = _Lookup_SinglePos_get_effective_value( + merger, merger.lookup_subtables[i], glyph + ) + if v is None: + v = otBase.ValueRecord(valueFormat) + values[j] = v + + merger.mergeLists(self.Value, padded) + + # Merge everything else; though, there shouldn't be anything else. :) + merger.mergeObjects( + self, lst, exclude=("Format", "Coverage", "Value", "ValueCount", "ValueFormat") + ) + self.ValueFormat = reduce( + int.__or__, [v.getEffectiveFormat() for v in self.Value], 0 + ) + + +@AligningMerger.merger(ot.PairSet) +def merge(merger, self, lst): + # Align them + glyphs, padded = _merge_GlyphOrders( + merger.font, + [[v.SecondGlyph for v in vs.PairValueRecord] for vs in lst], + [vs.PairValueRecord for vs in lst], + ) + + self.PairValueRecord = pvrs = [] + for glyph in glyphs: + pvr = ot.PairValueRecord() + pvr.SecondGlyph = glyph + pvr.Value1 = ( + otBase.ValueRecord(merger.valueFormat1) if merger.valueFormat1 else None + ) + pvr.Value2 = ( + otBase.ValueRecord(merger.valueFormat2) if merger.valueFormat2 else None + ) + pvrs.append(pvr) + self.PairValueCount = len(self.PairValueRecord) + + for i, values in enumerate(padded): + for j, glyph in enumerate(glyphs): + # Fill in value from other subtables + v = ot.PairValueRecord() + v.SecondGlyph = glyph + if values[j] is not None: + vpair = values[j] + else: + vpair = _Lookup_PairPos_get_effective_value_pair( + merger, merger.lookup_subtables[i], self._firstGlyph, glyph + ) + if vpair is None: + v1, v2 = None, None + else: + v1 = getattr(vpair, "Value1", None) + v2 = getattr(vpair, "Value2", None) + v.Value1 = ( + otBase.ValueRecord(merger.valueFormat1, src=v1) + if merger.valueFormat1 + else None + ) + v.Value2 = ( + otBase.ValueRecord(merger.valueFormat2, src=v2) + if merger.valueFormat2 + else None + ) + values[j] = v + del self._firstGlyph + + merger.mergeLists(self.PairValueRecord, padded) + + +def _PairPosFormat1_merge(self, lst, merger): + assert allEqual( + [l.ValueFormat2 == 0 for l in lst if l.PairSet] + ), "Report bug against fonttools." + + # Merge everything else; makes sure Format is the same. + merger.mergeObjects( + self, + lst, + exclude=("Coverage", "PairSet", "PairSetCount", "ValueFormat1", "ValueFormat2"), + ) + + empty = ot.PairSet() + empty.PairValueRecord = [] + empty.PairValueCount = 0 + + # Align them + glyphs, padded = _merge_GlyphOrders( + merger.font, + [v.Coverage.glyphs for v in lst], + [v.PairSet for v in lst], + default=empty, + ) + + self.Coverage.glyphs = glyphs + self.PairSet = [ot.PairSet() for _ in glyphs] + self.PairSetCount = len(self.PairSet) + for glyph, ps in zip(glyphs, self.PairSet): + ps._firstGlyph = glyph + + merger.mergeLists(self.PairSet, padded) + + +def _ClassDef_invert(self, allGlyphs=None): + if isinstance(self, dict): + classDefs = self + else: + classDefs = self.classDefs if self and self.classDefs else {} + m = max(classDefs.values()) if classDefs else 0 + + ret = [] + for _ in range(m + 1): + ret.append(set()) + + for k, v in classDefs.items(): + ret[v].add(k) + + # Class-0 is special. It's "everything else". + if allGlyphs is None: + ret[0] = None + else: + # Limit all classes to glyphs in allGlyphs. + # Collect anything without a non-zero class into class=zero. + ret[0] = class0 = set(allGlyphs) + for s in ret[1:]: + s.intersection_update(class0) + class0.difference_update(s) + + return ret + + +def _ClassDef_merge_classify(lst, allGlyphses=None): + self = ot.ClassDef() + self.classDefs = classDefs = {} + allGlyphsesWasNone = allGlyphses is None + if allGlyphsesWasNone: + allGlyphses = [None] * len(lst) + + classifier = classifyTools.Classifier() + for classDef, allGlyphs in zip(lst, allGlyphses): + sets = _ClassDef_invert(classDef, allGlyphs) + if allGlyphs is None: + sets = sets[1:] + classifier.update(sets) + classes = classifier.getClasses() + + if allGlyphsesWasNone: + classes.insert(0, set()) + + for i, classSet in enumerate(classes): + if i == 0: + continue + for g in classSet: + classDefs[g] = i + + return self, classes + + +def _PairPosFormat2_align_matrices(self, lst, font, transparent=False): + matrices = [l.Class1Record for l in lst] + + # Align first classes + self.ClassDef1, classes = _ClassDef_merge_classify( + [l.ClassDef1 for l in lst], [l.Coverage.glyphs for l in lst] + ) + self.Class1Count = len(classes) + new_matrices = [] + for l, matrix in zip(lst, matrices): + nullRow = None + coverage = set(l.Coverage.glyphs) + classDef1 = l.ClassDef1.classDefs + class1Records = [] + for classSet in classes: + exemplarGlyph = next(iter(classSet)) + if exemplarGlyph not in coverage: + # Follow-up to e6125b353e1f54a0280ded5434b8e40d042de69f, + # Fixes https://github.com/googlei18n/fontmake/issues/470 + # Again, revert 8d441779e5afc664960d848f62c7acdbfc71d7b9 + # when merger becomes selfless. + nullRow = None + if nullRow is None: + nullRow = ot.Class1Record() + class2records = nullRow.Class2Record = [] + # TODO: When merger becomes selfless, revert e6125b353e1f54a0280ded5434b8e40d042de69f + for _ in range(l.Class2Count): + if transparent: + rec2 = None + else: + rec2 = ot.Class2Record() + rec2.Value1 = ( + otBase.ValueRecord(self.ValueFormat1) + if self.ValueFormat1 + else None + ) + rec2.Value2 = ( + otBase.ValueRecord(self.ValueFormat2) + if self.ValueFormat2 + else None + ) + class2records.append(rec2) + rec1 = nullRow + else: + klass = classDef1.get(exemplarGlyph, 0) + rec1 = matrix[klass] # TODO handle out-of-range? + class1Records.append(rec1) + new_matrices.append(class1Records) + matrices = new_matrices + del new_matrices + + # Align second classes + self.ClassDef2, classes = _ClassDef_merge_classify([l.ClassDef2 for l in lst]) + self.Class2Count = len(classes) + new_matrices = [] + for l, matrix in zip(lst, matrices): + classDef2 = l.ClassDef2.classDefs + class1Records = [] + for rec1old in matrix: + oldClass2Records = rec1old.Class2Record + rec1new = ot.Class1Record() + class2Records = rec1new.Class2Record = [] + for classSet in classes: + if not classSet: # class=0 + rec2 = oldClass2Records[0] + else: + exemplarGlyph = next(iter(classSet)) + klass = classDef2.get(exemplarGlyph, 0) + rec2 = oldClass2Records[klass] + class2Records.append(copy.deepcopy(rec2)) + class1Records.append(rec1new) + new_matrices.append(class1Records) + matrices = new_matrices + del new_matrices + + return matrices + + +def _PairPosFormat2_merge(self, lst, merger): + assert allEqual( + [l.ValueFormat2 == 0 for l in lst if l.Class1Record] + ), "Report bug against fonttools." + + merger.mergeObjects( + self, + lst, + exclude=( + "Coverage", + "ClassDef1", + "Class1Count", + "ClassDef2", + "Class2Count", + "Class1Record", + "ValueFormat1", + "ValueFormat2", + ), + ) + + # Align coverages + glyphs, _ = _merge_GlyphOrders(merger.font, [v.Coverage.glyphs for v in lst]) + self.Coverage.glyphs = glyphs + + # Currently, if the coverage of PairPosFormat2 subtables are different, + # we do NOT bother walking down the subtable list when filling in new + # rows for alignment. As such, this is only correct if current subtable + # is the last subtable in the lookup. Ensure that. + # + # Note that our canonicalization process merges trailing PairPosFormat2's, + # so in reality this is rare. + for l, subtables in zip(lst, merger.lookup_subtables): + if l.Coverage.glyphs != glyphs: + assert l == subtables[-1] + + matrices = _PairPosFormat2_align_matrices(self, lst, merger.font) + + self.Class1Record = list(matrices[0]) # TODO move merger to be selfless + merger.mergeLists(self.Class1Record, matrices) + + +@AligningMerger.merger(ot.PairPos) +def merge(merger, self, lst): + merger.valueFormat1 = self.ValueFormat1 = reduce( + int.__or__, [l.ValueFormat1 for l in lst], 0 + ) + merger.valueFormat2 = self.ValueFormat2 = reduce( + int.__or__, [l.ValueFormat2 for l in lst], 0 + ) + + if self.Format == 1: + _PairPosFormat1_merge(self, lst, merger) + elif self.Format == 2: + _PairPosFormat2_merge(self, lst, merger) + else: + raise UnsupportedFormat(merger, subtable="pair positioning lookup") + + del merger.valueFormat1, merger.valueFormat2 + + # Now examine the list of value records, and update to the union of format values, + # as merge might have created new values. + vf1 = 0 + vf2 = 0 + if self.Format == 1: + for pairSet in self.PairSet: + for pairValueRecord in pairSet.PairValueRecord: + pv1 = getattr(pairValueRecord, "Value1", None) + if pv1 is not None: + vf1 |= pv1.getFormat() + pv2 = getattr(pairValueRecord, "Value2", None) + if pv2 is not None: + vf2 |= pv2.getFormat() + elif self.Format == 2: + for class1Record in self.Class1Record: + for class2Record in class1Record.Class2Record: + pv1 = getattr(class2Record, "Value1", None) + if pv1 is not None: + vf1 |= pv1.getFormat() + pv2 = getattr(class2Record, "Value2", None) + if pv2 is not None: + vf2 |= pv2.getFormat() + self.ValueFormat1 = vf1 + self.ValueFormat2 = vf2 + + +def _MarkBasePosFormat1_merge(self, lst, merger, Mark="Mark", Base="Base"): + self.ClassCount = max(l.ClassCount for l in lst) + + MarkCoverageGlyphs, MarkRecords = _merge_GlyphOrders( + merger.font, + [getattr(l, Mark + "Coverage").glyphs for l in lst], + [getattr(l, Mark + "Array").MarkRecord for l in lst], + ) + getattr(self, Mark + "Coverage").glyphs = MarkCoverageGlyphs + + BaseCoverageGlyphs, BaseRecords = _merge_GlyphOrders( + merger.font, + [getattr(l, Base + "Coverage").glyphs for l in lst], + [getattr(getattr(l, Base + "Array"), Base + "Record") for l in lst], + ) + getattr(self, Base + "Coverage").glyphs = BaseCoverageGlyphs + + # MarkArray + records = [] + for g, glyphRecords in zip(MarkCoverageGlyphs, zip(*MarkRecords)): + allClasses = [r.Class for r in glyphRecords if r is not None] + + # TODO Right now we require that all marks have same class in + # all masters that cover them. This is not required. + # + # We can relax that by just requiring that all marks that have + # the same class in a master, have the same class in every other + # master. Indeed, if, say, a sparse master only covers one mark, + # that mark probably will get class 0, which would possibly be + # different from its class in other masters. + # + # We can even go further and reclassify marks to support any + # input. But, since, it's unlikely that two marks being both, + # say, "top" in one master, and one being "top" and other being + # "top-right" in another master, we shouldn't do that, as any + # failures in that case will probably signify mistakes in the + # input masters. + + if not allEqual(allClasses): + raise ShouldBeConstant(merger, expected=allClasses[0], got=allClasses) + else: + rec = ot.MarkRecord() + rec.Class = allClasses[0] + allAnchors = [None if r is None else r.MarkAnchor for r in glyphRecords] + if allNone(allAnchors): + anchor = None + else: + anchor = ot.Anchor() + anchor.Format = 1 + merger.mergeThings(anchor, allAnchors) + rec.MarkAnchor = anchor + records.append(rec) + array = ot.MarkArray() + array.MarkRecord = records + array.MarkCount = len(records) + setattr(self, Mark + "Array", array) + + # BaseArray + records = [] + for g, glyphRecords in zip(BaseCoverageGlyphs, zip(*BaseRecords)): + if allNone(glyphRecords): + rec = None + else: + rec = getattr(ot, Base + "Record")() + anchors = [] + setattr(rec, Base + "Anchor", anchors) + glyphAnchors = [ + [] if r is None else getattr(r, Base + "Anchor") for r in glyphRecords + ] + for l in glyphAnchors: + l.extend([None] * (self.ClassCount - len(l))) + for allAnchors in zip(*glyphAnchors): + if allNone(allAnchors): + anchor = None + else: + anchor = ot.Anchor() + anchor.Format = 1 + merger.mergeThings(anchor, allAnchors) + anchors.append(anchor) + records.append(rec) + array = getattr(ot, Base + "Array")() + setattr(array, Base + "Record", records) + setattr(array, Base + "Count", len(records)) + setattr(self, Base + "Array", array) + + +@AligningMerger.merger(ot.MarkBasePos) +def merge(merger, self, lst): + if not allEqualTo(self.Format, (l.Format for l in lst)): + raise InconsistentFormats( + merger, + subtable="mark-to-base positioning lookup", + expected=self.Format, + got=[l.Format for l in lst], + ) + if self.Format == 1: + _MarkBasePosFormat1_merge(self, lst, merger) + else: + raise UnsupportedFormat(merger, subtable="mark-to-base positioning lookup") + + +@AligningMerger.merger(ot.MarkMarkPos) +def merge(merger, self, lst): + if not allEqualTo(self.Format, (l.Format for l in lst)): + raise InconsistentFormats( + merger, + subtable="mark-to-mark positioning lookup", + expected=self.Format, + got=[l.Format for l in lst], + ) + if self.Format == 1: + _MarkBasePosFormat1_merge(self, lst, merger, "Mark1", "Mark2") + else: + raise UnsupportedFormat(merger, subtable="mark-to-mark positioning lookup") + + +def _PairSet_flatten(lst, font): + self = ot.PairSet() + self.Coverage = ot.Coverage() + + # Align them + glyphs, padded = _merge_GlyphOrders( + font, + [[v.SecondGlyph for v in vs.PairValueRecord] for vs in lst], + [vs.PairValueRecord for vs in lst], + ) + + self.Coverage.glyphs = glyphs + self.PairValueRecord = pvrs = [] + for values in zip(*padded): + for v in values: + if v is not None: + pvrs.append(v) + break + else: + assert False + self.PairValueCount = len(self.PairValueRecord) + + return self + + +def _Lookup_PairPosFormat1_subtables_flatten(lst, font): + assert allEqual( + [l.ValueFormat2 == 0 for l in lst if l.PairSet] + ), "Report bug against fonttools." + + self = ot.PairPos() + self.Format = 1 + self.Coverage = ot.Coverage() + self.ValueFormat1 = reduce(int.__or__, [l.ValueFormat1 for l in lst], 0) + self.ValueFormat2 = reduce(int.__or__, [l.ValueFormat2 for l in lst], 0) + + # Align them + glyphs, padded = _merge_GlyphOrders( + font, [v.Coverage.glyphs for v in lst], [v.PairSet for v in lst] + ) + + self.Coverage.glyphs = glyphs + self.PairSet = [ + _PairSet_flatten([v for v in values if v is not None], font) + for values in zip(*padded) + ] + self.PairSetCount = len(self.PairSet) + return self + + +def _Lookup_PairPosFormat2_subtables_flatten(lst, font): + assert allEqual( + [l.ValueFormat2 == 0 for l in lst if l.Class1Record] + ), "Report bug against fonttools." + + self = ot.PairPos() + self.Format = 2 + self.Coverage = ot.Coverage() + self.ValueFormat1 = reduce(int.__or__, [l.ValueFormat1 for l in lst], 0) + self.ValueFormat2 = reduce(int.__or__, [l.ValueFormat2 for l in lst], 0) + + # Align them + glyphs, _ = _merge_GlyphOrders(font, [v.Coverage.glyphs for v in lst]) + self.Coverage.glyphs = glyphs + + matrices = _PairPosFormat2_align_matrices(self, lst, font, transparent=True) + + matrix = self.Class1Record = [] + for rows in zip(*matrices): + row = ot.Class1Record() + matrix.append(row) + row.Class2Record = [] + row = row.Class2Record + for cols in zip(*list(r.Class2Record for r in rows)): + col = next(iter(c for c in cols if c is not None)) + row.append(col) + + return self + + +def _Lookup_PairPos_subtables_canonicalize(lst, font): + """Merge multiple Format1 subtables at the beginning of lst, + and merge multiple consecutive Format2 subtables that have the same + Class2 (ie. were split because of offset overflows). Returns new list.""" + lst = list(lst) + + l = len(lst) + i = 0 + while i < l and lst[i].Format == 1: + i += 1 + lst[:i] = [_Lookup_PairPosFormat1_subtables_flatten(lst[:i], font)] + + l = len(lst) + i = l + while i > 0 and lst[i - 1].Format == 2: + i -= 1 + lst[i:] = [_Lookup_PairPosFormat2_subtables_flatten(lst[i:], font)] + + return lst + + +def _Lookup_SinglePos_subtables_flatten(lst, font, min_inclusive_rec_format): + glyphs, _ = _merge_GlyphOrders(font, [v.Coverage.glyphs for v in lst], None) + num_glyphs = len(glyphs) + new = ot.SinglePos() + new.Format = 2 + new.ValueFormat = min_inclusive_rec_format + new.Coverage = ot.Coverage() + new.Coverage.glyphs = glyphs + new.ValueCount = num_glyphs + new.Value = [None] * num_glyphs + for singlePos in lst: + if singlePos.Format == 1: + val_rec = singlePos.Value + for gname in singlePos.Coverage.glyphs: + i = glyphs.index(gname) + new.Value[i] = copy.deepcopy(val_rec) + elif singlePos.Format == 2: + for j, gname in enumerate(singlePos.Coverage.glyphs): + val_rec = singlePos.Value[j] + i = glyphs.index(gname) + new.Value[i] = copy.deepcopy(val_rec) + return [new] + + +@AligningMerger.merger(ot.CursivePos) +def merge(merger, self, lst): + # Align them + glyphs, padded = _merge_GlyphOrders( + merger.font, + [l.Coverage.glyphs for l in lst], + [l.EntryExitRecord for l in lst], + ) + + self.Format = 1 + self.Coverage = ot.Coverage() + self.Coverage.glyphs = glyphs + self.EntryExitRecord = [] + for _ in glyphs: + rec = ot.EntryExitRecord() + rec.EntryAnchor = ot.Anchor() + rec.EntryAnchor.Format = 1 + rec.ExitAnchor = ot.Anchor() + rec.ExitAnchor.Format = 1 + self.EntryExitRecord.append(rec) + merger.mergeLists(self.EntryExitRecord, padded) + self.EntryExitCount = len(self.EntryExitRecord) + + +@AligningMerger.merger(ot.EntryExitRecord) +def merge(merger, self, lst): + if all(master.EntryAnchor is None for master in lst): + self.EntryAnchor = None + if all(master.ExitAnchor is None for master in lst): + self.ExitAnchor = None + merger.mergeObjects(self, lst) + + +@AligningMerger.merger(ot.Lookup) +def merge(merger, self, lst): + subtables = merger.lookup_subtables = [l.SubTable for l in lst] + + # Remove Extension subtables + for l, sts in list(zip(lst, subtables)) + [(self, self.SubTable)]: + if not sts: + continue + if sts[0].__class__.__name__.startswith("Extension"): + if not allEqual([st.__class__ for st in sts]): + raise InconsistentExtensions( + merger, + expected="Extension", + got=[st.__class__.__name__ for st in sts], + ) + if not allEqual([st.ExtensionLookupType for st in sts]): + raise InconsistentExtensions(merger) + l.LookupType = sts[0].ExtensionLookupType + new_sts = [st.ExtSubTable for st in sts] + del sts[:] + sts.extend(new_sts) + + isPairPos = self.SubTable and isinstance(self.SubTable[0], ot.PairPos) + + if isPairPos: + # AFDKO and feaLib sometimes generate two Format1 subtables instead of one. + # Merge those before continuing. + # https://github.com/fonttools/fonttools/issues/719 + self.SubTable = _Lookup_PairPos_subtables_canonicalize( + self.SubTable, merger.font + ) + subtables = merger.lookup_subtables = [ + _Lookup_PairPos_subtables_canonicalize(st, merger.font) for st in subtables + ] + else: + isSinglePos = self.SubTable and isinstance(self.SubTable[0], ot.SinglePos) + if isSinglePos: + numSubtables = [len(st) for st in subtables] + if not all([nums == numSubtables[0] for nums in numSubtables]): + # Flatten list of SinglePos subtables to single Format 2 subtable, + # with all value records set to the rec format type. + # We use buildSinglePos() to optimize the lookup after merging. + valueFormatList = [t.ValueFormat for st in subtables for t in st] + # Find the minimum value record that can accomodate all the singlePos subtables. + mirf = reduce(ior, valueFormatList) + self.SubTable = _Lookup_SinglePos_subtables_flatten( + self.SubTable, merger.font, mirf + ) + subtables = merger.lookup_subtables = [ + _Lookup_SinglePos_subtables_flatten(st, merger.font, mirf) + for st in subtables + ] + flattened = True + else: + flattened = False + + merger.mergeLists(self.SubTable, subtables) + self.SubTableCount = len(self.SubTable) + + if isPairPos: + # If format-1 subtable created during canonicalization is empty, remove it. + assert len(self.SubTable) >= 1 and self.SubTable[0].Format == 1 + if not self.SubTable[0].Coverage.glyphs: + self.SubTable.pop(0) + self.SubTableCount -= 1 + + # If format-2 subtable created during canonicalization is empty, remove it. + assert len(self.SubTable) >= 1 and self.SubTable[-1].Format == 2 + if not self.SubTable[-1].Coverage.glyphs: + self.SubTable.pop(-1) + self.SubTableCount -= 1 + + # Compact the merged subtables + # This is a good moment to do it because the compaction should create + # smaller subtables, which may prevent overflows from happening. + # Keep reading the value from the ENV until ufo2ft switches to the config system + level = merger.font.cfg.get( + "fontTools.otlLib.optimize.gpos:COMPRESSION_LEVEL", + default=_compression_level_from_env(), + ) + if level != 0: + log.info("Compacting GPOS...") + self.SubTable = compact_pair_pos(merger.font, level, self.SubTable) + self.SubTableCount = len(self.SubTable) + + elif isSinglePos and flattened: + singlePosTable = self.SubTable[0] + glyphs = singlePosTable.Coverage.glyphs + # We know that singlePosTable is Format 2, as this is set + # in _Lookup_SinglePos_subtables_flatten. + singlePosMapping = { + gname: valRecord for gname, valRecord in zip(glyphs, singlePosTable.Value) + } + self.SubTable = buildSinglePos( + singlePosMapping, merger.font.getReverseGlyphMap() + ) + merger.mergeObjects(self, lst, exclude=["SubTable", "SubTableCount"]) + + del merger.lookup_subtables + + +# +# InstancerMerger +# + + +class InstancerMerger(AligningMerger): + """A merger that takes multiple master fonts, and instantiates + an instance.""" + + def __init__(self, font, model, location): + Merger.__init__(self, font) + self.model = model + self.location = location + self.masterScalars = model.getMasterScalars(location) + + +@InstancerMerger.merger(ot.CaretValue) +def merge(merger, self, lst): + assert self.Format == 1 + Coords = [a.Coordinate for a in lst] + model = merger.model + masterScalars = merger.masterScalars + self.Coordinate = otRound( + model.interpolateFromValuesAndScalars(Coords, masterScalars) + ) + + +@InstancerMerger.merger(ot.Anchor) +def merge(merger, self, lst): + assert self.Format == 1 + XCoords = [a.XCoordinate for a in lst] + YCoords = [a.YCoordinate for a in lst] + model = merger.model + masterScalars = merger.masterScalars + self.XCoordinate = otRound( + model.interpolateFromValuesAndScalars(XCoords, masterScalars) + ) + self.YCoordinate = otRound( + model.interpolateFromValuesAndScalars(YCoords, masterScalars) + ) + + +@InstancerMerger.merger(otBase.ValueRecord) +def merge(merger, self, lst): + model = merger.model + masterScalars = merger.masterScalars + # TODO Handle differing valueformats + for name, tableName in [ + ("XAdvance", "XAdvDevice"), + ("YAdvance", "YAdvDevice"), + ("XPlacement", "XPlaDevice"), + ("YPlacement", "YPlaDevice"), + ]: + assert not hasattr(self, tableName) + + if hasattr(self, name): + values = [getattr(a, name, 0) for a in lst] + value = otRound( + model.interpolateFromValuesAndScalars(values, masterScalars) + ) + setattr(self, name, value) + + +# +# MutatorMerger +# + + +class MutatorMerger(AligningMerger): + """A merger that takes a variable font, and instantiates + an instance. While there's no "merging" to be done per se, + the operation can benefit from many operations that the + aligning merger does.""" + + def __init__(self, font, instancer, deleteVariations=True): + Merger.__init__(self, font) + self.instancer = instancer + self.deleteVariations = deleteVariations + + +@MutatorMerger.merger(ot.CaretValue) +def merge(merger, self, lst): + # Hack till we become selfless. + self.__dict__ = lst[0].__dict__.copy() + + if self.Format != 3: + return + + instancer = merger.instancer + dev = self.DeviceTable + if merger.deleteVariations: + del self.DeviceTable + if dev: + assert dev.DeltaFormat == 0x8000 + varidx = (dev.StartSize << 16) + dev.EndSize + delta = otRound(instancer[varidx]) + self.Coordinate += delta + + if merger.deleteVariations: + self.Format = 1 + + +@MutatorMerger.merger(ot.Anchor) +def merge(merger, self, lst): + # Hack till we become selfless. + self.__dict__ = lst[0].__dict__.copy() + + if self.Format != 3: + return + + instancer = merger.instancer + for v in "XY": + tableName = v + "DeviceTable" + if not hasattr(self, tableName): + continue + dev = getattr(self, tableName) + if merger.deleteVariations: + delattr(self, tableName) + if dev is None: + continue + + assert dev.DeltaFormat == 0x8000 + varidx = (dev.StartSize << 16) + dev.EndSize + delta = otRound(instancer[varidx]) + + attr = v + "Coordinate" + setattr(self, attr, getattr(self, attr) + delta) + + if merger.deleteVariations: + self.Format = 1 + + +@MutatorMerger.merger(otBase.ValueRecord) +def merge(merger, self, lst): + # Hack till we become selfless. + self.__dict__ = lst[0].__dict__.copy() + + instancer = merger.instancer + for name, tableName in [ + ("XAdvance", "XAdvDevice"), + ("YAdvance", "YAdvDevice"), + ("XPlacement", "XPlaDevice"), + ("YPlacement", "YPlaDevice"), + ]: + if not hasattr(self, tableName): + continue + dev = getattr(self, tableName) + if merger.deleteVariations: + delattr(self, tableName) + if dev is None: + continue + + assert dev.DeltaFormat == 0x8000 + varidx = (dev.StartSize << 16) + dev.EndSize + delta = otRound(instancer[varidx]) + + setattr(self, name, getattr(self, name, 0) + delta) + + +# +# VariationMerger +# + + +class VariationMerger(AligningMerger): + """A merger that takes multiple master fonts, and builds a + variable font.""" + + def __init__(self, model, axisTags, font): + Merger.__init__(self, font) + self.store_builder = varStore.OnlineVarStoreBuilder(axisTags) + self.setModel(model) + + def setModel(self, model): + self.model = model + self.store_builder.setModel(model) + + def mergeThings(self, out, lst): + masterModel = None + origTTFs = None + if None in lst: + if allNone(lst): + if out is not None: + raise FoundANone(self, got=lst) + return + + # temporarily subset the list of master ttfs to the ones for which + # master values are not None + origTTFs = self.ttfs + if self.ttfs: + self.ttfs = subList([v is not None for v in lst], self.ttfs) + + masterModel = self.model + model, lst = masterModel.getSubModel(lst) + self.setModel(model) + + super(VariationMerger, self).mergeThings(out, lst) + + if masterModel: + self.setModel(masterModel) + if origTTFs: + self.ttfs = origTTFs + + +def buildVarDevTable(store_builder, master_values): + if allEqual(master_values): + return master_values[0], None + base, varIdx = store_builder.storeMasters(master_values) + return base, builder.buildVarDevTable(varIdx) + + +@VariationMerger.merger(ot.BaseCoord) +def merge(merger, self, lst): + if self.Format != 1: + raise UnsupportedFormat(merger, subtable="a baseline coordinate") + self.Coordinate, DeviceTable = buildVarDevTable( + merger.store_builder, [a.Coordinate for a in lst] + ) + if DeviceTable: + self.Format = 3 + self.DeviceTable = DeviceTable + + +@VariationMerger.merger(ot.CaretValue) +def merge(merger, self, lst): + if self.Format != 1: + raise UnsupportedFormat(merger, subtable="a caret") + self.Coordinate, DeviceTable = buildVarDevTable( + merger.store_builder, [a.Coordinate for a in lst] + ) + if DeviceTable: + self.Format = 3 + self.DeviceTable = DeviceTable + + +@VariationMerger.merger(ot.Anchor) +def merge(merger, self, lst): + if self.Format != 1: + raise UnsupportedFormat(merger, subtable="an anchor") + self.XCoordinate, XDeviceTable = buildVarDevTable( + merger.store_builder, [a.XCoordinate for a in lst] + ) + self.YCoordinate, YDeviceTable = buildVarDevTable( + merger.store_builder, [a.YCoordinate for a in lst] + ) + if XDeviceTable or YDeviceTable: + self.Format = 3 + self.XDeviceTable = XDeviceTable + self.YDeviceTable = YDeviceTable + + +@VariationMerger.merger(otBase.ValueRecord) +def merge(merger, self, lst): + for name, tableName in [ + ("XAdvance", "XAdvDevice"), + ("YAdvance", "YAdvDevice"), + ("XPlacement", "XPlaDevice"), + ("YPlacement", "YPlaDevice"), + ]: + if hasattr(self, name): + value, deviceTable = buildVarDevTable( + merger.store_builder, [getattr(a, name, 0) for a in lst] + ) + setattr(self, name, value) + if deviceTable: + setattr(self, tableName, deviceTable) + + +class COLRVariationMerger(VariationMerger): + """A specialized VariationMerger that takes multiple master fonts containing + COLRv1 tables, and builds a variable COLR font. + + COLR tables are special in that variable subtables can be associated with + multiple delta-set indices (via VarIndexBase). + They also contain tables that must change their type (not simply the Format) + as they become variable (e.g. Affine2x3 -> VarAffine2x3) so this merger takes + care of that too. + """ + + def __init__(self, model, axisTags, font, allowLayerReuse=True): + VariationMerger.__init__(self, model, axisTags, font) + # maps {tuple(varIdxes): VarIndexBase} to facilitate reuse of VarIndexBase + # between variable tables with same varIdxes. + self.varIndexCache = {} + # flat list of all the varIdxes generated while merging + self.varIdxes = [] + # set of id()s of the subtables that contain variations after merging + # and need to be upgraded to the associated VarType. + self.varTableIds = set() + # we keep these around for rebuilding a LayerList while merging PaintColrLayers + self.layers = [] + self.layerReuseCache = None + if allowLayerReuse: + self.layerReuseCache = LayerReuseCache() + # flag to ensure BaseGlyphList is fully merged before LayerList gets processed + self._doneBaseGlyphs = False + + def mergeTables(self, font, master_ttfs, tableTags=("COLR",)): + if "COLR" in tableTags and "COLR" in font: + # The merger modifies the destination COLR table in-place. If this contains + # multiple PaintColrLayers referencing the same layers from LayerList, it's + # a problem because we may risk modifying the same paint more than once, or + # worse, fail while attempting to do that. + # We don't know whether the master COLR table was built with layer reuse + # disabled, thus to be safe we rebuild its LayerList so that it contains only + # unique layers referenced from non-overlapping PaintColrLayers throughout + # the base paint graphs. + self.expandPaintColrLayers(font["COLR"].table) + VariationMerger.mergeTables(self, font, master_ttfs, tableTags) + + def checkFormatEnum(self, out, lst, validate=lambda _: True): + fmt = out.Format + formatEnum = out.formatEnum + ok = False + try: + fmt = formatEnum(fmt) + except ValueError: + pass + else: + ok = validate(fmt) + if not ok: + raise UnsupportedFormat(self, subtable=type(out).__name__, value=fmt) + expected = fmt + got = [] + for v in lst: + fmt = getattr(v, "Format", None) + try: + fmt = formatEnum(fmt) + except ValueError: + pass + got.append(fmt) + if not allEqualTo(expected, got): + raise InconsistentFormats( + self, + subtable=type(out).__name__, + expected=expected, + got=got, + ) + return expected + + def mergeSparseDict(self, out, lst): + for k in out.keys(): + try: + self.mergeThings(out[k], [v.get(k) for v in lst]) + except VarLibMergeError as e: + e.stack.append(f"[{k!r}]") + raise + + def mergeAttrs(self, out, lst, attrs): + for attr in attrs: + value = getattr(out, attr) + values = [getattr(item, attr) for item in lst] + try: + self.mergeThings(value, values) + except VarLibMergeError as e: + e.stack.append(f".{attr}") + raise + + def storeMastersForAttr(self, out, lst, attr): + master_values = [getattr(item, attr) for item in lst] + + # VarStore treats deltas for fixed-size floats as integers, so we + # must convert master values to int before storing them in the builder + # then back to float. + is_fixed_size_float = False + conv = out.getConverterByName(attr) + if isinstance(conv, BaseFixedValue): + is_fixed_size_float = True + master_values = [conv.toInt(v) for v in master_values] + + baseValue = master_values[0] + varIdx = ot.NO_VARIATION_INDEX + if not allEqual(master_values): + baseValue, varIdx = self.store_builder.storeMasters(master_values) + + if is_fixed_size_float: + baseValue = conv.fromInt(baseValue) + + return baseValue, varIdx + + def storeVariationIndices(self, varIdxes) -> int: + # try to reuse an existing VarIndexBase for the same varIdxes, or else + # create a new one + key = tuple(varIdxes) + varIndexBase = self.varIndexCache.get(key) + + if varIndexBase is None: + # scan for a full match anywhere in the self.varIdxes + for i in range(len(self.varIdxes) - len(varIdxes) + 1): + if self.varIdxes[i : i + len(varIdxes)] == varIdxes: + self.varIndexCache[key] = varIndexBase = i + break + + if varIndexBase is None: + # try find a partial match at the end of the self.varIdxes + for n in range(len(varIdxes) - 1, 0, -1): + if self.varIdxes[-n:] == varIdxes[:n]: + varIndexBase = len(self.varIdxes) - n + self.varIndexCache[key] = varIndexBase + self.varIdxes.extend(varIdxes[n:]) + break + + if varIndexBase is None: + # no match found, append at the end + self.varIndexCache[key] = varIndexBase = len(self.varIdxes) + self.varIdxes.extend(varIdxes) + + return varIndexBase + + def mergeVariableAttrs(self, out, lst, attrs) -> int: + varIndexBase = ot.NO_VARIATION_INDEX + varIdxes = [] + for attr in attrs: + baseValue, varIdx = self.storeMastersForAttr(out, lst, attr) + setattr(out, attr, baseValue) + varIdxes.append(varIdx) + + if any(v != ot.NO_VARIATION_INDEX for v in varIdxes): + varIndexBase = self.storeVariationIndices(varIdxes) + + return varIndexBase + + @classmethod + def convertSubTablesToVarType(cls, table): + for path in dfs_base_table( + table, + skip_root=True, + predicate=lambda path: ( + getattr(type(path[-1].value), "VarType", None) is not None + ), + ): + st = path[-1] + subTable = st.value + varType = type(subTable).VarType + newSubTable = varType() + newSubTable.__dict__.update(subTable.__dict__) + newSubTable.populateDefaults() + parent = path[-2].value + if st.index is not None: + getattr(parent, st.name)[st.index] = newSubTable + else: + setattr(parent, st.name, newSubTable) + + @staticmethod + def expandPaintColrLayers(colr): + """Rebuild LayerList without PaintColrLayers reuse. + + Each base paint graph is fully DFS-traversed (with exception of PaintColrGlyph + which are irrelevant for this); any layers referenced via PaintColrLayers are + collected into a new LayerList and duplicated when reuse is detected, to ensure + that all paints are distinct objects at the end of the process. + PaintColrLayers's FirstLayerIndex/NumLayers are updated so that no overlap + is left. Also, any consecutively nested PaintColrLayers are flattened. + The COLR table's LayerList is replaced with the new unique layers. + A side effect is also that any layer from the old LayerList which is not + referenced by any PaintColrLayers is dropped. + """ + if not colr.LayerList: + # if no LayerList, there's nothing to expand + return + uniqueLayerIDs = set() + newLayerList = [] + for rec in colr.BaseGlyphList.BaseGlyphPaintRecord: + frontier = [rec.Paint] + while frontier: + paint = frontier.pop() + if paint.Format == ot.PaintFormat.PaintColrGlyph: + # don't traverse these, we treat them as constant for merging + continue + elif paint.Format == ot.PaintFormat.PaintColrLayers: + # de-treeify any nested PaintColrLayers, append unique copies to + # the new layer list and update PaintColrLayers index/count + children = list(_flatten_layers(paint, colr)) + first_layer_index = len(newLayerList) + for layer in children: + if id(layer) in uniqueLayerIDs: + layer = copy.deepcopy(layer) + assert id(layer) not in uniqueLayerIDs + newLayerList.append(layer) + uniqueLayerIDs.add(id(layer)) + paint.FirstLayerIndex = first_layer_index + paint.NumLayers = len(children) + else: + children = paint.getChildren(colr) + frontier.extend(reversed(children)) + # sanity check all the new layers are distinct objects + assert len(newLayerList) == len(uniqueLayerIDs) + colr.LayerList.Paint = newLayerList + colr.LayerList.LayerCount = len(newLayerList) + + +@COLRVariationMerger.merger(ot.BaseGlyphList) +def merge(merger, self, lst): + # ignore BaseGlyphCount, allow sparse glyph sets across masters + out = {rec.BaseGlyph: rec for rec in self.BaseGlyphPaintRecord} + masters = [{rec.BaseGlyph: rec for rec in m.BaseGlyphPaintRecord} for m in lst] + + for i, g in enumerate(out.keys()): + try: + # missing base glyphs don't participate in the merge + merger.mergeThings(out[g], [v.get(g) for v in masters]) + except VarLibMergeError as e: + e.stack.append(f".BaseGlyphPaintRecord[{i}]") + e.cause["location"] = f"base glyph {g!r}" + raise + + merger._doneBaseGlyphs = True + + +@COLRVariationMerger.merger(ot.LayerList) +def merge(merger, self, lst): + # nothing to merge for LayerList, assuming we have already merged all PaintColrLayers + # found while traversing the paint graphs rooted at BaseGlyphPaintRecords. + assert merger._doneBaseGlyphs, "BaseGlyphList must be merged before LayerList" + # Simply flush the final list of layers and go home. + self.LayerCount = len(merger.layers) + self.Paint = merger.layers + + +def _flatten_layers(root, colr): + assert root.Format == ot.PaintFormat.PaintColrLayers + for paint in root.getChildren(colr): + if paint.Format == ot.PaintFormat.PaintColrLayers: + yield from _flatten_layers(paint, colr) + else: + yield paint + + +def _merge_PaintColrLayers(self, out, lst): + # we only enforce that the (flat) number of layers is the same across all masters + # but we allow FirstLayerIndex to differ to acommodate for sparse glyph sets. + + out_layers = list(_flatten_layers(out, self.font["COLR"].table)) + + # sanity check ttfs are subset to current values (see VariationMerger.mergeThings) + # before matching each master PaintColrLayers to its respective COLR by position + assert len(self.ttfs) == len(lst) + master_layerses = [ + list(_flatten_layers(lst[i], self.ttfs[i]["COLR"].table)) + for i in range(len(lst)) + ] + + try: + self.mergeLists(out_layers, master_layerses) + except VarLibMergeError as e: + # NOTE: This attribute doesn't actually exist in PaintColrLayers but it's + # handy to have it in the stack trace for debugging. + e.stack.append(".Layers") + raise + + # following block is very similar to LayerListBuilder._beforeBuildPaintColrLayers + # but I couldn't find a nice way to share the code between the two... + + if self.layerReuseCache is not None: + # successful reuse can make the list smaller + out_layers = self.layerReuseCache.try_reuse(out_layers) + + # if the list is still too big we need to tree-fy it + is_tree = len(out_layers) > MAX_PAINT_COLR_LAYER_COUNT + out_layers = build_n_ary_tree(out_layers, n=MAX_PAINT_COLR_LAYER_COUNT) + + # We now have a tree of sequences with Paint leaves. + # Convert the sequences into PaintColrLayers. + def listToColrLayers(paint): + if isinstance(paint, list): + layers = [listToColrLayers(l) for l in paint] + paint = ot.Paint() + paint.Format = int(ot.PaintFormat.PaintColrLayers) + paint.NumLayers = len(layers) + paint.FirstLayerIndex = len(self.layers) + self.layers.extend(layers) + if self.layerReuseCache is not None: + self.layerReuseCache.add(layers, paint.FirstLayerIndex) + return paint + + out_layers = [listToColrLayers(l) for l in out_layers] + + if len(out_layers) == 1 and out_layers[0].Format == ot.PaintFormat.PaintColrLayers: + # special case when the reuse cache finds a single perfect PaintColrLayers match + # (it can only come from a successful reuse, _flatten_layers has gotten rid of + # all nested PaintColrLayers already); we assign it directly and avoid creating + # an extra table + out.NumLayers = out_layers[0].NumLayers + out.FirstLayerIndex = out_layers[0].FirstLayerIndex + else: + out.NumLayers = len(out_layers) + out.FirstLayerIndex = len(self.layers) + + self.layers.extend(out_layers) + + # Register our parts for reuse provided we aren't a tree + # If we are a tree the leaves registered for reuse and that will suffice + if self.layerReuseCache is not None and not is_tree: + self.layerReuseCache.add(out_layers, out.FirstLayerIndex) + + +@COLRVariationMerger.merger((ot.Paint, ot.ClipBox)) +def merge(merger, self, lst): + fmt = merger.checkFormatEnum(self, lst, lambda fmt: not fmt.is_variable()) + + if fmt is ot.PaintFormat.PaintColrLayers: + _merge_PaintColrLayers(merger, self, lst) + return + + varFormat = fmt.as_variable() + + varAttrs = () + if varFormat is not None: + varAttrs = otBase.getVariableAttrs(type(self), varFormat) + staticAttrs = (c.name for c in self.getConverters() if c.name not in varAttrs) + + merger.mergeAttrs(self, lst, staticAttrs) + + varIndexBase = merger.mergeVariableAttrs(self, lst, varAttrs) + + subTables = [st.value for st in self.iterSubTables()] + + # Convert table to variable if itself has variations or any subtables have + isVariable = varIndexBase != ot.NO_VARIATION_INDEX or any( + id(table) in merger.varTableIds for table in subTables + ) + + if isVariable: + if varAttrs: + # Some PaintVar* don't have any scalar attributes that can vary, + # only indirect offsets to other variable subtables, thus have + # no VarIndexBase of their own (e.g. PaintVarTransform) + self.VarIndexBase = varIndexBase + + if subTables: + # Convert Affine2x3 -> VarAffine2x3, ColorLine -> VarColorLine, etc. + merger.convertSubTablesToVarType(self) + + assert varFormat is not None + self.Format = int(varFormat) + + +@COLRVariationMerger.merger((ot.Affine2x3, ot.ColorStop)) +def merge(merger, self, lst): + varType = type(self).VarType + + varAttrs = otBase.getVariableAttrs(varType) + staticAttrs = (c.name for c in self.getConverters() if c.name not in varAttrs) + + merger.mergeAttrs(self, lst, staticAttrs) + + varIndexBase = merger.mergeVariableAttrs(self, lst, varAttrs) + + if varIndexBase != ot.NO_VARIATION_INDEX: + self.VarIndexBase = varIndexBase + # mark as having variations so the parent table will convert to Var{Type} + merger.varTableIds.add(id(self)) + + +@COLRVariationMerger.merger(ot.ColorLine) +def merge(merger, self, lst): + merger.mergeAttrs(self, lst, (c.name for c in self.getConverters())) + + if any(id(stop) in merger.varTableIds for stop in self.ColorStop): + merger.convertSubTablesToVarType(self) + merger.varTableIds.add(id(self)) + + +@COLRVariationMerger.merger(ot.ClipList, "clips") +def merge(merger, self, lst): + # 'sparse' in that we allow non-default masters to omit ClipBox entries + # for some/all glyphs (i.e. they don't participate) + merger.mergeSparseDict(self, lst) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/models.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/models.py new file mode 100644 index 0000000000000000000000000000000000000000..05f92c252e0d0352155495f6d1622eacbf0a4beb --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/models.py @@ -0,0 +1,663 @@ +"""Variation fonts interpolation models.""" + +from __future__ import annotations + +__all__ = [ + "normalizeValue", + "normalizeLocation", + "supportScalar", + "piecewiseLinearMap", + "VariationModel", +] + +from collections.abc import Mapping +from typing import TYPE_CHECKING +from fontTools.misc.roundTools import noRound +from .errors import VariationModelError + + +if TYPE_CHECKING: + from typing import Mapping, Sequence + + +def nonNone(lst): + return [l for l in lst if l is not None] + + +def allNone(lst): + return all(l is None for l in lst) + + +def allEqualTo(ref, lst, mapper=None): + if mapper is None: + return all(ref == item for item in lst) + + mapped = mapper(ref) + return all(mapped == mapper(item) for item in lst) + + +def allEqual(lst, mapper=None): + if not lst: + return True + it = iter(lst) + try: + first = next(it) + except StopIteration: + return True + return allEqualTo(first, it, mapper=mapper) + + +def subList(truth, lst): + assert len(truth) == len(lst) + return [l for l, t in zip(lst, truth) if t] + + +def normalizeValue( + v: float, triple: Sequence[float], extrapolate: bool = False +) -> float: + """Normalizes value based on a min/default/max triple. + + >>> normalizeValue(400, (100, 400, 900)) + 0.0 + >>> normalizeValue(100, (100, 400, 900)) + -1.0 + >>> normalizeValue(650, (100, 400, 900)) + 0.5 + """ + lower, default, upper = triple + if not (lower <= default <= upper): + raise ValueError( + f"Invalid axis values, must be minimum, default, maximum: " + f"{lower:3.3f}, {default:3.3f}, {upper:3.3f}" + ) + if not extrapolate: + v = max(min(v, upper), lower) + + if v == default or lower == upper: + return 0.0 + + if (v < default and lower != default) or (v > default and upper == default): + return (v - default) / (default - lower) + else: + assert (v > default and upper != default) or ( + v < default and lower == default + ), f"Ooops... v={v}, triple=({lower}, {default}, {upper})" + return (v - default) / (upper - default) + + +def normalizeLocation( + location: Mapping[str, float], + axes: Mapping[str, tuple[float, float, float]], + extrapolate: bool = False, + *, + validate: bool = False, +) -> dict[str, float]: + """Normalizes location based on axis min/default/max values from axes. + + >>> axes = {"wght": (100, 400, 900)} + >>> normalizeLocation({"wght": 400}, axes) + {'wght': 0.0} + >>> normalizeLocation({"wght": 100}, axes) + {'wght': -1.0} + >>> normalizeLocation({"wght": 900}, axes) + {'wght': 1.0} + >>> normalizeLocation({"wght": 650}, axes) + {'wght': 0.5} + >>> normalizeLocation({"wght": 1000}, axes) + {'wght': 1.0} + >>> normalizeLocation({"wght": 0}, axes) + {'wght': -1.0} + >>> axes = {"wght": (0, 0, 1000)} + >>> normalizeLocation({"wght": 0}, axes) + {'wght': 0.0} + >>> normalizeLocation({"wght": -1}, axes) + {'wght': 0.0} + >>> normalizeLocation({"wght": 1000}, axes) + {'wght': 1.0} + >>> normalizeLocation({"wght": 500}, axes) + {'wght': 0.5} + >>> normalizeLocation({"wght": 1001}, axes) + {'wght': 1.0} + >>> axes = {"wght": (0, 1000, 1000)} + >>> normalizeLocation({"wght": 0}, axes) + {'wght': -1.0} + >>> normalizeLocation({"wght": -1}, axes) + {'wght': -1.0} + >>> normalizeLocation({"wght": 500}, axes) + {'wght': -0.5} + >>> normalizeLocation({"wght": 1000}, axes) + {'wght': 0.0} + >>> normalizeLocation({"wght": 1001}, axes) + {'wght': 0.0} + """ + if validate: + assert set(location.keys()) <= set(axes.keys()), set(location.keys()) - set( + axes.keys() + ) + out = {} + for tag, triple in axes.items(): + v = location.get(tag, triple[1]) + out[tag] = normalizeValue(v, triple, extrapolate=extrapolate) + return out + + +def supportScalar(location, support, ot=True, extrapolate=False, axisRanges=None): + """Returns the scalar multiplier at location, for a master + with support. If ot is True, then a peak value of zero + for support of an axis means "axis does not participate". That + is how OpenType Variation Font technology works. + + If extrapolate is True, axisRanges must be a dict that maps axis + names to (axisMin, axisMax) tuples. + + >>> supportScalar({}, {}) + 1.0 + >>> supportScalar({'wght':.2}, {}) + 1.0 + >>> supportScalar({'wght':.2}, {'wght':(0,2,3)}) + 0.1 + >>> supportScalar({'wght':2.5}, {'wght':(0,2,4)}) + 0.75 + >>> supportScalar({'wght':2.5, 'wdth':0}, {'wght':(0,2,4), 'wdth':(-1,0,+1)}) + 0.75 + >>> supportScalar({'wght':2.5, 'wdth':.5}, {'wght':(0,2,4), 'wdth':(-1,0,+1)}, ot=False) + 0.375 + >>> supportScalar({'wght':2.5, 'wdth':0}, {'wght':(0,2,4), 'wdth':(-1,0,+1)}) + 0.75 + >>> supportScalar({'wght':2.5, 'wdth':.5}, {'wght':(0,2,4), 'wdth':(-1,0,+1)}) + 0.75 + >>> supportScalar({'wght':3}, {'wght':(0,1,2)}, extrapolate=True, axisRanges={'wght':(0, 2)}) + -1.0 + >>> supportScalar({'wght':-1}, {'wght':(0,1,2)}, extrapolate=True, axisRanges={'wght':(0, 2)}) + -1.0 + >>> supportScalar({'wght':3}, {'wght':(0,2,2)}, extrapolate=True, axisRanges={'wght':(0, 2)}) + 1.5 + >>> supportScalar({'wght':-1}, {'wght':(0,2,2)}, extrapolate=True, axisRanges={'wght':(0, 2)}) + -0.5 + """ + if extrapolate and axisRanges is None: + raise TypeError("axisRanges must be passed when extrapolate is True") + scalar = 1.0 + for axis, (lower, peak, upper) in support.items(): + if ot: + # OpenType-specific case handling + if peak == 0.0: + continue + if lower > peak or peak > upper: + continue + if lower < 0.0 and upper > 0.0: + continue + v = location.get(axis, 0.0) + else: + assert axis in location + v = location[axis] + if v == peak: + continue + + if extrapolate: + axisMin, axisMax = axisRanges[axis] + if v < axisMin and lower <= axisMin: + if peak <= axisMin and peak < upper: + scalar *= (v - upper) / (peak - upper) + continue + elif axisMin < peak: + scalar *= (v - lower) / (peak - lower) + continue + elif axisMax < v and axisMax <= upper: + if axisMax <= peak and lower < peak: + scalar *= (v - lower) / (peak - lower) + continue + elif peak < axisMax: + scalar *= (v - upper) / (peak - upper) + continue + + if v <= lower or upper <= v: + scalar = 0.0 + break + + if v < peak: + scalar *= (v - lower) / (peak - lower) + else: # v > peak + scalar *= (v - upper) / (peak - upper) + return scalar + + +class VariationModel(object): + """Locations must have the base master at the origin (ie. 0). + + If axis-ranges are not provided, values are assumed to be normalized to + the range [-1, 1]. + + If the extrapolate argument is set to True, then values are extrapolated + outside the axis range. + + >>> from pprint import pprint + >>> axisRanges = {'wght': (-180, +180), 'wdth': (-1, +1)} + >>> locations = [ \ + {'wght':100}, \ + {'wght':-100}, \ + {'wght':-180}, \ + {'wdth':+.3}, \ + {'wght':+120,'wdth':.3}, \ + {'wght':+120,'wdth':.2}, \ + {}, \ + {'wght':+180,'wdth':.3}, \ + {'wght':+180}, \ + ] + >>> model = VariationModel(locations, axisOrder=['wght'], axisRanges=axisRanges) + >>> pprint(model.locations) + [{}, + {'wght': -100}, + {'wght': -180}, + {'wght': 100}, + {'wght': 180}, + {'wdth': 0.3}, + {'wdth': 0.3, 'wght': 180}, + {'wdth': 0.3, 'wght': 120}, + {'wdth': 0.2, 'wght': 120}] + >>> pprint(model.deltaWeights) + [{}, + {0: 1.0}, + {0: 1.0}, + {0: 1.0}, + {0: 1.0}, + {0: 1.0}, + {0: 1.0, 4: 1.0, 5: 1.0}, + {0: 1.0, 3: 0.75, 4: 0.25, 5: 1.0, 6: 0.6666666666666666}, + {0: 1.0, + 3: 0.75, + 4: 0.25, + 5: 0.6666666666666667, + 6: 0.4444444444444445, + 7: 0.6666666666666667}] + """ + + def __init__( + self, locations, axisOrder=None, extrapolate=False, *, axisRanges=None + ): + locations = [{k: v for k, v in loc.items() if v != 0.0} for loc in locations] + + if len(set(tuple(sorted(l.items())) for l in locations)) != len(locations): + raise VariationModelError("Locations must be unique.") + + self.origLocations = locations + self.axisOrder = axisOrder if axisOrder is not None else [] + self.extrapolate = extrapolate + if axisRanges is None: + if extrapolate: + axisRanges = self.computeAxisRanges(locations) + else: + allAxes = {axis for loc in locations for axis in loc.keys()} + axisRanges = {axis: (-1, 1) for axis in allAxes} + self.axisRanges = axisRanges + keyFunc = self.getMasterLocationsSortKeyFunc( + locations, axisOrder=self.axisOrder + ) + self.locations = sorted(locations, key=keyFunc) + + # Mapping from user's master order to our master order + self.mapping = [self.locations.index(l) for l in locations] + self.reverseMapping = [locations.index(l) for l in self.locations] + + self._computeMasterSupports() + self._subModels = {} + + def getSubModel(self, items): + """Return a sub-model and the items that are not None. + + The sub-model is necessary for working with the subset + of items when some are None. + + The sub-model is cached.""" + if None not in items: + return self, items + key = tuple(v is not None for v in items) + subModel = self._subModels.get(key) + if subModel is None: + subModel = VariationModel( + subList(key, self.origLocations), + self.axisOrder, + extrapolate=self.extrapolate, + axisRanges=self.axisRanges, + ) + self._subModels[key] = subModel + return subModel, subList(key, items) + + @staticmethod + def computeAxisRanges(locations): + axisRanges = {} + allAxes = {axis for loc in locations for axis in loc.keys()} + for loc in locations: + for axis in allAxes: + value = loc.get(axis, 0) + axisMin, axisMax = axisRanges.get(axis, (value, value)) + axisRanges[axis] = min(value, axisMin), max(value, axisMax) + return axisRanges + + @staticmethod + def getMasterLocationsSortKeyFunc(locations, axisOrder=[]): + if {} not in locations: + raise VariationModelError("Base master not found.") + axisPoints = {} + for loc in locations: + if len(loc) != 1: + continue + axis = next(iter(loc)) + value = loc[axis] + if axis not in axisPoints: + axisPoints[axis] = {0.0} + assert ( + value not in axisPoints[axis] + ), 'Value "%s" in axisPoints["%s"] --> %s' % (value, axis, axisPoints) + axisPoints[axis].add(value) + + def getKey(axisPoints, axisOrder): + def sign(v): + return -1 if v < 0 else +1 if v > 0 else 0 + + def key(loc): + rank = len(loc) + onPointAxes = [ + axis + for axis, value in loc.items() + if axis in axisPoints and value in axisPoints[axis] + ] + orderedAxes = [axis for axis in axisOrder if axis in loc] + orderedAxes.extend( + [axis for axis in sorted(loc.keys()) if axis not in axisOrder] + ) + return ( + rank, # First, order by increasing rank + -len(onPointAxes), # Next, by decreasing number of onPoint axes + tuple( + axisOrder.index(axis) if axis in axisOrder else 0x10000 + for axis in orderedAxes + ), # Next, by known axes + tuple(orderedAxes), # Next, by all axes + tuple( + sign(loc[axis]) for axis in orderedAxes + ), # Next, by signs of axis values + tuple( + abs(loc[axis]) for axis in orderedAxes + ), # Next, by absolute value of axis values + ) + + return key + + ret = getKey(axisPoints, axisOrder) + return ret + + def reorderMasters(self, master_list, mapping): + # For changing the master data order without + # recomputing supports and deltaWeights. + new_list = [master_list[idx] for idx in mapping] + self.origLocations = [self.origLocations[idx] for idx in mapping] + locations = [ + {k: v for k, v in loc.items() if v != 0.0} for loc in self.origLocations + ] + self.mapping = [self.locations.index(l) for l in locations] + self.reverseMapping = [locations.index(l) for l in self.locations] + self._subModels = {} + return new_list + + def _computeMasterSupports(self): + self.supports = [] + regions = self._locationsToRegions() + for i, region in enumerate(regions): + locAxes = set(region.keys()) + # Walk over previous masters now + for prev_region in regions[:i]: + # Master with different axes do not participate + if set(prev_region.keys()) != locAxes: + continue + # If it's NOT in the current box, it does not participate + relevant = True + for axis, (lower, peak, upper) in region.items(): + if not ( + prev_region[axis][1] == peak + or lower < prev_region[axis][1] < upper + ): + relevant = False + break + if not relevant: + continue + + # Split the box for new master; split in whatever direction + # that has largest range ratio. + # + # For symmetry, we actually cut across multiple axes + # if they have the largest, equal, ratio. + # https://github.com/fonttools/fonttools/commit/7ee81c8821671157968b097f3e55309a1faa511e#commitcomment-31054804 + + bestAxes = {} + bestRatio = -1 + for axis in prev_region.keys(): + val = prev_region[axis][1] + assert axis in region + lower, locV, upper = region[axis] + newLower, newUpper = lower, upper + if val < locV: + newLower = val + ratio = (val - locV) / (lower - locV) + elif locV < val: + newUpper = val + ratio = (val - locV) / (upper - locV) + else: # val == locV + # Can't split box in this direction. + continue + if ratio > bestRatio: + bestAxes = {} + bestRatio = ratio + if ratio == bestRatio: + bestAxes[axis] = (newLower, locV, newUpper) + + for axis, triple in bestAxes.items(): + region[axis] = triple + self.supports.append(region) + self._computeDeltaWeights() + + def _locationsToRegions(self): + locations = self.locations + axisRanges = self.axisRanges + + regions = [] + for loc in locations: + region = {} + for axis, locV in loc.items(): + if locV > 0: + region[axis] = (0, locV, axisRanges[axis][1]) + else: + region[axis] = (axisRanges[axis][0], locV, 0) + regions.append(region) + return regions + + def _computeDeltaWeights(self): + self.deltaWeights = [] + for i, loc in enumerate(self.locations): + deltaWeight = {} + # Walk over previous masters now, populate deltaWeight + for j, support in enumerate(self.supports[:i]): + scalar = supportScalar(loc, support) + if scalar: + deltaWeight[j] = scalar + self.deltaWeights.append(deltaWeight) + + def getDeltas(self, masterValues, *, round=noRound): + assert len(masterValues) == len(self.deltaWeights), ( + len(masterValues), + len(self.deltaWeights), + ) + mapping = self.reverseMapping + out = [] + for i, weights in enumerate(self.deltaWeights): + delta = masterValues[mapping[i]] + for j, weight in weights.items(): + if weight == 1: + delta -= out[j] + else: + delta -= out[j] * weight + out.append(round(delta)) + return out + + def getDeltasAndSupports(self, items, *, round=noRound): + model, items = self.getSubModel(items) + return model.getDeltas(items, round=round), model.supports + + def getScalars(self, loc): + """Return scalars for each delta, for the given location. + If interpolating many master-values at the same location, + this function allows speed up by fetching the scalars once + and using them with interpolateFromMastersAndScalars().""" + return [ + supportScalar( + loc, support, extrapolate=self.extrapolate, axisRanges=self.axisRanges + ) + for support in self.supports + ] + + def getMasterScalars(self, targetLocation): + """Return multipliers for each master, for the given location. + If interpolating many master-values at the same location, + this function allows speed up by fetching the scalars once + and using them with interpolateFromValuesAndScalars(). + + Note that the scalars used in interpolateFromMastersAndScalars(), + are *not* the same as the ones returned here. They are the result + of getScalars().""" + out = self.getScalars(targetLocation) + for i, weights in reversed(list(enumerate(self.deltaWeights))): + for j, weight in weights.items(): + out[j] -= out[i] * weight + + out = [out[self.mapping[i]] for i in range(len(out))] + return out + + @staticmethod + def interpolateFromValuesAndScalars(values, scalars): + """Interpolate from values and scalars coefficients. + + If the values are master-values, then the scalars should be + fetched from getMasterScalars(). + + If the values are deltas, then the scalars should be fetched + from getScalars(); in which case this is the same as + interpolateFromDeltasAndScalars(). + """ + v = None + assert len(values) == len(scalars) + for value, scalar in zip(values, scalars): + if not scalar: + continue + contribution = value * scalar + if v is None: + v = contribution + else: + v += contribution + return v + + @staticmethod + def interpolateFromDeltasAndScalars(deltas, scalars): + """Interpolate from deltas and scalars fetched from getScalars().""" + return VariationModel.interpolateFromValuesAndScalars(deltas, scalars) + + def interpolateFromDeltas(self, loc, deltas): + """Interpolate from deltas, at location loc.""" + scalars = self.getScalars(loc) + return self.interpolateFromDeltasAndScalars(deltas, scalars) + + def interpolateFromMasters(self, loc, masterValues, *, round=noRound): + """Interpolate from master-values, at location loc.""" + scalars = self.getMasterScalars(loc) + return self.interpolateFromValuesAndScalars(masterValues, scalars) + + def interpolateFromMastersAndScalars(self, masterValues, scalars, *, round=noRound): + """Interpolate from master-values, and scalars fetched from + getScalars(), which is useful when you want to interpolate + multiple master-values with the same location.""" + deltas = self.getDeltas(masterValues, round=round) + return self.interpolateFromDeltasAndScalars(deltas, scalars) + + +def piecewiseLinearMap(v: float, mapping: Mapping[float, float]) -> float: + keys = mapping.keys() + if not keys: + return v + if v in keys: + return mapping[v] + k = min(keys) + if v < k: + return v + mapping[k] - k + k = max(keys) + if v > k: + return v + mapping[k] - k + # Interpolate + a = max(k for k in keys if k < v) + b = min(k for k in keys if k > v) + va = mapping[a] + vb = mapping[b] + return va + (vb - va) * (v - a) / (b - a) + + +def main(args=None): + """Normalize locations on a given designspace""" + from fontTools import configLogger + import argparse + + parser = argparse.ArgumentParser( + "fonttools varLib.models", + description=main.__doc__, + ) + parser.add_argument( + "--loglevel", + metavar="LEVEL", + default="INFO", + help="Logging level (defaults to INFO)", + ) + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("-d", "--designspace", metavar="DESIGNSPACE", type=str) + group.add_argument( + "-l", + "--locations", + metavar="LOCATION", + nargs="+", + help="Master locations as comma-separate coordinates. One must be all zeros.", + ) + + args = parser.parse_args(args) + + configLogger(level=args.loglevel) + from pprint import pprint + + if args.designspace: + from fontTools.designspaceLib import DesignSpaceDocument + + doc = DesignSpaceDocument() + doc.read(args.designspace) + locs = [s.location for s in doc.sources] + print("Original locations:") + pprint(locs) + doc.normalize() + print("Normalized locations:") + locs = [s.location for s in doc.sources] + pprint(locs) + else: + axes = [chr(c) for c in range(ord("A"), ord("Z") + 1)] + locs = [ + dict(zip(axes, (float(v) for v in s.split(",")))) for s in args.locations + ] + + model = VariationModel(locs) + print("Sorted locations:") + pprint(model.locations) + print("Supports:") + pprint(model.supports) + + +if __name__ == "__main__": + import doctest, sys + + if len(sys.argv) > 1: + sys.exit(main()) + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/multiVarStore.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/multiVarStore.py new file mode 100644 index 0000000000000000000000000000000000000000..2d074a353d9b215967a17276f57d29e4ff4041fb --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/multiVarStore.py @@ -0,0 +1,253 @@ +from fontTools.misc.roundTools import noRound, otRound +from fontTools.misc.intTools import bit_count +from fontTools.misc.vector import Vector +from fontTools.ttLib.tables import otTables as ot +from fontTools.varLib.models import supportScalar +import fontTools.varLib.varStore # For monkey-patching +from fontTools.varLib.builder import ( + buildVarRegionList, + buildSparseVarRegionList, + buildSparseVarRegion, + buildMultiVarStore, + buildMultiVarData, +) +from fontTools.misc.iterTools import batched +from functools import partial +from collections import defaultdict +from heapq import heappush, heappop + + +NO_VARIATION_INDEX = ot.NO_VARIATION_INDEX +ot.MultiVarStore.NO_VARIATION_INDEX = NO_VARIATION_INDEX + + +def _getLocationKey(loc): + return tuple(sorted(loc.items(), key=lambda kv: kv[0])) + + +class OnlineMultiVarStoreBuilder(object): + def __init__(self, axisTags): + self._axisTags = axisTags + self._regionMap = {} + self._regionList = buildSparseVarRegionList([], axisTags) + self._store = buildMultiVarStore(self._regionList, []) + self._data = None + self._model = None + self._supports = None + self._varDataIndices = {} + self._varDataCaches = {} + self._cache = None + + def setModel(self, model): + self.setSupports(model.supports) + self._model = model + + def setSupports(self, supports): + self._model = None + self._supports = list(supports) + if not self._supports[0]: + del self._supports[0] # Drop base master support + self._cache = None + self._data = None + + def finish(self): + self._regionList.RegionCount = len(self._regionList.Region) + self._store.MultiVarDataCount = len(self._store.MultiVarData) + return self._store + + def _add_MultiVarData(self): + regionMap = self._regionMap + regionList = self._regionList + + regions = self._supports + regionIndices = [] + for region in regions: + key = _getLocationKey(region) + idx = regionMap.get(key) + if idx is None: + varRegion = buildSparseVarRegion(region, self._axisTags) + idx = regionMap[key] = len(regionList.Region) + regionList.Region.append(varRegion) + regionIndices.append(idx) + + # Check if we have one already... + key = tuple(regionIndices) + varDataIdx = self._varDataIndices.get(key) + if varDataIdx is not None: + self._outer = varDataIdx + self._data = self._store.MultiVarData[varDataIdx] + self._cache = self._varDataCaches[key] + if len(self._data.Item) == 0xFFFF: + # This is full. Need new one. + varDataIdx = None + + if varDataIdx is None: + self._data = buildMultiVarData(regionIndices, []) + self._outer = len(self._store.MultiVarData) + self._store.MultiVarData.append(self._data) + self._varDataIndices[key] = self._outer + if key not in self._varDataCaches: + self._varDataCaches[key] = {} + self._cache = self._varDataCaches[key] + + def storeMasters(self, master_values, *, round=round): + deltas = self._model.getDeltas(master_values, round=round) + base = deltas.pop(0) + return base, self.storeDeltas(deltas, round=noRound) + + def storeDeltas(self, deltas, *, round=round): + deltas = tuple(round(d) for d in deltas) + + if not any(deltas): + return NO_VARIATION_INDEX + + deltas_tuple = tuple(tuple(d) for d in deltas) + + if not self._data: + self._add_MultiVarData() + + varIdx = self._cache.get(deltas_tuple) + if varIdx is not None: + return varIdx + + inner = len(self._data.Item) + if inner == 0xFFFF: + # Full array. Start new one. + self._add_MultiVarData() + return self.storeDeltas(deltas, round=noRound) + self._data.addItem(deltas, round=noRound) + + varIdx = (self._outer << 16) + inner + self._cache[deltas_tuple] = varIdx + return varIdx + + +def MultiVarData_addItem(self, deltas, *, round=round): + deltas = tuple(round(d) for d in deltas) + + assert len(deltas) == self.VarRegionCount + + values = [] + for d in deltas: + values.extend(d) + + self.Item.append(values) + self.ItemCount = len(self.Item) + + +ot.MultiVarData.addItem = MultiVarData_addItem + + +def SparseVarRegion_get_support(self, fvar_axes): + return { + fvar_axes[reg.AxisIndex].axisTag: (reg.StartCoord, reg.PeakCoord, reg.EndCoord) + for reg in self.SparseVarRegionAxis + } + + +ot.SparseVarRegion.get_support = SparseVarRegion_get_support + + +def MultiVarStore___bool__(self): + return bool(self.MultiVarData) + + +ot.MultiVarStore.__bool__ = MultiVarStore___bool__ + + +class MultiVarStoreInstancer(object): + def __init__(self, multivarstore, fvar_axes, location={}): + self.fvar_axes = fvar_axes + assert multivarstore is None or multivarstore.Format == 1 + self._varData = multivarstore.MultiVarData if multivarstore else [] + self._regions = ( + multivarstore.SparseVarRegionList.Region if multivarstore else [] + ) + self.setLocation(location) + + def setLocation(self, location): + self.location = dict(location) + self._clearCaches() + + def _clearCaches(self): + self._scalars = {} + + def _getScalar(self, regionIdx): + scalar = self._scalars.get(regionIdx) + if scalar is None: + support = self._regions[regionIdx].get_support(self.fvar_axes) + scalar = supportScalar(self.location, support) + self._scalars[regionIdx] = scalar + return scalar + + @staticmethod + def interpolateFromDeltasAndScalars(deltas, scalars): + if not deltas: + return Vector([]) + assert len(deltas) % len(scalars) == 0, (len(deltas), len(scalars)) + m = len(deltas) // len(scalars) + delta = Vector([0] * m) + for d, s in zip(batched(deltas, m), scalars): + if not s: + continue + delta += Vector(d) * s + return delta + + def __getitem__(self, varidx): + major, minor = varidx >> 16, varidx & 0xFFFF + if varidx == NO_VARIATION_INDEX: + return Vector([]) + varData = self._varData + scalars = [self._getScalar(ri) for ri in varData[major].VarRegionIndex] + deltas = varData[major].Item[minor] + return self.interpolateFromDeltasAndScalars(deltas, scalars) + + def interpolateFromDeltas(self, varDataIndex, deltas): + varData = self._varData + scalars = [self._getScalar(ri) for ri in varData[varDataIndex].VarRegionIndex] + return self.interpolateFromDeltasAndScalars(deltas, scalars) + + +def MultiVarStore_subset_varidxes(self, varIdxes): + return ot.VarStore.subset_varidxes(self, varIdxes, VarData="MultiVarData") + + +def MultiVarStore_prune_regions(self): + return ot.VarStore.prune_regions( + self, VarData="MultiVarData", VarRegionList="SparseVarRegionList" + ) + + +ot.MultiVarStore.prune_regions = MultiVarStore_prune_regions +ot.MultiVarStore.subset_varidxes = MultiVarStore_subset_varidxes + + +def MultiVarStore_get_supports(self, major, fvarAxes): + supports = [] + varData = self.MultiVarData[major] + for regionIdx in varData.VarRegionIndex: + region = self.SparseVarRegionList.Region[regionIdx] + support = region.get_support(fvarAxes) + supports.append(support) + return supports + + +ot.MultiVarStore.get_supports = MultiVarStore_get_supports + + +def VARC_collect_varidxes(self, varidxes): + for glyph in self.VarCompositeGlyphs.VarCompositeGlyph: + for component in glyph.components: + varidxes.add(component.axisValuesVarIndex) + varidxes.add(component.transformVarIndex) + + +def VARC_remap_varidxes(self, varidxes_map): + for glyph in self.VarCompositeGlyphs.VarCompositeGlyph: + for component in glyph.components: + component.axisValuesVarIndex = varidxes_map[component.axisValuesVarIndex] + component.transformVarIndex = varidxes_map[component.transformVarIndex] + + +ot.VARC.collect_varidxes = VARC_collect_varidxes +ot.VARC.remap_varidxes = VARC_remap_varidxes diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/mutator.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..cfa7607c4f2201fe3c9069dbe06ee7b229826700 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/mutator.py @@ -0,0 +1,529 @@ +""" +Instantiate a variation font. Run, eg: + +.. code-block:: sh + + $ fonttools varLib.mutator ./NotoSansArabic-VF.ttf wght=140 wdth=85 + +.. warning:: + ``fontTools.varLib.mutator`` is deprecated in favor of :mod:`fontTools.varLib.instancer` + which provides equivalent full instancing and also supports partial instancing. + Please migrate CLI usage to ``fonttools varLib.instancer`` and API usage to + :func:`fontTools.varLib.instancer.instantiateVariableFont`. +""" + +from fontTools.misc.fixedTools import floatToFixedToFloat, floatToFixed +from fontTools.misc.loggingTools import deprecateFunction +from fontTools.misc.roundTools import otRound +from fontTools.pens.boundsPen import BoundsPen +from fontTools.ttLib import TTFont, newTable +from fontTools.ttLib.tables import ttProgram +from fontTools.ttLib.tables._g_l_y_f import ( + GlyphCoordinates, + flagOverlapSimple, + OVERLAP_COMPOUND, +) +from fontTools.varLib.models import ( + supportScalar, + normalizeLocation, + piecewiseLinearMap, +) +from fontTools.varLib.merger import MutatorMerger +from fontTools.varLib.varStore import VarStoreInstancer +from fontTools.varLib.mvar import MVAR_ENTRIES +from fontTools.varLib.iup import iup_delta +import fontTools.subset.cff +import os.path +import logging +from io import BytesIO + + +log = logging.getLogger("fontTools.varlib.mutator") + +# map 'wdth' axis (1..200) to OS/2.usWidthClass (1..9), rounding to closest +OS2_WIDTH_CLASS_VALUES = {} +percents = [50.0, 62.5, 75.0, 87.5, 100.0, 112.5, 125.0, 150.0, 200.0] +for i, (prev, curr) in enumerate(zip(percents[:-1], percents[1:]), start=1): + half = (prev + curr) / 2 + OS2_WIDTH_CLASS_VALUES[half] = i + + +def interpolate_cff2_PrivateDict(topDict, interpolateFromDeltas): + pd_blend_lists = ( + "BlueValues", + "OtherBlues", + "FamilyBlues", + "FamilyOtherBlues", + "StemSnapH", + "StemSnapV", + ) + pd_blend_values = ("BlueScale", "BlueShift", "BlueFuzz", "StdHW", "StdVW") + for fontDict in topDict.FDArray: + pd = fontDict.Private + vsindex = pd.vsindex if (hasattr(pd, "vsindex")) else 0 + for key, value in pd.rawDict.items(): + if (key in pd_blend_values) and isinstance(value, list): + delta = interpolateFromDeltas(vsindex, value[1:]) + pd.rawDict[key] = otRound(value[0] + delta) + elif (key in pd_blend_lists) and isinstance(value[0], list): + """If any argument in a BlueValues list is a blend list, + then they all are. The first value of each list is an + absolute value. The delta tuples are calculated from + relative master values, hence we need to append all the + deltas to date to each successive absolute value.""" + delta = 0 + for i, val_list in enumerate(value): + delta += otRound(interpolateFromDeltas(vsindex, val_list[1:])) + value[i] = val_list[0] + delta + + +def interpolate_cff2_charstrings(topDict, interpolateFromDeltas, glyphOrder): + charstrings = topDict.CharStrings + for gname in glyphOrder: + # Interpolate charstring + # e.g replace blend op args with regular args, + # and use and discard vsindex op. + charstring = charstrings[gname] + new_program = [] + vsindex = 0 + last_i = 0 + for i, token in enumerate(charstring.program): + if token == "vsindex": + vsindex = charstring.program[i - 1] + if last_i != 0: + new_program.extend(charstring.program[last_i : i - 1]) + last_i = i + 1 + elif token == "blend": + num_regions = charstring.getNumRegions(vsindex) + numMasters = 1 + num_regions + num_args = charstring.program[i - 1] + # The program list starting at program[i] is now: + # ..args for following operations + # num_args values from the default font + # num_args tuples, each with numMasters-1 delta values + # num_blend_args + # 'blend' + argi = i - (num_args * numMasters + 1) + end_args = tuplei = argi + num_args + while argi < end_args: + next_ti = tuplei + num_regions + deltas = charstring.program[tuplei:next_ti] + delta = interpolateFromDeltas(vsindex, deltas) + charstring.program[argi] += otRound(delta) + tuplei = next_ti + argi += 1 + new_program.extend(charstring.program[last_i:end_args]) + last_i = i + 1 + if last_i != 0: + new_program.extend(charstring.program[last_i:]) + charstring.program = new_program + + +def interpolate_cff2_metrics(varfont, topDict, glyphOrder, loc): + """Unlike TrueType glyphs, neither advance width nor bounding box + info is stored in a CFF2 charstring. The width data exists only in + the hmtx and HVAR tables. Since LSB data cannot be interpolated + reliably from the master LSB values in the hmtx table, we traverse + the charstring to determine the actual bound box.""" + + charstrings = topDict.CharStrings + boundsPen = BoundsPen(glyphOrder) + hmtx = varfont["hmtx"] + hvar_table = None + if "HVAR" in varfont: + hvar_table = varfont["HVAR"].table + fvar = varfont["fvar"] + varStoreInstancer = VarStoreInstancer(hvar_table.VarStore, fvar.axes, loc) + + for gid, gname in enumerate(glyphOrder): + entry = list(hmtx[gname]) + # get width delta. + if hvar_table: + if hvar_table.AdvWidthMap: + width_idx = hvar_table.AdvWidthMap.mapping[gname] + else: + width_idx = gid + width_delta = otRound(varStoreInstancer[width_idx]) + else: + width_delta = 0 + + # get LSB. + boundsPen.init() + charstring = charstrings[gname] + charstring.draw(boundsPen) + if boundsPen.bounds is None: + # Happens with non-marking glyphs + lsb_delta = 0 + else: + lsb = otRound(boundsPen.bounds[0]) + lsb_delta = entry[1] - lsb + + if lsb_delta or width_delta: + if width_delta: + entry[0] = max(0, entry[0] + width_delta) + if lsb_delta: + entry[1] = lsb + hmtx[gname] = tuple(entry) + + +@deprecateFunction( + "use fontTools.varLib.instancer.instantiateVariableFont instead " + "for either full or partial instancing", +) +def instantiateVariableFont(varfont, location, inplace=False, overlap=True): + """Generate a static instance from a variable TTFont and a dictionary + defining the desired location along the variable font's axes. + The location values must be specified as user-space coordinates, e.g.: + + .. code-block:: + + {'wght': 400, 'wdth': 100} + + By default, a new TTFont object is returned. If ``inplace`` is True, the + input varfont is modified and reduced to a static font. + + When the overlap parameter is defined as True, + OVERLAP_SIMPLE and OVERLAP_COMPOUND bits are set to 1. See + https://docs.microsoft.com/en-us/typography/opentype/spec/glyf + """ + if not inplace: + # make a copy to leave input varfont unmodified + stream = BytesIO() + varfont.save(stream) + stream.seek(0) + varfont = TTFont(stream) + + fvar = varfont["fvar"] + axes = {a.axisTag: (a.minValue, a.defaultValue, a.maxValue) for a in fvar.axes} + loc = normalizeLocation(location, axes) + if "avar" in varfont: + maps = varfont["avar"].segments + loc = {k: piecewiseLinearMap(v, maps[k]) for k, v in loc.items()} + # Quantize to F2Dot14, to avoid surprise interpolations. + loc = {k: floatToFixedToFloat(v, 14) for k, v in loc.items()} + # Location is normalized now + log.info("Normalized location: %s", loc) + + if "gvar" in varfont: + log.info("Mutating glyf/gvar tables") + gvar = varfont["gvar"] + glyf = varfont["glyf"] + hMetrics = varfont["hmtx"].metrics + vMetrics = getattr(varfont.get("vmtx"), "metrics", None) + # get list of glyph names in gvar sorted by component depth + glyphnames = sorted( + gvar.variations.keys(), + key=lambda name: ( + ( + glyf[name].getCompositeMaxpValues(glyf).maxComponentDepth + if glyf[name].isComposite() + else 0 + ), + name, + ), + ) + for glyphname in glyphnames: + variations = gvar.variations[glyphname] + coordinates, _ = glyf._getCoordinatesAndControls( + glyphname, hMetrics, vMetrics + ) + origCoords, endPts = None, None + for var in variations: + scalar = supportScalar(loc, var.axes) + if not scalar: + continue + delta = var.coordinates + if None in delta: + if origCoords is None: + origCoords, g = glyf._getCoordinatesAndControls( + glyphname, hMetrics, vMetrics + ) + delta = iup_delta(delta, origCoords, g.endPts) + coordinates += GlyphCoordinates(delta) * scalar + glyf._setCoordinates(glyphname, coordinates, hMetrics, vMetrics) + else: + glyf = None + + if "DSIG" in varfont: + del varfont["DSIG"] + + if "cvar" in varfont: + log.info("Mutating cvt/cvar tables") + cvar = varfont["cvar"] + cvt = varfont["cvt "] + deltas = {} + for var in cvar.variations: + scalar = supportScalar(loc, var.axes) + if not scalar: + continue + for i, c in enumerate(var.coordinates): + if c is not None: + deltas[i] = deltas.get(i, 0) + scalar * c + for i, delta in deltas.items(): + cvt[i] += otRound(delta) + + if "CFF2" in varfont: + log.info("Mutating CFF2 table") + glyphOrder = varfont.getGlyphOrder() + CFF2 = varfont["CFF2"] + topDict = CFF2.cff.topDictIndex[0] + vsInstancer = VarStoreInstancer(topDict.VarStore.otVarStore, fvar.axes, loc) + interpolateFromDeltas = vsInstancer.interpolateFromDeltas + interpolate_cff2_PrivateDict(topDict, interpolateFromDeltas) + CFF2.desubroutinize() + interpolate_cff2_charstrings(topDict, interpolateFromDeltas, glyphOrder) + interpolate_cff2_metrics(varfont, topDict, glyphOrder, loc) + del topDict.rawDict["VarStore"] + del topDict.VarStore + + if "MVAR" in varfont: + log.info("Mutating MVAR table") + mvar = varfont["MVAR"].table + varStoreInstancer = VarStoreInstancer(mvar.VarStore, fvar.axes, loc) + records = mvar.ValueRecord + for rec in records: + mvarTag = rec.ValueTag + if mvarTag not in MVAR_ENTRIES: + continue + tableTag, itemName = MVAR_ENTRIES[mvarTag] + delta = otRound(varStoreInstancer[rec.VarIdx]) + if not delta: + continue + setattr( + varfont[tableTag], + itemName, + getattr(varfont[tableTag], itemName) + delta, + ) + + log.info("Mutating FeatureVariations") + for tableTag in "GSUB", "GPOS": + if not tableTag in varfont: + continue + table = varfont[tableTag].table + if not getattr(table, "FeatureVariations", None): + continue + variations = table.FeatureVariations + for record in variations.FeatureVariationRecord: + applies = True + for condition in record.ConditionSet.ConditionTable: + if condition.Format == 1: + axisIdx = condition.AxisIndex + axisTag = fvar.axes[axisIdx].axisTag + Min = condition.FilterRangeMinValue + Max = condition.FilterRangeMaxValue + v = loc[axisTag] + if not (Min <= v <= Max): + applies = False + else: + applies = False + if not applies: + break + + if applies: + assert record.FeatureTableSubstitution.Version == 0x00010000 + for rec in record.FeatureTableSubstitution.SubstitutionRecord: + table.FeatureList.FeatureRecord[rec.FeatureIndex].Feature = ( + rec.Feature + ) + break + del table.FeatureVariations + + if "GDEF" in varfont and varfont["GDEF"].table.Version >= 0x00010003: + log.info("Mutating GDEF/GPOS/GSUB tables") + gdef = varfont["GDEF"].table + instancer = VarStoreInstancer(gdef.VarStore, fvar.axes, loc) + + merger = MutatorMerger(varfont, instancer) + merger.mergeTables(varfont, [varfont], ["GDEF", "GPOS"]) + + # Downgrade GDEF. + del gdef.VarStore + gdef.Version = 0x00010002 + if gdef.MarkGlyphSetsDef is None: + del gdef.MarkGlyphSetsDef + gdef.Version = 0x00010000 + + if not ( + gdef.LigCaretList + or gdef.MarkAttachClassDef + or gdef.GlyphClassDef + or gdef.AttachList + or (gdef.Version >= 0x00010002 and gdef.MarkGlyphSetsDef) + ): + del varfont["GDEF"] + + addidef = False + if glyf: + for glyph in glyf.glyphs.values(): + if hasattr(glyph, "program"): + instructions = glyph.program.getAssembly() + # If GETVARIATION opcode is used in bytecode of any glyph add IDEF + addidef = any(op.startswith("GETVARIATION") for op in instructions) + if addidef: + break + if overlap: + for glyph_name in glyf.keys(): + glyph = glyf[glyph_name] + # Set OVERLAP_COMPOUND bit for compound glyphs + if glyph.isComposite(): + glyph.components[0].flags |= OVERLAP_COMPOUND + # Set OVERLAP_SIMPLE bit for simple glyphs + elif glyph.numberOfContours > 0: + glyph.flags[0] |= flagOverlapSimple + if addidef: + log.info("Adding IDEF to fpgm table for GETVARIATION opcode") + asm = [] + if "fpgm" in varfont: + fpgm = varfont["fpgm"] + asm = fpgm.program.getAssembly() + else: + fpgm = newTable("fpgm") + fpgm.program = ttProgram.Program() + varfont["fpgm"] = fpgm + asm.append("PUSHB[000] 145") + asm.append("IDEF[ ]") + args = [str(len(loc))] + for a in fvar.axes: + args.append(str(floatToFixed(loc[a.axisTag], 14))) + asm.append("NPUSHW[ ] " + " ".join(args)) + asm.append("ENDF[ ]") + fpgm.program.fromAssembly(asm) + + # Change maxp attributes as IDEF is added + if "maxp" in varfont: + maxp = varfont["maxp"] + setattr( + maxp, "maxInstructionDefs", 1 + getattr(maxp, "maxInstructionDefs", 0) + ) + setattr( + maxp, + "maxStackElements", + max(len(loc), getattr(maxp, "maxStackElements", 0)), + ) + + if "name" in varfont: + log.info("Pruning name table") + exclude = {a.axisNameID for a in fvar.axes} + for i in fvar.instances: + exclude.add(i.subfamilyNameID) + exclude.add(i.postscriptNameID) + if "ltag" in varfont: + # Drop the whole 'ltag' table if all its language tags are referenced by + # name records to be pruned. + # TODO: prune unused ltag tags and re-enumerate langIDs accordingly + excludedUnicodeLangIDs = [ + n.langID + for n in varfont["name"].names + if n.nameID in exclude and n.platformID == 0 and n.langID != 0xFFFF + ] + if set(excludedUnicodeLangIDs) == set(range(len((varfont["ltag"].tags)))): + del varfont["ltag"] + varfont["name"].names[:] = [ + n + for n in varfont["name"].names + if n.nameID < 256 or n.nameID not in exclude + ] + + if "wght" in location and "OS/2" in varfont: + varfont["OS/2"].usWeightClass = otRound(max(1, min(location["wght"], 1000))) + if "wdth" in location: + wdth = location["wdth"] + for percent, widthClass in sorted(OS2_WIDTH_CLASS_VALUES.items()): + if wdth < percent: + varfont["OS/2"].usWidthClass = widthClass + break + else: + varfont["OS/2"].usWidthClass = 9 + if "slnt" in location and "post" in varfont: + varfont["post"].italicAngle = max(-90, min(location["slnt"], 90)) + + log.info("Removing variable tables") + for tag in ("avar", "cvar", "fvar", "gvar", "HVAR", "MVAR", "VVAR", "STAT"): + if tag in varfont: + del varfont[tag] + + return varfont + + +def main(args=None): + """Instantiate a variation font""" + from fontTools import configLogger + import argparse + + parser = argparse.ArgumentParser( + "fonttools varLib.mutator", description="Instantiate a variable font" + ) + parser.add_argument("input", metavar="INPUT.ttf", help="Input variable TTF file.") + parser.add_argument( + "locargs", + metavar="AXIS=LOC", + nargs="*", + help="List of space separated locations. A location consist in " + "the name of a variation axis, followed by '=' and a number. E.g.: " + " wght=700 wdth=80. The default is the location of the base master.", + ) + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT.ttf", + default=None, + help="Output instance TTF file (default: INPUT-instance.ttf).", + ) + parser.add_argument( + "--no-recalc-timestamp", + dest="recalc_timestamp", + action="store_false", + help="Don't set the output font's timestamp to the current time.", + ) + logging_group = parser.add_mutually_exclusive_group(required=False) + logging_group.add_argument( + "-v", "--verbose", action="store_true", help="Run more verbosely." + ) + logging_group.add_argument( + "-q", "--quiet", action="store_true", help="Turn verbosity off." + ) + parser.add_argument( + "--no-overlap", + dest="overlap", + action="store_false", + help="Don't set OVERLAP_SIMPLE/OVERLAP_COMPOUND glyf flags.", + ) + options = parser.parse_args(args) + + varfilename = options.input + outfile = ( + os.path.splitext(varfilename)[0] + "-instance.ttf" + if not options.output + else options.output + ) + configLogger( + level=("DEBUG" if options.verbose else "ERROR" if options.quiet else "INFO") + ) + + loc = {} + for arg in options.locargs: + try: + tag, val = arg.split("=") + assert len(tag) <= 4 + loc[tag.ljust(4)] = float(val) + except (ValueError, AssertionError): + parser.error("invalid location argument format: %r" % arg) + log.info("Location: %s", loc) + + log.info("Loading variable font") + varfont = TTFont(varfilename, recalcTimestamp=options.recalc_timestamp) + + instantiateVariableFont(varfont, loc, inplace=True, overlap=options.overlap) + + log.info("Saving instance font %s", outfile) + varfont.save(outfile) + + +if __name__ == "__main__": + import sys + + if len(sys.argv) > 1: + sys.exit(main()) + import doctest + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/mvar.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/mvar.py new file mode 100644 index 0000000000000000000000000000000000000000..653aeb45e0ff18e06c2dd04ad58085d77a73c1b5 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/mvar.py @@ -0,0 +1,40 @@ +MVAR_ENTRIES = { + "hasc": ("OS/2", "sTypoAscender"), # horizontal ascender + "hdsc": ("OS/2", "sTypoDescender"), # horizontal descender + "hlgp": ("OS/2", "sTypoLineGap"), # horizontal line gap + "hcla": ("OS/2", "usWinAscent"), # horizontal clipping ascent + "hcld": ("OS/2", "usWinDescent"), # horizontal clipping descent + "vasc": ("vhea", "ascent"), # vertical ascender + "vdsc": ("vhea", "descent"), # vertical descender + "vlgp": ("vhea", "lineGap"), # vertical line gap + "hcrs": ("hhea", "caretSlopeRise"), # horizontal caret rise + "hcrn": ("hhea", "caretSlopeRun"), # horizontal caret run + "hcof": ("hhea", "caretOffset"), # horizontal caret offset + "vcrs": ("vhea", "caretSlopeRise"), # vertical caret rise + "vcrn": ("vhea", "caretSlopeRun"), # vertical caret run + "vcof": ("vhea", "caretOffset"), # vertical caret offset + "xhgt": ("OS/2", "sxHeight"), # x height + "cpht": ("OS/2", "sCapHeight"), # cap height + "sbxs": ("OS/2", "ySubscriptXSize"), # subscript em x size + "sbys": ("OS/2", "ySubscriptYSize"), # subscript em y size + "sbxo": ("OS/2", "ySubscriptXOffset"), # subscript em x offset + "sbyo": ("OS/2", "ySubscriptYOffset"), # subscript em y offset + "spxs": ("OS/2", "ySuperscriptXSize"), # superscript em x size + "spys": ("OS/2", "ySuperscriptYSize"), # superscript em y size + "spxo": ("OS/2", "ySuperscriptXOffset"), # superscript em x offset + "spyo": ("OS/2", "ySuperscriptYOffset"), # superscript em y offset + "strs": ("OS/2", "yStrikeoutSize"), # strikeout size + "stro": ("OS/2", "yStrikeoutPosition"), # strikeout offset + "unds": ("post", "underlineThickness"), # underline size + "undo": ("post", "underlinePosition"), # underline offset + #'gsp0': ('gasp', 'gaspRange[0].rangeMaxPPEM'), # gaspRange[0] + #'gsp1': ('gasp', 'gaspRange[1].rangeMaxPPEM'), # gaspRange[1] + #'gsp2': ('gasp', 'gaspRange[2].rangeMaxPPEM'), # gaspRange[2] + #'gsp3': ('gasp', 'gaspRange[3].rangeMaxPPEM'), # gaspRange[3] + #'gsp4': ('gasp', 'gaspRange[4].rangeMaxPPEM'), # gaspRange[4] + #'gsp5': ('gasp', 'gaspRange[5].rangeMaxPPEM'), # gaspRange[5] + #'gsp6': ('gasp', 'gaspRange[6].rangeMaxPPEM'), # gaspRange[6] + #'gsp7': ('gasp', 'gaspRange[7].rangeMaxPPEM'), # gaspRange[7] + #'gsp8': ('gasp', 'gaspRange[8].rangeMaxPPEM'), # gaspRange[8] + #'gsp9': ('gasp', 'gaspRange[9].rangeMaxPPEM'), # gaspRange[9] +} diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/plot.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/plot.py new file mode 100644 index 0000000000000000000000000000000000000000..e0a7ca50d3f317d7c3219b77ff84f0f8bb310c6d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/plot.py @@ -0,0 +1,238 @@ +"""Visualize DesignSpaceDocument and resulting VariationModel.""" + +from fontTools.varLib.models import VariationModel, supportScalar +from fontTools.designspaceLib import DesignSpaceDocument +from matplotlib import pyplot +from mpl_toolkits.mplot3d import axes3d +from itertools import cycle +import math +import logging +import sys + +log = logging.getLogger(__name__) + + +def stops(support, count=10): + a, b, c = support + + return ( + [a + (b - a) * i / count for i in range(count)] + + [b + (c - b) * i / count for i in range(count)] + + [c] + ) + + +def _plotLocationsDots(locations, axes, subplot, **kwargs): + for loc, color in zip(locations, cycle(pyplot.cm.Set1.colors)): + if len(axes) == 1: + subplot.plot([loc.get(axes[0], 0)], [1.0], "o", color=color, **kwargs) + elif len(axes) == 2: + subplot.plot( + [loc.get(axes[0], 0)], + [loc.get(axes[1], 0)], + [1.0], + "o", + color=color, + **kwargs, + ) + else: + raise AssertionError(len(axes)) + + +def plotLocations(locations, fig, names=None, **kwargs): + n = len(locations) + cols = math.ceil(n**0.5) + rows = math.ceil(n / cols) + + if names is None: + names = [None] * len(locations) + + model = VariationModel(locations) + names = [names[model.reverseMapping[i]] for i in range(len(names))] + + axes = sorted(locations[0].keys()) + if len(axes) == 1: + _plotLocations2D(model, axes[0], fig, cols, rows, names=names, **kwargs) + elif len(axes) == 2: + _plotLocations3D(model, axes, fig, cols, rows, names=names, **kwargs) + else: + raise ValueError("Only 1 or 2 axes are supported") + + +def _plotLocations2D(model, axis, fig, cols, rows, names, **kwargs): + subplot = fig.add_subplot(111) + for i, (support, color, name) in enumerate( + zip(model.supports, cycle(pyplot.cm.Set1.colors), cycle(names)) + ): + if name is not None: + subplot.set_title(name) + subplot.set_xlabel(axis) + pyplot.xlim(-1.0, +1.0) + + Xs = support.get(axis, (-1.0, 0.0, +1.0)) + X, Y = [], [] + for x in stops(Xs): + y = supportScalar({axis: x}, support) + X.append(x) + Y.append(y) + subplot.plot(X, Y, color=color, **kwargs) + + _plotLocationsDots(model.locations, [axis], subplot) + + +def _plotLocations3D(model, axes, fig, rows, cols, names, **kwargs): + ax1, ax2 = axes + + axis3D = fig.add_subplot(111, projection="3d") + for i, (support, color, name) in enumerate( + zip(model.supports, cycle(pyplot.cm.Set1.colors), cycle(names)) + ): + if name is not None: + axis3D.set_title(name) + axis3D.set_xlabel(ax1) + axis3D.set_ylabel(ax2) + pyplot.xlim(-1.0, +1.0) + pyplot.ylim(-1.0, +1.0) + + Xs = support.get(ax1, (-1.0, 0.0, +1.0)) + Ys = support.get(ax2, (-1.0, 0.0, +1.0)) + for x in stops(Xs): + X, Y, Z = [], [], [] + for y in Ys: + z = supportScalar({ax1: x, ax2: y}, support) + X.append(x) + Y.append(y) + Z.append(z) + axis3D.plot(X, Y, Z, color=color, **kwargs) + for y in stops(Ys): + X, Y, Z = [], [], [] + for x in Xs: + z = supportScalar({ax1: x, ax2: y}, support) + X.append(x) + Y.append(y) + Z.append(z) + axis3D.plot(X, Y, Z, color=color, **kwargs) + + _plotLocationsDots(model.locations, [ax1, ax2], axis3D) + + +def plotDocument(doc, fig, **kwargs): + doc.normalize() + locations = [s.location for s in doc.sources] + names = [s.name for s in doc.sources] + plotLocations(locations, fig, names, **kwargs) + + +def _plotModelFromMasters2D(model, masterValues, fig, **kwargs): + assert len(model.axisOrder) == 1 + axis = model.axisOrder[0] + + axis_min = min(loc.get(axis, 0) for loc in model.locations) + axis_max = max(loc.get(axis, 0) for loc in model.locations) + + import numpy as np + + X = np.arange(axis_min, axis_max, (axis_max - axis_min) / 100) + Y = [] + + for x in X: + loc = {axis: x} + v = model.interpolateFromMasters(loc, masterValues) + Y.append(v) + + subplot = fig.add_subplot(111) + subplot.plot(X, Y, "-", **kwargs) + + +def _plotModelFromMasters3D(model, masterValues, fig, **kwargs): + assert len(model.axisOrder) == 2 + axis1, axis2 = model.axisOrder[0], model.axisOrder[1] + + axis1_min = min(loc.get(axis1, 0) for loc in model.locations) + axis1_max = max(loc.get(axis1, 0) for loc in model.locations) + axis2_min = min(loc.get(axis2, 0) for loc in model.locations) + axis2_max = max(loc.get(axis2, 0) for loc in model.locations) + + import numpy as np + + X = np.arange(axis1_min, axis1_max, (axis1_max - axis1_min) / 100) + Y = np.arange(axis2_min, axis2_max, (axis2_max - axis2_min) / 100) + X, Y = np.meshgrid(X, Y) + Z = [] + + for row_x, row_y in zip(X, Y): + z_row = [] + Z.append(z_row) + for x, y in zip(row_x, row_y): + loc = {axis1: x, axis2: y} + v = model.interpolateFromMasters(loc, masterValues) + z_row.append(v) + Z = np.array(Z) + + axis3D = fig.add_subplot(111, projection="3d") + axis3D.plot_surface(X, Y, Z, **kwargs) + + +def plotModelFromMasters(model, masterValues, fig, **kwargs): + """Plot a variation model and set of master values corresponding + to the locations to the model into a pyplot figure. Variation + model must have axisOrder of size 1 or 2.""" + if len(model.axisOrder) == 1: + _plotModelFromMasters2D(model, masterValues, fig, **kwargs) + elif len(model.axisOrder) == 2: + _plotModelFromMasters3D(model, masterValues, fig, **kwargs) + else: + raise ValueError("Only 1 or 2 axes are supported") + + +def main(args=None): + from fontTools import configLogger + + if args is None: + args = sys.argv[1:] + + # configure the library logger (for >= WARNING) + configLogger() + # comment this out to enable debug messages from logger + # log.setLevel(logging.DEBUG) + + if len(args) < 1: + print("usage: fonttools varLib.plot source.designspace", file=sys.stderr) + print(" or") + print("usage: fonttools varLib.plot location1 location2 ...", file=sys.stderr) + print(" or") + print( + "usage: fonttools varLib.plot location1=value1 location2=value2 ...", + file=sys.stderr, + ) + sys.exit(1) + + fig = pyplot.figure() + fig.set_tight_layout(True) + + if len(args) == 1 and args[0].endswith(".designspace"): + doc = DesignSpaceDocument() + doc.read(args[0]) + plotDocument(doc, fig) + else: + axes = [chr(c) for c in range(ord("A"), ord("Z") + 1)] + if "=" not in args[0]: + locs = [dict(zip(axes, (float(v) for v in s.split(",")))) for s in args] + plotLocations(locs, fig) + else: + locations = [] + masterValues = [] + for arg in args: + loc, v = arg.split("=") + locations.append(dict(zip(axes, (float(v) for v in loc.split(","))))) + masterValues.append(float(v)) + model = VariationModel(locations, axes[: len(locations[0])]) + plotModelFromMasters(model, masterValues, fig) + + pyplot.show() + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/stat.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/stat.py new file mode 100644 index 0000000000000000000000000000000000000000..eacbf580aed454fea16935c2c4ec2a3706752e66 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/stat.py @@ -0,0 +1,149 @@ +"""Extra methods for DesignSpaceDocument to generate its STAT table data.""" + +from __future__ import annotations + +from typing import Dict, List, Union + +import fontTools.otlLib.builder +from fontTools.designspaceLib import ( + AxisLabelDescriptor, + DesignSpaceDocument, + DesignSpaceDocumentError, + LocationLabelDescriptor, +) +from fontTools.designspaceLib.types import Region, getVFUserRegion, locationInRegion +from fontTools.ttLib import TTFont + + +def buildVFStatTable(ttFont: TTFont, doc: DesignSpaceDocument, vfName: str) -> None: + """Build the STAT table for the variable font identified by its name in + the given document. + + Knowing which variable we're building STAT data for is needed to subset + the STAT locations to only include what the variable font actually ships. + + .. versionadded:: 5.0 + + .. seealso:: + - :func:`getStatAxes()` + - :func:`getStatLocations()` + - :func:`fontTools.otlLib.builder.buildStatTable()` + """ + for vf in doc.getVariableFonts(): + if vf.name == vfName: + break + else: + raise DesignSpaceDocumentError( + f"Cannot find the variable font by name {vfName}" + ) + + region = getVFUserRegion(doc, vf) + + # if there are not currently any mac names don't add them here, that's inconsistent + # https://github.com/fonttools/fonttools/issues/683 + macNames = any( + nr.platformID == 1 for nr in getattr(ttFont.get("name"), "names", ()) + ) + + return fontTools.otlLib.builder.buildStatTable( + ttFont, + getStatAxes(doc, region), + getStatLocations(doc, region), + doc.elidedFallbackName if doc.elidedFallbackName is not None else 2, + macNames=macNames, + ) + + +def getStatAxes(doc: DesignSpaceDocument, userRegion: Region) -> List[Dict]: + """Return a list of axis dicts suitable for use as the ``axes`` + argument to :func:`fontTools.otlLib.builder.buildStatTable()`. + + .. versionadded:: 5.0 + """ + # First, get the axis labels with explicit ordering + # then append the others in the order they appear. + maxOrdering = max( + (axis.axisOrdering for axis in doc.axes if axis.axisOrdering is not None), + default=-1, + ) + axisOrderings = [] + for axis in doc.axes: + if axis.axisOrdering is not None: + axisOrderings.append(axis.axisOrdering) + else: + maxOrdering += 1 + axisOrderings.append(maxOrdering) + return [ + dict( + tag=axis.tag, + name={"en": axis.name, **axis.labelNames}, + ordering=ordering, + values=[ + _axisLabelToStatLocation(label) + for label in axis.axisLabels + if locationInRegion({axis.name: label.userValue}, userRegion) + ], + ) + for axis, ordering in zip(doc.axes, axisOrderings) + ] + + +def getStatLocations(doc: DesignSpaceDocument, userRegion: Region) -> List[Dict]: + """Return a list of location dicts suitable for use as the ``locations`` + argument to :func:`fontTools.otlLib.builder.buildStatTable()`. + + .. versionadded:: 5.0 + """ + axesByName = {axis.name: axis for axis in doc.axes} + return [ + dict( + name={"en": label.name, **label.labelNames}, + # Location in the designspace is keyed by axis name + # Location in buildStatTable by axis tag + location={ + axesByName[name].tag: value + for name, value in label.getFullUserLocation(doc).items() + }, + flags=_labelToFlags(label), + ) + for label in doc.locationLabels + if locationInRegion(label.getFullUserLocation(doc), userRegion) + ] + + +def _labelToFlags(label: Union[AxisLabelDescriptor, LocationLabelDescriptor]) -> int: + flags = 0 + if label.olderSibling: + flags |= 1 + if label.elidable: + flags |= 2 + return flags + + +def _axisLabelToStatLocation( + label: AxisLabelDescriptor, +) -> Dict: + label_format = label.getFormat() + name = {"en": label.name, **label.labelNames} + flags = _labelToFlags(label) + if label_format == 1: + return dict(name=name, value=label.userValue, flags=flags) + if label_format == 3: + return dict( + name=name, + value=label.userValue, + linkedValue=label.linkedUserValue, + flags=flags, + ) + if label_format == 2: + res = dict( + name=name, + nominalValue=label.userValue, + flags=flags, + ) + if label.userMinimum is not None: + res["rangeMinValue"] = label.userMinimum + if label.userMaximum is not None: + res["rangeMaxValue"] = label.userMaximum + return res + raise NotImplementedError("Unknown STAT label format") diff --git a/.venv/lib/python3.11/site-packages/fontTools/varLib/varStore.py b/.venv/lib/python3.11/site-packages/fontTools/varLib/varStore.py new file mode 100644 index 0000000000000000000000000000000000000000..c0329c09d252c9297d6ac6e49cbcc27a3326dff3 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/varLib/varStore.py @@ -0,0 +1,739 @@ +from fontTools.misc.roundTools import noRound, otRound +from fontTools.misc.intTools import bit_count +from fontTools.ttLib.tables import otTables as ot +from fontTools.varLib.models import supportScalar +from fontTools.varLib.builder import ( + buildVarRegionList, + buildVarStore, + buildVarRegion, + buildVarData, +) +from functools import partial +from collections import defaultdict +from heapq import heappush, heappop + + +NO_VARIATION_INDEX = ot.NO_VARIATION_INDEX +ot.VarStore.NO_VARIATION_INDEX = NO_VARIATION_INDEX + + +def _getLocationKey(loc): + return tuple(sorted(loc.items(), key=lambda kv: kv[0])) + + +class OnlineVarStoreBuilder(object): + def __init__(self, axisTags): + self._axisTags = axisTags + self._regionMap = {} + self._regionList = buildVarRegionList([], axisTags) + self._store = buildVarStore(self._regionList, []) + self._data = None + self._model = None + self._supports = None + self._varDataIndices = {} + self._varDataCaches = {} + self._cache = None + + def setModel(self, model): + self.setSupports(model.supports) + self._model = model + + def setSupports(self, supports): + self._model = None + self._supports = list(supports) + if self._supports and not self._supports[0]: + del self._supports[0] # Drop base master support + self._cache = None + self._data = None + + def finish(self, optimize=True): + self._regionList.RegionCount = len(self._regionList.Region) + self._store.VarDataCount = len(self._store.VarData) + for data in self._store.VarData: + data.ItemCount = len(data.Item) + data.calculateNumShorts(optimize=optimize) + return self._store + + def _add_VarData(self, num_items=1): + regionMap = self._regionMap + regionList = self._regionList + + regions = self._supports + regionIndices = [] + for region in regions: + key = _getLocationKey(region) + idx = regionMap.get(key) + if idx is None: + varRegion = buildVarRegion(region, self._axisTags) + idx = regionMap[key] = len(regionList.Region) + regionList.Region.append(varRegion) + regionIndices.append(idx) + + # Check if we have one already... + key = tuple(regionIndices) + varDataIdx = self._varDataIndices.get(key) + if varDataIdx is not None: + self._outer = varDataIdx + self._data = self._store.VarData[varDataIdx] + self._cache = self._varDataCaches[key] + if len(self._data.Item) + num_items > 0xFFFF: + # This is full. Need new one. + varDataIdx = None + + if varDataIdx is None: + self._data = buildVarData(regionIndices, [], optimize=False) + self._outer = len(self._store.VarData) + self._store.VarData.append(self._data) + self._varDataIndices[key] = self._outer + if key not in self._varDataCaches: + self._varDataCaches[key] = {} + self._cache = self._varDataCaches[key] + + def storeMasters(self, master_values, *, round=round): + deltas = self._model.getDeltas(master_values, round=round) + base = deltas.pop(0) + return base, self.storeDeltas(deltas, round=noRound) + + def storeMastersMany(self, master_values_list, *, round=round): + deltas_list = [ + self._model.getDeltas(master_values, round=round) + for master_values in master_values_list + ] + base_list = [deltas.pop(0) for deltas in deltas_list] + return base_list, self.storeDeltasMany(deltas_list, round=noRound) + + def storeDeltas(self, deltas, *, round=round): + deltas = [round(d) for d in deltas] + if len(deltas) == len(self._supports) + 1: + deltas = tuple(deltas[1:]) + else: + assert len(deltas) == len(self._supports) + deltas = tuple(deltas) + + if not self._data: + self._add_VarData() + + varIdx = self._cache.get(deltas) + if varIdx is not None: + return varIdx + + inner = len(self._data.Item) + if inner == 0xFFFF: + # Full array. Start new one. + self._add_VarData() + return self.storeDeltas(deltas, round=noRound) + self._data.addItem(deltas, round=noRound) + + varIdx = (self._outer << 16) + inner + self._cache[deltas] = varIdx + return varIdx + + def storeDeltasMany(self, deltas_list, *, round=round): + deltas_list = [[round(d) for d in deltas] for deltas in deltas_list] + deltas_list = tuple(tuple(deltas) for deltas in deltas_list) + + if not self._data: + self._add_VarData(len(deltas_list)) + + varIdx = self._cache.get(deltas_list) + if varIdx is not None: + return varIdx + + inner = len(self._data.Item) + if inner + len(deltas_list) > 0xFFFF: + # Full array. Start new one. + self._add_VarData(len(deltas_list)) + return self.storeDeltasMany(deltas_list, round=noRound) + for i, deltas in enumerate(deltas_list): + self._data.addItem(deltas, round=noRound) + + varIdx = (self._outer << 16) + inner + i + self._cache[deltas] = varIdx + + varIdx = (self._outer << 16) + inner + self._cache[deltas_list] = varIdx + + return varIdx + + +def VarData_addItem(self, deltas, *, round=round): + deltas = [round(d) for d in deltas] + + countUs = self.VarRegionCount + countThem = len(deltas) + if countUs + 1 == countThem: + deltas = list(deltas[1:]) + else: + assert countUs == countThem, (countUs, countThem) + deltas = list(deltas) + self.Item.append(deltas) + self.ItemCount = len(self.Item) + + +ot.VarData.addItem = VarData_addItem + + +def VarRegion_get_support(self, fvar_axes): + return { + fvar_axes[i].axisTag: (reg.StartCoord, reg.PeakCoord, reg.EndCoord) + for i, reg in enumerate(self.VarRegionAxis) + if reg.PeakCoord != 0 + } + + +ot.VarRegion.get_support = VarRegion_get_support + + +def VarStore___bool__(self): + return bool(self.VarData) + + +ot.VarStore.__bool__ = VarStore___bool__ + + +class VarStoreInstancer(object): + def __init__(self, varstore, fvar_axes, location={}): + self.fvar_axes = fvar_axes + assert varstore is None or varstore.Format == 1 + self._varData = varstore.VarData if varstore else [] + self._regions = varstore.VarRegionList.Region if varstore else [] + self.setLocation(location) + + def setLocation(self, location): + self.location = dict(location) + self._clearCaches() + + def _clearCaches(self): + self._scalars = {} + + def _getScalar(self, regionIdx): + scalar = self._scalars.get(regionIdx) + if scalar is None: + support = self._regions[regionIdx].get_support(self.fvar_axes) + scalar = supportScalar(self.location, support) + self._scalars[regionIdx] = scalar + return scalar + + @staticmethod + def interpolateFromDeltasAndScalars(deltas, scalars): + delta = 0.0 + for d, s in zip(deltas, scalars): + if not s: + continue + delta += d * s + return delta + + def __getitem__(self, varidx): + major, minor = varidx >> 16, varidx & 0xFFFF + if varidx == NO_VARIATION_INDEX: + return 0.0 + varData = self._varData + scalars = [self._getScalar(ri) for ri in varData[major].VarRegionIndex] + deltas = varData[major].Item[minor] + return self.interpolateFromDeltasAndScalars(deltas, scalars) + + def interpolateFromDeltas(self, varDataIndex, deltas): + varData = self._varData + scalars = [self._getScalar(ri) for ri in varData[varDataIndex].VarRegionIndex] + return self.interpolateFromDeltasAndScalars(deltas, scalars) + + +# +# Optimizations +# +# retainFirstMap - If true, major 0 mappings are retained. Deltas for unused indices are zeroed +# advIdxes - Set of major 0 indices for advance deltas to be listed first. Other major 0 indices follow. + + +def VarStore_subset_varidxes( + self, + varIdxes, + optimize=True, + retainFirstMap=False, + advIdxes=set(), + *, + VarData="VarData", +): + # Sort out used varIdxes by major/minor. + used = defaultdict(set) + for varIdx in varIdxes: + if varIdx == NO_VARIATION_INDEX: + continue + major = varIdx >> 16 + minor = varIdx & 0xFFFF + used[major].add(minor) + del varIdxes + + # + # Subset VarData + # + + varData = getattr(self, VarData) + newVarData = [] + varDataMap = {NO_VARIATION_INDEX: NO_VARIATION_INDEX} + for major, data in enumerate(varData): + usedMinors = used.get(major) + if usedMinors is None: + continue + newMajor = len(newVarData) + newVarData.append(data) + + items = data.Item + newItems = [] + if major == 0 and retainFirstMap: + for minor in range(len(items)): + newItems.append( + items[minor] if minor in usedMinors else [0] * len(items[minor]) + ) + varDataMap[minor] = minor + else: + if major == 0: + minors = sorted(advIdxes) + sorted(usedMinors - advIdxes) + else: + minors = sorted(usedMinors) + for minor in minors: + newMinor = len(newItems) + newItems.append(items[minor]) + varDataMap[(major << 16) + minor] = (newMajor << 16) + newMinor + + data.Item = newItems + data.ItemCount = len(data.Item) + + if VarData == "VarData": + data.calculateNumShorts(optimize=optimize) + + setattr(self, VarData, newVarData) + setattr(self, VarData + "Count", len(newVarData)) + + self.prune_regions() + + return varDataMap + + +ot.VarStore.subset_varidxes = VarStore_subset_varidxes + + +def VarStore_prune_regions(self, *, VarData="VarData", VarRegionList="VarRegionList"): + """Remove unused VarRegions.""" + # + # Subset VarRegionList + # + + # Collect. + usedRegions = set() + for data in getattr(self, VarData): + usedRegions.update(data.VarRegionIndex) + # Subset. + regionList = getattr(self, VarRegionList) + regions = regionList.Region + newRegions = [] + regionMap = {} + for i in sorted(usedRegions): + regionMap[i] = len(newRegions) + newRegions.append(regions[i]) + regionList.Region = newRegions + regionList.RegionCount = len(regionList.Region) + # Map. + for data in getattr(self, VarData): + data.VarRegionIndex = [regionMap[i] for i in data.VarRegionIndex] + + +ot.VarStore.prune_regions = VarStore_prune_regions + + +def _visit(self, func): + """Recurse down from self, if type of an object is ot.Device, + call func() on it. Works on otData-style classes.""" + + if type(self) == ot.Device: + func(self) + + elif isinstance(self, list): + for that in self: + _visit(that, func) + + elif hasattr(self, "getConverters") and not hasattr(self, "postRead"): + for conv in self.getConverters(): + that = getattr(self, conv.name, None) + if that is not None: + _visit(that, func) + + elif isinstance(self, ot.ValueRecord): + for that in self.__dict__.values(): + _visit(that, func) + + +def _Device_recordVarIdx(self, s): + """Add VarIdx in this Device table (if any) to the set s.""" + if self.DeltaFormat == 0x8000: + s.add((self.StartSize << 16) + self.EndSize) + + +def Object_collect_device_varidxes(self, varidxes): + adder = partial(_Device_recordVarIdx, s=varidxes) + _visit(self, adder) + + +ot.GDEF.collect_device_varidxes = Object_collect_device_varidxes +ot.GPOS.collect_device_varidxes = Object_collect_device_varidxes + + +def _Device_mapVarIdx(self, mapping, done): + """Map VarIdx in this Device table (if any) through mapping.""" + if id(self) in done: + return + done.add(id(self)) + if self.DeltaFormat == 0x8000: + varIdx = mapping[(self.StartSize << 16) + self.EndSize] + self.StartSize = varIdx >> 16 + self.EndSize = varIdx & 0xFFFF + + +def Object_remap_device_varidxes(self, varidxes_map): + mapper = partial(_Device_mapVarIdx, mapping=varidxes_map, done=set()) + _visit(self, mapper) + + +ot.GDEF.remap_device_varidxes = Object_remap_device_varidxes +ot.GPOS.remap_device_varidxes = Object_remap_device_varidxes + + +class _Encoding(object): + def __init__(self, chars): + self.chars = chars + self.width = bit_count(chars) + self.columns = self._columns(chars) + self.overhead = self._characteristic_overhead(self.columns) + self.items = set() + + def append(self, row): + self.items.add(row) + + def extend(self, lst): + self.items.update(lst) + + def width_sort_key(self): + return self.width, self.chars + + @staticmethod + def _characteristic_overhead(columns): + """Returns overhead in bytes of encoding this characteristic + as a VarData.""" + c = 4 + 6 # 4 bytes for LOffset, 6 bytes for VarData header + c += bit_count(columns) * 2 + return c + + @staticmethod + def _columns(chars): + cols = 0 + i = 1 + while chars: + if chars & 0b1111: + cols |= i + chars >>= 4 + i <<= 1 + return cols + + def gain_from_merging(self, other_encoding): + combined_chars = other_encoding.chars | self.chars + combined_width = bit_count(combined_chars) + combined_columns = self.columns | other_encoding.columns + combined_overhead = _Encoding._characteristic_overhead(combined_columns) + combined_gain = ( + +self.overhead + + other_encoding.overhead + - combined_overhead + - (combined_width - self.width) * len(self.items) + - (combined_width - other_encoding.width) * len(other_encoding.items) + ) + return combined_gain + + +class _EncodingDict(dict): + def __missing__(self, chars): + r = self[chars] = _Encoding(chars) + return r + + def add_row(self, row): + chars = self._row_characteristics(row) + self[chars].append(row) + + @staticmethod + def _row_characteristics(row): + """Returns encoding characteristics for a row.""" + longWords = False + + chars = 0 + i = 1 + for v in row: + if v: + chars += i + if not (-128 <= v <= 127): + chars += i * 0b0010 + if not (-32768 <= v <= 32767): + longWords = True + break + i <<= 4 + + if longWords: + # Redo; only allow 2byte/4byte encoding + chars = 0 + i = 1 + for v in row: + if v: + chars += i * 0b0011 + if not (-32768 <= v <= 32767): + chars += i * 0b1100 + i <<= 4 + + return chars + + +def VarStore_optimize(self, use_NO_VARIATION_INDEX=True, quantization=1): + """Optimize storage. Returns mapping from old VarIdxes to new ones.""" + + # Overview: + # + # For each VarData row, we first extend it with zeroes to have + # one column per region in VarRegionList. We then group the + # rows into _Encoding objects, by their "characteristic" bitmap. + # The characteristic bitmap is a binary number representing how + # many bytes each column of the data takes up to encode. Each + # column is encoded in four bits. For example, if a column has + # only values in the range -128..127, it would only have a single + # bit set in the characteristic bitmap for that column. If it has + # values in the range -32768..32767, it would have two bits set. + # The number of ones in the characteristic bitmap is the "width" + # of the encoding. + # + # Each encoding as such has a number of "active" (ie. non-zero) + # columns. The overhead of encoding the characteristic bitmap + # is 10 bytes, plus 2 bytes per active column. + # + # When an encoding is merged into another one, if the characteristic + # of the old encoding is a subset of the new one, then the overhead + # of the old encoding is completely eliminated. However, each row + # now would require more bytes to encode, to the tune of one byte + # per characteristic bit that is active in the new encoding but not + # in the old one. + # + # The "gain" of merging two encodings is how many bytes we save by doing so. + # + # High-level algorithm: + # + # - Each encoding has a minimal way to encode it. However, because + # of the overhead of encoding the characteristic bitmap, it may + # be beneficial to merge two encodings together, if there is + # gain in doing so. As such, we need to search for the best + # such successive merges. + # + # Algorithm: + # + # - Put all encodings into a "todo" list. + # + # - Sort todo list (for stability) by width_sort_key(), which is a tuple + # of the following items: + # * The "width" of the encoding. + # * The characteristic bitmap of the encoding, with higher-numbered + # columns compared first. + # + # - Make a priority-queue of the gain from combining each two + # encodings in the todo list. The priority queue is sorted by + # decreasing gain. Only positive gains are included. + # + # - While priority queue is not empty: + # - Pop the first item from the priority queue, + # - Merge the two encodings it represents, + # - Remove the two encodings from the todo list, + # - Insert positive gains from combining the new encoding with + # all existing todo list items into the priority queue, + # - If a todo list item with the same characteristic bitmap as + # the new encoding exists, remove it from the todo list and + # merge it into the new encoding. + # - Insert the new encoding into the todo list, + # + # - Encode all remaining items in the todo list. + # + # The output is then sorted for stability, in the following way: + # - The VarRegionList of the input is kept intact. + # - The VarData is sorted by the same width_sort_key() used at the beginning. + # - Within each VarData, the items are sorted as vectors of numbers. + # + # Finally, each VarData is optimized to remove the empty columns and + # reorder columns as needed. + + # TODO + # Check that no two VarRegions are the same; if they are, fold them. + + n = len(self.VarRegionList.Region) # Number of columns + zeroes = [0] * n + + front_mapping = {} # Map from old VarIdxes to full row tuples + + encodings = _EncodingDict() + + # Collect all items into a set of full rows (with lots of zeroes.) + for major, data in enumerate(self.VarData): + regionIndices = data.VarRegionIndex + + for minor, item in enumerate(data.Item): + row = list(zeroes) + + if quantization == 1: + for regionIdx, v in zip(regionIndices, item): + row[regionIdx] += v + else: + for regionIdx, v in zip(regionIndices, item): + row[regionIdx] += ( + round(v / quantization) * quantization + ) # TODO https://github.com/fonttools/fonttools/pull/3126#discussion_r1205439785 + + row = tuple(row) + + if use_NO_VARIATION_INDEX and not any(row): + front_mapping[(major << 16) + minor] = None + continue + + encodings.add_row(row) + front_mapping[(major << 16) + minor] = row + + # Prepare for the main algorithm. + todo = sorted(encodings.values(), key=_Encoding.width_sort_key) + del encodings + + # Repeatedly pick two best encodings to combine, and combine them. + + heap = [] + for i, encoding in enumerate(todo): + for j in range(i + 1, len(todo)): + other_encoding = todo[j] + combining_gain = encoding.gain_from_merging(other_encoding) + if combining_gain > 0: + heappush(heap, (-combining_gain, i, j)) + + while heap: + _, i, j = heappop(heap) + if todo[i] is None or todo[j] is None: + continue + + encoding, other_encoding = todo[i], todo[j] + todo[i], todo[j] = None, None + + # Combine the two encodings + combined_chars = other_encoding.chars | encoding.chars + combined_encoding = _Encoding(combined_chars) + combined_encoding.extend(encoding.items) + combined_encoding.extend(other_encoding.items) + + for k, enc in enumerate(todo): + if enc is None: + continue + + # In the unlikely event that the same encoding exists already, + # combine it. + if enc.chars == combined_chars: + combined_encoding.extend(enc.items) + todo[k] = None + continue + + combining_gain = combined_encoding.gain_from_merging(enc) + if combining_gain > 0: + heappush(heap, (-combining_gain, k, len(todo))) + + todo.append(combined_encoding) + + encodings = [encoding for encoding in todo if encoding is not None] + + # Assemble final store. + back_mapping = {} # Mapping from full rows to new VarIdxes + encodings.sort(key=_Encoding.width_sort_key) + self.VarData = [] + for encoding in encodings: + items = sorted(encoding.items) + + while items: + major = len(self.VarData) + data = ot.VarData() + self.VarData.append(data) + data.VarRegionIndex = range(n) + data.VarRegionCount = len(data.VarRegionIndex) + + # Each major can only encode up to 0xFFFF entries. + data.Item, items = items[:0xFFFF], items[0xFFFF:] + + for minor, item in enumerate(data.Item): + back_mapping[item] = (major << 16) + minor + + # Compile final mapping. + varidx_map = {NO_VARIATION_INDEX: NO_VARIATION_INDEX} + for k, v in front_mapping.items(): + varidx_map[k] = back_mapping[v] if v is not None else NO_VARIATION_INDEX + + # Recalculate things and go home. + self.VarRegionList.RegionCount = len(self.VarRegionList.Region) + self.VarDataCount = len(self.VarData) + for data in self.VarData: + data.ItemCount = len(data.Item) + data.optimize() + + # Remove unused regions. + self.prune_regions() + + return varidx_map + + +ot.VarStore.optimize = VarStore_optimize + + +def main(args=None): + """Optimize a font's GDEF variation store""" + from argparse import ArgumentParser + from fontTools import configLogger + from fontTools.ttLib import TTFont + from fontTools.ttLib.tables.otBase import OTTableWriter + + parser = ArgumentParser(prog="varLib.varStore", description=main.__doc__) + parser.add_argument("--quantization", type=int, default=1) + parser.add_argument("fontfile") + parser.add_argument("outfile", nargs="?") + options = parser.parse_args(args) + + # TODO: allow user to configure logging via command-line options + configLogger(level="INFO") + + quantization = options.quantization + fontfile = options.fontfile + outfile = options.outfile + + font = TTFont(fontfile) + gdef = font["GDEF"] + store = gdef.table.VarStore + + writer = OTTableWriter() + store.compile(writer, font) + size = len(writer.getAllData()) + print("Before: %7d bytes" % size) + + varidx_map = store.optimize(quantization=quantization) + + writer = OTTableWriter() + store.compile(writer, font) + size = len(writer.getAllData()) + print("After: %7d bytes" % size) + + if outfile is not None: + gdef.table.remap_device_varidxes(varidx_map) + if "GPOS" in font: + font["GPOS"].table.remap_device_varidxes(varidx_map) + + font.save(outfile) + + +if __name__ == "__main__": + import sys + + if len(sys.argv) > 1: + sys.exit(main()) + import doctest + + sys.exit(doctest.testmod().failed) diff --git a/.venv/lib/python3.11/site-packages/fontTools/voltLib/__init__.py b/.venv/lib/python3.11/site-packages/fontTools/voltLib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..886aa3a7864523656e609dd602683d73f985f467 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/voltLib/__init__.py @@ -0,0 +1,5 @@ +"""fontTools.voltLib -- a package for dealing with Visual OpenType Layout Tool +(VOLT) files.""" + +# See +# http://www.microsoft.com/typography/VOLT.mspx diff --git a/.venv/lib/python3.11/site-packages/fontTools/voltLib/__main__.py b/.venv/lib/python3.11/site-packages/fontTools/voltLib/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..aa2c3b3e610ddc1f9161dd2a956c783bd38f4079 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/voltLib/__main__.py @@ -0,0 +1,206 @@ +import argparse +import logging +import sys +from io import StringIO +from pathlib import Path + +from fontTools import configLogger +from fontTools.feaLib.builder import addOpenTypeFeaturesFromString +from fontTools.feaLib.error import FeatureLibError +from fontTools.feaLib.lexer import Lexer +from fontTools.misc.cliTools import makeOutputFileName +from fontTools.ttLib import TTFont, TTLibError +from fontTools.voltLib.parser import Parser +from fontTools.voltLib.voltToFea import TABLES, VoltToFea + +log = logging.getLogger("fontTools.feaLib") + +SUPPORTED_TABLES = TABLES + ["cmap"] + + +def invalid_fea_glyph_name(name): + """Check if the glyph name is valid according to FEA syntax.""" + if name[0] not in Lexer.CHAR_NAME_START_: + return True + if any(c not in Lexer.CHAR_NAME_CONTINUATION_ for c in name[1:]): + return True + return False + + +def sanitize_glyph_name(name): + """Sanitize the glyph name to ensure it is valid according to FEA syntax.""" + sanitized = "" + for i, c in enumerate(name): + if i == 0 and c not in Lexer.CHAR_NAME_START_: + sanitized += "a" + c + elif c not in Lexer.CHAR_NAME_CONTINUATION_: + sanitized += "_" + else: + sanitized += c + + return sanitized + + +def main(args=None): + """Build tables from a MS VOLT project into an OTF font""" + parser = argparse.ArgumentParser( + description="Use fontTools to compile MS VOLT projects." + ) + parser.add_argument( + "input", + metavar="INPUT", + help="Path to the input font/VTP file to process", + type=Path, + ) + parser.add_argument( + "-f", + "--font", + metavar="INPUT_FONT", + help="Path to the input font (if INPUT is a VTP file)", + type=Path, + ) + parser.add_argument( + "-o", + "--output", + dest="output", + metavar="OUTPUT", + help="Path to the output font.", + type=Path, + ) + parser.add_argument( + "-t", + "--tables", + metavar="TABLE_TAG", + choices=SUPPORTED_TABLES, + nargs="+", + help="Specify the table(s) to be built.", + ) + parser.add_argument( + "-F", + "--debug-feature-file", + help="Write the generated feature file to disk.", + action="store_true", + ) + parser.add_argument( + "--ship", + help="Remove source VOLT tables from output font.", + action="store_true", + ) + parser.add_argument( + "-v", + "--verbose", + help="Increase the logger verbosity. Multiple -v options are allowed.", + action="count", + default=0, + ) + parser.add_argument( + "-T", + "--traceback", + help="show traceback for exceptions.", + action="store_true", + ) + options = parser.parse_args(args) + + levels = ["WARNING", "INFO", "DEBUG"] + configLogger(level=levels[min(len(levels) - 1, options.verbose)]) + + output_font = options.output or Path( + makeOutputFileName(options.font or options.input) + ) + log.info(f"Compiling MS VOLT to '{output_font}'") + + file_or_path = options.input + font = None + + # If the input is a font file, extract the VOLT data from the "TSIV" table + try: + font = TTFont(file_or_path) + if "TSIV" in font: + file_or_path = StringIO(font["TSIV"].data.decode("utf-8")) + else: + log.error('"TSIV" table is missing') + return 1 + except TTLibError: + pass + + # If input is not a font file, the font must be provided + if font is None: + if not options.font: + log.error("Please provide an input font") + return 1 + font = TTFont(options.font) + + # FEA syntax does not allow some glyph names that VOLT accepts, so if we + # found such glyph name we will temporarily rename such glyphs. + glyphOrder = font.getGlyphOrder() + tempGlyphOrder = None + if any(invalid_fea_glyph_name(n) for n in glyphOrder): + tempGlyphOrder = [] + for n in glyphOrder: + if invalid_fea_glyph_name(n): + n = sanitize_glyph_name(n) + existing = set(tempGlyphOrder) | set(glyphOrder) + while n in existing: + n = "a" + n + tempGlyphOrder.append(n) + font.setGlyphOrder(tempGlyphOrder) + + doc = Parser(file_or_path).parse() + + log.info("Converting VTP data to FEA") + converter = VoltToFea(doc, font) + try: + fea = converter.convert(options.tables, ignore_unsupported_settings=True) + except NotImplementedError as e: + if options.traceback: + raise + location = getattr(e.args[0], "location", None) + message = f'"{e}" is not supported' + if location: + path, line, column = location + log.error(f"{path}:{line}:{column}: {message}") + else: + log.error(message) + return 1 + + fea_filename = options.input + if options.debug_feature_file: + fea_filename = output_font.with_suffix(".fea") + log.info(f"Writing FEA to '{fea_filename}'") + with open(fea_filename, "w") as fp: + fp.write(fea) + + log.info("Compiling FEA to OpenType tables") + try: + addOpenTypeFeaturesFromString( + font, + fea, + filename=fea_filename, + tables=options.tables, + ) + except FeatureLibError as e: + if options.traceback: + raise + log.error(e) + return 1 + + if options.ship: + for tag in ["TSIV", "TSIS", "TSIP", "TSID"]: + if tag in font: + del font[tag] + + # Restore original glyph names. + if tempGlyphOrder: + import io + + f = io.BytesIO() + font.save(f) + font = TTFont(f) + font.setGlyphOrder(glyphOrder) + font["post"].extraNames = [] + + font.save(output_font) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fontTools/voltLib/ast.py b/.venv/lib/python3.11/site-packages/fontTools/voltLib/ast.py new file mode 100644 index 0000000000000000000000000000000000000000..dba8f4d45b3e16a3f7a13aa43daf52f0f6bc45e0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/voltLib/ast.py @@ -0,0 +1,452 @@ +from fontTools.voltLib.error import VoltLibError +from typing import NamedTuple + + +class Pos(NamedTuple): + adv: int + dx: int + dy: int + adv_adjust_by: dict + dx_adjust_by: dict + dy_adjust_by: dict + + def __str__(self): + res = " POS" + for attr in ("adv", "dx", "dy"): + value = getattr(self, attr) + if value is not None: + res += f" {attr.upper()} {value}" + adjust_by = getattr(self, f"{attr}_adjust_by", {}) + for size, adjustment in adjust_by.items(): + res += f" ADJUST_BY {adjustment} AT {size}" + res += " END_POS" + return res + + +class Element(object): + def __init__(self, location=None): + self.location = location + + def build(self, builder): + pass + + def __str__(self): + raise NotImplementedError + + +class Statement(Element): + pass + + +class Expression(Element): + pass + + +class VoltFile(Statement): + def __init__(self): + Statement.__init__(self, location=None) + self.statements = [] + + def build(self, builder): + for s in self.statements: + s.build(builder) + + def __str__(self): + return "\n" + "\n".join(str(s) for s in self.statements) + " END\n" + + +class GlyphDefinition(Statement): + def __init__(self, name, gid, gunicode, gtype, components, location=None): + Statement.__init__(self, location) + self.name = name + self.id = gid + self.unicode = gunicode + self.type = gtype + self.components = components + + def __str__(self): + res = f'DEF_GLYPH "{self.name}" ID {self.id}' + if self.unicode is not None: + if len(self.unicode) > 1: + unicodes = ",".join(f"U+{u:04X}" for u in self.unicode) + res += f' UNICODEVALUES "{unicodes}"' + else: + res += f" UNICODE {self.unicode[0]}" + if self.type is not None: + res += f" TYPE {self.type}" + if self.components is not None: + res += f" COMPONENTS {self.components}" + res += " END_GLYPH" + return res + + +class GroupDefinition(Statement): + def __init__(self, name, enum, location=None): + Statement.__init__(self, location) + self.name = name + self.enum = enum + self.glyphs_ = None + + def glyphSet(self, groups=None): + if groups is not None and self.name in groups: + raise VoltLibError( + 'Group "%s" contains itself.' % (self.name), self.location + ) + if self.glyphs_ is None: + if groups is None: + groups = set({self.name}) + else: + groups.add(self.name) + self.glyphs_ = self.enum.glyphSet(groups) + return self.glyphs_ + + def __str__(self): + enum = self.enum and str(self.enum) or "" + return f'DEF_GROUP "{self.name}"\n{enum}\nEND_GROUP' + + +class GlyphName(Expression): + """A single glyph name, such as cedilla.""" + + def __init__(self, glyph, location=None): + Expression.__init__(self, location) + self.glyph = glyph + + def glyphSet(self): + return (self.glyph,) + + def __str__(self): + return f' GLYPH "{self.glyph}"' + + +class Enum(Expression): + """An enum""" + + def __init__(self, enum, location=None): + Expression.__init__(self, location) + self.enum = enum + + def __iter__(self): + for e in self.glyphSet(): + yield e + + def glyphSet(self, groups=None): + glyphs = [] + for element in self.enum: + if isinstance(element, (GroupName, Enum)): + glyphs.extend(element.glyphSet(groups)) + else: + glyphs.extend(element.glyphSet()) + return tuple(glyphs) + + def __str__(self): + enum = "".join(str(e) for e in self.enum) + return f" ENUM{enum} END_ENUM" + + +class GroupName(Expression): + """A glyph group""" + + def __init__(self, group, parser, location=None): + Expression.__init__(self, location) + self.group = group + self.parser_ = parser + + def glyphSet(self, groups=None): + group = self.parser_.resolve_group(self.group) + if group is not None: + self.glyphs_ = group.glyphSet(groups) + return self.glyphs_ + else: + raise VoltLibError( + 'Group "%s" is used but undefined.' % (self.group), self.location + ) + + def __str__(self): + return f' GROUP "{self.group}"' + + +class Range(Expression): + """A glyph range""" + + def __init__(self, start, end, parser, location=None): + Expression.__init__(self, location) + self.start = start + self.end = end + self.parser = parser + + def glyphSet(self): + return tuple(self.parser.glyph_range(self.start, self.end)) + + def __str__(self): + return f' RANGE "{self.start}" TO "{self.end}"' + + +class ScriptDefinition(Statement): + def __init__(self, name, tag, langs, location=None): + Statement.__init__(self, location) + self.name = name + self.tag = tag + self.langs = langs + + def __str__(self): + res = "DEF_SCRIPT" + if self.name is not None: + res += f' NAME "{self.name}"' + res += f' TAG "{self.tag}"\n\n' + for lang in self.langs: + res += f"{lang}" + res += "END_SCRIPT" + return res + + +class LangSysDefinition(Statement): + def __init__(self, name, tag, features, location=None): + Statement.__init__(self, location) + self.name = name + self.tag = tag + self.features = features + + def __str__(self): + res = "DEF_LANGSYS" + if self.name is not None: + res += f' NAME "{self.name}"' + res += f' TAG "{self.tag}"\n\n' + for feature in self.features: + res += f"{feature}" + res += "END_LANGSYS\n" + return res + + +class FeatureDefinition(Statement): + def __init__(self, name, tag, lookups, location=None): + Statement.__init__(self, location) + self.name = name + self.tag = tag + self.lookups = lookups + + def __str__(self): + res = f'DEF_FEATURE NAME "{self.name}" TAG "{self.tag}"\n' + res += " " + " ".join(f'LOOKUP "{l}"' for l in self.lookups) + "\n" + res += "END_FEATURE\n" + return res + + +class LookupDefinition(Statement): + def __init__( + self, + name, + process_base, + process_marks, + mark_glyph_set, + direction, + reversal, + comments, + context, + sub, + pos, + location=None, + ): + Statement.__init__(self, location) + self.name = name + self.process_base = process_base + self.process_marks = process_marks + self.mark_glyph_set = mark_glyph_set + self.direction = direction + self.reversal = reversal + self.comments = comments + self.context = context + self.sub = sub + self.pos = pos + + def __str__(self): + res = f'DEF_LOOKUP "{self.name}"' + res += f' {self.process_base and "PROCESS_BASE" or "SKIP_BASE"}' + if self.process_marks: + res += " PROCESS_MARKS " + if self.mark_glyph_set: + res += f'MARK_GLYPH_SET "{self.mark_glyph_set}"' + elif isinstance(self.process_marks, str): + res += f'"{self.process_marks}"' + else: + res += "ALL" + else: + res += " SKIP_MARKS" + if self.direction is not None: + res += f" DIRECTION {self.direction}" + if self.reversal: + res += " REVERSAL" + if self.comments is not None: + comments = self.comments.replace("\n", r"\n") + res += f'\nCOMMENTS "{comments}"' + if self.context: + res += "\n" + "\n".join(str(c) for c in self.context) + else: + res += "\nIN_CONTEXT\nEND_CONTEXT" + if self.sub: + res += f"\n{self.sub}" + if self.pos: + res += f"\n{self.pos}" + return res + + +class SubstitutionDefinition(Statement): + def __init__(self, mapping, location=None): + Statement.__init__(self, location) + self.mapping = mapping + + def __str__(self): + res = "AS_SUBSTITUTION\n" + for src, dst in self.mapping.items(): + src = "".join(str(s) for s in src) + dst = "".join(str(d) for d in dst) + res += f"SUB{src}\nWITH{dst}\nEND_SUB\n" + res += "END_SUBSTITUTION" + return res + + +class SubstitutionSingleDefinition(SubstitutionDefinition): + pass + + +class SubstitutionMultipleDefinition(SubstitutionDefinition): + pass + + +class SubstitutionLigatureDefinition(SubstitutionDefinition): + pass + + +class SubstitutionAlternateDefinition(SubstitutionDefinition): + pass + + +class SubstitutionReverseChainingSingleDefinition(SubstitutionDefinition): + pass + + +class PositionAttachDefinition(Statement): + def __init__(self, coverage, coverage_to, location=None): + Statement.__init__(self, location) + self.coverage = coverage + self.coverage_to = coverage_to + + def __str__(self): + coverage = "".join(str(c) for c in self.coverage) + res = f"AS_POSITION\nATTACH{coverage}\nTO" + for coverage, anchor in self.coverage_to: + coverage = "".join(str(c) for c in coverage) + res += f'{coverage} AT ANCHOR "{anchor}"' + res += "\nEND_ATTACH\nEND_POSITION" + return res + + +class PositionAttachCursiveDefinition(Statement): + def __init__(self, coverages_exit, coverages_enter, location=None): + Statement.__init__(self, location) + self.coverages_exit = coverages_exit + self.coverages_enter = coverages_enter + + def __str__(self): + res = "AS_POSITION\nATTACH_CURSIVE" + for coverage in self.coverages_exit: + coverage = "".join(str(c) for c in coverage) + res += f"\nEXIT {coverage}" + for coverage in self.coverages_enter: + coverage = "".join(str(c) for c in coverage) + res += f"\nENTER {coverage}" + res += "\nEND_ATTACH\nEND_POSITION" + return res + + +class PositionAdjustPairDefinition(Statement): + def __init__(self, coverages_1, coverages_2, adjust_pair, location=None): + Statement.__init__(self, location) + self.coverages_1 = coverages_1 + self.coverages_2 = coverages_2 + self.adjust_pair = adjust_pair + + def __str__(self): + res = "AS_POSITION\nADJUST_PAIR\n" + for coverage in self.coverages_1: + coverage = " ".join(str(c) for c in coverage) + res += f" FIRST {coverage}" + res += "\n" + for coverage in self.coverages_2: + coverage = " ".join(str(c) for c in coverage) + res += f" SECOND {coverage}" + res += "\n" + for (id_1, id_2), (pos_1, pos_2) in self.adjust_pair.items(): + res += f" {id_1} {id_2} BY{pos_1}{pos_2}\n" + res += "\nEND_ADJUST\nEND_POSITION" + return res + + +class PositionAdjustSingleDefinition(Statement): + def __init__(self, adjust_single, location=None): + Statement.__init__(self, location) + self.adjust_single = adjust_single + + def __str__(self): + res = "AS_POSITION\nADJUST_SINGLE" + for coverage, pos in self.adjust_single: + coverage = "".join(str(c) for c in coverage) + res += f"{coverage} BY{pos}" + res += "\nEND_ADJUST\nEND_POSITION" + return res + + +class ContextDefinition(Statement): + def __init__(self, ex_or_in, left=None, right=None, location=None): + Statement.__init__(self, location) + self.ex_or_in = ex_or_in + self.left = left if left is not None else [] + self.right = right if right is not None else [] + + def __str__(self): + res = self.ex_or_in + "\n" + for coverage in self.left: + coverage = "".join(str(c) for c in coverage) + res += f" LEFT{coverage}\n" + for coverage in self.right: + coverage = "".join(str(c) for c in coverage) + res += f" RIGHT{coverage}\n" + res += "END_CONTEXT" + return res + + +class AnchorDefinition(Statement): + def __init__(self, name, gid, glyph_name, component, locked, pos, location=None): + Statement.__init__(self, location) + self.name = name + self.gid = gid + self.glyph_name = glyph_name + self.component = component + self.locked = locked + self.pos = pos + + def __str__(self): + locked = self.locked and " LOCKED" or "" + return ( + f'DEF_ANCHOR "{self.name}"' + f" ON {self.gid}" + f" GLYPH {self.glyph_name}" + f" COMPONENT {self.component}" + f"{locked}" + f" AT {self.pos} END_ANCHOR" + ) + + +class SettingDefinition(Statement): + def __init__(self, name, value, location=None): + Statement.__init__(self, location) + self.name = name + self.value = value + + def __str__(self): + if self.value is True: + return f"{self.name}" + if isinstance(self.value, (tuple, list)): + value = " ".join(str(v) for v in self.value) + return f"{self.name} {value}" + return f"{self.name} {self.value}" diff --git a/.venv/lib/python3.11/site-packages/fontTools/voltLib/error.py b/.venv/lib/python3.11/site-packages/fontTools/voltLib/error.py new file mode 100644 index 0000000000000000000000000000000000000000..c51d3b8fdc45afdb7bafbeb13a951264e0228985 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/voltLib/error.py @@ -0,0 +1,12 @@ +class VoltLibError(Exception): + def __init__(self, message, location): + Exception.__init__(self, message) + self.location = location + + def __str__(self): + message = Exception.__str__(self) + if self.location: + path, line, column = self.location + return "%s:%d:%d: %s" % (path, line, column, message) + else: + return message diff --git a/.venv/lib/python3.11/site-packages/fontTools/voltLib/lexer.py b/.venv/lib/python3.11/site-packages/fontTools/voltLib/lexer.py new file mode 100644 index 0000000000000000000000000000000000000000..706b21bbb19717a32025e505c3ae4a2e5f2154ec --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/voltLib/lexer.py @@ -0,0 +1,99 @@ +from fontTools.voltLib.error import VoltLibError + + +class Lexer(object): + NUMBER = "NUMBER" + STRING = "STRING" + NAME = "NAME" + NEWLINE = "NEWLINE" + + CHAR_WHITESPACE_ = " \t" + CHAR_NEWLINE_ = "\r\n" + CHAR_DIGIT_ = "0123456789" + CHAR_UC_LETTER_ = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + CHAR_LC_LETTER_ = "abcdefghijklmnopqrstuvwxyz" + CHAR_UNDERSCORE_ = "_" + CHAR_PERIOD_ = "." + CHAR_NAME_START_ = ( + CHAR_UC_LETTER_ + CHAR_LC_LETTER_ + CHAR_PERIOD_ + CHAR_UNDERSCORE_ + ) + CHAR_NAME_CONTINUATION_ = CHAR_NAME_START_ + CHAR_DIGIT_ + + def __init__(self, text, filename): + self.filename_ = filename + self.line_ = 1 + self.pos_ = 0 + self.line_start_ = 0 + self.text_ = text + self.text_length_ = len(text) + + def __iter__(self): + return self + + def next(self): # Python 2 + return self.__next__() + + def __next__(self): # Python 3 + while True: + token_type, token, location = self.next_() + if token_type not in {Lexer.NEWLINE}: + return (token_type, token, location) + + def location_(self): + column = self.pos_ - self.line_start_ + 1 + return (self.filename_ or "<volt>", self.line_, column) + + def next_(self): + self.scan_over_(Lexer.CHAR_WHITESPACE_) + location = self.location_() + start = self.pos_ + text = self.text_ + limit = len(text) + if start >= limit: + raise StopIteration() + cur_char = text[start] + next_char = text[start + 1] if start + 1 < limit else None + + if cur_char == "\n": + self.pos_ += 1 + self.line_ += 1 + self.line_start_ = self.pos_ + return (Lexer.NEWLINE, None, location) + if cur_char == "\r": + self.pos_ += 2 if next_char == "\n" else 1 + self.line_ += 1 + self.line_start_ = self.pos_ + return (Lexer.NEWLINE, None, location) + if cur_char == '"': + self.pos_ += 1 + self.scan_until_('"\r\n') + if self.pos_ < self.text_length_ and self.text_[self.pos_] == '"': + self.pos_ += 1 + return (Lexer.STRING, text[start + 1 : self.pos_ - 1], location) + else: + raise VoltLibError("Expected '\"' to terminate string", location) + if cur_char in Lexer.CHAR_NAME_START_: + self.pos_ += 1 + self.scan_over_(Lexer.CHAR_NAME_CONTINUATION_) + token = text[start : self.pos_] + return (Lexer.NAME, token, location) + if cur_char in Lexer.CHAR_DIGIT_: + self.scan_over_(Lexer.CHAR_DIGIT_) + return (Lexer.NUMBER, int(text[start : self.pos_], 10), location) + if cur_char == "-" and next_char in Lexer.CHAR_DIGIT_: + self.pos_ += 1 + self.scan_over_(Lexer.CHAR_DIGIT_) + return (Lexer.NUMBER, int(text[start : self.pos_], 10), location) + raise VoltLibError("Unexpected character: '%s'" % cur_char, location) + + def scan_over_(self, valid): + p = self.pos_ + while p < self.text_length_ and self.text_[p] in valid: + p += 1 + self.pos_ = p + + def scan_until_(self, stop_at): + p = self.pos_ + while p < self.text_length_ and self.text_[p] not in stop_at: + p += 1 + self.pos_ = p diff --git a/.venv/lib/python3.11/site-packages/fontTools/voltLib/parser.py b/.venv/lib/python3.11/site-packages/fontTools/voltLib/parser.py new file mode 100644 index 0000000000000000000000000000000000000000..31a76e69bff9c2c74e79463e91684f6da605d119 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/voltLib/parser.py @@ -0,0 +1,664 @@ +import fontTools.voltLib.ast as ast +from fontTools.voltLib.lexer import Lexer +from fontTools.voltLib.error import VoltLibError +from io import open + +PARSE_FUNCS = { + "DEF_GLYPH": "parse_def_glyph_", + "DEF_GROUP": "parse_def_group_", + "DEF_SCRIPT": "parse_def_script_", + "DEF_LOOKUP": "parse_def_lookup_", + "DEF_ANCHOR": "parse_def_anchor_", + "GRID_PPEM": "parse_ppem_", + "PRESENTATION_PPEM": "parse_ppem_", + "PPOSITIONING_PPEM": "parse_ppem_", + "COMPILER_USEEXTENSIONLOOKUPS": "parse_noarg_option_", + "COMPILER_USEPAIRPOSFORMAT2": "parse_noarg_option_", + "CMAP_FORMAT": "parse_cmap_format", + "DO_NOT_TOUCH_CMAP": "parse_noarg_option_", +} + + +class Parser(object): + def __init__(self, path): + self.doc_ = ast.VoltFile() + self.glyphs_ = OrderedSymbolTable() + self.groups_ = SymbolTable() + self.anchors_ = {} # dictionary of SymbolTable() keyed by glyph + self.scripts_ = SymbolTable() + self.langs_ = SymbolTable() + self.lookups_ = SymbolTable() + self.next_token_type_, self.next_token_ = (None, None) + self.next_token_location_ = None + self.make_lexer_(path) + self.advance_lexer_() + + def make_lexer_(self, file_or_path): + if hasattr(file_or_path, "read"): + filename = getattr(file_or_path, "name", None) + data = file_or_path.read() + else: + filename = file_or_path + with open(file_or_path, "r") as f: + data = f.read() + self.lexer_ = Lexer(data, filename) + + def parse(self): + statements = self.doc_.statements + while self.next_token_type_ is not None: + self.advance_lexer_() + if self.cur_token_ in PARSE_FUNCS.keys(): + func = getattr(self, PARSE_FUNCS[self.cur_token_]) + statements.append(func()) + elif self.is_cur_keyword_("END"): + break + else: + raise VoltLibError( + "Expected " + ", ".join(sorted(PARSE_FUNCS.keys())), + self.cur_token_location_, + ) + return self.doc_ + + def parse_def_glyph_(self): + assert self.is_cur_keyword_("DEF_GLYPH") + location = self.cur_token_location_ + name = self.expect_string_() + self.expect_keyword_("ID") + gid = self.expect_number_() + if gid < 0: + raise VoltLibError("Invalid glyph ID", self.cur_token_location_) + gunicode = None + if self.next_token_ == "UNICODE": + self.expect_keyword_("UNICODE") + gunicode = [self.expect_number_()] + if gunicode[0] < 0: + raise VoltLibError("Invalid glyph UNICODE", self.cur_token_location_) + elif self.next_token_ == "UNICODEVALUES": + self.expect_keyword_("UNICODEVALUES") + gunicode = self.parse_unicode_values_() + gtype = None + if self.next_token_ == "TYPE": + self.expect_keyword_("TYPE") + gtype = self.expect_name_() + assert gtype in ("BASE", "LIGATURE", "MARK", "COMPONENT") + components = None + if self.next_token_ == "COMPONENTS": + self.expect_keyword_("COMPONENTS") + components = self.expect_number_() + self.expect_keyword_("END_GLYPH") + if self.glyphs_.resolve(name) is not None: + raise VoltLibError( + 'Glyph "%s" (gid %i) already defined' % (name, gid), location + ) + def_glyph = ast.GlyphDefinition( + name, gid, gunicode, gtype, components, location=location + ) + self.glyphs_.define(name, def_glyph) + return def_glyph + + def parse_def_group_(self): + assert self.is_cur_keyword_("DEF_GROUP") + location = self.cur_token_location_ + name = self.expect_string_() + enum = None + if self.next_token_ == "ENUM": + enum = self.parse_enum_() + self.expect_keyword_("END_GROUP") + if self.groups_.resolve(name) is not None: + raise VoltLibError( + 'Glyph group "%s" already defined, ' + "group names are case insensitive" % name, + location, + ) + def_group = ast.GroupDefinition(name, enum, location=location) + self.groups_.define(name, def_group) + return def_group + + def parse_def_script_(self): + assert self.is_cur_keyword_("DEF_SCRIPT") + location = self.cur_token_location_ + name = None + if self.next_token_ == "NAME": + self.expect_keyword_("NAME") + name = self.expect_string_() + self.expect_keyword_("TAG") + tag = self.expect_string_() + if self.scripts_.resolve(tag) is not None: + raise VoltLibError( + 'Script "%s" already defined, ' + "script tags are case insensitive" % tag, + location, + ) + self.langs_.enter_scope() + langs = [] + while self.next_token_ != "END_SCRIPT": + self.advance_lexer_() + lang = self.parse_langsys_() + self.expect_keyword_("END_LANGSYS") + if self.langs_.resolve(lang.tag) is not None: + raise VoltLibError( + 'Language "%s" already defined in script "%s", ' + "language tags are case insensitive" % (lang.tag, tag), + location, + ) + self.langs_.define(lang.tag, lang) + langs.append(lang) + self.expect_keyword_("END_SCRIPT") + self.langs_.exit_scope() + def_script = ast.ScriptDefinition(name, tag, langs, location=location) + self.scripts_.define(tag, def_script) + return def_script + + def parse_langsys_(self): + assert self.is_cur_keyword_("DEF_LANGSYS") + location = self.cur_token_location_ + name = None + if self.next_token_ == "NAME": + self.expect_keyword_("NAME") + name = self.expect_string_() + self.expect_keyword_("TAG") + tag = self.expect_string_() + features = [] + while self.next_token_ != "END_LANGSYS": + self.advance_lexer_() + feature = self.parse_feature_() + self.expect_keyword_("END_FEATURE") + features.append(feature) + def_langsys = ast.LangSysDefinition(name, tag, features, location=location) + return def_langsys + + def parse_feature_(self): + assert self.is_cur_keyword_("DEF_FEATURE") + location = self.cur_token_location_ + self.expect_keyword_("NAME") + name = self.expect_string_() + self.expect_keyword_("TAG") + tag = self.expect_string_() + lookups = [] + while self.next_token_ != "END_FEATURE": + # self.advance_lexer_() + self.expect_keyword_("LOOKUP") + lookup = self.expect_string_() + lookups.append(lookup) + feature = ast.FeatureDefinition(name, tag, lookups, location=location) + return feature + + def parse_def_lookup_(self): + assert self.is_cur_keyword_("DEF_LOOKUP") + location = self.cur_token_location_ + name = self.expect_string_() + if not name[0].isalpha(): + raise VoltLibError( + 'Lookup name "%s" must start with a letter' % name, location + ) + if self.lookups_.resolve(name) is not None: + raise VoltLibError( + 'Lookup "%s" already defined, ' + "lookup names are case insensitive" % name, + location, + ) + process_base = True + if self.next_token_ == "PROCESS_BASE": + self.advance_lexer_() + elif self.next_token_ == "SKIP_BASE": + self.advance_lexer_() + process_base = False + process_marks = True + mark_glyph_set = None + if self.next_token_ == "PROCESS_MARKS": + self.advance_lexer_() + if self.next_token_ == "MARK_GLYPH_SET": + self.advance_lexer_() + mark_glyph_set = self.expect_string_() + elif self.next_token_ == "ALL": + self.advance_lexer_() + elif self.next_token_ == "NONE": + self.advance_lexer_() + process_marks = False + elif self.next_token_type_ == Lexer.STRING: + process_marks = self.expect_string_() + else: + raise VoltLibError( + "Expected ALL, NONE, MARK_GLYPH_SET or an ID. " + "Got %s" % (self.next_token_type_), + location, + ) + elif self.next_token_ == "SKIP_MARKS": + self.advance_lexer_() + process_marks = False + direction = None + if self.next_token_ == "DIRECTION": + self.expect_keyword_("DIRECTION") + direction = self.expect_name_() + assert direction in ("LTR", "RTL") + reversal = None + if self.next_token_ == "REVERSAL": + self.expect_keyword_("REVERSAL") + reversal = True + comments = None + if self.next_token_ == "COMMENTS": + self.expect_keyword_("COMMENTS") + comments = self.expect_string_().replace(r"\n", "\n") + context = [] + while self.next_token_ in ("EXCEPT_CONTEXT", "IN_CONTEXT"): + context = self.parse_context_() + as_pos_or_sub = self.expect_name_() + sub = None + pos = None + if as_pos_or_sub == "AS_SUBSTITUTION": + sub = self.parse_substitution_(reversal) + elif as_pos_or_sub == "AS_POSITION": + pos = self.parse_position_() + else: + raise VoltLibError( + "Expected AS_SUBSTITUTION or AS_POSITION. " "Got %s" % (as_pos_or_sub), + location, + ) + def_lookup = ast.LookupDefinition( + name, + process_base, + process_marks, + mark_glyph_set, + direction, + reversal, + comments, + context, + sub, + pos, + location=location, + ) + self.lookups_.define(name, def_lookup) + return def_lookup + + def parse_context_(self): + location = self.cur_token_location_ + contexts = [] + while self.next_token_ in ("EXCEPT_CONTEXT", "IN_CONTEXT"): + side = None + coverage = None + ex_or_in = self.expect_name_() + # side_contexts = [] # XXX + if self.next_token_ != "END_CONTEXT": + left = [] + right = [] + while self.next_token_ in ("LEFT", "RIGHT"): + side = self.expect_name_() + coverage = self.parse_coverage_() + if side == "LEFT": + left.append(coverage) + else: + right.append(coverage) + self.expect_keyword_("END_CONTEXT") + context = ast.ContextDefinition( + ex_or_in, left, right, location=location + ) + contexts.append(context) + else: + self.expect_keyword_("END_CONTEXT") + return contexts + + def parse_substitution_(self, reversal): + assert self.is_cur_keyword_("AS_SUBSTITUTION") + location = self.cur_token_location_ + src = [] + dest = [] + if self.next_token_ != "SUB": + raise VoltLibError("Expected SUB", location) + while self.next_token_ == "SUB": + self.expect_keyword_("SUB") + src.append(self.parse_coverage_()) + self.expect_keyword_("WITH") + dest.append(self.parse_coverage_()) + self.expect_keyword_("END_SUB") + self.expect_keyword_("END_SUBSTITUTION") + max_src = max([len(cov) for cov in src]) + max_dest = max([len(cov) for cov in dest]) + + # many to many or mixed is invalid + if max_src > 1 and max_dest > 1: + raise VoltLibError("Invalid substitution type", location) + + mapping = dict(zip(tuple(src), tuple(dest))) + if max_src == 1 and max_dest == 1: + # Alternate substitutions are represented by adding multiple + # substitutions for the same glyph, so we detect that here + glyphs = [x.glyphSet() for cov in src for x in cov] # flatten src + if len(set(glyphs)) != len(glyphs): # src has duplicates + sub = ast.SubstitutionAlternateDefinition(mapping, location=location) + else: + if reversal: + # Reversal is valid only for single glyph substitutions + # and VOLT ignores it otherwise. + sub = ast.SubstitutionReverseChainingSingleDefinition( + mapping, location=location + ) + else: + sub = ast.SubstitutionSingleDefinition(mapping, location=location) + elif max_src == 1 and max_dest > 1: + sub = ast.SubstitutionMultipleDefinition(mapping, location=location) + elif max_src > 1 and max_dest == 1: + sub = ast.SubstitutionLigatureDefinition(mapping, location=location) + return sub + + def parse_position_(self): + assert self.is_cur_keyword_("AS_POSITION") + location = self.cur_token_location_ + pos_type = self.expect_name_() + if pos_type not in ("ATTACH", "ATTACH_CURSIVE", "ADJUST_PAIR", "ADJUST_SINGLE"): + raise VoltLibError( + "Expected ATTACH, ATTACH_CURSIVE, ADJUST_PAIR, ADJUST_SINGLE", location + ) + if pos_type == "ATTACH": + position = self.parse_attach_() + elif pos_type == "ATTACH_CURSIVE": + position = self.parse_attach_cursive_() + elif pos_type == "ADJUST_PAIR": + position = self.parse_adjust_pair_() + elif pos_type == "ADJUST_SINGLE": + position = self.parse_adjust_single_() + self.expect_keyword_("END_POSITION") + return position + + def parse_attach_(self): + assert self.is_cur_keyword_("ATTACH") + location = self.cur_token_location_ + coverage = self.parse_coverage_() + coverage_to = [] + self.expect_keyword_("TO") + while self.next_token_ != "END_ATTACH": + cov = self.parse_coverage_() + self.expect_keyword_("AT") + self.expect_keyword_("ANCHOR") + anchor_name = self.expect_string_() + coverage_to.append((cov, anchor_name)) + self.expect_keyword_("END_ATTACH") + position = ast.PositionAttachDefinition( + coverage, coverage_to, location=location + ) + return position + + def parse_attach_cursive_(self): + assert self.is_cur_keyword_("ATTACH_CURSIVE") + location = self.cur_token_location_ + coverages_exit = [] + coverages_enter = [] + while self.next_token_ != "ENTER": + self.expect_keyword_("EXIT") + coverages_exit.append(self.parse_coverage_()) + while self.next_token_ != "END_ATTACH": + self.expect_keyword_("ENTER") + coverages_enter.append(self.parse_coverage_()) + self.expect_keyword_("END_ATTACH") + position = ast.PositionAttachCursiveDefinition( + coverages_exit, coverages_enter, location=location + ) + return position + + def parse_adjust_pair_(self): + assert self.is_cur_keyword_("ADJUST_PAIR") + location = self.cur_token_location_ + coverages_1 = [] + coverages_2 = [] + adjust_pair = {} + while self.next_token_ == "FIRST": + self.advance_lexer_() + coverage_1 = self.parse_coverage_() + coverages_1.append(coverage_1) + while self.next_token_ == "SECOND": + self.advance_lexer_() + coverage_2 = self.parse_coverage_() + coverages_2.append(coverage_2) + while self.next_token_ != "END_ADJUST": + id_1 = self.expect_number_() + id_2 = self.expect_number_() + self.expect_keyword_("BY") + pos_1 = self.parse_pos_() + pos_2 = self.parse_pos_() + adjust_pair[(id_1, id_2)] = (pos_1, pos_2) + self.expect_keyword_("END_ADJUST") + position = ast.PositionAdjustPairDefinition( + coverages_1, coverages_2, adjust_pair, location=location + ) + return position + + def parse_adjust_single_(self): + assert self.is_cur_keyword_("ADJUST_SINGLE") + location = self.cur_token_location_ + adjust_single = [] + while self.next_token_ != "END_ADJUST": + coverages = self.parse_coverage_() + self.expect_keyword_("BY") + pos = self.parse_pos_() + adjust_single.append((coverages, pos)) + self.expect_keyword_("END_ADJUST") + position = ast.PositionAdjustSingleDefinition(adjust_single, location=location) + return position + + def parse_def_anchor_(self): + assert self.is_cur_keyword_("DEF_ANCHOR") + location = self.cur_token_location_ + name = self.expect_string_() + self.expect_keyword_("ON") + gid = self.expect_number_() + self.expect_keyword_("GLYPH") + glyph_name = self.expect_name_() + self.expect_keyword_("COMPONENT") + component = self.expect_number_() + # check for duplicate anchor names on this glyph + if glyph_name in self.anchors_: + anchor = self.anchors_[glyph_name].resolve(name) + if anchor is not None and anchor.component == component: + raise VoltLibError( + 'Anchor "%s" already defined, ' + "anchor names are case insensitive" % name, + location, + ) + if self.next_token_ == "LOCKED": + locked = True + self.advance_lexer_() + else: + locked = False + self.expect_keyword_("AT") + pos = self.parse_pos_() + self.expect_keyword_("END_ANCHOR") + anchor = ast.AnchorDefinition( + name, gid, glyph_name, component, locked, pos, location=location + ) + if glyph_name not in self.anchors_: + self.anchors_[glyph_name] = SymbolTable() + self.anchors_[glyph_name].define(name, anchor) + return anchor + + def parse_adjust_by_(self): + self.advance_lexer_() + assert self.is_cur_keyword_("ADJUST_BY") + adjustment = self.expect_number_() + self.expect_keyword_("AT") + size = self.expect_number_() + return adjustment, size + + def parse_pos_(self): + # VOLT syntax doesn't seem to take device Y advance + self.advance_lexer_() + location = self.cur_token_location_ + assert self.is_cur_keyword_("POS"), location + adv = None + dx = None + dy = None + adv_adjust_by = {} + dx_adjust_by = {} + dy_adjust_by = {} + if self.next_token_ == "ADV": + self.advance_lexer_() + adv = self.expect_number_() + while self.next_token_ == "ADJUST_BY": + adjustment, size = self.parse_adjust_by_() + adv_adjust_by[size] = adjustment + if self.next_token_ == "DX": + self.advance_lexer_() + dx = self.expect_number_() + while self.next_token_ == "ADJUST_BY": + adjustment, size = self.parse_adjust_by_() + dx_adjust_by[size] = adjustment + if self.next_token_ == "DY": + self.advance_lexer_() + dy = self.expect_number_() + while self.next_token_ == "ADJUST_BY": + adjustment, size = self.parse_adjust_by_() + dy_adjust_by[size] = adjustment + self.expect_keyword_("END_POS") + return ast.Pos(adv, dx, dy, adv_adjust_by, dx_adjust_by, dy_adjust_by) + + def parse_unicode_values_(self): + location = self.cur_token_location_ + try: + unicode_values = self.expect_string_().split(",") + unicode_values = [int(uni[2:], 16) for uni in unicode_values if uni != ""] + except ValueError as err: + raise VoltLibError(str(err), location) + return unicode_values if unicode_values != [] else None + + def parse_enum_(self): + self.expect_keyword_("ENUM") + location = self.cur_token_location_ + enum = ast.Enum(self.parse_coverage_(), location=location) + self.expect_keyword_("END_ENUM") + return enum + + def parse_coverage_(self): + coverage = [] + location = self.cur_token_location_ + while self.next_token_ in ("GLYPH", "GROUP", "RANGE", "ENUM"): + if self.next_token_ == "ENUM": + enum = self.parse_enum_() + coverage.append(enum) + elif self.next_token_ == "GLYPH": + self.expect_keyword_("GLYPH") + name = self.expect_string_() + coverage.append(ast.GlyphName(name, location=location)) + elif self.next_token_ == "GROUP": + self.expect_keyword_("GROUP") + name = self.expect_string_() + coverage.append(ast.GroupName(name, self, location=location)) + elif self.next_token_ == "RANGE": + self.expect_keyword_("RANGE") + start = self.expect_string_() + self.expect_keyword_("TO") + end = self.expect_string_() + coverage.append(ast.Range(start, end, self, location=location)) + return tuple(coverage) + + def resolve_group(self, group_name): + return self.groups_.resolve(group_name) + + def glyph_range(self, start, end): + return self.glyphs_.range(start, end) + + def parse_ppem_(self): + location = self.cur_token_location_ + ppem_name = self.cur_token_ + value = self.expect_number_() + setting = ast.SettingDefinition(ppem_name, value, location=location) + return setting + + def parse_noarg_option_(self): + location = self.cur_token_location_ + name = self.cur_token_ + value = True + setting = ast.SettingDefinition(name, value, location=location) + return setting + + def parse_cmap_format(self): + location = self.cur_token_location_ + name = self.cur_token_ + value = (self.expect_number_(), self.expect_number_(), self.expect_number_()) + setting = ast.SettingDefinition(name, value, location=location) + return setting + + def is_cur_keyword_(self, k): + return (self.cur_token_type_ is Lexer.NAME) and (self.cur_token_ == k) + + def expect_string_(self): + self.advance_lexer_() + if self.cur_token_type_ is not Lexer.STRING: + raise VoltLibError("Expected a string", self.cur_token_location_) + return self.cur_token_ + + def expect_keyword_(self, keyword): + self.advance_lexer_() + if self.cur_token_type_ is Lexer.NAME and self.cur_token_ == keyword: + return self.cur_token_ + raise VoltLibError('Expected "%s"' % keyword, self.cur_token_location_) + + def expect_name_(self): + self.advance_lexer_() + if self.cur_token_type_ is Lexer.NAME: + return self.cur_token_ + raise VoltLibError("Expected a name", self.cur_token_location_) + + def expect_number_(self): + self.advance_lexer_() + if self.cur_token_type_ is not Lexer.NUMBER: + raise VoltLibError("Expected a number", self.cur_token_location_) + return self.cur_token_ + + def advance_lexer_(self): + self.cur_token_type_, self.cur_token_, self.cur_token_location_ = ( + self.next_token_type_, + self.next_token_, + self.next_token_location_, + ) + try: + if self.is_cur_keyword_("END"): + raise StopIteration + ( + self.next_token_type_, + self.next_token_, + self.next_token_location_, + ) = self.lexer_.next() + except StopIteration: + self.next_token_type_, self.next_token_ = (None, None) + + +class SymbolTable(object): + def __init__(self): + self.scopes_ = [{}] + + def enter_scope(self): + self.scopes_.append({}) + + def exit_scope(self): + self.scopes_.pop() + + def define(self, name, item): + self.scopes_[-1][name] = item + + def resolve(self, name, case_insensitive=True): + for scope in reversed(self.scopes_): + item = scope.get(name) + if item: + return item + if case_insensitive: + for key in scope: + if key.lower() == name.lower(): + return scope[key] + return None + + +class OrderedSymbolTable(SymbolTable): + def __init__(self): + self.scopes_ = [{}] + + def enter_scope(self): + self.scopes_.append({}) + + def resolve(self, name, case_insensitive=False): + SymbolTable.resolve(self, name, case_insensitive=case_insensitive) + + def range(self, start, end): + for scope in reversed(self.scopes_): + if start in scope and end in scope: + start_idx = list(scope.keys()).index(start) + end_idx = list(scope.keys()).index(end) + return list(scope.keys())[start_idx : end_idx + 1] + return None diff --git a/.venv/lib/python3.11/site-packages/fontTools/voltLib/voltToFea.py b/.venv/lib/python3.11/site-packages/fontTools/voltLib/voltToFea.py new file mode 100644 index 0000000000000000000000000000000000000000..d552f4b52a80855338ef8ea7308a83ee051fc7a5 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fontTools/voltLib/voltToFea.py @@ -0,0 +1,911 @@ +"""\ +MS VOLT ``.vtp`` to AFDKO ``.fea`` OpenType Layout converter. + +Usage +----- + +To convert a VTP project file: + + +.. code-block:: sh + + $ fonttools voltLib.voltToFea input.vtp output.fea + +It is also possible convert font files with `TSIV` table (as saved from Volt), +in this case the glyph names used in the Volt project will be mapped to the +actual glyph names in the font files when written to the feature file: + +.. code-block:: sh + + $ fonttools voltLib.voltToFea input.ttf output.fea + +The ``--quiet`` option can be used to suppress warnings. + +The ``--traceback`` can be used to get Python traceback in case of exceptions, +instead of suppressing the traceback. + + +Limitations +----------- + +* Not all VOLT features are supported, the script will error if it it + encounters something it does not understand. Please report an issue if this + happens. +* AFDKO feature file syntax for mark positioning is awkward and does not allow + setting the mark coverage. It also defines mark anchors globally, as a result + some mark positioning lookups might cover many marks than what was in the VOLT + file. This should not be an issue in practice, but if it is then the only way + is to modify the VOLT file or the generated feature file manually to use unique + mark anchors for each lookup. +* VOLT allows subtable breaks in any lookup type, but AFDKO feature file + implementations vary in their support; currently AFDKO’s makeOTF supports + subtable breaks in pair positioning lookups only, while FontTools’ feaLib + support it for most substitution lookups and only some positioning lookups. +""" + +import logging +import re +from io import StringIO +from graphlib import TopologicalSorter + +from fontTools.feaLib import ast +from fontTools.ttLib import TTFont, TTLibError +from fontTools.voltLib import ast as VAst +from fontTools.voltLib.parser import Parser as VoltParser + +log = logging.getLogger("fontTools.voltLib.voltToFea") + +TABLES = ["GDEF", "GSUB", "GPOS"] + + +def _flatten_group(group): + ret = [] + if isinstance(group, (tuple, list)): + for item in group: + ret.extend(_flatten_group(item)) + elif hasattr(group, "enum"): + ret.extend(_flatten_group(group.enum)) + else: + ret.append(group) + return ret + + +# Topologically sort of group definitions to ensure that all groups are defined +# before they are referenced. This is necessary because FEA requires it but +# VOLT does not, see below. +def sort_groups(groups): + group_map = {group.name.lower(): group for group in groups} + graph = { + group.name.lower(): [ + x.group.lower() + for x in _flatten_group(group) + if isinstance(x, VAst.GroupName) + ] + for group in groups + } + sorter = TopologicalSorter(graph) + return [group_map[name] for name in sorter.static_order()] + + +class Lookup(ast.LookupBlock): + def __init__(self, name, use_extension=False, location=None): + super().__init__(name, use_extension, location) + self.chained = [] + + +class VoltToFea: + _NOT_LOOKUP_NAME_RE = re.compile(r"[^A-Za-z_0-9.]") + _NOT_CLASS_NAME_RE = re.compile(r"[^A-Za-z_0-9.\-]") + + def __init__(self, file_or_path, font=None): + if isinstance(file_or_path, VAst.VoltFile): + self._doc, self._file_or_path = file_or_path, None + else: + self._doc, self._file_or_path = None, file_or_path + self._font = font + + self._glyph_map = {} + self._glyph_order = None + + self._gdef = {} + self._glyphclasses = {} + self._features = {} + self._lookups = {} + + self._marks = set() + self._ligatures = {} + + self._markclasses = {} + self._anchors = {} + + self._settings = {} + + self._lookup_names = {} + self._class_names = {} + + def _lookupName(self, name): + if name not in self._lookup_names: + res = self._NOT_LOOKUP_NAME_RE.sub("_", name) + while res in self._lookup_names.values(): + res += "_" + self._lookup_names[name] = res + return self._lookup_names[name] + + def _className(self, name): + if name not in self._class_names: + res = self._NOT_CLASS_NAME_RE.sub("_", name) + while res in self._class_names.values(): + res += "_" + self._class_names[name] = res + return self._class_names[name] + + def _collectStatements(self, doc, tables, ignore_unsupported_settings=False): + # Collect glyph difinitions first, as we need them to map VOLT glyph names to font glyph name. + for statement in doc.statements: + if isinstance(statement, VAst.GlyphDefinition): + self._glyphDefinition(statement) + + # Collect and sort group definitions first, to make sure a group + # definition that references other groups comes after them since VOLT + # does not enforce such ordering, and feature file require it. + groups = [s for s in doc.statements if isinstance(s, VAst.GroupDefinition)] + for group in sort_groups(groups): + self._groupDefinition(group) + + for statement in doc.statements: + if isinstance(statement, VAst.AnchorDefinition): + if "GPOS" in tables: + self._anchorDefinition(statement) + elif isinstance(statement, VAst.SettingDefinition): + self._settingDefinition(statement, ignore_unsupported_settings) + elif isinstance(statement, (VAst.GlyphDefinition, VAst.GroupDefinition)): + pass # Handled above + elif isinstance(statement, VAst.ScriptDefinition): + self._scriptDefinition(statement) + elif not isinstance(statement, VAst.LookupDefinition): + raise NotImplementedError(statement) + + # Lookup definitions need to be handled last as they reference glyph + # and mark classes that might be defined after them. + for statement in doc.statements: + if isinstance(statement, VAst.LookupDefinition): + if statement.pos and "GPOS" not in tables: + continue + if statement.sub and "GSUB" not in tables: + continue + self._lookupDefinition(statement) + + def _buildFeatureFile(self, tables): + doc = ast.FeatureFile() + statements = doc.statements + + if self._glyphclasses: + statements.append(ast.Comment("# Glyph classes")) + statements.extend(self._glyphclasses.values()) + + if self._markclasses: + statements.append(ast.Comment("\n# Mark classes")) + statements.extend(c[1] for c in sorted(self._markclasses.items())) + + if self._lookups: + statements.append(ast.Comment("\n# Lookups")) + for lookup in self._lookups.values(): + statements.extend(lookup.chained) + statements.append(lookup) + + # Prune features + features = self._features.copy() + for feature_tag in features: + scripts = features[feature_tag] + for script_tag in scripts: + langs = scripts[script_tag] + for language_tag in langs: + langs[language_tag] = [ + l for l in langs[language_tag] if l.lower() in self._lookups + ] + scripts[script_tag] = {t: l for t, l in langs.items() if l} + features[feature_tag] = {t: s for t, s in scripts.items() if s} + features = {t: f for t, f in features.items() if f} + + if features: + statements.append(ast.Comment("# Features")) + for feature_tag, scripts in features.items(): + feature = ast.FeatureBlock(feature_tag) + script_tags = sorted(scripts, key=lambda k: 0 if k == "DFLT" else 1) + if feature_tag == "aalt" and len(script_tags) > 1: + log.warning( + "FEA syntax does not allow script statements in 'aalt' feature, " + "so only lookups from the first script will be included." + ) + script_tags = script_tags[:1] + for script_tag in script_tags: + if feature_tag != "aalt": + feature.statements.append(ast.ScriptStatement(script_tag)) + language_tags = sorted( + scripts[script_tag], + key=lambda k: 0 if k == "dflt" else 1, + ) + if feature_tag == "aalt" and len(language_tags) > 1: + log.warning( + "FEA syntax does not allow language statements in 'aalt' feature, " + "so only lookups from the first language will be included." + ) + language_tags = language_tags[:1] + for language_tag in language_tags: + if feature_tag != "aalt": + include_default = True if language_tag == "dflt" else False + feature.statements.append( + ast.LanguageStatement( + language_tag.ljust(4), + include_default=include_default, + ) + ) + for name in scripts[script_tag][language_tag]: + lookup = self._lookups[name.lower()] + lookupref = ast.LookupReferenceStatement(lookup) + feature.statements.append(lookupref) + statements.append(feature) + + if self._gdef and "GDEF" in tables: + classes = [] + for name in ("BASE", "MARK", "LIGATURE", "COMPONENT"): + if name in self._gdef: + classname = "GDEF_" + name.lower() + glyphclass = ast.GlyphClassDefinition(classname, self._gdef[name]) + statements.append(glyphclass) + classes.append(ast.GlyphClassName(glyphclass)) + else: + classes.append(None) + + gdef = ast.TableBlock("GDEF") + gdef.statements.append(ast.GlyphClassDefStatement(*classes)) + statements.append(gdef) + + return doc + + def convert(self, tables=None, ignore_unsupported_settings=False): + if self._doc is None: + self._doc = VoltParser(self._file_or_path).parse() + doc = self._doc + + if tables is None: + tables = TABLES + if self._font is not None: + self._glyph_order = self._font.getGlyphOrder() + + self._collectStatements(doc, tables, ignore_unsupported_settings) + fea = self._buildFeatureFile(tables) + return fea.asFea() + + def _glyphName(self, glyph): + try: + name = glyph.glyph + except AttributeError: + name = glyph + return ast.GlyphName(self._glyph_map.get(name, name)) + + def _groupName(self, group): + try: + name = group.group + except AttributeError: + name = group + return ast.GlyphClassName(self._glyphclasses[name.lower()]) + + def _glyphSet(self, item): + return [ + (self._glyphName(x) if isinstance(x, (str, VAst.GlyphName)) else x) + for x in item.glyphSet() + ] + + def _coverage(self, coverage, flatten=False): + items = [] + for item in coverage: + if isinstance(item, VAst.GlyphName): + items.append(self._glyphName(item)) + elif isinstance(item, VAst.GroupName): + items.append(self._groupName(item)) + elif isinstance(item, VAst.Enum): + item = self._coverage(item.enum, flatten=True) + if flatten: + items.extend(item) + else: + items.append(ast.GlyphClass(item)) + elif isinstance(item, VAst.Range): + item = self._glyphSet(item) + if flatten: + items.extend(item) + else: + items.append(ast.GlyphClass(item)) + else: + raise NotImplementedError(item) + return items + + def _context(self, context): + out = [] + for item in context: + coverage = self._coverage(item, flatten=True) + if len(coverage) > 1: + coverage = ast.GlyphClass(coverage) + else: + coverage = coverage[0] + out.append(coverage) + return out + + def _groupDefinition(self, group): + name = self._className(group.name) + glyphs = self._coverage(group.enum.enum, flatten=True) + glyphclass = ast.GlyphClass(glyphs) + classdef = ast.GlyphClassDefinition(name, glyphclass) + self._glyphclasses[group.name.lower()] = classdef + + def _glyphDefinition(self, glyph): + try: + self._glyph_map[glyph.name] = self._glyph_order[glyph.id] + except TypeError: + pass + + if glyph.type in ("BASE", "MARK", "LIGATURE", "COMPONENT"): + if glyph.type not in self._gdef: + self._gdef[glyph.type] = ast.GlyphClass() + self._gdef[glyph.type].glyphs.append(self._glyphName(glyph.name)) + + if glyph.type == "MARK": + self._marks.add(glyph.name) + elif glyph.type == "LIGATURE": + self._ligatures[glyph.name] = glyph.components + + def _scriptDefinition(self, script): + stag = script.tag + for lang in script.langs: + ltag = lang.tag + for feature in lang.features: + lookups = {l.split("\\")[0]: True for l in feature.lookups} + ftag = feature.tag + if ftag not in self._features: + self._features[ftag] = {} + if stag not in self._features[ftag]: + self._features[ftag][stag] = {} + assert ltag not in self._features[ftag][stag] + self._features[ftag][stag][ltag] = lookups.keys() + + def _settingDefinition(self, setting, ignore_unsupported=False): + if setting.name.startswith("COMPILER_"): + self._settings[setting.name] = setting.value + elif not ignore_unsupported: + log.warning(f"Unsupported setting ignored: {setting.name}") + + def _adjustment(self, adjustment): + adv, dx, dy, adv_adjust_by, dx_adjust_by, dy_adjust_by = adjustment + + adv_device = adv_adjust_by and adv_adjust_by.items() or None + dx_device = dx_adjust_by and dx_adjust_by.items() or None + dy_device = dy_adjust_by and dy_adjust_by.items() or None + + return ast.ValueRecord( + xPlacement=dx, + yPlacement=dy, + xAdvance=adv, + xPlaDevice=dx_device, + yPlaDevice=dy_device, + xAdvDevice=adv_device, + ) + + def _anchor(self, adjustment): + adv, dx, dy, adv_adjust_by, dx_adjust_by, dy_adjust_by = adjustment + + assert not adv_adjust_by + dx_device = dx_adjust_by and dx_adjust_by.items() or None + dy_device = dy_adjust_by and dy_adjust_by.items() or None + + return ast.Anchor( + dx or 0, + dy or 0, + xDeviceTable=dx_device or None, + yDeviceTable=dy_device or None, + ) + + def _anchorDefinition(self, anchordef): + anchorname = anchordef.name + glyphname = anchordef.glyph_name + anchor = self._anchor(anchordef.pos) + + if glyphname not in self._anchors: + self._anchors[glyphname] = {} + if anchorname.startswith("MARK_"): + anchorname = anchorname[:5] + anchorname[5:].lower() + else: + anchorname = anchorname.lower() + if anchorname not in self._anchors[glyphname]: + self._anchors[glyphname][anchorname] = {} + self._anchors[glyphname][anchorname][anchordef.component] = anchor + + def _gposLookup(self, lookup, fealookup): + statements = fealookup.statements + + pos = lookup.pos + if isinstance(pos, VAst.PositionAdjustPairDefinition): + for (idx1, idx2), (pos1, pos2) in pos.adjust_pair.items(): + coverage_1 = pos.coverages_1[idx1 - 1] + coverage_2 = pos.coverages_2[idx2 - 1] + + # If not both are groups, use “enum pos” otherwise makeotf will + # fail. + enumerated = False + for item in coverage_1 + coverage_2: + if not isinstance(item, VAst.GroupName): + enumerated = True + + glyphs1 = self._coverage(coverage_1) + glyphs2 = self._coverage(coverage_2) + record1 = self._adjustment(pos1) + record2 = self._adjustment(pos2) + assert len(glyphs1) == 1 + assert len(glyphs2) == 1 + statements.append( + ast.PairPosStatement( + glyphs1[0], record1, glyphs2[0], record2, enumerated=enumerated + ) + ) + elif isinstance(pos, VAst.PositionAdjustSingleDefinition): + for a, b in pos.adjust_single: + glyphs = self._coverage(a) + record = self._adjustment(b) + assert len(glyphs) == 1 + statements.append( + ast.SinglePosStatement([(glyphs[0], record)], [], [], False) + ) + elif isinstance(pos, VAst.PositionAttachDefinition): + anchors = {} + allmarks = set() + for coverage, anchorname in pos.coverage_to: + # In feature files mark classes are global, but in VOLT they + # are defined per-lookup. If we output mark class definitions + # for all marks that use a given anchor, we might end up with a + # mark used in two different classes in the same lookup, which + # is causes feature file compilation error. + # At the expense of uglier feature code, we make the mark class + # name by appending the current lookup name not the anchor + # name, and output mark class definitions only for marks used + # in this lookup. + classname = self._className(f"{anchorname}.{lookup.name}") + markclass = ast.MarkClass(classname) + + # Anchor names are case-insensitive in VOLT + anchorname = anchorname.lower() + + # We might still end in marks used in two different anchor + # classes, so we filter out already used marks. + marks = set() + for mark in coverage: + marks.update(mark.glyphSet()) + if not marks.isdisjoint(allmarks): + marks.difference_update(allmarks) + if not marks: + continue + allmarks.update(marks) + + for glyphname in marks: + glyph = self._glyphName(glyphname) + anchor = self._anchors[glyphname][f"MARK_{anchorname}"][1] + markdef = ast.MarkClassDefinition(markclass, anchor, glyph) + self._markclasses[(glyphname, classname)] = markdef + + for base in pos.coverage: + for name in base.glyphSet(): + if name not in anchors: + anchors[name] = [] + if (anchorname, classname) not in anchors[name]: + anchors[name].append((anchorname, classname)) + + is_ligature = all(n in self._ligatures for n in anchors) + is_mark = all(n in self._marks for n in anchors) + for name in anchors: + components = 1 + if is_ligature: + components = self._ligatures[name] + + marks = [[] for _ in range(components)] + for mark, classname in anchors[name]: + markclass = ast.MarkClass(classname) + for component in range(1, components + 1): + if component in self._anchors[name][mark]: + anchor = self._anchors[name][mark][component] + marks[component - 1].append((anchor, markclass)) + + base = self._glyphName(name) + if is_mark: + mark = ast.MarkMarkPosStatement(base, marks[0]) + elif is_ligature: + mark = ast.MarkLigPosStatement(base, marks) + else: + mark = ast.MarkBasePosStatement(base, marks[0]) + statements.append(mark) + elif isinstance(pos, VAst.PositionAttachCursiveDefinition): + # Collect enter and exit glyphs + enter_coverage = [] + for coverage in pos.coverages_enter: + for base in coverage: + for name in base.glyphSet(): + enter_coverage.append(name) + exit_coverage = [] + for coverage in pos.coverages_exit: + for base in coverage: + for name in base.glyphSet(): + exit_coverage.append(name) + + # Write enter anchors, also check if the glyph has exit anchor and + # write it, too. + for name in enter_coverage: + glyph = self._glyphName(name) + entry = self._anchors[name]["entry"][1] + exit = None + if name in exit_coverage: + exit = self._anchors[name]["exit"][1] + exit_coverage.pop(exit_coverage.index(name)) + statements.append(ast.CursivePosStatement(glyph, entry, exit)) + + # Write any remaining exit anchors. + for name in exit_coverage: + glyph = self._glyphName(name) + exit = self._anchors[name]["exit"][1] + statements.append(ast.CursivePosStatement(glyph, None, exit)) + else: + raise NotImplementedError(pos) + + def _gposContextLookup(self, lookup, prefix, suffix, ignore, fealookup, chained): + statements = fealookup.statements + + pos = lookup.pos + if isinstance(pos, VAst.PositionAdjustPairDefinition): + for (idx1, idx2), (pos1, pos2) in pos.adjust_pair.items(): + glyphs1 = self._coverage(pos.coverages_1[idx1 - 1]) + glyphs2 = self._coverage(pos.coverages_2[idx2 - 1]) + assert len(glyphs1) == 1 + assert len(glyphs2) == 1 + glyphs = (glyphs1[0], glyphs2[0]) + + if ignore: + statement = ast.IgnorePosStatement([(prefix, glyphs, suffix)]) + else: + statement = ast.ChainContextPosStatement( + prefix, glyphs, suffix, [chained, chained] + ) + statements.append(statement) + elif isinstance(pos, VAst.PositionAdjustSingleDefinition): + glyphs = [ast.GlyphClass()] + for a, _ in pos.adjust_single: + glyphs[0].extend(self._coverage(a, flatten=True)) + + if ignore: + statement = ast.IgnorePosStatement([(prefix, glyphs, suffix)]) + else: + statement = ast.ChainContextPosStatement( + prefix, glyphs, suffix, [chained] + ) + statements.append(statement) + elif isinstance(pos, VAst.PositionAttachDefinition): + glyphs = [ast.GlyphClass()] + for coverage, _ in pos.coverage_to: + glyphs[0].extend(self._coverage(coverage, flatten=True)) + + if ignore: + statement = ast.IgnorePosStatement([(prefix, glyphs, suffix)]) + else: + statement = ast.ChainContextPosStatement( + prefix, glyphs, suffix, [chained] + ) + statements.append(statement) + else: + raise NotImplementedError(pos) + + def _gsubLookup(self, lookup, fealookup): + statements = fealookup.statements + + sub = lookup.sub + + # Alternate substitutions are represented by adding multiple + # substitutions for the same glyph, so we need to collect them into one + # to many mapping. + if isinstance(sub, VAst.SubstitutionAlternateDefinition): + alternates = {} + for key, val in sub.mapping.items(): + if not key or not val: + path, line, column = sub.location + log.warning(f"{path}:{line}:{column}: Ignoring empty substitution") + continue + glyphs = self._coverage(key) + replacements = self._coverage(val) + assert len(glyphs) == 1 + for src_glyph, repl_glyph in zip( + glyphs[0].glyphSet(), replacements[0].glyphSet() + ): + alternates.setdefault(str(self._glyphName(src_glyph)), []).append( + str(self._glyphName(repl_glyph)) + ) + + for glyph, replacements in alternates.items(): + statement = ast.AlternateSubstStatement( + [], glyph, [], ast.GlyphClass(replacements) + ) + statements.append(statement) + return + + for key, val in sub.mapping.items(): + if not key or not val: + path, line, column = sub.location + log.warning(f"{path}:{line}:{column}: Ignoring empty substitution") + continue + glyphs = self._coverage(key) + replacements = self._coverage(val) + if isinstance(sub, VAst.SubstitutionSingleDefinition): + assert len(glyphs) == 1 + assert len(replacements) == 1 + statements.append( + ast.SingleSubstStatement(glyphs, replacements, [], [], False) + ) + elif isinstance(sub, VAst.SubstitutionReverseChainingSingleDefinition): + # This is handled in gsubContextLookup() + pass + elif isinstance(sub, VAst.SubstitutionMultipleDefinition): + assert len(glyphs) == 1 + statements.append( + ast.MultipleSubstStatement([], glyphs[0], [], replacements) + ) + elif isinstance(sub, VAst.SubstitutionLigatureDefinition): + assert len(replacements) == 1 + statement = ast.LigatureSubstStatement( + [], glyphs, [], replacements[0], False + ) + + # If any of the input glyphs is a group, we need to + # explode the substitution into multiple ligature substitutions + # since feature file syntax does not support classes in + # ligature substitutions. + n = max(len(x.glyphSet()) for x in glyphs) + if n > 1: + # All input should either be groups of the same length or single glyphs + assert all(len(x.glyphSet()) in (n, 1) for x in glyphs) + glyphs = [x.glyphSet() for x in glyphs] + glyphs = [([x[0]] * n if len(x) == 1 else x) for x in glyphs] + + # In this case ligature replacements must be a group of the same length + # as the input groups, or a single glyph. VOLT + # allows the replacement glyphs to be longer and truncates them. + # So well allow that and zip() below will do the truncation + # for us. + replacement = replacements[0].glyphSet() + if len(replacement) == 1: + replacement = [replacement[0]] * n + assert len(replacement) >= n + + # Add the unexploded statement commented out for reference. + statements.append(ast.Comment(f"# {statement}")) + + for zipped in zip(*glyphs, replacement): + zipped = [self._glyphName(x) for x in zipped] + statements.append( + ast.LigatureSubstStatement( + [], zipped[:-1], [], zipped[-1], False + ) + ) + else: + statements.append(statement) + else: + raise NotImplementedError(sub) + + def _gsubContextLookup(self, lookup, prefix, suffix, ignore, fealookup, chained): + statements = fealookup.statements + + sub = lookup.sub + + if isinstance(sub, VAst.SubstitutionReverseChainingSingleDefinition): + # Reverse substitutions is a special case, it can’t use chained lookups. + for key, val in sub.mapping.items(): + if not key or not val: + path, line, column = sub.location + log.warning(f"{path}:{line}:{column}: Ignoring empty substitution") + continue + glyphs = self._coverage(key) + replacements = self._coverage(val) + statements.append( + ast.ReverseChainSingleSubstStatement( + prefix, suffix, glyphs, replacements + ) + ) + fealookup.chained = [] + return + + if not isinstance( + sub, + ( + VAst.SubstitutionSingleDefinition, + VAst.SubstitutionMultipleDefinition, + VAst.SubstitutionLigatureDefinition, + VAst.SubstitutionAlternateDefinition, + ), + ): + raise NotImplementedError(type(sub)) + + glyphs = [] + for key, val in sub.mapping.items(): + if not key or not val: + path, line, column = sub.location + log.warning(f"{path}:{line}:{column}: Ignoring empty substitution") + continue + glyphs.extend(self._coverage(key, flatten=True)) + + if len(glyphs) > 1: + glyphs = [ast.GlyphClass(glyphs)] + if ignore: + statements.append(ast.IgnoreSubstStatement([(prefix, glyphs, suffix)])) + else: + statements.append( + ast.ChainContextSubstStatement(prefix, glyphs, suffix, [chained]) + ) + + def _lookupDefinition(self, lookup): + mark_attachement = None + mark_filtering = None + + flags = 0 + if lookup.direction == "RTL": + flags |= 1 + if not lookup.process_base: + flags |= 2 + # FIXME: Does VOLT support this? + # if not lookup.process_ligatures: + # flags |= 4 + if not lookup.process_marks: + flags |= 8 + elif isinstance(lookup.process_marks, str): + mark_attachement = self._groupName(lookup.process_marks) + elif lookup.mark_glyph_set is not None: + mark_filtering = self._groupName(lookup.mark_glyph_set) + + lookupflags = None + if flags or mark_attachement is not None or mark_filtering is not None: + lookupflags = ast.LookupFlagStatement( + flags, mark_attachement, mark_filtering + ) + + use_extension = False + if self._settings.get("COMPILER_USEEXTENSIONLOOKUPS"): + use_extension = True + + if "\\" in lookup.name: + # Merge sub lookups as subtables (lookups named “base\sub”), + # makeotf/feaLib will issue a warning and ignore the subtable + # statement if it is not a pairpos lookup, though. + name = lookup.name.split("\\")[0] + if name.lower() not in self._lookups: + fealookup = Lookup( + self._lookupName(name), + use_extension=use_extension, + ) + if lookupflags is not None: + fealookup.statements.append(lookupflags) + fealookup.statements.append(ast.Comment("# " + lookup.name)) + else: + fealookup = self._lookups[name.lower()] + fealookup.statements.append(ast.SubtableStatement()) + fealookup.statements.append(ast.Comment("# " + lookup.name)) + self._lookups[name.lower()] = fealookup + else: + fealookup = Lookup( + self._lookupName(lookup.name), + use_extension=use_extension, + ) + if lookupflags is not None: + fealookup.statements.append(lookupflags) + self._lookups[lookup.name.lower()] = fealookup + + if lookup.comments is not None: + fealookup.statements.append(ast.Comment("# " + lookup.comments)) + + contexts = [] + for context in lookup.context: + prefix = self._context(context.left) + suffix = self._context(context.right) + ignore = context.ex_or_in == "EXCEPT_CONTEXT" + contexts.append([prefix, suffix, ignore]) + # It seems that VOLT will create contextual substitution using + # only the input if there is no other contexts in this lookup. + if ignore and len(lookup.context) == 1: + contexts.append([[], [], False]) + + if contexts: + chained = ast.LookupBlock( + self._lookupName(lookup.name + " chained"), + use_extension=use_extension, + ) + fealookup.chained.append(chained) + if lookup.sub is not None: + self._gsubLookup(lookup, chained) + elif lookup.pos is not None: + self._gposLookup(lookup, chained) + for prefix, suffix, ignore in contexts: + if lookup.sub is not None: + self._gsubContextLookup( + lookup, prefix, suffix, ignore, fealookup, chained + ) + elif lookup.pos is not None: + self._gposContextLookup( + lookup, prefix, suffix, ignore, fealookup, chained + ) + else: + if lookup.sub is not None: + self._gsubLookup(lookup, fealookup) + elif lookup.pos is not None: + self._gposLookup(lookup, fealookup) + + +def main(args=None): + """Convert MS VOLT to AFDKO feature files.""" + + import argparse + from pathlib import Path + + from fontTools import configLogger + + parser = argparse.ArgumentParser( + "fonttools voltLib.voltToFea", description=main.__doc__ + ) + parser.add_argument( + "input", metavar="INPUT", type=Path, help="input font/VTP file to process" + ) + parser.add_argument( + "featurefile", metavar="OUTPUT", type=Path, help="output feature file" + ) + parser.add_argument( + "-t", + "--table", + action="append", + choices=TABLES, + dest="tables", + help="List of tables to write, by default all tables are written", + ) + parser.add_argument( + "-q", "--quiet", action="store_true", help="Suppress non-error messages" + ) + parser.add_argument( + "--traceback", action="store_true", help="Don’t catch exceptions" + ) + + options = parser.parse_args(args) + + configLogger(level=("ERROR" if options.quiet else "INFO")) + + file_or_path = options.input + font = None + try: + font = TTFont(file_or_path) + if "TSIV" in font: + file_or_path = StringIO(font["TSIV"].data.decode("utf-8")) + else: + log.error('"TSIV" table is missing, font was not saved from VOLT?') + return 1 + except TTLibError: + pass + + converter = VoltToFea(file_or_path, font) + try: + fea = converter.convert(options.tables) + except NotImplementedError as e: + if options.traceback: + raise + location = getattr(e.args[0], "location", None) + message = f'"{e}" is not supported' + if location: + path, line, column = location + log.error(f"{path}:{line}:{column}: {message}") + else: + log.error(message) + return 1 + with open(options.featurefile, "w") as feafile: + feafile.write(fea) + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/INSTALLER b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/METADATA b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..d32a85a7f228e7913407f0cb16fb1bf46754f5c4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/METADATA @@ -0,0 +1,2339 @@ +Metadata-Version: 2.4 +Name: fonttools +Version: 4.63.0 +Summary: Tools to manipulate font files +Home-page: http://github.com/fonttools/fonttools +Author: Just van Rossum +Author-email: just@letterror.com +Maintainer: Behdad Esfahbod +Maintainer-email: behdad@behdad.org +License: MIT +Platform: Any +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Environment :: Other Environment +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: End Users/Desktop +Classifier: Natural Language :: English +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: 3 +Classifier: Topic :: Text Processing :: Fonts +Classifier: Topic :: Multimedia :: Graphics +Classifier: Topic :: Multimedia :: Graphics :: Graphics Conversion +Requires-Python: >=3.10 +Description-Content-Type: text/x-rst +License-File: LICENSE +License-File: LICENSE.external +Provides-Extra: ufo +Provides-Extra: lxml +Requires-Dist: lxml>=4.0; extra == "lxml" +Provides-Extra: woff +Requires-Dist: brotli>=1.0.1; platform_python_implementation == "CPython" and extra == "woff" +Requires-Dist: brotlicffi>=0.8.0; platform_python_implementation != "CPython" and extra == "woff" +Requires-Dist: zopfli>=0.1.4; extra == "woff" +Provides-Extra: unicode +Requires-Dist: unicodedata2>=17.0.0; python_version <= "3.14" and extra == "unicode" +Provides-Extra: graphite +Requires-Dist: lz4>=1.7.4.2; extra == "graphite" +Provides-Extra: interpolatable +Requires-Dist: scipy; platform_python_implementation != "PyPy" and extra == "interpolatable" +Requires-Dist: munkres; platform_python_implementation == "PyPy" and extra == "interpolatable" +Requires-Dist: pycairo; extra == "interpolatable" +Provides-Extra: plot +Requires-Dist: matplotlib; extra == "plot" +Provides-Extra: symfont +Requires-Dist: sympy; extra == "symfont" +Provides-Extra: type1 +Requires-Dist: xattr; sys_platform == "darwin" and extra == "type1" +Provides-Extra: pathops +Requires-Dist: skia-pathops>=0.5.0; extra == "pathops" +Provides-Extra: repacker +Requires-Dist: uharfbuzz>=0.45.0; extra == "repacker" +Provides-Extra: all +Requires-Dist: lxml>=4.0; extra == "all" +Requires-Dist: brotli>=1.0.1; platform_python_implementation == "CPython" and extra == "all" +Requires-Dist: brotlicffi>=0.8.0; platform_python_implementation != "CPython" and extra == "all" +Requires-Dist: zopfli>=0.1.4; extra == "all" +Requires-Dist: unicodedata2>=17.0.0; python_version <= "3.14" and extra == "all" +Requires-Dist: lz4>=1.7.4.2; extra == "all" +Requires-Dist: scipy; platform_python_implementation != "PyPy" and extra == "all" +Requires-Dist: munkres; platform_python_implementation == "PyPy" and extra == "all" +Requires-Dist: pycairo; extra == "all" +Requires-Dist: matplotlib; extra == "all" +Requires-Dist: sympy; extra == "all" +Requires-Dist: xattr; sys_platform == "darwin" and extra == "all" +Requires-Dist: skia-pathops>=0.5.0; extra == "all" +Requires-Dist: uharfbuzz>=0.45.0; extra == "all" +Dynamic: author +Dynamic: author-email +Dynamic: classifier +Dynamic: description +Dynamic: description-content-type +Dynamic: home-page +Dynamic: license +Dynamic: license-file +Dynamic: maintainer +Dynamic: maintainer-email +Dynamic: platform +Dynamic: provides-extra +Dynamic: requires-python +Dynamic: summary + +|CI Build Status| |Coverage Status| |PyPI| |Gitter Chat| + +What is this? +~~~~~~~~~~~~~ + +| fontTools is a library for manipulating fonts, written in Python. The + project includes the TTX tool, that can convert TrueType and OpenType + fonts to and from an XML text format, which is also called TTX. It + supports TrueType, OpenType, AFM and to an extent Type 1 and some + Mac-specific formats. The project has an `MIT open-source + license <LICENSE>`__. +| Among other things this means you can use it free of charge. + +`User documentation <https://fonttools.readthedocs.io/en/latest/>`_ and +`developer documentation <https://fonttools.readthedocs.io/en/latest/developer.html>`_ +are available at `Read the Docs <https://fonttools.readthedocs.io/>`_. + +Installation +~~~~~~~~~~~~ + +FontTools requires `Python <http://www.python.org/download/>`__ 3.10 +or later. We try to follow the same schedule of minimum Python version support as +NumPy (see `NEP 29 <https://numpy.org/neps/nep-0029-deprecation_policy.html>`__). + +The package is listed in the Python Package Index (PyPI), so you can +install it with `pip <https://pip.pypa.io>`__: + +.. code:: sh + + pip install fonttools + +If you would like to contribute to its development, you can clone the +repository from GitHub, install the package in 'editable' mode and +modify the source code in place. We recommend creating a virtual +environment, using `virtualenv <https://virtualenv.pypa.io>`__ or +Python 3 `venv <https://docs.python.org/3/library/venv.html>`__ module. + +.. code:: sh + + # download the source code to 'fonttools' folder + git clone https://github.com/fonttools/fonttools.git + cd fonttools + + # create new virtual environment called e.g. 'fonttools-venv', or anything you like + python -m virtualenv fonttools-venv + + # source the `activate` shell script to enter the environment (Unix-like); to exit, just type `deactivate` + . fonttools-venv/bin/activate + + # to activate the virtual environment in Windows `cmd.exe`, do + fonttools-venv\Scripts\activate.bat + + # install in 'editable' mode + pip install -e . + +Optional Requirements +--------------------- + +The ``fontTools`` package currently has no (required) external dependencies +besides the modules included in the Python Standard Library. +However, a few extra dependencies are required by some of its modules, which +are needed to unlock optional features. +The ``fonttools`` PyPI distribution also supports so-called "extras", i.e. a +set of keywords that describe a group of additional dependencies, which can be +used when installing via pip, or when specifying a requirement. +For example: + +.. code:: sh + + pip install fonttools[ufo,lxml,woff,unicode] + +This command will install fonttools, as well as the optional dependencies that +are required to unlock the extra features named "ufo", etc. + +- ``Lib/fontTools/misc/etree.py`` + + The module exports a ElementTree-like API for reading/writing XML files, and + allows to use as the backend either the built-in ``xml.etree`` module or + `lxml <https://lxml.de>`__. The latter is preferred whenever present, + as it is generally faster and more secure. + + *Extra:* ``lxml`` + +- ``Lib/fontTools/ttLib/woff2.py`` + + Module to compress/decompress WOFF 2.0 web fonts; it requires: + + * `brotli <https://pypi.python.org/pypi/Brotli>`__: Python bindings of + the Brotli compression library. + + *Extra:* ``woff`` + +- ``Lib/fontTools/ttLib/sfnt.py`` + + To better compress WOFF 1.0 web fonts, the following module can be used + instead of the built-in ``zlib`` library: + + * `zopfli <https://pypi.python.org/pypi/zopfli>`__: Python bindings of + the Zopfli compression library. + + *Extra:* ``woff`` + +- ``Lib/fontTools/unicode.py`` + + To display the Unicode character names when dumping the ``cmap`` table + with ``ttx`` we use the ``unicodedata`` module in the Standard Library. + The version included in there varies between different Python versions. + To use the latest available data, you can install: + + * `unicodedata2 <https://pypi.python.org/pypi/unicodedata2>`__: + ``unicodedata`` backport for Python 3.x updated to the latest Unicode + version 17.0. + + *Extra:* ``unicode`` + +- ``Lib/fontTools/varLib/interpolatable.py`` + + Module for finding wrong contour/component order between different masters. + It requires one of the following packages in order to solve the so-called + "minimum weight perfect matching problem in bipartite graphs", or + the Assignment problem: + + * `scipy <https://pypi.python.org/pypi/scipy>`__: the Scientific Library + for Python, which internally uses `NumPy <https://pypi.python.org/pypi/numpy>`__ + arrays and hence is very fast; + * `munkres <https://pypi.python.org/pypi/munkres>`__: a pure-Python + module that implements the Hungarian or Kuhn-Munkres algorithm. Slower than + SciPy, but useful for minimalistic systems where adding SciPy is undesirable. + + This ensures both performance (via SciPy) and minimal footprint (via Munkres) + are possible. + + To plot the results to a PDF or HTML format, you also need to install: + + * `pycairo <https://pypi.org/project/pycairo/>`__: Python bindings for the + Cairo graphics library. Note that wheels are currently only available for + Windows, for other platforms see pycairo's `installation instructions + <https://pycairo.readthedocs.io/en/latest/getting_started.html>`__. + + *Extra:* ``interpolatable`` + +- ``Lib/fontTools/varLib/plot.py`` + + Module for visualizing DesignSpaceDocument and resulting VariationModel. + + * `matplotlib <https://pypi.org/pypi/matplotlib>`__: 2D plotting library. + + *Extra:* ``plot`` + +- ``Lib/fontTools/misc/symfont.py`` + + Advanced module for symbolic font statistics analysis; it requires: + + * `sympy <https://pypi.python.org/pypi/sympy>`__: the Python library for + symbolic mathematics. + + *Extra:* ``symfont`` + +- ``Lib/fontTools/t1Lib.py`` + + To get the file creator and type of Macintosh PostScript Type 1 fonts + on Python 3 you need to install the following module, as the old ``MacOS`` + module is no longer included in Mac Python: + + * `xattr <https://pypi.python.org/pypi/xattr>`__: Python wrapper for + extended filesystem attributes (macOS platform only). + + *Extra:* ``type1`` + +- ``Lib/fontTools/ttLib/removeOverlaps.py`` + + Simplify TrueType glyphs by merging overlapping contours and components. + + * `skia-pathops <https://pypi.python.org/pypi/skia-pathops>`__: Python + bindings for the Skia library's PathOps module, performing boolean + operations on paths (union, intersection, etc.). + + *Extra:* ``pathops`` + +- ``Lib/fontTools/ufoLib`` + + Package for reading and writing UFO source files; if available, it will use: + + * `fs <https://pypi.org/pypi/fs>`__: (aka ``pyfilesystem2``) filesystem abstraction layer + + for reading and writing UFOs to the local filesystem or zip files (.ufoz), instead of + the built-in ``fontTools.misc.filesystem`` package. + The reader and writer classes can in theory also accept any object compatible the + ``fs.base.FS`` interface, although not all have been tested. + +- ``Lib/fontTools/pens/cocoaPen.py`` and ``Lib/fontTools/pens/quartzPen.py`` + + Pens for drawing glyphs with Cocoa ``NSBezierPath`` or ``CGPath`` require: + + * `PyObjC <https://pypi.python.org/pypi/pyobjc>`__: the bridge between + Python and the Objective-C runtime (macOS platform only). + +- ``Lib/fontTools/pens/qtPen.py`` + + Pen for drawing glyphs with Qt's ``QPainterPath``, requires: + + * `PyQt5 <https://pypi.python.org/pypi/PyQt5>`__: Python bindings for + the Qt cross platform UI and application toolkit. + +- ``Lib/fontTools/pens/reportLabPen.py`` + + Pen to drawing glyphs as PNG images, requires: + + * `reportlab <https://pypi.python.org/pypi/reportlab>`__: Python toolkit + for generating PDFs and graphics. + +- ``Lib/fontTools/pens/freetypePen.py`` + + Pen to drawing glyphs with FreeType as raster images, requires: + + * `freetype-py <https://pypi.python.org/pypi/freetype-py>`__: Python binding + for the FreeType library. + +- ``Lib/fontTools/ttLib/tables/otBase.py`` + + Use the Harfbuzz library to serialize GPOS/GSUB using ``hb_repack`` method, requires: + + * `uharfbuzz <https://pypi.python.org/pypi/uharfbuzz>`__: Streamlined Cython + bindings for the harfbuzz shaping engine + + *Extra:* ``repacker`` + +How to make a new release +~~~~~~~~~~~~~~~~~~~~~~~~~ + +1) Update ``NEWS.rst`` with all the changes since the last release. Write a + changelog entry for each PR, with one or two short sentences summarizing it, + as well as links to the PR and relevant issues addressed by the PR. Do not + put a new title, the next command will do it for you. +2) Use semantic versioning to decide whether the new release will be a 'major', + 'minor' or 'patch' release. It's usually one of the latter two, depending on + whether new backward compatible APIs were added, or simply some bugs were fixed. +3) From inside a venv, first do ``pip install -r dev-requirements.txt``, then run + the ``python setup.py release`` command from the tip of the ``main`` branch. + By default this bumps the third or 'patch' digit only, unless you pass ``--major`` + or ``--minor`` to bump respectively the first or second digit. + This bumps the package version string, extracts the changes since the latest + version from ``NEWS.rst``, and uses that text to create an annotated git tag + (or a signed git tag if you pass the ``--sign`` option and your git and Github + account are configured for `signing commits <https://docs.github.com/en/github/authenticating-to-github/managing-commit-signature-verification/signing-commits>`__ + using a GPG key). + It also commits an additional version bump which opens the main branch for + the subsequent developmental cycle +4) Push both the tag and commit to the upstream repository, by running the command + ``git push --follow-tags``. Note: it may push other local tags as well, be + careful. +5) Let the CI build the wheel and source distribution packages and verify both + get uploaded to the Python Package Index (PyPI). +6) [Optional] Go to fonttools `Github Releases <https://github.com/fonttools/fonttools/releases>`__ + page and create a new release, copy-pasting the content of the git tag + message. This way, the release notes are nicely formatted as markdown, and + users watching the repo will get an email notification. One day we shall + automate that too. + + +Acknowledgments +~~~~~~~~~~~~~~~~ + +In alphabetical order: + +aschmitz, Olivier Berten, Samyak Bhuta, Erik van Blokland, Petr van Blokland, +Jelle Bosma, Sascha Brawer, Tom Byrer, Antonio Cavedoni, Frédéric Coiffier, +Vincent Connare, David Corbett, Simon Cozens, Dave Crossland, Simon Daniels, +Peter Dekkers, Behdad Esfahbod, Behnam Esfahbod, Hannes Famira, Sam Fishman, +Matt Fontaine, Takaaki Fuji, Rob Hagemans, Yannis Haralambous, Greg Hitchcock, +Jeremie Hornus, Khaled Hosny, John Hudson, Denis Moyogo Jacquerye, Jack Jansen, +Tom Kacvinsky, Jens Kutilek, Antoine Leca, Werner Lemberg, Tal Leming, Liang Hai, Peter +Lofting, Cosimo Lupo, Olli Meier, Masaya Nakamura, Dave Opstad, Laurence Penney, +Roozbeh Pournader, Garret Rieger, Read Roberts, Colin Rofls, Guido van Rossum, +Just van Rossum, Andreas Seidel, Georg Seifert, Chris Simpkins, Miguel Sousa, +Adam Twardoch, Adrien Tétar, Vitaly Volkov, Paul Wise. + +Copyrights +~~~~~~~~~~ + +| Copyright (c) 1999-2004 Just van Rossum, LettError + (just@letterror.com) +| See `LICENSE <LICENSE>`__ for the full license. + +Copyright (c) 2000 BeOpen.com. All Rights Reserved. + +Copyright (c) 1995-2001 Corporation for National Research Initiatives. +All Rights Reserved. + +Copyright (c) 1991-1995 Stichting Mathematisch Centrum, Amsterdam. All +Rights Reserved. + +Have fun! + +.. |CI Build Status| image:: https://github.com/fonttools/fonttools/workflows/Test/badge.svg + :target: https://github.com/fonttools/fonttools/actions?query=workflow%3ATest +.. |Coverage Status| image:: https://codecov.io/gh/fonttools/fonttools/branch/main/graph/badge.svg + :target: https://codecov.io/gh/fonttools/fonttools +.. |PyPI| image:: https://img.shields.io/pypi/v/fonttools.svg + :target: https://pypi.org/project/FontTools +.. |Gitter Chat| image:: https://badges.gitter.im/fonttools-dev/Lobby.svg + :alt: Join the chat at https://gitter.im/fonttools-dev/Lobby + :target: https://gitter.im/fonttools-dev/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge + +Changelog +~~~~~~~~~ + +4.63.0 (released 2026-05-14) +---------------------------- + +- [ttLib] Add support for Apple Color Emoji ``bgcl`` table (#4065). +- [ttLib] Add support for ``IFT`` and ``IFTX`` tables (Incremental Font Transfer, + PatchMapFormat2) (#4070, #4072). +- [otData] Introduce ``FieldSpec`` dataclass for OpenType table schema definitions, + replacing raw tuples in ``otData.py`` (#4076). +- [Feat] Show ``name`` table strings as comments next to label IDs in TTX output, + matching the convention used by ``fvar``, ``STAT``, ``trak`` (#4089). +- [cu2qu] Fix Cython complex-division rounding difference in + ``split_cubic_into_three`` that could cause ±1 off-curve coordinate shifts + (#3928, #4083). +- [designspaceLib] Fix ``map_backward`` for many-to-one (flat-segment) axis maps + that silently dropped entries via dict comprehension + (googlefonts/ufo2ft#978, #4085). +- [OS/2] Fix ``setUnicodeRanges`` to accept reserved bits 123-127, restoring + round-trip with ``getUnicodeRanges`` and fixing ``recalcUnicodeRanges`` crash + in the subsetter (#4087, #4088). +- [cython] Declare Cython extensions as free-threading compatible on Python 3.13+, + so that importing them on free-threaded Python no longer re-enables the GIL + (#4073, #4090). + +4.62.1 (released 2026-03-13) +---------------------------- + +- [feaLib] Extend contextual rule merging to all rule types: single subst, GSUB/GPOS + named lookups, ignore rules, and chained alternate subst (#4061). + +4.62.0 (released 2026-03-09) +---------------------------- + +- [diff] Add new ``fonttools diff`` command for comparing font files, imported from the + ``fdiff`` project and heavily reworked (#1190, #4007, #4009, #4011, #4013, #4019). +- [feaLib] Fix ``VariableScalar`` interpolation bug with non-linear avar mappings. Also + decouple ``VariableScalar`` from compiled fonts, allowing it to work with designspace data + before compilation (#3938, #4054). +- [feaLib] Fix ``VariableScalar`` axis ordering and iterative delta rounding to match fontc + behavior (#4053). +- [feaLib] Merge chained multi subst rules with same context into a single subtable instead of + emitting one subtable per glyph (#4016, #4058). +- [feaLib] Pass location to ``ConditionsetStatement`` to fix glyphsLib round-tripping + (fontra/fontra-glyphs#130, #4057). +- [feaLib] Write ``0xFFFF`` instead of ``0`` for missing nameIDs in ``cv`` feature params + (#4010, #4012). +- [cmap] Fix ``CmapSubtable.__lt__()`` ``TypeError`` on Python 3 when subtables share the + same encoding record, and add compile-time validation for unique encoding records (#4035, + #4055). +- [svgLib] Skip non-element XML nodes (comments, processing instructions) when drawing SVG + paths (#4042, #4043). +- [glifLib] Fix regression reading glyph outlines when ``glyphObject=None`` (#4030, #4031). +- [pointPen] Fix ``SegmentToPointPen`` edge case: only remove a duplicate final point on + ``closePath()`` if it is an on-curve point (#4014, #4015). +- [cffLib] **SECURITY** Replace ``eval()`` with ``safeEval()`` in ``parseBlendList()`` to + prevent arbitrary code execution from crafted TTX files (#4039, #4040). +- [ttLib] Remove defunct Adobe SING Glyphlet tables (``META``, ``SING``, ``GMAP``, ``GPKG``) + (#4044). +- [varLib.interpolatable] Various bugfixes: fix swapped nodeTypes assignment, duplicate + kink-detector condition, typos, CFF2 vsindex parsing, glyph existence check, and plot + helpers (#4046). +- [varLib.models] Fix ``getSubModel`` not forwarding ``extrapolate``/``axisRanges``; check + location uniqueness after stripping zeros (#4047). +- [varLib] Fix ``--variable-fonts`` filter in ``build_many``; remove dead code and fix + comments (#4048). +- [avar] Preserve existing name table in build; keep ``unbuild`` return types consistent; + validate ``map`` CLI coordinates (#4051). +- [cu2qu/qu2cu] Add input validation: reject non-positive tolerances, validate curve inputs + and list lengths (#4052). +- [colorLib] Raise a clear ``ColorLibError`` when base glyphs are missing from glyphMap, + instead of a confusing ``KeyError`` (#4041). +- [glyf] Remove unnecessary ``fvar`` table dependency (#4017). +- [fvar/trak] Remove unnecessary ``name`` table dependency (#4018). +- [ufoLib] Relax guideline validation to follow the updated spec (#3537, #3553). +- [ttFont] Fix ``saveXML`` regression with empty table lists, clarify docstring (#4025, #4026, + #4056). +- [setup.py] Link ``libm`` for Cython extensions using math functions (#4028, #4029). +- Add typing annotations for ``DSIG``, ``DefaultTable``, ``ttProgram`` (#4033). + +4.61.1 (released 2025-12-12) +---------------------------- + +- [otlLib] buildCoverage: return empty Coverage instead of None (#4003, #4004). +- [instancer] bug fix in ``avar2`` full instancing (#4002). +- [designspaceLib] Preserve empty conditionsets when serializing to XML (#4001). +- [fontBu ilder] Fix FontBuilder ``setupOS2()`` default params globally polluted (#3996, #3997). +- [ttFont] Add more typing annotations to ttFont, xmlWriter, sfnt, varLib.models and others (#3952, #3826). +- Explicitly test and declare support for Python 3.14, even though we were already shipping pre-built wheels for it (#3990). + +4.60.2 (released 2025-12-09) +---------------------------- + +- **Backport release** Same as 4.61.0 but without "Drop support for EOL Python 3.9" change to allow + downstream projects still on Python 3.9 to avail of the security fix for CVE-2025-66034 (#3994, #3999). + +4.61.0 (released 2025-11-28) +---------------------------- + +- [varLib.main]: **SECURITY** Only use basename(vf.filename) to prevent path traversal attacks when + running ``fonttools varLib`` command, or code which invokes ``fonttools.varLib.main()``. + Fixes CVE-2025-66034, see: + https://github.com/fonttools/fonttools/security/advisories/GHSA-768j-98cg-p3fv. +- [feaLib] Sort BaseLangSysRecords by tag (#3986). +- Drop support for EOL Python 3.9 (#3982). +- [instancer] Support --remove-overlaps for fonts with CFF2 table (#3975). +- [CFF2ToCFF] Add --remove-overlaps option (#3976). +- [feaLib] Raise an error for rsub with NULL target (#3979). +- [bezierTools] Fix logic bug in curveCurveIntersections (#3963). +- [feaLib] Error when condition sets have the same name (#3958). +- [cu2qu.ufo] skip processing empty glyphs to support sparse kerning masters (#3956). +- [unicodedata] Update to Unicode 17. Require ``unicodedata2 >= 17.0.0`` when installed with 'unicode' extra. + +4.60.1 (released 2025-09-29) +---------------------------- + +- [ufoLib] Reverted accidental method name change in ``UFOReader.getKerningGroupConversionRenameMaps`` + that broke compatibility with downstream projects like defcon (#3948, #3947, robotools/defcon#478). +- [ufoLib] Added test coverage for ``getKerningGroupConversionRenameMaps`` method (#3950). +- [subset] Don't try to subset BASE table; pass it through by default instead (#3949). +- [subset] Remove empty BaseRecord entries in MarkBasePos lookups (#3897, #3892). +- [subset] Add pruning for MarkLigPos and MarkMarkPos lookups (#3946). +- [subset] Remove duplicate features when subsetting (#3945). +- [Docs] Added documentation for the visitor module (#3944). + +4.60.0 (released 2025-09-17) +---------------------------- + +- [pointPen] Allow ``reverseFlipped`` parameter of ``DecomposingPointPen`` to take a ``ReverseFlipped`` + enum value to control whether/how to reverse contour direction of flipped components, in addition to + the existing True/False. This allows to set ``ReverseFlipped.ON_CURVE_FIRST`` to ensure that + the decomposed outline starts with an on-curve point before being reversed, for better consistency + with other segment-oriented contour transformations. The change is backward compatible, and the + default behavior hasn't changed (#3934). +- [filterPen] Added ``ContourFilterPointPen``, base pen for buffered contour operations, and + ``OnCurveStartPointPen`` filter to ensure contours start with an on-curve point (#3934). +- [cu2qu] Fixed difference in cython vs pure-python complex division by real number (#3930). +- [varLib.avar] Refactored and added some new sub-modules and scripts (#3926). + * ``varLib.avar.build`` module to build avar (and a missing fvar) binaries into a possibly empty TTFont, + * ``varLib.avar.unbuild`` module to print a .designspace snippet that would generate the same avar binary, + * ``varLib.avar.map`` module to take TTFont and do the mapping, in user/normalized space, + * ``varLib.avar.plan`` module moved from ``varLib.avarPlanner``. + The bare ``fonttools varLib.avar`` script is deprecated, in favour of ``fonttools varLib.avar.build`` (or ``unbuild``). +- [interpolatable] Clarify ``linear_sum_assignment`` backend options and minimal dependency + usage (#3927). +- [post] Speed up ``build_psNameMapping`` (#3923). +- [ufoLib] Added typing annotations to fontTools.ufoLib (#3875). + +4.59.2 (released 2025-08-27) +---------------------------- + +- [varLib] Clear ``USE_MY_METRICS`` component flags when inconsistent across masters (#3912). +- [varLib.instancer] Avoid negative advance width/height values when instatiating HVAR/VVAR, + (unlikely in well-behaved fonts) (#3918). +- [subset] Fix shaping behaviour when pruning empty mark sets (#3915, harfbuzz/harfbuzz#5499). +- [cu2qu] Fixed ``dot()`` product of perpendicular vectors not always returning exactly 0.0 + in all Python implementations (#3911) +- [varLib.instancer] Implemented fully-instantiating ``avar2`` fonts (#3909). +- [feaLib] Allow float values in ``VariableScalar``'s axis locations (#3906, #3907). +- [cu2qu] Handle special case in ``calc_intersect`` for degenerate cubic curves where 3 to 4 + control points are equal (#3904). + +4.59.1 (released 2025-08-14) +---------------------------- + +- [featureVars] Update OS/2.usMaxContext if possible after addFeatureVariationsRaw (#3894). +- [vhmtx] raise TTLibError('not enough data...') when hmtx/vmtx are truncated (#3843, #3901). +- [feaLib] Combine duplicate features that have the same set of lookups regardless of the order in which those lookups are added to the feature (#3895). +- [varLib] Deprecate ``varLib.mutator`` in favor of ``varLib.instancer``. The latter + provides equivalent full (static font) instancing in addition to partial VF instancing. + CLI users should replace ``fonttools varLib.mutator`` with ``fonttools varLib.instancer``. + API users should migrate to ``fontTools.varLib.instancer.instantiateVariableFont`` (#2680). + + +4.59.0 (released 2025-07-16) +---------------------------- + +- Removed hard-dependency on pyfilesystem2 (``fs`` package) from ``fonttools[ufo]`` extra. + This is replaced by the `fontTools.misc.filesystem` package, a stdlib-only, drop-in + replacement for the subset of the pyfilesystem2's API used by ``fontTools.ufoLib``. + The latter should continue to work with the upstream ``fs`` (we even test with/without). + Clients who wish to continue using ``fs`` can do so by depending on it directly instead + of via the ``fonttools[ufo]`` extra (#3885, #3620). +- [xmlWriter] Replace illegal XML characters (e.g. control or non-characters) with "?" + when dumping to ttx (#3868, #71). +- [varLib.hvar] Fixed vertical metrics fields copy/pasta error (#3884). +- Micro optimizations in ttLib and sstruct modules (#3878, #3879). +- [unicodedata] Add Garay script to RTL_SCRIPTS (#3882). +- [roundingPen] Remove unreliable kwarg usage. Argument names aren’t consistent among + point pens’ ``.addComponent()`` implementations, in particular ``baseGlyphName`` + vs ``glyphName`` (#3880). + +4.58.5 (released 2025-07-03) +---------------------------- + +- [feaLib] Don't try to combine ligature & multisub rules (#3874). +- [feaLib/ast] Use weakref proxies to avoid cycles in visitor (#3873). +- [varLib.instancer] Fixed instancing CFF2 fonts where VarData contains more than 64k items (#3858). + +4.58.4 (released 2025-06-13) +---------------------------- + +- [feaLib] Allow for empty MarkFilter & MarkAttach sets (#3856). + +4.58.3 (released 2025-06-13) +---------------------------- + +- [feaLib] Fixed iterable check for Python 3.13.4 and newer (#3854, #3855). + +4.58.2 (released 2025-06-06) +---------------------------- + +- [ttLib.reorderGlyphs] Handle CFF2 when reordering glyphs (#3852) +- [subset] Copy name IDs in use before scrapping or scrambling them for webfonts (#3853) + +4.58.1 (released 2025-05-28) +---------------------------- + +- [varLib] Make sure that fvar named instances only reuse name ID 2 or 17 if they are at the default location across all axes, to match OT spec requirement (#3831). +- [feaLib] Improve single substitution promotion to multiple/ligature substitutions, fixing a few bugs as well (#3849). +- [loggingTools] Make ``Timer._time`` a static method that doesn't take self, makes it easier to override (#3836). +- [featureVars] Use ``None`` for empty ConditionSet, which translates to a null offset in the compiled table (#3850). +- [feaLib] Raise an error on conflicting ligature substitution rules instead of silently taking the last one (#3835). +- Add typing annotations to T2CharStringPen (#3837). +- [feaLib] Add single substitutions that were promoted to multiple or ligature substitutions to ``aalt`` feature (#3847). +- [featureVars] Create a default ``LangSys`` in a ``ScriptRecord`` if missing when adding feature variations to existing GSUB later in the build (#3838). +- [symfont] Added a ``main()``. +- [cffLib.specializer] Fix rmoveto merging when blends used (#3839, #3840). +- [pyftmerge] Add support for cmap format 14 in the merge tool (#3830). +- [varLib.instancer/cff2] Fix vsindex of Private dicts when instantiating (#3828, #3232). +- Update text file read to use UTF-8 with optional BOM so it works with e.g. Windows Notepad.exe (#3824). +- [varLib] Ensure that instances only reuse name ID 2 or 17 if they are at the default location across all axes (#3831). +- [varLib] Create a dflt LangSys in a ScriptRecord when adding variations later, to fix an avoidable crash in an edge case (#3838). + +4.58.0 (released 2025-05-10) +---------------------------- + +- Drop Python 3.8, require 3.9+ (#3819) +- [HVAR, VVAR] Prune unused regions when using a direct mapping (#3797) +- [Docs] Improvements to ufoLib documentation (#3721) +- [Docs] Improvements to varLib documentation (#3727) +- [Docs] Improvements to Pens and pen-module documentation (#3724) +- [Docs] Miscellany updates to docs (misc modules and smaller modules) (#3730) +- [subset] Close codepoints over BiDi mirror variants. (#3801) +- [feaLib] Fix serializing ChainContextPosStatement and + ChainContextSubstStatement in some rare cases (#3788) +- [designspaceLib] Clarify user expectations for getStatNames (#2892) +- [GVAR] Add support for new `GVAR` table (#3728) +- [TSI0, TSI5] Derive number of entries to decompile from data length (#2477) +- [ttLib] Fix `AttributeError` when reporting table overflow (#3808) +- [ttLib] Apply rounding more often in getCoordinates (#3798) +- [ttLib] Ignore component bounds if empty (#3799) +- [ttLib] Change the separator for duplicate glyph names from "#" to "." (#3809) +- [feaLib] Support subtable breaks in CursivePos, MarkBasePos, MarkToLigPos and + MarkToMarkPos lookups (#3800, #3807) +- [feaLib] If the same lookup has single substitutions and ligature + substitutions, upgrade single substitutions to ligature substitutions with + one input glyph (#3805) +- [feaLib] Correctly handle <NULL> in single pos lookups (#3803) +- [feaLib] Remove duplicates from class pair pos classes instead of raising an + error (#3804) +- [feaLib] Support creating extension lookups using useExtenion lookup flag + instead of silently ignoring it (#3811) +- [STAT] Add typing for the simpler STAT arguments (#3812) +- [otlLib.builder] Add future import for annotations (#3814) +- [cffLib] Fix reading supplement encoding (#3813) +- [voltLib] Add some missing functionality and fixes to voltLib and VoltToFea, + making the conversion to feature files more robust. Add also `fonttools + voltLib` command line tool to compile VOLT sources directly (doing an + intermediate fea conversion internally) (#3818) +- [pens] Add some PointPen annotations (#3820) + +4.57.0 (released 2025-04-03) +---------------------------- + +- [ttLib.__main__] Add `--no-recalc-timestamp` flag (#3771) +- [ttLib.__main__] Add `-b` (recalcBBoxes=False) flag (#3772) +- [cmap] Speed up glyphOrder loading from cmap (#3774) +- [ttLib.__main__] Improvements around the `-t` flag (#3776) +- [Debg] Fix parsing from XML; add roundtrip tests (#3781) +- [fealib] Support \*Base.MinMax tables (#3783, #3786) +- [config] Add OPTIMIZE_FONT_SPEED (#3784) +- [varLib.hvar] New module to add HVAR table to the font (#3780) +- [otlLib.optimize] Fix crash when the provided TTF does not contain a `GPOS` (#3794) + +4.56.0 (released 2025-02-07) +---------------------------- + +- [varStore] Sort the input todo list with the same sorting key used for the opimizer's output (#3767). +- [otData] Fix DeviceTable's ``DeltaValue`` repeat value which caused a crash after importing from XML and then compiling a GPOS containing Device tables (#3758). +- [feaLib] Make ``FeatureLibError`` pickleable, so client can e.g. use feaLib to can compile features in parallel with multiprocessing (#3762). +- [varLib/gvar] Removed workaround for old, long-fixed macOS bug about composite glyphs with all zero deltas (#1381, #1788). +- [Docs] Updated ttLib documentation, beefed up TTFont and TTGlyphSet explanations (#3720). + +4.55.8 (released 2025-01-29) +---------------------------- + +- [MetaTools] Fixed bug in buildUCD.py script whereby the first non-header line of some UCD text file was being skipped. This affected in particular the U+00B7 (MIDDLE DOT) entry of ScriptExtensions.txt (#3756). + +4.55.7 (released 2025-01-28) +---------------------------- + +- Shorten the changelog included in PyPI package description to accommodate maximum length limit imposed by Azure DevOps. No actual code changes since v4.55.6 (#3754). + +4.55.6 (released 2025-01-24) +---------------------------- + +- [glyf] Fixed regression introduced in 4.55.5 when computing bounds of nested composite glyphs with transformed components (#3752). + +4.55.5 (released 2025-01-23) +---------------------------- + +- [glyf] Fixed recalcBounds of transformed components with unrounded coordinates (#3750). +- [feaLib] Allow duplicate script/language statements (#3749). + +4.55.4 (released 2025-01-21) +---------------------------- + +- [bezierTools] Fixed ``splitCubicAtT`` sometimes not returning identical start/end points as result of numerical precision (#3742, #3743). +- [feaLib/ast] Fixed docstring of ``AlternateSubstStatement`` (#3735). +- [transform] Typing fixes (#3734). + +4.55.3 (released 2024-12-10) +---------------------------- + +- [Docs] fill out ttLib table section [#3716] +- [feaLib] More efficient inline format 4 lookups [#3726] + +4.55.2 (released 2024-12-05) +---------------------------- + +- [Docs] update Sphinx config (#3712) +- [designspaceLib] Allow axisOrdering to be set to zero (#3715) +- [feaLib] Don’t modify variable anchors in place (#3717) + +4.55.1 (released 2024-12-02) +---------------------------- + +- [ttGlyphSet] Support VARC CFF2 fonts (#3683) +- [DecomposedTransform] Document and implement always skewY == 0 (#3697) +- [varLib] "Fix" cython iup issue? (#3704) +- Cython minor refactor (#3705) + + +4.55.0 (released 2024-11-14) +---------------------------- + +- [cffLib.specializer] Adjust stack use calculation (#3689) +- [varLib] Lets not add mac names if the rest of name doesn't have them (#3688) +- [ttLib.reorderGlyphs] Update CFF table charstrings and charset (#3682) +- [cffLib.specializer] Add cmdline to specialize a CFF2 font (#3675, #3679) +- [CFF2] Lift uint16 VariationStore.length limitation (#3674) +- [subset] consider variation selectors subsetting cmap14 (#3672) +- [varLib.interpolatable] Support CFF2 fonts (#3670) +- Set isfinal to true in XML parser for proper resource cleanup (#3669) +- [removeOverlaps] Fix CFF CharString width (#3659) +- [glyf] Add optimizeSize option (#3657) +- Python 3.13 support (#3656) +- [TupleVariation] Optimize for loading speed, not size (#3650, #3653) + + +4.54.1 (released 2024-09-24) +---------------------------- + +- [unicodedata] Update to Unicode 16 +- [subset] Escape ``\\`` in doc string + +4.54.0 (released 2024-09-23) +---------------------------- + +- [Docs] Small docs cleanups by @n8willis (#3611) +- [Docs] cleanup code blocks by @n8willis (#3627) +- [Docs] fix Sphinx builds by @n8willis (#3625) +- [merge] Minor fixes to documentation for merge by @drj11 (#3588) +- [subset] Small tweaks to pyftsubset documentation by @RoelN (#3633) +- [Tests] Do not require fonttools command to be available by @behdad (#3612) +- [Tests] subset_test: add failing test to reproduce issue #3616 by @anthrotype (#3622) +- [ttLib] NameRecordVisitor: include whole sequence of character variants' UI labels, not just the first by @anthrotype (#3617) +- [varLib.avar] Reconstruct mappings from binary by @behdad (#3598) +- [varLib.instancer] Fix visual artefacts with partial L2 instancing by @Hoolean (#3635) +- [varLib.interpolatable] Support discrete axes in .designspace by @behdad (#3599) +- [varLib.models] By default, assume OpenType-like normalized space by @behdad (#3601) + +4.53.1 (released 2024-07-05) +---------------------------- + +- [feaLib] Improve the sharing of inline chained lookups (#3559) +- [otlLib] Correct the calculation of OS/2.usMaxContext with reversed chaining contextual single substitutions (#3569) +- [misc.visitor] Visitors search the inheritance chain of objects they are visiting (#3581) + +4.53.0 (released 2024-05-31) +---------------------------- + +- [ttLib.removeOverlaps] Support CFF table to aid in downconverting CFF2 fonts (#3528) +- [avar] Fix crash when accessing not-yet-existing attribute (#3550) +- [docs] Add buildMathTable to otlLib.builder documentation (#3540) +- [feaLib] Allow UTF-8 with BOM when reading features (#3495) +- [SVGPathPen] Revert rounding coordinates to two decimal places by default (#3543) +- [varLib.instancer] Refix output filename decision-making (#3545, #3544, #3548) + +4.52.4 (released 2024-05-27) +---------------------------- + +- [varLib.cff] Restore and deprecate convertCFFtoCFF2 that was removed in 4.52.0 + release as it is used by downstream projects (#3535). + +4.52.3 (released 2024-05-27) +---------------------------- + +- Fixed a small syntax error in the reStructuredText-formatted NEWS.rst file + which caused the upload to PyPI to fail for 4.52.2. No other code changes. + +4.52.2 (released 2024-05-27) +---------------------------- + +- [varLib.interpolatable] Ensure that scipy/numpy output is JSON-serializable + (#3522, #3526). +- [housekeeping] Regenerate table lists, to fix pyinstaller packaging of the new + ``VARC`` table (#3531, #3529). +- [cffLib] Make CFFToCFF2 and CFF2ToCFF more robust (#3521, #3525). + +4.52.1 (released 2024-05-24) +---------------------------- + +- Fixed a small syntax error in the reStructuredText-formatted NEWS.rst file + which caused the upload to PyPI to fail for 4.52.0. No other code changes. + +4.52.0 (released 2024-05-24) +---------------------------- + +- Added support for the new ``VARC`` (Variable Composite) table that is being + proposed to OpenType spec (#3395). For more info: + https://github.com/harfbuzz/boring-expansion-spec/blob/main/VARC.md +- [ttLib.__main__] Fixed decompiling all tables (90fed08). +- [feaLib] Don't reference the same lookup index multiple times within the same + feature record, it is only applied once anyway (#3520). +- [cffLib] Moved methods to desubroutinize, remove hints and unused subroutines + from subset module to cffLib (#3517). +- [varLib.instancer] Added support for partial-instancing CFF2 tables! Also, added + method to down-convert from CFF2 to CFF 1.0, and CLI entry points to convert + CFF<->CFF2 (#3506). +- [subset] Prune unused user name IDs even with --name-IDs='*' (#3410). +- [ttx] use GNU-style getopt to intermix options and positional arguments (#3509). +- [feaLib.variableScalar] Fixed ``value_at_location()`` method (#3491) +- [psCharStrings] Shorten output of ``encodeFloat`` (#3492). +- [bezierTools] Fix infinite-recursion in ``calcCubicArcLength`` (#3502). +- [avar2] Implement ``avar2`` support in ``TTFont.getGlyphSet()`` (#3473). + +4.51.0 (released 2024-04-05) +---------------------------- + +- [ttLib] Optimization on loading aux fields (#3464). +- [ttFont] Add reorderGlyphs (#3468). + +4.50.0 (released 2024-03-15) +---------------------------- + +- [pens] Added decomposing filter pens that draw components as regular contours (#3460). +- [instancer] Drop explicit no-op axes from TupleVariations (#3457). +- [cu2qu/ufo] Return set of modified glyph names from fonts_to_quadratic (#3456). + +4.49.0 (released 2024-02-15) +---------------------------- + +- [otlLib] Add API for building ``MATH`` table (#3446) + +4.48.1 (released 2024-02-06) +---------------------------- + +- Fixed uploading wheels to PyPI, no code changes since v4.48.0. + +4.48.0 (released 2024-02-06) +---------------------------- + +- [varLib] Do not log when there are no OTL tables to be merged. +- [setup.py] Do not restrict lxml<5 any more, tests pass just fine with lxml>=5. +- [feaLib] Remove glyph and class names length restrictions in FEA (#3424). +- [roundingPens] Added ``transformRoundFunc`` parameter to the rounding pens to allow + for custom rounding of the components' transforms (#3426). +- [feaLib] Keep declaration order of ligature components within a ligature set, instead + of sorting by glyph name (#3429). +- [feaLib] Fixed ordering of alternates in ``aalt`` lookups, following the declaration + order of feature references within the ``aalt`` feature block (#3430). +- [varLib.instancer] Fixed a bug in the instancer's IUP optimization (#3432). +- [sbix] Support sbix glyphs with new graphicType "flip" (#3433). +- [svgPathPen] Added ``--glyphs`` option to dump the SVG paths for the named glyphs + in the font (0572f78). +- [designspaceLib] Added "description" attribute to ``<mappings>`` and ``<mapping>`` + elements, and allow multiple ``<mappings>`` elements to group ``<mapping>`` elements + that are logically related (#3435, #3437). +- [otlLib] Correctly choose the most compact GSUB contextual lookup format (#3439). + +4.47.2 (released 2024-01-11) +---------------------------- + +Minor release to fix uploading wheels to PyPI. + +4.47.1 (released 2024-01-11) +---------------------------- + +- [merge] Improve help message and add standard command line options (#3408) +- [otlLib] Pass ``ttFont`` to ``name.addName`` in ``buildStatTable`` (#3406) +- [featureVars] Re-use ``FeatureVariationRecord``'s when possible (#3413) + +4.47.0 (released 2023-12-18) +---------------------------- + +- [varLib.models] New API for VariationModel: ``getMasterScalars`` and + ``interpolateFromValuesAndScalars``. +- [varLib.interpolatable] Various bugfixes and rendering improvements. In particular, + add a Summary page in the front, and an Index and Table-of-Contents in the back. + Change the page size to Letter. +- [Docs/designspaceLib] Defined a new ``public.fontInfo`` lib key, not used anywhere yet (#3358). + +4.46.0 (released 2023-12-02) +---------------------------- + +- [featureVars] Allow to register the same set of substitution rules to multiple features. + The ``addFeatureVariations`` function can now take a list of featureTags; similarly, the + lib key 'com.github.fonttools.varLib.featureVarsFeatureTag' can now take a + comma-separateed string of feature tags (e.g. "salt,ss01") instead of a single tag (#3360). +- [featureVars] Don't overwrite GSUB FeatureVariations, but append new records to it + for features which are not already there. But raise ``VarLibError`` if the feature tag + already has feature variations associated with it (#3363). +- [varLib] Added ``addGSUBFeatureVariations`` function to add GSUB Feature Variations + to an existing variable font from rules defined in a DesignSpace document (#3362). +- [varLib.interpolatable] Various bugfixes and rendering improvements. In particular, + a new test for "underweight" glyphs. The new test reports quite a few false-positives + though. Please send feedback. + +4.45.1 (released 2023-11-23) +---------------------------- + +- [varLib.interpolatable] Various bugfixes and improvements, better reporting, reduced + false positives. +- [ttGlyphSet] Added option to not recalculate glyf bounds (#3348). + +4.45.0 (released 2023-11-20) +---------------------------- + +- [varLib.interpolatable] Vastly improved algorithms. Also available now is ``--pdf`` + and ``--html`` options to generate a PDF or HTML report of the interpolation issues. + The PDF/HTML report showcases the problematic masters, the interpolated broken + glyph, as well as the proposed fixed version. + +4.44.3 (released 2023-11-15) +---------------------------- + +- [subset] Only prune codepage ranges for OS/2.version >= 1, ignore otherwise (#3334). +- [instancer] Ensure hhea vertical metrics stay in sync with OS/2 ones after instancing + MVAR table containing 'hasc', 'hdsc' or 'hlgp' tags (#3297). + +4.44.2 (released 2023-11-14) +---------------------------- + +- [glyf] Have ``Glyph.recalcBounds`` skip empty components (base glyph with no contours) + when computing the bounding box of composite glyphs. This simply restores the existing + behavior before some changes were introduced in fonttools 4.44.0 (#3333). + +4.44.1 (released 2023-11-14) +---------------------------- + +- [feaLib] Ensure variable mark anchors are deep-copied while building since they + get modified in-place and later reused (#3330). +- [OS/2|subset] Added method to ``recalcCodePageRanges`` to OS/2 table class; added + ``--prune-codepage-ranges`` to `fonttools subset` command (#3328, #2607). + +4.44.0 (released 2023-11-03) +---------------------------- + +- [instancer] Recalc OS/2 AvgCharWidth after instancing if default changes (#3317). +- [otlLib] Make ClassDefBuilder class order match varLib.merger's, i.e. large + classes first, then glyph lexicographic order (#3321, #3324). +- [instancer] Allow not specifying any of min:default:max values and let be filled + up with fvar's values (#3322, #3323). +- [instancer] When running --update-name-table ignore axes that have no STAT axis + values (#3318, #3319). +- [Debg] When dumping to ttx, write the embedded JSON as multi-line string with + indentation (92cbfee0d). +- [varStore] Handle > 65535 items per encoding by splitting VarData subtable (#3310). +- [subset] Handle null-offsets in MarkLigPos subtables. +- [subset] Keep East Asian spacing fatures vhal, halt, chws, vchw by default (#3305). +- [instancer.solver] Fixed case where axisDef < lower and upper < axisMax (#3304). +- [glyf] Speed up compilation, mostly around ``recalcBounds`` (#3301). +- [varLib.interpolatable] Speed it up when working on variable fonts, plus various + micro-optimizations (#3300). +- Require unicodedata2 >= 15.1.0 when installed with 'unicode' extra, contains UCD 15.1. + +4.43.1 (released 2023-10-06) +---------------------------- + +- [EBDT] Fixed TypeError exception in `_reverseBytes` method triggered when dumping + some bitmap fonts with `ttx -z bitwise` option (#3162). +- [v/hhea] Fixed UnboundLocalError exception in ``recalc`` method when no vmtx or hmtx + tables are present (#3290). +- [bezierTools] Fixed incorrectly typed cython local variable leading to TypeError when + calling ``calcQuadraticArcLength`` (#3288). +- [feaLib/otlLib] Better error message when building Coverage table with missing glyph (#3286). + +4.43.0 (released 2023-09-29) +---------------------------- + +- [subset] Set up lxml ``XMLParser(resolve_entities=False)`` when parsing OT-SVG documents + to prevent XML External Entity (XXE) attacks (9f61271dc): + https://codeql.github.com/codeql-query-help/python/py-xxe/ +- [varLib.iup] Added workaround for a Cython bug in ``iup_delta_optimize`` that was + leading to IUP tolerance being incorrectly initialised, resulting in sub-optimal deltas + (60126435d, cython/cython#5732). +- [varLib] Added new command-line entry point ``fonttools varLib.avar`` to add an + ``avar`` table to an existing VF from axes mappings in a .designspace file (0a3360e52). +- [instancer] Fixed bug whereby no longer used variation regions were not correctly pruned + after VarData optimization (#3268). +- Added support for Python 3.12 (#3283). + +4.42.1 (released 2023-08-20) +---------------------------- + +- [t1Lib] Fixed several Type 1 issues (#3238, #3240). +- [otBase/packer] Allow sharing tables reached by different offset sizes (#3241, #3236). +- [varLib/merger] Fix Cursive attachment merging error when all anchors are NULL (#3248, #3247). +- [ttLib] Fixed warning when calling ``addMultilingualName`` and ``ttFont`` parameter was not + passed on to ``findMultilingualName`` (#3253). + +4.42.0 (released 2023-08-02) +---------------------------- + +- [varLib] Use sentinel value 0xFFFF to mark a glyph advance in hmtx/vmtx as non + participating, allowing sparse masters to contain glyphs for variation purposes other + than {H,V}VAR (#3235). +- [varLib/cff] Treat empty glyphs in non-default masters as missing, thus not participating + in CFF2 delta computation, similarly to how varLib already treats them for gvar (#3234). +- Added varLib.avarPlanner script to deduce 'correct' avar v1 axis mappings based on + glyph average weights (#3223). + +4.41.1 (released 2023-07-21) +---------------------------- + +- [subset] Fixed perf regression in v4.41.0 by making ``NameRecordVisitor`` only visit + tables that do contain nameID references (#3213, #3214). +- [varLib.instancer] Support instancing fonts containing null ConditionSet offsets in + FeatureVariationRecords (#3211, #3212). +- [statisticsPen] Report font glyph-average weight/width and font-wide slant. +- [fontBuilder] Fixed head.created date incorrectly set to 0 instead of the current + timestamp, regression introduced in v4.40.0 (#3210). +- [varLib.merger] Support sparse ``CursivePos`` masters (#3209). + +4.41.0 (released 2023-07-12) +---------------------------- + +- [fontBuilder] Fixed bug in setupOS2 with default panose attribute incorrectly being + set to a dict instead of a Panose object (#3201). +- [name] Added method to ``removeUnusedNameRecords`` in the user range (#3185). +- [varLib.instancer] Fixed issue with L4 instancing (moving default) (#3179). +- [cffLib] Use latin1 so we can roundtrip non-ASCII in {Full,Font,Family}Name (#3202). +- [designspaceLib] Mark <source name="..."> as optional in docs (as it is in the code). +- [glyf-1] Fixed drawPoints() bug whereby last cubic segment becomes quadratic (#3189, #3190). +- [fontBuilder] Propagate the 'hidden' flag to the fvar Axis instance (#3184). +- [fontBuilder] Update setupAvar() to also support avar 2, fixing ``_add_avar()`` call + site (#3183). +- Added new ``voltLib.voltToFea`` submodule (originally Tiro Typeworks' "Volto") for + converting VOLT OpenType Layout sources to FEA format (#3164). + +4.40.0 (released 2023-06-12) +---------------------------- + +- Published native binary wheels to PyPI for all the python minor versions and platform + and architectures currently supported that would benefit from this. They will include + precompiled Cython-accelerated modules (e.g. cu2qu) without requiring to compile them + from source. The pure-python wheel and source distribution will continue to be + published as always (pip will automatically chose them when no binary wheel is + available for the given platform, e.g. pypy). Use ``pip install --no-binary=fonttools fonttools`` + to expliclity request pip to install from the pure-python source. +- [designspaceLib|varLib] Add initial support for specifying axis mappings and build + ``avar2`` table from those (#3123). +- [feaLib] Support variable ligature caret position (#3130). +- [varLib|glyf] Added option to --drop-implied-oncurves; test for impliable oncurve + points either before or after rounding (#3146, #3147, #3155, #3156). +- [TTGlyphPointPen] Don't error with empty contours, simply ignore them (#3145). +- [sfnt] Fixed str vs bytes remnant of py3 transition in code dealing with de/compiling + WOFF metadata (#3129). +- [instancer-solver] Fixed bug when moving default instance with sparse masters (#3139, #3140). +- [feaLib] Simplify variable scalars that don’t vary (#3132). +- [pens] Added filter pen that explicitly emits closing line when lastPt != movePt (#3100). +- [varStore] Improve optimize algorithm and better document the algorithm (#3124, #3127). + Added ``quantization`` option (#3126). +- Added CI workflow config file for building native binary wheels (#3121). +- [fontBuilder] Added glyphDataFormat=0 option; raise error when glyphs contain cubic + outlines but glyphDataFormat was not explicitly set to 1 (#3113, #3119). +- [subset] Prune emptied GDEF.MarkGlyphSetsDef and remap indices; ensure GDEF is + subsetted before GSUB and GPOS (#3114, #3118). +- [xmlReader] Fixed issue whereby DSIG table data was incorrectly parsed (#3115, #2614). +- [varLib/merger] Fixed merging of SinglePos with pos=0 (#3111, #3112). +- [feaLib] Demote "Feature has not been defined" error to a warning when building aalt + and referenced feature is empty (#3110). +- [feaLib] Dedupe multiple substitutions with classes (#3105). + +4.39.4 (released 2023-05-10) +---------------------------- + +- [varLib.interpolatable] Allow for sparse masters (#3075) +- [merge] Handle differing default/nominalWidthX in CFF (#3070) +- [ttLib] Add missing main.py file to ttLib package (#3088) +- [ttx] Fix missing composite instructions in XML (#3092) +- [ttx] Fix split tables option to work on filenames containing '%' (#3096) +- [featureVars] Process lookups for features other than rvrn last (#3099) +- [feaLib] support multiple substitution with classes (#3103) + +4.39.3 (released 2023-03-28) +---------------------------- + +- [sbix] Fixed TypeError when compiling empty glyphs whose imageData is None, regression + was introduced in v4.39 (#3059). +- [ttFont] Fixed AttributeError on python <= 3.10 when opening a TTFont from a tempfile + SpooledTemporaryFile, seekable method only added on python 3.11 (#3052). + +4.39.2 (released 2023-03-16) +---------------------------- + +- [varLib] Fixed regression introduced in 4.39.1 whereby an incomplete 'STAT' table + would be built even though a DesignSpace v5 did contain 'STAT' definitions (#3045, #3046). + +4.39.1 (released 2023-03-16) +---------------------------- + +- [avar2] Added experimental support for reading/writing avar version 2 as specified in + this draft proposal: https://github.com/harfbuzz/boring-expansion-spec/blob/main/avar2.md +- [glifLib] Wrap underlying XML library exceptions with GlifLibError when parsing GLIFs, + and also print the name and path of the glyph that fails to be parsed (#3042). +- [feaLib] Consult avar for normalizing user-space values in ConditionSets and in + VariableScalars (#3042, #3043). +- [ttProgram] Handle string input to Program.fromAssembly() (#3038). +- [otlLib] Added a config option to emit GPOS 7 lookups, currently disabled by default + because of a macOS bug (#3034). +- [COLRv1] Added method to automatically compute ClipBoxes (#3027). +- [ttFont] Fixed getGlyphID to raise KeyError on missing glyphs instead of returning + None. The regression was introduced in v4.27.0 (#3032). +- [sbix] Fixed UnboundLocalError: cannot access local variable 'rawdata' (#3031). +- [varLib] When building VF, do not overwrite a pre-existing ``STAT`` table that was built + with feaLib from FEA feature file. Also, added support for building multiple VFs + defined in Designspace v5 from ``fonttools varLib`` script (#3024). +- [mtiLib] Only add ``Debg`` table with lookup names when ``FONTTOOLS_LOOKUP_DEBUGGING`` + env variable is set (#3023). + +4.39.0 (released 2023-03-06) +---------------------------- + +- [mtiLib] Optionally add `Debg` debug info for MTI feature builds (#3018). +- [ttx] Support reading input file from standard input using special `-` character, + similar to existing `-o -` option to write output to standard output (#3020). +- [cython] Prevent ``cython.compiled`` raise AttributeError if cython not installed + properly (#3017). +- [OS/2] Guard against ZeroDivisionError when calculating xAvgCharWidth in the unlikely + scenario no glyph has non-zero advance (#3015). +- [subset] Recompute xAvgCharWidth independently of --no-prune-unicode-ranges, + previously the two options were involuntarily bundled together (#3012). +- [fontBuilder] Add ``debug`` parameter to addOpenTypeFeatures method to add source + debugging information to the font in the ``Debg`` private table (#3008). +- [name] Make NameRecord `__lt__` comparison not fail on Unicode encoding errors (#3006). +- [featureVars] Fixed bug in ``overlayBox`` (#3003, #3005). +- [glyf] Added experimental support for cubic bezier curves in TrueType glyf table, as + outlined in glyf v1 proposal (#2988): + https://github.com/harfbuzz/boring-expansion-spec/blob/main/glyf1-cubicOutlines.md +- Added new qu2cu module and related qu2cuPen, the reverse of cu2qu for converting + TrueType quadratic splines to cubic bezier curves (#2993). +- [glyf] Added experimental support for reading and writing Variable Composites/Components + as defined in glyf v1 spec proposal (#2958): + https://github.com/harfbuzz/boring-expansion-spec/blob/main/glyf1-varComposites.md. +- [pens]: Added `addVarComponent` method to pen protocols' base classes, which pens can implement + to handle varcomponents (by default they get decomposed) (#2958). +- [misc.transform] Added DecomposedTransform class which implements an affine transformation + with separate translate, rotation, scale, skew, and transformation-center components (#2598) +- [sbix] Ensure Glyph.referenceGlyphName is set; fixes error after dumping and + re-compiling sbix table with 'dupe' glyphs (#2984). +- [feaLib] Be cleverer when merging chained single substitutions into same lookup + when they are specified using the inline notation (#2150, #2974). +- [instancer] Clamp user-inputted axis ranges to those of fvar (#2959). +- [otBase/subset] Define ``__getstate__`` for BaseTable so that a copied/pickled 'lazy' + object gets its own OTTableReader to read from; incidentally fixes a bug while + subsetting COLRv1 table containing ClipBoxes on python 3.11 (#2965, #2968). +- [sbix] Handle glyphs with "dupe" graphic type on compile correctly (#2963). +- [glyf] ``endPointsOfContours`` field should be unsigned! Kudos to behdad for + spotting one of the oldest bugs in FT. Probably nobody has ever dared to make + glyphs with more than 32767 points... (#2957). +- [feaLib] Fixed handling of ``ignore`` statements with unmarked glyphs to match + makeotf behavior, which assumes the first glyph is marked (#2950). +- Reformatted code with ``black`` and enforce new code style via CI check (#2925). +- [feaLib] Sort name table entries following OT spec prescribed order in the builder (#2927). +- [cu2quPen] Add Cu2QuMultiPen that converts multiple outlines at a time in + interpolation compatible way; its methods take a list of tuples arguments + that would normally be passed to individual segment pens, and at the end it + dispatches the converted outlines to each pen (#2912). +- [reverseContourPen/ttGlyphPen] Add outputImpliedClosingLine option (#2913, #2914, + #2921, #2922, #2995). +- [gvar] Avoid expanding all glyphs unnecessarily upon compile (#2918). +- [scaleUpem] Fixed bug whereby CFF2 vsindex was scaled; it should not (#2893, #2894). +- [designspaceLib] Add DS.getAxisByTag and refactor getAxis (#2891). +- [unicodedata] map Zmth<->math in ot_tag_{to,from}_script (#1737, #2889). +- [woff2] Support encoding/decoding OVERLAP_SIMPLE glyf flags (#2576, #2884). +- [instancer] Update OS/2 class and post.italicAngle when default moved (L4) +- Dropped support for Python 3.7 which reached EOL, fontTools requires 3.8+. +- [instancer] Fixed instantiateFeatureVariations logic when a rule range becomes + default-applicable (#2737, #2880). +- [ttLib] Add main to ttFont and ttCollection that just decompile and re-compile the + input font (#2869). +- [featureVars] Insert 'rvrn' lookup at the beginning of LookupList, to work around bug + in Apple implementation of 'rvrn' feature which the spec says it should be processed + early whereas on macOS 10.15 it follows lookup order (#2140, #2867). +- [instancer/mutator] Remove 'DSIG' table if present. +- [svgPathPen] Don't close path in endPath(), assume open unless closePath() (#2089, #2865). + +4.38.0 (released 2022-10-21) +---------------------------- + +- [varLib.instancer] Added support for L4 instancing, i.e. moving the default value of + an axis while keeping it variable. Thanks Behdad! (#2728, #2861). + It's now also possible to restrict an axis min/max values beyond the current default + value, e.g. a font wght has min=100, def=400, max=900 and you want a partial VF that + only varies between 500 and 700, you can now do that. + You can either specify two min/max values (wght=500:700), and the new default will be + set to either the minimum or maximum, depending on which one is closer to the current + default (e.g. 500 in this case). Or you can specify three values (e.g. wght=500:600:700) + to specify the new default value explicitly. +- [otlLib/featureVars] Set a few Count values so one doesn't need to compile the font + to update them (#2860). +- [varLib.models] Make extrapolation work for 2-master models as well where one master + is at the default location (#2843, #2846). + Add optional extrapolate=False to normalizeLocation() (#2847, #2849). +- [varLib.cff] Fixed sub-optimal packing of CFF2 deltas by no longer rounding them to + integer (#2838). +- [scaleUpem] Calculate numShorts in VarData after scale; handle CFF hintmasks (#2840). + +4.37.4 (released 2022-09-30) +---------------------------- + +- [subset] Keep nameIDs used by CPAL palette entry labels (#2837). +- [varLib] Avoid negative hmtx values when creating font from variable CFF2 font (#2827). +- [instancer] Don't prune stat.ElidedFallbackNameID (#2828). +- [unicodedata] Update Scripts/Blocks to Unicode 15.0 (#2833). + +4.37.3 (released 2022-09-20) +---------------------------- + +- Fix arguments in calls to (glyf) glyph.draw() and drawPoints(), whereby offset wasn't + correctly passed down; this fix also exposed a second bug, where lsb and tsb were not + set (#2824, #2825, adobe-type-tools/afdko#1560). + +4.37.2 (released 2022-09-15) +---------------------------- + +- [subset] Keep CPAL table and don't attempt to prune unused color indices if OT-SVG + table is present even if COLR table was subsetted away; OT-SVG may be referencing the + CPAL table; for now we assume that's the case (#2814, #2815). +- [varLib.instancer] Downgrade GPOS/GSUB version if there are no more FeatureVariations + after instancing (#2812). +- [subset] Added ``--no-lazy`` to optionally load fonts eagerly (mostly to ease + debugging of table lazy loading, no practical effects) (#2807). +- [varLib] Avoid building empty COLR.DeltaSetIndexMap with only identity mappings (#2803). +- [feaLib] Allow multiple value record types (by promoting to the most general format) + within the same PairPos subtable; e.g. this allows variable and non variable kerning + rules to share the same subtable. This also fixes a bug whereby some kerning pairs + would become unreachable while shapiong because of premature subtable splitting (#2772, #2776). +- [feaLib] Speed up ``VarScalar`` by caching models for recurring master locations (#2798). +- [feaLib] Optionally cythonize ``feaLib.lexer``, speeds up parsing FEA a bit (#2799). +- [designspaceLib] Avoid crash when handling unbounded rule conditions (#2797). +- [post] Don't crash if ``post`` legacy format 1 is malformed/improperly used (#2786) +- [gvar] Don't be "lazy" (load all glyph variations up front) when TTFont.lazy=False (#2771). +- [TTFont] Added ``normalizeLocation`` method to normalize a location dict from the + font's defined axes space (also known as "user space") into the normalized (-1..+1) + space. It applies ``avar`` mapping if the font contains an ``avar`` table (#2789). +- [TTVarGlyphSet] Support drawing glyph instances from CFF2 variable glyph set (#2784). +- [fontBuilder] Do not error when building cmap if there are zero code points (#2785). +- [varLib.plot] Added ability to plot a variation model and set of accompaning master + values corresponding to the model's master locations into a pyplot figure (#2767). +- [Snippets] Added ``statShape.py`` script to draw statistical shape of a glyph as an + ellips (requires pycairo) (baecd88). +- [TTVarGlyphSet] implement drawPoints natively, avoiding going through + SegmentToPointPen (#2778). +- [TTVarGlyphSet] Fixed bug whereby drawing a composite glyph multiple times, its + components would shif; needed an extra copy (#2774). + +4.37.1 (released 2022-08-24) +---------------------------- + +- [subset] Fixed regression introduced with v4.37.0 while subsetting the VarStore of + ``HVAR`` and ``VVAR`` tables, whereby an ``AttributeError: subset_varidxes`` was + thrown because an apparently unused import statement (with the side-effect of + dynamically binding that ``subset_varidxes`` method to the VarStore class) had been + accidentally deleted in an unrelated PR (#2679, #2773). +- [pens] Added ``cairoPen`` (#2678). +- [gvar] Read ``gvar`` more lazily by not parsing all of the ``glyf`` table (#2771). +- [ttGlyphSet] Make ``drawPoints(pointPen)`` method work for CFF fonts as well via + adapter pen (#2770). + +4.37.0 (released 2022-08-23) +---------------------------- + +- [varLib.models] Reverted PR #2717 which added support for "narrow tents" in v4.36.0, + as it introduced a regression (#2764, #2765). It will be restored in upcoming release + once we found a solution to the bug. +- [cff.specializer] Fixed issue in charstring generalizer with the ``blend`` operator + (#2750, #1975). +- [varLib.models] Added support for extrapolation (#2757). +- [ttGlyphSet] Ensure the newly added ``_TTVarGlyphSet`` inherits from ``_TTGlyphSet`` + to keep backward compatibility with existing API (#2762). +- [kern] Allow compiling legacy kern tables with more than 64k entries (d21cfdede). +- [visitor] Added new visitor API to traverse tree of objects and dispatch based + on the attribute type: cf. ``fontTools.misc.visitor`` and ``fontTools.ttLib.ttVisitor``. Added ``fontTools.ttLib.scaleUpem`` module that uses the latter to + change a font's units-per-em and scale all the related fields accordingly (#2718, + #2755). + +4.36.0 (released 2022-08-17) +---------------------------- + +- [varLib.models] Use a simpler model that generates narrower "tents" (regions, master + supports) whenever possible: specifically when any two axes that actively "cooperate" + (have masters at non-zero positions for both axes) have a complete set of intermediates. + The simpler algorithm produces fewer overlapping regions and behaves better with + respect to rounding at the peak positions than the generic solver, always matching + intermediate masters exactly, instead of maximally 0.5 units off. This may be useful + when 100% metrics compatibility is desired (#2218, #2717). +- [feaLib] Remove warning when about ``GDEF`` not being built when explicitly not + requested; don't build one unconditonally even when not requested (#2744, also works + around #2747). +- [ttFont] ``TTFont.getGlyphSet`` method now supports selecting a location that + represents an instance of a variable font (supports both user-scale and normalized + axes coordinates via the ``normalized=False`` parameter). Currently this only works + for TrueType-flavored variable fonts (#2738). + +4.35.0 (released 2022-08-15) +---------------------------- + +- [otData/otConverters] Added support for 'biased' PaintSweepGradient start/end angles + to match latest COLRv1 spec (#2743). +- [varLib.instancer] Fixed bug in ``_instantiateFeatureVariations`` when at the same + time pinning one axis and restricting the range of a subsequent axis; the wrong axis + tag was being used in the latter step (as the records' axisIdx was updated in the + preceding step but looked up using the old axes order in the following step) (#2733, + #2734). +- [mtiLib] Pad script tags with space when less than 4 char long (#1727). +- [merge] Use ``'.'`` instead of ``'#'`` in duplicate glyph names (#2742). +- [gvar] Added support for lazily loading glyph variations (#2741). +- [varLib] In ``build_many``, we forgot to pass on ``colr_layer_reuse`` parameter to + the ``build`` method (#2730). +- [svgPathPen] Add a main that prints SVG for input text (6df779fd). +- [cffLib.width] Fixed off-by-one in optimized values; previous code didn't match the + code block above it (2963fa50). +- [varLib.interpolatable] Support reading .designspace and .glyphs files (via optional + ``glyphsLib``). +- Compile some modules with Cython when available and building/installing fonttools + from source: ``varLib.iup`` (35% faster), ``pens.momentsPen`` (makes + ``varLib.interpolatable`` 3x faster). +- [feaLib] Allow features to be built for VF without also building a GDEF table (e.g. + only build GSUB); warn when GDEF would be needed but isn't requested (#2705, 2694). +- [otBase] Fixed ``AttributeError`` when uharfbuzz < 0.23.0 and 'repack' method is + missing (32aa8eaf). Use new ``uharfbuzz.repack_with_tag`` when available (since + uharfbuzz>=0.30.0), enables table-specific optimizations to be performed during + repacking (#2724). +- [statisticsPen] By default report all glyphs (4139d891). Avoid division-by-zero + (52b28f90). +- [feaLib] Added missing required argument to FeatureLibError exception (#2693) +- [varLib.merge] Fixed error during error reporting (#2689). Fixed undefined + ``NotANone`` variable (#2714). + +4.34.4 (released 2022-07-07) +---------------------------- + +- Fixed typo in varLib/merger.py that causes NameError merging COLR glyphs + containing more than 255 layers (#2685). + +4.34.3 (released 2022-07-07) +---------------------------- + +- [designspaceLib] Don't make up bad PS names when no STAT data (#2684) + +4.34.2 (released 2022-07-06) +---------------------------- + +- [varStore/subset] fixed KeyError exception to do with NO_VARIATION_INDEX while + subsetting varidxes in GPOS/GDEF (a08140d). + +4.34.1 (released 2022-07-06) +---------------------------- + +- [instancer] When optimizing HVAR/VVAR VarStore, use_NO_VARIATION_INDEX=False to avoid + including NO_VARIATION_INDEX in AdvWidthMap, RsbMap, LsbMap mappings, which would + push the VarIdx width to maximum (4bytes), which is not desirable. This also fixes + a hard crash when attempting to subset a varfont after it had been partially instanced + with use_NO_VARIATION_INDEX=True. + +4.34.0 (released 2022-07-06) +---------------------------- + +- [instancer] Set RIBBI bits in head and OS/2 table when cutting instances and the + subfamily nameID=2 contains strings like 'Italic' or 'Bold' (#2673). +- [otTraverse] Addded module containing methods for traversing trees of otData tables + (#2660). +- [otTables] Made DeltaSetIndexMap TTX dump less verbose by omitting no-op entries + (#2660). +- [colorLib.builder] Added option to disable PaintColrLayers's reuse of layers from + LayerList (#2660). +- [varLib] Added support for merging multiple master COLRv1 tables into a variable + COLR table (#2660, #2328). Base color glyphs of same name in different masters must have + identical paint graph structure (incl. number of layers, palette indices, number + of color line stops, corresponding paint formats at each level of the graph), + but can differ in the variable fields (e.g. PaintSolid.Alpha). PaintVar* tables + are produced when this happens and a VarStore/DeltaSetIndexMap is added to the + variable COLR table. It is possible for non-default masters to be 'sparse', i.e. + omit some of the color glyphs present in the default master. +- [feaLib] Let the Parser set nameIDs 1 through 6 that were previously reserved (#2675). +- [varLib.varStore] Support NO_VARIATION_INDEX in optimizer and instancer. +- [feaLib] Show all missing glyphs at once at end of parsing (#2665). +- [varLib.iup] Rewrite force-set conditions and limit DP loopback length (#2651). + For Noto Sans, IUP time drops from 23s down to 9s, with only a slight size increase + in the final font. This basically turns the algorithm from O(n^3) into O(n). +- [featureVars] Report about missing glyphs in substitution rules (#2654). +- [mutator/instancer] Added CLI flag to --no-recalc-timestamp (#2649). +- [SVG] Allow individual SVG documents in SVG OT table to be compressed on uncompressed, + and remember that when roundtripping to/from ttx. The SVG.docList is now a list + of SVGDocument namedtuple-like dataclass containing an extra ``compressed`` field, + and no longer a bare 3-tuple (#2645). +- [designspaceLib] Check for descriptor types with hasattr() to allow custom classes + that don't inherit the default descriptors (#2634). +- [subset] Enable sharing across subtables of extension lookups for harfbuzz packing + (#2626). Updated how table packing falls back to fontTools from harfbuzz (#2668). +- [subset] Updated default feature tags following current Harfbuzz (#2637). +- [svgLib] Fixed regex for real number to support e.g. 1e-4 in addition to 1.0e-4. + Support parsing negative rx, ry on arc commands (#2596, #2611). +- [subset] Fixed subsetting SinglePosFormat2 when ValueFormat=0 (#2603). + +4.33.3 (released 2022-04-26) +---------------------------- + +- [designspaceLib] Fixed typo in ``deepcopyExceptFonts`` method, preventing font + references to be transferred (#2600). Fixed another typo in the name of ``Range`` + dataclass's ``__post_init__`` magic method (#2597). + +4.33.2 (released 2022-04-22) +---------------------------- + +- [otBase] Make logging less verbose when harfbuzz fails to serialize. Do not exit + at the first failure but continue attempting to fix offset overflow error using + the pure-python serializer even when the ``USE_HARFBUZZ_REPACKER`` option was + explicitly set to ``True``. This is normal with fonts with relatively large + tables, at least until hb.repack implements proper table splitting. + +4.33.1 (released 2022-04-22) +---------------------------- + +- [otlLib] Put back the ``FONTTOOLS_GPOS_COMPACT_MODE`` environment variable to fix + regression in ufo2ft (and thus fontmake) introduced with v4.33.0 (#2592, #2593). + This is deprecated and will be removed one ufo2ft gets updated to use the new + config setup. + +4.33.0 (released 2022-04-21) +---------------------------- + +- [OS/2 / merge] Automatically recalculate ``OS/2.xAvgCharWidth`` after merging + fonts with ``fontTools.merge`` (#2591, #2538). +- [misc/config] Added ``fontTools.misc.configTools`` module, a generic configuration + system (#2416, #2439). + Added ``fontTools.config`` module, a fontTools-specific configuration + system using ``configTools`` above. + Attached a ``Config`` object to ``TTFont``. +- [otlLib] Replaced environment variable for GPOS compression level with an + equivalent option using the new config system. +- [designspaceLib] Incremented format version to 5.0 (#2436). + Added discrete axes, variable fonts, STAT information, either design- or + user-space location on instances. + Added ``fontTools.designspaceLib.split`` module to split a designspace + into sub-spaces that interpolate and that represent the variable fonts + listed in the document. + Made instance names optional and allow computing them from STAT data instead. + Added ``fontTools.designspaceLib.statNames`` module. + Allow instances to have the same location as a previously defined STAT label. + Deprecated some attributes: + ``SourceDescriptor``: ``copyLib``, ``copyInfo``, ``copyGroups``, ``copyFeatures``. + ``InstanceDescriptor``: ``kerning``, ``info``; ``glyphs``: use rules or sparse + sources. + For both, ``location``: use the more explicit designLocation. + Note: all are soft deprecations and existing code should keep working. + Updated documentation for Python methods and the XML format. +- [varLib] Added ``build_many`` to build several variable fonts from a single + designspace document (#2436). + Added ``fontTools.varLib.stat`` module to build STAT tables from a designspace + document. +- [otBase] Try to use the Harfbuzz Repacker for packing GSUB/GPOS tables when + ``uharfbuzz`` python bindings are available (#2552). Disable it by setting the + "fontTools.ttLib.tables.otBase:USE_HARFBUZZ_REPACKER" config option to ``False``. + If the option is set explicitly to ``True`` but ``uharfbuzz`` can't be imported + or fails to serialize for any reasons, an error will be raised (ImportError or + uharfbuzz errors). +- [CFF/T2] Ensure that ``pen.closePath()`` gets called for CFF2 charstrings (#2577). + Handle implicit CFF2 closePath within ``T2OutlineExtractor`` (#2580). + +4.32.0 (released 2022-04-08) +---------------------------- + +- [otlLib] Disable GPOS7 optimization to work around bug in Apple CoreText. + Always force Chaining GPOS8 for now (#2540). +- [glifLib] Added ``outputImpliedClosingLine=False`` parameter to ``Glyph.draw()``, + to control behaviour of ``PointToSegmentPen`` (6b4e2e7). +- [varLib.interpolatable] Check for wrong contour starting point (#2571). +- [cffLib] Remove leftover ``GlobalState`` class and fix calls to ``TopDictIndex()`` + (#2569, #2570). +- [instancer] Clear ``AxisValueArray`` if it is empty after instantiating (#2563). + +4.31.2 (released 2022-03-22) +---------------------------- + +- [varLib] fix instantiation of GPOS SinglePos values (#2555). + +4.31.1 (released 2022-03-18) +---------------------------- + +- [subset] fix subsetting OT-SVG when glyph id attribute is on the root ``<svg>`` + element (#2553). + +4.31.0 (released 2022-03-18) +---------------------------- + +- [ttCollection] Fixed 'ResourceWarning: unclosed file' warning (#2549). +- [varLib.merger] Handle merging SinglePos with valueformat=0 (#2550). +- [ttFont] Update glyf's glyphOrder when calling TTFont.setGlyphOrder() (#2544). +- [ttFont] Added ``ensureDecompiled`` method to load all tables irrespective + of the ``lazy`` attribute (#2551). +- [otBase] Added ``iterSubTable`` method to iterate over BaseTable's children of + type BaseTable; useful for traversing a tree of otTables (#2551). + +4.30.0 (released 2022-03-10) +---------------------------- + +- [varLib] Added debug logger showing the glyph name for which ``gvar`` is built (#2542). +- [varLib.errors] Fixed undefined names in ``FoundANone`` and ``UnsupportedFormat`` + exceptions (ac4d5611). +- [otlLib.builder] Added ``windowsNames`` and ``macNames`` (bool) parameters to the + ``buildStatTabe`` function, so that one can select whether to only add one or both + of the two sets (#2528). +- [t1Lib] Added the ability to recreate PostScript stream (#2504). +- [name] Added ``getFirstDebugName``, ``getBest{Family,SubFamily,Full}Name`` methods (#2526). + +4.29.1 (released 2022-02-01) +---------------------------- + +- [colorLib] Fixed rounding issue with radial gradient's start/end circles inside + one another (#2521). +- [freetypePen] Handle rotate/skew transform when auto-computing width/height of the + buffer; raise PenError wen missing moveTo (#2517) + +4.29.0 (released 2022-01-24) +---------------------------- + +- [ufoLib] Fixed illegal characters and expanded reserved filenames (#2506). +- [COLRv1] Don't emit useless PaintColrLayers of lenght=1 in LayerListBuilder (#2513). +- [ttx] Removed legacy ``waitForKeyPress`` method on Windows (#2509). +- [pens] Added FreeTypePen that uses ``freetype-py`` and the pen protocol for + rasterizating outline paths (#2494). +- [unicodedata] Updated the script direction list to Unicode 14.0 (#2484). + Bumped unicodedata2 dependency to 14.0 (#2499). +- [psLib] Fixed type of ``fontName`` in ``suckfont`` (#2496). + +4.28.5 (released 2021-12-19) +---------------------------- + +- [svgPathPen] Continuation of #2471: make sure all occurrences of ``str()`` are now + replaced with user-defined ``ntos`` callable. +- [merge] Refactored code into submodules, plus several bugfixes and improvements: + fixed duplicate-glyph-resolution GSUB-lookup generation code; use tolerance in glyph + comparison for empty glyph's width; ignore space of default ignorable glyphs; + downgrade duplicates-resolution missing-GSUB from assert to warn; added --drop-tables + option (#2473, #2475, #2476). + +4.28.4 (released 2021-12-15) +---------------------------- + +- [merge] Merge GDEF marksets in Lookups properly (#2474). +- [feaLib] Have ``fontTools feaLib`` script exit with error code when build fails (#2459) +- [svgPathPen] Added ``ntos`` option to customize number formatting (e.g. rounding) (#2471). +- [subset] Speed up subsetting of large CFF fonts (#2467). +- [otTables] Speculatively promote lookups to extension to speed up compilation. If the + offset to lookup N is too big to fit in a ushort, the offset to lookup N+1 is going to + be too big as well, so we promote to extension all lookups from lookup N onwards (#2465). + +4.28.3 (released 2021-12-03) +---------------------------- + +- [subset] Fixed bug while subsetting ``COLR`` table, whereby incomplete layer records + pointing to missing glyphs were being retained leading to ``struct.error`` upon + compiling. Make it so that ``glyf`` glyph closure, which follows the ``COLR`` glyph + closure, does not influence the ``COLR`` table subsetting (#2461, #2462). +- [docs] Fully document the ``cmap`` and ``glyf`` tables (#2454, #2457). +- [colorLib.unbuilder] Fixed CLI by deleting no longer existing parameter (180bb1867). + +4.28.2 (released 2021-11-22) +---------------------------- + +- [otlLib] Remove duplicates when building coverage (#2433). +- [docs] Add interrogate configuration (#2443). +- [docs] Remove comment about missing “start” optional argument to ``calcChecksum`` (#2448). +- [cu2qu/cli] Adapt to the latest ufoLib2. +- [subset] Support subsetting SVG table and remove it from the list of drop by default tables (#534). +- [subset] add ``--pretty-svg`` option to pretty print SVG table contents (#2452). +- [merge] Support merging ``CFF`` tables (CID-keyed ``CFF`` is still not supported) (#2447). +- [merge] Support ``--output-file`` (#2447). +- [docs] Split table docs into individual pages (#2444). +- [feaLib] Forbid empty classes (#2446). +- [docs] Improve documentation for ``fontTools.ttLib.ttFont`` (#2442). + +4.28.1 (released 2021-11-08) +---------------------------- + +- [subset] Fixed AttributeError while traversing a color glyph's Paint graph when there is no + LayerList, which is optional (#2441). + +4.28.0 (released 2021-11-05) +---------------------------- + +- Dropped support for EOL Python 3.6, require Python 3.7 (#2417). +- [ufoLib/glifLib] Make filename-clash checks faster by using a set instead of a list (#2422). +- [subset] Don't crash if optional ClipList and LayerList are ``None`` (empty) (#2424, 2439). +- [OT-SVG] Removed support for old deprecated version 1 and embedded color palettes, + which were never officially part of the OpenType SVG spec. Upon compile, reuse offsets + to SVG documents that are identical (#2430). +- [feaLib] Added support for Variable Feature File syntax. This is experimental and subject + to change until it is finalized in the Adobe FEA spec (#2432). +- [unicodedata] Update Scripts/ScriptExtensions/Blocks to UnicodeData 14.0 (#2437). + +4.27.1 (released 2021-09-23) +---------------------------- + +- [otlLib] Fixed error when chained contextual lookup builder overflows (#2404, #2411). +- [bezierTools] Fixed two floating-point bugs: one when computing `t` for a point + lying on an almost horizontal/vertical line; another when computing the intersection + point between a curve and a line (#2413). + +4.27.0 (released 2021-09-14) +---------------------------- + +- [ttLib/otTables] Cleaned up virtual GID handling: allow virtual GIDs in ``Coverage`` + and ``ClassDef`` readers; removed unused ``allowVID`` argument from ``TTFont`` + constructor, and ``requireReal`` argument in ``TTFont.getGlyphID`` method. + Make ``TTFont.setGlyphOrder`` clear reverse glyphOrder map, and assume ``glyphOrder`` + internal attribute is never modified outside setGlyphOrder; added ``TTFont.getGlyphNameMany`` + and ``getGlyphIDMany`` (#1536, #1654, #2334, #2398). +- [py23] Dropped internal use of ``fontTools.py23`` module to fix deprecation warnings + in client code that imports from fontTools (#2234, #2399, #2400). +- [subset] Fix subsetting COLRv1 clip boxes when font is loaded lazily (#2408). + +4.26.2 (released 2021-08-09) +---------------------------- + +- [otTables] Added missing ``CompositeMode.PLUS`` operator (#2390). + +4.26.1 (released 2021-08-03) +---------------------------- + +- [transform] Added ``transformVector`` and ``transformVectors`` methods to the + ``Transform`` class. Similar to ``transformPoint`` but ignore the translation + part (#2386). + +4.26.0 (released 2021-08-03) +---------------------------- + +- [xmlWriter] Default to ``"\n"`` for ``newlinestr`` instead of platform-specific + ``os.linesep`` (#2384). +- [otData] Define COLRv1 ClipList and ClipBox (#2379). +- [removeOverlaps/instancer] Added --ignore-overlap-errors option to work around + Skia PathOps.Simplify bug (#2382, #2363, google/fonts#3365). +- NOTE: This will be the last version to support Python 3.6. FontTools will require + Python 3.7 or above from the next release (#2350) + +4.25.2 (released 2021-07-26) +---------------------------- + +- [COLRv1] Various changes to sync with the latest CORLv1 draft spec. In particular: + define COLR.VarIndexMap, remove/inline ColorIndex struct, add VarIndexBase to ``PaintVar*`` tables (#2372); + add reduced-precicion specialized transform Paints; + define Angle as fraction of half circle encoded as F2Dot14; + use FWORD (int16) for all Paint center coordinates; + change PaintTransform to have an offset to Affine2x3; +- [ttLib] when importing XML, only set sfntVersion if the font has no reader and is empty (#2376) + +4.25.1 (released 2021-07-16) +---------------------------- + +- [ttGlyphPen] Fixed bug in ``TTGlyphPointPen``, whereby open contours (i.e. starting + with segmentType "move") would throw ``NotImplementedError``. They are now treated + as if they are closed, like with the ``TTGlyphPen`` (#2364, #2366). + +4.25.0 (released 2021-07-05) +---------------------------- + +- [tfmLib] Added new library for parsing TeX Font Metric (TFM) files (#2354). +- [TupleVariation] Make shared tuples order deterministic on python < 3.7 where + Counter (subclass of dict) doesn't remember insertion order (#2351, #2353). +- [otData] Renamed COLRv1 structs to remove 'v1' suffix and match the updated draft + spec: 'LayerV1List' -> 'LayerList', 'BaseGlyphV1List' -> 'BaseGlyphList', + 'BaseGlyphV1Record' -> 'BaseGlyphPaintRecord' (#2346). + Added 8 new ``PaintScale*`` tables: with/without centers, uniform vs non-uniform. + Added ``*AroundCenter`` variants to ``PaintRotate`` and ``PaintSkew``: the default + versions no longer have centerX/Y, but default to origin. + ``PaintRotate``, ``PaintSkew`` and ``PaintComposite`` formats were re-numbered. + NOTE: these are breaking changes; clients using the experimental COLRv1 API will + have to be updated (#2348). +- [pointPens] Allow ``GuessSmoothPointPen`` to accept a tolerance. Fixed call to + ``math.atan2`` with x/y parameters inverted. Sync the code with fontPens (#2344). +- [post] Fixed parsing ``post`` table format 2.0 when it contains extra garbage + at the end of the stringData array (#2314). +- [subset] drop empty features unless 'size' with FeatureParams table (#2324). +- [otlLib] Added ``otlLib.optimize`` module; added GPOS compaction algorithm. + The compaction can be run on existing fonts with ``fonttools otlLib.optimize`` + or using the snippet ``compact_gpos.py``. There's experimental support for + compacting fonts at compilation time using an environment variable, but that + might be removed later (#2326). + +4.24.4 (released 2021-05-25) +---------------------------- + +- [subset/instancer] Fixed ``AttributeError`` when instantiating a VF that + contains GPOS ValueRecords with ``Device`` tables but without the respective + non-Device values (e.g. ``XAdvDevice`` without ``XAdvance``). When not + explicitly set, the latter are assumed to be 0 (#2323). + +4.24.3 (released 2021-05-20) +---------------------------- + +- [otTables] Fixed ``AttributeError`` in methods that split LigatureSubst, + MultipleSubst and AlternateSubst subtables when an offset overflow occurs. + The ``Format`` attribute was removed in v4.22.0 (#2319). + +4.24.2 (released 2021-05-20) +---------------------------- + +- [ttGlyphPen] Fixed typing annotation of TTGlyphPen glyphSet parameter (#2315). +- Fixed two instances of DeprecationWarning: invalid escape sequence (#2311). + +4.24.1 (released 2021-05-20) +---------------------------- + +- [subset] Fixed AttributeError when SinglePos subtable has None Value (ValueFormat 0) + (#2312, #2313). + +4.24.0 (released 2021-05-17) +---------------------------- + +- [pens] Add ``ttGlyphPen.TTGlyphPointPen`` similar to ``TTGlyphPen`` (#2205). + +4.23.1 (released 2021-05-14) +---------------------------- + +- [subset] Fix ``KeyError`` after subsetting ``COLR`` table that initially contains + both v0 and v1 color glyphs when the subset only requested v1 glyphs; we were + not pruning the v0 portion of the table (#2308). +- [colorLib] Set ``LayerV1List`` attribute to ``None`` when empty, it's optional + in CORLv1 (#2308). + +4.23.0 (released 2021-05-13) +---------------------------- + +- [designspaceLib] Allow to use ``\\UNC`` absolute paths on Windows (#2299, #2306). +- [varLib.merger] Fixed bug where ``VarLibMergeError`` was raised with incorrect + parameters (#2300). +- [feaLib] Allow substituting a glyph class with ``NULL`` to delete multiple glyphs + (#2303). +- [glyf] Fixed ``NameError`` exception in ``getPhantomPoints`` (#2295, #2305). +- [removeOverlaps] Retry pathops.simplify after rounding path coordinates to integers + if it fails the first time using floats, to work around a rare and hard to debug + Skia bug (#2288). +- [varLib] Added support for building, reading, writing and optimizing 32-bit + ``ItemVariationStore`` as used in COLRv1 table (#2285). +- [otBase/otConverters] Add array readers/writers for int types (#2285). +- [feaLib] Allow more than one lookahead glyph/class in contextual positioning with + "value at end" (#2293, #2294). +- [COLRv1] Default varIdx should be 0xFFFFFFFF (#2297, #2298). +- [pens] Make RecordingPointPen actually pass on identifiers; replace asserts with + explicit ``PenError`` exception (#2284). +- [mutator] Round lsb for CF2 fonts as well (#2286). + +4.22.1 (released 2021-04-26) +---------------------------- + +- [feaLib] Skip references to named lookups if the lookup block definition + is empty, similarly to makeotf. This also fixes an ``AttributeError`` while + generating ``aalt`` feature (#2276, #2277). +- [subset] Fixed bug with ``--no-hinting`` implementation for Device tables (#2272, + #2275). The previous code was alwyas dropping Device tables if no-hinting was + requested, but some Device tables (DeltaFormat=0x8000) are also used to encode + variation indices and need to be retained. +- [otBase] Fixed bug in getting the ValueRecordSize when decompiling ``MVAR`` + table with ``lazy=True`` (#2273, #2274). +- [varLib/glyf/gvar] Optimized and simplified ``GlyphCoordinates`` and + ``TupleVariation`` classes, use ``bytearray`` where possible, refactored + phantom-points calculations. We measured about 30% speedup in total time + of loading master ttfs, building gvar, and saving (#2261, #2266). +- [subset] Fixed ``AssertionError`` while pruning unused CPAL palettes when + ``0xFFFF`` is present (#2257, #2259). + +4.22.0 (released 2021-04-01) +---------------------------- + +- [ttLib] Remove .Format from Coverage, ClassDef, SingleSubst, LigatureSubst, + AlternateSubst, MultipleSubst (#2238). + ATTENTION: This will change your TTX dumps! +- [misc.arrayTools] move Vector to its own submodule, and rewrite as a tuple + subclass (#2201). +- [docs] Added a terminology section for varLib (#2209). +- [varLib] Move rounding to VariationModel, to avoid error accumulation from + multiple deltas (#2214) +- [varLib] Explain merge errors in more human-friendly terms (#2223, #2226) +- [otlLib] Correct some documentation (#2225) +- [varLib/otlLib] Allow merging into VariationFont without first saving GPOS + PairPos2 (#2229) +- [subset] Improve PairPosFormat2 subsetting (#2221) +- [ttLib] TTFont.save: create file on disk as late as possible (#2253) +- [cffLib] Add missing CFF2 dict operators LanguageGroup and ExpansionFactor + (#2249) + ATTENTION: This will change your TTX dumps! + +4.21.1 (released 2021-02-26) +---------------------------- + +- [pens] Reverted breaking change that turned ``AbstractPen`` and ``AbstractPointPen`` + into abstract base classes (#2164, #2198). + +4.21.0 (released 2021-02-26) +---------------------------- + +- [feaLib] Indent anchor statements in ``asFea()`` to make them more legible and + diff-able (#2193). +- [pens] Turn ``AbstractPen`` and ``AbstractPointPen`` into abstract base classes + (#2164). +- [feaLib] Added support for parsing and building ``STAT`` table from AFDKO feature + files (#2039). +- [instancer] Added option to update name table of generated instance using ``STAT`` + table's axis values (#2189). +- [bezierTools] Added functions to compute bezier point-at-time, as well as line-line, + curve-line and curve-curve intersections (#2192). + +4.20.0 (released 2021-02-15) +---------------------------- + +- [COLRv1] Added ``unbuildColrV1`` to deconstruct COLRv1 otTables to raw json-able + data structure; it does the reverse of ``buildColrV1`` (#2171). +- [feaLib] Allow ``sub X by NULL`` sequence to delete a glyph (#2170). +- [arrayTools] Fixed ``Vector`` division (#2173). +- [COLRv1] Define new ``PaintSweepGradient`` (#2172). +- [otTables] Moved ``Paint.Format`` enum class outside of ``Paint`` class definition, + now named ``PaintFormat``. It was clashing with paint instance ``Format`` attribute + and thus was breaking lazy load of COLR table which relies on magic ``__getattr__`` + (#2175). +- [COLRv1] Replace hand-coded builder functions with otData-driven dynamic + implementation (#2181). +- [COLRv1] Define additional static (non-variable) Paint formats (#2181). +- [subset] Added support for subsetting COLR v1 and CPAL tables (#2174, #2177). +- [fontBuilder] Allow ``setupFvar`` to optionally take ``designspaceLib.AxisDescriptor`` + objects. Added new ``setupAvar`` method. Support localised names for axes and + named instances (#2185). + +4.19.1 (released 2021-01-28) +---------------------------- + +- [woff2] An initial off-curve point with an overlap flag now stays an off-curve + point after compression. + +4.19.0 (released 2021-01-25) +---------------------------- + +- [codecs] Handle ``errors`` parameter different from 'strict' for the custom + extended mac encodings (#2137, #2132). +- [featureVars] Raise better error message when a script is missing the required + default language system (#2154). +- [COLRv1] Avoid abrupt change caused by rounding ``PaintRadialGradient.c0`` when + the start circle almost touches the end circle's perimeter (#2148). +- [COLRv1] Support building unlimited lists of paints as 255-ary trees of + ``PaintColrLayers`` tables (#2153). +- [subset] Prune redundant format-12 cmap subtables when all non-BMP characters + are dropped (#2146). +- [basePen] Raise ``MissingComponentError`` instead of bare ``KeyError`` when a + referenced component is missing (#2145). + +4.18.2 (released 2020-12-16) +---------------------------- + +- [COLRv1] Implemented ``PaintTranslate`` paint format (#2129). +- [varLib.cff] Fixed unbound local variable error (#1787). +- [otlLib] Don't crash when creating OpenType class definitions if some glyphs + occur more than once (#2125). + +4.18.1 (released 2020-12-09) +---------------------------- + +- [colorLib] Speed optimization for ``LayerV1ListBuilder`` (#2119). +- [mutator] Fixed missing tab in ``interpolate_cff2_metrics`` (0957dc7a). + +4.18.0 (released 2020-12-04) +---------------------------- + +- [COLRv1] Update to latest draft: added ``PaintRotate`` and ``PaintSkew`` (#2118). +- [woff2] Support new ``brotlicffi`` bindings for PyPy (#2117). +- [glifLib] Added ``expectContentsFile`` parameter to ``GlyphSet``, for use when + reading existing UFOs, to comply with the specification stating that a + ``contents.plist`` file must exist in a glyph set (#2114). +- [subset] Allow ``LangSys`` tags in ``--layout-scripts`` option (#2112). For example: + ``--layout-scripts=arab.dflt,arab.URD,latn``; this will keep ``DefaultLangSys`` + and ``URD`` language for ``arab`` script, and all languages for ``latn`` script. +- [varLib.interpolatable] Allow UFOs to be checked; report open paths, non existant + glyphs; add a ``--json`` option to produce a machine-readable list of + incompatibilities +- [pens] Added ``QuartzPen`` to create ``CGPath`` from glyph outlines on macOS. + Requires pyobjc (#2107). +- [feaLib] You can export ``FONTTOOLS_LOOKUP_DEBUGGING=1`` to enable feature file + debugging info stored in ``Debg`` table (#2106). +- [otlLib] Build more efficient format 1 and format 2 contextual lookups whenever + possible (#2101). + +4.17.1 (released 2020-11-16) +---------------------------- + +- [colorLib] Fixed regression in 4.17.0 when building COLR v0 table; when color + layers are stored in UFO lib plist, we can't distinguish tuples from lists so + we need to accept either types (e5439eb9, googlefonts/ufo2ft/issues#426). + +4.17.0 (released 2020-11-12) +---------------------------- + +- [colorLib/otData] Updated to latest draft ``COLR`` v1 spec (#2092). +- [svgLib] Fixed parsing error when arc commands' boolean flags are not separated + by space or comma (#2094). +- [varLib] Interpret empty non-default glyphs as 'missing', if the default glyph is + not empty (#2082). +- [feaLib.builder] Only stash lookup location for ``Debg`` if ``Builder.buildLookups_`` + has cooperated (#2065, #2067). +- [varLib] Fixed bug in VarStore optimizer (#2073, #2083). +- [varLib] Add designspace lib key for custom feavar feature tag (#2080). +- Add HashPointPen adapted from psautohint. With this pen, a hash value of a glyph + can be computed, which can later be used to detect glyph changes (#2005). + +4.16.1 (released 2020-10-05) +---------------------------- + +- [varLib.instancer] Fixed ``TypeError`` exception when instantiating a VF with + a GSUB table 1.1 in which ``FeatureVariations`` attribute is present but set to + ``None`` -- indicating that optional ``FeatureVariations`` is missing (#2077). +- [glifLib] Make ``x`` and ``y`` attributes of the ``point`` element required + even when validation is turned off, and raise a meaningful ``GlifLibError`` + message when that happens (#2075). + +4.16.0 (released 2020-09-30) +---------------------------- + +- [removeOverlaps] Added new module and ``removeOverlaps`` function that merges + overlapping contours and components in TrueType glyphs. It requires the + `skia-pathops <https://github.com/fonttools/skia-pathops>`__ module. + Note that removing overlaps invalidates the TrueType hinting (#2068). +- [varLib.instancer] Added ``--remove-overlaps`` command-line option. + The ``overlap`` option in ``instantiateVariableFont`` now takes an ``OverlapMode`` + enum: 0: KEEP_AND_DONT_SET_FLAGS, 1: KEEP_AND_SET_FLAGS (default), and 2: REMOVE. + The latter is equivalent to calling ``removeOverlaps`` on the generated static + instance. The option continues to accept ``bool`` value for backward compatibility. + + +4.15.0 (released 2020-09-21) +---------------------------- + +- [plistlib] Added typing annotations to plistlib module. Set up mypy static + typechecker to run automatically on CI (#2061). +- [ttLib] Implement private ``Debg`` table, a reverse-DNS namespaced JSON dict. +- [feaLib] Optionally add an entry into the ``Debg`` table with the original + lookup name (if any), feature name / script / language combination (if any), + and original source filename and line location. Annotate the ttx output for + a lookup with the information from the Debg table (#2052). +- [sfnt] Disabled checksum checking by default in ``SFNTReader`` (#2058). +- [Docs] Document ``mtiLib`` module (#2027). +- [varLib.interpolatable] Added checks for contour node count and operation type + of each node (#2054). +- [ttLib] Added API to register custom table packer/unpacker classes (#2055). + +4.14.0 (released 2020-08-19) +---------------------------- + +- [feaLib] Allow anonymous classes in LookupFlags definitions (#2037). +- [Docs] Better document DesignSpace rules processing order (#2041). +- [ttLib] Fixed 21-year old bug in ``maxp.maxComponentDepth`` calculation (#2044, + #2045). +- [varLib.models] Fixed misspelled argument name in CLI entry point (81d0042a). +- [subset] When subsetting GSUB v1.1, fixed TypeError by checking whether the + optional FeatureVariations table is present (e63ecc5b). +- [Snippets] Added snippet to show how to decompose glyphs in a TTF (#2030). +- [otlLib] Generate GSUB type 5 and GPOS type 7 contextual lookups where appropriate + (#2016). + +4.13.0 (released 2020-07-10) +---------------------------- + +- [feaLib/otlLib] Moved lookup subtable builders from feaLib to otlLib; refactored + some common code (#2004, #2007). +- [docs] Document otlLib module (#2009). +- [glifLib] Fixed bug with some UFO .glif filenames clashing on case-insensitive + filesystems (#2001, #2002). +- [colorLib] Updated COLRv1 implementation following changes in the draft spec: + (#2008, googlefonts/colr-gradients-spec#24). + +4.12.1 (released 2020-06-16) +---------------------------- + +- [_n_a_m_e] Fixed error in ``addMultilingualName`` with one-character names. + Only attempt to recovered malformed UTF-16 data from a ``bytes`` string, + not from unicode ``str`` (#1997, #1998). + +4.12.0 (released 2020-06-09) +---------------------------- + +- [otlLib/varLib] Ensure that the ``AxisNameID`` in the ``STAT`` and ``fvar`` + tables is grater than 255 as per OpenType spec (#1985, #1986). +- [docs] Document more modules in ``fontTools.misc`` package: ``filenames``, + ``fixedTools``, ``intTools``, ``loggingTools``, ``macCreatorType``, ``macRes``, + ``plistlib`` (#1981). +- [OS/2] Don't calculate whole sets of unicode codepoints, use faster and more memory + efficient ranges and bisect lookups (#1984). +- [voltLib] Support writing back abstract syntax tree as VOLT data (#1983). +- [voltLib] Accept DO_NOT_TOUCH_CMAP keyword (#1987). +- [subset/merge] Fixed a namespace clash involving a private helper class (#1955). + +4.11.0 (released 2020-05-28) +---------------------------- + +- [feaLib] Introduced ``includeDir`` parameter on Parser and IncludingLexer to + explicitly specify the directory to search when ``include()`` statements are + encountered (#1973). +- [ufoLib] Silently delete duplicate glyphs within the same kerning group when reading + groups (#1970). +- [ttLib] Set version of COLR table when decompiling COLRv1 (commit 9d8a7e2). + +4.10.2 (released 2020-05-20) +---------------------------- + +- [sfnt] Fixed ``NameError: SimpleNamespace`` while reading TTC header. The regression + was introduced with 4.10.1 after removing ``py23`` star import. + +4.10.1 (released 2020-05-19) +---------------------------- + +- [sfnt] Make ``SFNTReader`` pickleable even when TTFont is loaded with lazy=True + option and thus keeps a reference to an external file (#1962, #1967). +- [feaLib.ast] Restore backward compatibility (broken in 4.10 with #1905) for + ``ChainContextPosStatement`` and ``ChainContextSubstStatement`` classes. + Make them accept either list of lookups or list of lists of lookups (#1961). +- [docs] Document some modules in ``fontTools.misc`` package: ``arrayTools``, + ``bezierTools`` ``cliTools`` and ``eexec`` (#1956). +- [ttLib._n_a_m_e] Fixed ``findMultilingualName()`` when name record's ``string`` is + encoded as bytes sequence (#1963). + +4.10.0 (released 2020-05-15) +---------------------------- + +- [varLib] Allow feature variations to be active across the entire space (#1957). +- [ufoLib] Added support for ``formatVersionMinor`` in UFO's ``fontinfo.plist`` and for + ``formatMinor`` attribute in GLIF file as discussed in unified-font-object/ufo-spec#78. + No changes in reading or writing UFOs until an upcoming (non-0) minor update of the + UFO specification is published (#1786). +- [merge] Fixed merging fonts with different versions of ``OS/2`` table (#1865, #1952). +- [subset] Fixed ``AttributeError`` while subsetting ``ContextSubst`` and ``ContextPos`` + Format 3 subtable (#1879, #1944). +- [ttLib.table._m_e_t_a] if data happens to be ascii, emit comment in TTX (#1938). +- [feaLib] Support multiple lookups per glyph position (#1905). +- [psCharStrings] Use inheritance to avoid repeated code in initializer (#1932). +- [Doc] Improved documentation for the following modules: ``afmLib`` (#1933), ``agl`` + (#1934), ``cffLib`` (#1935), ``cu2qu`` (#1937), ``encodings`` (#1940), ``feaLib`` + (#1941), ``merge`` (#1949). +- [Doc] Split off developer-centric info to new page, making front page of docs more + user-focused. List all utilities and sub-modules with brief descriptions. + Make README more concise and focused (#1914). +- [otlLib] Add function to build STAT table from high-level description (#1926). +- [ttLib._n_a_m_e] Add ``findMultilingualName()`` method (#1921). +- [unicodedata] Update ``RTL_SCRIPTS`` for Unicode 13.0 (#1925). +- [gvar] Sort ``gvar`` XML output by glyph name, not glyph order (#1907, #1908). +- [Doc] Added help options to ``fonttools`` command line tool (#1913, #1920). + Ensure all fonttools CLI tools have help documentation (#1948). +- [ufoLib] Only write fontinfo.plist when there actually is content (#1911). + +4.9.0 (released 2020-04-29) +--------------------------- + +- [subset] Fixed subsetting of FeatureVariations table. The subsetter no longer drops + FeatureVariationRecords that have empty substitutions as that will keep the search + going and thus change the logic. It will only drop empty records that occur at the + end of the FeatureVariationRecords array (#1881). +- [subset] Remove FeatureVariations table and downgrade GSUB/GPOS to version 0x10000 + when FeatureVariations contain no FeatureVariationRecords after subsetting (#1903). +- [agl] Add support for legacy Adobe Glyph List of glyph names in ``fontTools.agl`` + (#1895). +- [feaLib] Ignore superfluous script statements (#1883). +- [feaLib] Hide traceback by default on ``fonttools feaLib`` command line. + Use ``--traceback`` option to show (#1898). +- [feaLib] Check lookup index in chaining sub/pos lookups and print better error + message (#1896, #1897). +- [feaLib] Fix building chained alt substitutions (#1902). +- [Doc] Included all fontTools modules in the sphinx-generated documentation, and + published it to ReadTheDocs for continuous documentation of the fontTools project + (#1333). Check it out at https://fonttools.readthedocs.io/. Thanks to Chris Simpkins! +- [transform] The ``Transform`` class is now subclass of ``typing.NamedTuple``. No + change in functionality (#1904). + + +4.8.1 (released 2020-04-17) +--------------------------- + +- [feaLib] Fixed ``AttributeError: 'NoneType' has no attribute 'getAlternateGlyphs'`` + when ``aalt`` feature references a chain contextual substitution lookup + (googlefonts/fontmake#648, #1878). + +4.8.0 (released 2020-04-16) +--------------------------- + +- [feaLib] If Parser is initialized without a ``glyphNames`` parameter, it cannot + distinguish between a glyph name containing an hyphen, or a range of glyph names; + instead of raising an error, it now interprets them as literal glyph names, while + also outputting a logging warning to alert user about the ambiguity (#1768, #1870). +- [feaLib] When serializing AST to string, emit spaces around hyphens that denote + ranges. Also, fixed an issue with CID ranges when round-tripping AST->string->AST + (#1872). +- [Snippets/otf2ttf] In otf2ttf.py script update LSB in hmtx to match xMin (#1873). +- [colorLib] Added experimental support for building ``COLR`` v1 tables as per + the `colr-gradients-spec <https://github.com/googlefonts/colr-gradients-spec/blob/main/colr-gradients-spec.md>`__ + draft proposal. **NOTE**: both the API and the XML dump of ``COLR`` v1 are + susceptible to change while the proposal is being discussed and formalized (#1822). + +4.7.0 (released 2020-04-03) +--------------------------- + +- [cu2qu] Added ``fontTools.cu2qu`` package, imported from the original + `cu2qu <https://github.com/googlefonts/cu2qu>`__ project. The ``cu2qu.pens`` module + was moved to ``fontTools.pens.cu2quPen``. The optional cu2qu extension module + can be compiled by installing `Cython <https://cython.org/>`__ before installing + fonttools from source (i.e. git repo or sdist tarball). The wheel package that + is published on PyPI (i.e. the one ``pip`` downloads, unless ``--no-binary`` + option is used), will continue to be pure-Python for now (#1868). + +4.6.0 (released 2020-03-24) +--------------------------- + +- [varLib] Added support for building variable ``BASE`` table version 1.1 (#1858). +- [CPAL] Added ``fromRGBA`` method to ``Color`` class (#1861). + + +4.5.0 (released 2020-03-20) +--------------------------- + +- [designspaceLib] Added ``add{Axis,Source,Instance,Rule}Descriptor`` methods to + ``DesignSpaceDocument`` class, to initialize new descriptor objects using keyword + arguments, and at the same time append them to the current document (#1860). +- [unicodedata] Update to Unicode 13.0 (#1859). + +4.4.3 (released 2020-03-13) +--------------------------- + +- [varLib] Always build ``gvar`` table for TrueType-flavored Variable Fonts, + even if it contains no variation data. The table is required according to + the OpenType spec (#1855, #1857). + +4.4.2 (released 2020-03-12) +--------------------------- + +- [ttx] Annotate ``LookupFlag`` in XML dump with comment explaining what bits + are set and what they mean (#1850). +- [feaLib] Added more descriptive message to ``IncludedFeaNotFound`` error (#1842). + +4.4.1 (released 2020-02-26) +--------------------------- + +- [woff2] Skip normalizing ``glyf`` and ``loca`` tables if these are missing from + a font (e.g. in NotoColorEmoji using ``CBDT/CBLC`` tables). +- [timeTools] Use non-localized date parsing in ``timestampFromString``, to fix + error when non-English ``LC_TIME`` locale is set (#1838, #1839). +- [fontBuilder] Make sure the CFF table generated by fontBuilder can be used by varLib + without having to compile and decompile the table first. This was breaking in + converting the CFF table to CFF2 due to some unset attributes (#1836). + +4.4.0 (released 2020-02-18) +--------------------------- + +- [colorLib] Added ``fontTools.colorLib.builder`` module, initially with ``buildCOLR`` + and ``buildCPAL`` public functions. More color font formats will follow (#1827). +- [fontBuilder] Added ``setupCOLR`` and ``setupCPAL`` methods (#1826). +- [ttGlyphPen] Quantize ``GlyphComponent.transform`` floats to ``F2Dot14`` to fix + round-trip issue when computing bounding boxes of transformed components (#1830). +- [glyf] If a component uses reference points (``firstPt`` and ``secondPt``) for + alignment (instead of X and Y offsets), compute the effective translation offset + *after* having applied any transform (#1831). +- [glyf] When all glyphs have zero contours, compile ``glyf`` table data as a single + null byte in order to pass validation by OTS and Windows (#1829). +- [feaLib] Parsing feature code now ensures that referenced glyph names are part of + the known glyph set, unless a glyph set was not provided. +- [varLib] When filling in the default axis value for a missing location of a source or + instance, correctly map the value forward. +- [varLib] The avar table can now contain mapping output values that are greater than + OR EQUAL to the preceeding value, as the avar specification allows this. +- [varLib] The errors of the module are now ordered hierarchically below VarLibError. + See #1821. + +4.3.0 (released 2020-02-03) +--------------------------- + +- [EBLC/CBLC] Fixed incorrect padding length calculation for Format 3 IndexSubTable + (#1817, #1818). +- [varLib] Fixed error when merging OTL tables and TTFonts were loaded as ``lazy=True`` + (#1808, #1809). +- [varLib] Allow to use master fonts containing ``CFF2`` table when building VF (#1816). +- [ttLib] Make ``recalcBBoxes`` option work also with ``CFF2`` table (#1816). +- [feaLib] Don't reset ``lookupflag`` in lookups defined inside feature blocks. + They will now inherit the current ``lookupflag`` of the feature. This is what + Adobe ``makeotf`` also does in this case (#1815). +- [feaLib] Fixed bug with mixed single/multiple substitutions. If a single substitution + involved a glyph class, we were incorrectly using only the first glyph in the class + (#1814). + +4.2.5 (released 2020-01-29) +--------------------------- + +- [feaLib] Do not fail on duplicate multiple substitutions, only warn (#1811). +- [subset] Optimize SinglePos subtables to Format 1 if all ValueRecords are the same + (#1802). + +4.2.4 (released 2020-01-09) +--------------------------- + +- [unicodedata] Update RTL_SCRIPTS for Unicode 11 and 12. + +4.2.3 (released 2020-01-07) +--------------------------- + +- [otTables] Fixed bug when splitting `MarkBasePos` subtables as offsets overflow. + The mark class values in the split subtable were not being updated, leading to + invalid mark-base attachments (#1797, googlefonts/noto-source#145). +- [feaLib] Only log a warning instead of error when features contain duplicate + substitutions (#1767). +- [glifLib] Strip XML comments when parsing with lxml (#1784, #1785). + +4.2.2 (released 2019-12-12) +--------------------------- + +- [subset] Fixed issue with subsetting FeatureVariations table when the index + of features changes as features get dropped. The feature index need to be + remapped to point to index of the remaining features (#1777, #1782). +- [fontBuilder] Added `addFeatureVariations` method to `FontBuilder` class. This + is a shorthand for calling `featureVars.addFeatureVariations` on the builder's + TTFont object (#1781). +- [glyf] Fixed the flags bug in glyph.drawPoints() like we did for glyph.draw() + (#1771, #1774). + +4.2.1 (released 2019-12-06) +--------------------------- + +- [glyf] Use the ``flagOnCurve`` bit mask in ``glyph.draw()``, so that we ignore + the ``overlap`` flag that may be set when instantiating variable fonts (#1771). + +4.2.0 (released 2019-11-28) +--------------------------- + +- [pens] Added the following pens: + + * ``roundingPen.RoundingPen``: filter pen that rounds coordinates and components' + offsets to integer; + * ``roundingPen.RoundingPointPen``: like the above, but using PointPen protocol. + * ``filterPen.FilterPointPen``: base class for filter point pens; + * ``transformPen.TransformPointPen``: filter point pen to apply affine transform; + * ``recordingPen.RecordingPointPen``: records and replays point-pen commands. + +- [ttGlyphPen] Always round float coordinates and component offsets to integers + (#1763). +- [ufoLib] When converting kerning groups from UFO2 to UFO3, avoid confusing + groups with the same name as one of the glyphs (#1761, #1762, + unified-font-object/ufo-spec#98). + +4.1.0 (released 2019-11-18) +--------------------------- + +- [instancer] Implemented restricting axis ranges (level 3 partial instancing). + You can now pass ``{axis_tag: (min, max)}`` tuples as input to the + ``instantiateVariableFont`` function. Note that changing the default axis + position is not supported yet. The command-line script also accepts axis ranges + in the form of colon-separated float values, e.g. ``wght=400:700`` (#1753, #1537). +- [instancer] Never drop STAT ``DesignAxis`` records, but only prune out-of-range + ``AxisValue`` records. +- [otBase/otTables] Enforce that VarStore.RegionAxisCount == fvar.axisCount, even + when regions list is empty to appease OTS < v8.0 (#1752). +- [designspaceLib] Defined new ``processing`` attribute for ``<rules>`` element, + with values "first" or "last", plus other editorial changes to DesignSpace + specification. Bumped format version to 4.1 (#1750). +- [varLib] Improved error message when masters' glyph orders do not match (#1758, + #1759). +- [featureVars] Allow to specify custom feature tag in ``addFeatureVariations``; + allow said feature to already exist, in which case we append new lookup indices + to existing features. Implemented ``<rules>`` attribute ``processing`` according to + DesignSpace specification update in #1750. Depending on this flag, we generate + either an 'rvrn' (always processed first) or a 'rclt' feature (follows lookup order, + therefore last) (#1747, #1625, #1371). +- [ttCollection] Added support for context manager auto-closing via ``with`` statement + like with ``TTFont`` (#1751). +- [unicodedata] Require unicodedata2 >= 12.1.0. +- [py2.py3] Removed yet more PY2 vestiges (#1743). +- [_n_a_m_e] Fixed issue when comparing NameRecords with different string types (#1742). +- [fixedTools] Changed ``fixedToFloat`` to not do any rounding but simply return + ``value / (1 << precisionBits)``. Added ``floatToFixedToStr`` and + ``strToFixedToFloat`` functions to be used when loading from or dumping to XML. + Fixed values (e.g. fvar axes and instance coordinates, avar mappings, etc.) are + are now stored as un-rounded decimal floats upon decompiling (#1740, #737). +- [feaLib] Fixed handling of multiple ``LigatureCaret`` statements for the same glyph. + Only the first rule per glyph is used, additional ones are ignored (#1733). + +4.0.2 (released 2019-09-26) +--------------------------- + +- [voltLib] Added support for ``ALL`` and ``NONE`` in ``PROCESS_MARKS`` (#1732). +- [Silf] Fixed issue in ``Silf`` table compilation and decompilation regarding str vs + bytes in python3 (#1728). +- [merge] Handle duplicate glyph names better: instead of appending font index to + all glyph names, use similar code like we use in ``post`` and ``CFF`` tables (#1729). + +4.0.1 (released 2019-09-11) +--------------------------- + +- [otTables] Support fixing offset overflows in ``MultipleSubst`` lookup subtables + (#1706). +- [subset] Prune empty strikes in ``EBDT`` and ``CBDT`` table data (#1698, #1633). +- [pens] Fixed issue in ``PointToSegmentPen`` when last point of closed contour has + same coordinates as the starting point and was incorrectly dropped (#1720). +- [Graphite] Fixed ``Sill`` table output to pass OTS (#1705). +- [name] Added ``removeNames`` method to ``table__n_a_m_e`` class (#1719). +- [ttLib] Added aliases for renamed entries ``ascender`` and ``descender`` in + ``hhea`` table (#1715). + +4.0.0 (released 2019-08-22) +--------------------------- + +- NOTE: The v4.x version series only supports Python 3.6 or greater. You can keep + using fonttools 3.x if you need support for Python 2. +- [py23] Removed all the python2-only code since it is no longer reachable, thus + unused; only the Python3 symbols were kept, but these are no-op. The module is now + DEPRECATED and will removed in the future. +- [ttLib] Fixed UnboundLocalError for empty loca/glyph tables (#1680). Also, allow + the glyf table to be incomplete when dumping to XML (#1681). +- [varLib.models] Fixed KeyError while sorting masters and there are no on-axis for + a given axis (38a8eb0e). +- [cffLib] Make sure glyph names are unique (#1699). +- [feaLib] Fix feature parser to correctly handle octal numbers (#1700). + +\... see `here <https://github.com/fonttools/fonttools/blob/main/NEWS.rst>`__ for earlier changes diff --git a/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/RECORD b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..347b55f49cd44315cebbc36e2607ef12c1148e09 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/RECORD @@ -0,0 +1,365 @@ +../../../bin/fonttools,sha256=fzbGKLEikoobsqlbiVYe7QQxzp6Fipo00o8yTgy1U8o,349 +../../../bin/pyftmerge,sha256=Jk-SxL3A9GTbss0pZjD96Sj-tSyzgaAdH5lxOphZ7kI,346 +../../../bin/pyftsubset,sha256=tmdHJI0oMoZK7Hex_PvABFpD1jnL8bOGYblgEgfc6y8,347 +../../../bin/ttx,sha256=2JJFm7Lqg7YbGEWW148_xlf6FE4D8gw_RzdAo6iUQa4,344 +../../../share/man/man1/ttx.1,sha256=cLbm_pOOj1C76T2QXvDxzwDj9gk-GTd5RztvTMsouFw,5377 +fontTools/__init__.py,sha256=yAGxcHJPzTgLB4szimvj-C4mSPOUESVqivNB_-W0gR4,183 +fontTools/__main__.py,sha256=VjkGh1UD-i1zTDA1dXo1uecSs6PxHdGQ5vlCk_mCCYs,925 +fontTools/afmLib.py,sha256=1MagIItOzRV4vV5kKPxeDZbPJsfxLB3wdHLFkQvl0uk,13164 +fontTools/agl.py,sha256=05bm8Uq45uVWW8nPbP6xbNgmFyxQr8sWhYAiP0VSjnI,112975 +fontTools/annotations.py,sha256=BdIIriNYDzBfgniwWFg_u71qLZnV0sCcn4-VAkXkYNM,1225 +fontTools/cffLib/CFF2ToCFF.py,sha256=7h0fK53ji4JlA4mAxlVb54eQTM4N2e3LAd4wwW3TLyE,8201 +fontTools/cffLib/CFFToCFF2.py,sha256=Qnk7lYlsTRHnlZQ6NXNdr_f4MJwZQ21kcS08KFbsyY8,10119 +fontTools/cffLib/__init__.py,sha256=RG1jUl82tehcsjI0SbsKJ_-MhVTvKxQ2ThgWOI3SA0k,107890 +fontTools/cffLib/specializer.py,sha256=vsOPkR_jHNe6tESQEjmm0i76y7sWI5MKo3bsTmI3sNM,32609 +fontTools/cffLib/transforms.py,sha256=SEIZc1XxWYiVXVBsoNm6LTvM9SUN7Z76QOaSAlR1ZCo,17455 +fontTools/cffLib/width.py,sha256=IqGL0CLyCZqi_hvsHySG08qpYxS3kaqW-tsAT-bjHV4,6074 +fontTools/colorLib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fontTools/colorLib/builder.py,sha256=XChSOAAWbp-kQqQ3tspEwEkxl0uHE1I3HRAq5p9SOrk,23711 +fontTools/colorLib/errors.py,sha256=CsaviiRxxrpgVX4blm7KCyK8553ljwL44xkJOeC5U7U,41 +fontTools/colorLib/geometry.py,sha256=3ScySrR2YDJa7d5K5_xM5Yt1-3NCV-ry8ikYA5VwVbI,5518 +fontTools/colorLib/table_builder.py,sha256=ZeltWY6n-YPiJv_hQ1iBXoEFAG70EKxZyScgsMKUFGU,7469 +fontTools/colorLib/unbuilder.py,sha256=iW-E5I39WsV82K3NgCO4Cjzwm1WqzGrtypHt8epwbHM,2142 +fontTools/config/__init__.py,sha256=JICOHIz06KuHCiBmrxj-ga19P6ZTuLXh0lHmPh-Ra1w,3154 +fontTools/cu2qu/__init__.py,sha256=Cuc7Uglb0nSgaraTxXY5J8bReznH5wApW0uakN7MycY,618 +fontTools/cu2qu/__main__.py,sha256=kTUI-jczsHeelULLlory74QEeFjZWp9zigCc7PrdVQY,92 +fontTools/cu2qu/benchmark.py,sha256=wasPJmf8q9k9UHjpHChC3WQAGbBAyHN9PvJzXvWC0Fw,1296 +fontTools/cu2qu/cli.py,sha256=cT6419FTnrGVm4pJeH3bOenQq7LHGAj-GGrtvtTzg4Y,6075 +fontTools/cu2qu/cu2qu.c,sha256=DLQ4jdgnufi0A9SXTNP-FKxOjU-v27o83MJnefK4gsE,650930 +fontTools/cu2qu/cu2qu.cpython-311-x86_64-linux-gnu.so,sha256=lO-BmOCkZBPQXz13VzzHeQTrVDMCWnXy3MROHGx_TVU,1108968 +fontTools/cu2qu/cu2qu.py,sha256=yDuJRHUwbS6EzhDn6UFX95ziB0ebvEzysdZ72qHg8o0,18283 +fontTools/cu2qu/errors.py,sha256=PyJNMy8lHDtKpfFkc0nkM8F4jNLZAC4lPQCN1Km4bpg,2441 +fontTools/cu2qu/ufo.py,sha256=F54FtS-1Wnb6Wurrxp4xlw_m7Y_pRCtNZUqY62V_GRw,13124 +fontTools/designspaceLib/__init__.py,sha256=K_MqE93yFETrORGcjfv7npUkvwNiXuSc4YsTl2eYFuI,130368 +fontTools/designspaceLib/__main__.py,sha256=xhtYXo1T1tsykhQDD0tcconSNYgWL5hoTBORpVDUYrc,103 +fontTools/designspaceLib/split.py,sha256=FB1NuvhUO453UXveQZi9oyrW_caoCPM3RADp1rYWkDs,19239 +fontTools/designspaceLib/statNames.py,sha256=gXGKWVr1ju2_oL-R_DkyoZ3GlI7mfLORovpk1Ebgmvc,9237 +fontTools/designspaceLib/types.py,sha256=ofK65qXNADqcpl7zI72Pa5s07-cm7G41iEmLVV44-Es,5320 +fontTools/diff/__init__.py,sha256=DgYBwtNerim9gxmYvy3KT8TEx6zHw3zDFuAaAPAQp1c,13626 +fontTools/diff/__main__.py,sha256=442JjNXoCFuVK3UktJ3S4v-3J0Z1hordmoxyEjzUtTE,93 +fontTools/diff/color.py,sha256=irtBTlKWAc2FQvY8tI3_hkVCkfKGxTUqu3F7XFNb-Yc,1387 +fontTools/diff/diff.py,sha256=Sw_QdyYkTJilzDaQMo5ijCIVXeI5bDcNzUZrB7e2U0o,8964 +fontTools/diff/utils.py,sha256=OTdXNfK4JO8VvT4eueZ-XtYf01lOop0XGBIp84BtPz8,1020 +fontTools/encodings/MacRoman.py,sha256=4vEooUDm2gLCG8KIIDhRxm5-A64w7XrhP9cjDRr2Eo0,3576 +fontTools/encodings/StandardEncoding.py,sha256=Eo3AGE8FE_p-IVYYuV097KouSsF3UrXoRRN0XyvYbrs,3581 +fontTools/encodings/__init__.py,sha256=DJBWmoX_Haau7qlgmvWyfbhSzrX2qL636Rns7CG01pk,75 +fontTools/encodings/codecs.py,sha256=u50ruwz9fcRsrUrRGpR17Cr55Ovn1fvCHCKrElVumDE,4721 +fontTools/feaLib/__init__.py,sha256=jlIru2ghxvb1HhC5Je2BCXjFJmFQlYKpruorPoz3BvQ,213 +fontTools/feaLib/__main__.py,sha256=Df2PA6LXwna98lSXiL7R4as_ZEdWCIk3egSM5w7GpvM,2240 +fontTools/feaLib/ast.py,sha256=48Y6vpSD_wYfucWyh_bQtzf2AQFX-pOwBvsxdcpVDz0,74158 +fontTools/feaLib/builder.py,sha256=PBEAwdjxnAIjTrlIIevCzXR6Sgvz_DODIGnGL_KMYJg,75405 +fontTools/feaLib/error.py,sha256=Bz_5tNcNVcY7_nrAmFlQNhQldtqZWd8WUGQ2E3PWhZo,648 +fontTools/feaLib/lexer.c,sha256=EC3ITEOrpFlChkY9DZ8L_IEKKMCfiXrPFynbcmRix2I,748779 +fontTools/feaLib/lexer.cpython-311-x86_64-linux-gnu.so,sha256=DkJab1Tyjj_JoyomiPivt_hJQWcFj9wwt29W0l7nLB4,1512208 +fontTools/feaLib/lexer.py,sha256=emyMPmRoqNZkzxnJyI6JRCCtXrbCOFofwa9O6ABGLiw,11121 +fontTools/feaLib/location.py,sha256=JXzHqGV56EHdcq823AwA5oaK05hf_1ySWpScbo3zGC0,234 +fontTools/feaLib/lookupDebugInfo.py,sha256=gVRr5-APWfT_a5-25hRuawSVX8fEvXVsOSLWkH91T2w,304 +fontTools/feaLib/parser.py,sha256=2QxPRhc9c5OVyw7UnrgAuLwKVCI5xa_6Rp9NhwPOF9Y,99974 +fontTools/feaLib/variableScalar.py,sha256=DQZDWHV3dDnnGRcF1w4jrmHILoCFhPWsO1VZAoUu-78,9671 +fontTools/fontBuilder.py,sha256=d9bBaId9K6KPpR5nP0l5Ou_8Pp7hr4gRuLeQV0NmwUY,34280 +fontTools/help.py,sha256=bAjatvIhV7TJyXI7WhsxdYO4YVlhScZXu_kRtHANEPo,1125 +fontTools/merge/__init__.py,sha256=8i6ownyQTAOBKWnTEHvvCYFw64Mv7Z1HPBgJI-ZiuKo,8256 +fontTools/merge/__main__.py,sha256=hDx3gfbUBO83AJKumSEhiV-xqNTJNNgK2uFjazOGTmw,94 +fontTools/merge/base.py,sha256=l0G1Px98E9ZdVuFLMUBKWdtr7Jb8JX8vxcjeaDUUnzY,2389 +fontTools/merge/cmap.py,sha256=HpthxVH5lA7VegJ8yHoBjd9vrFBV7UB5OknKGYpxWY8,6728 +fontTools/merge/layout.py,sha256=fkMPGPLxEdxohS3scVM4W7LmNthSz-UPyocsffe2KqE,16075 +fontTools/merge/options.py,sha256=xko_1-WErcNQkirECzIOOYxSJR_bRtdQYQYOtmgccYI,2501 +fontTools/merge/tables.py,sha256=7SzXYL04awDEDhvU2-9T_8A2gAjvgGyYAHUICUJOpZg,10958 +fontTools/merge/unicode.py,sha256=kb1Jrfuoq1KUcVhhSKnflAED_wMZxXDjVwB-CI9k05Y,4273 +fontTools/merge/util.py,sha256=BH3bZWNFy-Tsj1cth7aSpGVJ18YXKXqDakPn6Wzku6U,3378 +fontTools/misc/__init__.py,sha256=DJBWmoX_Haau7qlgmvWyfbhSzrX2qL636Rns7CG01pk,75 +fontTools/misc/arrayTools.py,sha256=jZk__GE-K9VViZE_H-LPPj0smWbKng-yfPE8BfGp8HI,11483 +fontTools/misc/bezierTools.c,sha256=zz4veIZquc91rQR25WyU1BQRj7rHTh0Tefpky9RBLn8,1837643 +fontTools/misc/bezierTools.cpython-311-x86_64-linux-gnu.so,sha256=YHC-axO_6pocwUQBh5VHwcMWloRUeiE5Rjmga3J8AOM,5095288 +fontTools/misc/bezierTools.py,sha256=2xvnsKboUZ-Ln1bbiJk2392foh3vRhulmpqaWl1a-lQ,45240 +fontTools/misc/classifyTools.py,sha256=zcg3EM4GOerBW9c063ljaLllgeeZ772EpFZjp9CdgLI,5613 +fontTools/misc/cliTools.py,sha256=qCznJMLCQu3ZHQD_4ctUnr3TkfAUdkGl-UuxZUrppy0,1862 +fontTools/misc/configTools.py,sha256=tDede_q8h81w6mDh132_F5WzfWkwmDGAF8nwj3dHVus,11229 +fontTools/misc/cython.py,sha256=eyLcL2Bw-SSToYro8f44dkkYRlQfiFbhcza0afS-qHE,682 +fontTools/misc/dictTools.py,sha256=VxjarsGJuk_wa3z29FSCtKZNCFfXtMBiNEu0RPAlpDk,2417 +fontTools/misc/eexec.py,sha256=GNn2OCRvO1HbbIeDPxk9i0glO7cux_AQaoVMXhBR8y8,3331 +fontTools/misc/encodingTools.py,sha256=hCv5PFfnXQJVCZA8Wyn1vr3vzLBbUuEPtGk5CzWM9RY,2073 +fontTools/misc/enumTools.py,sha256=YQZW-d2ES9KFFkAXOUMIBbRUk6v_3BT6Q7lXE1ufhxA,502 +fontTools/misc/etree.py,sha256=ZzJc6TvAS579deAgZLVDvTY_HeTm-ZsKJ5s3LYhZSSY,16304 +fontTools/misc/filenames.py,sha256=MMCO3xjk1pcDc-baobcKd8IdoFPt-bcGqu8t8HUGAkI,8223 +fontTools/misc/filesystem/__init__.py,sha256=iwoOj6DpXKk8q-NRRHqOfRxFF6lcXIhsIA46j-cZswU,2011 +fontTools/misc/filesystem/_base.py,sha256=p74O7xREadfPQgzPJ9mP3ehu0ZHDgsmXlpsL9CnTRso,4010 +fontTools/misc/filesystem/_copy.py,sha256=ifMSs-A_bz1Aa4tIQrlUd9HtdJQ5fp5M3B6mbYuDtXI,1361 +fontTools/misc/filesystem/_errors.py,sha256=-YziRB1BT1I80ypmufvCR-M_4XoerCHyBVqX-cRnIzU,641 +fontTools/misc/filesystem/_info.py,sha256=pbV7bDTJ5F8ms6alK34J0FZYWzmRO7FT0NM3yRA3czo,2013 +fontTools/misc/filesystem/_osfs.py,sha256=RkKCE2IxcRaj7gyqFW10LhEm_-VJYtXxsS5s0DCXihM,5737 +fontTools/misc/filesystem/_path.py,sha256=frP6ZLmMeP9E3NiwoCbbgBPWpQbLBRh7T-0vOE-EuPo,1745 +fontTools/misc/filesystem/_subfs.py,sha256=vRotQwyLVfINbR88xIBQUbq9j4Kmg1_mJvEhnpvK_t4,3028 +fontTools/misc/filesystem/_tempfs.py,sha256=9FUXdCBTwFtZMFx8ghYuZVYoQdDb0tDB-jXNu3D-Qy0,924 +fontTools/misc/filesystem/_tools.py,sha256=r75dpadp7C9EdQ6r7pJQKZlCZDUJzVq2ikb_LXN-wCI,972 +fontTools/misc/filesystem/_walk.py,sha256=KMQ-GavWYr4SsA5V8ohLPmz3boilvY2P0JKrLoxW6NU,1655 +fontTools/misc/filesystem/_zipfs.py,sha256=i3qolbkDRntB_oL3v79KuEgfVlVojecPBnBA0X04PWc,6301 +fontTools/misc/fixedTools.py,sha256=XycD5QpaejJE-GFaFMlLlku6RG0ipJowuamMXjFrMjQ,7668 +fontTools/misc/iftSparseBitSet.py,sha256=nhqQIa3o-P5frMdk4XlySzEAmGc7igouYlhBSR8U7ss,10521 +fontTools/misc/intTools.py,sha256=l6pjk4UYlXcyLtfC0DdOC5RL6UJ8ihRR0zRiYow5xA8,586 +fontTools/misc/iterTools.py,sha256=17H6LPZszp32bTKoNorp6uZF1PKj47BAbe5QG8irUjo,390 +fontTools/misc/lazyTools.py,sha256=BC6MmF-OzJ3GrBD8TYDZ-VCSN4UOx0pN0r3oF4GSoiw,1020 +fontTools/misc/loggingTools.py,sha256=NOYROsLK5TzONK5967OGdVonNyXC6kP_CmPr7M2PW_c,19933 +fontTools/misc/macCreatorType.py,sha256=Je9jtqUr7EPbpH3QxlVl3pizoQ-1AOPMBIctHIMTM3k,1593 +fontTools/misc/macRes.py,sha256=GT_pnfPw2NCvvOF86nHLAnOtZ6SMHqEuLntaplXzvHM,8579 +fontTools/misc/plistlib/__init__.py,sha256=1HfhHPt3As6u2eRSlFfl6XdnXv_ypQImeQdWIw6wK7Y,21113 +fontTools/misc/plistlib/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fontTools/misc/psCharStrings.py,sha256=Zn8mr7NRTfEJbnaXbOnSarrFBPcvPM0AupAxF2C80LY,43468 +fontTools/misc/psLib.py,sha256=ioIPm5x3MHkBXF2vzNkC4iVZYobrkWcyvFhmYsjOrPY,12099 +fontTools/misc/psOperators.py,sha256=9SLl5PPBulLo0Xxg_dqlJMitNIBdiGKdkXhOWsNSYZE,15700 +fontTools/misc/py23.py,sha256=aPVCEUz_deggwLBCeTSsccX6QgJavZqvdVtuhpzrPvA,2238 +fontTools/misc/roundTools.py,sha256=1RSXZ0gyi1qW42tz6WSBMJD1FlPdtgqKfWixVN9bd78,3173 +fontTools/misc/sstruct.py,sha256=vUODd2CKvHLtjr7yn1K94Hui_yxOPWKmlgAmBMm3KDQ,7009 +fontTools/misc/symfont.py,sha256=x5ZwqK9Ik9orG6qSftgVGygBFE1wTSngrMK2We1Z5AM,6977 +fontTools/misc/testTools.py,sha256=3vj_KllUQVEiVFbS0SzTmeuKv44-L-disI1dZ4XhOfw,7052 +fontTools/misc/textTools.py,sha256=wNjH5zl1v9qNfmTl4BL52IO2IG1H5xY3o_pslqPPRjc,3483 +fontTools/misc/timeTools.py,sha256=e9h5pgzL04tBDXmCv_8eRGB4boFV8GKXlS6dq3ggEpw,2234 +fontTools/misc/transform.py,sha256=OR8dPsAw87z77gkZQMq00iUkDWLIxYv-12XiKH1-erk,15798 +fontTools/misc/treeTools.py,sha256=tLWkwyDHeZUPVOGNnJeD4Pn7x2bQeZetwJKaEAW2J2M,1269 +fontTools/misc/vector.py,sha256=6lqZcDjAgHJFQgjzD-ULQ_PrigAMfeZKaBZmAfcC0ig,4062 +fontTools/misc/visitor.py,sha256=zwBAVfZ3MTsrbhNFj03pSSjNRyT6oGkare-kfWkN5ns,5754 +fontTools/misc/xmlReader.py,sha256=igut4_d13RT4WarliqVvuuPybO1uSXVeoBOeW4j0_e4,6580 +fontTools/misc/xmlWriter.py,sha256=EIdcS_0y9gvpc82QvZC9VcerAqEsP8eeC0gqMoPK4jc,7374 +fontTools/mtiLib/__init__.py,sha256=EzYwNaENLf906h1THBeq6nSRHUKpOAYxuzO9x9PHzh8,46602 +fontTools/mtiLib/__main__.py,sha256=gd8X89jnZOe-752k7uaR1lWoiju-2zIT5Yx35Kl0Xek,94 +fontTools/otlLib/__init__.py,sha256=D2leUW-3gsUTOFcJYGC18edBYjIJ804ut4qitJYWsaQ,45 +fontTools/otlLib/builder.py,sha256=SFwSVBHxdvSr0hEkBzzdMrBXDrppN1TJIwR3ndYrdtQ,129529 +fontTools/otlLib/error.py,sha256=cthuhBuOwZYpkTLi5gFPupUxkXkCHe-L_YgkE7N1wCI,335 +fontTools/otlLib/maxContextCalc.py,sha256=3es4Kt84TaZ49sA2ev1zrlwPJikJCAECx5KavwhyB-I,3175 +fontTools/otlLib/optimize/__init__.py,sha256=UUQRpNkHU2RczCRt-Gz7sEiYE9AQq9BHLXZEOyvsnX4,1530 +fontTools/otlLib/optimize/__main__.py,sha256=BvP472kA9KxBb9RMyyehPNevAfpmgW9MfdazkUiAO3M,104 +fontTools/otlLib/optimize/gpos.py,sha256=htOgSP743DZDUKF3eWAeJ-kdqNYOnpGXdlV-rbEXJ1A,17668 +fontTools/pens/__init__.py,sha256=DJBWmoX_Haau7qlgmvWyfbhSzrX2qL636Rns7CG01pk,75 +fontTools/pens/areaPen.py,sha256=Y1WkmqzcC4z_bpGAR0IZUKrtHFtxKUQBmr5-64_zCOk,1472 +fontTools/pens/basePen.py,sha256=eIGSKrKm6w4LLHuG6XJoQZ3eObtoKV5P6aF4gT4sk7U,17073 +fontTools/pens/boundsPen.py,sha256=wE3owOQA8DfhH-zBGC3lJvnVwp-oyIt0KZrEqXbmS9I,3129 +fontTools/pens/cairoPen.py,sha256=wuuOJ1qQDSt_K3zscM2nukRyHZTZMwMzzCXCirfq_qQ,592 +fontTools/pens/cocoaPen.py,sha256=IJRQcAxRuVOTQ90bB_Bgjnmz7px_ST5uLF9CW-Y0KPY,612 +fontTools/pens/cu2quPen.py,sha256=gMUwFUsm_-WzBlDjTMQiNnEuI2heomGeOJBX81zYXPo,13007 +fontTools/pens/explicitClosingLinePen.py,sha256=kKKtdZiwaf8Cj4_ytrIDdGB2GMpPPDXm5Nwbw5WDgwU,3219 +fontTools/pens/filterPen.py,sha256=REgspXaaSvF3XUwqe40abe3X_E7-WbBP13IqLUUBLCw,14703 +fontTools/pens/freetypePen.py,sha256=HD-gXJSbgImJdBc8sIBk0HWBdjv3WKFofs6PgCCsGOY,19908 +fontTools/pens/hashPointPen.py,sha256=gElrFyQoOQp3ZbpKHRWPwC61A9OgT2Js8crVUD8BQAY,3573 +fontTools/pens/momentsPen.c,sha256=Oa69GAyoaCsPcsCt58hToFUgpg_tiNCWhQq_xutJ_1U,568777 +fontTools/pens/momentsPen.cpython-311-x86_64-linux-gnu.so,sha256=0rgFLRLgYozkdtcGvGxForBC_v_sEhqc6DbphF-cMdU,926528 +fontTools/pens/momentsPen.py,sha256=kjLVXhGe55Abl__Yr1gob0bl0dHe7fPSwyr7TRJnbug,25658 +fontTools/pens/perimeterPen.py,sha256=lr6NzrIWxi4TXBJPbcJsKzqABWfQeil2Bgm9BgUD3N4,2153 +fontTools/pens/pointInsidePen.py,sha256=noEUvBQIeAheDMJwzvvfnEiKhmwbS1i0RQE9jik6Gl4,6355 +fontTools/pens/pointPen.py,sha256=mXO5Tv25kyAdZ9FX29IoTQSjeh00kmtv3SZIMru8ZvI,24604 +fontTools/pens/qtPen.py,sha256=QRNLIry2rQl4E_7ct2tu10-qLHneQp0XV7FfaZ-tcL8,634 +fontTools/pens/qu2cuPen.py,sha256=pRST43-rUpzlOP83Z_Rr0IvIQBCx6RWI6nnNaitQcLk,3985 +fontTools/pens/quartzPen.py,sha256=EH482Kz_xsqYhVRovv6N_T1CXaSvOzUKPLxTaN956tU,1287 +fontTools/pens/recordingPen.py,sha256=VgFZ4NMhnZt1qSTzFEU0cma-gw3kBe47bfSxPYH73rs,12489 +fontTools/pens/reportLabPen.py,sha256=kpfMfOLXt2vOQ5smPsU82ft80FpCPWJzQLl7ENOH8Ew,2066 +fontTools/pens/reverseContourPen.py,sha256=oz64ZRhLAvT7DYMAwGKoLzZXQK8l81jRiYnTZkW6a-Y,4022 +fontTools/pens/roundingPen.py,sha256=vh_FjikRd82-S4I8glgGMGEuGrj5IkCjRT_wmZ8jfqY,4620 +fontTools/pens/statisticsPen.py,sha256=piWK6NjjWqk9MLROjeE2-4EsxVYMyNU7UQFGD_trE9g,9808 +fontTools/pens/svgPathPen.py,sha256=T3b6SZS9B9sVWMK9mSFDtjHeviQs_yOJOZKq5Sg5Zdg,8572 +fontTools/pens/t2CharStringPen.py,sha256=GgGklb5XsCer0w37ujgRLRXx-EuzdFsyCYuzCx4n-Qs,2931 +fontTools/pens/teePen.py,sha256=P1ARJOCMJ6MxK-PB1yZ-ips3CUfnadWYnQ_do6VIasQ,1290 +fontTools/pens/transformPen.py,sha256=s0kUyQdnemUwHvYr2SFboFmh4WY1S9OHBL8L4PJKRwE,4056 +fontTools/pens/ttGlyphPen.py,sha256=yLtB-E5pTQR59OKVYySttWBu1xC2vR8ezSaRhIMtVwg,11870 +fontTools/pens/wxPen.py,sha256=W9RRHlBWHp-CVC4Exvk3ytBmRaB4-LgJPP5Bv7o9BA0,680 +fontTools/qu2cu/__init__.py,sha256=Jfm1JljXbt91w4gyvZn6jzEmVnhRx50sh2fDongrOsE,618 +fontTools/qu2cu/__main__.py,sha256=9FWf6SIZaRaC8SiL0LhjAWC2yIdY9N_9wlRko8m1l2Q,93 +fontTools/qu2cu/benchmark.py,sha256=GMcr_4r7L6K9SmJ13itt-_XKhnKqSVUDPlXUG6IZmmM,1400 +fontTools/qu2cu/cli.py,sha256=rTuUuk0UX7ki3oIU6rG38U7v3FJapEt4flY3GHP8pPo,3840 +fontTools/qu2cu/qu2cu.c,sha256=ujc5hBu52ZqNMJ2MA4qoGFZnGkOIrV2viE7l_8un7Wk,734539 +fontTools/qu2cu/qu2cu.cpython-311-x86_64-linux-gnu.so,sha256=CHI6RvasVIZ11ze5289KIpk6Vay5TPuBE9k27VXnmaE,1354448 +fontTools/qu2cu/qu2cu.py,sha256=DR6f4X0o3bEmhE0zHlwzAozndWtdN0yZ9j751H3rsnE,13117 +fontTools/subset/__init__.py,sha256=R9VoZ2QWhqENHC5Ct1wyhLhEU-xo4mUXwCWl6EZGgwQ,143263 +fontTools/subset/__main__.py,sha256=bhtfP2SqP4k799pxtksFgnC-XGNQDr3LcO4lc8T5e5g,95 +fontTools/subset/cff.py,sha256=rqMRJOlX5FacV1LW8aDlVOglgEM87TkMA9bdsYenask,6145 +fontTools/subset/svg.py,sha256=sbuR0UHVB_9vYIb6dWCbZ5yCqushkpp5lg-f9NeatBM,9297 +fontTools/subset/util.py,sha256=9SXFYb5Ef9Z58uXmYPCQil8B2i3Q7aFB_1fFDFSppdU,754 +fontTools/svgLib/__init__.py,sha256=IGCLwSbU8jLhq6HI2vSdPQgNs6zDUi5774TgX5MCXPY,75 +fontTools/svgLib/path/__init__.py,sha256=C82fh7xH6ZHsSFVnV848-xeDezpokx1EwTmayJCouFU,1996 +fontTools/svgLib/path/arc.py,sha256=-f5Ym6q4tDWQ76sMNSTUTWgL_7AfgXojvBhtBS7bWwQ,5812 +fontTools/svgLib/path/parser.py,sha256=8T6okMstvgM9ufb2zBcwSzsuuoYbqfnUjNYgb6kjznU,10788 +fontTools/svgLib/path/shapes.py,sha256=5NYE-JSSNsxUt2T20ZK3TIFWrNg3cH3ETvOTwak-6cA,5387 +fontTools/t1Lib/__init__.py,sha256=p42y70wEIbuX0IIxZG7-b_I-gHto1VLy0gLsDvxCfkw,20865 +fontTools/tfmLib.py,sha256=UMbkM73JXRJVS9t2B-BJc13rSjImaWBuzCoehLwHFhs,14270 +fontTools/ttLib/__init__.py,sha256=1k7qp9z04gA3m6GvxDaINjqrKbzOkdTA_4RnqW_-LrA,661 +fontTools/ttLib/__main__.py,sha256=lHMPWsnzjKPuMFavf6i1gpk9KexiAk4qzgDd50Mbby0,4733 +fontTools/ttLib/macUtils.py,sha256=lj3oeFpyjV7ko_JqnluneITmAtlc119J-vwTTg2s73A,1737 +fontTools/ttLib/removeOverlaps.py,sha256=ny3XhFw-RrriJ6n0sSh4SaVaKc3GhmCVZMw8g4oI0TY,12744 +fontTools/ttLib/reorderGlyphs.py,sha256=TbxLxqPTUGiKRX3ulGFCwVm2lEisFYlX6caONJr_4oY,10371 +fontTools/ttLib/scaleUpem.py,sha256=U_-NGkwfS9GRIackdEXjGYZ-wSomcUPXQahDneLeArI,14618 +fontTools/ttLib/sfnt.py,sha256=D5f7kvZX7rXDcmDqVBXqPPki4k27QgiBFYi8ziH2OiU,22981 +fontTools/ttLib/standardGlyphOrder.py,sha256=7AY_fVWdtwZ4iv5uWdyKAUcbEQiSDt1lN4sqx9xXwE0,5785 +fontTools/ttLib/tables/B_A_S_E_.py,sha256=H71A9pJ850mvjbrWHqy8iFI2Dxg7102YRtAkfdCooig,369 +fontTools/ttLib/tables/BitmapGlyphMetrics.py,sha256=9gcGPVzsxEYnVBO7YLWfeOuht9PaCl09GmbAqDYqKi0,1769 +fontTools/ttLib/tables/C_B_D_T_.py,sha256=5LNdc8FMir1kC5fvp5iHwWfeuE-RuqdxAArFXaqPjQ0,3646 +fontTools/ttLib/tables/C_B_L_C_.py,sha256=YXlwovoCHYx8THLQD9iBU_VGoaB9LFObEKtL6ddD320,520 +fontTools/ttLib/tables/C_F_F_.py,sha256=yg3mUtYBudgmpG7Bz475j_DNnCelsgrTsM8DH1uR4ek,1978 +fontTools/ttLib/tables/C_F_F__2.py,sha256=YoHfJQdF-ezx4OwRQ2Y2O7rRJEPjOkf3Hx5Y11Xq0AM,807 +fontTools/ttLib/tables/C_O_L_R_.py,sha256=SHwFVNVmoUQR2e87KuTSe-J9LfeegS4f2hEpee29_2o,5993 +fontTools/ttLib/tables/C_P_A_L_.py,sha256=odFjqM4GnjXyQYGEC-e0Gvqms1jQ5zHHG3SDg7y-BI0,11942 +fontTools/ttLib/tables/D_S_I_G_.py,sha256=ahBwKMgJf9hPm3ETuF1IvG4d1_UlSW0UdzO85DibOck,5903 +fontTools/ttLib/tables/D__e_b_g.py,sha256=KDnfkNOUnm3F13wD_j3YNBOvYadZ40Gf_0170hFkJp0,1134 +fontTools/ttLib/tables/DefaultTable.py,sha256=moj0d0iIyZQt_7EVK024tP7JjJSP9HIwamQ-Big252A,1913 +fontTools/ttLib/tables/E_B_D_T_.py,sha256=uOpmt25gOJQeO1u1IGAyPWgVmh-4vSZqrQEHvOYwbwg,32534 +fontTools/ttLib/tables/E_B_L_C_.py,sha256=LfEVzBg_yWr9dhChzS0U2G-7wNOwzwB0LWoXIUYNKKM,30054 +fontTools/ttLib/tables/F_F_T_M_.py,sha256=_450vdbEH7Y-0_rOwb3Q0hg-Qq2W8C_sHljy7rZtqqM,1683 +fontTools/ttLib/tables/F__e_a_t.py,sha256=LAk4RCO66ErHFKpcm-z7YKD33_7w3yZ1NFmyF6GRGUM,5947 +fontTools/ttLib/tables/G_D_E_F_.py,sha256=QXiILFCRnPNZcwpub6ojN5S9WP6y56LsXi25pUWLgp4,299 +fontTools/ttLib/tables/G_P_O_S_.py,sha256=UkP3mlnyvQg-jj6ZBOh6j-OieVg_goJQ31nlLvoLGSI,397 +fontTools/ttLib/tables/G_S_U_B_.py,sha256=cwFMKO-pgwsn1H8Q9Jb58Z6ZrBrCoN0sqJB0YunBfSk,294 +fontTools/ttLib/tables/G_V_A_R_.py,sha256=13oO2dD-L4yfkrBuR-KN2rc40wh5lLIlx_khwMz5GH4,94 +fontTools/ttLib/tables/G__l_a_t.py,sha256=Xh3IzFgYlvNjrAOn7Ja73DrWrQTJgJxmDFSUKS6yHdM,8645 +fontTools/ttLib/tables/G__l_o_c.py,sha256=5DsxGzaG7HyJVvLlKQeff1lXt-XPWaHNNaf-EYwsKh4,2685 +fontTools/ttLib/tables/H_V_A_R_.py,sha256=6kPLDUGT8EussA3y9gKr_mrgY5PNv7YaK1V0keMXD9w,313 +fontTools/ttLib/tables/I_F_T_.py,sha256=d63nXOfLF7WqGrNj99PcLkvLIiCQ3JdZl5Sf9a7ITGk,272 +fontTools/ttLib/tables/I_F_T_X_.py,sha256=5A7KfKBr_7FgtkQV6NT2YS1eGlrWrOnjYrLnG-BGFNY,274 +fontTools/ttLib/tables/J_S_T_F_.py,sha256=Q9TEf3OuyDIxZlmoz9a3c-mDMlJK6YBQ9KcYmiwFRbU,315 +fontTools/ttLib/tables/L_T_S_H_.py,sha256=Iu6syJFuhJj0_7Aan2NPlDuQDIq-AzLwsOQbXVTnlL0,2189 +fontTools/ttLib/tables/M_A_T_H_.py,sha256=-TVu9Nlcs-1shkElbIk-CWtUwXUMdycHFkjvPE8C_fs,342 +fontTools/ttLib/tables/M_V_A_R_.py,sha256=67cEuiTw5y5W1Zk98L_S_SmJINIfy_mzWCkyHcujz94,308 +fontTools/ttLib/tables/O_S_2f_2.py,sha256=DwgOLNqcA0IIWLS8_G5Pgy41aH-HCEkOXlP1qGu2jlk,28030 +fontTools/ttLib/tables/S_T_A_T_.py,sha256=y9NiWCtnlZtMjw4K9_SdA84Xa-dJk7G5eb2dSe6ciWc,498 +fontTools/ttLib/tables/S_V_G_.py,sha256=vT6QTW5ArtskVUxnPEH_ZxKz4DF4v1pKbylN6DG0R3o,7676 +fontTools/ttLib/tables/S__i_l_f.py,sha256=lPQV2RdhcJRgfDzHp_dkgSxVUUdkcAnY1Bz7V18Gt9U,34985 +fontTools/ttLib/tables/S__i_l_l.py,sha256=Vjtn7SI83vaLGIuQf2e-jhZSFOXb9vXB4jwqznjqnMc,3224 +fontTools/ttLib/tables/T_S_I_B_.py,sha256=3WhEtyNnjYumcowD0GpjubrgnS-RzouZxCxEe4yLDo8,341 +fontTools/ttLib/tables/T_S_I_C_.py,sha256=hAV9Hq_ALsWaducDpw1tDRREvFL7hx7onnUF0sXTelU,381 +fontTools/ttLib/tables/T_S_I_D_.py,sha256=TsdX-G2xxVQO9sSE1wE_xDRx-gor5YiXTHeUthMwCPY,341 +fontTools/ttLib/tables/T_S_I_J_.py,sha256=x8Tlvi6aTxoQcI12UL7muoWF1Q61iBDseAS1mRdOYrg,341 +fontTools/ttLib/tables/T_S_I_P_.py,sha256=-il2ucTBOghVBY7cmleHdLZc3W3CKh7-iPPT0A3KBzk,341 +fontTools/ttLib/tables/T_S_I_S_.py,sha256=tVBnl63vyZUIq93oM6dEjHCXvPn9vt5vvL3jG59b0Lg,341 +fontTools/ttLib/tables/T_S_I_V_.py,sha256=iUWxz2MSrtw7mzuQZj30QAJrCPnyJ4GincFfySFUNAg,855 +fontTools/ttLib/tables/T_S_I__0.py,sha256=O-2oI0eBgt4mP15-UwH0_0r7YWi3EEEhG-4etqDueGI,2505 +fontTools/ttLib/tables/T_S_I__1.py,sha256=nSUhni-fvYmeKXW4zLfP3FG_3LQU2QKPKS1_gKY5lYg,6971 +fontTools/ttLib/tables/T_S_I__2.py,sha256=q2rub-d77iWWiBM6awO0-TCl-Xq7kalPobHYC2QEOfc,496 +fontTools/ttLib/tables/T_S_I__3.py,sha256=0LcvvCzVZJzyz7i4zjIkUuYXEqXwOCs9WeCsgDFqKJ8,543 +fontTools/ttLib/tables/T_S_I__5.py,sha256=hhvJn6jiXs8kuBtun8krNUTXTljH-eKxaxXM1T-7SXM,1905 +fontTools/ttLib/tables/T_T_F_A_.py,sha256=LuT0w__AMtawnsBMobhEMW9gp2yk0mA5ZRzwF45c0UI,392 +fontTools/ttLib/tables/TupleVariation.py,sha256=1Cot3Eoj-O_JfEPUKd2wJh1jJoqs8eiD0smaQxu9FlM,32196 +fontTools/ttLib/tables/V_A_R_C_.py,sha256=3jFX50J6X-Cc4dwwiztKKsDTRXVHTXlVdQH328UN1-k,289 +fontTools/ttLib/tables/V_D_M_X_.py,sha256=RbHl7vvO9pcjT_kKvcCmcByQj39n4PmVeq55wD5C14g,10437 +fontTools/ttLib/tables/V_O_R_G_.py,sha256=Cn3OxjVtcO-Uvp61P5c2336V9iEbuGr6vWAXnSIaihk,5965 +fontTools/ttLib/tables/V_V_A_R_.py,sha256=Cstw6tc_U4-EmTriRItBSpvTJODAjMFQjfyTaxLzsbI,319 +fontTools/ttLib/tables/__init__.py,sha256=WRAkPL6KLlvwjJoJo3z3btfFJ71S9z4qLmbxMYmlJlA,2622 +fontTools/ttLib/tables/_a_n_k_r.py,sha256=MpAzIifmIi_3gx2oP6PC3R2lu36Ewsr2-W1rXjsz2Ug,483 +fontTools/ttLib/tables/_a_v_a_r.py,sha256=_qplWEwFos0VwRTdZa7LhfLIWw-CF1G76NbiRM_Rgvs,7374 +fontTools/ttLib/tables/_b_g_c_l.py,sha256=e8-vrpQlyh2CBnIRIYAK9xcMtWXdXr0i1Gmz13eTOsk,5484 +fontTools/ttLib/tables/_b_s_l_n.py,sha256=_848o7SQqztzBDfHYei-80u9ltxIHVBzXu1dYHLV57M,465 +fontTools/ttLib/tables/_c_i_d_g.py,sha256=yt8rVIadpJSDUCoVH4dZetNiy0Azm5ESAxHjB2BX_eA,913 +fontTools/ttLib/tables/_c_m_a_p.py,sha256=1bfeetlu0fMcipG1n5pS6XccmeDrtUoq_KSSNt8EVNM,62544 +fontTools/ttLib/tables/_c_v_a_r.py,sha256=35ayk2kX1pcLGwyx0y4I1l-r7LHgdKv0ulVx8oBPteI,3527 +fontTools/ttLib/tables/_c_v_t.py,sha256=1_RhEcTmhWQWQp7Hsj8UsByKmXCIppZyIbIArGywEEM,1618 +fontTools/ttLib/tables/_f_e_a_t.py,sha256=Fi1XnjhkCG0tp43AcvpIaivD-YRFpufo6feGIrenQDo,469 +fontTools/ttLib/tables/_f_p_g_m.py,sha256=uZHZzqL6OdLn_Hxskv-xf3XuE4fyaSv_jbALEjwXYug,1633 +fontTools/ttLib/tables/_f_v_a_r.py,sha256=NTpSm77_UDcPtQMg8etrkRVmm6P23vESpe_WAbFP03o,8808 +fontTools/ttLib/tables/_g_a_s_p.py,sha256=YvhAVDvdssN2fjPMTfSrO4WBCfTuh9T2cU5zquDVnSw,2203 +fontTools/ttLib/tables/_g_c_i_d.py,sha256=AJ4uV7PTHbnsw4Tfw8c2Ezh0VMox3oAH0qhhq7y8hdM,362 +fontTools/ttLib/tables/_g_l_y_f.py,sha256=O9TATqc30B_Z7h-HwBfAeRaOM0Al84VpcReAfFsYodU,85308 +fontTools/ttLib/tables/_g_v_a_r.py,sha256=E9WCKjeITUfd5hcJLQ0rjQFBtZdxw1eswFlWp1U6bD4,12196 +fontTools/ttLib/tables/_h_d_m_x.py,sha256=wMrO4D04QNT8u30p8AV-aG3bndXCq4wlPNvtbd8ip7c,4252 +fontTools/ttLib/tables/_h_e_a_d.py,sha256=yY2GTFq6Mn6nN8EegbMVJRMUWIqDYFln3FhTk3ziw6s,4926 +fontTools/ttLib/tables/_h_h_e_a.py,sha256=X4t1aF1MZMuz3phCVSFwKcNTeoZdx-042wFtHc-nK9w,4767 +fontTools/ttLib/tables/_h_m_t_x.py,sha256=rbxr3cy9-9Jm0HCGIWQiX6fGH5iu6yojp9kfgWrW2Ks,6192 +fontTools/ttLib/tables/_k_e_r_n.py,sha256=DQNLmD_HEdDKPfp4tamOd9W3T5a1lXFM5tDaWrKl164,10794 +fontTools/ttLib/tables/_l_c_a_r.py,sha256=8W6xFOj-sm003MCXX4bIHxs9ntfVvT0FXYllPxa3z4I,390 +fontTools/ttLib/tables/_l_o_c_a.py,sha256=yxiwLKXLZjNju5XYmLb6EhNLec1d7ezEDDe1dszceHo,2180 +fontTools/ttLib/tables/_l_t_a_g.py,sha256=9YpApjI-rZ4e3HeT8Pj-osiHl3uALD9JXg5O7pqk9L0,2552 +fontTools/ttLib/tables/_m_a_x_p.py,sha256=cIDIZWse9czwwsnlxIh3qwgwaXbt7PQAjXKAcmMDspY,5264 +fontTools/ttLib/tables/_m_e_t_a.py,sha256=A0CZPEAVxYrpytjXUGQJCTddwG8KrvUVbtBe3A1MqgI,3913 +fontTools/ttLib/tables/_m_o_r_t.py,sha256=u35tYqn3cjzKxeCF0FUFeLtaf36mjDDSN08uuk0Kme8,487 +fontTools/ttLib/tables/_m_o_r_x.py,sha256=OwamVpIO7REDnFr95HuFPoY_0U6i9zQPb11K1sFTvDY,548 +fontTools/ttLib/tables/_n_a_m_e.py,sha256=86_0fUeA5_c-GY5ZnkqUI0jyWwMh1mn6yVOf6KKqIlU,41266 +fontTools/ttLib/tables/_o_p_b_d.py,sha256=TNZv_2YTrj4dGzd6wA9Jb-KGZ99un177s5p3LlfxQ74,448 +fontTools/ttLib/tables/_p_o_s_t.py,sha256=9siVXSisWGdTfj_mC1E9dUDz9Jdm1i3QzI-l3i3VWME,11671 +fontTools/ttLib/tables/_p_r_e_p.py,sha256=CcKr4HrswkupLmbJdrJLTM-z9XgLefQyv8467j9V0zs,427 +fontTools/ttLib/tables/_p_r_o_p.py,sha256=Eg8x5qWyXDzPezMafFu0s0qyPDHj-sPsFxGtE6h29qo,427 +fontTools/ttLib/tables/_s_b_i_x.py,sha256=tkkKbNKNYkUhZJuN0kl7q37x5KK5OovB06y28obPV6A,4865 +fontTools/ttLib/tables/_t_r_a_k.py,sha256=YJHZy28-h7LlTGykF7SuU8lrgD9lQgznkUxz-TxdFUU,11350 +fontTools/ttLib/tables/_v_h_e_a.py,sha256=FuULIBl4OQyUeLPOFEY8buB0pAnQhGa1-5a6kN9i5Sc,4459 +fontTools/ttLib/tables/_v_m_t_x.py,sha256=AUuxtyQvMWrTBNbOIaL6uKcB_DNpNb0YX28JIuTHw_Y,500 +fontTools/ttLib/tables/asciiTable.py,sha256=4c69jsAirUnDEpylf9CYBoCKTzwbmfbtUAOrtPnpHjY,637 +fontTools/ttLib/tables/grUtils.py,sha256=hcOJ5oJPOd2uJWnWA7qwR7AfL37YZ5zUT7g8o5BBV80,2270 +fontTools/ttLib/tables/otBase.py,sha256=Cje7tkdabZ8fowzz1dXrBcSOtRdpBgvpAq5tQBUKkXA,53237 +fontTools/ttLib/tables/otConverters.py,sha256=fgVqc1I_99rbVC9dkF8RFidTZLAyrFKaxBqXdcnProY,85940 +fontTools/ttLib/tables/otData.py,sha256=RyvA-k_3FgwL_hkrGd_IdZHGXeZVdKKr9Xq6ZK5uCek,203917 +fontTools/ttLib/tables/otDataSchema.py,sha256=JWAnyuhMTfT2satixLYsL8ci0jvWArQqOM3D_9Y1gqA,1667 +fontTools/ttLib/tables/otTables.py,sha256=QlTXte1aHGTQ4m7N1O5rCNq9gh7yo-4c7GDF7d_u76c,98116 +fontTools/ttLib/tables/otTraverse.py,sha256=HznEVAlVf_8eyqjsO2edgELtMlXnjnUqccK3PytvVUE,5518 +fontTools/ttLib/tables/sbixGlyph.py,sha256=tjEUPVRfx6gr5yme8UytGTtVrimKN5qmbzT1GZPjXiM,5796 +fontTools/ttLib/tables/sbixStrike.py,sha256=dL8O9K8R4S6RVQDP-PVjIPBrvbqbE9zwra0uRL0nLq0,6651 +fontTools/ttLib/tables/table_API_readme.txt,sha256=eZlRTLUkLzc_9Ot3pdfhyMb3ahU0_Iipx0vSbzOVGy8,2748 +fontTools/ttLib/tables/ttProgram.py,sha256=LT9NCxufFPlA6dRkYdYsawmOXIJ5rhauQ4nfy4_z8tQ,36005 +fontTools/ttLib/ttCollection.py,sha256=aRph2MkBK3kd9-JCLqhJ1EN9pffN_lVX6WWmOTTewc8,3963 +fontTools/ttLib/ttFont.py,sha256=otE7aE6JjnE98OtfQYF-ve7AnNaLpBMq1R6z88_rnDw,63517 +fontTools/ttLib/ttGlyphSet.py,sha256=cUBhMGa5hszeVqOm2KpOdeJh-LsiqE7RNdyIUPZ2vO8,17476 +fontTools/ttLib/ttVisitor.py,sha256=_tah4C42Tv6Pm9QeLNQwwVCxqI4VNEAqYCbmThp6cvY,1025 +fontTools/ttLib/woff2.py,sha256=6LPISeBQ1dubzKjWrUcYm_vgETC46BTLY4XkG52qvSA,60921 +fontTools/ttx.py,sha256=FxuGubujWCGJWSTrJEjoNH--25fVIPy-ZRtYy9H6iTk,17277 +fontTools/ufoLib/__init__.py,sha256=chTCZoHcVj85nXt66BIcr4BrxgeFTQHVSq_HOkRQKQs,98982 +fontTools/ufoLib/converters.py,sha256=YnBKr8kmyjwLcq8LdD46ubGOgyL9Pxt9avlvZn9anKI,13444 +fontTools/ufoLib/errors.py,sha256=9f8l5NaFAj3BZPa6Bbqt06FL4afffLuMzy4nPf-eOlE,845 +fontTools/ufoLib/etree.py,sha256=T3sjLTgjMAq6VyYRicWPaMIVBJ2YSuwZxV6Vc5yZtQI,231 +fontTools/ufoLib/filenames.py,sha256=hoyUhzzQMDaeckT7UdreISANq4-gLR2jGyk5yAyYtOA,10654 +fontTools/ufoLib/glifLib.py,sha256=VBbz67ece59g_re8GxtEWDP3zh6Fqsf3lmFXr7zxDoA,77495 +fontTools/ufoLib/kerning.py,sha256=o1BeJDVZ_CZZPzmOPwRKTqglYmhA_JZPjwq2JLgdQIk,4836 +fontTools/ufoLib/plistlib.py,sha256=jzMGOGvHO6XvS-IO8hS04ur7r8-v2dnVq-vKMoJZvqQ,1510 +fontTools/ufoLib/pointPen.py,sha256=CuREcm3IYteZNBDAd_ZRAV4XqBsy0s07jdWc4en9r-8,244 +fontTools/ufoLib/utils.py,sha256=nZoJJqHXQSL-LXYE58_WHA97XlbTkEbYkdH3GL32SmQ,3192 +fontTools/ufoLib/validators.py,sha256=Il9qYYVj9Po51_IGVGmeVE2SXf77pFccgJpVLkgAFKk,31991 +fontTools/unicode.py,sha256=ZZ7OMmWvIyV1IL1k6ioTzaRAh3tUvm6gvK7QgFbOIHY,1237 +fontTools/unicodedata/Blocks.py,sha256=frShN07WqD1uLt-V0TNlUZPnn0-cKc0Nuhtd2uNDoSw,33139 +fontTools/unicodedata/Mirrored.py,sha256=kdhwCWOWaArmfNkDah0Thv-67M9wWz45R5IMPhqyzFM,9242 +fontTools/unicodedata/OTTags.py,sha256=wOPpbMsNcp_gdvPFeITtgVMnTN8TJSNAsVEdu_nuPXE,1196 +fontTools/unicodedata/ScriptExtensions.py,sha256=udL53Wdd5n-ZFEVHeIC0LpKKBqPHD2WqRiBbzXs2U7U,28647 +fontTools/unicodedata/Scripts.py,sha256=wBD6SElfM3w-u6JcgyNsyofXPvqW1e4YltgeHE4XaOc,131148 +fontTools/unicodedata/__init__.py,sha256=4eR0Luk4QJHC9YDG0_wi6UrIDawED36e6Xbf3S5hDPg,9085 +fontTools/varLib/__init__.py,sha256=C1tfl_FZ-ek_5cXqtKSXRLu8dqAg_DL2UdbvLusYBxQ,56777 +fontTools/varLib/__main__.py,sha256=wbdYC5bPjWCxA0I4SKcLO88gl-UMtsYS8MxdW9ySTkY,95 +fontTools/varLib/avar/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fontTools/varLib/avar/__main__.py,sha256=ew1fJpg81GpYbdkrp4I7ntWdbhkDeg5fNn1SrpF254k,1770 +fontTools/varLib/avar/build.py,sha256=g0rlRT1HqVFv7pIzY0MN8phqZhEuJ0YO2S-mauW4frA,2089 +fontTools/varLib/avar/map.py,sha256=dtCkJ7dP6Cvav8pyO3GJIaXq9OIPuSdyaRgWCmKeFag,3482 +fontTools/varLib/avar/plan.py,sha256=qDKZjQq6OfeXBr3T3DZnrmauyMm5ek0rm5YVosAO_UA,27354 +fontTools/varLib/avar/unbuild.py,sha256=Es_D9yiqp-ib-5bxfxCsZF9Jq7f60eb2aVfx-Eepv10,10517 +fontTools/varLib/avarPlanner.py,sha256=CabG5xB57FMdwmN1acpVMvyOucVPxCdRdJSK2gu7gb4,109 +fontTools/varLib/builder.py,sha256=mSKOCcnnw-WzmZs15FayoqCDh77Ts7o9Tre9psh8CUc,6609 +fontTools/varLib/cff.py,sha256=EVgaQcoROIrYQsRuftnxFuGGldEPYbrIh5yBckylJC4,22901 +fontTools/varLib/errors.py,sha256=dMo8eGj76I7H4hrBEiNbYrGs2J1K1SwdsUyTHpkVOrQ,6934 +fontTools/varLib/featureVars.py,sha256=kp4gPjKyyGRu7yBWxgd1N4OkKnU9V45QwiWSHs6OWd0,26180 +fontTools/varLib/hvar.py,sha256=1IvL5BneTkg8jJYicH0TSQViB6D0vBEesLdlfqoLBX4,3695 +fontTools/varLib/instancer/__init__.py,sha256=59DxY9wZuxRbUfj1KnY9FeYmDIr2AoHv7e244J8Fl3s,75423 +fontTools/varLib/instancer/__main__.py,sha256=zfULwcP01FhplS1IlcMgNQnLxk5RVfmOuinWjqeid-g,104 +fontTools/varLib/instancer/featureVars.py,sha256=oPqSlnHLMDTtOsmQMi6gkzLox7ymCrqlRAkvC_EJ4bc,7110 +fontTools/varLib/instancer/names.py,sha256=IPRqel_M8zVU0jl30WsfgufxUm9PBBQDQCY3VHapeHc,14950 +fontTools/varLib/instancer/solver.py,sha256=uMePwX0BVT5F94kUvDglsI4_F0nEH67F7RFuJ6tQwQ0,11002 +fontTools/varLib/interpolatable.py,sha256=WP553NVSShJp8oGktsYlYrv0fLDY2gRs6dNSZRSpNd8,45227 +fontTools/varLib/interpolatableHelpers.py,sha256=9EZUwcgXLeKAn_mPA2E5mCqWkpXREB71a5efsmzyk9o,10286 +fontTools/varLib/interpolatablePlot.py,sha256=14AMcuUIrgQItBfeWlAP6FFFtmvoh97Hwjicj1Fj5KA,44272 +fontTools/varLib/interpolatableTestContourOrder.py,sha256=xIPmAKLDq_5wtI9hDtYRbprfNT9Bj-uaQmklBth8Xh0,2970 +fontTools/varLib/interpolatableTestStartingPoint.py,sha256=WrV4VzmL6YmaLmThXrMd_U9A1dlzStV-AqkamIRNh24,4364 +fontTools/varLib/interpolate_layout.py,sha256=22VjGZuV2YiAe2MpdTf0xPVz1x2G84bcOL0vOeBpGQM,3689 +fontTools/varLib/iup.c,sha256=YaaxnrOIaGpf4PAvDLIYOrPO-AUi61Mx1zHsLTJ-xHA,835847 +fontTools/varLib/iup.cpython-311-x86_64-linux-gnu.so,sha256=gJShTaj6HY0o9A-LvCeq7nrLT8lWIInjFX4yO7nlecg,1685576 +fontTools/varLib/iup.py,sha256=mKq_GRWuUg4yTmw2V32nu0v2r-SzzN7xS7rIbV0mYuc,14984 +fontTools/varLib/merger.py,sha256=E59oli4AwqWZ-FgnuStMSBvsB-FHe-55esXTYUqGeJ8,60802 +fontTools/varLib/models.py,sha256=2cr3hjwKd6E6uciOVZixh0Z-TATEtUa-LEGsDK3zvqk,22910 +fontTools/varLib/multiVarStore.py,sha256=eQEuWNY01YF5zDpy1UwNtvOYyD6c0FLxpH-QFpX1i78,8305 +fontTools/varLib/mutator.py,sha256=kzXiLFxRLgU2pcHzOzh9u0n0KkO3DuBk06xZ_RPhWz8,19804 +fontTools/varLib/mvar.py,sha256=LTV77vH_3Ecg_qKBO5xQzjLOlJir_ppEr7mPVZRgad8,2449 +fontTools/varLib/plot.py,sha256=NoSZkJ5ndxNcDvJIvd5pQ9_jX6X1oM1K2G_tR4sdPVs,7494 +fontTools/varLib/stat.py,sha256=XuNKKZxGlBrl4OGFDAwVXhpBwJi23U3BdHmNTKoJnvE,4811 +fontTools/varLib/varStore.py,sha256=2QA9SDI6jQyQ_zq82OOwa3FBkfl-ksaSo1KGmVFpa9Q,24069 +fontTools/voltLib/__init__.py,sha256=ZZ1AsTx1VlDn40Kupce-fM3meOWugy3RZraBW9LG-9M,151 +fontTools/voltLib/__main__.py,sha256=uVtABLzMeHtvKL8zetf4rpC4aB8BkYr5QLSegNjZZZI,5928 +fontTools/voltLib/ast.py,sha256=arA9W3Gqo6OqljwNNKnMAojz-C5LStbC5SgjJh7buKk,13300 +fontTools/voltLib/error.py,sha256=phcQOQj-xOspCXu9hBJQRhSOBDzxHRgZd3fWQOFNJzw,395 +fontTools/voltLib/lexer.py,sha256=OvuETOSvlS6v7iCVeJ3IdH2Cg71n3OJoEyiB3-h6vhE,3368 +fontTools/voltLib/parser.py,sha256=rkw2IHBZPsrhGVC7Kw7V501m0u52kh1JSM5HXp-xchM,25396 +fontTools/voltLib/voltToFea.py,sha256=Z2yvnaZLQXzPLT86Uta0zRsXIYgj6NnvZtSWt5xmw2s,36549 +fonttools-4.63.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +fonttools-4.63.0.dist-info/METADATA,sha256=h4N02BG_qDwEXFXqu8qgnmRvQm_Ud3tef3FuTiGY-8E,118704 +fonttools-4.63.0.dist-info/RECORD,, +fonttools-4.63.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fonttools-4.63.0.dist-info/WHEEL,sha256=LzEbpIS1rN6HUKgX_HYQou3zndu2RacYEOrIqPjDwB0,151 +fonttools-4.63.0.dist-info/entry_points.txt,sha256=8kVHddxfFWA44FSD4mBpmC-4uCynQnkoz_9aNJb227Y,147 +fonttools-4.63.0.dist-info/licenses/LICENSE,sha256=Z4cgj4P2Wcy8IiOy_elS_6b36KymLxqKK_W8UbsbI4M,1072 +fonttools-4.63.0.dist-info/licenses/LICENSE.external,sha256=lKg6ruBymg8wLTSsxKzsvZ1YNm8mJCkHX-VX5KVLLmk,20022 +fonttools-4.63.0.dist-info/top_level.txt,sha256=rRgRylrXzekqWOsrhygzib12pQ7WILf7UGjqEwkIFDM,10 diff --git a/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/REQUESTED b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/WHEEL b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..f6f776ab3392f40d51ed119d76e69162a0381a16 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: setuptools (82.0.1) +Root-Is-Purelib: false +Tag: cp311-cp311-manylinux_2_17_x86_64 +Tag: cp311-cp311-manylinux2014_x86_64 + diff --git a/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/entry_points.txt b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..87ae781f169a63f0cf672a9050474035bfa5add4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/entry_points.txt @@ -0,0 +1,5 @@ +[console_scripts] +fonttools = fontTools.__main__:main +pyftmerge = fontTools.merge:main +pyftsubset = fontTools.subset:main +ttx = fontTools.ttx:main diff --git a/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/licenses/LICENSE b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..cc633905d333c4b42c1a0c8b34e9f734adeb6e1e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Just van Rossum + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/licenses/LICENSE.external b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/licenses/LICENSE.external new file mode 100644 index 0000000000000000000000000000000000000000..5c45052f870940119ee6bc2b51ab3a7bc43d01d7 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/licenses/LICENSE.external @@ -0,0 +1,388 @@ +FontTools includes the following font projects for testing purposes, which are +under SIL Open Font License, Version 1.1: + +Lobster + Copyright (c) 2010, Pablo Impallari (www.impallari.com|impallari@gmail.com), + with Reserved Font Name Lobster. + This Font Software is licensed under the SIL Open Font License, Version 1.1. + +Noto Fonts + This Font Software is licensed under the SIL Open Font License, Version 1.1. + +XITS font project + Copyright (c) 2001-2010 by the STI Pub Companies, consisting of the American + Institute of Physics, the American Chemical Society, the American + Mathematical Society, the American Physical Society, Elsevier, Inc., and The + Institute of Electrical and Electronic Engineers, Inc. (www.stixfonts.org), + with Reserved Font Name STIX Fonts, STIX Fonts (TM) is a trademark of The + Institute of Electrical and Electronics Engineers, Inc. + + Portions copyright (c) 1998-2003 by MicroPress, Inc. + (www.micropress-inc.com), with Reserved Font Name TM Math. To obtain + additional mathematical fonts, please contact MicroPress, Inc., 68-30 Harrow + Street, Forest Hills, NY 11375, USA, Phone: (718) 575-1816. + + Portions copyright (c) 1990 by Elsevier, Inc. + + This Font Software is licensed under the SIL Open Font License, Version 1.1. + +Iosevka + Copyright (c) 2015-2020 Belleve Invis (belleve@typeof.net). + This Font Software is licensed under the SIL Open Font License, Version 1.1. + +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font +creation efforts of academic and linguistic communities, and to +provide a free and open framework in which fonts may be shared and +improved in partnership with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply to +any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software +components as distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, +deleting, or substituting -- in part or in whole -- any of the +components of the Original Version, by changing formats or by porting +the Font Software to a new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, +modify, redistribute, and sell modified and unmodified copies of the +Font Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, in +Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the +corresponding Copyright Holder. This restriction only applies to the +primary font name as presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created using +the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. + +===== + +FontTools includes Adobe AGL & AGLFN, which is under 3-clauses BSD license: + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +Neither the name of Adobe Systems Incorporated nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +===== + +FontTools includes cu2qu, which is Copyright 2016 Google Inc. All Rights Reserved. +Licensed under the Apache License, Version 2.0, a copy of which is reproduced below: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +===== + +FontTools includes code in `fontTools.misc.filesystem` which is derived from: + +PyFilesystem2 (i.e. the `fs` package) by Will McGugan +Licensed under the MIT License +https://github.com/PyFilesystem/pyfilesystem2 + +Copyright (c) 2017-2021 The PyFilesystem2 contributors +Copyright (c) 2016-2019 Will McGugan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/top_level.txt b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..9af65ba39d292309497df4accdc44bd6f8143d10 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fonttools-4.63.0.dist-info/top_level.txt @@ -0,0 +1 @@ +fontTools diff --git a/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/INSTALLER b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/METADATA b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..50fd308daef5f325d3825526faad0a8678cd0dee --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/METADATA @@ -0,0 +1,257 @@ +Metadata-Version: 2.4 +Name: fsspec +Version: 2026.6.0 +Summary: File-system specification +Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html +Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/ +Project-URL: Homepage, https://github.com/fsspec/filesystem_spec +Maintainer-email: Martin Durant <mdurant@anaconda.com> +License-Expression: BSD-3-Clause +License-File: LICENSE +Keywords: file +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Requires-Python: >=3.10 +Provides-Extra: abfs +Requires-Dist: adlfs; extra == 'abfs' +Provides-Extra: adl +Requires-Dist: adlfs; extra == 'adl' +Provides-Extra: arrow +Requires-Dist: pyarrow>=1; extra == 'arrow' +Provides-Extra: dask +Requires-Dist: dask; extra == 'dask' +Requires-Dist: distributed; extra == 'dask' +Provides-Extra: dev +Requires-Dist: pre-commit; extra == 'dev' +Requires-Dist: ruff>=0.5; extra == 'dev' +Provides-Extra: doc +Requires-Dist: numpydoc; extra == 'doc' +Requires-Dist: sphinx; extra == 'doc' +Requires-Dist: sphinx-design; extra == 'doc' +Requires-Dist: sphinx-rtd-theme; extra == 'doc' +Requires-Dist: yarl; extra == 'doc' +Provides-Extra: dropbox +Requires-Dist: dropbox; extra == 'dropbox' +Requires-Dist: dropboxdrivefs; extra == 'dropbox' +Requires-Dist: requests; extra == 'dropbox' +Provides-Extra: entrypoints +Provides-Extra: full +Requires-Dist: adlfs; extra == 'full' +Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'full' +Requires-Dist: dask; extra == 'full' +Requires-Dist: distributed; extra == 'full' +Requires-Dist: dropbox; extra == 'full' +Requires-Dist: dropboxdrivefs; extra == 'full' +Requires-Dist: fusepy; extra == 'full' +Requires-Dist: gcsfs>2024.2.0; extra == 'full' +Requires-Dist: libarchive-c; extra == 'full' +Requires-Dist: ocifs; extra == 'full' +Requires-Dist: panel; extra == 'full' +Requires-Dist: paramiko; extra == 'full' +Requires-Dist: pyarrow>=1; extra == 'full' +Requires-Dist: pygit2; extra == 'full' +Requires-Dist: requests; extra == 'full' +Requires-Dist: s3fs>2024.2.0; extra == 'full' +Requires-Dist: smbprotocol; extra == 'full' +Requires-Dist: tqdm; extra == 'full' +Provides-Extra: fuse +Requires-Dist: fusepy; extra == 'fuse' +Provides-Extra: gcs +Requires-Dist: gcsfs>2024.2.0; extra == 'gcs' +Provides-Extra: git +Requires-Dist: pygit2; extra == 'git' +Provides-Extra: github +Requires-Dist: requests; extra == 'github' +Provides-Extra: gs +Requires-Dist: gcsfs; extra == 'gs' +Provides-Extra: gui +Requires-Dist: panel; extra == 'gui' +Provides-Extra: hdfs +Requires-Dist: pyarrow>=1; extra == 'hdfs' +Provides-Extra: http +Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'http' +Provides-Extra: libarchive +Requires-Dist: libarchive-c; extra == 'libarchive' +Provides-Extra: oci +Requires-Dist: ocifs; extra == 'oci' +Provides-Extra: s3 +Requires-Dist: s3fs>2024.2.0; extra == 's3' +Provides-Extra: sftp +Requires-Dist: paramiko; extra == 'sftp' +Provides-Extra: smb +Requires-Dist: smbprotocol; extra == 'smb' +Provides-Extra: ssh +Requires-Dist: paramiko; extra == 'ssh' +Provides-Extra: test +Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test' +Requires-Dist: numpy; extra == 'test' +Requires-Dist: pytest; extra == 'test' +Requires-Dist: pytest-asyncio!=0.22.0; extra == 'test' +Requires-Dist: pytest-benchmark; extra == 'test' +Requires-Dist: pytest-cov; extra == 'test' +Requires-Dist: pytest-mock; extra == 'test' +Requires-Dist: pytest-recording; extra == 'test' +Requires-Dist: pytest-rerunfailures; extra == 'test' +Requires-Dist: requests; extra == 'test' +Provides-Extra: test-downstream +Requires-Dist: aiobotocore<3.0.0,>=2.5.4; extra == 'test-downstream' +Requires-Dist: dask[dataframe,test]; extra == 'test-downstream' +Requires-Dist: moto[server]<5,>4; extra == 'test-downstream' +Requires-Dist: pytest-timeout; extra == 'test-downstream' +Requires-Dist: xarray; extra == 'test-downstream' +Provides-Extra: test-full +Requires-Dist: adlfs; extra == 'test-full' +Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test-full' +Requires-Dist: backports-zstd; (python_version < '3.14') and extra == 'test-full' +Requires-Dist: cloudpickle; extra == 'test-full' +Requires-Dist: dask; extra == 'test-full' +Requires-Dist: distributed; extra == 'test-full' +Requires-Dist: dropbox; extra == 'test-full' +Requires-Dist: dropboxdrivefs; extra == 'test-full' +Requires-Dist: fastparquet; extra == 'test-full' +Requires-Dist: fusepy; extra == 'test-full' +Requires-Dist: gcsfs; extra == 'test-full' +Requires-Dist: jinja2; extra == 'test-full' +Requires-Dist: kerchunk; extra == 'test-full' +Requires-Dist: libarchive-c; extra == 'test-full' +Requires-Dist: lz4; extra == 'test-full' +Requires-Dist: notebook; extra == 'test-full' +Requires-Dist: numpy; extra == 'test-full' +Requires-Dist: ocifs; extra == 'test-full' +Requires-Dist: pandas<3.0.0; extra == 'test-full' +Requires-Dist: panel; extra == 'test-full' +Requires-Dist: paramiko; extra == 'test-full' +Requires-Dist: pyarrow; extra == 'test-full' +Requires-Dist: pyarrow>=1; extra == 'test-full' +Requires-Dist: pyftpdlib; extra == 'test-full' +Requires-Dist: pygit2; extra == 'test-full' +Requires-Dist: pytest; extra == 'test-full' +Requires-Dist: pytest-asyncio!=0.22.0; extra == 'test-full' +Requires-Dist: pytest-benchmark; extra == 'test-full' +Requires-Dist: pytest-cov; extra == 'test-full' +Requires-Dist: pytest-mock; extra == 'test-full' +Requires-Dist: pytest-recording; extra == 'test-full' +Requires-Dist: pytest-rerunfailures; extra == 'test-full' +Requires-Dist: python-snappy; extra == 'test-full' +Requires-Dist: requests; extra == 'test-full' +Requires-Dist: smbprotocol; extra == 'test-full' +Requires-Dist: tqdm; extra == 'test-full' +Requires-Dist: urllib3; extra == 'test-full' +Requires-Dist: zarr<3.2.0; extra == 'test-full' +Requires-Dist: zstandard; (python_version < '3.14') and extra == 'test-full' +Provides-Extra: tqdm +Requires-Dist: tqdm; extra == 'tqdm' +Description-Content-Type: text/markdown + +# filesystem_spec + +[![PyPI version](https://badge.fury.io/py/fsspec.svg)](https://pypi.python.org/pypi/fsspec/) +[![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec) +![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg) +[![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest) + +A specification for pythonic filesystems. + +## Install + +```bash +pip install fsspec +``` + +would install the base fsspec. Various optionally supported features might require specification of custom +extra require, e.g. `pip install fsspec[ssh]` will install dependencies for `ssh` backends support. +Use `pip install fsspec[full]` for installation of all known extra dependencies. + +Up-to-date package also provided through conda-forge distribution: + +```bash +conda install -c conda-forge fsspec +``` + + +## Purpose + +To produce a template or specification for a file-system interface, that specific implementations should follow, +so that applications making use of them can rely on a common behaviour and not have to worry about the specific +internal implementation decisions with any given backend. Many such implementations are included in this package, +or in sister projects such as `s3fs` and `gcsfs`. + +In addition, if this is well-designed, then additional functionality, such as a key-value store or FUSE +mounting of the file-system implementation may be available for all implementations "for free". + +## Documentation + +Please refer to [RTD](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest) + +## Develop + +fsspec uses GitHub Actions for CI. Environment files can be found +in the "ci/" directory. Note that the main environment is called "py38", +but it is expected that the version of python installed be adjustable at +CI runtime. For local use, pick a version suitable for you. + +```bash +# For a new environment (mamba / conda). +mamba create -n fsspec -c conda-forge python=3.10 -y +conda activate fsspec + +# Standard dev install with docs and tests. +pip install -e ".[dev,doc,test]" + +# Full tests except for downstream +pip install s3fs +pip uninstall s3fs +pip install -e .[dev,doc,test_full] +pip install s3fs --no-deps +pytest -v + +# Downstream tests. +sh install_s3fs.sh +# Windows powershell. +install_s3fs.sh +``` + +### Testing + +Tests can be run in the dev environment, if activated, via ``pytest fsspec``. + +The full fsspec suite requires a system-level docker, docker-compose, and fuse +installation. If only making changes to one backend implementation, it is +not generally necessary to run all tests locally. + +It is expected that contributors ensure that any change to fsspec does not +cause issues or regressions for either other fsspec-related packages such +as gcsfs and s3fs, nor for downstream users of fsspec. The "downstream" CI +run and corresponding environment file run a set of tests from the dask +test suite, and very minimal tests against pandas and zarr from the +test_downstream.py module in this repo. + +### Code Formatting + +fsspec uses [Black](https://black.readthedocs.io/en/stable) to ensure +a consistent code format throughout the project. +Run ``black fsspec`` from the root of the filesystem_spec repository to +auto-format your code. Additionally, many editors have plugins that will apply +``black`` as you edit files. ``black`` is included in the ``tox`` environments. + +Optionally, you may wish to setup [pre-commit hooks](https://pre-commit.com) to +automatically run ``black`` when you make a git commit. +Run ``pre-commit install --install-hooks`` from the root of the +filesystem_spec repository to setup pre-commit hooks. ``black`` will now be run +before you commit, reformatting any changed files. You can format without +committing via ``pre-commit run`` or skip these checks with ``git commit +--no-verify``. + +## Support + +Work on this repository is supported in part by: + +"Anaconda, Inc. - Advancing AI through open source." + +<a href="https://anaconda.com/"><img src="https://camo.githubusercontent.com/b8555ef2222598ed37ce38ac86955febbd25de7619931bb7dd3c58432181d3b6/68747470733a2f2f626565776172652e6f72672f636f6d6d756e6974792f6d656d626572732f616e61636f6e64612f616e61636f6e64612d6c617267652e706e67" alt="anaconda logo" width="40%"/></a> diff --git a/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/RECORD b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..3c128efac23adb16383a6548e982d572fb47eed6 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/RECORD @@ -0,0 +1,63 @@ +fsspec-2026.6.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +fsspec-2026.6.0.dist-info/METADATA,sha256=IMhcAkppoFkjfDp5hdOBo3RQF64dY0xVSwxZAd3oEJU,10530 +fsspec-2026.6.0.dist-info/RECORD,, +fsspec-2026.6.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fsspec-2026.6.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87 +fsspec-2026.6.0.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513 +fsspec/__init__.py,sha256=L7qwNBU1iMNQd8Of87HYSNFT9gWlNMSESaJC8fY0AaQ,2053 +fsspec/_version.py,sha256=UvxfmLfdYeeNmxMYI29CuetrWuGdmBzeCcZ6C6RpX24,526 +fsspec/archive.py,sha256=vM6t_lgV6lBWbBYwpm3S4ofBQFQxUPr5KkDQrrQcQro,2411 +fsspec/asyn.py,sha256=i2pfNovARPVnV4pg93gMUZnnkMT_sCQGF0RKPCVZP5w,38733 +fsspec/caching.py,sha256=YJm7SjQTxOgyavKvAE0CST_GR0mpnEjnIIWCdE1QbLk,34024 +fsspec/callbacks.py,sha256=BDIwLzK6rr_0V5ch557fSzsivCElpdqhXr5dZ9Te-EE,9210 +fsspec/compression.py,sha256=3v_Fe39gzRRWfaeXpzNjAGPqgTzmETYRCo3qHVqD3po,5132 +fsspec/config.py,sha256=mHKzAgjXa_LwqtO_VoTJJ568VqwYkbK3gUp4J6JZJp4,4238 +fsspec/conftest.py,sha256=uWfm_Qs5alPRxOhRpDfQ0-1jqSJ54pni4y96IxOREXM,3446 +fsspec/core.py,sha256=lc7XSnZU6_C6xljp7Z_xEGN3V7704hbeQLkxvPP0wds,24173 +fsspec/dircache.py,sha256=YzogWJrhEastHU7vWz-cJiJ7sdtLXFXhEpInGKd4EcM,2717 +fsspec/exceptions.py,sha256=pauSLDMxzTJMOjvX1WEUK0cMyFkrFxpWJsyFywav7A8,331 +fsspec/fuse.py,sha256=Q-3NOOyLqBfYa4Db5E19z_ZY36zzYHtIs1mOUasItBQ,10177 +fsspec/generic.py,sha256=9QHQYMNb-8w8-eYuIqShcTjO_LeHXFoQTyt8J5oEq5Q,13482 +fsspec/gui.py,sha256=CQ7QsrTpaDlWSLNOpwNoJc7khOcYXIZxmrAJN9bHWQU,14002 +fsspec/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fsspec/implementations/arrow.py,sha256=8FhvcvOYLZNMMegCYFFCEHgEqig8AkOU7Ehb8XfcgnA,8890 +fsspec/implementations/asyn_wrapper.py,sha256=3lfJkGs6D_AwRBdxTSYlL-RCVdaXBZ9Itys2P5o5Si0,3738 +fsspec/implementations/cache_mapper.py,sha256=W4wlxyPxZbSp9ItJ0pYRVBMh6bw9eFypgP6kUYuuiI4,2421 +fsspec/implementations/cache_metadata.py,sha256=EPhQV27lgPlYREYprDET1vcMed15pVNMit-Ta1ljoyE,7942 +fsspec/implementations/cached.py,sha256=38B4qVDDXRh65Zb_JXy2-lJOxtdgUdBgNdy-oBe6jGw,36375 +fsspec/implementations/chained.py,sha256=iGivpNaHUFjB_ea0-HAPhcmm6CL8qnDf270PSj7JwuE,680 +fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466 +fsspec/implementations/data.py,sha256=9tMcy1fyX1auvzFX2aANPK9CfPoarnxTCDLhX12usvs,2131 +fsspec/implementations/dbfs.py,sha256=yHuLUGNmQd5pUCt9vpXleQNpjOpIZZwVZA08YauQU4U,16249 +fsspec/implementations/dirfs.py,sha256=5JAp_nn9v1OjKY3MwuRkLrqeZ8gGYlILSQRqkQQdz7k,12624 +fsspec/implementations/ftp.py,sha256=XkD7MiHczFllZZQWndpz9sHBJkzFfRcfkKAfey6KnSI,13465 +fsspec/implementations/gist.py,sha256=Y6jTDrE-wuTwvpPyAQDuuOMBGxlajafKWoB1_yX6jdY,8528 +fsspec/implementations/git.py,sha256=qBDWMz5LNllPqVjr5jf_1FuNha4P5lyQI3IlhYg-wUE,3731 +fsspec/implementations/github.py,sha256=aCsZL8UvXZgdkcB1RUs3DdLeNrjLKcFsFYeQFDWbBFo,11653 +fsspec/implementations/http.py,sha256=1uWwxllOjPe8mQD54IrCYQJHU1YaAXh_cDmXBk-AoQ8,30894 +fsspec/implementations/http_sync.py,sha256=UmBqd938ebwVjYgVtzg-ysG3ZoGhIJw0wFtQAfxV3Aw,30332 +fsspec/implementations/jupyter.py,sha256=q1PlQ66AAswGFyr8MFKWyobaV2YekMWRtqENBDQtD28,4002 +fsspec/implementations/libarchive.py,sha256=SpIA1F-zf7kb2-VYUVuhMrXTBOhBxUXKgEW1RaAdDoA,7098 +fsspec/implementations/local.py,sha256=aVEUnAB2hkxmweXLCiB4XD_KPwDL-gMX4dN9Fz-pk7k,17161 +fsspec/implementations/memory.py,sha256=TDdLtSPWXxZKrrVGwmc3uS3oK_2mlcVTk2BiqR8IeII,10507 +fsspec/implementations/reference.py,sha256=YKLzPBF6bLpOnHY9iM0MR5JLGRi5CB6x1cWIgMb6Ahg,49974 +fsspec/implementations/sftp.py,sha256=v7t-LpD4hEycHnw1kbepyoNgrhClshXQXitFuY4mBWE,5951 +fsspec/implementations/smb.py,sha256=5fhu8h06nOLBPh2c48aT7WBRqh9cEcbIwtyu06wTjec,15236 +fsspec/implementations/tar.py,sha256=0gqTHdwmwQI1Ia8rNQjuAVjNgrEFKPbdPkmndR9vpmk,4451 +fsspec/implementations/webhdfs.py,sha256=osF2m0nhDil6sbMzYW_4DZzhxF4ygtb59XDiybd9Fyg,17589 +fsspec/implementations/zip.py,sha256=GPDJh4UtUNJmaqijUQ0eiEZIGpwusQjkIcbhsrmLSi0,6263 +fsspec/json.py,sha256=4EBZ-xOmRiyxmIqPIwxmDImosRQ7io7qBM2xjJPsEE4,3768 +fsspec/mapping.py,sha256=m2ndB_gtRBXYmNJg0Ie1-BVR75TFleHmIQBzC-yWhjU,8343 +fsspec/parquet.py,sha256=0SzfswUTc2k5nAj7xZNYPPyvUkFNXznFRXXtBRhr-iE,20506 +fsspec/registry.py,sha256=KeDNL9HOyHX_D4D4qQGAI-Y3DQhiLeUC1VK2I81NZaE,12344 +fsspec/spec.py,sha256=7G8EFQn3gMSMP6PxaFgM9Vf93DhFc_-1ABivc_069fo,78097 +fsspec/tests/abstract/__init__.py,sha256=4xUJrv7gDgc85xAOz1p-V_K1hrsdMWTSa0rviALlJk8,10181 +fsspec/tests/abstract/common.py,sha256=1GQwNo5AONzAnzZj0fWgn8NJPLXALehbsuGxS3FzWVU,4973 +fsspec/tests/abstract/copy.py,sha256=gU5-d97U3RSde35Vp4RxPY4rWwL744HiSrJ8IBOp9-8,19967 +fsspec/tests/abstract/get.py,sha256=vNR4HztvTR7Cj56AMo7_tx7TeYz1Jgr_2Wb8Lv-UiBY,20755 +fsspec/tests/abstract/mv.py,sha256=k8eUEBIrRrGMsBY5OOaDXdGnQUKGwDIfQyduB6YD3Ns,1982 +fsspec/tests/abstract/open.py,sha256=Fi2PBPYLbRqysF8cFm0rwnB41kMdQVYjq8cGyDXp3BU,329 +fsspec/tests/abstract/pipe.py,sha256=LFzIrLCB5GLXf9rzFKJmE8AdG7LQ_h4bJo70r8FLPqM,402 +fsspec/tests/abstract/put.py,sha256=7aih17OKB_IZZh1Mkq1eBDIjobhtMQmI8x-Pw-S_aZk,21201 +fsspec/transaction.py,sha256=xliRG6U2Zf3khG4xcw9WiB-yAoqJSHEGK_VjHOdtgo0,2398 +fsspec/utils.py,sha256=E24ji0XLWC6n3bw2sHA28OYxrGU9Wy_al2XydsRgrRk,23623 diff --git a/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/REQUESTED b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/WHEEL b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..bd842267444de3fff37e8c9981cd8662490531b4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.30.1 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/licenses/LICENSE b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..67590a5e5be5a5a2dde3fe53a7512e404a896c22 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec-2026.6.0.dist-info/licenses/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2018, Martin Durant +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/.venv/lib/python3.11/site-packages/fsspec/__init__.py b/.venv/lib/python3.11/site-packages/fsspec/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..452c78a055e72a6d04f1013d1a98fda33fdc449e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/__init__.py @@ -0,0 +1,71 @@ +from . import caching +from ._version import __version__ # noqa: F401 +from .callbacks import Callback +from .compression import available_compressions +from .core import get_fs_token_paths, open, open_files, open_local, url_to_fs +from .exceptions import FSTimeoutError +from .mapping import FSMap, get_mapper +from .registry import ( + available_protocols, + filesystem, + get_filesystem_class, + register_implementation, + registry, +) +from .spec import AbstractFileSystem + +__all__ = [ + "AbstractFileSystem", + "FSTimeoutError", + "FSMap", + "filesystem", + "register_implementation", + "get_filesystem_class", + "get_fs_token_paths", + "get_mapper", + "open", + "open_files", + "open_local", + "registry", + "caching", + "Callback", + "available_protocols", + "available_compressions", + "url_to_fs", +] + + +def process_entries(): + try: + from importlib.metadata import entry_points + except ImportError: + return + if entry_points is not None: + try: + eps = entry_points() + except TypeError: + pass # importlib-metadata < 0.8 + else: + if hasattr(eps, "select"): # Python 3.10+ / importlib_metadata >= 3.9.0 + specs = eps.select(group="fsspec.specs") + else: + specs = eps.get("fsspec.specs", []) + registered_names = {} + for spec in specs: + err_msg = f"Unable to load filesystem from {spec}" + name = spec.name + if name in registered_names: + continue + registered_names[name] = True + register_implementation( + name, + spec.value.replace(":", "."), + errtxt=err_msg, + # We take our implementations as the ones to overload with if + # for some reason we encounter some, may be the same, already + # registered + clobber=True, + ) + + +process_entries() diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea2b7b2c1ebade7ea5a549cc5e694cff3b292d88 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/_version.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/_version.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbc0a2749a0abb17606a782c4f1f8754177d82af Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/_version.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/caching.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/caching.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0d199f37ce573cd571ff91246f4e03aca3acaaad Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/caching.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/callbacks.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/callbacks.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aebea9347f645eb12a957484e493851172619aa1 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/callbacks.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/compression.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/compression.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87ec31eff1077e30b1fa2f609b52318e0a093ad3 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/compression.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/config.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/config.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6a2d2f40fb1bef09a4024a0180b7adccd3046153 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/config.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/core.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/core.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0d695a001bcfce5acd99254cdcbb65e0ae55b5eb Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/core.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/dircache.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/dircache.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..381e0a98e5128f1bf466448ba3610a7a0371ec61 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/dircache.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/exceptions.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/exceptions.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dcc6b6750c03a7e560c657c032e9fb2fb336d5cb Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/exceptions.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/mapping.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/mapping.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2eb8b800adc0b64ffd435dd6fa106674336526c8 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/mapping.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/registry.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/registry.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..050e234488a47e9dd4afb6a3e029009e98d60a2c Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/registry.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/spec.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/spec.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..89c67c4ada190d2df7c2dc6526629b38c212e880 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/spec.cpython-311.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d23d758f77275004987d311f2a35a3a3799283d0f23c390d582af81ac28350ae +size 102302 diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/transaction.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/transaction.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93557db465bbf506baf80b4005be05a5f868d331 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/transaction.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/__pycache__/utils.cpython-311.pyc b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/utils.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..367704e2ce955670edebfcffdfcc961272a0edf8 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/fsspec/__pycache__/utils.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/fsspec/_version.py b/.venv/lib/python3.11/site-packages/fsspec/_version.py new file mode 100644 index 0000000000000000000000000000000000000000..5c0ab79355ccd6ed9a64788ef39a5f49f5880a70 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/_version.py @@ -0,0 +1,24 @@ +# file generated by vcs-versioning +# don't change, don't track in version control +from __future__ import annotations + +__all__ = [ + "__version__", + "__version_tuple__", + "version", + "version_tuple", + "__commit_id__", + "commit_id", +] + +version: str +__version__: str +__version_tuple__: tuple[int | str, ...] +version_tuple: tuple[int | str, ...] +commit_id: str | None +__commit_id__: str | None + +__version__ = version = '2026.6.0' +__version_tuple__ = version_tuple = (2026, 6, 0) + +__commit_id__ = commit_id = None diff --git a/.venv/lib/python3.11/site-packages/fsspec/archive.py b/.venv/lib/python3.11/site-packages/fsspec/archive.py new file mode 100644 index 0000000000000000000000000000000000000000..13a4da8df7c9405297cdd7d37476be2f725b2f57 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/archive.py @@ -0,0 +1,75 @@ +import operator + +from fsspec import AbstractFileSystem +from fsspec.utils import tokenize + + +class AbstractArchiveFileSystem(AbstractFileSystem): + """ + A generic superclass for implementing Archive-based filesystems. + + Currently, it is shared amongst + :class:`~fsspec.implementations.zip.ZipFileSystem`, + :class:`~fsspec.implementations.libarchive.LibArchiveFileSystem` and + :class:`~fsspec.implementations.tar.TarFileSystem`. + """ + + def __str__(self): + return f"<Archive-like object {type(self).__name__} at {id(self)}>" + + __repr__ = __str__ + + def ukey(self, path): + return tokenize(path, self.fo, self.protocol) + + def _all_dirnames(self, paths): + """Returns *all* directory names for each path in paths, including intermediate + ones. + + Parameters + ---------- + paths: Iterable of path strings + """ + if len(paths) == 0: + return set() + + dirnames = {self._parent(path) for path in paths} - {self.root_marker} + return dirnames | self._all_dirnames(dirnames) + + def info(self, path, **kwargs): + self._get_dirs() + path = self._strip_protocol(path) + if path in {"", "/"} and self.dir_cache: + return {"name": "", "type": "directory", "size": 0} + if path in self.dir_cache: + return self.dir_cache[path] + elif path + "/" in self.dir_cache: + return self.dir_cache[path + "/"] + else: + raise FileNotFoundError(path) + + def ls(self, path, detail=True, **kwargs): + self._get_dirs() + paths = {} + for p, f in self.dir_cache.items(): + p = p.rstrip("/") + if "/" in p: + root = p.rsplit("/", 1)[0] + else: + root = "" + if root == path.rstrip("/"): + paths[p] = f + elif all( + (a == b) + for a, b in zip(path.split("/"), [""] + p.strip("/").split("/")) + ): + # root directory entry + ppath = p.rstrip("/").split("/", 1)[0] + if ppath not in paths: + out = {"name": ppath, "size": 0, "type": "directory"} + paths[ppath] = out + if detail: + out = sorted(paths.values(), key=operator.itemgetter("name")) + return out + else: + return sorted(paths) diff --git a/.venv/lib/python3.11/site-packages/fsspec/asyn.py b/.venv/lib/python3.11/site-packages/fsspec/asyn.py new file mode 100644 index 0000000000000000000000000000000000000000..f6a58fc2a81b7c8927d1ad932f2865ead94b9f99 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/asyn.py @@ -0,0 +1,1158 @@ +import asyncio +import asyncio.events +import functools +import inspect +import io +import numbers +import os +import re +import threading +from collections.abc import Iterable +from glob import has_magic +from typing import TYPE_CHECKING + +from .callbacks import DEFAULT_CALLBACK +from .exceptions import FSTimeoutError +from .implementations.local import LocalFileSystem, make_path_posix, trailing_sep +from .spec import AbstractBufferedFile, AbstractFileSystem +from .utils import glob_translate, is_exception, other_paths + +private = re.compile("_[^_]") +iothread = [None] # dedicated fsspec IO thread +loop = [None] # global event loop for any non-async instance +_lock = None # global lock placeholder +get_running_loop = asyncio.get_running_loop + + +def get_lock(): + """Allocate or return a threading lock. + + The lock is allocated on first use to allow setting one lock per forked process. + """ + global _lock + if not _lock: + _lock = threading.Lock() + return _lock + + +def reset_lock(): + """Reset the global lock. + + This should be called only on the init of a forked process to reset the lock to + None, enabling the new forked process to get a new lock. + """ + global _lock + + iothread[0] = None + loop[0] = None + _lock = None + + +async def _runner(event, coro, result, timeout=None): + timeout = timeout if timeout else None # convert 0 or 0.0 to None + if timeout is not None: + coro = asyncio.wait_for(coro, timeout=timeout) + try: + result[0] = await coro + except Exception as ex: + result[0] = ex + finally: + event.set() + + +def sync(loop, func, *args, timeout=None, **kwargs): + """ + Make loop run coroutine until it returns. Runs in other thread + + Examples + -------- + >>> fsspec.asyn.sync(fsspec.asyn.get_loop(), func, *args, + timeout=timeout, **kwargs) + """ + timeout = timeout if timeout else None # convert 0 or 0.0 to None + # NB: if the loop is not running *yet*, it is OK to submit work + # and we will wait for it + if loop is None or loop.is_closed(): + raise RuntimeError("Loop is not running") + try: + loop0 = asyncio.events.get_running_loop() + if loop0 is loop: + raise NotImplementedError("Calling sync() from within a running loop") + except NotImplementedError: + raise + except RuntimeError: + pass + coro = func(*args, **kwargs) + result = [None] + event = threading.Event() + asyncio.run_coroutine_threadsafe(_runner(event, coro, result, timeout), loop) + while True: + # this loops allows thread to get interrupted + if event.wait(1): + break + if timeout is not None: + timeout -= 1 + if timeout < 0: + raise FSTimeoutError + + return_result = result[0] + if isinstance(return_result, asyncio.TimeoutError): + # suppress asyncio.TimeoutError, raise FSTimeoutError + raise FSTimeoutError from return_result + elif isinstance(return_result, BaseException): + raise return_result + else: + return return_result + + +def sync_wrapper(func, obj=None): + """Given a function, make so can be called in blocking contexts + + Leave obj=None if defining within a class. Pass the instance if attaching + as an attribute of the instance. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + self = obj or args[0] + return sync(self.loop, func, *args, **kwargs) + + return wrapper + + +def async_gen_wrapper(func, obj=None): + """Given a async generator, make so can be called in blocking contexts""" + + @functools.wraps(func) + def wrapper(*args, **kwargs): + self = obj or args[0] + gen = func(*args, **kwargs) + while True: + try: + yield sync(self.loop, gen.__anext__) + except StopAsyncIteration: + break + + return wrapper + + +def get_loop(): + """Create or return the default fsspec IO loop + + The loop will be running on a separate thread. + """ + if loop[0] is None: + with get_lock(): + # repeat the check just in case the loop got filled between the + # previous two calls from another thread + if loop[0] is None: + loop[0] = asyncio.new_event_loop() + th = threading.Thread(target=loop[0].run_forever, name="fsspecIO") + th.daemon = True + th.start() + iothread[0] = th + return loop[0] + + +def reset_after_fork(): + global lock + loop[0] = None + iothread[0] = None + lock = None + + +if hasattr(os, "register_at_fork"): + # should be posix; this will do nothing for spawn or forkserver subprocesses + os.register_at_fork(after_in_child=reset_after_fork) + + +if TYPE_CHECKING: + import resource + + ResourceError = resource.error +else: + try: + import resource + except ImportError: + resource = None + ResourceError = OSError + else: + ResourceError = getattr(resource, "error", OSError) + +_DEFAULT_BATCH_SIZE = 128 +_NOFILES_DEFAULT_BATCH_SIZE = 1280 + + +def _get_batch_size(nofiles=False): + from fsspec.config import conf + + if nofiles: + if "nofiles_gather_batch_size" in conf: + return conf["nofiles_gather_batch_size"] + else: + if "gather_batch_size" in conf: + return conf["gather_batch_size"] + if nofiles: + return _NOFILES_DEFAULT_BATCH_SIZE + if resource is None: + return _DEFAULT_BATCH_SIZE + + try: + soft_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + except (ImportError, ValueError, ResourceError): + return _DEFAULT_BATCH_SIZE + + if soft_limit == resource.RLIM_INFINITY: + return -1 + else: + return soft_limit // 8 + + +def running_async() -> bool: + """Being executed by an event loop?""" + try: + asyncio.get_running_loop() + return True + except RuntimeError: + return False + + +async def _run_coros_in_chunks( + coros, + batch_size=None, + callback=DEFAULT_CALLBACK, + timeout=None, + return_exceptions=False, + nofiles=False, +): + """Run the given coroutines in chunks. + + Parameters + ---------- + coros: list of coroutines to run + batch_size: int or None + Number of coroutines to submit/wait on simultaneously. + If -1, then it will not be any throttling. If + None, it will be inferred from _get_batch_size() + callback: fsspec.callbacks.Callback instance + Gets a relative_update when each coroutine completes + timeout: number or None + If given, each coroutine times out after this time. Note that, since + there are multiple batches, the total run time of this function will in + general be longer + return_exceptions: bool + Same meaning as in asyncio.gather + nofiles: bool + If inferring the batch_size, does this operation involve local files? + If yes, you normally expect smaller batches. + """ + + if batch_size is None: + batch_size = _get_batch_size(nofiles=nofiles) + + if batch_size == -1: + batch_size = len(coros) + elif batch_size <= 0: + raise ValueError + + async def _run_coro(coro, i): + try: + return await asyncio.wait_for(coro, timeout=timeout), i + except Exception as e: + if not return_exceptions: + raise + return e, i + finally: + callback.relative_update(1) + + i = 0 + n = len(coros) + results = [None] * n + pending = set() + + while pending or i < n: + while len(pending) < batch_size and i < n: + pending.add(asyncio.ensure_future(_run_coro(coros[i], i))) + i += 1 + + if not pending: + break + + done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + first_exc = None + while done: + task = done.pop() + try: + result, k = await task + results[k] = result + except Exception as exc: + if first_exc is None: + first_exc = exc + + if first_exc is not None: + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + raise first_exc + + return results + + +# these methods should be implemented as async by any async-able backend +async_methods = [ + "_ls", + "_cat_file", + "_get_file", + "_put_file", + "_rm_file", + "_cp_file", + "_pipe_file", + "_expand_path", + "_info", + "_isfile", + "_isdir", + "_exists", + "_walk", + "_glob", + "_find", + "_du", + "_size", + "_mkdir", + "_makedirs", +] + + +class AsyncFileSystem(AbstractFileSystem): + """Async file operations, default implementations + + Passes bulk operations to asyncio.gather for concurrent operation. + + Implementations that have concurrent batch operations and/or async methods + should inherit from this class instead of AbstractFileSystem. Docstrings are + copied from the un-underscored method in AbstractFileSystem, if not given. + """ + + # note that methods do not have docstring here; they will be copied + # for _* methods and inferred for overridden methods. + + async_impl = True + mirror_sync_methods = True + disable_throttling = False + + def __init__(self, *args, asynchronous=False, loop=None, batch_size=None, **kwargs): + self.asynchronous = asynchronous + self._pid = os.getpid() + if not asynchronous: + self._loop = loop or get_loop() + else: + self._loop = None + self.batch_size = batch_size + super().__init__(*args, **kwargs) + + @property + def loop(self): + if self._pid != os.getpid(): + raise RuntimeError("This class is not fork-safe") + return self._loop + + async def _rm_file(self, path, **kwargs): + if ( + inspect.iscoroutinefunction(self._rm) + and type(self)._rm is not AsyncFileSystem._rm + ): + return await self._rm(path, recursive=False, batch_size=1, **kwargs) + raise NotImplementedError + + async def _rm(self, path, recursive=False, batch_size=None, **kwargs): + # TODO: implement on_error + batch_size = batch_size or self.batch_size + path = await self._expand_path(path, recursive=recursive) + return await _run_coros_in_chunks( + [self._rm_file(p, **kwargs) for p in reversed(path)], + batch_size=batch_size, + nofiles=True, + ) + + async def _cp_file(self, path1, path2, **kwargs): + raise NotImplementedError + + async def _mv_file(self, path1, path2): + await self._cp_file(path1, path2) + await self._rm_file(path1) + + async def _copy( + self, + path1, + path2, + recursive=False, + on_error=None, + maxdepth=None, + batch_size=None, + **kwargs, + ): + if on_error is None and recursive: + on_error = "ignore" + elif on_error is None: + on_error = "raise" + + if isinstance(path1, list) and isinstance(path2, list): + # No need to expand paths when both source and destination + # are provided as lists + paths1 = path1 + paths2 = path2 + else: + source_is_str = isinstance(path1, str) + paths1 = await self._expand_path( + path1, maxdepth=maxdepth, recursive=recursive + ) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + paths1 = [ + p for p in paths1 if not (trailing_sep(p) or await self._isdir(p)) + ] + if not paths1: + return + + source_is_file = len(paths1) == 1 + dest_is_dir = isinstance(path2, str) and ( + trailing_sep(path2) or await self._isdir(path2) + ) + + exists = source_is_str and ( + (has_magic(path1) and source_is_file) + or (not has_magic(path1) and dest_is_dir and not trailing_sep(path1)) + ) + paths2 = other_paths( + paths1, + path2, + exists=exists, + flatten=not source_is_str, + ) + + batch_size = batch_size or self.batch_size + coros = [self._cp_file(p1, p2, **kwargs) for p1, p2 in zip(paths1, paths2)] + result = await _run_coros_in_chunks( + coros, batch_size=batch_size, return_exceptions=True, nofiles=True + ) + + for ex in filter(is_exception, result): + if on_error == "ignore" and isinstance(ex, FileNotFoundError): + continue + raise ex + + async def _pipe_file(self, path, value, mode="overwrite", **kwargs): + raise NotImplementedError + + async def _pipe(self, path, value=None, batch_size=None, **kwargs): + if isinstance(path, str): + path = {path: value} + batch_size = batch_size or self.batch_size + return await _run_coros_in_chunks( + [self._pipe_file(k, v, **kwargs) for k, v in path.items()], + batch_size=batch_size, + nofiles=True, + ) + + async def _process_limits(self, url, start, end): + """Helper for "Range"-based _cat_file""" + size = None + suff = False + if start is not None and start < 0: + # if start is negative and end None, end is the "suffix length" + if end is None: + end = -start + start = "" + suff = True + else: + size = size or (await self._info(url))["size"] + start = size + start + elif start is None: + start = 0 + if not suff: + if end is not None and end < 0: + if start is not None: + size = size or (await self._info(url))["size"] + end = size + end + elif end is None: + end = "" + if isinstance(end, numbers.Integral): + end -= 1 # bytes range is inclusive + return f"bytes={start}-{end}" + + async def _cat_file(self, path, start=None, end=None, **kwargs): + raise NotImplementedError + + async def _cat( + self, path, recursive=False, on_error="raise", batch_size=None, **kwargs + ): + paths = await self._expand_path(path, recursive=recursive) + coros = [self._cat_file(path, **kwargs) for path in paths] + batch_size = batch_size or self.batch_size + out = await _run_coros_in_chunks( + coros, batch_size=batch_size, nofiles=True, return_exceptions=True + ) + if on_error == "raise": + ex = next(filter(is_exception, out), False) + if ex: + raise ex + if ( + len(paths) > 1 + or isinstance(path, list) + or paths[0] != self._strip_protocol(path) + ): + return { + k: v + for k, v in zip(paths, out) + if on_error != "omit" or not is_exception(v) + } + else: + return out[0] + + async def _cat_ranges( + self, + paths, + starts, + ends, + max_gap=None, + batch_size=None, + on_error="return", + **kwargs, + ): + """Get the contents of byte ranges from one or more files + + Parameters + ---------- + paths: list + A list of of filepaths on this filesystems + starts, ends: int or list + Bytes limits of the read. If using a single int, the same value will be + used to read all the specified files. + on_error: "return" or "raise" + If "return" (default), any per-range exception is placed in the output + list at the corresponding position. Otherwise the first such exception + is raised. Matches ``AbstractFileSystem.cat_ranges``. + """ + if max_gap is not None: + # use utils.merge_offset_ranges + raise NotImplementedError + if not isinstance(paths, list): + raise TypeError + if not isinstance(starts, Iterable): + starts = [starts] * len(paths) + if not isinstance(ends, Iterable): + ends = [ends] * len(paths) + if len(starts) != len(paths) or len(ends) != len(paths): + raise ValueError + coros = [ + self._cat_file(p, start=s, end=e, **kwargs) + for p, s, e in zip(paths, starts, ends) + ] + batch_size = batch_size or self.batch_size + out = await _run_coros_in_chunks( + coros, batch_size=batch_size, nofiles=True, return_exceptions=True + ) + if on_error != "return": + ex = next(filter(is_exception, out), None) + if ex is not None: + raise ex + return out + + async def _put_file(self, lpath, rpath, mode="overwrite", **kwargs): + raise NotImplementedError + + async def _put( + self, + lpath, + rpath, + recursive=False, + callback=DEFAULT_CALLBACK, + batch_size=None, + maxdepth=None, + **kwargs, + ): + """Copy file(s) from local. + + Copies a specific file or tree of files (if recursive=True). If rpath + ends with a "/", it will be assumed to be a directory, and target files + will go within. + + The put_file method will be called concurrently on a batch of files. The + batch_size option can configure the amount of futures that can be executed + at the same time. If it is -1, then all the files will be uploaded concurrently. + The default can be set for this instance by passing "batch_size" in the + constructor, or for all instances by setting the "gather_batch_size" key + in ``fsspec.config.conf``, falling back to 1/8th of the system limit . + """ + if isinstance(lpath, list) and isinstance(rpath, list): + # No need to expand paths when both source and destination + # are provided as lists + rpaths = rpath + lpaths = lpath + else: + source_is_str = isinstance(lpath, str) + if source_is_str: + lpath = make_path_posix(lpath) + fs = LocalFileSystem() + lpaths = fs.expand_path(lpath, recursive=recursive, maxdepth=maxdepth) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))] + if not lpaths: + return + + source_is_file = len(lpaths) == 1 + dest_is_dir = isinstance(rpath, str) and ( + trailing_sep(rpath) or await self._isdir(rpath) + ) + + rpath = self._strip_protocol(rpath) + exists = source_is_str and ( + (has_magic(lpath) and source_is_file) + or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath)) + ) + rpaths = other_paths( + lpaths, + rpath, + exists=exists, + flatten=not source_is_str, + ) + + is_dir = {l: os.path.isdir(l) for l in lpaths} + rdirs = [r for l, r in zip(lpaths, rpaths) if is_dir[l]] + file_pairs = [(l, r) for l, r in zip(lpaths, rpaths) if not is_dir[l]] + + await asyncio.gather(*[self._makedirs(d, exist_ok=True) for d in rdirs]) + batch_size = batch_size or self.batch_size + + coros = [] + callback.set_size(len(file_pairs)) + for lfile, rfile in file_pairs: + put_file = callback.branch_coro(self._put_file) + coros.append(put_file(lfile, rfile, **kwargs)) + + return await _run_coros_in_chunks( + coros, batch_size=batch_size, callback=callback + ) + + async def _get_file(self, rpath, lpath, **kwargs): + raise NotImplementedError + + async def _get( + self, + rpath, + lpath, + recursive=False, + callback=DEFAULT_CALLBACK, + maxdepth=None, + **kwargs, + ): + """Copy file(s) to local. + + Copies a specific file or tree of files (if recursive=True). If lpath + ends with a "/", it will be assumed to be a directory, and target files + will go within. Can submit a list of paths, which may be glob-patterns + and will be expanded. + + The get_file method will be called concurrently on a batch of files. The + batch_size option can configure the amount of futures that can be executed + at the same time. If it is -1, then all the files will be uploaded concurrently. + The default can be set for this instance by passing "batch_size" in the + constructor, or for all instances by setting the "gather_batch_size" key + in ``fsspec.config.conf``, falling back to 1/8th of the system limit . + """ + if isinstance(lpath, list) and isinstance(rpath, list): + # No need to expand paths when both source and destination + # are provided as lists + rpaths = rpath + lpaths = lpath + else: + source_is_str = isinstance(rpath, str) + # First check for rpath trailing slash as _strip_protocol removes it. + source_not_trailing_sep = source_is_str and not trailing_sep(rpath) + rpath = self._strip_protocol(rpath) + rpaths = await self._expand_path( + rpath, recursive=recursive, maxdepth=maxdepth + ) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + rpaths = [ + p for p in rpaths if not (trailing_sep(p) or await self._isdir(p)) + ] + if not rpaths: + return + + lpath = make_path_posix(lpath) + source_is_file = len(rpaths) == 1 + dest_is_dir = isinstance(lpath, str) and ( + trailing_sep(lpath) or LocalFileSystem().isdir(lpath) + ) + + exists = source_is_str and ( + (has_magic(rpath) and source_is_file) + or (not has_magic(rpath) and dest_is_dir and source_not_trailing_sep) + ) + lpaths = other_paths( + rpaths, + lpath, + exists=exists, + flatten=not source_is_str, + ) + + [os.makedirs(os.path.dirname(lp), exist_ok=True) for lp in lpaths] + batch_size = kwargs.pop("batch_size", self.batch_size) + + coros = [] + callback.set_size(len(lpaths)) + for lpath, rpath in zip(lpaths, rpaths): + get_file = callback.branch_coro(self._get_file) + coros.append(get_file(rpath, lpath, **kwargs)) + return await _run_coros_in_chunks( + coros, batch_size=batch_size, callback=callback + ) + + async def _isfile(self, path): + try: + return (await self._info(path))["type"] == "file" + except: # noqa: E722 + return False + + async def _isdir(self, path): + try: + return (await self._info(path))["type"] == "directory" + except OSError: + return False + + async def _size(self, path): + return (await self._info(path)).get("size", None) + + async def _sizes(self, paths, batch_size=None): + batch_size = batch_size or self.batch_size + return await _run_coros_in_chunks( + [self._size(p) for p in paths], batch_size=batch_size + ) + + async def _exists(self, path, **kwargs): + try: + await self._info(path, **kwargs) + return True + except FileNotFoundError: + return False + + async def _info(self, path, **kwargs): + raise NotImplementedError + + async def _ls(self, path, detail=True, **kwargs): + raise NotImplementedError + + async def _walk(self, path, maxdepth=None, on_error="omit", **kwargs): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + path = self._strip_protocol(path) + full_dirs = {} + dirs = {} + files = {} + + detail = kwargs.pop("detail", False) + try: + listing = await self._ls(path, detail=True, **kwargs) + except (FileNotFoundError, OSError) as e: + if on_error == "raise": + raise + elif callable(on_error): + on_error(e) + if detail: + yield path, {}, {} + else: + yield path, [], [] + return + + for info in listing: + # each info name must be at least [path]/part , but here + # we check also for names like [path]/part/ + pathname = info["name"].rstrip("/") + name = pathname.rsplit("/", 1)[-1] + if info["type"] == "directory" and pathname != path: + # do not include "self" path + full_dirs[name] = pathname + dirs[name] = info + elif pathname == path: + # file-like with same name as give path + files[""] = info + else: + files[name] = info + + if detail: + yield path, dirs, files + else: + yield path, list(dirs), list(files) + + if maxdepth is not None: + maxdepth -= 1 + if maxdepth < 1: + return + + for d in dirs: + async for _ in self._walk( + full_dirs[d], maxdepth=maxdepth, detail=detail, **kwargs + ): + yield _ + + async def _glob(self, path, maxdepth=None, **kwargs): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + import re + + seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,) + ends_with_sep = path.endswith(seps) # _strip_protocol strips trailing slash + path = self._strip_protocol(path) + append_slash_to_dirname = ends_with_sep or path.endswith( + tuple(sep + "**" for sep in seps) + ) + idx_star = path.find("*") if path.find("*") >= 0 else len(path) + idx_qmark = path.find("?") if path.find("?") >= 0 else len(path) + idx_brace = path.find("[") if path.find("[") >= 0 else len(path) + + min_idx = min(idx_star, idx_qmark, idx_brace) + + detail = kwargs.pop("detail", False) + withdirs = kwargs.pop("withdirs", True) + + if not has_magic(path): + if await self._exists(path, **kwargs): + if not detail: + return [path] + else: + return {path: await self._info(path, **kwargs)} + else: + if not detail: + return [] # glob of non-existent returns empty + else: + return {} + elif "/" in path[:min_idx]: + first_wildcard_idx = min_idx + min_idx = path[:min_idx].rindex("/") + root = path[ + : min_idx + 1 + ] # everything up to the last / before the first wildcard + prefix = path[ + min_idx + 1 : first_wildcard_idx + ] # stem between last "/" and first wildcard + depth = path[min_idx + 1 :].count("/") + 1 + else: + root = "" + prefix = path[:min_idx] # stem up to the first wildcard + depth = path[min_idx + 1 :].count("/") + 1 + + if "**" in path: + if maxdepth is not None: + idx_double_stars = path.find("**") + depth_double_stars = path[idx_double_stars:].count("/") + 1 + depth = depth - depth_double_stars + maxdepth + else: + depth = None + + # Pass the filename stem as prefix= so backends that support it such as + # gcsfs, s3fs and adlfs can filter server-side up to the first wildcard. + if prefix: + kwargs["prefix"] = prefix + allpaths = await self._find( + root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs + ) + + pattern = glob_translate(path + ("/" if ends_with_sep else "")) + pattern = re.compile(pattern) + + out = { + p: info + for p, info in sorted(allpaths.items()) + if pattern.match( + p + "/" + if append_slash_to_dirname and info["type"] == "directory" + else p + ) + } + + if detail: + return out + else: + return list(out) + + async def _du(self, path, total=True, maxdepth=None, **kwargs): + sizes = {} + # async for? + for f in await self._find(path, maxdepth=maxdepth, **kwargs): + info = await self._info(f) + sizes[info["name"]] = info["size"] + if total: + return sum(sizes.values()) + else: + return sizes + + async def _find(self, path, maxdepth=None, withdirs=False, **kwargs): + path = self._strip_protocol(path) + out = {} + detail = kwargs.pop("detail", False) + + # Add the root directory if withdirs is requested + # This is needed for posix glob compliance + if withdirs and path != "" and await self._isdir(path): + out[path] = await self._info(path) + + # async for? + async for _, dirs, files in self._walk(path, maxdepth, detail=True, **kwargs): + if withdirs: + files.update(dirs) + out.update({info["name"]: info for name, info in files.items()}) + if not out and (await self._isfile(path)): + # walk works on directories, but find should also return [path] + # when path happens to be a file + out[path] = {} + names = sorted(out) + if not detail: + return names + else: + return {name: out[name] for name in names} + + async def _expand_path( + self, path, recursive=False, maxdepth=None, assume_literal=False + ): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + if isinstance(path, str): + out = await self._expand_path([path], recursive, maxdepth) + else: + out = set() + path = [self._strip_protocol(p) for p in path] + for p in path: # can gather here + if not assume_literal and has_magic(p): + bit = set(await self._glob(p, maxdepth=maxdepth)) + out |= bit + if recursive: + # glob call above expanded one depth so if maxdepth is defined + # then decrement it in expand_path call below. If it is zero + # after decrementing then avoid expand_path call. + if maxdepth is not None and maxdepth <= 1: + continue + out |= set( + await self._expand_path( + list(bit), + recursive=recursive, + maxdepth=maxdepth - 1 if maxdepth is not None else None, + assume_literal=True, + ) + ) + continue + elif recursive: + rec = set(await self._find(p, maxdepth=maxdepth, withdirs=True)) + out |= rec + if p not in out and (recursive is False or (await self._exists(p))): + # should only check once, for the root + out.add(p) + if not out: + raise FileNotFoundError(path) + return sorted(out) + + async def _mkdir(self, path, create_parents=True, **kwargs): + pass # not necessary to implement, may not have directories + + async def _makedirs(self, path, exist_ok=False): + pass # not necessary to implement, may not have directories + + async def open_async(self, path, mode="rb", **kwargs): + if "b" not in mode or kwargs.get("compression"): + raise ValueError + raise NotImplementedError + + +def mirror_sync_methods(obj): + """Populate sync and async methods for obj + + For each method will create a sync version if the name refers to an async method + (coroutine) and there is no override in the child class; will create an async + method for the corresponding sync method if there is no implementation. + + Uses the methods specified in + - async_methods: the set that an implementation is expected to provide + - default_async_methods: that can be derived from their sync version in + AbstractFileSystem + - AsyncFileSystem: async-specific default coroutines + """ + from fsspec import AbstractFileSystem + + for method in set(async_methods + dir(AsyncFileSystem)): + if not method.startswith("_"): + continue + smethod = method[1:] + if private.match(method): + isco = inspect.iscoroutinefunction(getattr(obj, method, None)) + unsync = getattr(getattr(obj, smethod, False), "__func__", None) + is_default = unsync is getattr(AbstractFileSystem, smethod, "") + if isco and is_default: + mth = sync_wrapper(getattr(obj, method), obj=obj) + elif inspect.isasyncgenfunction(getattr(obj, method, None)) and is_default: + mth = async_gen_wrapper(getattr(obj, method), obj=obj) + else: + continue + setattr(obj, smethod, mth) + if not mth.__doc__: + mth.__doc__ = getattr( + getattr(AbstractFileSystem, smethod, None), "__doc__", "" + ) + + +class FSSpecCoroutineCancel(Exception): + pass + + +def _dump_running_tasks( + printout=True, cancel=True, exc=FSSpecCoroutineCancel, with_task=False +): + import traceback + + tasks = [t for t in asyncio.tasks.all_tasks(loop[0]) if not t.done()] + if printout: + [task.print_stack() for task in tasks] + out = [ + { + "locals": task._coro.cr_frame.f_locals, + "file": task._coro.cr_frame.f_code.co_filename, + "firstline": task._coro.cr_frame.f_code.co_firstlineno, + "linelo": task._coro.cr_frame.f_lineno, + "stack": traceback.format_stack(task._coro.cr_frame), + "task": task if with_task else None, + } + for task in tasks + ] + if cancel: + for t in tasks: + cbs = t._callbacks + t.cancel() + asyncio.futures.Future.set_exception(t, exc) + asyncio.futures.Future.cancel(t) + [cb[0](t) for cb in cbs] # cancels any dependent concurrent.futures + try: + t._coro.throw(exc) # exits coro, unless explicitly handled + except exc: + pass + return out + + +class AbstractAsyncStreamedFile(AbstractBufferedFile): + # no read buffering, and always auto-commit + # TODO: readahead might still be useful here, but needs async version + + async def read(self, length=-1): + """ + Return data from cache, or fetch pieces as necessary + + Parameters + ---------- + length: int (-1) + Number of bytes to read; if <0, all remaining bytes. + """ + length = -1 if length is None else int(length) + if self.mode != "rb": + raise ValueError("File not in read mode") + if length < 0: + length = self.size - self.loc + if self.closed: + raise ValueError("I/O operation on closed file.") + if length == 0: + # don't even bother calling fetch + return b"" + out = await self._fetch_range(self.loc, self.loc + length) + self.loc += len(out) + return out + + async def write(self, data): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + data: bytes + Set of bytes to be written. + """ + if self.mode not in {"wb", "ab"}: + raise ValueError("File not in write mode") + if self.closed: + raise ValueError("I/O operation on closed file.") + if self.forced: + raise ValueError("This file has been force-flushed, can only close") + out = self.buffer.write(data) + self.loc += out + if self.buffer.tell() >= self.blocksize: + await self.flush() + return out + + async def close(self): + """Close file + + Finalizes writes, discards cache + """ + if getattr(self, "_unclosable", False): + return + if self.closed: + return + if self.mode == "rb": + self.cache = None + else: + if not self.forced: + await self.flush(force=True) + + if self.fs is not None: + self.fs.invalidate_cache(self.path) + self.fs.invalidate_cache(self.fs._parent(self.path)) + + self.closed = True + + async def flush(self, force=False): + if self.closed: + raise ValueError("Flush on closed file") + if force and self.forced: + raise ValueError("Force flush cannot be called more than once") + if force: + self.forced = True + + if self.mode not in {"wb", "ab"}: + # no-op to flush on read-mode + return + + if not force and self.buffer.tell() < self.blocksize: + # Defer write on small block + return + + if self.offset is None: + # Initialize a multipart upload + self.offset = 0 + try: + await self._initiate_upload() + except: + self.closed = True + raise + + if await self._upload_chunk(final=force) is not False: + self.offset += self.buffer.seek(0, 2) + self.buffer = io.BytesIO() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + async def _fetch_range(self, start, end): + raise NotImplementedError + + async def _initiate_upload(self): + pass + + async def _upload_chunk(self, final=False): + raise NotImplementedError diff --git a/.venv/lib/python3.11/site-packages/fsspec/caching.py b/.venv/lib/python3.11/site-packages/fsspec/caching.py new file mode 100644 index 0000000000000000000000000000000000000000..3499b4d26c240e14500232cfc752df6e6d90fd53 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/caching.py @@ -0,0 +1,1004 @@ +from __future__ import annotations + +import collections +import functools +import logging +import math +import os +import threading +from collections import OrderedDict +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor +from itertools import groupby +from operator import itemgetter +from typing import TYPE_CHECKING, Any, ClassVar, Generic, NamedTuple, TypeVar + +if TYPE_CHECKING: + import mmap + + from typing_extensions import ParamSpec + + P = ParamSpec("P") +else: + P = TypeVar("P") + +T = TypeVar("T") + + +logger = logging.getLogger("fsspec.caching") + +Fetcher = Callable[[int, int], bytes] # Maps (start, end) to bytes +MultiFetcher = Callable[[list[int, int]], bytes] # Maps [(start, end)] to bytes + + +class BaseCache: + """Pass-though cache: doesn't keep anything, calls every time + + Acts as base class for other cachers + + Parameters + ---------- + blocksize: int + How far to read ahead in numbers of bytes + fetcher: func + Function of the form f(start, end) which gets bytes from remote as + specified + size: int + How big this file is + """ + + name: ClassVar[str] = "none" + + def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: + self.blocksize = blocksize + self.nblocks = 0 + self.fetcher = fetcher + self.size = size + self.hit_count = 0 + self.miss_count = 0 + # the bytes that we actually requested + self.total_requested_bytes = 0 + + def _fetch(self, start: int | None, stop: int | None) -> bytes: + if start is None: + start = 0 + if stop is None: + stop = self.size + if start >= self.size or start >= stop: + return b"" + return self.fetcher(start, stop) + + def _reset_stats(self) -> None: + """Reset hit and miss counts for a more ganular report e.g. by file.""" + self.hit_count = 0 + self.miss_count = 0 + self.total_requested_bytes = 0 + + def _log_stats(self) -> str: + """Return a formatted string of the cache statistics.""" + if self.hit_count == 0 and self.miss_count == 0: + # a cache that does nothing, this is for logs only + return "" + return f" , {self.name}: {self.hit_count} hits, {self.miss_count} misses, {self.total_requested_bytes} total requested bytes" + + def __repr__(self) -> str: + # TODO: use rich for better formatting + return f""" + <{self.__class__.__name__}: + block size : {self.blocksize} + block count : {self.nblocks} + file size : {self.size} + cache hits : {self.hit_count} + cache misses: {self.miss_count} + total requested bytes: {self.total_requested_bytes}> + """ + + +class MMapCache(BaseCache): + """memory-mapped sparse file cache + + Opens temporary file, which is filled blocks-wise when data is requested. + Ensure there is enough disc space in the temporary location. + + This cache method might only work on posix + + Parameters + ---------- + blocksize: int + How far to read ahead in numbers of bytes + fetcher: Fetcher + Function of the form f(start, end) which gets bytes from remote as + specified + size: int + How big this file is + location: str + Where to create the temporary file. If None, a temporary file is + created using tempfile.TemporaryFile(). + blocks: set[int] + Set of block numbers that have already been fetched. If None, an empty + set is created. + multi_fetcher: MultiFetcher + Function of the form f([(start, end)]) which gets bytes from remote + as specified. This function is used to fetch multiple blocks at once. + If not specified, the fetcher function is used instead. + """ + + name = "mmap" + + def __init__( + self, + blocksize: int, + fetcher: Fetcher, + size: int, + location: str | None = None, + blocks: set[int] | None = None, + multi_fetcher: MultiFetcher | None = None, + ) -> None: + super().__init__(blocksize, fetcher, size) + self.blocks = set() if blocks is None else blocks + self.location = location + self.multi_fetcher = multi_fetcher + self.cache = self._makefile() + + def _makefile(self) -> mmap.mmap | bytearray: + import mmap + import tempfile + + if self.size == 0: + return bytearray() + + # posix version + if self.location is None or not os.path.exists(self.location): + if self.location is None: + fd = tempfile.TemporaryFile() + self.blocks = set() + else: + fd = open(self.location, "wb+") + fd.seek(self.size - 1) + fd.write(b"1") + fd.flush() + else: + fd = open(self.location, "r+b") + + return mmap.mmap(fd.fileno(), self.size) + + def _fetch(self, start: int | None, end: int | None) -> bytes: + logger.debug(f"MMap cache fetching {start}-{end}") + if start is None: + start = 0 + if end is None: + end = self.size + if start >= self.size or start >= end: + return b"" + start_block = start // self.blocksize + end_block = end // self.blocksize + block_range = range(start_block, end_block + 1) + # Determine which blocks need to be fetched. This sequence is sorted by construction. + need = (i for i in block_range if i not in self.blocks) + # Count the number of blocks already cached + self.hit_count += sum(1 for i in block_range if i in self.blocks) + + ranges = [] + + # Consolidate needed blocks. + # Algorithm adapted from Python 2.x itertools documentation. + # We are grouping an enumerated sequence of blocks. By comparing when the difference + # between an ascending range (provided by enumerate) and the needed block numbers + # we can detect when the block number skips values. The key computes this difference. + # Whenever the difference changes, we know that we have previously cached block(s), + # and a new group is started. In other words, this algorithm neatly groups + # runs of consecutive block numbers so they can be fetched together. + for _, _blocks in groupby(enumerate(need), key=lambda x: x[0] - x[1]): + # Extract the blocks from the enumerated sequence + _blocks = tuple(map(itemgetter(1), _blocks)) + # Compute start of first block + sstart = _blocks[0] * self.blocksize + # Compute the end of the last block. Last block may not be full size. + send = min(_blocks[-1] * self.blocksize + self.blocksize, self.size) + + # Fetch bytes (could be multiple consecutive blocks) + self.total_requested_bytes += send - sstart + logger.debug( + f"MMap get blocks {_blocks[0]}-{_blocks[-1]} ({sstart}-{send})" + ) + ranges.append((sstart, send)) + + # Update set of cached blocks + self.blocks.update(_blocks) + # Update cache statistics with number of blocks we had to cache + self.miss_count += len(_blocks) + + if not ranges: + return self.cache[start:end] + + if self.multi_fetcher: + logger.debug(f"MMap get blocks {ranges}") + for idx, r in enumerate(self.multi_fetcher(ranges)): + sstart, send = ranges[idx] + logger.debug(f"MMap copy block ({sstart}-{send}") + self.cache[sstart:send] = r + else: + for sstart, send in ranges: + logger.debug(f"MMap get block ({sstart}-{send}") + self.cache[sstart:send] = self.fetcher(sstart, send) + + return self.cache[start:end] + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__.copy() + # Remove the unpicklable entries. + del state["cache"] + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + # Restore instance attributes + self.__dict__.update(state) + self.cache = self._makefile() + + +class ReadAheadCache(BaseCache): + """Cache which reads only when we get beyond a block of data + + This is a much simpler version of BytesCache, and does not attempt to + fill holes in the cache or keep fragments alive. It is best suited to + many small reads in a sequential order (e.g., reading lines from a file). + """ + + name = "readahead" + + def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: + super().__init__(blocksize, fetcher, size) + self.cache = b"" + self.start = 0 + self.end = 0 + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if start is None: + start = 0 + if end is None or end > self.size: + end = self.size + if start >= self.size or start >= end: + return b"" + l = end - start + if start >= self.start and end <= self.end: + # cache hit + self.hit_count += 1 + return self.cache[start - self.start : end - self.start] + elif self.start <= start < self.end: + # partial hit + self.miss_count += 1 + part = self.cache[start - self.start :] + l -= len(part) + start = self.end + else: + # miss + self.miss_count += 1 + part = b"" + end = min(self.size, end + self.blocksize) + self.total_requested_bytes += end - start + self.cache = self.fetcher(start, end) # new block replaces old + self.start = start + self.end = self.start + len(self.cache) + return part + self.cache[:l] + + +class FirstChunkCache(BaseCache): + """Caches the first block of a file only + + This may be useful for file types where the metadata is stored in the header, + but is randomly accessed. + """ + + name = "first" + + def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: + if blocksize > size: + # this will buffer the whole thing + blocksize = size + super().__init__(blocksize, fetcher, size) + self.cache: bytes | None = None + + def _fetch(self, start: int | None, end: int | None) -> bytes: + start = start or 0 + if start > self.size: + logger.debug("FirstChunkCache: requested start > file size") + return b"" + + end = min(end, self.size) + + if start < self.blocksize: + if self.cache is None: + self.miss_count += 1 + if end > self.blocksize: + self.total_requested_bytes += end + data = self.fetcher(0, end) + self.cache = data[: self.blocksize] + return data[start:] + self.cache = self.fetcher(0, self.blocksize) + self.total_requested_bytes += self.blocksize + part = self.cache[start:end] + if end > self.blocksize: + self.total_requested_bytes += end - self.blocksize + part += self.fetcher(self.blocksize, end) + self.hit_count += 1 + return part + else: + self.miss_count += 1 + self.total_requested_bytes += end - start + return self.fetcher(start, end) + + +class BlockCache(BaseCache): + """ + Cache holding memory as a set of blocks. + + Requests are only ever made ``blocksize`` at a time, and are + stored in an LRU cache. The least recently accessed block is + discarded when more than ``maxblocks`` are stored. + + Parameters + ---------- + blocksize : int + The number of bytes to store in each block. + Requests are only ever made for ``blocksize``, so this + should balance the overhead of making a request against + the granularity of the blocks. + fetcher : Callable + size : int + The total size of the file being cached. + maxblocks : int + The maximum number of blocks to cache for. The maximum memory + use for this cache is then ``blocksize * maxblocks``. + """ + + name = "blockcache" + + def __init__( + self, blocksize: int, fetcher: Fetcher, size: int, maxblocks: int = 32 + ) -> None: + super().__init__(blocksize, fetcher, size) + self.nblocks = math.ceil(size / blocksize) + self.maxblocks = maxblocks + self._fetch_block_cached = functools.lru_cache(maxblocks)(self._fetch_block) + + def cache_info(self): + """ + The statistics on the block cache. + + Returns + ------- + NamedTuple + Returned directly from the LRU Cache used internally. + """ + return self._fetch_block_cached.cache_info() + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__ + del state["_fetch_block_cached"] + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__.update(state) + self._fetch_block_cached = functools.lru_cache(state["maxblocks"])( + self._fetch_block + ) + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if start is None: + start = 0 + if end is None: + end = self.size + if start >= self.size or start >= end: + return b"" + + return self._read_cache( + start, end, start // self.blocksize, (end - 1) // self.blocksize + ) + + def _fetch_block(self, block_number: int) -> bytes: + """ + Fetch the block of data for `block_number`. + """ + if block_number > self.nblocks: + raise ValueError( + f"'block_number={block_number}' is greater than " + f"the number of blocks ({self.nblocks})" + ) + + start = block_number * self.blocksize + end = start + self.blocksize + self.total_requested_bytes += end - start + self.miss_count += 1 + logger.info("BlockCache fetching block %d", block_number) + block_contents = super()._fetch(start, end) + return block_contents + + def _read_cache( + self, start: int, end: int, start_block_number: int, end_block_number: int + ) -> bytes: + """ + Read from our block cache. + + Parameters + ---------- + start, end : int + The start and end byte positions. + start_block_number, end_block_number : int + The start and end block numbers. + """ + start_pos = start % self.blocksize + end_pos = end % self.blocksize + if end_pos == 0: + end_pos = self.blocksize + + self.hit_count += 1 + if start_block_number == end_block_number: + block: bytes = self._fetch_block_cached(start_block_number) + return block[start_pos:end_pos] + + else: + # read from the initial + out = [self._fetch_block_cached(start_block_number)[start_pos:]] + + # intermediate blocks + # Note: it'd be nice to combine these into one big request. However + # that doesn't play nicely with our LRU cache. + out.extend( + map( + self._fetch_block_cached, + range(start_block_number + 1, end_block_number), + ) + ) + + # final block + out.append(self._fetch_block_cached(end_block_number)[:end_pos]) + + return b"".join(out) + + +class BytesCache(BaseCache): + """Cache which holds data in a in-memory bytes object + + Implements read-ahead by the block size, for semi-random reads progressing + through the file. + + Parameters + ---------- + trim: bool + As we read more data, whether to discard the start of the buffer when + we are more than a blocksize ahead of it. + """ + + name: ClassVar[str] = "bytes" + + def __init__( + self, blocksize: int, fetcher: Fetcher, size: int, trim: bool = True + ) -> None: + super().__init__(blocksize, fetcher, size) + self.cache = b"" + self.start: int | None = None + self.end: int | None = None + self.trim = trim + + def _fetch(self, start: int | None, end: int | None) -> bytes: + # TODO: only set start/end after fetch, in case it fails? + # is this where retry logic might go? + if start is None: + start = 0 + if end is None: + end = self.size + if start >= self.size or start >= end: + return b"" + if ( + self.start is not None + and start >= self.start + and self.end is not None + and end < self.end + ): + # cache hit: we have all the required data + offset = start - self.start + self.hit_count += 1 + return self.cache[offset : offset + end - start] + + if self.blocksize: + bend = min(self.size, end + self.blocksize) + else: + bend = end + + if bend == start or start > self.size: + return b"" + + if (self.start is None or start < self.start) and ( + self.end is None or end > self.end + ): + # First read, or extending both before and after + self.total_requested_bytes += bend - start + self.miss_count += 1 + self.cache = self.fetcher(start, bend) + self.start = start + else: + assert self.start is not None + assert self.end is not None + self.miss_count += 1 + + if start < self.start: + if self.end is None or self.end - end > self.blocksize: + self.total_requested_bytes += bend - start + self.cache = self.fetcher(start, bend) + self.start = start + else: + self.total_requested_bytes += self.start - start + new = self.fetcher(start, self.start) + self.start = start + self.cache = new + self.cache + elif self.end is not None and bend > self.end: + if self.end > self.size: + pass + elif end - self.end > self.blocksize: + self.total_requested_bytes += bend - start + self.cache = self.fetcher(start, bend) + self.start = start + else: + self.total_requested_bytes += bend - self.end + new = self.fetcher(self.end, bend) + self.cache = self.cache + new + + self.end = self.start + len(self.cache) + offset = start - self.start + out = self.cache[offset : offset + end - start] + if self.trim: + num = (self.end - self.start) // (self.blocksize + 1) + if num > 1: + self.start += self.blocksize * num + self.cache = self.cache[self.blocksize * num :] + return out + + def __len__(self) -> int: + return len(self.cache) + + +class AllBytes(BaseCache): + """Cache entire contents of the file""" + + name: ClassVar[str] = "all" + + def __init__( + self, + blocksize: int | None = None, + fetcher: Fetcher | None = None, + size: int | None = None, + data: bytes | None = None, + ) -> None: + super().__init__(blocksize, fetcher, size) # type: ignore[arg-type] + if data is None: + self.miss_count += 1 + self.total_requested_bytes += self.size + data = self.fetcher(0, self.size) + self.data = data + + def _fetch(self, start: int | None, stop: int | None) -> bytes: + self.hit_count += 1 + return self.data[start:stop] + + +class KnownPartsOfAFile(BaseCache): + """ + Cache holding known file parts. + + Parameters + ---------- + blocksize: int + How far to read ahead in numbers of bytes + fetcher: func + Function of the form f(start, end) which gets bytes from remote as + specified + size: int + How big this file is + data: dict + A dictionary mapping explicit `(start, stop)` file-offset tuples + with known bytes. + strict: bool, default True + Whether to fetch reads that go beyond a known byte-range boundary. + If `False`, any read that ends outside a known part will be zero + padded. Note that zero padding will not be used for reads that + begin outside a known byte-range. + """ + + name: ClassVar[str] = "parts" + + def __init__( + self, + blocksize: int, + fetcher: Fetcher, + size: int, + data: dict[tuple[int, int], bytes] | None = None, + strict: bool = False, + **_: Any, + ): + super().__init__(blocksize, fetcher, size) + self.strict = strict + + # simple consolidation of contiguous blocks + if data: + old_offsets = sorted(data.keys()) + offsets = [old_offsets[0]] + blocks = [data.pop(old_offsets[0])] + for start, stop in old_offsets[1:]: + start0, stop0 = offsets[-1] + if start == stop0: + offsets[-1] = (start0, stop) + blocks[-1] += data.pop((start, stop)) + else: + offsets.append((start, stop)) + blocks.append(data.pop((start, stop))) + + self.data = dict(zip(offsets, blocks)) + else: + self.data = {} + + @property + def size(self): + return sum(_[1] - _[0] for _ in self.data) + + @size.setter + def size(self, value): + pass + + @property + def nblocks(self): + return len(self.data) + + @nblocks.setter + def nblocks(self, value): + pass + + def _fetch(self, start: int | None, stop: int | None) -> bytes: + logger.debug("Known parts request %s %s", start, stop) + if start is None: + start = 0 + if stop is None: + stop = self.size + self.total_requested_bytes += stop - start + out = b"" + started = False + loc_old = 0 + for loc0, loc1 in sorted(self.data): + if (loc0 <= start < loc1) and (loc0 <= stop <= loc1): + # entirely within the block + off = start - loc0 + self.hit_count += 1 + return self.data[(loc0, loc1)][off : off + stop - start] + if stop <= loc0: + break + if started and loc0 > loc_old: + # a gap where we need data + self.miss_count += 1 + if self.strict: + raise ValueError + out += b"\x00" * (loc0 - loc_old) + if loc0 <= start < loc1: + # found the start + self.hit_count += 1 + off = start - loc0 + out = self.data[(loc0, loc1)][off : off + stop - start] + started = True + elif start < loc0 and stop > loc1: + # the whole block + self.hit_count += 1 + out += self.data[(loc0, loc1)] + elif loc0 <= stop <= loc1: + # end block + self.hit_count += 1 + out = out + self.data[(loc0, loc1)][: stop - loc0] + return out + loc_old = loc1 + self.miss_count += 1 + if started and not self.strict: + out = out + b"\x00" * (stop - loc_old) + return out + raise ValueError + + +class UpdatableLRU(Generic[P, T]): + """ + Custom implementation of LRU cache that allows updating keys + + Used by BackgroundBlockCache + """ + + class CacheInfo(NamedTuple): + hits: int + misses: int + maxsize: int + currsize: int + + def __init__(self, func: Callable[P, T], max_size: int = 128) -> None: + self._cache: OrderedDict[Any, T] = collections.OrderedDict() + self._func = func + self._max_size = max_size + self._hits = 0 + self._misses = 0 + self._lock = threading.Lock() + + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: + if kwargs: + raise TypeError(f"Got unexpected keyword argument {kwargs.keys()}") + with self._lock: + if args in self._cache: + self._cache.move_to_end(args) + self._hits += 1 + return self._cache[args] + + result = self._func(*args, **kwargs) + + with self._lock: + self._cache[args] = result + self._misses += 1 + if len(self._cache) > self._max_size: + self._cache.popitem(last=False) + + return result + + def is_key_cached(self, *args: Any) -> bool: + with self._lock: + return args in self._cache + + def add_key(self, result: T, *args: Any) -> None: + with self._lock: + self._cache[args] = result + if len(self._cache) > self._max_size: + self._cache.popitem(last=False) + + def cache_info(self) -> UpdatableLRU.CacheInfo: + with self._lock: + return self.CacheInfo( + maxsize=self._max_size, + currsize=len(self._cache), + hits=self._hits, + misses=self._misses, + ) + + +class BackgroundBlockCache(BaseCache): + """ + Cache holding memory as a set of blocks with pre-loading of + the next block in the background. + + Requests are only ever made ``blocksize`` at a time, and are + stored in an LRU cache. The least recently accessed block is + discarded when more than ``maxblocks`` are stored. If the + next block is not in cache, it is loaded in a separate thread + in non-blocking way. + + Parameters + ---------- + blocksize : int + The number of bytes to store in each block. + Requests are only ever made for ``blocksize``, so this + should balance the overhead of making a request against + the granularity of the blocks. + fetcher : Callable + size : int + The total size of the file being cached. + maxblocks : int + The maximum number of blocks to cache for. The maximum memory + use for this cache is then ``blocksize * maxblocks``. + """ + + name: ClassVar[str] = "background" + + def __init__( + self, blocksize: int, fetcher: Fetcher, size: int, maxblocks: int = 32 + ) -> None: + super().__init__(blocksize, fetcher, size) + self.nblocks = math.ceil(size / blocksize) + self.maxblocks = maxblocks + self._fetch_block_cached = UpdatableLRU(self._fetch_block, maxblocks) + + self._thread_executor = ThreadPoolExecutor(max_workers=1) + self._fetch_future_block_number: int | None = None + self._fetch_future: Future[bytes] | None = None + self._fetch_future_lock = threading.Lock() + + def cache_info(self) -> UpdatableLRU.CacheInfo: + """ + The statistics on the block cache. + + Returns + ------- + NamedTuple + Returned directly from the LRU Cache used internally. + """ + return self._fetch_block_cached.cache_info() + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__ + del state["_fetch_block_cached"] + del state["_thread_executor"] + del state["_fetch_future_block_number"] + del state["_fetch_future"] + del state["_fetch_future_lock"] + return state + + def __setstate__(self, state) -> None: + self.__dict__.update(state) + self._fetch_block_cached = UpdatableLRU(self._fetch_block, state["maxblocks"]) + self._thread_executor = ThreadPoolExecutor(max_workers=1) + self._fetch_future_block_number = None + self._fetch_future = None + self._fetch_future_lock = threading.Lock() + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if start is None: + start = 0 + if end is None: + end = self.size + if start >= self.size or start >= end: + return b"" + + # byte position -> block numbers + start_block_number = start // self.blocksize + end_block_number = end // self.blocksize + + fetch_future_block_number = None + fetch_future = None + with self._fetch_future_lock: + # Background thread is running. Check we we can or must join it. + if self._fetch_future is not None: + assert self._fetch_future_block_number is not None + if self._fetch_future.done(): + logger.info("BlockCache joined background fetch without waiting.") + self._fetch_block_cached.add_key( + self._fetch_future.result(), self._fetch_future_block_number + ) + # Cleanup the fetch variables. Done with fetching the block. + self._fetch_future_block_number = None + self._fetch_future = None + else: + # Must join if we need the block for the current fetch + must_join = bool( + start_block_number + <= self._fetch_future_block_number + <= end_block_number + ) + if must_join: + # Copy to the local variables to release lock + # before waiting for result + fetch_future_block_number = self._fetch_future_block_number + fetch_future = self._fetch_future + + # Cleanup the fetch variables. Have a local copy. + self._fetch_future_block_number = None + self._fetch_future = None + + # Need to wait for the future for the current read + if fetch_future is not None: + logger.info("BlockCache waiting for background fetch.") + # Wait until result and put it in cache + self._fetch_block_cached.add_key( + fetch_future.result(), fetch_future_block_number + ) + + # these are cached, so safe to do multiple calls for the same start and end. + for block_number in range(start_block_number, end_block_number + 1): + self._fetch_block_cached(block_number) + + # fetch next block in the background if nothing is running in the background, + # the block is within file and it is not already cached + end_block_plus_1 = end_block_number + 1 + with self._fetch_future_lock: + if ( + self._fetch_future is None + and end_block_plus_1 <= self.nblocks + and not self._fetch_block_cached.is_key_cached(end_block_plus_1) + ): + self._fetch_future_block_number = end_block_plus_1 + self._fetch_future = self._thread_executor.submit( + self._fetch_block, end_block_plus_1, "async" + ) + + return self._read_cache( + start, + end, + start_block_number=start_block_number, + end_block_number=end_block_number, + ) + + def _fetch_block(self, block_number: int, log_info: str = "sync") -> bytes: + """ + Fetch the block of data for `block_number`. + """ + if block_number > self.nblocks: + raise ValueError( + f"'block_number={block_number}' is greater than " + f"the number of blocks ({self.nblocks})" + ) + + start = block_number * self.blocksize + end = start + self.blocksize + logger.info("BlockCache fetching block (%s) %d", log_info, block_number) + self.total_requested_bytes += end - start + self.miss_count += 1 + block_contents = super()._fetch(start, end) + return block_contents + + def _read_cache( + self, start: int, end: int, start_block_number: int, end_block_number: int + ) -> bytes: + """ + Read from our block cache. + + Parameters + ---------- + start, end : int + The start and end byte positions. + start_block_number, end_block_number : int + The start and end block numbers. + """ + start_pos = start % self.blocksize + end_pos = end % self.blocksize + + # kind of pointless to count this as a hit, but it is + self.hit_count += 1 + + if start_block_number == end_block_number: + block = self._fetch_block_cached(start_block_number) + return block[start_pos:end_pos] + + else: + # read from the initial + out = [self._fetch_block_cached(start_block_number)[start_pos:]] + + # intermediate blocks + # Note: it'd be nice to combine these into one big request. However + # that doesn't play nicely with our LRU cache. + out.extend( + map( + self._fetch_block_cached, + range(start_block_number + 1, end_block_number), + ) + ) + + # final block + out.append(self._fetch_block_cached(end_block_number)[:end_pos]) + + return b"".join(out) + + +caches: dict[str | None, type[BaseCache]] = { + # one custom case + None: BaseCache, +} + + +def register_cache(cls: type[BaseCache], clobber: bool = False) -> None: + """'Register' cache implementation. + + Parameters + ---------- + clobber: bool, optional + If set to True (default is False) - allow to overwrite existing + entry. + + Raises + ------ + ValueError + """ + name = cls.name + if not clobber and name in caches: + raise ValueError(f"Cache with name {name!r} is already known: {caches[name]}") + caches[name] = cls + + +for c in ( + BaseCache, + MMapCache, + BytesCache, + ReadAheadCache, + BlockCache, + FirstChunkCache, + AllBytes, + KnownPartsOfAFile, + BackgroundBlockCache, +): + register_cache(c) diff --git a/.venv/lib/python3.11/site-packages/fsspec/callbacks.py b/.venv/lib/python3.11/site-packages/fsspec/callbacks.py new file mode 100644 index 0000000000000000000000000000000000000000..7ca99ca6ac3cd69b28bcd1550f6550e8e648c5fe --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/callbacks.py @@ -0,0 +1,324 @@ +from functools import wraps + + +class Callback: + """ + Base class and interface for callback mechanism + + This class can be used directly for monitoring file transfers by + providing ``callback=Callback(hooks=...)`` (see the ``hooks`` argument, + below), or subclassed for more specialised behaviour. + + Parameters + ---------- + size: int (optional) + Nominal quantity for the value that corresponds to a complete + transfer, e.g., total number of tiles or total number of + bytes + value: int (0) + Starting internal counter value + hooks: dict or None + A dict of named functions to be called on each update. The signature + of these must be ``f(size, value, **kwargs)`` + """ + + def __init__(self, size=None, value=0, hooks=None, **kwargs): + self.size = size + self.value = value + self.hooks = hooks or {} + self.kw = kwargs + + def __enter__(self): + return self + + def __exit__(self, *exc_args): + self.close() + + def close(self): + """Close callback.""" + + def branched(self, path_1, path_2, **kwargs): + """ + Return callback for child transfers + + If this callback is operating at a higher level, e.g., put, which may + trigger transfers that can also be monitored. The function returns a callback + that has to be passed to the child method, e.g., put_file, + as `callback=` argument. + + The implementation uses `callback.branch` for compatibility. + When implementing callbacks, it is recommended to override this function instead + of `branch` and avoid calling `super().branched(...)`. + + Prefer using this function over `branch`. + + Parameters + ---------- + path_1: str + Child's source path + path_2: str + Child's destination path + **kwargs: + Arbitrary keyword arguments + + Returns + ------- + callback: Callback + A callback instance to be passed to the child method + """ + self.branch(path_1, path_2, kwargs) + # mutate kwargs so that we can force the caller to pass "callback=" explicitly + return kwargs.pop("callback", DEFAULT_CALLBACK) + + def branch_coro(self, fn): + """ + Wraps a coroutine, and pass a new child callback to it. + """ + + @wraps(fn) + async def func(path1, path2: str, **kwargs): + with self.branched(path1, path2, **kwargs) as child: + return await fn(path1, path2, callback=child, **kwargs) + + return func + + def set_size(self, size): + """ + Set the internal maximum size attribute + + Usually called if not initially set at instantiation. Note that this + triggers a ``call()``. + + Parameters + ---------- + size: int + """ + self.size = size + self.call() + + def absolute_update(self, value): + """ + Set the internal value state + + Triggers ``call()`` + + Parameters + ---------- + value: int + """ + self.value = value + self.call() + + def relative_update(self, inc=1): + """ + Delta increment the internal counter + + Triggers ``call()`` + + Parameters + ---------- + inc: int + """ + self.value += inc + self.call() + + def call(self, hook_name=None, **kwargs): + """ + Execute hook(s) with current state + + Each function is passed the internal size and current value + + Parameters + ---------- + hook_name: str or None + If given, execute on this hook + kwargs: passed on to (all) hook(s) + """ + if not self.hooks: + return + kw = self.kw.copy() + kw.update(kwargs) + if hook_name: + if hook_name not in self.hooks: + return + return self.hooks[hook_name](self.size, self.value, **kw) + for hook in self.hooks.values() or []: + hook(self.size, self.value, **kw) + + def wrap(self, iterable): + """ + Wrap an iterable to call ``relative_update`` on each iterations + + Parameters + ---------- + iterable: Iterable + The iterable that is being wrapped + """ + for item in iterable: + self.relative_update() + yield item + + def branch(self, path_1, path_2, kwargs): + """ + Set callbacks for child transfers + + If this callback is operating at a higher level, e.g., put, which may + trigger transfers that can also be monitored. The passed kwargs are + to be *mutated* to add ``callback=``, if this class supports branching + to children. + + Parameters + ---------- + path_1: str + Child's source path + path_2: str + Child's destination path + kwargs: dict + arguments passed to child method, e.g., put_file. + + Returns + ------- + + """ + return None + + def no_op(self, *_, **__): + pass + + def __getattr__(self, item): + """ + If undefined methods are called on this class, nothing happens + """ + return self.no_op + + @classmethod + def as_callback(cls, maybe_callback=None): + """Transform callback=... into Callback instance + + For the special value of ``None``, return the global instance of + ``NoOpCallback``. This is an alternative to including + ``callback=DEFAULT_CALLBACK`` directly in a method signature. + """ + if maybe_callback is None: + return DEFAULT_CALLBACK + return maybe_callback + + +class NoOpCallback(Callback): + """ + This implementation of Callback does exactly nothing + """ + + def call(self, *args, **kwargs): + return None + + +class DotPrinterCallback(Callback): + """ + Simple example Callback implementation + + Almost identical to Callback with a hook that prints a char; here we + demonstrate how the outer layer may print "#" and the inner layer "." + """ + + def __init__(self, chr_to_print="#", **kwargs): + self.chr = chr_to_print + super().__init__(**kwargs) + + def branch(self, path_1, path_2, kwargs): + """Mutate kwargs to add new instance with different print char""" + kwargs["callback"] = DotPrinterCallback(".") + + def call(self, **kwargs): + """Just outputs a character""" + print(self.chr, end="") + + +class TqdmCallback(Callback): + """ + A callback to display a progress bar using tqdm + + Parameters + ---------- + tqdm_kwargs : dict, (optional) + Any argument accepted by the tqdm constructor. + See the `tqdm doc <https://tqdm.github.io/docs/tqdm/#__init__>`_. + Will be forwarded to `tqdm_cls`. + tqdm_cls: (optional) + subclass of `tqdm.tqdm`. If not passed, it will default to `tqdm.tqdm`. + + Examples + -------- + >>> import fsspec + >>> from fsspec.callbacks import TqdmCallback + >>> fs = fsspec.filesystem("memory") + >>> path2distant_data = "/your-path" + >>> fs.upload( + ".", + path2distant_data, + recursive=True, + callback=TqdmCallback(), + ) + + You can forward args to tqdm using the ``tqdm_kwargs`` parameter. + + >>> fs.upload( + ".", + path2distant_data, + recursive=True, + callback=TqdmCallback(tqdm_kwargs={"desc": "Your tqdm description"}), + ) + + You can also customize the progress bar by passing a subclass of `tqdm`. + + .. code-block:: python + + class TqdmFormat(tqdm): + '''Provides a `total_time` format parameter''' + @property + def format_dict(self): + d = super().format_dict + total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) + d.update(total_time=self.format_interval(total_time) + " in total") + return d + + >>> with TqdmCallback( + tqdm_kwargs={ + "desc": "desc", + "bar_format": "{total_time}: {percentage:.0f}%|{bar}{r_bar}", + }, + tqdm_cls=TqdmFormat, + ) as callback: + fs.upload(".", path2distant_data, recursive=True, callback=callback) + """ + + def __init__(self, tqdm_kwargs=None, *args, **kwargs): + try: + from tqdm import tqdm + + except ImportError as exce: + raise ImportError( + "Using TqdmCallback requires tqdm to be installed" + ) from exce + + self._tqdm_cls = kwargs.pop("tqdm_cls", tqdm) + self._tqdm_kwargs = tqdm_kwargs or {} + self.tqdm = None + super().__init__(*args, **kwargs) + + def call(self, *args, **kwargs): + if self.tqdm is None: + self.tqdm = self._tqdm_cls(total=self.size, **self._tqdm_kwargs) + self.tqdm.total = self.size + self.tqdm.update(self.value - self.tqdm.n) + + def close(self): + if self.tqdm is not None: + self.tqdm.close() + self.tqdm = None + + def __del__(self): + return self.close() + + +DEFAULT_CALLBACK = _DEFAULT_CALLBACK = NoOpCallback() diff --git a/.venv/lib/python3.11/site-packages/fsspec/compression.py b/.venv/lib/python3.11/site-packages/fsspec/compression.py new file mode 100644 index 0000000000000000000000000000000000000000..11c2e3d3f142d95186663fa5a747911e66832266 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/compression.py @@ -0,0 +1,185 @@ +"""Helper functions for a standard streaming compression API""" + +import sys +from zipfile import ZipFile + +import fsspec.utils +from fsspec.spec import AbstractBufferedFile + + +def noop_file(file, mode, **kwargs): + return file + + +# TODO: files should also be available as contexts +# should be functions of the form func(infile, mode=, **kwargs) -> file-like +compr = {None: noop_file} + + +def register_compression(name, callback, extensions, force=False): + """Register an "inferable" file compression type. + + Registers transparent file compression type for use with fsspec.open. + Compression can be specified by name in open, or "infer"-ed for any files + ending with the given extensions. + + Args: + name: (str) The compression type name. Eg. "gzip". + callback: A callable of form (infile, mode, **kwargs) -> file-like. + Accepts an input file-like object, the target mode and kwargs. + Returns a wrapped file-like object. + extensions: (str, Iterable[str]) A file extension, or list of file + extensions for which to infer this compression scheme. Eg. "gz". + force: (bool) Force re-registration of compression type or extensions. + + Raises: + ValueError: If name or extensions already registered, and not force. + + """ + if isinstance(extensions, str): + extensions = [extensions] + + # Validate registration + if name in compr and not force: + raise ValueError(f"Duplicate compression registration: {name}") + + for ext in extensions: + if ext in fsspec.utils.compressions and not force: + raise ValueError(f"Duplicate compression file extension: {ext} ({name})") + + compr[name] = callback + + for ext in extensions: + fsspec.utils.compressions[ext] = name + + +def unzip(infile, mode="rb", filename=None, **kwargs): + if "r" not in mode: + filename = filename or "file" + z = ZipFile(infile, mode="w", **kwargs) + fo = z.open(filename, mode="w") + fo.close = lambda closer=fo.close: closer() or z.close() + return fo + z = ZipFile(infile) + if filename is None: + filename = z.namelist()[0] + return z.open(filename, mode="r", **kwargs) + + +register_compression("zip", unzip, "zip") + +try: + from bz2 import BZ2File +except ImportError: + pass +else: + register_compression("bz2", BZ2File, "bz2") + +try: # pragma: no cover + from isal import igzip + + def isal(infile, mode="rb", **kwargs): + return igzip.IGzipFile(fileobj=infile, mode=mode, **kwargs) + + register_compression("gzip", isal, "gz") +except ImportError: + from gzip import GzipFile + + register_compression( + "gzip", lambda f, **kwargs: GzipFile(fileobj=f, **kwargs), "gz" + ) + +try: + from lzma import LZMAFile + + register_compression("lzma", LZMAFile, "lzma") + register_compression("xz", LZMAFile, "xz") +except ImportError: + pass + +try: + import lzmaffi + + register_compression("lzma", lzmaffi.LZMAFile, "lzma", force=True) + register_compression("xz", lzmaffi.LZMAFile, "xz", force=True) +except ImportError: + pass + + +class SnappyFile(AbstractBufferedFile): + def __init__(self, infile, mode, **kwargs): + import snappy + + super().__init__( + fs=None, path="snappy", mode=mode.strip("b") + "b", size=999999999, **kwargs + ) + self.infile = infile + if "r" in mode: + self.codec = snappy.StreamDecompressor() + else: + self.codec = snappy.StreamCompressor() + + def _upload_chunk(self, final=False): + self.buffer.seek(0) + out = self.codec.add_chunk(self.buffer.read()) + self.infile.write(out) + return True + + def seek(self, loc, whence=0): + raise NotImplementedError("SnappyFile is not seekable") + + def seekable(self): + return False + + def _fetch_range(self, start, end): + """Get the specified set of bytes from remote""" + data = self.infile.read(end - start) + return self.codec.decompress(data) + + +try: + import snappy + + snappy.compress(b"") + # Snappy may use the .sz file extension, but this is not part of the + # standard implementation. + register_compression("snappy", SnappyFile, []) + +except (ImportError, NameError, AttributeError): + pass + +try: + import lz4.frame + + register_compression("lz4", lz4.frame.open, "lz4") +except ImportError: + pass + +try: + if sys.version_info >= (3, 14): + from compression import zstd + else: + from backports import zstd + + register_compression("zstd", zstd.ZstdFile, "zst") +except ImportError: + try: + import zstandard as zstd + + def zstandard_file(infile, mode="rb"): + if "r" in mode: + cctx = zstd.ZstdDecompressor() + return cctx.stream_reader(infile) + else: + cctx = zstd.ZstdCompressor(level=10) + return cctx.stream_writer(infile) + + register_compression("zstd", zstandard_file, "zst") + except ImportError: + pass + pass + + +def available_compressions(): + """Return a list of the implemented compressions.""" + return list(compr) diff --git a/.venv/lib/python3.11/site-packages/fsspec/config.py b/.venv/lib/python3.11/site-packages/fsspec/config.py new file mode 100644 index 0000000000000000000000000000000000000000..19f68071ca026dc2bb11da70effb0c517de5380b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/config.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import configparser +import json +import os +import warnings +from typing import Any + +conf: dict[str, dict[str, Any]] = {} +default_conf_dir = os.path.join(os.path.expanduser("~"), ".config/fsspec") +conf_dir = os.environ.get("FSSPEC_CONFIG_DIR", default_conf_dir) + + +def set_conf_env(conf_dict, envdict=os.environ): + """Set config values from environment variables + + Looks for variables of the form ``FSSPEC_<protocol>`` and + ``FSSPEC_<protocol>_<kwarg>``. For ``FSSPEC_<protocol>`` the value is parsed + as a json dictionary and used to ``update`` the config of the + corresponding protocol. For ``FSSPEC_<protocol>_<kwarg>`` there is no + attempt to convert the string value, but the kwarg keys will be lower-cased. + + The ``FSSPEC_<protocol>_<kwarg>`` variables are applied after the + ``FSSPEC_<protocol>`` ones. + + Parameters + ---------- + conf_dict : dict(str, dict) + This dict will be mutated + envdict : dict-like(str, str) + Source for the values - usually the real environment + """ + envdict = dict(envdict) + kwarg_keys = [] + for key in envdict: + if key.startswith("FSSPEC_") and len(key) > 7 and key[7] != "_": + try: + value = json.loads(envdict[key]) + envdict[key] = value + except json.decoder.JSONDecodeError: + value = envdict[key] + if key.count("_") > 1: + kwarg_keys.append(key) + continue + else: + if isinstance(value, dict): + _, proto = key.split("_", 1) + conf_dict.setdefault(proto.lower(), {}).update(value) + else: + warnings.warn( + f"Ignoring environment variable {key} due to not being a dict:" + f" {type(value)}" + ) + elif key.startswith("FSSPEC"): + warnings.warn( + f"Ignoring environment variable {key} due to having an unexpected name" + ) + + for key in kwarg_keys: + _, proto, kwarg = key.split("_", 2) + conf_dict.setdefault(proto.lower(), {})[kwarg.lower()] = envdict[key] + + +def set_conf_files(cdir, conf_dict): + """Set config values from files + + Scans for INI and JSON files in the given dictionary, and uses their + contents to set the config. In case of repeated values, later values + win. + + In the case of INI files, all values are strings, and these will not + be converted. + + Parameters + ---------- + cdir : str + Directory to search + conf_dict : dict(str, dict) + This dict will be mutated + """ + if not os.path.isdir(cdir): + return + allfiles = sorted(os.listdir(cdir)) + for fn in allfiles: + if fn.endswith(".ini"): + ini = configparser.ConfigParser() + ini.read(os.path.join(cdir, fn)) + for key in ini: + if key == "DEFAULT": + continue + conf_dict.setdefault(key, {}).update(dict(ini[key])) + if fn.endswith(".json"): + with open(os.path.join(cdir, fn)) as f: + js = json.load(f) + for key in js: + conf_dict.setdefault(key, {}).update(dict(js[key])) + + +def apply_config(cls, kwargs, conf_dict=None): + """Supply default values for kwargs when instantiating class + + Augments the passed kwargs, by finding entries in the config dict + which match the classes ``.protocol`` attribute (one or more str) + + Parameters + ---------- + cls : file system implementation + kwargs : dict + conf_dict : dict of dict + Typically this is the global configuration + + Returns + ------- + dict : the modified set of kwargs + """ + if conf_dict is None: + conf_dict = conf + protos = cls.protocol if isinstance(cls.protocol, (tuple, list)) else [cls.protocol] + kw = {} + for proto in protos: + # default kwargs from the current state of the config + if proto in conf_dict: + kw.update(conf_dict[proto]) + # explicit kwargs always win + kw.update(**kwargs) + kwargs = kw + return kwargs + + +set_conf_files(conf_dir, conf) +set_conf_env(conf) diff --git a/.venv/lib/python3.11/site-packages/fsspec/conftest.py b/.venv/lib/python3.11/site-packages/fsspec/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..f05eb5c30d42b0c1c5cc432f9c217d8f0e01f412 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/conftest.py @@ -0,0 +1,125 @@ +import os +import shutil +import subprocess +import sys +import time +from collections import deque +from collections.abc import Generator, Sequence + +import pytest + +import fsspec + + +@pytest.fixture() +def m(): + """ + Fixture providing a memory filesystem. + """ + m = fsspec.filesystem("memory") + m.store.clear() + m.pseudo_dirs.clear() + m.pseudo_dirs.append("") + try: + yield m + finally: + m.store.clear() + m.pseudo_dirs.clear() + m.pseudo_dirs.append("") + + +class InstanceCacheInspector: + """ + Helper class to inspect instance caches of filesystem classes in tests. + """ + + def clear(self) -> None: + """ + Clear instance caches of all currently imported filesystem classes. + """ + classes = deque([fsspec.spec.AbstractFileSystem]) + while classes: + cls = classes.popleft() + cls.clear_instance_cache() + classes.extend(cls.__subclasses__()) + + def gather_counts(self, *, omit_zero: bool = True) -> dict[str, int]: + """ + Gather counts of filesystem instances in the instance caches + of all currently imported filesystem classes. + + Parameters + ---------- + omit_zero: + Whether to omit instance types with no cached instances. + """ + out: dict[str, int] = {} + classes = deque([fsspec.spec.AbstractFileSystem]) + while classes: + cls = classes.popleft() + count = len(cls._cache) # there is no public interface for the cache + # note: skip intermediate AbstractFileSystem subclasses + # if they proxy the protocol attribute via a property. + if isinstance(cls.protocol, (Sequence, str)): + key = cls.protocol if isinstance(cls.protocol, str) else cls.protocol[0] + if count or not omit_zero: + out[key] = count + classes.extend(cls.__subclasses__()) + return out + + +@pytest.fixture(scope="function", autouse=True) +def instance_caches() -> Generator[InstanceCacheInspector, None, None]: + """ + Fixture to ensure empty filesystem instance caches before and after a test. + + Used by default for all tests. + Clears caches of all imported filesystem classes. + Can be used to write test assertions about instance caches. + + Usage: + + def test_something(instance_caches): + # Test code here + fsspec.open("file://abc") + fsspec.open("memory://foo/bar") + + # Test assertion + assert instance_caches.gather_counts() == {"file": 1, "memory": 1} + + Returns + ------- + instance_caches: An instance cache inspector for clearing and inspecting caches. + """ + ic = InstanceCacheInspector() + + ic.clear() + try: + yield ic + finally: + ic.clear() + + +@pytest.fixture(scope="function") +def ftp_writable(tmpdir): + """ + Fixture providing a writable FTP filesystem. + """ + pytest.importorskip("pyftpdlib") + + d = str(tmpdir) + with open(os.path.join(d, "out"), "wb") as f: + f.write(b"hello" * 10000) + P = subprocess.Popen( + [sys.executable, "-m", "pyftpdlib", "-d", d, "-u", "user", "-P", "pass", "-w"] + ) + try: + time.sleep(1) + yield "localhost", 2121, "user", "pass" + finally: + P.terminate() + P.wait() + try: + shutil.rmtree(tmpdir) + except Exception: + pass diff --git a/.venv/lib/python3.11/site-packages/fsspec/core.py b/.venv/lib/python3.11/site-packages/fsspec/core.py new file mode 100644 index 0000000000000000000000000000000000000000..5876bfcefc176b3b3aed0e16b54fa3809a5a0eee --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/core.py @@ -0,0 +1,760 @@ +from __future__ import annotations + +import io +import logging +import os +import re +from glob import has_magic +from pathlib import Path + +# for backwards compat, we export cache things from here too +from fsspec.caching import ( # noqa: F401 + BaseCache, + BlockCache, + BytesCache, + MMapCache, + ReadAheadCache, + caches, +) +from fsspec.compression import compr +from fsspec.config import conf +from fsspec.registry import available_protocols, filesystem, get_filesystem_class +from fsspec.utils import ( + _unstrip_protocol, + build_name_function, + infer_compression, + stringify_path, +) + +logger = logging.getLogger("fsspec") + + +class OpenFile: + """ + File-like object to be used in a context + + Can layer (buffered) text-mode and compression over any file-system, which + are typically binary-only. + + These instances are safe to serialize, as the low-level file object + is not created until invoked using ``with``. + + Parameters + ---------- + fs: FileSystem + The file system to use for opening the file. Should be a subclass or duck-type + with ``fsspec.spec.AbstractFileSystem`` + path: str + Location to open + mode: str like 'rb', optional + Mode of the opened file + compression: str or None, optional + Compression to apply + encoding: str or None, optional + The encoding to use if opened in text mode. + errors: str or None, optional + How to handle encoding errors if opened in text mode. + newline: None or str + Passed to TextIOWrapper in text mode, how to handle line endings. + autoopen: bool + If True, calls open() immediately. Mostly used by pickle + pos: int + If given and autoopen is True, seek to this location immediately + """ + + def __init__( + self, + fs, + path, + mode="rb", + compression=None, + encoding=None, + errors=None, + newline=None, + ): + self.fs = fs + self.path = path + self.mode = mode + self.compression = get_compression(path, compression) + self.encoding = encoding + self.errors = errors + self.newline = newline + self.fobjects = [] + + def __reduce__(self): + return ( + OpenFile, + ( + self.fs, + self.path, + self.mode, + self.compression, + self.encoding, + self.errors, + self.newline, + ), + ) + + def __repr__(self): + return f"<OpenFile '{self.path}'>" + + def __enter__(self): + mode = self.mode.replace("t", "").replace("b", "") + "b" + + try: + f = self.fs.open(self.path, mode=mode) + except FileNotFoundError as e: + if has_magic(self.path): + raise FileNotFoundError( + "%s not found. The URL contains glob characters: you maybe needed\n" + "to pass expand=True in fsspec.open() or the storage_options of \n" + "your library. You can also set the config value 'open_expand'\n" + "before import, or fsspec.core.DEFAULT_EXPAND at runtime, to True.", + self.path, + ) from e + raise + + self.fobjects = [f] + + if self.compression is not None: + compress = compr[self.compression] + f = compress(f, mode=mode[0]) + self.fobjects.append(f) + + if "b" not in self.mode: + # assume, for example, that 'r' is equivalent to 'rt' as in builtin + f = PickleableTextIOWrapper( + f, encoding=self.encoding, errors=self.errors, newline=self.newline + ) + self.fobjects.append(f) + + return self.fobjects[-1] + + def __exit__(self, *args): + self.close() + + @property + def full_name(self): + return _unstrip_protocol(self.path, self.fs) + + def open(self): + """Materialise this as a real open file without context + + The OpenFile object should be explicitly closed to avoid enclosed file + instances persisting. You must, therefore, keep a reference to the OpenFile + during the life of the file-like it generates. + """ + return self.__enter__() + + def close(self): + """Close all encapsulated file objects""" + for f in reversed(self.fobjects): + if "r" not in self.mode and not f.closed: + f.flush() + f.close() + self.fobjects.clear() + + +class OpenFiles(list): + """List of OpenFile instances + + Can be used in a single context, which opens and closes all of the + contained files. Normal list access to get the elements works as + normal. + + A special case is made for caching filesystems - the files will + be down/uploaded together at the start or end of the context, and + this may happen concurrently, if the target filesystem supports it. + """ + + def __init__(self, *args, mode="rb", fs=None): + self.mode = mode + self.fs = fs + self.files = [] + super().__init__(*args) + + def __enter__(self): + if self.fs is None: + raise ValueError("Context has already been used") + + fs = self.fs + while True: + if hasattr(fs, "open_many"): + # check for concurrent cache download; or set up for upload + self.files = fs.open_many(self) + return self.files + if hasattr(fs, "fs") and fs.fs is not None: + fs = fs.fs + else: + break + return [s.__enter__() for s in self] + + def __exit__(self, *args): + fs = self.fs + [s.__exit__(*args) for s in self] + if "r" not in self.mode: + while True: + if hasattr(fs, "open_many"): + # check for concurrent cache upload + fs.commit_many(self.files) + return + if hasattr(fs, "fs") and fs.fs is not None: + fs = fs.fs + else: + break + + def __getitem__(self, item): + out = super().__getitem__(item) + if isinstance(item, slice): + return OpenFiles(out, mode=self.mode, fs=self.fs) + return out + + def __repr__(self): + return f"<List of {len(self)} OpenFile instances>" + + +def open_files( + urlpath, + mode="rb", + compression=None, + encoding="utf8", + errors=None, + name_function=None, + num=1, + protocol=None, + newline=None, + auto_mkdir=True, + expand=True, + **kwargs, +): + """Given a path or paths, return a list of ``OpenFile`` objects. + + For writing, a str path must contain the "*" character, which will be filled + in by increasing numbers, e.g., "part*" -> "part1", "part2" if num=2. + + For either reading or writing, can instead provide explicit list of paths. + + Parameters + ---------- + urlpath: string or list + Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` + to read from alternative filesystems. To read from multiple files you + can pass a globstring or a list of paths, with the caveat that they + must all have the same protocol. + mode: 'rb', 'wt', etc. + compression: string or None + If given, open file using compression codec. Can either be a compression + name (a key in ``fsspec.compression.compr``) or "infer" to guess the + compression from the filename suffix. + encoding: str + For text mode only + errors: None or str + Passed to TextIOWrapper in text mode + name_function: function or None + if opening a set of files for writing, those files do not yet exist, + so we need to generate their names by formatting the urlpath for + each sequence number + num: int [1] + if writing mode, number of files we expect to create (passed to + name+function) + protocol: str or None + If given, overrides the protocol found in the URL. + newline: bytes or None + Used for line terminator in text mode. If None, uses system default; + if blank, uses no translation. + auto_mkdir: bool (True) + If in write mode, this will ensure the target directory exists before + writing, by calling ``fs.mkdirs(exist_ok=True)``. + expand: bool + **kwargs: dict + Extra options that make sense to a particular storage connection, e.g. + host, port, username, password, etc. + + Examples + -------- + >>> files = open_files('2015-*-*.csv') # doctest: +SKIP + >>> files = open_files( + ... 's3://bucket/2015-*-*.csv.gz', compression='gzip' + ... ) # doctest: +SKIP + + Returns + ------- + An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can + be used as a single context + + Notes + ----- + For a full list of the available protocols and the implementations that + they map across to see the latest online documentation: + + - For implementations built into ``fsspec`` see + https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations + - For implementations in separate packages see + https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations + """ + fs, fs_token, paths = get_fs_token_paths( + urlpath, + mode, + num=num, + name_function=name_function, + storage_options=kwargs, + protocol=protocol, + expand=expand, + ) + if fs.protocol == "file": + fs.auto_mkdir = auto_mkdir + elif "r" not in mode and auto_mkdir: + parents = {fs._parent(path) for path in paths} + for parent in parents: + try: + fs.makedirs(parent, exist_ok=True) + except PermissionError: + pass + return OpenFiles( + [ + OpenFile( + fs, + path, + mode=mode, + compression=compression, + encoding=encoding, + errors=errors, + newline=newline, + ) + for path in paths + ], + mode=mode, + fs=fs, + ) + + +def _un_chain(path, kwargs): + # Avoid a circular import + from fsspec.implementations.chained import ChainedFileSystem + + if "::" in path: + x = re.compile(".*[^a-z]+.*") # test for non protocol-like single word + known_protocols = set(available_protocols()) + bits = [] + + # split on '::', then ensure each bit has a protocol + for p in path.split("::"): + if p in known_protocols: + bits.append(p + "://") + elif "://" in p or x.match(p): + bits.append(p) + else: + bits.append(p + "://") + else: + bits = [path] + + # [[url, protocol, kwargs], ...] + out = [] + previous_bit = None + kwargs = kwargs.copy() + + for bit in reversed(bits): + protocol = kwargs.pop("protocol", None) or split_protocol(bit)[0] or "file" + cls = get_filesystem_class(protocol) + extra_kwargs = cls._get_kwargs_from_urls(bit) + kws = kwargs.pop(protocol, {}) + + if bit is bits[0]: + kws.update(kwargs) + + kw = dict( + **{k: v for k, v in extra_kwargs.items() if k not in kws or v != kws[k]}, + **kws, + ) + bit = cls._strip_protocol(bit) + + if ( + "target_protocol" not in kw + and issubclass(cls, ChainedFileSystem) + and not bit + ): + # replace bit if we are chaining and no path given + bit = previous_bit + + out.append((bit, protocol, kw)) + previous_bit = bit + + out.reverse() + return out + + +def url_to_fs(url, **kwargs): + """ + Turn fully-qualified and potentially chained URL into filesystem instance + + Parameters + ---------- + url : str + The fsspec-compatible URL + **kwargs: dict + Extra options that make sense to a particular storage connection, e.g. + host, port, username, password, etc. + + Returns + ------- + filesystem : FileSystem + The new filesystem discovered from ``url`` and created with + ``**kwargs``. + urlpath : str + The file-systems-specific URL for ``url``. + """ + url = stringify_path(url) + # non-FS arguments that appear in fsspec.open() + # inspect could keep this in sync with open()'s signature + known_kwargs = { + "compression", + "encoding", + "errors", + "expand", + "mode", + "name_function", + "newline", + "num", + } + kwargs = {k: v for k, v in kwargs.items() if k not in known_kwargs} + chain = _un_chain(url, kwargs) + inkwargs = {} + # Reverse iterate the chain, creating a nested target_* structure + for i, ch in enumerate(reversed(chain)): + urls, protocol, kw = ch + if i == len(chain) - 1: + inkwargs = dict(**kw, **inkwargs) + continue + inkwargs["target_options"] = dict(**kw, **inkwargs) + inkwargs["target_protocol"] = protocol + inkwargs["fo"] = urls + urlpath, protocol, _ = chain[0] + fs = filesystem(protocol, **inkwargs) + return fs, urlpath + + +DEFAULT_EXPAND = conf.get("open_expand", False) + + +def open( + urlpath, + mode="rb", + compression=None, + encoding="utf8", + errors=None, + protocol=None, + newline=None, + expand=None, + **kwargs, +): + """Given a path or paths, return one ``OpenFile`` object. + + Parameters + ---------- + urlpath: string or list + Absolute or relative filepath. Prefix with a protocol like ``s3://`` + to read from alternative filesystems. Should not include glob + character(s). + mode: 'rb', 'wt', etc. + compression: string or None + If given, open file using compression codec. Can either be a compression + name (a key in ``fsspec.compression.compr``) or "infer" to guess the + compression from the filename suffix. + encoding: str + For text mode only + errors: None or str + Passed to TextIOWrapper in text mode + protocol: str or None + If given, overrides the protocol found in the URL. + newline: bytes or None + Used for line terminator in text mode. If None, uses system default; + if blank, uses no translation. + expand: bool or None + Whether to regard file paths containing special glob characters as needing + expansion (finding the first match) or absolute. Setting False allows using + paths which do embed such characters. If None (default), this argument + takes its value from the DEFAULT_EXPAND module variable, which takes + its initial value from the "open_expand" config value at startup, which will + be False if not set. + **kwargs: dict + Extra options that make sense to a particular storage connection, e.g. + host, port, username, password, etc. + + Examples + -------- + >>> openfile = open('2015-01-01.csv') # doctest: +SKIP + >>> openfile = open( + ... 's3://bucket/2015-01-01.csv.gz', compression='gzip' + ... ) # doctest: +SKIP + >>> with openfile as f: + ... df = pd.read_csv(f) # doctest: +SKIP + ... + + Returns + ------- + ``OpenFile`` object. + + Notes + ----- + For a full list of the available protocols and the implementations that + they map across to see the latest online documentation: + + - For implementations built into ``fsspec`` see + https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations + - For implementations in separate packages see + https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations + """ + expand = DEFAULT_EXPAND if expand is None else expand + out = open_files( + urlpath=[urlpath], + mode=mode, + compression=compression, + encoding=encoding, + errors=errors, + protocol=protocol, + newline=newline, + expand=expand, + **kwargs, + ) + if not out: + raise FileNotFoundError(urlpath) + return out[0] + + +def open_local( + url: str | list[str] | Path | list[Path], + mode: str = "rb", + **storage_options: dict, +) -> str | list[str]: + """Open file(s) which can be resolved to local + + For files which either are local, or get downloaded upon open + (e.g., by file caching) + + Parameters + ---------- + url: str or list(str) + mode: str + Must be read mode + storage_options: + passed on to FS for or used by open_files (e.g., compression) + """ + if "r" not in mode: + raise ValueError("Can only ensure local files when reading") + of = open_files(url, mode=mode, **storage_options) + if not getattr(of[0].fs, "local_file", False): + raise ValueError( + "open_local can only be used on a filesystem which" + " has attribute local_file=True" + ) + with of as files: + paths = [f.name for f in files] + if (isinstance(url, str) and not has_magic(url)) or isinstance(url, Path): + return paths[0] + return paths + + +def get_compression(urlpath, compression): + if compression == "infer": + compression = infer_compression(urlpath) + if compression is not None and compression not in compr: + raise ValueError(f"Compression type {compression} not supported") + return compression + + +def split_protocol(urlpath): + """Return protocol, path pair""" + urlpath = stringify_path(urlpath) + if "://" in urlpath: + protocol, path = urlpath.split("://", 1) + if len(protocol) > 1: + # excludes Windows paths + return protocol, path + if urlpath.startswith("data:"): + return urlpath.split(":", 1) + return None, urlpath + + +def strip_protocol(urlpath): + """Return only path part of full URL, according to appropriate backend""" + protocol, _ = split_protocol(urlpath) + cls = get_filesystem_class(protocol) + return cls._strip_protocol(urlpath) + + +def expand_paths_if_needed(paths, mode, num, fs, name_function): + """Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` + in them (read mode). + + :param paths: list of paths + mode: str + Mode in which to open files. + num: int + If opening in writing mode, number of files we expect to create. + fs: filesystem object + name_function: callable + If opening in writing mode, this callable is used to generate path + names. Names are generated for each partition by + ``urlpath.replace('*', name_function(partition_index))``. + :return: list of paths + """ + expanded_paths = [] + paths = list(paths) + + if "w" in mode: # read mode + if sum(1 for p in paths if "*" in p) > 1: + raise ValueError( + "When writing data, only one filename mask can be specified." + ) + num = max(num, len(paths)) + + for curr_path in paths: + if "*" in curr_path: + # expand using name_function + expanded_paths.extend(_expand_paths(curr_path, name_function, num)) + else: + expanded_paths.append(curr_path) + # if we generated more paths that asked for, trim the list + if len(expanded_paths) > num: + expanded_paths = expanded_paths[:num] + + else: # read mode + for curr_path in paths: + if has_magic(curr_path): + # expand using glob + expanded_paths.extend(fs.glob(curr_path)) + else: + expanded_paths.append(curr_path) + + return expanded_paths + + +def get_fs_token_paths( + urlpath, + mode="rb", + num=1, + name_function=None, + storage_options=None, + protocol=None, + expand=True, +): + """Filesystem, deterministic token, and paths from a urlpath and options. + + Parameters + ---------- + urlpath: string or iterable + Absolute or relative filepath, URL (may include protocols like + ``s3://``), or globstring pointing to data. + mode: str, optional + Mode in which to open files. + num: int, optional + If opening in writing mode, number of files we expect to create. + name_function: callable, optional + If opening in writing mode, this callable is used to generate path + names. Names are generated for each partition by + ``urlpath.replace('*', name_function(partition_index))``. + storage_options: dict, optional + Additional keywords to pass to the filesystem class. + protocol: str or None + To override the protocol specifier in the URL + expand: bool + Expand string paths for writing, assuming the path is a directory + """ + if isinstance(urlpath, (list, tuple, set)): + if not urlpath: + raise ValueError("empty urlpath sequence") + urlpath0 = stringify_path(next(iter(urlpath))) + else: + urlpath0 = stringify_path(urlpath) + storage_options = storage_options or {} + if protocol: + storage_options["protocol"] = protocol + chain = _un_chain(urlpath0, storage_options or {}) + inkwargs = {} + # Reverse iterate the chain, creating a nested target_* structure + for i, ch in enumerate(reversed(chain)): + urls, nested_protocol, kw = ch + if i == len(chain) - 1: + inkwargs = dict(**kw, **inkwargs) + continue + inkwargs["target_options"] = dict(**kw, **inkwargs) + inkwargs["target_protocol"] = nested_protocol + inkwargs["fo"] = urls + paths, protocol, _ = chain[0] + fs = filesystem(protocol, **inkwargs) + if isinstance(urlpath, (list, tuple, set)): + pchains = [ + _un_chain(stringify_path(u), storage_options or {})[0] for u in urlpath + ] + if len({pc[1] for pc in pchains}) > 1: + raise ValueError("Protocol mismatch getting fs from %s", urlpath) + paths = [pc[0] for pc in pchains] + else: + paths = fs._strip_protocol(paths) + if isinstance(paths, (list, tuple, set)): + if expand: + paths = expand_paths_if_needed(paths, mode, num, fs, name_function) + elif not isinstance(paths, list): + paths = list(paths) + else: + if ("w" in mode or "x" in mode) and expand: + paths = _expand_paths(paths, name_function, num) + elif "*" in paths: + paths = [f for f in sorted(fs.glob(paths)) if not fs.isdir(f)] + else: + paths = [paths] + + return fs, fs._fs_token, paths + + +def _expand_paths(path, name_function, num): + if isinstance(path, str): + if path.count("*") > 1: + raise ValueError("Output path spec must contain exactly one '*'.") + elif "*" not in path: + path = os.path.join(path, "*.part") + + if name_function is None: + name_function = build_name_function(num - 1) + + paths = [path.replace("*", name_function(i)) for i in range(num)] + if paths != sorted(paths): + logger.warning( + "In order to preserve order between partitions" + " paths created with ``name_function`` should " + "sort to partition order" + ) + elif isinstance(path, (tuple, list)): + assert len(path) == num + paths = list(path) + else: + raise ValueError( + "Path should be either\n" + "1. A list of paths: ['foo.json', 'bar.json', ...]\n" + "2. A directory: 'foo/\n" + "3. A path with a '*' in it: 'foo.*.json'" + ) + return paths + + +class PickleableTextIOWrapper(io.TextIOWrapper): + """TextIOWrapper cannot be pickled. This solves it. + + Requires that ``buffer`` be pickleable, which all instances of + AbstractBufferedFile are. + """ + + def __init__( + self, + buffer, + encoding=None, + errors=None, + newline=None, + line_buffering=False, + write_through=False, + ): + self.args = buffer, encoding, errors, newline, line_buffering, write_through + super().__init__(*self.args) + + def __reduce__(self): + return PickleableTextIOWrapper, self.args diff --git a/.venv/lib/python3.11/site-packages/fsspec/dircache.py b/.venv/lib/python3.11/site-packages/fsspec/dircache.py new file mode 100644 index 0000000000000000000000000000000000000000..eca19566b135e5a7a4f6e7407d56411ec58bfe44 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/dircache.py @@ -0,0 +1,98 @@ +import time +from collections.abc import MutableMapping +from functools import lru_cache + + +class DirCache(MutableMapping): + """ + Caching of directory listings, in a structure like:: + + {"path0": [ + {"name": "path0/file0", + "size": 123, + "type": "file", + ... + }, + {"name": "path0/file1", + }, + ... + ], + "path1": [...] + } + + Parameters to this class control listing expiry or indeed turn + caching off + """ + + def __init__( + self, + use_listings_cache=True, + listings_expiry_time=None, + max_paths=None, + **kwargs, + ): + """ + + Parameters + ---------- + use_listings_cache: bool + If False, this cache never returns items, but always reports KeyError, + and setting items has no effect + listings_expiry_time: int or float (optional) + Time in seconds that a listing is considered valid. If None, + listings do not expire. + max_paths: int (optional) + The number of most recent listings that are considered valid; 'recent' + refers to when the entry was set. + """ + self._cache = {} + self._times = {} + if max_paths: + self._q = lru_cache(max_paths + 1)(lambda key: self._cache.pop(key, None)) + self.use_listings_cache = use_listings_cache + self.listings_expiry_time = listings_expiry_time + self.max_paths = max_paths + + def __getitem__(self, item): + if self.listings_expiry_time is not None: + if self._times.get(item, 0) - time.time() < -self.listings_expiry_time: + del self._cache[item] + if self.max_paths: + self._q(item) + return self._cache[item] # maybe raises KeyError + + def clear(self): + self._cache.clear() + + def __len__(self): + return len(self._cache) + + def __contains__(self, item): + try: + self[item] + return True + except KeyError: + return False + + def __setitem__(self, key, value): + if not self.use_listings_cache: + return + if self.max_paths: + self._q(key) + self._cache[key] = value + if self.listings_expiry_time is not None: + self._times[key] = time.time() + + def __delitem__(self, key): + del self._cache[key] + + def __iter__(self): + entries = list(self._cache) + + return (k for k in entries if k in self) + + def __reduce__(self): + return ( + DirCache, + (self.use_listings_cache, self.listings_expiry_time, self.max_paths), + ) diff --git a/.venv/lib/python3.11/site-packages/fsspec/exceptions.py b/.venv/lib/python3.11/site-packages/fsspec/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..ae8905475f02655f4fc5863931d99ca9da55db78 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/exceptions.py @@ -0,0 +1,18 @@ +""" +fsspec user-defined exception classes +""" + +import asyncio + + +class BlocksizeMismatchError(ValueError): + """ + Raised when a cached file is opened with a different blocksize than it was + written with + """ + + +class FSTimeoutError(asyncio.TimeoutError): + """ + Raised when a fsspec function timed out occurs + """ diff --git a/.venv/lib/python3.11/site-packages/fsspec/fuse.py b/.venv/lib/python3.11/site-packages/fsspec/fuse.py new file mode 100644 index 0000000000000000000000000000000000000000..566d520fce3e94e3bbaee48c3c6acc9f1db315a8 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/fuse.py @@ -0,0 +1,324 @@ +import argparse +import logging +import os +import stat +import threading +import time +from errno import EIO, ENOENT + +from fuse import FUSE, FuseOSError, LoggingMixIn, Operations + +from fsspec import __version__ +from fsspec.core import url_to_fs + +logger = logging.getLogger("fsspec.fuse") + + +class FUSEr(Operations): + def __init__(self, fs, path, ready_file=False): + self.fs = fs + self.cache = {} + self.root = path.rstrip("/") + "/" + self.counter = 0 + logger.info("Starting FUSE at %s", path) + self._ready_file = ready_file + + def getattr(self, path, fh=None): + logger.debug("getattr %s", path) + if self._ready_file and path in ["/.fuse_ready", ".fuse_ready"]: + return {"type": "file", "st_size": 5} + + path = "".join([self.root, path.lstrip("/")]).rstrip("/") + try: + info = self.fs.info(path) + except FileNotFoundError as exc: + raise FuseOSError(ENOENT) from exc + + data = {"st_uid": info.get("uid", 1000), "st_gid": info.get("gid", 1000)} + perm = info.get("mode", 0o777) + + if info["type"] != "file": + data["st_mode"] = stat.S_IFDIR | perm + data["st_size"] = 0 + data["st_blksize"] = 0 + else: + data["st_mode"] = stat.S_IFREG | perm + data["st_size"] = info["size"] + data["st_blksize"] = 5 * 2**20 + data["st_nlink"] = 1 + data["st_atime"] = info["atime"] if "atime" in info else time.time() + data["st_ctime"] = info["ctime"] if "ctime" in info else time.time() + data["st_mtime"] = info["mtime"] if "mtime" in info else time.time() + return data + + def readdir(self, path, fh): + logger.debug("readdir %s", path) + path = "".join([self.root, path.lstrip("/")]) + files = self.fs.ls(path, False) + files = [os.path.basename(f.rstrip("/")) for f in files] + return [".", ".."] + files + + def mkdir(self, path, mode): + path = "".join([self.root, path.lstrip("/")]) + self.fs.mkdir(path) + return 0 + + def rmdir(self, path): + path = "".join([self.root, path.lstrip("/")]) + self.fs.rmdir(path) + return 0 + + def read(self, path, size, offset, fh): + logger.debug("read %s", (path, size, offset)) + if self._ready_file and path in ["/.fuse_ready", ".fuse_ready"]: + # status indicator + return b"ready" + + f = self.cache[fh] + f.seek(offset) + out = f.read(size) + return out + + def write(self, path, data, offset, fh): + logger.debug("write %s", (path, offset)) + f = self.cache[fh] + f.seek(offset) + f.write(data) + return len(data) + + def create(self, path, flags, fi=None): + logger.debug("create %s", (path, flags)) + fn = "".join([self.root, path.lstrip("/")]) + self.fs.touch(fn) # OS will want to get attributes immediately + f = self.fs.open(fn, "wb") + self.cache[self.counter] = f + self.counter += 1 + return self.counter - 1 + + def open(self, path, flags): + logger.debug("open %s", (path, flags)) + fn = "".join([self.root, path.lstrip("/")]) + if flags % 2 == 0: + # read + mode = "rb" + else: + # write/create + mode = "wb" + self.cache[self.counter] = self.fs.open(fn, mode) + self.counter += 1 + return self.counter - 1 + + def truncate(self, path, length, fh=None): + fn = "".join([self.root, path.lstrip("/")]) + if length != 0: + raise NotImplementedError + # maybe should be no-op since open with write sets size to zero anyway + self.fs.touch(fn) + + def unlink(self, path): + fn = "".join([self.root, path.lstrip("/")]) + try: + self.fs.rm(fn, False) + except (OSError, FileNotFoundError) as exc: + raise FuseOSError(EIO) from exc + + def release(self, path, fh): + try: + if fh in self.cache: + f = self.cache[fh] + f.close() + self.cache.pop(fh) + except Exception as e: + print(e) + return 0 + + def chmod(self, path, mode): + if hasattr(self.fs, "chmod"): + path = "".join([self.root, path.lstrip("/")]) + return self.fs.chmod(path, mode) + raise NotImplementedError + + +def run( + fs, + path, + mount_point, + foreground=True, + threads=False, + ready_file=False, + ops_class=FUSEr, +): + """Mount stuff in a local directory + + This uses fusepy to make it appear as if a given path on an fsspec + instance is in fact resident within the local file-system. + + This requires that fusepy by installed, and that FUSE be available on + the system (typically requiring a package to be installed with + apt, yum, brew, etc.). + + Parameters + ---------- + fs: file-system instance + From one of the compatible implementations + path: str + Location on that file-system to regard as the root directory to + mount. Note that you typically should include the terminating "/" + character. + mount_point: str + An empty directory on the local file-system where the contents of + the remote path will appear. + foreground: bool + Whether or not calling this function will block. Operation will + typically be more stable if True. + threads: bool + Whether or not to create threads when responding to file operations + within the mounter directory. Operation will typically be more + stable if False. + ready_file: bool + Whether the FUSE process is ready. The ``.fuse_ready`` file will + exist in the ``mount_point`` directory if True. Debugging purpose. + ops_class: FUSEr or Subclass of FUSEr + To override the default behavior of FUSEr. For Example, logging + to file. + + """ + func = lambda: FUSE( + ops_class(fs, path, ready_file=ready_file), + mount_point, + nothreads=not threads, + foreground=foreground, + ) + if not foreground: + th = threading.Thread(target=func) + th.daemon = True + th.start() + return th + else: # pragma: no cover + try: + func() + except KeyboardInterrupt: + pass + + +def main(args): + """Mount filesystem from chained URL to MOUNT_POINT. + + Examples: + + python3 -m fsspec.fuse memory /usr/share /tmp/mem + + python3 -m fsspec.fuse local /tmp/source /tmp/local \\ + -l /tmp/fsspecfuse.log + + You can also mount chained-URLs and use special settings: + + python3 -m fsspec.fuse 'filecache::zip::file://data.zip' \\ + / /tmp/zip \\ + -o 'filecache-cache_storage=/tmp/simplecache' + + You can specify the type of the setting by using `[int]` or `[bool]`, + (`true`, `yes`, `1` represents the Boolean value `True`): + + python3 -m fsspec.fuse 'simplecache::ftp://ftp1.at.proftpd.org' \\ + /historic/packages/RPMS /tmp/ftp \\ + -o 'simplecache-cache_storage=/tmp/simplecache' \\ + -o 'simplecache-check_files=false[bool]' \\ + -o 'ftp-listings_expiry_time=60[int]' \\ + -o 'ftp-username=anonymous' \\ + -o 'ftp-password=xieyanbo' + """ + + class RawDescriptionArgumentParser(argparse.ArgumentParser): + def format_help(self): + usage = super().format_help() + parts = usage.split("\n\n") + parts[1] = self.description.rstrip() + return "\n\n".join(parts) + + parser = RawDescriptionArgumentParser(prog="fsspec.fuse", description=main.__doc__) + parser.add_argument("--version", action="version", version=__version__) + parser.add_argument("url", type=str, help="fs url") + parser.add_argument("source_path", type=str, help="source directory in fs") + parser.add_argument("mount_point", type=str, help="local directory") + parser.add_argument( + "-o", + "--option", + action="append", + help="Any options of protocol included in the chained URL", + ) + parser.add_argument( + "-l", "--log-file", type=str, help="Logging FUSE debug info (Default: '')" + ) + parser.add_argument( + "-f", + "--foreground", + action="store_false", + help="Running in foreground or not (Default: False)", + ) + parser.add_argument( + "-t", + "--threads", + action="store_false", + help="Running with threads support (Default: False)", + ) + parser.add_argument( + "-r", + "--ready-file", + action="store_false", + help="The `.fuse_ready` file will exist after FUSE is ready. " + "(Debugging purpose, Default: False)", + ) + args = parser.parse_args(args) + + kwargs = {} + for item in args.option or []: + key, sep, value = item.partition("=") + if not sep: + parser.error(message=f"Wrong option: {item!r}") + val = value.lower() + if val.endswith("[int]"): + value = int(value[: -len("[int]")]) + elif val.endswith("[bool]"): + value = val[: -len("[bool]")] in ["1", "yes", "true"] + + if "-" in key: + fs_name, setting_name = key.split("-", 1) + if fs_name in kwargs: + kwargs[fs_name][setting_name] = value + else: + kwargs[fs_name] = {setting_name: value} + else: + kwargs[key] = value + + if args.log_file: + logging.basicConfig( + level=logging.DEBUG, + filename=args.log_file, + format="%(asctime)s %(message)s", + ) + + class LoggingFUSEr(FUSEr, LoggingMixIn): + pass + + fuser = LoggingFUSEr + else: + fuser = FUSEr + + fs, url_path = url_to_fs(args.url, **kwargs) + logger.debug("Mounting %s to %s", url_path, str(args.mount_point)) + run( + fs, + args.source_path, + args.mount_point, + foreground=args.foreground, + threads=args.threads, + ready_file=args.ready_file, + ops_class=fuser, + ) + + +if __name__ == "__main__": + import sys + + main(sys.argv[1:]) diff --git a/.venv/lib/python3.11/site-packages/fsspec/generic.py b/.venv/lib/python3.11/site-packages/fsspec/generic.py new file mode 100644 index 0000000000000000000000000000000000000000..0a641b0e2bcf70729a44064319eecb3647450379 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/generic.py @@ -0,0 +1,396 @@ +from __future__ import annotations + +import inspect +import logging +import os +import shutil +import uuid + +from .asyn import AsyncFileSystem, _run_coros_in_chunks, sync_wrapper +from .callbacks import DEFAULT_CALLBACK +from .core import filesystem, get_filesystem_class, split_protocol, url_to_fs + +_generic_fs = {} +logger = logging.getLogger("fsspec.generic") + + +def set_generic_fs(protocol, **storage_options): + """Populate the dict used for method=="generic" lookups""" + _generic_fs[protocol] = filesystem(protocol, **storage_options) + + +def _resolve_fs(url, method, protocol=None, storage_options=None): + """Pick instance of backend FS""" + url = url[0] if isinstance(url, (list, tuple)) else url + protocol = protocol or split_protocol(url)[0] + storage_options = storage_options or {} + if method == "default": + return filesystem(protocol) + if method == "generic": + return _generic_fs[protocol] + if method == "current": + cls = get_filesystem_class(protocol) + return cls.current() + if method == "options": + fs, _ = url_to_fs(url, **storage_options.get(protocol, {})) + return fs + raise ValueError(f"Unknown FS resolution method: {method}") + + +def rsync( + source, + destination, + delete_missing=False, + source_field="size", + dest_field="size", + update_cond="different", + inst_kwargs=None, + fs=None, + **kwargs, +): + """Sync files between two directory trees + + (experimental) + + Parameters + ---------- + source: str + Root of the directory tree to take files from. This must be a directory, but + do not include any terminating "/" character + destination: str + Root path to copy into. The contents of this location should be + identical to the contents of ``source`` when done. This will be made a + directory, and the terminal "/" should not be included. + delete_missing: bool + If there are paths in the destination that don't exist in the + source and this is True, delete them. Otherwise, leave them alone. + source_field: str | callable + If ``update_field`` is "different", this is the key in the info + of source files to consider for difference. Maybe a function of the + info dict. + dest_field: str | callable + If ``update_field`` is "different", this is the key in the info + of destination files to consider for difference. May be a function of + the info dict. + update_cond: "different"|"always"|"never" + If "always", every file is copied, regardless of whether it exists in + the destination. If "never", files that exist in the destination are + not copied again. If "different" (default), only copy if the info + fields given by ``source_field`` and ``dest_field`` (usually "size") + are different. Other comparisons may be added in the future. + inst_kwargs: dict|None + If ``fs`` is None, use this set of keyword arguments to make a + GenericFileSystem instance + fs: GenericFileSystem|None + Instance to use if explicitly given. The instance defines how to + to make downstream file system instances from paths. + + Returns + ------- + dict of the copy operations that were performed, {source: destination} + """ + fs = fs or GenericFileSystem(**(inst_kwargs or {})) + source = fs._strip_protocol(source) + destination = fs._strip_protocol(destination) + allfiles = fs.find(source, withdirs=True, detail=True) + if not fs.isdir(source): + raise ValueError("Can only rsync on a directory") + otherfiles = fs.find(destination, withdirs=True, detail=True) + dirs = [ + a + for a, v in allfiles.items() + if v["type"] == "directory" and a.replace(source, destination) not in otherfiles + ] + logger.debug(f"{len(dirs)} directories to create") + if dirs: + fs.make_many_dirs( + [dirn.replace(source, destination) for dirn in dirs], exist_ok=True + ) + allfiles = {a: v for a, v in allfiles.items() if v["type"] == "file"} + logger.debug(f"{len(allfiles)} files to consider for copy") + to_delete = [ + o + for o, v in otherfiles.items() + if o.replace(destination, source) not in allfiles and v["type"] == "file" + ] + for k, v in allfiles.copy().items(): + otherfile = k.replace(source, destination) + if otherfile in otherfiles: + if update_cond == "always": + allfiles[k] = otherfile + elif update_cond == "never": + allfiles.pop(k) + elif update_cond == "different": + inf1 = source_field(v) if callable(source_field) else v[source_field] + v2 = otherfiles[otherfile] + inf2 = dest_field(v2) if callable(dest_field) else v2[dest_field] + if inf1 != inf2: + # details mismatch, make copy + allfiles[k] = otherfile + else: + # details match, don't copy + allfiles.pop(k) + else: + # file not in target yet + allfiles[k] = otherfile + logger.debug(f"{len(allfiles)} files to copy") + if allfiles: + source_files, target_files = zip(*allfiles.items()) + fs.cp(source_files, target_files, **kwargs) + logger.debug(f"{len(to_delete)} files to delete") + if delete_missing and to_delete: + fs.rm(to_delete) + return allfiles + + +class GenericFileSystem(AsyncFileSystem): + """Wrapper over all other FS types + + <experimental!> + + This implementation is a single unified interface to be able to run FS operations + over generic URLs, and dispatch to the specific implementations using the URL + protocol prefix. + + Note: instances of this FS are always async, even if you never use it with any async + backend. + """ + + protocol = "generic" # there is no real reason to ever use a protocol with this FS + + def __init__(self, default_method="default", storage_options=None, **kwargs): + """ + + Parameters + ---------- + default_method: str (optional) + Defines how to configure backend FS instances. Options are: + - "default": instantiate like FSClass(), with no + extra arguments; this is the default instance of that FS, and can be + configured via the config system + - "generic": takes instances from the `_generic_fs` dict in this module, + which you must populate before use. Keys are by protocol + - "options": expects storage_options, a dict mapping protocol to + kwargs to use when constructing the filesystem + - "current": takes the most recently instantiated version of each FS + """ + self.method = default_method + self.st_opts = storage_options + super().__init__(**kwargs) + + def _parent(self, path): + fs = _resolve_fs(path, self.method, storage_options=self.st_opts) + return fs.unstrip_protocol(fs._parent(path)) + + def _strip_protocol(self, path): + # normalization only + fs = _resolve_fs(path, self.method, storage_options=self.st_opts) + return fs.unstrip_protocol(fs._strip_protocol(path)) + + async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): + fs = _resolve_fs(path, self.method, storage_options=self.st_opts) + if fs.async_impl: + out = await fs._find( + path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs + ) + else: + out = fs.find( + path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs + ) + result = {} + for k, v in out.items(): + v = v.copy() # don't corrupt target FS dircache + name = fs.unstrip_protocol(k) + v["name"] = name + result[name] = v + if detail: + return result + return list(result) + + async def _info(self, url, **kwargs): + fs = _resolve_fs(url, self.method) + if fs.async_impl: + out = await fs._info(url, **kwargs) + else: + out = fs.info(url, **kwargs) + out = out.copy() # don't edit originals + out["name"] = fs.unstrip_protocol(out["name"]) + return out + + async def _ls( + self, + url, + detail=True, + **kwargs, + ): + fs = _resolve_fs(url, self.method) + if fs.async_impl: + out = await fs._ls(url, detail=True, **kwargs) + else: + out = fs.ls(url, detail=True, **kwargs) + out = [o.copy() for o in out] # don't edit originals + for o in out: + o["name"] = fs.unstrip_protocol(o["name"]) + if detail: + return out + else: + return [o["name"] for o in out] + + async def _cat_file( + self, + url, + **kwargs, + ): + fs = _resolve_fs(url, self.method) + if fs.async_impl: + return await fs._cat_file(url, **kwargs) + else: + return fs.cat_file(url, **kwargs) + + async def _pipe_file( + self, + path, + value, + **kwargs, + ): + fs = _resolve_fs(path, self.method, storage_options=self.st_opts) + if fs.async_impl: + return await fs._pipe_file(path, value, **kwargs) + else: + return fs.pipe_file(path, value, **kwargs) + + async def _rm(self, url, **kwargs): + urls = url + if isinstance(urls, str): + urls = [urls] + fs = _resolve_fs(urls[0], self.method) + if fs.async_impl: + await fs._rm(urls, **kwargs) + else: + fs.rm(url, **kwargs) + + async def _makedirs(self, path, exist_ok=False): + logger.debug("Make dir %s", path) + fs = _resolve_fs(path, self.method, storage_options=self.st_opts) + if fs.async_impl: + await fs._makedirs(path, exist_ok=exist_ok) + else: + fs.makedirs(path, exist_ok=exist_ok) + + def rsync(self, source, destination, **kwargs): + """Sync files between two directory trees + + See `func:rsync` for more details. + """ + rsync(source, destination, fs=self, **kwargs) + + async def _cp_file( + self, + url, + url2, + blocksize=2**20, + callback=DEFAULT_CALLBACK, + tempdir: str | None = None, + **kwargs, + ): + fs = _resolve_fs(url, self.method) + fs2 = _resolve_fs(url2, self.method) + if fs is fs2: + # pure remote + if fs.async_impl: + return await fs._copy(url, url2, **kwargs) + else: + return fs.copy(url, url2, **kwargs) + await copy_file_op(fs, [url], fs2, [url2], tempdir, 1, on_error="raise") + + async def _make_many_dirs(self, urls, exist_ok=True): + fs = _resolve_fs(urls[0], self.method) + if fs.async_impl: + coros = [fs._makedirs(u, exist_ok=exist_ok) for u in urls] + await _run_coros_in_chunks(coros) + else: + for u in urls: + fs.makedirs(u, exist_ok=exist_ok) + + make_many_dirs = sync_wrapper(_make_many_dirs) + + async def _copy( + self, + path1: list[str], + path2: list[str], + recursive: bool = False, + on_error: str = "ignore", + maxdepth: int | None = None, + batch_size: int | None = None, + tempdir: str | None = None, + **kwargs, + ): + # TODO: special case for one FS being local, which can use get/put + # TODO: special case for one being memFS, which can use cat/pipe + if recursive: + raise NotImplementedError("Please use fsspec.generic.rsync") + path1 = [path1] if isinstance(path1, str) else path1 + path2 = [path2] if isinstance(path2, str) else path2 + + fs = _resolve_fs(path1, self.method) + fs2 = _resolve_fs(path2, self.method) + + if fs is fs2: + if fs.async_impl: + return await fs._copy(path1, path2, **kwargs) + else: + return fs.copy(path1, path2, **kwargs) + + await copy_file_op( + fs, path1, fs2, path2, tempdir, batch_size, on_error=on_error + ) + + +async def copy_file_op( + fs1, url1, fs2, url2, tempdir=None, batch_size=20, on_error="ignore" +): + import tempfile + + tempdir = tempdir or tempfile.mkdtemp() + try: + coros = [ + _copy_file_op( + fs1, + u1, + fs2, + u2, + os.path.join(tempdir, uuid.uuid4().hex), + ) + for u1, u2 in zip(url1, url2) + ] + out = await _run_coros_in_chunks( + coros, batch_size=batch_size, return_exceptions=True + ) + finally: + shutil.rmtree(tempdir) + if on_error == "return": + return out + elif on_error == "raise": + for o in out: + if isinstance(o, Exception): + raise o + + +async def _copy_file_op(fs1, url1, fs2, url2, local, on_error="ignore"): + if fs1.async_impl: + await fs1._get_file(url1, local) + else: + fs1.get_file(url1, local) + if fs2.async_impl: + await fs2._put_file(local, url2) + else: + fs2.put_file(local, url2) + os.unlink(local) + logger.debug("Copy %s -> %s; done", url1, url2) + + +async def maybe_await(cor): + if inspect.iscoroutine(cor): + return await cor + else: + return cor diff --git a/.venv/lib/python3.11/site-packages/fsspec/gui.py b/.venv/lib/python3.11/site-packages/fsspec/gui.py new file mode 100644 index 0000000000000000000000000000000000000000..9d914c8beb6cabb2c2700eb8eee31028559be2bd --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/gui.py @@ -0,0 +1,417 @@ +import ast +import contextlib +import logging +import os +import re +from collections.abc import Sequence +from typing import ClassVar + +import panel as pn + +from .core import OpenFile, get_filesystem_class, split_protocol +from .registry import known_implementations + +pn.extension() +logger = logging.getLogger("fsspec.gui") + + +class SigSlot: + """Signal-slot mixin, for Panel event passing + + Include this class in a widget manager's superclasses to be able to + register events and callbacks on Panel widgets managed by that class. + + The method ``_register`` should be called as widgets are added, and external + code should call ``connect`` to associate callbacks. + + By default, all signals emit a DEBUG logging statement. + """ + + # names of signals that this class may emit each of which must be + # set by _register for any new instance + signals: ClassVar[Sequence[str]] = [] + # names of actions that this class may respond to + slots: ClassVar[Sequence[str]] = [] + + # each of which must be a method name + + def __init__(self): + self._ignoring_events = False + self._sigs = {} + self._map = {} + self._setup() + + def _setup(self): + """Create GUI elements and register signals""" + self.panel = pn.pane.PaneBase() + # no signals to set up in the base class + + def _register( + self, widget, name, thing="value", log_level=logging.DEBUG, auto=False + ): + """Watch the given attribute of a widget and assign it a named event + + This is normally called at the time a widget is instantiated, in the + class which owns it. + + Parameters + ---------- + widget : pn.layout.Panel or None + Widget to watch. If None, an anonymous signal not associated with + any widget. + name : str + Name of this event + thing : str + Attribute of the given widget to watch + log_level : int + When the signal is triggered, a logging event of the given level + will be fired in the dfviz logger. + auto : bool + If True, automatically connects with a method in this class of the + same name. + """ + if name not in self.signals: + raise ValueError(f"Attempt to assign an undeclared signal: {name}") + self._sigs[name] = { + "widget": widget, + "callbacks": [], + "thing": thing, + "log": log_level, + } + wn = "-".join( + [ + getattr(widget, "name", str(widget)) if widget is not None else "none", + thing, + ] + ) + self._map[wn] = name + if widget is not None: + widget.param.watch(self._signal, thing, onlychanged=True) + if auto and hasattr(self, name): + self.connect(name, getattr(self, name)) + + def _repr_mimebundle_(self, *args, **kwargs): + """Display in a notebook or a server""" + try: + return self.panel._repr_mimebundle_(*args, **kwargs) + except (ValueError, AttributeError) as exc: + raise NotImplementedError( + "Panel does not seem to be set up properly" + ) from exc + + def connect(self, signal, slot): + """Associate call back with given event + + The callback must be a function which takes the "new" value of the + watched attribute as the only parameter. If the callback return False, + this cancels any further processing of the given event. + + Alternatively, the callback can be a string, in which case it means + emitting the correspondingly-named event (i.e., connect to self) + """ + self._sigs[signal]["callbacks"].append(slot) + + def _signal(self, event): + """This is called by a an action on a widget + + Within an self.ignore_events context, nothing happens. + + Tests can execute this method by directly changing the values of + widget components. + """ + if not self._ignoring_events: + wn = "-".join([event.obj.name, event.name]) + if wn in self._map and self._map[wn] in self._sigs: + self._emit(self._map[wn], event.new) + + @contextlib.contextmanager + def ignore_events(self): + """Temporarily turn off events processing in this instance + + (does not propagate to children) + """ + self._ignoring_events = True + try: + yield + finally: + self._ignoring_events = False + + def _emit(self, sig, value=None): + """An event happened, call its callbacks + + This method can be used in tests to simulate message passing without + directly changing visual elements. + + Calling of callbacks will halt whenever one returns False. + """ + logger.log(self._sigs[sig]["log"], f"{sig}: {value}") + for callback in self._sigs[sig]["callbacks"]: + if isinstance(callback, str): + self._emit(callback) + else: + try: + # running callbacks should not break the interface + ret = callback(value) + if ret is False: + break + except Exception as e: + logger.exception( + "Exception (%s) while executing callback for signal: %s", + e, + sig, + ) + + def show(self, threads=False): + """Open a new browser tab and display this instance's interface""" + self.panel.show(threads=threads, verbose=False) + return self + + +class SingleSelect(SigSlot): + """A multiselect which only allows you to select one item for an event""" + + signals = ["_selected", "selected"] # the first is internal + slots = ["set_options", "set_selection", "add", "clear", "select"] + + def __init__(self, **kwargs): + self.kwargs = kwargs + super().__init__() + + def _setup(self): + self.panel = pn.widgets.MultiSelect(**self.kwargs) + self._register(self.panel, "_selected", "value") + self._register(None, "selected") + self.connect("_selected", self.select_one) + + def _signal(self, *args, **kwargs): + super()._signal(*args, **kwargs) + + def select_one(self, *_): + with self.ignore_events(): + val = [self.panel.value[-1]] if self.panel.value else [] + self.panel.value = val + self._emit("selected", self.panel.value) + + def set_options(self, options): + self.panel.options = options + + def clear(self): + self.panel.options = [] + + @property + def value(self): + return self.panel.value + + def set_selection(self, selection): + self.panel.value = [selection] + + +class FileSelector(SigSlot): + """Panel-based graphical file selector widget + + Instances of this widget are interactive and can be displayed in jupyter by having + them as the output of a cell, or in a separate browser tab using ``.show()``. + """ + + signals = [ + "protocol_changed", + "selection_changed", + "directory_entered", + "home_clicked", + "up_clicked", + "go_clicked", + "filters_changed", + ] + slots = ["set_filters", "go_home"] + + def __init__(self, url=None, filters=None, ignore=None, kwargs=None): + """ + + Parameters + ---------- + url : str (optional) + Initial value of the URL to populate the dialog; should include protocol + filters : list(str) (optional) + File endings to include in the listings. If not included, all files are + allowed. Does not affect directories. + If given, the endings will appear as checkboxes in the interface + ignore : list(str) (optional) + Regex(s) of file basename patterns to ignore, e.g., "\\." for typical + hidden files on posix + kwargs : dict (optional) + To pass to file system instance + """ + if url: + self.init_protocol, url = split_protocol(url) + else: + self.init_protocol, url = "file", os.getcwd() + self.init_url = url + self.init_kwargs = (kwargs if isinstance(kwargs, str) else str(kwargs)) or "{}" + self.filters = filters + self.ignore = [re.compile(i) for i in ignore or []] + self._fs = None + super().__init__() + + def _setup(self): + self.url = pn.widgets.TextInput( + name="url", + value=self.init_url, + align="end", + sizing_mode="stretch_width", + width_policy="max", + ) + self.protocol = pn.widgets.Select( + options=sorted(known_implementations), + value=self.init_protocol, + name="protocol", + align="center", + ) + self.kwargs = pn.widgets.TextInput( + name="kwargs", value=self.init_kwargs, align="center" + ) + self.go = pn.widgets.Button(name="⇨", align="end", width=45) + self.main = SingleSelect(size=10) + self.home = pn.widgets.Button(name="🏠", width=40, height=30, align="end") + self.up = pn.widgets.Button(name="‹", width=30, height=30, align="end") + + self._register(self.protocol, "protocol_changed", auto=True) + self._register(self.go, "go_clicked", "clicks", auto=True) + self._register(self.up, "up_clicked", "clicks", auto=True) + self._register(self.home, "home_clicked", "clicks", auto=True) + self._register(None, "selection_changed") + self.main.connect("selected", self.selection_changed) + self._register(None, "directory_entered") + self.prev_protocol = self.protocol.value + self.prev_kwargs = self.storage_options + + self.filter_sel = pn.widgets.CheckBoxGroup( + value=[], options=[], inline=False, align="end", width_policy="min" + ) + self._register(self.filter_sel, "filters_changed", auto=True) + + self.panel = pn.Column( + pn.Row(self.protocol, self.kwargs), + pn.Row(self.home, self.up, self.url, self.go, self.filter_sel), + self.main.panel, + ) + self.set_filters(self.filters) + self.go_clicked() + + def set_filters(self, filters=None): + self.filters = filters + if filters: + self.filter_sel.options = filters + self.filter_sel.value = filters + else: + self.filter_sel.options = [] + self.filter_sel.value = [] + + @property + def storage_options(self): + """Value of the kwargs box as a dictionary""" + return ast.literal_eval(self.kwargs.value) or {} + + @property + def fs(self): + """Current filesystem instance""" + if self._fs is None: + cls = get_filesystem_class(self.protocol.value) + self._fs = cls(**self.storage_options) + return self._fs + + @property + def urlpath(self): + """URL of currently selected item""" + return ( + (f"{self.protocol.value}://{self.main.value[0]}") + if self.main.value + else None + ) + + def open_file(self, mode="rb", compression=None, encoding=None): + """Create OpenFile instance for the currently selected item + + For example, in a notebook you might do something like + + .. code-block:: + + [ ]: sel = FileSelector(); sel + + # user selects their file + + [ ]: with sel.open_file('rb') as f: + ... out = f.read() + + Parameters + ---------- + mode: str (optional) + Open mode for the file. + compression: str (optional) + The interact with the file as compressed. Set to 'infer' to guess + compression from the file ending + encoding: str (optional) + If using text mode, use this encoding; defaults to UTF8. + """ + if self.urlpath is None: + raise ValueError("No file selected") + return OpenFile(self.fs, self.urlpath, mode, compression, encoding) + + def filters_changed(self, values): + self.filters = values + self.go_clicked() + + def selection_changed(self, *_): + if self.urlpath is None: + return + if self.fs.isdir(self.urlpath): + self.url.value = self.fs._strip_protocol(self.urlpath) + self.go_clicked() + + def go_clicked(self, *_): + if ( + self.prev_protocol != self.protocol.value + or self.prev_kwargs != self.storage_options + ): + self._fs = None # causes fs to be recreated + self.prev_protocol = self.protocol.value + self.prev_kwargs = self.storage_options + listing = sorted( + self.fs.ls(self.url.value, detail=True), key=lambda x: x["name"] + ) + listing = [ + l + for l in listing + if not any(i.match(l["name"].rsplit("/", 1)[-1]) for i in self.ignore) + ] + folders = { + "📁 " + o["name"].rsplit("/", 1)[-1]: o["name"] + for o in listing + if o["type"] == "directory" + } + files = { + "📄 " + o["name"].rsplit("/", 1)[-1]: o["name"] + for o in listing + if o["type"] == "file" + } + if self.filters: + files = { + k: v + for k, v in files.items() + if any(v.endswith(ext) for ext in self.filters) + } + self.main.set_options(dict(**folders, **files)) + + def protocol_changed(self, *_): + self._fs = None + self.main.options = [] + self.url.value = "" + + def home_clicked(self, *_): + self.protocol.value = self.init_protocol + self.kwargs.value = self.init_kwargs + self.url.value = self.init_url + self.go_clicked() + + def up_clicked(self, *_): + self.url.value = self.fs._parent(self.url.value) + self.go_clicked() diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/__init__.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/arrow.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/arrow.py new file mode 100644 index 0000000000000000000000000000000000000000..227d50930763f56bc8c01556249eb1306347350b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/arrow.py @@ -0,0 +1,312 @@ +import errno +import io +import os +import secrets +import shutil +from contextlib import suppress +from functools import cached_property, wraps +from urllib.parse import parse_qs + +from fsspec.spec import AbstractFileSystem +from fsspec.utils import ( + get_package_version_without_import, + infer_storage_options, + mirror_from, + tokenize, +) + + +def wrap_exceptions(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except OSError as exception: + if not exception.args: + raise + + message, *args = exception.args + if isinstance(message, str) and "does not exist" in message: + raise FileNotFoundError(errno.ENOENT, message) from exception + else: + raise + + return wrapper + + +PYARROW_VERSION = None + + +class ArrowFSWrapper(AbstractFileSystem): + """FSSpec-compatible wrapper of pyarrow.fs.FileSystem. + + Parameters + ---------- + fs : pyarrow.fs.FileSystem + + """ + + root_marker = "/" + + def __init__(self, fs, **kwargs): + global PYARROW_VERSION + PYARROW_VERSION = get_package_version_without_import("pyarrow") + self.fs = fs + super().__init__(**kwargs) + + @property + def protocol(self): + return self.fs.type_name + + @cached_property + def fsid(self): + return "hdfs_" + tokenize(self.fs.host, self.fs.port) + + @classmethod + def _strip_protocol(cls, path): + ops = infer_storage_options(path) + path = ops["path"] + if path.startswith("//"): + # special case for "hdfs://path" (without the triple slash) + path = path[1:] + return path + + def ls(self, path, detail=False, **kwargs): + path = self._strip_protocol(path) + from pyarrow.fs import FileSelector + + try: + entries = [ + self._make_entry(entry) + for entry in self.fs.get_file_info(FileSelector(path)) + ] + except (FileNotFoundError, NotADirectoryError): + entries = [self.info(path, **kwargs)] + if detail: + return entries + else: + return [entry["name"] for entry in entries] + + def info(self, path, **kwargs): + path = self._strip_protocol(path) + [info] = self.fs.get_file_info([path]) + return self._make_entry(info) + + def exists(self, path): + path = self._strip_protocol(path) + try: + self.info(path) + except FileNotFoundError: + return False + else: + return True + + def _make_entry(self, info): + from pyarrow.fs import FileType + + if info.type is FileType.Directory: + kind = "directory" + elif info.type is FileType.File: + kind = "file" + elif info.type is FileType.NotFound: + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), info.path) + else: + kind = "other" + + return { + "name": info.path, + "size": info.size, + "type": kind, + "mtime": info.mtime, + } + + @wrap_exceptions + def cp_file(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1).rstrip("/") + path2 = self._strip_protocol(path2).rstrip("/") + + with self._open(path1, "rb") as lstream: + tmp_fname = f"{path2}.tmp.{secrets.token_hex(6)}" + try: + with self.open(tmp_fname, "wb") as rstream: + shutil.copyfileobj(lstream, rstream) + self.fs.move(tmp_fname, path2) + except BaseException: + with suppress(FileNotFoundError): + self.fs.delete_file(tmp_fname) + raise + + @wrap_exceptions + def mv(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1).rstrip("/") + path2 = self._strip_protocol(path2).rstrip("/") + self.fs.move(path1, path2) + + @wrap_exceptions + def rm_file(self, path): + path = self._strip_protocol(path) + self.fs.delete_file(path) + + @wrap_exceptions + def rm(self, path, recursive=False, maxdepth=None): + path = self._strip_protocol(path).rstrip("/") + if self.isdir(path): + if recursive: + self.fs.delete_dir(path) + else: + raise ValueError("Can't delete directories without recursive=False") + else: + self.fs.delete_file(path) + + @wrap_exceptions + def _open(self, path, mode="rb", block_size=None, seekable=True, **kwargs): + if mode == "rb": + if seekable: + method = self.fs.open_input_file + else: + method = self.fs.open_input_stream + elif mode == "wb": + method = self.fs.open_output_stream + elif mode == "ab": + method = self.fs.open_append_stream + else: + raise ValueError(f"unsupported mode for Arrow filesystem: {mode!r}") + + _kwargs = {} + if mode != "rb" or not seekable: + if int(PYARROW_VERSION.split(".")[0]) >= 4: + # disable compression auto-detection + _kwargs["compression"] = None + stream = method(path, **_kwargs) + + return ArrowFile(self, stream, path, mode, block_size, **kwargs) + + @wrap_exceptions + def mkdir(self, path, create_parents=True, **kwargs): + path = self._strip_protocol(path) + if create_parents: + self.makedirs(path, exist_ok=True) + else: + self.fs.create_dir(path, recursive=False) + + @wrap_exceptions + def makedirs(self, path, exist_ok=False): + path = self._strip_protocol(path) + self.fs.create_dir(path, recursive=True) + + @wrap_exceptions + def rmdir(self, path): + path = self._strip_protocol(path) + self.fs.delete_dir(path) + + @wrap_exceptions + def modified(self, path): + path = self._strip_protocol(path) + return self.fs.get_file_info(path).mtime + + def cat_file(self, path, start=None, end=None, **kwargs): + kwargs.setdefault("seekable", start not in [None, 0]) + return super().cat_file(path, start=None, end=None, **kwargs) + + def get_file(self, rpath, lpath, **kwargs): + kwargs.setdefault("seekable", False) + super().get_file(rpath, lpath, **kwargs) + + +@mirror_from( + "stream", + [ + "read", + "seek", + "tell", + "write", + "readable", + "writable", + "close", + "seekable", + ], +) +class ArrowFile(io.IOBase): + def __init__(self, fs, stream, path, mode, block_size=None, **kwargs): + self.path = path + self.mode = mode + + self.fs = fs + self.stream = stream + + self.blocksize = self.block_size = block_size + self.kwargs = kwargs + + def __enter__(self): + return self + + @property + def size(self): + if self.stream.seekable(): + return self.stream.size() + return None + + def __exit__(self, *args): + return self.close() + + +class HadoopFileSystem(ArrowFSWrapper): + """A wrapper on top of the pyarrow.fs.HadoopFileSystem + to connect it's interface with fsspec""" + + protocol = "hdfs" + + def __init__( + self, + host="default", + port=0, + user=None, + kerb_ticket=None, + replication=3, + extra_conf=None, + **kwargs, + ): + """ + + Parameters + ---------- + host: str + Hostname, IP or "default" to try to read from Hadoop config + port: int + Port to connect on, or default from Hadoop config if 0 + user: str or None + If given, connect as this username + kerb_ticket: str or None + If given, use this ticket for authentication + replication: int + set replication factor of file for write operations. default value is 3. + extra_conf: None or dict + Passed on to HadoopFileSystem + """ + from pyarrow.fs import HadoopFileSystem + + fs = HadoopFileSystem( + host=host, + port=port, + user=user, + kerb_ticket=kerb_ticket, + replication=replication, + extra_conf=extra_conf, + ) + super().__init__(fs=fs, **kwargs) + + @staticmethod + def _get_kwargs_from_urls(path): + ops = infer_storage_options(path) + out = {} + if ops.get("host", None): + out["host"] = ops["host"] + if ops.get("username", None): + out["user"] = ops["username"] + if ops.get("port", None): + out["port"] = ops["port"] + if ops.get("url_query", None): + queries = parse_qs(ops["url_query"]) + if queries.get("replication", None): + out["replication"] = int(queries["replication"][0]) + return out diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/asyn_wrapper.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/asyn_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..91db5eb48d00e36b46d9deb49504a7d2ad76d690 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/asyn_wrapper.py @@ -0,0 +1,124 @@ +import asyncio +import functools +import inspect + +import fsspec +from fsspec.asyn import AsyncFileSystem, running_async + +from .chained import ChainedFileSystem + + +def async_wrapper(func, obj=None, semaphore=None): + """ + Wraps a synchronous function to make it awaitable. + + Parameters + ---------- + func : callable + The synchronous function to wrap. + obj : object, optional + The instance to bind the function to, if applicable. + semaphore : asyncio.Semaphore, optional + A semaphore to limit concurrent calls. + + Returns + ------- + coroutine + An awaitable version of the function. + """ + + @functools.wraps(func) + async def wrapper(*args, **kwargs): + if semaphore: + async with semaphore: + return await asyncio.to_thread(func, *args, **kwargs) + return await asyncio.to_thread(func, *args, **kwargs) + + return wrapper + + +class AsyncFileSystemWrapper(AsyncFileSystem, ChainedFileSystem): + """ + A wrapper class to convert a synchronous filesystem into an asynchronous one. + + This class takes an existing synchronous filesystem implementation and wraps all + its methods to provide an asynchronous interface. + + Parameters + ---------- + sync_fs : AbstractFileSystem + The synchronous filesystem instance to wrap. + """ + + protocol = "asyncwrapper", "async_wrapper" + cachable = False + + def __init__( + self, + fs=None, + asynchronous=None, + target_protocol=None, + target_options=None, + semaphore=None, + max_concurrent_tasks=None, + **kwargs, + ): + if asynchronous is None: + asynchronous = running_async() + super().__init__(asynchronous=asynchronous, **kwargs) + if fs is not None: + self.sync_fs = fs + else: + self.sync_fs = fsspec.filesystem(target_protocol, **target_options) + self.protocol = self.sync_fs.protocol + self.semaphore = semaphore + self._wrap_all_sync_methods() + + @property + def fsid(self): + return f"async_{self.sync_fs.fsid}" + + def _wrap_all_sync_methods(self): + """ + Wrap all synchronous methods of the underlying filesystem with asynchronous versions. + """ + excluded_methods = {"open"} + for method_name in dir(self.sync_fs): + if method_name.startswith("_") or method_name in excluded_methods: + continue + + attr = inspect.getattr_static(self.sync_fs, method_name) + if isinstance(attr, property): + continue + + method = getattr(self.sync_fs, method_name) + if callable(method) and not inspect.iscoroutinefunction(method): + async_method = async_wrapper(method, obj=self, semaphore=self.semaphore) + setattr(self, f"_{method_name}", async_method) + + @classmethod + def wrap_class(cls, sync_fs_class): + """ + Create a new class that can be used to instantiate an AsyncFileSystemWrapper + with lazy instantiation of the underlying synchronous filesystem. + + Parameters + ---------- + sync_fs_class : type + The class of the synchronous filesystem to wrap. + + Returns + ------- + type + A new class that wraps the provided synchronous filesystem class. + """ + + class GeneratedAsyncFileSystemWrapper(cls): + def __init__(self, *args, **kwargs): + sync_fs = sync_fs_class(*args, **kwargs) + super().__init__(sync_fs) + + GeneratedAsyncFileSystemWrapper.__name__ = ( + f"Async{sync_fs_class.__name__}Wrapper" + ) + return GeneratedAsyncFileSystemWrapper diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/cache_mapper.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/cache_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..6e7c7d88afdddf12f77b26bb635bd8bf1e2bd7f1 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/cache_mapper.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import abc +import hashlib + +from fsspec.implementations.local import make_path_posix + + +class AbstractCacheMapper(abc.ABC): + """Abstract super-class for mappers from remote URLs to local cached + basenames. + """ + + @abc.abstractmethod + def __call__(self, path: str) -> str: ... + + def __eq__(self, other: object) -> bool: + # Identity only depends on class. When derived classes have attributes + # they will need to be included. + return isinstance(other, type(self)) + + def __hash__(self) -> int: + # Identity only depends on class. When derived classes have attributes + # they will need to be included. + return hash(type(self)) + + +class BasenameCacheMapper(AbstractCacheMapper): + """Cache mapper that uses the basename of the remote URL and a fixed number + of directory levels above this. + + The default is zero directory levels, meaning different paths with the same + basename will have the same cached basename. + """ + + def __init__(self, directory_levels: int = 0): + if directory_levels < 0: + raise ValueError( + "BasenameCacheMapper requires zero or positive directory_levels" + ) + self.directory_levels = directory_levels + + # Separator for directories when encoded as strings. + self._separator = "_@_" + + def __call__(self, path: str) -> str: + path = make_path_posix(path) + prefix, *bits = path.rsplit("/", self.directory_levels + 1) + if bits: + return self._separator.join(bits) + else: + return prefix # No separator found, simple filename + + def __eq__(self, other: object) -> bool: + return super().__eq__(other) and self.directory_levels == other.directory_levels + + def __hash__(self) -> int: + return super().__hash__() ^ hash(self.directory_levels) + + +class HashCacheMapper(AbstractCacheMapper): + """Cache mapper that uses a hash of the remote URL.""" + + def __call__(self, path: str) -> str: + return hashlib.sha256(path.encode()).hexdigest() + + +def create_cache_mapper(same_names: bool) -> AbstractCacheMapper: + """Factory method to create cache mapper for backward compatibility with + ``CachingFileSystem`` constructor using ``same_names`` kwarg. + """ + if same_names: + return BasenameCacheMapper() + else: + return HashCacheMapper() diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/cache_metadata.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/cache_metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..2a48231cac9f984f3e41bc9f81a113ae28a1f5dd --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/cache_metadata.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import os +import time +from typing import TYPE_CHECKING + +from fsspec.utils import atomic_write + +try: + import ujson as json +except ImportError: + if not TYPE_CHECKING: + import json + +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Any, Literal, TypeAlias + + from .cached import CachingFileSystem + + Detail: TypeAlias = dict[str, Any] + + +class CacheMetadata: + """Cache metadata. + + All reading and writing of cache metadata is performed by this class, + accessing the cached files and blocks is not. + + Metadata is stored in a single file per storage directory in JSON format. + For backward compatibility. No longer supports pickle. + """ + + def __init__(self, storage: list[str]): + """ + + Parameters + ---------- + storage: list[str] + Directories containing cached files, must be at least one. Metadata + is stored in the last of these directories by convention. + """ + if not storage: + raise ValueError("CacheMetadata expects at least one storage location") + + self._storage = storage + self.cached_files: list[Detail] = [{}] + + def _load(self, fn: str) -> Detail: + """Low-level function to load metadata from specific file""" + with open(fn, "r") as f: + loaded = json.load(f) + for c in loaded.values(): + if isinstance(c.get("blocks"), list): + c["blocks"] = set(c["blocks"]) + return loaded + + def _save(self, metadata_to_save: Detail, fn: str) -> None: + """Low-level function to save metadata to specific file""" + with atomic_write(fn, mode="w") as f: + json.dump(metadata_to_save, f) + + def _scan_locations( + self, writable_only: bool = False + ) -> Iterator[tuple[str, str, bool]]: + """Yield locations (filenames) where metadata is stored, and whether + writable or not. + + Parameters + ---------- + writable: bool + Set to True to only yield writable locations. + + Returns + ------- + Yields (str, str, bool) + """ + n = len(self._storage) + for i, storage in enumerate(self._storage): + writable = i == n - 1 + if writable_only and not writable: + continue + yield os.path.join(storage, "cache"), storage, writable + + def check_file( + self, path: str, cfs: CachingFileSystem | None + ) -> Literal[False] | tuple[Detail, str]: + """If path is in cache return its details, otherwise return ``False``. + + If the optional CachingFileSystem is specified then it is used to + perform extra checks to reject possible matches, such as if they are + too old. + """ + for (fn, base, _), cache in zip(self._scan_locations(), self.cached_files): + if path not in cache: + continue + detail = cache[path].copy() + + if cfs is not None: + if cfs.check_files and detail["uid"] != cfs.fs.ukey(path): + # Wrong file as determined by hash of file properties + continue + if cfs.expiry and time.time() - detail["time"] > cfs.expiry: + # Cached file has expired + continue + + fn = os.path.join(base, detail["fn"]) + if os.path.exists(fn): + return detail, fn + return False + + def clear_expired(self, expiry_time: int) -> tuple[list[str], bool]: + """Remove expired metadata from the cache. + + Returns names of files corresponding to expired metadata and a boolean + flag indicating whether the writable cache is empty. Caller is + responsible for deleting the expired files. + """ + expired_files = [] + for path, detail in self.cached_files[-1].copy().items(): + if time.time() - detail["time"] > expiry_time: + fn = detail.get("fn", "") + if not fn: + raise RuntimeError( + f"Cache metadata does not contain 'fn' for {path}" + ) + fn = os.path.join(self._storage[-1], fn) + expired_files.append(fn) + self.cached_files[-1].pop(path) + + if self.cached_files[-1]: + cache_path = os.path.join(self._storage[-1], "cache") + self._save(self.cached_files[-1], cache_path) + + writable_cache_empty = not self.cached_files[-1] + return expired_files, writable_cache_empty + + def load(self) -> None: + """Load all metadata from disk and store in ``self.cached_files``""" + cached_files = [] + for fn, _, _ in self._scan_locations(): + if os.path.exists(fn): + # TODO: consolidate blocks here + cached_files.append(self._load(fn)) + else: + cached_files.append({}) + self.cached_files = cached_files or [{}] + + def on_close_cached_file(self, f: Any, path: str) -> None: + """Perform side-effect actions on closing a cached file. + + The actual closing of the file is the responsibility of the caller. + """ + # File must be writable, so in self.cached_files[-1] + c = self.cached_files[-1][path] + if c["blocks"] is not True and len(c["blocks"]) * f.blocksize >= f.size: + c["blocks"] = True + + def pop_file(self, path: str) -> str | None: + """Remove metadata of cached file. + + If path is in the cache, return the filename of the cached file, + otherwise return ``None``. Caller is responsible for deleting the + cached file. + """ + details = self.check_file(path, None) + if not details: + return None + _, fn = details + if fn.startswith(self._storage[-1]): + self.cached_files[-1].pop(path) + self.save() + else: + raise PermissionError( + "Can only delete cached file in last, writable cache location" + ) + return fn + + def save(self) -> None: + """Save metadata to disk""" + for (fn, _, writable), cache in zip(self._scan_locations(), self.cached_files): + if not writable: + continue + + if os.path.exists(fn): + cached_files = self._load(fn) + for k, c in cached_files.items(): + if k in cache: + if c["blocks"] is True or cache[k]["blocks"] is True: + c["blocks"] = True + else: + # self.cached_files[*][*]["blocks"] must continue to + # point to the same set object so that updates + # performed by MMapCache are propagated back to + # self.cached_files. + blocks = cache[k]["blocks"] + blocks.update(c["blocks"]) + c["blocks"] = blocks + c["time"] = max(c["time"], cache[k]["time"]) + c["uid"] = cache[k]["uid"] + + # Files can be added to cache after it was written once + for k, c in cache.items(): + if k not in cached_files: + cached_files[k] = c + else: + cached_files = cache + cache = {k: v.copy() for k, v in cached_files.items()} + for c in cache.values(): + if isinstance(c["blocks"], set): + c["blocks"] = list(c["blocks"]) + self._save(cache, fn) + self.cached_files[-1] = cached_files + + def update_file(self, path: str, detail: Detail) -> None: + """Update metadata for specific file in memory, do not save""" + self.cached_files[-1][path] = detail diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/cached.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/cached.py new file mode 100644 index 0000000000000000000000000000000000000000..c140f8d617d7cfe5781b01344ab3fb77e0fe3775 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/cached.py @@ -0,0 +1,1024 @@ +from __future__ import annotations + +import inspect +import logging +import os +import tempfile +import time +import weakref +from collections.abc import Callable +from shutil import rmtree +from typing import TYPE_CHECKING, Any, ClassVar + +from fsspec import filesystem +from fsspec.callbacks import DEFAULT_CALLBACK +from fsspec.compression import compr +from fsspec.core import BaseCache, MMapCache +from fsspec.exceptions import BlocksizeMismatchError +from fsspec.implementations.cache_mapper import create_cache_mapper +from fsspec.implementations.cache_metadata import CacheMetadata +from fsspec.implementations.chained import ChainedFileSystem +from fsspec.implementations.local import LocalFileSystem +from fsspec.spec import AbstractBufferedFile +from fsspec.transaction import Transaction +from fsspec.utils import infer_compression + +if TYPE_CHECKING: + from fsspec.implementations.cache_mapper import AbstractCacheMapper + +logger = logging.getLogger("fsspec.cached") + + +class WriteCachedTransaction(Transaction): + def complete(self, commit=True): + rpaths = [f.path for f in self.files] + lpaths = [f.fn for f in self.files] + if commit: + self.fs.put(lpaths, rpaths) + self.files.clear() + self.fs._intrans = False + self.fs._transaction = None + self.fs = None # break cycle + + +class CachingFileSystem(ChainedFileSystem): + """Locally caching filesystem, layer over any other FS + + This class implements chunk-wise local storage of remote files, for quick + access after the initial download. The files are stored in a given + directory with hashes of URLs for the filenames. If no directory is given, + a temporary one is used, which should be cleaned up by the OS after the + process ends. The files themselves are sparse (as implemented in + :class:`~fsspec.caching.MMapCache`), so only the data which is accessed + takes up space. + + Restrictions: + + - the block-size must be the same for each access of a given file, unless + all blocks of the file have already been read + - caching can only be applied to file-systems which produce files + derived from fsspec.spec.AbstractBufferedFile ; LocalFileSystem is also + allowed, for testing + """ + + protocol: ClassVar[str | tuple[str, ...]] = ("blockcache", "cached") + _strip_tokenize_options = ("fo",) + + def __init__( + self, + target_protocol=None, + cache_storage="TMP", + cache_check=10, + check_files=False, + expiry_time=604800, + target_options=None, + fs=None, + same_names: bool | None = None, + compression=None, + cache_mapper: AbstractCacheMapper | None = None, + **kwargs, + ): + """ + + Parameters + ---------- + target_protocol: str (optional) + Target filesystem protocol. Provide either this or ``fs``. + cache_storage: str or list(str) + Location to store files. If "TMP", this is a temporary directory, + and will be cleaned up by the OS when this process ends (or later). + If a list, each location will be tried in the order given, but + only the last will be considered writable. + cache_check: int + Number of seconds between reload of cache metadata + check_files: bool + Whether to explicitly see if the UID of the remote file matches + the stored one before using. Warning: some file systems such as + HTTP cannot reliably give a unique hash of the contents of some + path, so be sure to set this option to False. + expiry_time: int + The time in seconds after which a local copy is considered useless. + Set to falsy to prevent expiry. The default is equivalent to one + week. + target_options: dict or None + Passed to the instantiation of the FS, if fs is None. + fs: filesystem instance + The target filesystem to run against. Provide this or ``protocol``. + same_names: bool (optional) + By default, target URLs are hashed using a ``HashCacheMapper`` so + that files from different backends with the same basename do not + conflict. If this argument is ``true``, a ``BasenameCacheMapper`` + is used instead. Other cache mapper options are available by using + the ``cache_mapper`` keyword argument. Only one of this and + ``cache_mapper`` should be specified. + compression: str (optional) + To decompress on download. Can be 'infer' (guess from the URL name), + one of the entries in ``fsspec.compression.compr``, or None for no + decompression. + cache_mapper: AbstractCacheMapper (optional) + The object use to map from original filenames to cached filenames. + Only one of this and ``same_names`` should be specified. + """ + super().__init__(**kwargs) + if fs is None and target_protocol is None: + raise ValueError( + "Please provide filesystem instance(fs) or target_protocol" + ) + if not (fs is None) ^ (target_protocol is None): + raise ValueError( + "Both filesystems (fs) and target_protocol may not be both given." + ) + if cache_storage == "TMP": + tempdir = tempfile.mkdtemp() + storage = [tempdir] + weakref.finalize(self, self._remove_tempdir, tempdir) + else: + if isinstance(cache_storage, str): + storage = [cache_storage] + else: + storage = cache_storage + os.makedirs(storage[-1], exist_ok=True) + self.storage = storage + self.kwargs = target_options or {} + self.cache_check = cache_check + self.check_files = check_files + self.expiry = expiry_time + self.compression = compression + + # Size of cache in bytes. If None then the size is unknown and will be + # recalculated the next time cache_size() is called. On writes to the + # cache this is reset to None. + self._cache_size = None + + if same_names is not None and cache_mapper is not None: + raise ValueError( + "Cannot specify both same_names and cache_mapper in " + "CachingFileSystem.__init__" + ) + if cache_mapper is not None: + self._mapper = cache_mapper + else: + self._mapper = create_cache_mapper( + same_names if same_names is not None else False + ) + + self.target_protocol = ( + target_protocol + if isinstance(target_protocol, str) + else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0]) + ) + self._metadata = CacheMetadata(self.storage) + self.load_cache() + self.fs = fs if fs is not None else filesystem(target_protocol, **self.kwargs) + + def _strip_protocol(path): + # acts as a method, since each instance has a difference target + return self.fs._strip_protocol(type(self)._strip_protocol(path)) + + self._strip_protocol: Callable = _strip_protocol + + @staticmethod + def _remove_tempdir(tempdir): + try: + rmtree(tempdir) + except Exception: + pass + + def _mkcache(self): + os.makedirs(self.storage[-1], exist_ok=True) + + def cache_size(self): + """Return size of cache in bytes. + + If more than one cache directory is in use, only the size of the last + one (the writable cache directory) is returned. + """ + if self._cache_size is None: + cache_dir = self.storage[-1] + self._cache_size = filesystem("file").du(cache_dir, withdirs=True) + return self._cache_size + + def load_cache(self): + """Read set of stored blocks from file""" + self._metadata.load() + self._mkcache() + self.last_cache = time.time() + + def save_cache(self): + """Save set of stored blocks from file""" + self._mkcache() + self._metadata.save() + self.last_cache = time.time() + self._cache_size = None + + def _check_cache(self): + """Reload caches if time elapsed or any disappeared""" + self._mkcache() + if not self.cache_check: + # explicitly told not to bother checking + return + timecond = time.time() - self.last_cache > self.cache_check + existcond = all(os.path.exists(storage) for storage in self.storage) + if timecond or not existcond: + self.load_cache() + + def _check_file(self, path): + """Is path in cache and still valid""" + path = self._strip_protocol(path) + self._check_cache() + return self._metadata.check_file(path, self) + + def clear_cache(self): + """Remove all files and metadata from the cache + + In the case of multiple cache locations, this clears only the last one, + which is assumed to be the read/write one. + """ + rmtree(self.storage[-1]) + self.load_cache() + self._cache_size = None + + def clear_expired_cache(self, expiry_time=None): + """Remove all expired files and metadata from the cache + + In the case of multiple cache locations, this clears only the last one, + which is assumed to be the read/write one. + + Parameters + ---------- + expiry_time: int + The time in seconds after which a local copy is considered useless. + If not defined the default is equivalent to the attribute from the + file caching instantiation. + """ + + if not expiry_time: + expiry_time = self.expiry + + self._check_cache() + + expired_files, writable_cache_empty = self._metadata.clear_expired(expiry_time) + for fn in expired_files: + if os.path.exists(fn): + os.remove(fn) + + if writable_cache_empty: + rmtree(self.storage[-1]) + self.load_cache() + + self._cache_size = None + + def pop_from_cache(self, path): + """Remove cached version of given file + + Deletes local copy of the given (remote) path. If it is found in a cache + location which is not the last, it is assumed to be read-only, and + raises PermissionError + """ + path = self._strip_protocol(path) + fn = self._metadata.pop_file(path) + if fn is not None: + os.remove(fn) + self._cache_size = None + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + """Wrap the target _open + + If the whole file exists in the cache, just open it locally and + return that. + + Otherwise, open the file on the target FS, and make it have a mmap + cache pointing to the location which we determine, in our cache. + The ``blocks`` instance is shared, so as the mmap cache instance + updates, so does the entry in our ``cached_files`` attribute. + We monkey-patch this file, so that when it closes, we call + ``close_and_update`` to save the state of the blocks. + """ + path = self._strip_protocol(path) + + path = self.fs._strip_protocol(path) + if "r" not in mode: + return self.fs._open( + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_options=cache_options, + **kwargs, + ) + detail = self._check_file(path) + if detail: + # file is in cache + detail, fn = detail + hash, blocks = detail["fn"], detail["blocks"] + if blocks is True: + # stored file is complete + logger.debug("Opening local copy of %s", path) + return open(fn, mode) + # TODO: action where partial file exists in read-only cache + logger.debug("Opening partially cached copy of %s", path) + else: + hash = self._mapper(path) + fn = os.path.join(self.storage[-1], hash) + blocks = set() + detail = { + "original": path, + "fn": hash, + "blocks": blocks, + "time": time.time(), + "uid": self.fs.ukey(path), + } + self._metadata.update_file(path, detail) + logger.debug("Creating local sparse file for %s", path) + + # explicitly submitting the size to the open call will avoid extra + # operations when opening. This is particularly relevant + # for any file that is read over a network, e.g. S3. + size = detail.get("size") + + # call target filesystems open + self._mkcache() + f = self.fs._open( + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_options=cache_options, + cache_type="none", + size=size, + **kwargs, + ) + + # set size if not already set + if size is None: + detail["size"] = f.size + self._metadata.update_file(path, detail) + + if self.compression: + comp = ( + infer_compression(path) + if self.compression == "infer" + else self.compression + ) + f = compr[comp](f, mode="rb") + if "blocksize" in detail: + if detail["blocksize"] != f.blocksize: + raise BlocksizeMismatchError( + f"Cached file must be reopened with same block" + f" size as original (old: {detail['blocksize']}," + f" new {f.blocksize})" + ) + else: + detail["blocksize"] = f.blocksize + + def _fetch_ranges(ranges): + return self.fs.cat_ranges( + [path] * len(ranges), + [r[0] for r in ranges], + [r[1] for r in ranges], + **kwargs, + ) + + multi_fetcher = None if self.compression else _fetch_ranges + f.cache = MMapCache( + f.blocksize, f._fetch_range, f.size, fn, blocks, multi_fetcher=multi_fetcher + ) + close = f.close + f.close = lambda: self.close_and_update(f, close) + self.save_cache() + return f + + def _parent(self, path): + return self.fs._parent(path) + + def hash_name(self, path: str, *args: Any) -> str: + # Kept for backward compatibility with downstream libraries. + # Ignores extra arguments, previously same_name boolean. + return self._mapper(path) + + def close_and_update(self, f, close): + """Called when a file is closing, so store the set of blocks""" + if f.closed: + return + path = self._strip_protocol(f.path) + self._metadata.on_close_cached_file(f, path) + try: + logger.debug("going to save") + self.save_cache() + logger.debug("saved") + except OSError: + logger.debug("Cache saving failed while closing file") + except NameError: + logger.debug("Cache save failed due to interpreter shutdown") + close() + f.closed = True + + def ls(self, path, detail=True): + return self.fs.ls(path, detail) + + def __getattribute__(self, item): + if item in { + "load_cache", + "_get_cached_file_before_open", + "_open", + "save_cache", + "close_and_update", + "__init__", + "__getattribute__", + "__reduce__", + "_make_local_details", + "open", + "cat", + "cat_file", + "_cat_file", + "cat_ranges", + "_cat_ranges", + "get", + "read_block", + "tail", + "head", + "info", + "ls", + "exists", + "isfile", + "isdir", + "_check_file", + "_check_cache", + "_mkcache", + "clear_cache", + "clear_expired_cache", + "pop_from_cache", + "local_file", + "_paths_from_path", + "get_mapper", + "open_many", + "commit_many", + "hash_name", + "__hash__", + "__eq__", + "to_json", + "to_dict", + "cache_size", + "pipe_file", + "pipe", + "start_transaction", + "end_transaction", + }: + # all the methods defined in this class. Note `open` here, since + # it calls `_open`, but is actually in superclass + if hasattr(type(self), item): + return lambda *args, **kw: getattr(type(self), item).__get__(self)( + *args, **kw + ) + # method is in the whitelist but not defined on this subclass; + # fall through to delegate to the wrapped filesystem below + if item in ["__reduce_ex__"]: + raise AttributeError + if item in ["transaction"]: + # property + return type(self).transaction.__get__(self) + if item in {"_cache", "transaction_type", "protocol"}: + # class attributes + return getattr(type(self), item) + if item == "__class__": + return type(self) + d = object.__getattribute__(self, "__dict__") + fs = d.get("fs", None) # fs is not immediately defined + if item in d: + return d[item] + elif fs is not None: + if item in fs.__dict__: + # attribute of instance + return fs.__dict__[item] + # attributed belonging to the target filesystem + cls = type(fs) + m = getattr(cls, item) + if (inspect.isfunction(m) or inspect.isdatadescriptor(m)) and ( + not hasattr(m, "__self__") or m.__self__ is None + ): + # instance method + return m.__get__(fs, cls) + return m # class method or attribute + else: + # attributes of the superclass, while target is being set up + return super().__getattribute__(item) + + def __eq__(self, other): + """Test for equality.""" + if self is other: + return True + if not isinstance(other, type(self)): + return False + return ( + self.storage == other.storage + and self.kwargs == other.kwargs + and self.cache_check == other.cache_check + and self.check_files == other.check_files + and self.expiry == other.expiry + and self.compression == other.compression + and self._mapper == other._mapper + and self.target_protocol == other.target_protocol + ) + + def __hash__(self): + """Calculate hash.""" + return ( + hash(tuple(self.storage)) + ^ hash(str(self.kwargs)) + ^ hash(self.cache_check) + ^ hash(self.check_files) + ^ hash(self.expiry) + ^ hash(self.compression) + ^ hash(self._mapper) + ^ hash(self.target_protocol) + ) + + +class WholeFileCacheFileSystem(CachingFileSystem): + """Caches whole remote files on first access + + This class is intended as a layer over any other file system, and + will make a local copy of each file accessed, so that all subsequent + reads are local. This is similar to ``CachingFileSystem``, but without + the block-wise functionality and so can work even when sparse files + are not allowed. See its docstring for definition of the init + arguments. + + The class still needs access to the remote store for listing files, + and may refresh cached files. + """ + + protocol = "filecache" + local_file = True + + def open_many(self, open_files, **kwargs): + paths = [of.path for of in open_files] + if "r" in open_files.mode: + self._mkcache() + else: + return [ + LocalTempFile( + self.fs, + path, + mode=open_files.mode, + fn=os.path.join(self.storage[-1], self._mapper(path)), + **kwargs, + ) + for path in paths + ] + + if self.compression: + raise NotImplementedError + details = [self._check_file(sp) for sp in paths] + downpath = [p for p, d in zip(paths, details) if not d] + downfn0 = [ + os.path.join(self.storage[-1], self._mapper(p)) + for p, d in zip(paths, details) + ] # keep these path names for opening later + downfn = [fn for fn, d in zip(downfn0, details) if not d] + if downpath: + # skip if all files are already cached and up to date + self.fs.get(downpath, downfn) + + # update metadata - only happens when downloads are successful + newdetail = [ + { + "original": path, + "fn": self._mapper(path), + "blocks": True, + "time": time.time(), + "uid": self.fs.ukey(path), + } + for path in downpath + ] + for path, detail in zip(downpath, newdetail): + self._metadata.update_file(path, detail) + self.save_cache() + + def firstpart(fn): + # helper to adapt both whole-file and simple-cache + return fn[1] if isinstance(fn, tuple) else fn + + return [ + open(firstpart(fn0) if fn0 else fn1, mode=open_files.mode) + for fn0, fn1 in zip(details, downfn0) + ] + + def commit_many(self, open_files): + self.fs.put([f.fn for f in open_files], [f.path for f in open_files]) + [f.close() for f in open_files] + for f in open_files: + # in case autocommit is off, and so close did not already delete + try: + os.remove(f.name) + except FileNotFoundError: + pass + self._cache_size = None + + def _make_local_details(self, path): + hash = self._mapper(path) + fn = os.path.join(self.storage[-1], hash) + detail = { + "original": path, + "fn": hash, + "blocks": True, + "time": time.time(), + "uid": self.fs.ukey(path), + } + self._metadata.update_file(path, detail) + logger.debug("Copying %s to local cache", path) + return fn + + def cat( + self, + path, + recursive=False, + on_error="raise", + callback=DEFAULT_CALLBACK, + **kwargs, + ): + paths = self.expand_path( + path, recursive=recursive, maxdepth=kwargs.get("maxdepth") + ) + getpaths = [] + storepaths = [] + fns = [] + out = {} + for p in paths.copy(): + try: + detail = self._check_file(p) + if not detail: + fn = self._make_local_details(p) + getpaths.append(p) + storepaths.append(fn) + else: + detail, fn = detail if isinstance(detail, tuple) else (None, detail) + fns.append(fn) + except Exception as e: + if on_error == "raise": + raise + if on_error == "return": + out[p] = e + paths.remove(p) + + if getpaths: + self.fs.get(getpaths, storepaths) + self.save_cache() + + callback.set_size(len(paths)) + for p, fn in zip(paths, fns): + with open(fn, "rb") as f: + out[p] = f.read() + callback.relative_update(1) + if isinstance(path, str) and len(paths) == 1 and recursive is False: + out = out[paths[0]] + return out + + def _get_cached_file_before_open(self, path, **kwargs): + fn = self._make_local_details(path) + # call target filesystems open + self._mkcache() + if self.compression: + with self.fs._open(path, mode="rb", **kwargs) as f, open(fn, "wb") as f2: + if isinstance(f, AbstractBufferedFile): + # want no type of caching if just downloading whole thing + f.cache = BaseCache(0, f.cache.fetcher, f.size) + comp = ( + infer_compression(path) + if self.compression == "infer" + else self.compression + ) + f = compr[comp](f, mode="rb") + data = True + while data: + block = getattr(f, "blocksize", 5 * 2**20) + data = f.read(block) + f2.write(data) + else: + self.fs.get_file(path, fn) + self.save_cache() + + def _open(self, path, mode="rb", **kwargs): + path = self._strip_protocol(path) + # For read (or append), (try) download from remote + if "r" in mode or "a" in mode: + if not self._check_file(path): + if self.fs.exists(path): + self._get_cached_file_before_open(path, **kwargs) + elif "r" in mode: + raise FileNotFoundError(path) + + detail, fn = self._check_file(path) + _, blocks = detail["fn"], detail["blocks"] + if blocks is True: + logger.debug("Opening local copy of %s", path) + else: + raise ValueError( + f"Attempt to open partially cached file {path}" + f" as a wholly cached file" + ) + + # Just reading does not need special file handling + if "r" in mode and "+" not in mode: + # In order to support downstream filesystems to be able to + # infer the compression from the original filename, like + # the `TarFileSystem`, let's extend the `io.BufferedReader` + # fileobject protocol by adding a dedicated attribute + # `original`. + f = open(fn, mode) + f.original = detail.get("original") + return f + + hash = self._mapper(path) + fn = os.path.join(self.storage[-1], hash) + user_specified_kwargs = { + k: v + for k, v in kwargs.items() + # those kwargs were added by open(), we don't want them + if k not in ["autocommit", "block_size", "cache_options"] + } + return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs) + + async def _cat_file(self, path, start=None, end=None, **kwargs): + logger.debug("async cat_file %s", path) + path = self._strip_protocol(path) + sha = self._mapper(path) + fn = self._check_file(path) + + if not fn: + fn = os.path.join(self.storage[-1], sha) + await self.fs._get_file(path, fn, **kwargs) + + with open(fn, "rb") as f: # noqa ASYNC230 + if start: + f.seek(start) + size = -1 if end is None else end - f.tell() + return f.read(size) + + async def _cat_ranges( + self, paths, starts, ends, max_gap=None, on_error="return", **kwargs + ): + logger.debug("async cat ranges %s", paths) + lpaths = [] + rset = set() + download = [] + rpaths = [] + for p in paths: + fn = self._check_file(p) + if fn is None and p not in rset: + sha = self._mapper(p) + fn = os.path.join(self.storage[-1], sha) + download.append(fn) + rset.add(p) + rpaths.append(p) + lpaths.append(fn) + if download: + await self.fs._get(rpaths, download, on_error=on_error) + + return LocalFileSystem().cat_ranges( + lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs + ) + + +class SimpleCacheFileSystem(WholeFileCacheFileSystem): + """Caches whole remote files on first access + + This class is intended as a layer over any other file system, and + will make a local copy of each file accessed, so that all subsequent + reads are local. This implementation only copies whole files, and + does not keep any metadata about the download time or file details. + It is therefore safer to use in multi-threaded/concurrent situations. + + This is the only of the caching filesystems that supports write: you will + be given a real local open file, and upon close and commit, it will be + uploaded to the target filesystem; the writability or the target URL is + not checked until that time. + + """ + + protocol = "simplecache" + local_file = True + transaction_type = WriteCachedTransaction + + def __init__(self, **kwargs): + kw = kwargs.copy() + for key in ["cache_check", "expiry_time", "check_files"]: + kw[key] = False + super().__init__(**kw) + for storage in self.storage: + if not os.path.exists(storage): + os.makedirs(storage, exist_ok=True) + + def _check_file(self, path): + self._check_cache() + sha = self._mapper(path) + for storage in self.storage: + fn = os.path.join(storage, sha) + if os.path.exists(fn): + return fn + + def save_cache(self): + pass + + def load_cache(self): + pass + + def pipe_file(self, path, value=None, **kwargs): + if self._intrans: + with self.open(path, "wb") as f: + f.write(value) + else: + super().pipe_file(path, value) + + def ls(self, path, detail=True, **kwargs): + path = self._strip_protocol(path) + details = [] + try: + details = self.fs.ls( + path, detail=True, **kwargs + ).copy() # don't edit original! + except FileNotFoundError as e: + ex = e + else: + ex = None + if self._intrans: + path1 = path.rstrip("/") + "/" + for f in self.transaction.files: + if f.path == path: + details.append( + {"name": path, "size": f.size or f.tell(), "type": "file"} + ) + elif f.path.startswith(path1): + if f.path.count("/") == path1.count("/"): + details.append( + {"name": f.path, "size": f.size or f.tell(), "type": "file"} + ) + else: + dname = "/".join(f.path.split("/")[: path1.count("/") + 1]) + details.append({"name": dname, "size": 0, "type": "directory"}) + if ex is not None and not details: + raise ex + if detail: + return details + return sorted(_["name"] for _ in details) + + def info(self, path, **kwargs): + path = self._strip_protocol(path) + if self._intrans: + f = [_ for _ in self.transaction.files if _.path == path] + if f: + size = os.path.getsize(f[0].fn) if f[0].closed else f[0].tell() + return {"name": path, "size": size, "type": "file"} + f = any(_.path.startswith(path + "/") for _ in self.transaction.files) + if f: + return {"name": path, "size": 0, "type": "directory"} + return self.fs.info(path, **kwargs) + + def pipe(self, path, value=None, **kwargs): + if isinstance(path, str): + self.pipe_file(self._strip_protocol(path), value, **kwargs) + elif isinstance(path, dict): + for k, v in path.items(): + self.pipe_file(self._strip_protocol(k), v, **kwargs) + else: + raise ValueError("path must be str or dict") + + def cat_ranges( + self, paths, starts, ends, max_gap=None, on_error="return", **kwargs + ): + logger.debug("cat ranges %s", paths) + lpaths = [self._check_file(p) for p in paths] + rpaths = [p for l, p in zip(lpaths, paths) if l is False] + lpaths = [l for l, p in zip(lpaths, paths) if l is False] + self.fs.get(rpaths, lpaths) + paths = [self._check_file(p) for p in paths] + return LocalFileSystem().cat_ranges( + paths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs + ) + + def _get_cached_file_before_open(self, path, **kwargs): + sha = self._mapper(path) + fn = os.path.join(self.storage[-1], sha) + logger.debug("Copying %s to local cache", path) + + self._mkcache() + self._cache_size = None + + if self.compression: + with self.fs._open(path, mode="rb", **kwargs) as f, open(fn, "wb") as f2: + if isinstance(f, AbstractBufferedFile): + # want no type of caching if just downloading whole thing + f.cache = BaseCache(0, f.cache.fetcher, f.size) + comp = ( + infer_compression(path) + if self.compression == "infer" + else self.compression + ) + f = compr[comp](f, mode="rb") + data = True + while data: + block = getattr(f, "blocksize", 5 * 2**20) + data = f.read(block) + f2.write(data) + else: + self.fs.get_file(path, fn) + + def _open(self, path, mode="rb", **kwargs): + path = self._strip_protocol(path) + sha = self._mapper(path) + + # For read (or append), (try) download from remote + if "r" in mode or "a" in mode: + if not self._check_file(path): + # append does not require an existing file but read does + if self.fs.exists(path): + self._get_cached_file_before_open(path, **kwargs) + elif "r" in mode: + raise FileNotFoundError(path) + + fn = self._check_file(path) + # Just reading does not need special file handling + if "r" in mode and "+" not in mode: + return open(fn, mode) + + fn = os.path.join(self.storage[-1], sha) + user_specified_kwargs = { + k: v + for k, v in kwargs.items() + if k not in ["autocommit", "block_size", "cache_options"] + } # those were added by open() + return LocalTempFile( + self, + path, + mode=mode, + autocommit=not self._intrans, + fn=fn, + **user_specified_kwargs, + ) + + +class LocalTempFile: + """A temporary local file, which will be uploaded on commit""" + + def __init__(self, fs, path, fn, mode="wb", autocommit=True, seek=0, **kwargs): + self.fn = fn + self.fh = open(fn, mode) + self.mode = mode + if seek: + self.fh.seek(seek) + self.path = path + self.size = None + self.fs = fs + self.closed = False + self.autocommit = autocommit + self.kwargs = kwargs + + def __reduce__(self): + # always open in r+b to allow continuing writing at a location + return ( + LocalTempFile, + (self.fs, self.path, self.fn, "r+b", self.autocommit, self.tell()), + ) + + def __enter__(self): + return self.fh + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + # self.size = self.fh.tell() + if self.closed: + return + self.fh.close() + self.closed = True + if self.autocommit: + self.commit() + + def discard(self): + self.fh.close() + os.remove(self.fn) + + def commit(self): + # calling put() with list arguments avoids path expansion and additional operations + # like isdir() + self.fs.put([self.fn], [self.path], **self.kwargs) + # we do not delete the local copy, it's still in the cache. + + @property + def name(self): + return self.fn + + def __repr__(self) -> str: + return f"LocalTempFile: {self.path}" + + def __getattr__(self, item): + return getattr(self.fh, item) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/chained.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/chained.py new file mode 100644 index 0000000000000000000000000000000000000000..bfce64334e8db0272eefa96b4428b23524b059f0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/chained.py @@ -0,0 +1,23 @@ +from typing import ClassVar + +from fsspec import AbstractFileSystem + +__all__ = ("ChainedFileSystem",) + + +class ChainedFileSystem(AbstractFileSystem): + """Chained filesystem base class. + + A chained filesystem is designed to be layered over another FS. + This is useful to implement things like caching. + + This base class does very little on its own, but is used as a marker + that the class is designed for chaining. + + Right now this is only used in `url_to_fs` to provide the path argument + (`fo`) to the chained filesystem from the underlying filesystem. + + Additional functionality may be added in the future. + """ + + protocol: ClassVar[str] = "chained" diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/dask.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/dask.py new file mode 100644 index 0000000000000000000000000000000000000000..3e1276463db6866665e6a0fe114efc247971b57e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/dask.py @@ -0,0 +1,152 @@ +import dask +from distributed.client import Client, _get_global_client +from distributed.worker import Worker + +from fsspec import filesystem +from fsspec.spec import AbstractBufferedFile, AbstractFileSystem +from fsspec.utils import infer_storage_options + + +def _get_client(client): + if client is None: + return _get_global_client() + elif isinstance(client, Client): + return client + else: + # e.g., connection string + return Client(client) + + +def _in_worker(): + return bool(Worker._instances) + + +class DaskWorkerFileSystem(AbstractFileSystem): + """View files accessible to a worker as any other remote file-system + + When instances are run on the worker, uses the real filesystem. When + run on the client, they call the worker to provide information or data. + + **Warning** this implementation is experimental, and read-only for now. + """ + + def __init__( + self, target_protocol=None, target_options=None, fs=None, client=None, **kwargs + ): + super().__init__(**kwargs) + if not (fs is None) ^ (target_protocol is None): + raise ValueError( + "Please provide one of filesystem instance (fs) or" + " target_protocol, not both" + ) + self.target_protocol = target_protocol + self.target_options = target_options + self.worker = None + self.client = client + self.fs = fs + self._determine_worker() + + @staticmethod + def _get_kwargs_from_urls(path): + so = infer_storage_options(path) + if "host" in so and "port" in so: + return {"client": f"{so['host']}:{so['port']}"} + else: + return {} + + def _determine_worker(self): + if _in_worker(): + self.worker = True + if self.fs is None: + self.fs = filesystem( + self.target_protocol, **(self.target_options or {}) + ) + else: + self.worker = False + self.client = _get_client(self.client) + self.rfs = dask.delayed(self) + + def mkdir(self, *args, **kwargs): + if self.worker: + self.fs.mkdir(*args, **kwargs) + else: + self.rfs.mkdir(*args, **kwargs).compute() + + def rm(self, *args, **kwargs): + if self.worker: + self.fs.rm(*args, **kwargs) + else: + self.rfs.rm(*args, **kwargs).compute() + + def copy(self, *args, **kwargs): + if self.worker: + self.fs.copy(*args, **kwargs) + else: + self.rfs.copy(*args, **kwargs).compute() + + def mv(self, *args, **kwargs): + if self.worker: + self.fs.mv(*args, **kwargs) + else: + self.rfs.mv(*args, **kwargs).compute() + + def ls(self, *args, **kwargs): + if self.worker: + return self.fs.ls(*args, **kwargs) + else: + return self.rfs.ls(*args, **kwargs).compute() + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + if self.worker: + return self.fs._open( + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_options=cache_options, + **kwargs, + ) + else: + return DaskFile( + fs=self, + path=path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_options=cache_options, + **kwargs, + ) + + def fetch_range(self, path, mode, start, end): + if self.worker: + with self._open(path, mode) as f: + f.seek(start) + return f.read(end - start) + else: + return self.rfs.fetch_range(path, mode, start, end).compute() + + +class DaskFile(AbstractBufferedFile): + def __init__(self, mode="rb", **kwargs): + if mode != "rb": + raise ValueError('Remote dask files can only be opened in "rb" mode') + super().__init__(**kwargs) + + def _upload_chunk(self, final=False): + pass + + def _initiate_upload(self): + """Create remote file/upload""" + pass + + def _fetch_range(self, start, end): + """Get the specified set of bytes from remote""" + return self.fs.fetch_range(self.path, self.mode, start, end) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/data.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/data.py new file mode 100644 index 0000000000000000000000000000000000000000..fad1b56d0940a6e5232497475ab32a504e19ec9a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/data.py @@ -0,0 +1,71 @@ +import base64 +import io +from urllib.parse import unquote + +from fsspec import AbstractFileSystem +from fsspec.utils import stringify_path + + +class DataFileSystem(AbstractFileSystem): + """A handy decoder for data-URLs + + Example + ------- + >>> with fsspec.open("data:,Hello%2C%20World%21") as f: + ... print(f.read()) + b"Hello, World!" + + See https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs + """ + + protocol = "data" + + def __init__(self, **kwargs): + """No parameters for this filesystem""" + super().__init__(**kwargs) + + @classmethod + def _strip_protocol(cls, path): + if isinstance(path, list): + return [cls._strip_protocol(p) for p in path] + path = stringify_path(path) + if path.startswith("data://"): + path = path[7:] + elif path.startswith("data:"): + path = path[5:] + # Do NOT strip trailing slashes, as they may be meaningful base64 characters + # or percent-encoded data content + return path + + def cat_file(self, path, start=None, end=None, **kwargs): + pref, data = path.split(",", 1) + if pref.endswith("base64"): + return base64.b64decode(data)[start:end] + return unquote(data).encode()[start:end] + + def info(self, path, **kwargs): + pref, name = path.split(",", 1) + data = self.cat_file(path) + mime = pref.split(":", 1)[1].split(";", 1)[0] + return {"name": name, "size": len(data), "type": "file", "mimetype": mime} + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + if "r" not in mode: + raise ValueError("Read only filesystem") + return io.BytesIO(self.cat_file(path)) + + @staticmethod + def encode(data: bytes, mime: str | None = None): + """Format the given data into data-URL syntax + + This version always base64 encodes, even when the data is ascii/url-safe. + """ + return f"data:{mime or ''};base64,{base64.b64encode(data).decode()}" diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/dbfs.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/dbfs.py new file mode 100644 index 0000000000000000000000000000000000000000..1b7f03f66c94dccf7880546728ce1c3f110bc10a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/dbfs.py @@ -0,0 +1,497 @@ +from __future__ import annotations + +import base64 +import urllib + +import requests +from requests.adapters import HTTPAdapter, Retry +from typing_extensions import override + +from fsspec import AbstractFileSystem +from fsspec.spec import AbstractBufferedFile + + +class DatabricksException(Exception): + """ + Helper class for exceptions raised in this module. + """ + + def __init__(self, error_code, message, details=None): + """Create a new DatabricksException""" + super().__init__(message) + + self.error_code = error_code + self.message = message + self.details = details + + +class DatabricksFileSystem(AbstractFileSystem): + """ + Get access to the Databricks filesystem implementation over HTTP. + Can be used inside and outside of a databricks cluster. + """ + + def __init__(self, instance, token, **kwargs): + """ + Create a new DatabricksFileSystem. + + Parameters + ---------- + instance: str + The instance URL of the databricks cluster. + For example for an Azure databricks cluster, this + has the form adb-<some-number>.<two digits>.azuredatabricks.net. + token: str + Your personal token. Find out more + here: https://docs.databricks.com/dev-tools/api/latest/authentication.html + """ + self.instance = instance + self.token = token + self.session = requests.Session() + self.retries = Retry( + total=10, + backoff_factor=0.05, + status_forcelist=[408, 429, 500, 502, 503, 504], + ) + + self.session.mount("https://", HTTPAdapter(max_retries=self.retries)) + self.session.headers.update({"Authorization": f"Bearer {self.token}"}) + + super().__init__(**kwargs) + + @override + def _ls_from_cache(self, path) -> list[dict[str, str | int]] | None: + """Check cache for listing + + Returns listing, if found (may be empty list for a directory that + exists but contains nothing), None if not in cache. + """ + self.dircache.pop(path.rstrip("/"), None) + + parent = self._parent(path) + if parent in self.dircache: + for entry in self.dircache[parent]: + if entry["name"] == path.rstrip("/"): + if entry["type"] != "directory": + return [entry] + return [] + raise FileNotFoundError(path) + + def ls(self, path, detail=True, **kwargs): + """ + List the contents of the given path. + + Parameters + ---------- + path: str + Absolute path + detail: bool + Return not only the list of filenames, + but also additional information on file sizes + and types. + """ + try: + out = self._ls_from_cache(path) + except FileNotFoundError: + # This happens if the `path`'s parent was cached, but `path` is not + # there. This suggests that `path` is new since the parent was + # cached. Attempt to invalidate parent's cache before continuing. + self.dircache.pop(self._parent(path), None) + out = None + + if not out: + try: + r = self._send_to_api( + method="get", endpoint="list", json={"path": path} + ) + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + raise FileNotFoundError(e.message) from e + + raise + files = r.get("files", []) + out = [ + { + "name": o["path"], + "type": "directory" if o["is_dir"] else "file", + "size": o["file_size"], + } + for o in files + ] + self.dircache[path] = out + + if detail: + return out + return [o["name"] for o in out] + + def makedirs(self, path, exist_ok=True): + """ + Create a given absolute path and all of its parents. + + Parameters + ---------- + path: str + Absolute path to create + exist_ok: bool + If false, checks if the folder + exists before creating it (and raises an + Exception if this is the case) + """ + if not exist_ok: + try: + # If the following succeeds, the path is already present + self._send_to_api( + method="get", endpoint="get-status", json={"path": path} + ) + raise FileExistsError(f"Path {path} already exists") + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + pass + + try: + self._send_to_api(method="post", endpoint="mkdirs", json={"path": path}) + except DatabricksException as e: + if e.error_code == "RESOURCE_ALREADY_EXISTS": + raise FileExistsError(e.message) from e + + raise + self.invalidate_cache(self._parent(path)) + + def mkdir(self, path, create_parents=True, **kwargs): + """ + Create a given absolute path and all of its parents. + + Parameters + ---------- + path: str + Absolute path to create + create_parents: bool + Whether to create all parents or not. + "False" is not implemented so far. + """ + if not create_parents: + raise NotImplementedError + + self.mkdirs(path, **kwargs) + + def rm(self, path, recursive=False, **kwargs): + """ + Remove the file or folder at the given absolute path. + + Parameters + ---------- + path: str + Absolute path what to remove + recursive: bool + Recursively delete all files in a folder. + """ + try: + self._send_to_api( + method="post", + endpoint="delete", + json={"path": path, "recursive": recursive}, + ) + except DatabricksException as e: + # This is not really an exception, it just means + # not everything was deleted so far + if e.error_code == "PARTIAL_DELETE": + self.rm(path=path, recursive=recursive) + elif e.error_code == "IO_ERROR": + # Using the same exception as the os module would use here + raise OSError(e.message) from e + + raise + self.invalidate_cache(self._parent(path)) + + def mv( + self, source_path, destination_path, recursive=False, maxdepth=None, **kwargs + ): + """ + Move a source to a destination path. + + A note from the original [databricks API manual] + (https://docs.databricks.com/dev-tools/api/latest/dbfs.html#move). + + When moving a large number of files the API call will time out after + approximately 60s, potentially resulting in partially moved data. + Therefore, for operations that move more than 10k files, we strongly + discourage using the DBFS REST API. + + Parameters + ---------- + source_path: str + From where to move (absolute path) + destination_path: str + To where to move (absolute path) + recursive: bool + Not implemented to far. + maxdepth: + Not implemented to far. + """ + if recursive: + raise NotImplementedError + if maxdepth: + raise NotImplementedError + + try: + self._send_to_api( + method="post", + endpoint="move", + json={"source_path": source_path, "destination_path": destination_path}, + ) + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + raise FileNotFoundError(e.message) from e + elif e.error_code == "RESOURCE_ALREADY_EXISTS": + raise FileExistsError(e.message) from e + + raise + self.invalidate_cache(self._parent(source_path)) + self.invalidate_cache(self._parent(destination_path)) + + def _open(self, path, mode="rb", block_size="default", **kwargs): + """ + Overwrite the base class method to make sure to create a DBFile. + All arguments are copied from the base method. + + Only the default blocksize is allowed. + """ + return DatabricksFile(self, path, mode=mode, block_size=block_size, **kwargs) + + def _send_to_api(self, method, endpoint, json): + """ + Send the given json to the DBFS API + using a get or post request (specified by the argument `method`). + + Parameters + ---------- + method: str + Which http method to use for communication; "get" or "post". + endpoint: str + Where to send the request to (last part of the API URL) + json: dict + Dictionary of information to send + """ + if method == "post": + session_call = self.session.post + elif method == "get": + session_call = self.session.get + else: + raise ValueError(f"Do not understand method {method}") + + url = urllib.parse.urljoin(f"https://{self.instance}/api/2.0/dbfs/", endpoint) + + r = session_call(url, json=json) + + # The DBFS API will return a json, also in case of an exception. + # We want to preserve this information as good as possible. + try: + r.raise_for_status() + except requests.HTTPError as e: + # try to extract json error message + # if that fails, fall back to the original exception + try: + exception_json = e.response.json() + except Exception: + raise e from None + + raise DatabricksException(**exception_json) from e + + return r.json() + + def _create_handle(self, path, overwrite=True): + """ + Internal function to create a handle, which can be used to + write blocks of a file to DBFS. + A handle has a unique identifier which needs to be passed + whenever written during this transaction. + The handle is active for 10 minutes - after that a new + write transaction needs to be created. + Make sure to close the handle after you are finished. + + Parameters + ---------- + path: str + Absolute path for this file. + overwrite: bool + If a file already exist at this location, either overwrite + it or raise an exception. + """ + try: + r = self._send_to_api( + method="post", + endpoint="create", + json={"path": path, "overwrite": overwrite}, + ) + return r["handle"] + except DatabricksException as e: + if e.error_code == "RESOURCE_ALREADY_EXISTS": + raise FileExistsError(e.message) from e + + raise + + def _close_handle(self, handle): + """ + Close a handle, which was opened by :func:`_create_handle`. + + Parameters + ---------- + handle: str + Which handle to close. + """ + try: + self._send_to_api(method="post", endpoint="close", json={"handle": handle}) + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + raise FileNotFoundError(e.message) from e + + raise + + def _add_data(self, handle, data): + """ + Upload data to an already opened file handle + (opened by :func:`_create_handle`). + The maximal allowed data size is 1MB after + conversion to base64. + Remember to close the handle when you are finished. + + Parameters + ---------- + handle: str + Which handle to upload data to. + data: bytes + Block of data to add to the handle. + """ + data = base64.b64encode(data).decode() + try: + self._send_to_api( + method="post", + endpoint="add-block", + json={"handle": handle, "data": data}, + ) + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + raise FileNotFoundError(e.message) from e + elif e.error_code == "MAX_BLOCK_SIZE_EXCEEDED": + raise ValueError(e.message) from e + + raise + + def _get_data(self, path, start, end): + """ + Download data in bytes from a given absolute path in a block + from [start, start+length]. + The maximum number of allowed bytes to read is 1MB. + + Parameters + ---------- + path: str + Absolute path to download data from + start: int + Start position of the block + end: int + End position of the block + """ + try: + r = self._send_to_api( + method="get", + endpoint="read", + json={"path": path, "offset": start, "length": end - start}, + ) + return base64.b64decode(r["data"]) + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + raise FileNotFoundError(e.message) from e + elif e.error_code in ["INVALID_PARAMETER_VALUE", "MAX_READ_SIZE_EXCEEDED"]: + raise ValueError(e.message) from e + + raise + + def invalidate_cache(self, path=None): + if path is None: + self.dircache.clear() + else: + self.dircache.pop(path, None) + super().invalidate_cache(path) + + +class DatabricksFile(AbstractBufferedFile): + """ + Helper class for files referenced in the DatabricksFileSystem. + """ + + DEFAULT_BLOCK_SIZE = 1 * 2**20 # only allowed block size + + def __init__( + self, + fs, + path, + mode="rb", + block_size="default", + autocommit=True, + cache_type="readahead", + cache_options=None, + **kwargs, + ): + """ + Create a new instance of the DatabricksFile. + + The blocksize needs to be the default one. + """ + if block_size is None or block_size == "default": + block_size = self.DEFAULT_BLOCK_SIZE + + if block_size != self.DEFAULT_BLOCK_SIZE: + raise ValueError( + f"Only the default block size is allowed, not {block_size}" + ) + + super().__init__( + fs, + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_type=cache_type, + cache_options=cache_options or {}, + **kwargs, + ) + + def _initiate_upload(self): + """Internal function to start a file upload""" + self.handle = self.fs._create_handle(self.path) + + def _upload_chunk(self, final=False): + """Internal function to add a chunk of data to a started upload""" + self.buffer.seek(0) + data = self.buffer.getvalue() + + data_chunks = [ + data[start:end] for start, end in self._to_sized_blocks(len(data)) + ] + + for data_chunk in data_chunks: + self.fs._add_data(handle=self.handle, data=data_chunk) + + if final: + self.fs._close_handle(handle=self.handle) + return True + + def _fetch_range(self, start, end): + """Internal function to download a block of data""" + return_buffer = b"" + length = end - start + for chunk_start, chunk_end in self._to_sized_blocks(length, start): + return_buffer += self.fs._get_data( + path=self.path, start=chunk_start, end=chunk_end + ) + + return return_buffer + + def _to_sized_blocks(self, length, start=0): + """Helper function to split a range from 0 to total_length into blocksizes""" + end = start + length + for data_chunk in range(start, end, self.blocksize): + data_start = data_chunk + data_end = min(end, data_chunk + self.blocksize) + yield data_start, data_end diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/dirfs.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/dirfs.py new file mode 100644 index 0000000000000000000000000000000000000000..65b9b5da1219eb3a2e642c6dfb7f508f629d4e37 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/dirfs.py @@ -0,0 +1,404 @@ +from .. import filesystem +from ..asyn import AsyncFileSystem +from .chained import ChainedFileSystem + + +class DirFileSystem(AsyncFileSystem, ChainedFileSystem): + """Directory prefix filesystem + + The DirFileSystem is a filesystem-wrapper. It assumes every path it is dealing with + is relative to the `path`. After performing the necessary paths operation it + delegates everything to the wrapped filesystem. + """ + + protocol = "dir" + + def __init__( + self, + path=None, + fs=None, + fo=None, + target_protocol=None, + target_options=None, + **storage_options, + ): + """ + Parameters + ---------- + path: str + Path to the directory. + fs: AbstractFileSystem + An instantiated filesystem to wrap. + target_protocol, target_options: + if fs is none, construct it from these + fo: str + Alternate for path; do not provide both + """ + super().__init__(**storage_options) + if fs is None: + fs = filesystem(protocol=target_protocol, **(target_options or {})) + path = path or fo + + if self.asynchronous and not fs.async_impl: + raise ValueError("can't use asynchronous with non-async fs") + + if fs.async_impl and self.asynchronous != fs.asynchronous: + raise ValueError("both dirfs and fs should be in the same sync/async mode") + + self.path = fs._strip_protocol(path) + self.fs = fs + + def _join(self, path): + if isinstance(path, str): + if not self.path: + return path + if not path: + return self.path + return self.fs.sep.join((self.path, self._strip_protocol(path))) + if isinstance(path, dict): + return {self._join(_path): value for _path, value in path.items()} + return [self._join(_path) for _path in path] + + def _relpath(self, path): + if isinstance(path, str): + if not self.path: + return path + # We need to account for S3FileSystem returning paths that do not + # start with a '/' + if path == self.path or ( + self.path.startswith(self.fs.sep) and path == self.path[1:] + ): + return "" + prefix = self.path + self.fs.sep + if self.path.startswith(self.fs.sep) and not path.startswith(self.fs.sep): + prefix = prefix[1:] + assert path.startswith(prefix) + return path[len(prefix) :] + return [self._relpath(_path) for _path in path] + + # Wrappers below + + @property + def sep(self): + return self.fs.sep + + async def set_session(self, *args, **kwargs): + return await self.fs.set_session(*args, **kwargs) + + async def _rm_file(self, path, **kwargs): + return await self.fs._rm_file(self._join(path), **kwargs) + + def rm_file(self, path, **kwargs): + return self.fs.rm_file(self._join(path), **kwargs) + + async def _rm(self, path, *args, **kwargs): + return await self.fs._rm(self._join(path), *args, **kwargs) + + def rm(self, path, *args, **kwargs): + return self.fs.rm(self._join(path), *args, **kwargs) + + def delete(self, path, recursive=False, maxdepth=None): + return self.fs.delete(self._join(path), recursive=recursive, maxdepth=maxdepth) + + async def _cp_file(self, path1, path2, **kwargs): + return await self.fs._cp_file(self._join(path1), self._join(path2), **kwargs) + + def cp_file(self, path1, path2, **kwargs): + return self.fs.cp_file(self._join(path1), self._join(path2), **kwargs) + + async def _copy( + self, + path1, + path2, + *args, + **kwargs, + ): + return await self.fs._copy( + self._join(path1), + self._join(path2), + *args, + **kwargs, + ) + + def copy(self, path1, path2, *args, **kwargs): + return self.fs.copy( + self._join(path1), + self._join(path2), + *args, + **kwargs, + ) + + async def _pipe(self, path, *args, **kwargs): + return await self.fs._pipe(self._join(path), *args, **kwargs) + + def pipe(self, path, *args, **kwargs): + return self.fs.pipe(self._join(path), *args, **kwargs) + + async def _pipe_file(self, path, *args, **kwargs): + return await self.fs._pipe_file(self._join(path), *args, **kwargs) + + def pipe_file(self, path, *args, **kwargs): + return self.fs.pipe_file(self._join(path), *args, **kwargs) + + def write_text( + self, path, value, encoding=None, errors=None, newline=None, **kwargs + ): + return self.fs.write_text( + self._join(path), + value, + encoding=encoding, + errors=errors, + newline=newline, + **kwargs, + ) + + async def _cat_file(self, path, *args, **kwargs): + return await self.fs._cat_file(self._join(path), *args, **kwargs) + + def cat_file(self, path, *args, **kwargs): + return self.fs.cat_file(self._join(path), *args, **kwargs) + + async def _cat(self, path, *args, **kwargs): + ret = await self.fs._cat( + self._join(path), + *args, + **kwargs, + ) + + if isinstance(ret, dict): + return {self._relpath(key): value for key, value in ret.items()} + + return ret + + def cat(self, path, *args, **kwargs): + ret = self.fs.cat( + self._join(path), + *args, + **kwargs, + ) + + if isinstance(ret, dict): + return {self._relpath(key): value for key, value in ret.items()} + + return ret + + async def _put_file(self, lpath, rpath, **kwargs): + return await self.fs._put_file(lpath, self._join(rpath), **kwargs) + + def put_file(self, lpath, rpath, **kwargs): + return self.fs.put_file(lpath, self._join(rpath), **kwargs) + + async def _put( + self, + lpath, + rpath, + *args, + **kwargs, + ): + return await self.fs._put( + lpath, + self._join(rpath), + *args, + **kwargs, + ) + + def put(self, lpath, rpath, *args, **kwargs): + return self.fs.put( + lpath, + self._join(rpath), + *args, + **kwargs, + ) + + async def _get_file(self, rpath, lpath, **kwargs): + return await self.fs._get_file(self._join(rpath), lpath, **kwargs) + + def get_file(self, rpath, lpath, **kwargs): + return self.fs.get_file(self._join(rpath), lpath, **kwargs) + + async def _get(self, rpath, *args, **kwargs): + return await self.fs._get(self._join(rpath), *args, **kwargs) + + def get(self, rpath, *args, **kwargs): + return self.fs.get(self._join(rpath), *args, **kwargs) + + async def _isfile(self, path): + return await self.fs._isfile(self._join(path)) + + def isfile(self, path): + return self.fs.isfile(self._join(path)) + + async def _isdir(self, path): + return await self.fs._isdir(self._join(path)) + + def isdir(self, path): + return self.fs.isdir(self._join(path)) + + async def _size(self, path): + return await self.fs._size(self._join(path)) + + def size(self, path): + return self.fs.size(self._join(path)) + + async def _exists(self, path): + return await self.fs._exists(self._join(path)) + + def exists(self, path): + return self.fs.exists(self._join(path)) + + async def _info(self, path, **kwargs): + info = await self.fs._info(self._join(path), **kwargs) + info = info.copy() + info["name"] = self._relpath(info["name"]) + return info + + def info(self, path, **kwargs): + info = self.fs.info(self._join(path), **kwargs) + info = info.copy() + info["name"] = self._relpath(info["name"]) + return info + + async def _ls(self, path, detail=True, **kwargs): + ret = (await self.fs._ls(self._join(path), detail=detail, **kwargs)).copy() + if detail: + out = [] + for entry in ret: + entry = entry.copy() + entry["name"] = self._relpath(entry["name"]) + out.append(entry) + return out + + return self._relpath(ret) + + def ls(self, path, detail=True, **kwargs): + ret = self.fs.ls(self._join(path), detail=detail, **kwargs).copy() + if detail: + out = [] + for entry in ret: + entry = entry.copy() + entry["name"] = self._relpath(entry["name"]) + out.append(entry) + return out + + return self._relpath(ret) + + async def _walk(self, path, *args, **kwargs): + async for root, dirs, files in self.fs._walk(self._join(path), *args, **kwargs): + yield self._relpath(root), dirs, files + + def walk(self, path, *args, **kwargs): + for root, dirs, files in self.fs.walk(self._join(path), *args, **kwargs): + yield self._relpath(root), dirs, files + + async def _glob(self, path, **kwargs): + detail = kwargs.get("detail", False) + ret = await self.fs._glob(self._join(path), **kwargs) + if detail: + return {self._relpath(path): info for path, info in ret.items()} + return self._relpath(ret) + + def glob(self, path, **kwargs): + detail = kwargs.get("detail", False) + ret = self.fs.glob(self._join(path), **kwargs) + if detail: + return {self._relpath(path): info for path, info in ret.items()} + return self._relpath(ret) + + async def _du(self, path, *args, **kwargs): + total = kwargs.get("total", True) + ret = await self.fs._du(self._join(path), *args, **kwargs) + if total: + return ret + + return {self._relpath(path): size for path, size in ret.items()} + + def du(self, path, *args, **kwargs): + total = kwargs.get("total", True) + ret = self.fs.du(self._join(path), *args, **kwargs) + if total: + return ret + + return {self._relpath(path): size for path, size in ret.items()} + + async def _find(self, path, *args, **kwargs): + detail = kwargs.get("detail", False) + ret = await self.fs._find(self._join(path), *args, **kwargs) + if detail: + return {self._relpath(path): info for path, info in ret.items()} + return self._relpath(ret) + + def find(self, path, *args, **kwargs): + detail = kwargs.get("detail", False) + ret = self.fs.find(self._join(path), *args, **kwargs) + if detail: + return {self._relpath(path): info for path, info in ret.items()} + return self._relpath(ret) + + async def _expand_path(self, path, *args, **kwargs): + return self._relpath( + await self.fs._expand_path(self._join(path), *args, **kwargs) + ) + + def expand_path(self, path, *args, **kwargs): + return self._relpath(self.fs.expand_path(self._join(path), *args, **kwargs)) + + async def _mkdir(self, path, *args, **kwargs): + return await self.fs._mkdir(self._join(path), *args, **kwargs) + + def mkdir(self, path, *args, **kwargs): + return self.fs.mkdir(self._join(path), *args, **kwargs) + + async def _makedirs(self, path, *args, **kwargs): + return await self.fs._makedirs(self._join(path), *args, **kwargs) + + def makedirs(self, path, *args, **kwargs): + return self.fs.makedirs(self._join(path), *args, **kwargs) + + def rmdir(self, path): + return self.fs.rmdir(self._join(path)) + + def mv(self, path1, path2, **kwargs): + return self.fs.mv( + self._join(path1), + self._join(path2), + **kwargs, + ) + + def touch(self, path, **kwargs): + return self.fs.touch(self._join(path), **kwargs) + + def created(self, path): + return self.fs.created(self._join(path)) + + def modified(self, path): + return self.fs.modified(self._join(path)) + + def sign(self, path, *args, **kwargs): + return self.fs.sign(self._join(path), *args, **kwargs) + + def __repr__(self): + return f"{self.__class__.__qualname__}(path='{self.path}', fs={self.fs})" + + def open( + self, + path, + *args, + **kwargs, + ): + return self.fs.open( + self._join(path), + *args, + **kwargs, + ) + + async def open_async( + self, + path, + *args, + **kwargs, + ): + return await self.fs.open_async( + self._join(path), + *args, + **kwargs, + ) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/ftp.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/ftp.py new file mode 100644 index 0000000000000000000000000000000000000000..053472e0474aa89200f37e0293b358fcc62d1dea --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/ftp.py @@ -0,0 +1,442 @@ +import os +import ssl +import uuid +from ftplib import FTP, FTP_TLS, Error, error_perm +from typing import Any + +from ..spec import AbstractBufferedFile, AbstractFileSystem +from ..utils import infer_storage_options, isfilelike + +SECURITY_PROTOCOL_MAP = { + "tls": ssl.PROTOCOL_TLS, + "tlsv1": ssl.PROTOCOL_TLSv1, + "tlsv1_1": ssl.PROTOCOL_TLSv1_1, + "tlsv1_2": ssl.PROTOCOL_TLSv1_2, + "sslv23": ssl.PROTOCOL_SSLv23, +} + + +class ImplicitFTPTLS(FTP_TLS): + """ + FTP_TLS subclass that automatically wraps sockets in SSL + to support implicit FTPS. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._sock = None + + @property + def sock(self): + """Return the socket.""" + return self._sock + + @sock.setter + def sock(self, value): + """When modifying the socket, ensure that it is ssl wrapped.""" + if value is not None and not isinstance(value, ssl.SSLSocket): + value = self.context.wrap_socket(value) + self._sock = value + + +class FTPFileSystem(AbstractFileSystem): + """A filesystem over classic FTP""" + + root_marker = "/" + cachable = False + protocol = "ftp" + + def __init__( + self, + host, + port=21, + username=None, + password=None, + acct=None, + block_size=None, + tempdir=None, + timeout=30, + encoding="utf-8", + tls=False, + **kwargs, + ): + """ + You can use _get_kwargs_from_urls to get some kwargs from + a reasonable FTP url. + + Authentication will be anonymous if username/password are not + given. + + Parameters + ---------- + host: str + The remote server name/ip to connect to + port: int + Port to connect with + username: str or None + If authenticating, the user's identifier + password: str of None + User's password on the server, if using + acct: str or None + Some servers also need an "account" string for auth + block_size: int or None + If given, the read-ahead or write buffer size. + tempdir: str + Directory on remote to put temporary files when in a transaction + timeout: int + Timeout of the ftp connection in seconds + encoding: str + Encoding to use for directories and filenames in FTP connection + tls: bool or str + Enable FTP-TLS for secure connections: + - False: Plain FTP (default) + - True: Explicit TLS (FTPS with AUTH TLS command) + - "tls": Auto-negotiate highest protocol + - "tlsv1": TLS v1.0 + - "tlsv1_1": TLS v1.1 + - "tlsv1_2": TLS v1.2 + """ + super().__init__(**kwargs) + self.host = host + self.port = port + self.tempdir = tempdir or "/tmp" + self.cred = username or "", password or "", acct or "" + self.timeout = timeout + self.encoding = encoding + if block_size is not None: + self.blocksize = block_size + else: + self.blocksize = 2**16 + self.tls = tls + self._connect() + if isinstance(self.tls, bool) and self.tls: + self.ftp.prot_p() + + def _connect(self): + security = None + if self.tls: + if isinstance(self.tls, str): + ftp_cls = ImplicitFTPTLS + security = SECURITY_PROTOCOL_MAP.get( + self.tls, + f"Not supported {self.tls} protocol", + ) + if isinstance(security, str): + raise ValueError(security) + else: + ftp_cls = FTP_TLS + else: + ftp_cls = FTP + self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding) + if security: + self.ftp.ssl_version = security + self.ftp.connect(self.host, self.port) + self.ftp.login(*self.cred) + + @classmethod + def _strip_protocol(cls, path): + return "/" + infer_storage_options(path)["path"].lstrip("/").rstrip("/") + + @staticmethod + def _get_kwargs_from_urls(urlpath): + out = infer_storage_options(urlpath) + out.pop("path", None) + out.pop("protocol", None) + return out + + def ls(self, path, detail=True, **kwargs): + path = self._strip_protocol(path) + out = [] + if path not in self.dircache: + try: + try: + out = [ + (fn, details) + for (fn, details) in self.ftp.mlsd(path) + if fn not in [".", ".."] + and details["type"] not in ["pdir", "cdir"] + ] + except error_perm: + out = _mlsd2(self.ftp, path) # Not platform independent + for fn, details in out: + details["name"] = "/".join( + ["" if path == "/" else path, fn.lstrip("/")] + ) + if details["type"] == "file": + details["size"] = int(details["size"]) + else: + details["size"] = 0 + if details["type"] == "dir": + details["type"] = "directory" + self.dircache[path] = out + except Error: + try: + info = self.info(path) + if info["type"] == "file": + out = [(path, info)] + except (Error, IndexError) as exc: + raise FileNotFoundError(path) from exc + files = self.dircache.get(path, out) + if not detail: + return sorted([fn for fn, details in files]) + return [details for fn, details in files] + + def info(self, path, **kwargs): + # implement with direct method + path = self._strip_protocol(path) + if path == "/": + # special case, since this dir has no real entry + return {"name": "/", "size": 0, "type": "directory"} + files = self.ls(self._parent(path).lstrip("/"), True) + try: + out = next(f for f in files if f["name"] == path) + except StopIteration as exc: + raise FileNotFoundError(path) from exc + return out + + def get_file(self, rpath, lpath, **kwargs): + if self.isdir(rpath): + if not os.path.exists(lpath): + os.mkdir(lpath) + return + if isfilelike(lpath): + outfile = lpath + else: + outfile = open(lpath, "wb") + + def cb(x): + outfile.write(x) + + self.ftp.retrbinary( + f"RETR {rpath}", + blocksize=self.blocksize, + callback=cb, + ) + if not isfilelike(lpath): + outfile.close() + + def cat_file(self, path, start=None, end=None, **kwargs): + if end is not None: + return super().cat_file(path, start, end, **kwargs) + out = [] + + def cb(x): + out.append(x) + + try: + self.ftp.retrbinary( + f"RETR {path}", + blocksize=self.blocksize, + rest=start, + callback=cb, + ) + except (Error, error_perm) as orig_exc: + raise FileNotFoundError(path) from orig_exc + return b"".join(out) + + def _open( + self, + path, + mode="rb", + block_size=None, + cache_options=None, + autocommit=True, + **kwargs, + ): + path = self._strip_protocol(path) + block_size = block_size or self.blocksize + return FTPFile( + self, + path, + mode=mode, + block_size=block_size, + tempdir=self.tempdir, + autocommit=autocommit, + cache_options=cache_options, + ) + + def _rm(self, path): + path = self._strip_protocol(path) + self.ftp.delete(path) + self.invalidate_cache(self._parent(path)) + + def rm(self, path, recursive=False, maxdepth=None): + paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) + for p in reversed(paths): + if self.isfile(p): + self.rm_file(p) + else: + self.rmdir(p) + + def mkdir(self, path: str, create_parents: bool = True, **kwargs: Any) -> None: + path = self._strip_protocol(path) + parent = self._parent(path) + if parent != self.root_marker and not self.exists(parent) and create_parents: + self.mkdir(parent, create_parents=create_parents) + + self.ftp.mkd(path) + self.invalidate_cache(self._parent(path)) + + def makedirs(self, path: str, exist_ok: bool = False) -> None: + path = self._strip_protocol(path) + if self.exists(path): + # NB: "/" does not "exist" as it has no directory entry + if not exist_ok: + raise FileExistsError(f"{path} exists without `exist_ok`") + # exists_ok=True -> no-op + else: + self.mkdir(path, create_parents=True) + + def rmdir(self, path): + path = self._strip_protocol(path) + self.ftp.rmd(path) + self.invalidate_cache(self._parent(path)) + + def mv(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1) + path2 = self._strip_protocol(path2) + self.ftp.rename(path1, path2) + self.invalidate_cache(self._parent(path1)) + self.invalidate_cache(self._parent(path2)) + + def __del__(self): + self.ftp.close() + + def invalidate_cache(self, path=None): + if path is None: + self.dircache.clear() + else: + self.dircache.pop(path, None) + super().invalidate_cache(path) + + +class TransferDone(Exception): + """Internal exception to break out of transfer""" + + pass + + +class FTPFile(AbstractBufferedFile): + """Interact with a remote FTP file with read/write buffering""" + + def __init__( + self, + fs, + path, + mode="rb", + block_size="default", + autocommit=True, + cache_type="readahead", + cache_options=None, + **kwargs, + ): + super().__init__( + fs, + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_type=cache_type, + cache_options=cache_options, + **kwargs, + ) + if not autocommit: + self.target = self.path + self.path = "/".join([kwargs["tempdir"], str(uuid.uuid4())]) + + def commit(self): + self.fs.mv(self.path, self.target) + + def discard(self): + self.fs.rm(self.path) + + def _fetch_range(self, start, end): + """Get bytes between given byte limits + + Implemented by raising an exception in the fetch callback when the + number of bytes received reaches the requested amount. + + Will fail if the server does not respect the REST command on + retrieve requests. + """ + out = [] + total = [0] + + def callback(x): + total[0] += len(x) + if total[0] > end - start: + out.append(x[: (end - start) - total[0]]) + if end < self.size: + raise TransferDone + else: + out.append(x) + + if total[0] == end - start and end < self.size: + raise TransferDone + + try: + self.fs.ftp.retrbinary( + f"RETR {self.path}", + blocksize=self.blocksize, + rest=start, + callback=callback, + ) + except TransferDone: + try: + # stop transfer, we got enough bytes for this block + self.fs.ftp.abort() + self.fs.ftp.getmultiline() + except Error: + self.fs._connect() + + return b"".join(out) + + def _upload_chunk(self, final=False): + self.buffer.seek(0) + self.fs.ftp.storbinary( + f"STOR {self.path}", self.buffer, blocksize=self.blocksize, rest=self.offset + ) + return True + + +def _mlsd2(ftp, path="."): + """ + Fall back to using `dir` instead of `mlsd` if not supported. + + This parses a Linux style `ls -l` response to `dir`, but the response may + be platform dependent. + + Parameters + ---------- + ftp: ftplib.FTP + path: str + Expects to be given path, but defaults to ".". + """ + lines = [] + minfo = [] + ftp.dir(path, lines.append) + for line in lines: + split_line = line.split(maxsplit=8) + if len(split_line) < 9: + continue + name = split_line[8] + unix_mode = split_line[0] + if unix_mode[0] == "l" and " -> " in name: + # Symbolic link: "<name> -> <target>"; keep only the link name. + name = name.split(" -> ", 1)[0] + this = ( + name, + { + "modify": " ".join(split_line[5:8]), + "unix.owner": split_line[2], + "unix.group": split_line[3], + "unix.mode": unix_mode, + "size": split_line[4], + }, + ) + if this[1]["unix.mode"][0] == "d": + this[1]["type"] = "dir" + else: + this[1]["type"] = "file" + minfo.append(this) + return minfo diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/gist.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/gist.py new file mode 100644 index 0000000000000000000000000000000000000000..ad9ac0b6a1cdbcfba6188e2cdeab2350bb9aad0a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/gist.py @@ -0,0 +1,241 @@ +import requests + +from ..spec import AbstractFileSystem +from ..utils import infer_storage_options +from .memory import MemoryFile + + +class GistFileSystem(AbstractFileSystem): + """ + Interface to files in a single GitHub Gist. + + Provides read-only access to a gist's files. Gists do not contain + subdirectories, so file listing is straightforward. + + Parameters + ---------- + gist_id: str + The ID of the gist you want to access (the long hex value from the URL). + filenames: list[str] (optional) + If provided, only make a file system representing these files, and do not fetch + the list of all files for this gist. + sha: str (optional) + If provided, fetch a particular revision of the gist. If omitted, + the latest revision is used. + username: str (optional) + GitHub username for authentication. + token: str (optional) + GitHub personal access token (required if username is given), or. + timeout: (float, float) or float, optional + Connect and read timeouts for requests (default 60s each). + kwargs: dict + Stored on `self.request_kw` and passed to `requests.get` when fetching Gist + metadata or reading ("opening") a file. + """ + + protocol = "gist" + gist_url = "https://api.github.com/gists/{gist_id}" + gist_rev_url = "https://api.github.com/gists/{gist_id}/{sha}" + + def __init__( + self, + gist_id, + filenames=None, + sha=None, + username=None, + token=None, + timeout=None, + **kwargs, + ): + super().__init__() + self.gist_id = gist_id + self.filenames = filenames + self.sha = sha # revision of the gist (optional) + if username is not None and token is None: + raise ValueError("User auth requires a token") + self.username = username + self.token = token + self.request_kw = kwargs + # Default timeouts to 60s connect/read if none provided + self.timeout = timeout if timeout is not None else (60, 60) + + # We use a single-level "directory" cache, because a gist is essentially flat + self.dircache[""] = self._fetch_file_list() + + @property + def kw(self): + """Auth parameters passed to 'requests' if we have username/token.""" + kw = { + "headers": { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + } + kw.update(self.request_kw) + if self.username and self.token: + kw["auth"] = (self.username, self.token) + elif self.token: + kw["headers"]["Authorization"] = f"Bearer {self.token}" + return kw + + def _fetch_gist_metadata(self): + """ + Fetch the JSON metadata for this gist (possibly for a specific revision). + """ + if self.sha: + url = self.gist_rev_url.format(gist_id=self.gist_id, sha=self.sha) + else: + url = self.gist_url.format(gist_id=self.gist_id) + + r = requests.get(url, timeout=self.timeout, **self.kw) + if r.status_code == 404: + raise FileNotFoundError( + f"Gist not found: {self.gist_id}@{self.sha or 'latest'}" + ) + r.raise_for_status() + return r.json() + + def _fetch_file_list(self): + """ + Returns a list of dicts describing each file in the gist. These get stored + in self.dircache[""]. + """ + meta = self._fetch_gist_metadata() + if self.filenames: + available_files = meta.get("files", {}) + files = {} + for fn in self.filenames: + if fn not in available_files: + raise FileNotFoundError(fn) + files[fn] = available_files[fn] + else: + files = meta.get("files", {}) + + out = [] + for fname, finfo in files.items(): + if finfo is None: + # Occasionally GitHub returns a file entry with null if it was deleted + continue + # Build a directory entry + out.append( + { + "name": fname, # file's name + "type": "file", # gists have no subdirectories + "size": finfo.get("size", 0), # file size in bytes + "raw_url": finfo.get("raw_url"), + } + ) + return out + + @classmethod + def _strip_protocol(cls, path): + """ + Remove 'gist://' from the path, if present. + """ + # The default infer_storage_options can handle gist://username:token@id/file + # or gist://id/file, but let's ensure we handle a normal usage too. + # We'll just strip the protocol prefix if it exists. + path = infer_storage_options(path).get("path", path) + return path.lstrip("/") + + @staticmethod + def _get_kwargs_from_urls(path): + """ + Parse 'gist://' style URLs into GistFileSystem constructor kwargs. + For example: + gist://:TOKEN@<gist_id>/file.txt + gist://username:TOKEN@<gist_id>/file.txt + """ + so = infer_storage_options(path) + out = {} + if "username" in so and so["username"]: + out["username"] = so["username"] + if "password" in so and so["password"]: + out["token"] = so["password"] + if "host" in so and so["host"]: + # We interpret 'host' as the gist ID + out["gist_id"] = so["host"] + + # Extract SHA and filename from path + if "path" in so and so["path"]: + path_parts = so["path"].rsplit("/", 2)[-2:] + if len(path_parts) == 2: + if path_parts[0]: # SHA present + out["sha"] = path_parts[0] + if path_parts[1]: # filename also present + out["filenames"] = [path_parts[1]] + + return out + + def ls(self, path="", detail=False, **kwargs): + """ + List files in the gist. Gists are single-level, so any 'path' is basically + the filename, or empty for all files. + + Parameters + ---------- + path : str, optional + The filename to list. If empty, returns all files in the gist. + detail : bool, default False + If True, return a list of dicts; if False, return a list of filenames. + """ + path = self._strip_protocol(path or "") + # If path is empty, return all + if path == "": + results = self.dircache[""] + else: + # We want just the single file with this name + all_files = self.dircache[""] + results = [f for f in all_files if f["name"] == path] + if not results: + raise FileNotFoundError(path) + if detail: + return results + else: + return sorted(f["name"] for f in results) + + def _open(self, path, mode="rb", block_size=None, **kwargs): + """ + Read a single file from the gist. + """ + if mode != "rb": + raise NotImplementedError("GitHub Gist FS is read-only (no write).") + + path = self._strip_protocol(path) + # Find the file entry in our dircache + matches = [f for f in self.dircache[""] if f["name"] == path] + if not matches: + raise FileNotFoundError(path) + finfo = matches[0] + + raw_url = finfo.get("raw_url") + if not raw_url: + raise FileNotFoundError(f"No raw_url for file: {path}") + + r = requests.get(raw_url, timeout=self.timeout, **self.kw) + if r.status_code == 404: + raise FileNotFoundError(path) + r.raise_for_status() + return MemoryFile(path, None, r.content) + + def cat(self, path, recursive=False, on_error="raise", **kwargs): + """ + Return {path: contents} for the given file or files. If 'recursive' is True, + and path is empty, returns all files in the gist. + """ + paths = self.expand_path(path, recursive=recursive) + out = {} + for p in paths: + try: + with self.open(p, "rb") as f: + out[p] = f.read() + except FileNotFoundError as e: + if on_error == "raise": + raise e + elif on_error == "omit": + pass # skip + else: + out[p] = e + if len(paths) == 1 and paths[0] == path: + return out[path] + return out diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/git.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/git.py new file mode 100644 index 0000000000000000000000000000000000000000..808d293a1c991ea87d19a2129f3e56d9b813daaa --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/git.py @@ -0,0 +1,114 @@ +import os + +import pygit2 + +from fsspec.spec import AbstractFileSystem + +from .memory import MemoryFile + + +class GitFileSystem(AbstractFileSystem): + """Browse the files of a local git repo at any hash/tag/branch + + (experimental backend) + """ + + root_marker = "" + cachable = True + + def __init__(self, path=None, fo=None, ref=None, **kwargs): + """ + + Parameters + ---------- + path: str (optional) + Local location of the repo (uses current directory if not given). + May be deprecated in favour of ``fo``. When used with a higher + level function such as fsspec.open(), may be of the form + "git://[path-to-repo[:]][ref@]path/to/file" (but the actual + file path should not contain "@" or ":"). + fo: str (optional) + Same as ``path``, but passed as part of a chained URL. This one + takes precedence if both are given. + ref: str (optional) + Reference to work with, could be a hash, tag or branch name. Defaults + to current working tree. Note that ``ls`` and ``open`` also take hash, + so this becomes the default for those operations + kwargs + """ + super().__init__(**kwargs) + self.repo = pygit2.Repository(fo or path or os.getcwd()) + self.ref = ref or "master" + + @classmethod + def _strip_protocol(cls, path): + path = super()._strip_protocol(path).lstrip("/") + if ":" in path: + path = path.split(":", 1)[1] + if "@" in path: + path = path.split("@", 1)[1] + return path.lstrip("/") + + def _path_to_object(self, path, ref): + comm, ref = self.repo.resolve_refish(ref or self.ref) + parts = path.split("/") + tree = comm.tree + for part in parts: + if part and isinstance(tree, pygit2.Tree): + if part not in tree: + raise FileNotFoundError(path) + tree = tree[part] + return tree + + @staticmethod + def _get_kwargs_from_urls(path): + path = path.removeprefix("git://") + out = {} + if ":" in path: + out["path"], path = path.split(":", 1) + if "@" in path: + out["ref"], path = path.split("@", 1) + return out + + @staticmethod + def _object_to_info(obj, path=None): + # obj.name and obj.filemode are None for the root tree! + is_dir = isinstance(obj, pygit2.Tree) + return { + "type": "directory" if is_dir else "file", + "name": ( + "/".join([path, obj.name or ""]).lstrip("/") if path else obj.name + ), + "hex": str(obj.id), + "mode": "100644" if obj.filemode is None else f"{obj.filemode:o}", + "size": 0 if is_dir else obj.size, + } + + def ls(self, path, detail=True, ref=None, **kwargs): + tree = self._path_to_object(self._strip_protocol(path), ref) + return [ + GitFileSystem._object_to_info(obj, path) + if detail + else GitFileSystem._object_to_info(obj, path)["name"] + for obj in (tree if isinstance(tree, pygit2.Tree) else [tree]) + ] + + def info(self, path, ref=None, **kwargs): + tree = self._path_to_object(self._strip_protocol(path), ref) + return GitFileSystem._object_to_info(tree, path) + + def ukey(self, path, ref=None): + return self.info(path, ref=ref)["hex"] + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + ref=None, + **kwargs, + ): + obj = self._path_to_object(path, ref or self.ref) + return MemoryFile(data=obj.data) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/github.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/github.py new file mode 100644 index 0000000000000000000000000000000000000000..3630f6db54413e2c396f6cc1b6b10cd379200043 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/github.py @@ -0,0 +1,333 @@ +import base64 +import re + +import requests + +from ..spec import AbstractFileSystem +from ..utils import infer_storage_options +from .memory import MemoryFile + + +class GithubFileSystem(AbstractFileSystem): + """Interface to files in github + + An instance of this class provides the files residing within a remote github + repository. You may specify a point in the repos history, by SHA, branch + or tag (default is current master). + + For files less than 1 MB in size, file content is returned directly in a + MemoryFile. For larger files, or for files tracked by git-lfs, file content + is returned as an HTTPFile wrapping the ``download_url`` provided by the + GitHub API. + + When using fsspec.open, allows URIs of the form: + + - "github://path/file", in which case you must specify org, repo and + may specify sha in the extra args + - 'github://org:repo@/precip/catalog.yml', where the org and repo are + part of the URI + - 'github://org:repo@sha/precip/catalog.yml', where the sha is also included + + ``sha`` can be the full or abbreviated hex of the commit you want to fetch + from, or a branch or tag name (so long as it doesn't contain special characters + like "/", "?", which would have to be HTTP-encoded). + + For authorised access, you must provide username and token, which can be made + at https://github.com/settings/tokens + """ + + url = "https://api.github.com/repos/{org}/{repo}/git/trees/{sha}" + content_url = "https://api.github.com/repos/{org}/{repo}/contents/{path}?ref={sha}" + protocol = "github" + timeout = (60, 60) # connect, read timeouts + + def __init__( + self, org, repo, sha=None, username=None, token=None, timeout=None, **kwargs + ): + super().__init__(**kwargs) + self.org = org + self.repo = repo + if (username is None) ^ (token is None): + raise ValueError("Auth required both username and token") + self.username = username + self.token = token + if timeout is not None: + self.timeout = timeout + if sha is None: + # look up default branch (not necessarily "master") + u = "https://api.github.com/repos/{org}/{repo}" + r = requests.get( + u.format(org=org, repo=repo), timeout=self.timeout, **self.kw + ) + r.raise_for_status() + sha = r.json()["default_branch"] + + self.root = sha + self.ls("") + try: + from .http import HTTPFileSystem + + self.http_fs = HTTPFileSystem(**kwargs) + except ImportError: + self.http_fs = None + + @property + def kw(self): + if self.username: + return {"auth": (self.username, self.token)} + return {} + + @classmethod + def repos(cls, org_or_user, is_org=True): + """List repo names for given org or user + + This may become the top level of the FS + + Parameters + ---------- + org_or_user: str + Name of the github org or user to query + is_org: bool (default True) + Whether the name is an organisation (True) or user (False) + + Returns + ------- + List of string + """ + r = requests.get( + f"https://api.github.com/{['users', 'orgs'][is_org]}/{org_or_user}/repos", + timeout=cls.timeout, + ) + r.raise_for_status() + return [repo["name"] for repo in r.json()] + + @property + def tags(self): + """Names of tags in the repo""" + r = requests.get( + f"https://api.github.com/repos/{self.org}/{self.repo}/tags", + timeout=self.timeout, + **self.kw, + ) + r.raise_for_status() + return [t["name"] for t in r.json()] + + @property + def branches(self): + """Names of branches in the repo""" + r = requests.get( + f"https://api.github.com/repos/{self.org}/{self.repo}/branches", + timeout=self.timeout, + **self.kw, + ) + r.raise_for_status() + return [t["name"] for t in r.json()] + + @property + def refs(self): + """Named references, tags and branches""" + return {"tags": self.tags, "branches": self.branches} + + def ls(self, path, detail=False, sha=None, _sha=None, **kwargs): + """List files at given path + + Parameters + ---------- + path: str + Location to list, relative to repo root + detail: bool + If True, returns list of dicts, one per file; if False, returns + list of full filenames only + sha: str (optional) + List at the given point in the repo history, branch or tag name or commit + SHA + _sha: str (optional) + List this specific tree object (used internally to descend into trees) + """ + path = self._strip_protocol(path) + if path == "": + _sha = sha or self.root + if _sha is None: + parts = path.rstrip("/").split("/") + so_far = "" + _sha = sha or self.root + for part in parts: + out = self.ls(so_far, True, sha=sha, _sha=_sha) + so_far += "/" + part if so_far else part + out = [o for o in out if o["name"] == so_far] + if not out: + raise FileNotFoundError(path) + out = out[0] + if out["type"] == "file": + if detail: + return [out] + else: + return path + _sha = out["sha"] + if path not in self.dircache or sha not in [self.root, None]: + r = requests.get( + self.url.format(org=self.org, repo=self.repo, sha=_sha), + timeout=self.timeout, + **self.kw, + ) + if r.status_code == 404: + raise FileNotFoundError(path) + r.raise_for_status() + types = {"blob": "file", "tree": "directory"} + out = [ + { + "name": path + "/" + f["path"] if path else f["path"], + "mode": f["mode"], + "type": types[f["type"]], + "size": f.get("size", 0), + "sha": f["sha"], + } + for f in r.json()["tree"] + if f["type"] in types + ] + if sha in [self.root, None]: + self.dircache[path] = out + else: + out = self.dircache[path] + if detail: + return out + else: + return sorted([f["name"] for f in out]) + + def invalidate_cache(self, path=None): + self.dircache.clear() + + @classmethod + def _strip_protocol(cls, path): + opts = infer_storage_options(path) + if "username" not in opts: + return super()._strip_protocol(path) + return opts["path"].lstrip("/") + + @staticmethod + def _get_kwargs_from_urls(path): + opts = infer_storage_options(path) + if "username" not in opts: + return {} + out = {"org": opts["username"], "repo": opts["password"]} + if opts["host"]: + out["sha"] = opts["host"] + return out + + def _open( + self, + path, + mode="rb", + block_size=None, + cache_options=None, + sha=None, + **kwargs, + ): + if mode != "rb": + raise NotImplementedError + + # construct a url to hit the GitHub API's repo contents API + url = self.content_url.format( + org=self.org, repo=self.repo, path=path, sha=sha or self.root + ) + + # make a request to this API, and parse the response as JSON + r = requests.get(url, timeout=self.timeout, **self.kw) + if r.status_code == 404: + raise FileNotFoundError(path) + r.raise_for_status() + content_json = r.json() + + # if the response's content key is not empty, try to parse it as base64 + if content_json["content"]: + content = base64.b64decode(content_json["content"]) + + # as long as the content does not start with the string + # "version https://git-lfs.github.com/" + # then it is probably not a git-lfs pointer and we can just return + # the content directly + if not content.startswith(b"version https://git-lfs.github.com/"): + return MemoryFile(None, None, content) + + # we land here if the content was not present in the first response + # (regular file over 1MB or git-lfs tracked file) + # in this case, we get let the HTTPFileSystem handle the download + if self.http_fs is None: + raise ImportError( + "Please install fsspec[http] to access github files >1 MB " + "or git-lfs tracked files." + ) + return self.http_fs.open( + content_json["download_url"], + mode=mode, + block_size=block_size, + cache_options=cache_options, + **kwargs, + ) + + def rm(self, path, recursive=False, maxdepth=None, message=None): + path = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) + for p in reversed(path): + self.rm_file(p, message=message) + + def rm_file(self, path, message=None, **kwargs): + """ + Remove a file from a specified branch using a given commit message. + + Since Github DELETE operation requires a branch name, and we can't reliably + determine whether the provided SHA refers to a branch, tag, or commit, we + assume it's a branch. If it's not, the user will encounter an error when + attempting to retrieve the file SHA or delete the file. + + Parameters + ---------- + path: str + The file's location relative to the repository root. + message: str, optional + The commit message for the deletion. + """ + + if not self.username: + raise ValueError("Authentication required") + + path = self._strip_protocol(path) + + # Attempt to get SHA from cache or Github API + sha = self._get_sha_from_cache(path) + if not sha: + url = self.content_url.format( + org=self.org, repo=self.repo, path=path.lstrip("/"), sha=self.root + ) + r = requests.get(url, timeout=self.timeout, **self.kw) + if r.status_code == 404: + raise FileNotFoundError(path) + r.raise_for_status() + sha = r.json()["sha"] + + # Delete the file + delete_url = self.content_url.format( + org=self.org, repo=self.repo, path=path, sha=self.root + ) + branch = self.root + data = { + "message": message or f"Delete {path}", + "sha": sha, + **({"branch": branch} if branch else {}), + } + + r = requests.delete(delete_url, json=data, timeout=self.timeout, **self.kw) + error_message = r.json().get("message", "") + if re.search(r"Branch .+ not found", error_message): + error = "Remove only works when the filesystem is initialised from a branch or default (None)" + raise ValueError(error) + r.raise_for_status() + + self.invalidate_cache(path) + + def _get_sha_from_cache(self, path): + for entries in self.dircache.values(): + for entry in entries: + entry_path = entry.get("name") + if entry_path and entry_path == path and "sha" in entry: + return entry["sha"] + return None diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/http.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/http.py new file mode 100644 index 0000000000000000000000000000000000000000..aff1a955c27091229513b988627de9e806dca8c5 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/http.py @@ -0,0 +1,902 @@ +import asyncio +import io +import logging +import re +import weakref +from copy import copy +from urllib.parse import urlparse + +import aiohttp +import yarl + +from fsspec.asyn import AbstractAsyncStreamedFile, AsyncFileSystem, sync, sync_wrapper +from fsspec.callbacks import DEFAULT_CALLBACK +from fsspec.exceptions import FSTimeoutError +from fsspec.spec import AbstractBufferedFile +from fsspec.utils import ( + DEFAULT_BLOCK_SIZE, + glob_translate, + isfilelike, + nullcontext, + tokenize, +) + +from ..caching import AllBytes + +# https://stackoverflow.com/a/15926317/3821154 +ex = re.compile(r"""<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P<url>[^"']+)""") +ex2 = re.compile(r"""(?P<url>http[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)""") +logger = logging.getLogger("fsspec.http") + + +async def get_client(**kwargs): + return aiohttp.ClientSession(**kwargs) + + +class HTTPFileSystem(AsyncFileSystem): + """ + Simple File-System for fetching data via HTTP(S) + + ``ls()`` is implemented by loading the parent page and doing a regex + match on the result. If simple_link=True, anything of the form + "http(s)://server.com/stuff?thing=other"; otherwise only links within + HTML href tags will be used. + + URLs are passed unfiltered to aiohttp, so all addresses are accessible. Where URLs are + supplied by a user, the calling application may wish to filter to prevent scanning. + """ + + protocol = ("http", "https") + sep = "/" + + def __init__( + self, + simple_links=True, + block_size=None, + same_scheme=True, + size_policy=None, + cache_type="bytes", + cache_options=None, + asynchronous=False, + loop=None, + client_kwargs=None, + get_client=get_client, + encoded=False, + **storage_options, + ): + """ + NB: if this is called async, you must await set_client + + Parameters + ---------- + block_size: int + Blocks to read bytes; if 0, will default to raw requests file-like + objects instead of HTTPFile instances + simple_links: bool + If True, will consider both HTML <a> tags and anything that looks + like a URL; if False, will consider only the former. + same_scheme: True + When doing ls/glob, if this is True, only consider paths that have + http/https matching the input URLs. + size_policy: this argument is deprecated + client_kwargs: dict + Passed to aiohttp.ClientSession, see + https://docs.aiohttp.org/en/stable/client_reference.html + For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}`` + get_client: Callable[..., aiohttp.ClientSession] + A callable, which takes keyword arguments and constructs + an aiohttp.ClientSession. Its state will be managed by + the HTTPFileSystem class. + storage_options: key-value + Any other parameters passed on to requests + cache_type, cache_options: defaults used in open() + """ + super().__init__(self, asynchronous=asynchronous, loop=loop, **storage_options) + self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE + self.simple_links = simple_links + self.same_schema = same_scheme + self.cache_type = cache_type + self.cache_options = cache_options + self.client_kwargs = client_kwargs or {} + self.get_client = get_client + self.encoded = encoded + self.kwargs = storage_options + self._session = None + + # Clean caching-related parameters from `storage_options` + # before propagating them as `request_options` through `self.kwargs`. + # TODO: Maybe rename `self.kwargs` to `self.request_options` to make + # it clearer. + request_options = copy(storage_options) + self.use_listings_cache = request_options.pop("use_listings_cache", False) + request_options.pop("listings_expiry_time", None) + request_options.pop("max_paths", None) + request_options.pop("skip_instance_cache", None) + self.kwargs = request_options + + @property + def fsid(self): + return "http" + + def encode_url(self, url): + return yarl.URL(url, encoded=self.encoded) + + @staticmethod + def close_session(loop, session): + if loop is not None and loop.is_running(): + try: + sync(loop, session.close, timeout=0.1) + return + except (TimeoutError, FSTimeoutError, NotImplementedError): + pass + connector = getattr(session, "_connector", None) + if connector is not None: + # close after loop is dead + connector._close() + + async def set_session(self): + if self._session is None: + self._session = await self.get_client(loop=self.loop, **self.client_kwargs) + if not self.asynchronous: + weakref.finalize(self, self.close_session, self.loop, self._session) + return self._session + + @classmethod + def _strip_protocol(cls, path): + """For HTTP, we always want to keep the full URL""" + return path + + @classmethod + def _parent(cls, path): + # override, since _strip_protocol is different for URLs + par = super()._parent(path) + if len(par) > 7: # "http://..." + return par + return "" + + async def _ls_real(self, url, detail=True, **kwargs): + # ignoring URL-encoded arguments + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(url) + session = await self.set_session() + async with session.get(self.encode_url(url), **self.kwargs) as r: + self._raise_not_found_for_status(r, url) + + if "Content-Type" in r.headers: + mimetype = r.headers["Content-Type"].partition(";")[0] + else: + mimetype = None + + if mimetype in ("text/html", None): + try: + text = await r.text(errors="ignore") + if self.simple_links: + links = ex2.findall(text) + [u[2] for u in ex.findall(text)] + else: + links = [u[2] for u in ex.findall(text)] + except UnicodeDecodeError: + links = [] # binary, not HTML + else: + links = [] + + out = set() + parts = urlparse(url) + for l in links: + if isinstance(l, tuple): + l = l[1] + if l.startswith("/") and len(l) > 1: + # absolute URL on this server + l = f"{parts.scheme}://{parts.netloc}{l}" + if l.startswith("http"): + if self.same_schema and l.startswith(url.rstrip("/") + "/"): + out.add(l) + elif l.replace("https", "http").startswith( + url.replace("https", "http").rstrip("/") + "/" + ): + # allowed to cross http <-> https + out.add(l) + else: + if l not in ["..", "../"]: + # Ignore FTP-like "parent" + out.add("/".join([url.rstrip("/"), l.lstrip("/")])) + if not out and url.endswith("/"): + out = await self._ls_real(url.rstrip("/"), detail=False) + if detail: + return [ + { + "name": u, + "size": None, + "type": "directory" if u.endswith("/") else "file", + } + for u in out + ] + else: + return sorted(out) + + async def _ls(self, url, detail=True, **kwargs): + if self.use_listings_cache and url in self.dircache: + out = self.dircache[url] + else: + out = await self._ls_real(url, detail=detail, **kwargs) + self.dircache[url] = out + return out + + ls = sync_wrapper(_ls) + + def _raise_not_found_for_status(self, response, url): + """ + Raises FileNotFoundError for 404s, otherwise uses raise_for_status. + """ + if response.status == 404: + raise FileNotFoundError(url) + response.raise_for_status() + + async def _cat_file(self, url, start=None, end=None, **kwargs): + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(url) + + if start is not None or end is not None: + if start == end: + return b"" + headers = kw.pop("headers", {}).copy() + + headers["Range"] = await self._process_limits(url, start, end) + kw["headers"] = headers + session = await self.set_session() + async with session.get(self.encode_url(url), **kw) as r: + out = await r.read() + self._raise_not_found_for_status(r, url) + return out + + async def _get_file( + self, rpath, lpath, chunk_size=5 * 2**20, callback=DEFAULT_CALLBACK, **kwargs + ): + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(rpath) + session = await self.set_session() + async with session.get(self.encode_url(rpath), **kw) as r: + try: + size = int(r.headers["content-length"]) + except (ValueError, KeyError): + size = None + + callback.set_size(size) + self._raise_not_found_for_status(r, rpath) + if isfilelike(lpath): + outfile = lpath + else: + outfile = open(lpath, "wb") # noqa: ASYNC230 + + try: + chunk = True + while chunk: + chunk = await r.content.read(chunk_size) + outfile.write(chunk) + callback.relative_update(len(chunk)) + finally: + if not isfilelike(lpath): + outfile.close() + + async def _put_file( + self, + lpath, + rpath, + chunk_size=5 * 2**20, + callback=DEFAULT_CALLBACK, + method="post", + mode="overwrite", + **kwargs, + ): + if mode != "overwrite": + raise NotImplementedError("Exclusive write") + + async def gen_chunks(): + # Support passing arbitrary file-like objects + # and use them instead of streams. + if isinstance(lpath, io.IOBase): + context = nullcontext(lpath) + use_seek = False # might not support seeking + else: + context = open(lpath, "rb") # noqa: ASYNC230 + use_seek = True + + with context as f: + if use_seek: + callback.set_size(f.seek(0, 2)) + f.seek(0) + else: + callback.set_size(getattr(f, "size", None)) + + chunk = f.read(chunk_size) + while chunk: + yield chunk + callback.relative_update(len(chunk)) + chunk = f.read(chunk_size) + + kw = self.kwargs.copy() + kw.update(kwargs) + session = await self.set_session() + + method = method.lower() + if method not in ("post", "put"): + raise ValueError( + f"method has to be either 'post' or 'put', not: {method!r}" + ) + + meth = getattr(session, method) + async with meth(self.encode_url(rpath), data=gen_chunks(), **kw) as resp: + self._raise_not_found_for_status(resp, rpath) + + async def _exists(self, path, strict=False, **kwargs): + kw = self.kwargs.copy() + kw.update(kwargs) + try: + logger.debug(path) + session = await self.set_session() + r = await session.get(self.encode_url(path), **kw) + async with r: + if strict: + self._raise_not_found_for_status(r, path) + return r.status < 400 + except FileNotFoundError: + return False + except aiohttp.ClientError: + if strict: + raise + return False + + async def _isfile(self, path, **kwargs): + return await self._exists(path, **kwargs) + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=None, # XXX: This differs from the base class. + cache_type=None, + cache_options=None, + size=None, + **kwargs, + ): + """Make a file-like object + + Parameters + ---------- + path: str + Full URL with protocol + mode: string + must be "rb" + block_size: int or None + Bytes to download in one request; use instance value if None. If + zero, will return a streaming Requests file-like instance. + kwargs: key-value + Any other parameters, passed to requests calls + """ + if mode != "rb": + raise NotImplementedError + block_size = block_size if block_size is not None else self.block_size + kw = self.kwargs.copy() + kw["asynchronous"] = self.asynchronous + kw.update(kwargs) + info = {} + size = size or info.update(self.info(path, **kwargs)) or info["size"] + session = sync(self.loop, self.set_session) + if block_size and size and info.get("partial", True): + return HTTPFile( + self, + path, + session=session, + block_size=block_size, + mode=mode, + size=size, + cache_type=cache_type or self.cache_type, + cache_options=cache_options or self.cache_options, + loop=self.loop, + **kw, + ) + else: + return HTTPStreamFile( + self, + path, + mode=mode, + loop=self.loop, + session=session, + **kw, + ) + + async def open_async(self, path, mode="rb", size=None, **kwargs): + session = await self.set_session() + if size is None: + try: + size = (await self._info(path, **kwargs))["size"] + except FileNotFoundError: + pass + return AsyncStreamFile( + self, + path, + loop=self.loop, + session=session, + size=size, + **kwargs, + ) + + def ukey(self, url): + """Unique identifier; assume HTTP files are static, unchanging""" + return tokenize(url, self.kwargs, self.protocol) + + async def _info(self, url, **kwargs): + """Get info of URL + + Tries to access location via HEAD, and then GET methods, but does + not fetch the data. + + It is possible that the server does not supply any size information, in + which case size will be given as None (and certain operations on the + corresponding file will not work). + """ + info = {} + session = await self.set_session() + + for policy in ["head", "get"]: + try: + info.update( + await _file_info( + self.encode_url(url), + size_policy=policy, + session=session, + **self.kwargs, + **kwargs, + ) + ) + if info.get("size") is not None: + break + except Exception as exc: + if policy == "get": + # If get failed, then raise a FileNotFoundError + raise FileNotFoundError(url) from exc + logger.debug("", exc_info=exc) + + return {"name": url, "size": None, **info, "type": "file"} + + async def _glob(self, path, maxdepth=None, **kwargs): + """ + Find files by glob-matching. + + This implementation is idntical to the one in AbstractFileSystem, + but "?" is not considered as a character for globbing, because it is + so common in URLs, often identifying the "query" part. + """ + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + import re + + ends_with_slash = path.endswith("/") # _strip_protocol strips trailing slash + path = self._strip_protocol(path) + append_slash_to_dirname = ends_with_slash or path.endswith(("/**", "/*")) + idx_star = path.find("*") if path.find("*") >= 0 else len(path) + idx_brace = path.find("[") if path.find("[") >= 0 else len(path) + + min_idx = min(idx_star, idx_brace) + + detail = kwargs.pop("detail", False) + + if not has_magic(path): + if await self._exists(path, **kwargs): + if not detail: + return [path] + else: + return {path: await self._info(path, **kwargs)} + else: + if not detail: + return [] # glob of non-existent returns empty + else: + return {} + elif "/" in path[:min_idx]: + min_idx = path[:min_idx].rindex("/") + root = path[: min_idx + 1] + depth = path[min_idx + 1 :].count("/") + 1 + else: + root = "" + depth = path[min_idx + 1 :].count("/") + 1 + + if "**" in path: + if maxdepth is not None: + idx_double_stars = path.find("**") + depth_double_stars = path[idx_double_stars:].count("/") + 1 + depth = depth - depth_double_stars + maxdepth + else: + depth = None + + allpaths = await self._find( + root, maxdepth=depth, withdirs=True, detail=True, **kwargs + ) + + pattern = glob_translate(path + ("/" if ends_with_slash else "")) + pattern = re.compile(pattern) + + out = { + ( + p.rstrip("/") + if not append_slash_to_dirname + and info["type"] == "directory" + and p.endswith("/") + else p + ): info + for p, info in sorted(allpaths.items()) + if pattern.match(p.rstrip("/")) + } + + if detail: + return out + else: + return list(out) + + async def _isdir(self, path): + # override, since all URLs are (also) files + try: + return bool(await self._ls(path)) + except (FileNotFoundError, ValueError): + return False + + async def _pipe_file(self, path, value, mode="overwrite", **kwargs): + """ + Write bytes to a remote file over HTTP. + + Parameters + ---------- + path : str + Target URL where the data should be written + value : bytes + Data to be written + mode : str + How to write to the file - 'overwrite' or 'append' + **kwargs : dict + Additional parameters to pass to the HTTP request + """ + url = self._strip_protocol(path) + headers = kwargs.pop("headers", {}) + headers["Content-Length"] = str(len(value)) + + session = await self.set_session() + + async with session.put( + self.encode_url(url), data=value, headers=headers, **kwargs + ) as r: + r.raise_for_status() + + +class HTTPFile(AbstractBufferedFile): + """ + A file-like object pointing to a remote HTTP(S) resource + + Supports only reading, with read-ahead of a predetermined block-size. + + In the case that the server does not supply the filesize, only reading of + the complete file in one go is supported. + + Parameters + ---------- + url: str + Full URL of the remote resource, including the protocol + session: aiohttp.ClientSession or None + All calls will be made within this session, to avoid restarting + connections where the server allows this + block_size: int or None + The amount of read-ahead to do, in bytes. Default is 5MB, or the value + configured for the FileSystem creating this file + size: None or int + If given, this is the size of the file in bytes, and we don't attempt + to call the server to find the value. + kwargs: all other key-values are passed to requests calls. + """ + + def __init__( + self, + fs, + url, + session=None, + block_size=None, + mode="rb", + cache_type="bytes", + cache_options=None, + size=None, + loop=None, + asynchronous=False, + **kwargs, + ): + if mode != "rb": + raise NotImplementedError("File mode not supported") + self.asynchronous = asynchronous + self.loop = loop + self.url = url + self.session = session + self.details = {"name": url, "size": size, "type": "file"} + super().__init__( + fs=fs, + path=url, + mode=mode, + block_size=block_size, + cache_type=cache_type, + cache_options=cache_options, + **kwargs, + ) + + def read(self, length=-1): + """Read bytes from file + + Parameters + ---------- + length: int + Read up to this many bytes. If negative, read all content to end of + file. If the server has not supplied the filesize, attempting to + read only part of the data will raise a ValueError. + """ + if ( + (length < 0 and self.loc == 0) # explicit read all + # but not when the size is known and fits into a block anyways + and not (self.size is not None and self.size <= self.blocksize) + ): + self._fetch_all() + if self.size is None: + if length < 0: + self._fetch_all() + else: + length = min(self.size - self.loc, length) + return super().read(length) + + async def async_fetch_all(self): + """Read whole file in one shot, without caching + + This is only called when position is still at zero, + and read() is called without a byte-count. + """ + logger.debug(f"Fetch all for {self}") + if not isinstance(self.cache, AllBytes): + r = await self.session.get(self.fs.encode_url(self.url), **self.kwargs) + async with r: + r.raise_for_status() + out = await r.read() + self.cache = AllBytes( + size=len(out), fetcher=None, blocksize=None, data=out + ) + self.size = len(out) + + _fetch_all = sync_wrapper(async_fetch_all) + + def _parse_content_range(self, headers): + """Parse the Content-Range header""" + s = headers.get("Content-Range", "") + m = re.match(r"bytes (\d+-\d+|\*)/(\d+|\*)", s) + if not m: + return None, None, None + + if m[1] == "*": + start = end = None + else: + start, end = [int(x) for x in m[1].split("-")] + total = None if m[2] == "*" else int(m[2]) + return start, end, total + + async def async_fetch_range(self, start, end): + """Download a block of data + + The expectation is that the server returns only the requested bytes, + with HTTP code 206. If this is not the case, we first check the headers, + and then stream the output - if the data size is bigger than we + requested, an exception is raised. + """ + logger.debug(f"Fetch range for {self}: {start}-{end}") + kwargs = self.kwargs.copy() + headers = kwargs.pop("headers", {}).copy() + headers["Range"] = f"bytes={start}-{end - 1}" + logger.debug(f"{self.url} : {headers['Range']}") + r = await self.session.get( + self.fs.encode_url(self.url), headers=headers, **kwargs + ) + async with r: + if r.status == 416: + # range request outside file + return b"" + r.raise_for_status() + + # If the server has handled the range request, it should reply + # with status 206 (partial content). But we'll guess that a suitable + # Content-Range header or a Content-Length no more than the + # requested range also mean we have got the desired range. + response_is_range = ( + r.status == 206 + or self._parse_content_range(r.headers)[0] == start + or int(r.headers.get("Content-Length", end + 1)) <= end - start + ) + + if response_is_range: + # partial content, as expected + out = await r.read() + elif start > 0: + raise ValueError( + "The HTTP server doesn't appear to support range requests. " + "Only reading this file from the beginning is supported. " + "Open with block_size=0 for a streaming file interface." + ) + else: + # Response is not a range, but we want the start of the file, + # so we can read the required amount anyway. + cl = 0 + out = [] + while True: + chunk = await r.content.read(2**20) + # data size unknown, let's read until we have enough + if chunk: + out.append(chunk) + cl += len(chunk) + if cl > end - start: + break + else: + break + out = b"".join(out)[: end - start] + return out + + _fetch_range = sync_wrapper(async_fetch_range) + + +magic_check = re.compile("([*[])") + + +def has_magic(s): + match = magic_check.search(s) + return match is not None + + +class HTTPStreamFile(AbstractBufferedFile): + def __init__(self, fs, url, mode="rb", loop=None, session=None, **kwargs): + self.asynchronous = kwargs.pop("asynchronous", False) + self.url = url + self.loop = loop + self.session = session + if mode != "rb": + raise ValueError + self.details = {"name": url, "size": None} + super().__init__(fs=fs, path=url, mode=mode, cache_type="none", **kwargs) + + async def cor(): + r = await self.session.get(self.fs.encode_url(url), **kwargs).__aenter__() + self.fs._raise_not_found_for_status(r, url) + return r + + self.r = sync(self.loop, cor) + self.loop = fs.loop + + def seek(self, loc, whence=0): + if loc == 0 and whence == 1: + return + if loc == self.loc and whence == 0: + return + raise ValueError("Cannot seek streaming HTTP file") + + async def _read(self, num=-1): + out = await self.r.content.read(num) + self.loc += len(out) + return out + + read = sync_wrapper(_read) + + async def _close(self): + self.r.close() + + def close(self): + asyncio.run_coroutine_threadsafe(self._close(), self.loop) + super().close() + + +class AsyncStreamFile(AbstractAsyncStreamedFile): + def __init__( + self, fs, url, mode="rb", loop=None, session=None, size=None, **kwargs + ): + self.url = url + self.session = session + self.r = None + if mode != "rb": + raise ValueError + self.details = {"name": url, "size": None} + self.kwargs = kwargs + super().__init__(fs=fs, path=url, mode=mode, cache_type="none") + self.size = size + + async def read(self, num=-1): + if self.r is None: + r = await self.session.get( + self.fs.encode_url(self.url), **self.kwargs + ).__aenter__() + self.fs._raise_not_found_for_status(r, self.url) + self.r = r + out = await self.r.content.read(num) + self.loc += len(out) + return out + + async def close(self): + if self.r is not None: + self.r.close() + self.r = None + await super().close() + + +async def get_range(session, url, start, end, file=None, **kwargs): + # explicit get a range when we know it must be safe + kwargs = kwargs.copy() + headers = kwargs.pop("headers", {}).copy() + headers["Range"] = f"bytes={start}-{end - 1}" + r = await session.get(url, headers=headers, **kwargs) + r.raise_for_status() + async with r: + out = await r.read() + if file: + with open(file, "r+b") as f: # noqa: ASYNC230 + f.seek(start) + f.write(out) + else: + return out + + +async def _file_info(url, session, size_policy="head", **kwargs): + """Call HEAD on the server to get details about the file (size/checksum etc.) + + Default operation is to explicitly allow redirects and use encoding + 'identity' (no compression) to get the true size of the target. + """ + logger.debug("Retrieve file size for %s", url) + kwargs = kwargs.copy() + ar = kwargs.pop("allow_redirects", True) + head = kwargs.get("headers", {}).copy() + head["Accept-Encoding"] = "identity" + kwargs["headers"] = head + + info = {} + if size_policy == "head": + r = await session.head(url, allow_redirects=ar, **kwargs) + elif size_policy == "get": + r = await session.get(url, allow_redirects=ar, **kwargs) + else: + raise TypeError(f'size_policy must be "head" or "get", got {size_policy}') + async with r: + r.raise_for_status() + + if "Content-Length" in r.headers: + # Some servers may choose to ignore Accept-Encoding and return + # compressed content, in which case the returned size is unreliable. + if "Content-Encoding" not in r.headers or r.headers["Content-Encoding"] in [ + "identity", + "", + ]: + info["size"] = int(r.headers["Content-Length"]) + elif "Content-Range" in r.headers: + info["size"] = int(r.headers["Content-Range"].split("/")[1]) + + if "Content-Type" in r.headers: + info["mimetype"] = r.headers["Content-Type"].partition(";")[0] + + if r.headers.get("Accept-Ranges") == "none": + # Some servers may explicitly discourage partial content requests, but + # the lack of "Accept-Ranges" does not always indicate they would fail + info["partial"] = False + + info["url"] = str(r.url) + + for checksum_field in ["ETag", "Content-MD5", "Digest", "Last-Modified"]: + if r.headers.get(checksum_field): + info[checksum_field] = r.headers[checksum_field] + + return info + + +async def _file_size(url, session=None, *args, **kwargs): + if session is None: + session = await get_client() + info = await _file_info(url, session=session, *args, **kwargs) + return info.get("size") + + +file_size = sync_wrapper(_file_size) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/http_sync.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/http_sync.py new file mode 100644 index 0000000000000000000000000000000000000000..a67ea3ea5fee9e6b51f7f3f66773e8cf65735e52 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/http_sync.py @@ -0,0 +1,937 @@ +"""This file is largely copied from http.py""" + +import io +import logging +import re +import urllib.error +import urllib.parse +from copy import copy +from json import dumps, loads +from urllib.parse import urlparse + +try: + import yarl +except (ImportError, ModuleNotFoundError, OSError): + yarl = False + +from fsspec.callbacks import _DEFAULT_CALLBACK +from fsspec.registry import register_implementation +from fsspec.spec import AbstractBufferedFile, AbstractFileSystem +from fsspec.utils import DEFAULT_BLOCK_SIZE, isfilelike, nullcontext, tokenize + +from ..caching import AllBytes + +# https://stackoverflow.com/a/15926317/3821154 +ex = re.compile(r"""<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P<url>[^"']+)""") +ex2 = re.compile(r"""(?P<url>http[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)""") +logger = logging.getLogger("fsspec.http") + + +class JsHttpException(urllib.error.HTTPError): ... + + +class StreamIO(io.BytesIO): + # fake class, so you can set attributes on it + # will eventually actually stream + ... + + +class ResponseProxy: + """Looks like a requests response""" + + def __init__(self, req, stream=False): + self.request = req + self.stream = stream + self._data = None + self._headers = None + + @property + def raw(self): + if self._data is None: + b = self.request.response.to_bytes() + if self.stream: + self._data = StreamIO(b) + else: + self._data = b + return self._data + + def close(self): + if hasattr(self, "_data"): + del self._data + + @property + def headers(self): + if self._headers is None: + self._headers = dict( + [ + _.split(": ") + for _ in self.request.getAllResponseHeaders().strip().split("\r\n") + ] + ) + return self._headers + + @property + def status_code(self): + return int(self.request.status) + + def raise_for_status(self): + if not self.ok: + raise JsHttpException( + self.url, self.status_code, self.reason, self.headers, None + ) + + def iter_content(self, chunksize, *_, **__): + while True: + out = self.raw.read(chunksize) + if out: + yield out + else: + break + + @property + def reason(self): + return self.request.statusText + + @property + def ok(self): + return self.status_code < 400 + + @property + def url(self): + return self.request.response.responseURL + + @property + def text(self): + # TODO: encoding from headers + return self.content.decode() + + @property + def content(self): + self.stream = False + return self.raw + + def json(self): + return loads(self.text) + + +class RequestsSessionShim: + def __init__(self): + self.headers = {} + + def request( + self, + method, + url, + params=None, + data=None, + headers=None, + cookies=None, + files=None, + auth=None, + timeout=None, + allow_redirects=None, + proxies=None, + hooks=None, + stream=None, + verify=None, + cert=None, + json=None, + ): + from js import Blob, XMLHttpRequest + + logger.debug("JS request: %s %s", method, url) + + if cert or verify or proxies or files or cookies or hooks: + raise NotImplementedError + if data and json: + raise ValueError("Use json= or data=, not both") + req = XMLHttpRequest.new() + extra = auth if auth else () + if params: + url = f"{url}?{urllib.parse.urlencode(params)}" + req.open(method, url, False, *extra) + if timeout: + req.timeout = timeout + if headers: + for k, v in headers.items(): + req.setRequestHeader(k, v) + + req.setRequestHeader("Accept", "application/octet-stream") + req.responseType = "arraybuffer" + if json: + blob = Blob.new([dumps(data)], {type: "application/json"}) + req.send(blob) + elif data: + if isinstance(data, io.IOBase): + data = data.read() + blob = Blob.new([data], {type: "application/octet-stream"}) + req.send(blob) + else: + req.send(None) + return ResponseProxy(req, stream=stream) + + def get(self, url, **kwargs): + return self.request("GET", url, **kwargs) + + def head(self, url, **kwargs): + return self.request("HEAD", url, **kwargs) + + def post(self, url, **kwargs): + return self.request("POST}", url, **kwargs) + + def put(self, url, **kwargs): + return self.request("PUT", url, **kwargs) + + def patch(self, url, **kwargs): + return self.request("PATCH", url, **kwargs) + + def delete(self, url, **kwargs): + return self.request("DELETE", url, **kwargs) + + +class HTTPFileSystem(AbstractFileSystem): + """ + Simple File-System for fetching data via HTTP(S) + + This is the BLOCKING version of the normal HTTPFileSystem. It uses + requests in normal python and the JS runtime in pyodide. + + ***This implementation is extremely experimental, do not use unless + you are testing pyodide/pyscript integration*** + """ + + protocol = ("http", "https", "sync-http", "sync-https") + sep = "/" + + def __init__( + self, + simple_links=True, + block_size=None, + same_scheme=True, + cache_type="readahead", + cache_options=None, + client_kwargs=None, + encoded=False, + **storage_options, + ): + """ + + Parameters + ---------- + block_size: int + Blocks to read bytes; if 0, will default to raw requests file-like + objects instead of HTTPFile instances + simple_links: bool + If True, will consider both HTML <a> tags and anything that looks + like a URL; if False, will consider only the former. + same_scheme: True + When doing ls/glob, if this is True, only consider paths that have + http/https matching the input URLs. + size_policy: this argument is deprecated + client_kwargs: dict + Passed to aiohttp.ClientSession, see + https://docs.aiohttp.org/en/stable/client_reference.html + For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}`` + storage_options: key-value + Any other parameters passed on to requests + cache_type, cache_options: defaults used in open + """ + super().__init__(self, **storage_options) + self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE + self.simple_links = simple_links + self.same_schema = same_scheme + self.cache_type = cache_type + self.cache_options = cache_options + self.client_kwargs = client_kwargs or {} + self.encoded = encoded + self.kwargs = storage_options + + try: + import js # noqa: F401 + + logger.debug("Starting JS session") + self.session = RequestsSessionShim() + self.js = True + except Exception as e: + import requests + + logger.debug("Starting cpython session because of: %s", e) + self.session = requests.Session(**(client_kwargs or {})) + self.js = False + + request_options = copy(storage_options) + self.use_listings_cache = request_options.pop("use_listings_cache", False) + request_options.pop("listings_expiry_time", None) + request_options.pop("max_paths", None) + request_options.pop("skip_instance_cache", None) + self.kwargs = request_options + + @property + def fsid(self): + return "sync-http" + + def encode_url(self, url): + if yarl: + return yarl.URL(url, encoded=self.encoded) + return url + + @classmethod + def _strip_protocol(cls, path: str) -> str: + """For HTTP, we always want to keep the full URL""" + path = path.replace("sync-http://", "http://").replace( + "sync-https://", "https://" + ) + return path + + @classmethod + def _parent(cls, path): + # override, since _strip_protocol is different for URLs + par = super()._parent(path) + if len(par) > 7: # "http://..." + return par + return "" + + def _ls_real(self, url, detail=True, **kwargs): + # ignoring URL-encoded arguments + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(url) + r = self.session.get(self.encode_url(url), **self.kwargs) + self._raise_not_found_for_status(r, url) + text = r.text + if self.simple_links: + links = ex2.findall(text) + [u[2] for u in ex.findall(text)] + else: + links = [u[2] for u in ex.findall(text)] + out = set() + parts = urlparse(url) + for l in links: + if isinstance(l, tuple): + l = l[1] + if l.startswith("/") and len(l) > 1: + # absolute URL on this server + l = parts.scheme + "://" + parts.netloc + l + if l.startswith("http"): + if self.same_schema and l.startswith(url.rstrip("/") + "/"): + out.add(l) + elif l.replace("https", "http").startswith( + url.replace("https", "http").rstrip("/") + "/" + ): + # allowed to cross http <-> https + out.add(l) + else: + if l not in ["..", "../"]: + # Ignore FTP-like "parent" + out.add("/".join([url.rstrip("/"), l.lstrip("/")])) + if not out and url.endswith("/"): + out = self._ls_real(url.rstrip("/"), detail=False) + if detail: + return [ + { + "name": u, + "size": None, + "type": "directory" if u.endswith("/") else "file", + } + for u in out + ] + else: + return sorted(out) + + def ls(self, url, detail=True, **kwargs): + if self.use_listings_cache and url in self.dircache: + out = self.dircache[url] + else: + out = self._ls_real(url, detail=detail, **kwargs) + self.dircache[url] = out + return out + + def _raise_not_found_for_status(self, response, url): + """ + Raises FileNotFoundError for 404s, otherwise uses raise_for_status. + """ + if response.status_code == 404: + raise FileNotFoundError(url) + response.raise_for_status() + + def cat_file(self, url, start=None, end=None, **kwargs): + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(url) + + if start is not None or end is not None: + if start == end: + return b"" + headers = kw.pop("headers", {}).copy() + + headers["Range"] = self._process_limits(url, start, end) + kw["headers"] = headers + r = self.session.get(self.encode_url(url), **kw) + self._raise_not_found_for_status(r, url) + return r.content + + def get_file( + self, rpath, lpath, chunk_size=5 * 2**20, callback=_DEFAULT_CALLBACK, **kwargs + ): + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(rpath) + r = self.session.get(self.encode_url(rpath), **kw) + try: + size = int( + r.headers.get("content-length", None) + or r.headers.get("Content-Length", None) + ) + except (ValueError, KeyError, TypeError): + size = None + + callback.set_size(size) + self._raise_not_found_for_status(r, rpath) + if not isfilelike(lpath): + lpath = open(lpath, "wb") + for chunk in r.iter_content(chunk_size, decode_unicode=False): + lpath.write(chunk) + callback.relative_update(len(chunk)) + + def put_file( + self, + lpath, + rpath, + chunk_size=5 * 2**20, + callback=_DEFAULT_CALLBACK, + method="post", + **kwargs, + ): + def gen_chunks(): + # Support passing arbitrary file-like objects + # and use them instead of streams. + if isinstance(lpath, io.IOBase): + context = nullcontext(lpath) + use_seek = False # might not support seeking + else: + context = open(lpath, "rb") + use_seek = True + + with context as f: + if use_seek: + callback.set_size(f.seek(0, 2)) + f.seek(0) + else: + callback.set_size(getattr(f, "size", None)) + + chunk = f.read(chunk_size) + while chunk: + yield chunk + callback.relative_update(len(chunk)) + chunk = f.read(chunk_size) + + kw = self.kwargs.copy() + kw.update(kwargs) + + method = method.lower() + if method not in ("post", "put"): + raise ValueError( + f"method has to be either 'post' or 'put', not: {method!r}" + ) + + meth = getattr(self.session, method) + resp = meth(rpath, data=gen_chunks(), **kw) + self._raise_not_found_for_status(resp, rpath) + + def _process_limits(self, url, start, end): + """Helper for "Range"-based _cat_file""" + size = None + suff = False + if start is not None and start < 0: + # if start is negative and end None, end is the "suffix length" + if end is None: + end = -start + start = "" + suff = True + else: + size = size or self.info(url)["size"] + start = size + start + elif start is None: + start = 0 + if not suff: + if end is not None and end < 0: + if start is not None: + size = size or self.info(url)["size"] + end = size + end + elif end is None: + end = "" + if isinstance(end, int): + end -= 1 # bytes range is inclusive + return f"bytes={start}-{end}" + + def exists(self, path, strict=False, **kwargs): + kw = self.kwargs.copy() + kw.update(kwargs) + try: + logger.debug(path) + r = self.session.get(self.encode_url(path), **kw) + if strict: + self._raise_not_found_for_status(r, path) + return r.status_code < 400 + except FileNotFoundError: + return False + except Exception: + if strict: + raise + return False + + def isfile(self, path, **kwargs): + return self.exists(path, **kwargs) + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=None, # XXX: This differs from the base class. + cache_type=None, + cache_options=None, + size=None, + **kwargs, + ): + """Make a file-like object + + Parameters + ---------- + path: str + Full URL with protocol + mode: string + must be "rb" + block_size: int or None + Bytes to download in one request; use instance value if None. If + zero, will return a streaming Requests file-like instance. + kwargs: key-value + Any other parameters, passed to requests calls + """ + if mode != "rb": + raise NotImplementedError + block_size = block_size if block_size is not None else self.block_size + kw = self.kwargs.copy() + kw.update(kwargs) + size = size or self.info(path, **kwargs)["size"] + if block_size and size: + return HTTPFile( + self, + path, + session=self.session, + block_size=block_size, + mode=mode, + size=size, + cache_type=cache_type or self.cache_type, + cache_options=cache_options or self.cache_options, + **kw, + ) + else: + return HTTPStreamFile( + self, + path, + mode=mode, + session=self.session, + **kw, + ) + + def ukey(self, url): + """Unique identifier; assume HTTP files are static, unchanging""" + return tokenize(url, self.kwargs, self.protocol) + + def info(self, url, **kwargs): + """Get info of URL + + Tries to access location via HEAD, and then GET methods, but does + not fetch the data. + + It is possible that the server does not supply any size information, in + which case size will be given as None (and certain operations on the + corresponding file will not work). + """ + info = {} + for policy in ["head", "get"]: + try: + info.update( + _file_info( + self.encode_url(url), + size_policy=policy, + session=self.session, + **self.kwargs, + **kwargs, + ) + ) + if info.get("size") is not None: + break + except Exception as exc: + if policy == "get": + # If get failed, then raise a FileNotFoundError + raise FileNotFoundError(url) from exc + logger.debug(str(exc)) + + return {"name": url, "size": None, **info, "type": "file"} + + def glob(self, path, maxdepth=None, **kwargs): + """ + Find files by glob-matching. + + This implementation is idntical to the one in AbstractFileSystem, + but "?" is not considered as a character for globbing, because it is + so common in URLs, often identifying the "query" part. + """ + import re + + ends = path.endswith("/") + path = self._strip_protocol(path) + indstar = path.find("*") if path.find("*") >= 0 else len(path) + indbrace = path.find("[") if path.find("[") >= 0 else len(path) + + ind = min(indstar, indbrace) + + detail = kwargs.pop("detail", False) + + if not has_magic(path): + root = path + depth = 1 + if ends: + path += "/*" + elif self.exists(path): + if not detail: + return [path] + else: + return {path: self.info(path)} + else: + if not detail: + return [] # glob of non-existent returns empty + else: + return {} + elif "/" in path[:ind]: + ind2 = path[:ind].rindex("/") + root = path[: ind2 + 1] + depth = None if "**" in path else path[ind2 + 1 :].count("/") + 1 + else: + root = "" + depth = None if "**" in path else path[ind + 1 :].count("/") + 1 + + allpaths = self.find( + root, maxdepth=maxdepth or depth, withdirs=True, detail=True, **kwargs + ) + # Escape characters special to python regex, leaving our supported + # special characters in place. + # See https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html + # for shell globbing details. + pattern = ( + "^" + + ( + path.replace("\\", r"\\") + .replace(".", r"\.") + .replace("+", r"\+") + .replace("//", "/") + .replace("(", r"\(") + .replace(")", r"\)") + .replace("|", r"\|") + .replace("^", r"\^") + .replace("$", r"\$") + .replace("{", r"\{") + .replace("}", r"\}") + .rstrip("/") + ) + + "$" + ) + pattern = re.sub("[*]{2}", "=PLACEHOLDER=", pattern) + pattern = re.sub("[*]", "[^/]*", pattern) + pattern = re.compile(pattern.replace("=PLACEHOLDER=", ".*")) + out = { + p: allpaths[p] + for p in sorted(allpaths) + if pattern.match(p.replace("//", "/").rstrip("/")) + } + if detail: + return out + else: + return list(out) + + def isdir(self, path): + # override, since all URLs are (also) files + try: + return bool(self.ls(path)) + except (FileNotFoundError, ValueError): + return False + + +class HTTPFile(AbstractBufferedFile): + """ + A file-like object pointing to a remove HTTP(S) resource + + Supports only reading, with read-ahead of a predermined block-size. + + In the case that the server does not supply the filesize, only reading of + the complete file in one go is supported. + + Parameters + ---------- + url: str + Full URL of the remote resource, including the protocol + session: requests.Session or None + All calls will be made within this session, to avoid restarting + connections where the server allows this + block_size: int or None + The amount of read-ahead to do, in bytes. Default is 5MB, or the value + configured for the FileSystem creating this file + size: None or int + If given, this is the size of the file in bytes, and we don't attempt + to call the server to find the value. + kwargs: all other key-values are passed to requests calls. + """ + + def __init__( + self, + fs, + url, + session=None, + block_size=None, + mode="rb", + cache_type="bytes", + cache_options=None, + size=None, + **kwargs, + ): + if mode != "rb": + raise NotImplementedError("File mode not supported") + self.url = url + self.session = session + self.details = {"name": url, "size": size, "type": "file"} + super().__init__( + fs=fs, + path=url, + mode=mode, + block_size=block_size, + cache_type=cache_type, + cache_options=cache_options, + **kwargs, + ) + + def read(self, length=-1): + """Read bytes from file + + Parameters + ---------- + length: int + Read up to this many bytes. If negative, read all content to end of + file. If the server has not supplied the filesize, attempting to + read only part of the data will raise a ValueError. + """ + if ( + (length < 0 and self.loc == 0) # explicit read all + # but not when the size is known and fits into a block anyways + and not (self.size is not None and self.size <= self.blocksize) + ): + self._fetch_all() + if self.size is None: + if length < 0: + self._fetch_all() + else: + length = min(self.size - self.loc, length) + return super().read(length) + + def _fetch_all(self): + """Read whole file in one shot, without caching + + This is only called when position is still at zero, + and read() is called without a byte-count. + """ + logger.debug(f"Fetch all for {self}") + if not isinstance(self.cache, AllBytes): + r = self.session.get(self.fs.encode_url(self.url), **self.kwargs) + r.raise_for_status() + out = r.content + self.cache = AllBytes(size=len(out), fetcher=None, blocksize=None, data=out) + self.size = len(out) + + def _parse_content_range(self, headers): + """Parse the Content-Range header""" + s = headers.get("Content-Range", "") + m = re.match(r"bytes (\d+-\d+|\*)/(\d+|\*)", s) + if not m: + return None, None, None + + if m[1] == "*": + start = end = None + else: + start, end = [int(x) for x in m[1].split("-")] + total = None if m[2] == "*" else int(m[2]) + return start, end, total + + def _fetch_range(self, start, end): + """Download a block of data + + The expectation is that the server returns only the requested bytes, + with HTTP code 206. If this is not the case, we first check the headers, + and then stream the output - if the data size is bigger than we + requested, an exception is raised. + """ + logger.debug(f"Fetch range for {self}: {start}-{end}") + kwargs = self.kwargs.copy() + headers = kwargs.pop("headers", {}).copy() + headers["Range"] = f"bytes={start}-{end - 1}" + logger.debug("%s : %s", self.url, headers["Range"]) + r = self.session.get(self.fs.encode_url(self.url), headers=headers, **kwargs) + if r.status_code == 416: + # range request outside file + return b"" + r.raise_for_status() + + # If the server has handled the range request, it should reply + # with status 206 (partial content). But we'll guess that a suitable + # Content-Range header or a Content-Length no more than the + # requested range also mean we have got the desired range. + cl = r.headers.get("Content-Length", r.headers.get("content-length", end + 1)) + response_is_range = ( + r.status_code == 206 + or self._parse_content_range(r.headers)[0] == start + or int(cl) <= end - start + ) + + if response_is_range: + # partial content, as expected + out = r.content + elif start > 0: + raise ValueError( + "The HTTP server doesn't appear to support range requests. " + "Only reading this file from the beginning is supported. " + "Open with block_size=0 for a streaming file interface." + ) + else: + # Response is not a range, but we want the start of the file, + # so we can read the required amount anyway. + cl = 0 + out = [] + for chunk in r.iter_content(2**20, False): + out.append(chunk) + cl += len(chunk) + out = b"".join(out)[: end - start] + return out + + +magic_check = re.compile("([*[])") + + +def has_magic(s): + match = magic_check.search(s) + return match is not None + + +class HTTPStreamFile(AbstractBufferedFile): + def __init__(self, fs, url, mode="rb", session=None, **kwargs): + self.url = url + self.session = session + if mode != "rb": + raise ValueError + self.details = {"name": url, "size": None} + super().__init__(fs=fs, path=url, mode=mode, cache_type="readahead", **kwargs) + + r = self.session.get(self.fs.encode_url(url), stream=True, **kwargs) + self.fs._raise_not_found_for_status(r, url) + self.it = r.iter_content(1024, False) + self.leftover = b"" + + self.r = r + + def seek(self, *args, **kwargs): + raise ValueError("Cannot seek streaming HTTP file") + + def read(self, num=-1): + bufs = [self.leftover] + leng = len(self.leftover) + while leng < num or num < 0: + try: + out = self.it.__next__() + except StopIteration: + break + if out: + bufs.append(out) + else: + break + leng += len(out) + out = b"".join(bufs) + if num >= 0: + self.leftover = out[num:] + out = out[:num] + else: + self.leftover = b"" + self.loc += len(out) + return out + + def close(self): + self.r.close() + self.closed = True + + +def get_range(session, url, start, end, **kwargs): + # explicit get a range when we know it must be safe + kwargs = kwargs.copy() + headers = kwargs.pop("headers", {}).copy() + headers["Range"] = f"bytes={start}-{end - 1}" + r = session.get(url, headers=headers, **kwargs) + r.raise_for_status() + return r.content + + +def _file_info(url, session, size_policy="head", **kwargs): + """Call HEAD on the server to get details about the file (size/checksum etc.) + + Default operation is to explicitly allow redirects and use encoding + 'identity' (no compression) to get the true size of the target. + """ + logger.debug("Retrieve file size for %s", url) + kwargs = kwargs.copy() + ar = kwargs.pop("allow_redirects", True) + head = kwargs.get("headers", {}).copy() + # TODO: not allowed in JS + # head["Accept-Encoding"] = "identity" + kwargs["headers"] = head + + info = {} + if size_policy == "head": + r = session.head(url, allow_redirects=ar, **kwargs) + elif size_policy == "get": + r = session.get(url, allow_redirects=ar, **kwargs) + else: + raise TypeError(f'size_policy must be "head" or "get", got {size_policy}') + r.raise_for_status() + + # TODO: + # recognise lack of 'Accept-Ranges', + # or 'Accept-Ranges': 'none' (not 'bytes') + # to mean streaming only, no random access => return None + if "Content-Length" in r.headers: + info["size"] = int(r.headers["Content-Length"]) + elif "Content-Range" in r.headers: + info["size"] = int(r.headers["Content-Range"].split("/")[1]) + elif "content-length" in r.headers: + info["size"] = int(r.headers["content-length"]) + elif "content-range" in r.headers: + info["size"] = int(r.headers["content-range"].split("/")[1]) + + for checksum_field in ["ETag", "Content-MD5", "Digest"]: + if r.headers.get(checksum_field): + info[checksum_field] = r.headers[checksum_field] + + return info + + +# importing this is enough to register it +def register(): + register_implementation("http", HTTPFileSystem, clobber=True) + register_implementation("https", HTTPFileSystem, clobber=True) + register_implementation("sync-http", HTTPFileSystem, clobber=True) + register_implementation("sync-https", HTTPFileSystem, clobber=True) + + +register() + + +def unregister(): + from fsspec.implementations.http import HTTPFileSystem + + register_implementation("http", HTTPFileSystem, clobber=True) + register_implementation("https", HTTPFileSystem, clobber=True) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/jupyter.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/jupyter.py new file mode 100644 index 0000000000000000000000000000000000000000..e5571ed56582170051f3b7cd903093eed4c65244 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/jupyter.py @@ -0,0 +1,129 @@ +import base64 +import io +import re + +import requests + +import fsspec + + +class JupyterFileSystem(fsspec.AbstractFileSystem): + """View of the files as seen by a Jupyter server (notebook or lab)""" + + protocol = ("jupyter", "jlab") + + def __init__(self, url, tok=None, **kwargs): + """ + + Parameters + ---------- + url : str + Base URL of the server, like "http://127.0.0.1:8888". May include + token in the string, which is given by the process when starting up + tok : str + If the token is obtained separately, can be given here + kwargs + """ + if "?" in url: + if tok is None: + try: + tok = re.findall("token=([a-z0-9]+)", url)[0] + except IndexError as e: + raise ValueError("Could not determine token") from e + url = url.split("?", 1)[0] + self.url = url.rstrip("/") + "/api/contents" + self.session = requests.Session() + if tok: + self.session.headers["Authorization"] = f"token {tok}" + + super().__init__(**kwargs) + + def ls(self, path, detail=True, **kwargs): + path = self._strip_protocol(path) + r = self.session.get(f"{self.url}/{path}") + if r.status_code == 404: + raise FileNotFoundError(path) + r.raise_for_status() + out = r.json() + + if out["type"] == "directory": + out = out["content"] + else: + out = [out] + for o in out: + o["name"] = o.pop("path") + o.pop("content") + if o["type"] == "notebook": + o["type"] = "file" + if detail: + return out + return [o["name"] for o in out] + + def cat_file(self, path, start=None, end=None, **kwargs): + path = self._strip_protocol(path) + r = self.session.get(f"{self.url}/{path}") + if r.status_code == 404: + raise FileNotFoundError(path) + r.raise_for_status() + out = r.json() + if out["format"] == "text": + # data should be binary + b = out["content"].encode() + else: + b = base64.b64decode(out["content"]) + return b[start:end] + + def pipe_file(self, path, value, **_): + path = self._strip_protocol(path) + json = { + "name": path.rsplit("/", 1)[-1], + "path": path, + "size": len(value), + "content": base64.b64encode(value).decode(), + "format": "base64", + "type": "file", + } + self.session.put(f"{self.url}/{path}", json=json) + + def mkdir(self, path, create_parents=True, **kwargs): + path = self._strip_protocol(path) + if create_parents and "/" in path: + self.mkdir(path.rsplit("/", 1)[0], True) + json = { + "name": path.rsplit("/", 1)[-1], + "path": path, + "size": None, + "content": None, + "type": "directory", + } + self.session.put(f"{self.url}/{path}", json=json) + + def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs): + if path1 == path2: + return + self.session.patch(f"{self.url}/{path1}", json={"path": path2}) + + def _rm(self, path): + path = self._strip_protocol(path) + self.session.delete(f"{self.url}/{path}") + + def _open(self, path, mode="rb", **kwargs): + path = self._strip_protocol(path) + if mode == "rb": + data = self.cat_file(path) + return io.BytesIO(data) + else: + return SimpleFileWriter(self, path, mode="wb") + + +class SimpleFileWriter(fsspec.spec.AbstractBufferedFile): + def _upload_chunk(self, final=False): + """Never uploads a chunk until file is done + + Not suitable for large files + """ + if final is False: + return False + self.buffer.seek(0) + data = self.buffer.read() + self.fs.pipe_file(self.path, data) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/libarchive.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/libarchive.py new file mode 100644 index 0000000000000000000000000000000000000000..6f8e750002df72865d611b48022e6634f9572614 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/libarchive.py @@ -0,0 +1,213 @@ +from contextlib import contextmanager +from ctypes import ( + CFUNCTYPE, + POINTER, + c_int, + c_longlong, + c_void_p, + cast, + create_string_buffer, +) + +import libarchive +import libarchive.ffi as ffi + +from fsspec import open_files +from fsspec.archive import AbstractArchiveFileSystem +from fsspec.implementations.memory import MemoryFile +from fsspec.utils import DEFAULT_BLOCK_SIZE + +# Libarchive requires seekable files or memory only for certain archive +# types. However, since we read the directory first to cache the contents +# and also allow random access to any file, the file-like object needs +# to be seekable no matter what. + +# Seek call-backs (not provided in the libarchive python wrapper) +SEEK_CALLBACK = CFUNCTYPE(c_longlong, c_int, c_void_p, c_longlong, c_int) +read_set_seek_callback = ffi.ffi( + "read_set_seek_callback", [ffi.c_archive_p, SEEK_CALLBACK], c_int, ffi.check_int +) +new_api = hasattr(ffi, "NO_OPEN_CB") + + +@contextmanager +def custom_reader(file, format_name="all", filter_name="all", block_size=ffi.page_size): + """Read an archive from a seekable file-like object. + + The `file` object must support the standard `readinto` and 'seek' methods. + """ + buf = create_string_buffer(block_size) + buf_p = cast(buf, c_void_p) + + def read_func(archive_p, context, ptrptr): + # readinto the buffer, returns number of bytes read + length = file.readinto(buf) + # write the address of the buffer into the pointer + ptrptr = cast(ptrptr, POINTER(c_void_p)) + ptrptr[0] = buf_p + # tell libarchive how much data was written into the buffer + return length + + def seek_func(archive_p, context, offset, whence): + file.seek(offset, whence) + # tell libarchvie the current position + return file.tell() + + read_cb = ffi.READ_CALLBACK(read_func) + seek_cb = SEEK_CALLBACK(seek_func) + + if new_api: + open_cb = ffi.NO_OPEN_CB + close_cb = ffi.NO_CLOSE_CB + else: + open_cb = libarchive.read.OPEN_CALLBACK(ffi.VOID_CB) + close_cb = libarchive.read.CLOSE_CALLBACK(ffi.VOID_CB) + + with libarchive.read.new_archive_read(format_name, filter_name) as archive_p: + read_set_seek_callback(archive_p, seek_cb) + ffi.read_open(archive_p, None, open_cb, read_cb, close_cb) + yield libarchive.read.ArchiveRead(archive_p) + + +class LibArchiveFileSystem(AbstractArchiveFileSystem): + """Compressed archives as a file-system (read-only) + + Supports the following formats: + tar, pax , cpio, ISO9660, zip, mtree, shar, ar, raw, xar, lha/lzh, rar + Microsoft CAB, 7-Zip, WARC + + See the libarchive documentation for further restrictions. + https://www.libarchive.org/ + + Keeps file object open while instance lives. It only works in seekable + file-like objects. In case the filesystem does not support this kind of + file object, it is recommended to cache locally. + + This class is pickleable, but not necessarily thread-safe (depends on the + platform). See libarchive documentation for details. + """ + + root_marker = "" + protocol = "libarchive" + cachable = False + + def __init__( + self, + fo="", + mode="r", + target_protocol=None, + target_options=None, + block_size=DEFAULT_BLOCK_SIZE, + **kwargs, + ): + """ + Parameters + ---------- + fo: str or file-like + Contains ZIP, and must exist. If a str, will fetch file using + :meth:`~fsspec.open_files`, which must return one file exactly. + mode: str + Currently, only 'r' accepted + target_protocol: str (optional) + If ``fo`` is a string, this value can be used to override the + FS protocol inferred from a URL + target_options: dict (optional) + Kwargs passed when instantiating the target FS, if ``fo`` is + a string. + """ + super().__init__(self, **kwargs) + if mode != "r": + raise ValueError("Only read from archive files accepted") + if isinstance(fo, str): + files = open_files(fo, protocol=target_protocol, **(target_options or {})) + if len(files) != 1: + raise ValueError( + f'Path "{fo}" did not resolve to exactly one file: "{files}"' + ) + fo = files[0] + self.of = fo + self.fo = fo.__enter__() # the whole instance is a context + self.block_size = block_size + self.dir_cache = None + + @contextmanager + def _open_archive(self): + self.fo.seek(0) + with custom_reader(self.fo, block_size=self.block_size) as arc: + yield arc + + @classmethod + def _strip_protocol(cls, path): + # file paths are always relative to the archive root + return super()._strip_protocol(path).lstrip("/") + + def _get_dirs(self): + fields = { + "name": "pathname", + "size": "size", + "created": "ctime", + "mode": "mode", + "uid": "uid", + "gid": "gid", + "mtime": "mtime", + } + + if self.dir_cache is not None: + return + + self.dir_cache = {} + list_names = [] + with self._open_archive() as arc: + for entry in arc: + if not entry.isdir and not entry.isfile: + # Skip symbolic links, fifo entries, etc. + continue + self.dir_cache.update( + { + dirname: {"name": dirname, "size": 0, "type": "directory"} + for dirname in self._all_dirnames(set(entry.name)) + } + ) + f = {key: getattr(entry, fields[key]) for key in fields} + f["type"] = "directory" if entry.isdir else "file" + list_names.append(entry.name) + + self.dir_cache[f["name"]] = f + # libarchive does not seem to return an entry for the directories (at least + # not in all formats), so get the directories names from the files names + self.dir_cache.update( + { + dirname: {"name": dirname, "size": 0, "type": "directory"} + for dirname in self._all_dirnames(list_names) + } + ) + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + path = self._strip_protocol(path) + if mode != "rb": + raise NotImplementedError + + data = b"" + with self._open_archive() as arc: + for entry in arc: + if entry.pathname != path: + continue + + if entry.size == 0: + # empty file, so there are no blocks + break + + for block in entry.get_blocks(entry.size): + data = block + break + else: + raise ValueError + return MemoryFile(fs=self, path=path, data=data) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/local.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/local.py new file mode 100644 index 0000000000000000000000000000000000000000..b549b93e22c56fc67c00a3a87718999d2039d4c2 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/local.py @@ -0,0 +1,517 @@ +import datetime +import io +import logging +import os +import os.path as osp +import shutil +import stat +import tempfile +from functools import lru_cache + +from fsspec import AbstractFileSystem +from fsspec.compression import compr +from fsspec.core import get_compression +from fsspec.utils import isfilelike, stringify_path + +logger = logging.getLogger("fsspec.local") + + +class LocalFileSystem(AbstractFileSystem): + """Interface to files on local storage + + Parameters + ---------- + auto_mkdir: bool + Whether, when opening a file, the directory containing it should + be created (if it doesn't already exist). This is assumed by pyarrow + code. + """ + + root_marker = "/" + protocol = "file", "local" + local_file = True + + def __init__(self, auto_mkdir=False, **kwargs): + super().__init__(**kwargs) + self.auto_mkdir = auto_mkdir + + @property + def fsid(self): + return "local" + + def mkdir(self, path, create_parents=True, **kwargs): + path = self._strip_protocol(path) + if self.exists(path): + raise FileExistsError(path) + if create_parents: + self.makedirs(path, exist_ok=True) + else: + os.mkdir(path, **kwargs) + + def makedirs(self, path, exist_ok=False): + path = self._strip_protocol(path) + os.makedirs(path, exist_ok=exist_ok) + + def rmdir(self, path): + path = self._strip_protocol(path) + os.rmdir(path) + + def ls(self, path, detail=False, **kwargs): + path = self._strip_protocol(path) + path_info = self.info(path) + infos = [] + if path_info["type"] == "directory": + with os.scandir(path) as it: + for f in it: + try: + # Only get the info if requested since it is a bit expensive (the stat call inside) + # The strip_protocol is also used in info() and calls make_path_posix to always return posix paths + info = self.info(f) if detail else self._strip_protocol(f.path) + infos.append(info) + except FileNotFoundError: + pass + else: + infos = [path_info] if detail else [path_info["name"]] + + return infos + + def info(self, path, **kwargs): + if isinstance(path, os.DirEntry): + # scandir DirEntry + out = path.stat(follow_symlinks=False) + link = path.is_symlink() + if path.is_dir(follow_symlinks=False): + t = "directory" + elif path.is_file(follow_symlinks=False): + t = "file" + else: + t = "other" + + size = out.st_size + if link: + try: + out2 = path.stat(follow_symlinks=True) + size = out2.st_size + except OSError: + size = 0 + path = self._strip_protocol(path.path) + else: + # str or path-like + path = self._strip_protocol(path) + out = os.stat(path, follow_symlinks=False) + link = stat.S_ISLNK(out.st_mode) + if link: + out = os.stat(path, follow_symlinks=True) + size = out.st_size + if stat.S_ISDIR(out.st_mode): + t = "directory" + elif stat.S_ISREG(out.st_mode): + t = "file" + else: + t = "other" + + # Check for the 'st_birthtime' attribute, which is not always present; fallback to st_ctime + created_time = getattr(out, "st_birthtime", out.st_ctime) + + result = { + "name": path, + "size": size, + "type": t, + "created": created_time, + "islink": link, + } + for field in ["mode", "uid", "gid", "mtime", "ino", "nlink"]: + result[field] = getattr(out, f"st_{field}") + if link: + result["destination"] = os.readlink(path) + return result + + def lexists(self, path, **kwargs): + return osp.lexists(path) + + def cp_file(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1) + path2 = self._strip_protocol(path2) + if self.auto_mkdir: + self.makedirs(self._parent(path2), exist_ok=True) + if self.isfile(path1): + shutil.copyfile(path1, path2) + elif self.isdir(path1): + self.mkdirs(path2, exist_ok=True) + else: + raise FileNotFoundError(path1) + + def isfile(self, path): + path = self._strip_protocol(path) + return os.path.isfile(path) + + def isdir(self, path): + path = self._strip_protocol(path) + return os.path.isdir(path) + + def get_file(self, path1, path2, callback=None, **kwargs): + if isfilelike(path2): + with open(path1, "rb") as f: + shutil.copyfileobj(f, path2) + else: + return self.cp_file(path1, path2, **kwargs) + + def put_file(self, path1, path2, callback=None, **kwargs): + return self.cp_file(path1, path2, **kwargs) + + def mv(self, path1, path2, recursive: bool = True, **kwargs): + """Move files/directories + For the specific case of local, all ops on directories are recursive and + the recursive= kwarg is ignored. + """ + path1 = self._strip_protocol(path1) + path2 = self._strip_protocol(path2) + + if self.auto_mkdir: + self.makedirs(self._parent(path2), exist_ok=True) + + shutil.move(path1, path2) + + def link(self, src, dst, **kwargs): + src = self._strip_protocol(src) + dst = self._strip_protocol(dst) + os.link(src, dst, **kwargs) + + def symlink(self, src, dst, **kwargs): + src = self._strip_protocol(src) + dst = self._strip_protocol(dst) + os.symlink(src, dst, **kwargs) + + def islink(self, path) -> bool: + return os.path.islink(self._strip_protocol(path)) + + def rm_file(self, path): + os.remove(self._strip_protocol(path)) + + def rm(self, path, recursive=False, maxdepth=None): + if not isinstance(path, list): + path = [path] + + for p in path: + p = self._strip_protocol(p) + if self.isdir(p): + if not recursive: + raise ValueError("Cannot delete directory, set recursive=True") + if osp.abspath(p) == os.getcwd(): + raise ValueError("Cannot delete current working directory") + shutil.rmtree(p) + else: + os.remove(p) + + def unstrip_protocol(self, name): + protocol = self.protocol if isinstance(self.protocol, str) else self.protocol[0] + name = self._strip_protocol(name) # normalise for local/win/... + return f"{protocol}://{name}" + + def _open(self, path, mode="rb", block_size=None, **kwargs): + path = self._strip_protocol(path) + if self.auto_mkdir and "w" in mode: + self.makedirs(self._parent(path), exist_ok=True) + return LocalFileOpener(path, mode, fs=self, **kwargs) + + def touch(self, path, truncate=True, **kwargs): + path = self._strip_protocol(path) + if self.auto_mkdir: + self.makedirs(self._parent(path), exist_ok=True) + if self.exists(path): + os.utime(path, None) + else: + open(path, "a").close() + if truncate: + os.truncate(path, 0) + + def created(self, path): + info = self.info(path=path) + return datetime.datetime.fromtimestamp( + info["created"], tz=datetime.timezone.utc + ) + + def modified(self, path): + info = self.info(path=path) + return datetime.datetime.fromtimestamp(info["mtime"], tz=datetime.timezone.utc) + + @classmethod + def _parent(cls, path): + path = cls._strip_protocol(path) + if os.sep == "/": + # posix native + return path.rsplit("/", 1)[0] or "/" + else: + # NT + path_ = path.rsplit("/", 1)[0] + if len(path_) <= 3: + if path_[1:2] == ":": + # nt root (something like c:/) + return path_[0] + ":/" + # More cases may be required here + return path_ + + @classmethod + def _strip_protocol(cls, path): + path = stringify_path(path) + protos = (cls.protocol,) if isinstance(cls.protocol, str) else cls.protocol + prefixes = (protocol + sep for protocol in protos for sep in ("://", ":")) + for prefix in prefixes: + if path.startswith(prefix): + path = path.removeprefix(prefix) + break + + path = make_path_posix(path) + if os.sep != "/": + # This code-path is a stripped down version of + # > drive, path = ntpath.splitdrive(path) + if path[1:2] == ":": + # Absolute drive-letter path, e.g. X:\Windows + # Relative path with drive, e.g. X:Windows + drive, path = path[:2], path[2:] + elif path[:2] == "//": + # UNC drives, e.g. \\server\share or \\?\UNC\server\share + # Device drives, e.g. \\.\device or \\?\device + if (index1 := path.find("/", 2)) == -1 or ( + index2 := path.find("/", index1 + 1) + ) == -1: + drive, path = path, "" + else: + drive, path = path[:index2], path[index2:] + else: + # Relative path, e.g. Windows + drive = "" + + path = path.rstrip("/") or cls.root_marker + return drive + path + + else: + return path.rstrip("/") or cls.root_marker + + def _isfilestore(self): + # Inheriting from DaskFileSystem makes this False (S3, etc. were) + # the original motivation. But we are a posix-like file system. + # See https://github.com/dask/dask/issues/5526 + return True + + def chmod(self, path, mode): + path = stringify_path(path) + return os.chmod(path, mode) + + +def make_path_posix(path): + """Make path generic and absolute for current OS""" + if not isinstance(path, str): + if isinstance(path, (list, set, tuple)): + return type(path)(make_path_posix(p) for p in path) + else: + path = stringify_path(path) + if not isinstance(path, str): + raise TypeError(f"could not convert {path!r} to string") + if os.sep == "/": + # Native posix + if path.startswith("/"): + # most common fast case for posix + return path + elif path.startswith("~"): + return osp.expanduser(path) + elif path.startswith("./"): + path = path[2:] + elif path == ".": + path = "" + return f"{os.getcwd()}/{path}" + else: + # NT handling + if path[0:1] == "/" and path[2:3] == ":": + # path is like "/c:/local/path" + path = path[1:] + if path[1:2] == ":": + # windows full path like "C:\\local\\path" + if len(path) <= 3: + # nt root (something like c:/) + return path[0] + ":/" + path = path.replace("\\", "/") + return path + elif path[0:1] == "~": + return make_path_posix(osp.expanduser(path)) + elif path.startswith(("\\\\", "//")): + # windows UNC/DFS-style paths + return "//" + path[2:].replace("\\", "/") + elif path.startswith(("\\", "/")): + # windows relative path with root + path = path.replace("\\", "/") + return f"{osp.splitdrive(os.getcwd())[0]}{path}" + else: + path = path.replace("\\", "/") + if path.startswith("./"): + path = path[2:] + elif path == ".": + path = "" + return f"{make_path_posix(os.getcwd())}/{path}" + + +def trailing_sep(path): + """Return True if the path ends with a path separator. + + A forward slash is always considered a path separator, even on Operating + Systems that normally use a backslash. + """ + # TODO: if all incoming paths were posix-compliant then separator would + # always be a forward slash, simplifying this function. + # See https://github.com/fsspec/filesystem_spec/pull/1250 + return path.endswith(os.sep) or (os.altsep is not None and path.endswith(os.altsep)) + + +@lru_cache(maxsize=1) +def get_umask(mask: int = 0o666) -> int: + """Get the current umask. + + Follows https://stackoverflow.com/a/44130549 to get the umask. + Temporarily sets the umask to the given value, and then resets it to the + original value. + """ + value = os.umask(mask) + os.umask(value) + return value + + +class LocalFileOpener(io.IOBase): + def __init__( + self, path, mode, autocommit=True, fs=None, compression=None, **kwargs + ): + logger.debug("open file: %s", path) + self.path = path + self.mode = mode + self.fs = fs + self.f = None + self.autocommit = autocommit + self.compression = get_compression(path, compression) + self.blocksize = io.DEFAULT_BUFFER_SIZE + self._open() + + def _open(self): + if self.f is None or self.f.closed: + if self.autocommit or "w" not in self.mode: + self.f = open(self.path, mode=self.mode) + if self.compression: + compress = compr[self.compression] + self.f = compress(self.f, mode=self.mode) + else: + # TODO: check if path is writable? + i, name = tempfile.mkstemp() + os.close(i) # we want normal open and normal buffered file + self.temp = name + self.f = open(name, mode=self.mode) + if "w" not in self.mode: + self.size = self.f.seek(0, 2) + self.f.seek(0) + self.f.size = self.size + + def _fetch_range(self, start, end): + # probably only used by cached FS + if "r" not in self.mode: + raise ValueError + self._open() + self.f.seek(start) + return self.f.read(end - start) + + def __setstate__(self, state): + self.f = None + loc = state.pop("loc", None) + self.__dict__.update(state) + if "r" in state["mode"]: + self.f = None + self._open() + self.f.seek(loc) + + def __getstate__(self): + d = self.__dict__.copy() + d.pop("f") + if "r" in self.mode: + d["loc"] = self.f.tell() + else: + if not self.f.closed: + raise ValueError("Cannot serialise open write-mode local file") + return d + + def commit(self): + if self.autocommit: + raise RuntimeError("Can only commit if not already set to autocommit") + try: + shutil.move(self.temp, self.path) + except PermissionError as e: + # shutil.move raises PermissionError if os.rename + # and the default copy2 fallback with shutil.copystats fail. + # The file should be there nonetheless, but without copied permissions. + # If it doesn't exist, there was no permission to create the file. + if not os.path.exists(self.path): + raise e + else: + # If PermissionError is not raised, permissions can be set. + try: + mask = 0o666 + os.chmod(self.path, mask & ~get_umask(mask)) + except RuntimeError: + pass + + def discard(self): + if self.autocommit: + raise RuntimeError("Cannot discard if set to autocommit") + os.remove(self.temp) + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return "r" not in self.mode + + def read(self, *args, **kwargs): + return self.f.read(*args, **kwargs) + + def write(self, *args, **kwargs): + return self.f.write(*args, **kwargs) + + def tell(self, *args, **kwargs): + return self.f.tell(*args, **kwargs) + + def seek(self, *args, **kwargs): + return self.f.seek(*args, **kwargs) + + def seekable(self, *args, **kwargs): + return self.f.seekable(*args, **kwargs) + + def readline(self, *args, **kwargs): + return self.f.readline(*args, **kwargs) + + def readlines(self, *args, **kwargs): + return self.f.readlines(*args, **kwargs) + + def close(self): + return self.f.close() + + def truncate(self, size=None) -> int: + return self.f.truncate(size) + + @property + def closed(self): + return self.f.closed + + def fileno(self): + return self.raw.fileno() + + def flush(self) -> None: + self.f.flush() + + def __iter__(self): + return self.f.__iter__() + + def __getattr__(self, item): + return getattr(self.f, item) + + def __enter__(self): + self._incontext = True + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._incontext = False + self.f.__exit__(exc_type, exc_value, traceback) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/memory.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..f6b67bbc84e7aa625eee5609c20f6a893ddd349e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/memory.py @@ -0,0 +1,311 @@ +from __future__ import annotations + +import logging +from datetime import datetime, timezone +from errno import ENOTEMPTY +from io import BytesIO +from pathlib import PurePath, PureWindowsPath +from typing import Any, ClassVar + +from fsspec import AbstractFileSystem +from fsspec.implementations.local import LocalFileSystem +from fsspec.utils import stringify_path + +logger = logging.getLogger("fsspec.memoryfs") + + +class MemoryFileSystem(AbstractFileSystem): + """A filesystem based on a dict of BytesIO objects + + This is a global filesystem so instances of this class all point to the same + in memory filesystem. + """ + + store: ClassVar[dict[str, Any]] = {} # global, do not overwrite! + pseudo_dirs = [""] # global, do not overwrite! + protocol = "memory" + root_marker = "/" + + @classmethod + def _strip_protocol(cls, path): + if isinstance(path, PurePath): + if isinstance(path, PureWindowsPath): + return LocalFileSystem._strip_protocol(path) + else: + path = stringify_path(path) + + path = path.removeprefix("memory://") + if "::" in path or "://" in path: + return path.rstrip("/") + path = path.lstrip("/").rstrip("/") + return "/" + path if path else "" + + def ls(self, path, detail=True, **kwargs): + path = self._strip_protocol(path) + if path in self.store: + # there is a key with this exact name + if not detail: + return [path] + return [ + { + "name": path, + "size": self.store[path].size, + "type": "file", + "created": self.store[path].created.timestamp(), + } + ] + paths = set() + starter = path + "/" + out = [] + for p2 in tuple(self.store): + if p2.startswith(starter): + if "/" not in p2[len(starter) :]: + # exact child + out.append( + { + "name": p2, + "size": self.store[p2].size, + "type": "file", + "created": self.store[p2].created.timestamp(), + } + ) + elif len(p2) > len(starter): + # implied child directory + ppath = starter + p2[len(starter) :].split("/", 1)[0] + if ppath not in paths: + out = out or [] + out.append( + { + "name": ppath, + "size": 0, + "type": "directory", + } + ) + paths.add(ppath) + for p2 in self.pseudo_dirs: + if p2.startswith(starter): + if "/" not in p2[len(starter) :]: + # exact child pdir + if p2 not in paths: + out.append({"name": p2, "size": 0, "type": "directory"}) + paths.add(p2) + else: + # directory implied by deeper pdir + ppath = starter + p2[len(starter) :].split("/", 1)[0] + if ppath not in paths: + out.append({"name": ppath, "size": 0, "type": "directory"}) + paths.add(ppath) + if not out: + if path in self.pseudo_dirs: + # empty dir + return [] + raise FileNotFoundError(path) + if detail: + return out + return sorted([f["name"] for f in out]) + + def mkdir(self, path, create_parents=True, **kwargs): + path = self._strip_protocol(path) + if path in self.store or path in self.pseudo_dirs: + raise FileExistsError(path) + if self._parent(path).strip("/") and self.isfile(self._parent(path)): + raise NotADirectoryError(self._parent(path)) + if create_parents and self._parent(path).strip("/"): + try: + self.mkdir(self._parent(path), create_parents, **kwargs) + except FileExistsError: + pass + if path and path not in self.pseudo_dirs: + self.pseudo_dirs.append(path) + + def makedirs(self, path, exist_ok=False): + try: + self.mkdir(path, create_parents=True) + except FileExistsError: + if not exist_ok: + raise + + def pipe_file(self, path, value, mode="overwrite", **kwargs): + """Set the bytes of given file + + Avoids copies of the data if possible + """ + mode = "xb" if mode == "create" else "wb" + self.open(path, mode=mode, data=value) + + def rmdir(self, path): + path = self._strip_protocol(path) + if path == "": + # silently avoid deleting FS root + return + if path in self.pseudo_dirs: + if not self.ls(path): + self.pseudo_dirs.remove(path) + else: + raise OSError(ENOTEMPTY, "Directory not empty", path) + else: + raise FileNotFoundError(path) + + def info(self, path, **kwargs): + logger.debug("info: %s", path) + path = self._strip_protocol(path) + if path in self.pseudo_dirs or any( + p.startswith(path + "/") for p in list(self.store) + self.pseudo_dirs + ): + return { + "name": path, + "size": 0, + "type": "directory", + } + elif path in self.store: + filelike = self.store[path] + return { + "name": path, + "size": filelike.size, + "type": "file", + "created": getattr(filelike, "created", None), + } + else: + raise FileNotFoundError(path) + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + path = self._strip_protocol(path) + if "x" in mode and self.exists(path): + raise FileExistsError + if path in self.pseudo_dirs: + raise IsADirectoryError(path) + parent = path + while len(parent) > 1: + parent = self._parent(parent) + if self.isfile(parent): + raise FileExistsError(parent) + if mode in ["rb", "ab", "r+b", "a+b"]: + if path in self.store: + f = self.store[path] + if "a" in mode: + # position at the end of file + f.seek(0, 2) + else: + # position at the beginning of file + f.seek(0) + return f + else: + raise FileNotFoundError(path) + elif mode in {"wb", "w+b", "xb", "x+b"}: + if "x" in mode and self.exists(path): + raise FileExistsError + m = MemoryFile(self, path, kwargs.get("data")) + if not self._intrans: + m.commit() + return m + else: + name = self.__class__.__name__ + raise ValueError(f"unsupported file mode for {name}: {mode!r}") + + def cp_file(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1) + path2 = self._strip_protocol(path2) + if self.isfile(path1): + self.store[path2] = MemoryFile( + self, path2, self.store[path1].getvalue() + ) # implicit copy + elif self.isdir(path1): + if path2 not in self.pseudo_dirs: + self.pseudo_dirs.append(path2) + else: + raise FileNotFoundError(path1) + + def cat_file(self, path, start=None, end=None, **kwargs): + logger.debug("cat: %s", path) + path = self._strip_protocol(path) + try: + return bytes(self.store[path].getbuffer()[start:end]) + except KeyError as e: + raise FileNotFoundError(path) from e + + def _rm(self, path): + path = self._strip_protocol(path) + try: + del self.store[path] + except KeyError as e: + raise FileNotFoundError(path) from e + + def modified(self, path): + path = self._strip_protocol(path) + try: + return self.store[path].modified + except KeyError as e: + raise FileNotFoundError(path) from e + + def created(self, path): + path = self._strip_protocol(path) + try: + return self.store[path].created + except KeyError as e: + raise FileNotFoundError(path) from e + + def isfile(self, path): + path = self._strip_protocol(path) + return path in self.store + + def rm(self, path, recursive=False, maxdepth=None): + if isinstance(path, str): + path = self._strip_protocol(path) + else: + path = [self._strip_protocol(p) for p in path] + paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) + for p in reversed(paths): + if self.isfile(p): + self.rm_file(p) + # If the expanded path doesn't exist, it is only because the expanded + # path was a directory that does not exist in self.pseudo_dirs. This + # is possible if you directly create files without making the + # directories first. + elif not self.exists(p): + continue + else: + self.rmdir(p) + + +class MemoryFile(BytesIO): + """A BytesIO which can't close and works as a context manager + + Can initialise with data. Each path should only be active once at any moment. + + No need to provide fs, path if auto-committing (default) + """ + + def __init__(self, fs=None, path=None, data=None): + logger.debug("open file %s", path) + self.fs = fs + self.path = path + self.created = datetime.now(tz=timezone.utc) + self.modified = datetime.now(tz=timezone.utc) + if data: + super().__init__(data) + self.seek(0) + + @property + def size(self): + return self.getbuffer().nbytes + + def __enter__(self): + return self + + def close(self): + pass + + def discard(self): + pass + + def commit(self): + self.fs.store[self.path] = self + self.modified = datetime.now(tz=timezone.utc) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/reference.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/reference.py new file mode 100644 index 0000000000000000000000000000000000000000..1b5e1b1b6bc938f1cbc1c595e761e4d062aea31d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/reference.py @@ -0,0 +1,1339 @@ +import base64 +import collections +import io +import itertools +import logging +import math +import os +from functools import lru_cache +from itertools import chain +from typing import TYPE_CHECKING, Literal + +import fsspec.core +from fsspec.spec import AbstractBufferedFile + +try: + import ujson as json +except ImportError: + if not TYPE_CHECKING: + import json + +from fsspec.asyn import AsyncFileSystem +from fsspec.callbacks import DEFAULT_CALLBACK +from fsspec.core import filesystem, open, split_protocol +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +from fsspec.utils import ( + isfilelike, + merge_offset_ranges, + other_paths, +) + +logger = logging.getLogger("fsspec.reference") + + +class ReferenceNotReachable(RuntimeError): + def __init__(self, reference, target, *args): + super().__init__(*args) + self.reference = reference + self.target = target + + def __str__(self): + return f'Reference "{self.reference}" failed to fetch target {self.target}' + + +def _first(d): + return next(iter(d.values())) + + +def _prot_in_references(path, references): + ref = references.get(path) + if isinstance(ref, (list, tuple)) and isinstance(ref[0], str): + return split_protocol(ref[0])[0] if ref[0] else ref[0] + + +def _protocol_groups(paths, references): + if isinstance(paths, str): + return {_prot_in_references(paths, references): [paths]} + out = {} + for path in paths: + protocol = _prot_in_references(path, references) + out.setdefault(protocol, []).append(path) + return out + + +class RefsValuesView(collections.abc.ValuesView): + def __iter__(self): + for val in self._mapping.zmetadata.values(): + yield json.dumps(val).encode() + yield from self._mapping._items.values() + for field in self._mapping.listdir(): + chunk_sizes = self._mapping._get_chunk_sizes(field) + if len(chunk_sizes) == 0: + yield self._mapping[field + "/0"] + continue + yield from self._mapping._generate_all_records(field) + + +class RefsItemsView(collections.abc.ItemsView): + def __iter__(self): + return zip(self._mapping.keys(), self._mapping.values()) + + +def ravel_multi_index(idx, sizes): + val = 0 + mult = 1 + for i, s in zip(idx[::-1], sizes[::-1]): + val += i * mult + mult *= s + return val + + +class LazyReferenceMapper(collections.abc.MutableMapping): + """This interface can be used to read/write references from Parquet stores. + It is not intended for other types of references. + It can be used with Kerchunk's MultiZarrToZarr method to combine + references into a parquet store. + Examples of this use-case can be found here: + https://fsspec.github.io/kerchunk/advanced.html?highlight=parquet#parquet-storage""" + + # import is class level to prevent numpy dep requirement for fsspec + @property + def np(self): + import numpy as np + + return np + + @property + def pd(self): + import pandas as pd + + return pd + + def __init__( + self, + root, + fs=None, + out_root=None, + cache_size=128, + categorical_threshold=10, + engine: Literal["fastparquet", "pyarrow"] = "fastparquet", + ): + """ + + This instance will be writable, storing changes in memory until full partitions + are accumulated or .flush() is called. + + To create an empty lazy store, use .create() + + Parameters + ---------- + root : str + Root of parquet store + fs : fsspec.AbstractFileSystem + fsspec filesystem object, default is local filesystem. + cache_size : int, default=128 + Maximum size of LRU cache, where cache_size*record_size denotes + the total number of references that can be loaded in memory at once. + categorical_threshold : int + Encode urls as pandas.Categorical to reduce memory footprint if the ratio + of the number of unique urls to total number of refs for each variable + is greater than or equal to this number. (default 10) + engine: Literal["fastparquet","pyarrow"] + Engine choice for reading parquet files. (default is "fastparquet") + """ + + self.root = root + self.chunk_sizes = {} + self.cat_thresh = categorical_threshold + self.engine = engine + self.cache_size = cache_size + self.url = self.root + "/{field}/refs.{record}.parq" + # TODO: derive fs from `root` + self.fs = fsspec.filesystem("file") if fs is None else fs + self.out_root = self.fs.unstrip_protocol(out_root or self.root) + + from importlib.util import find_spec + + if self.engine == "pyarrow" and find_spec("pyarrow") is None: + raise ImportError("engine choice `pyarrow` is not installed.") + + # Apply `lru_cache` decorator manually per instance. + # This way `self` reference is not held on class level. + # WARNING: However, this means that self and its members are not reflected + # in the cache key, so we expect they won't be mutated once a value is cached. + self.listdir = lru_cache()(self.listdir) + self._key_to_record = lru_cache(maxsize=4096)(self._key_to_record) + + def __getattr__(self, item): + if item in ("_items", "record_size", "zmetadata"): + self.setup() + # avoid possible recursion if setup fails somehow + return self.__dict__[item] + raise AttributeError(item) + + def setup(self): + self._items = {} + self._items[".zmetadata"] = self.fs.cat_file( + "/".join([self.root, ".zmetadata"]) + ) + met = json.loads(self._items[".zmetadata"]) + self.record_size = met["record_size"] + self.zmetadata = met["metadata"] + + # Define function to open and decompress refs + @lru_cache(maxsize=self.cache_size) + def open_refs(field, record): + """cached parquet file loader""" + path = self.url.format(field=field, record=record) + data = io.BytesIO(self.fs.cat_file(path)) + try: + df = self.pd.read_parquet(data, engine=self.engine) + refs = {c: df[c].to_numpy(copy=True) for c in df.columns} + except OSError: + refs = None + return refs + + self.open_refs = open_refs + + @staticmethod + def create(root, storage_options=None, fs=None, record_size=10000, **kwargs): + """Make empty parquet reference set + + First deletes the contents of the given directory, if it exists. + + Parameters + ---------- + root: str + Directory to contain the output; will be created + storage_options: dict | None + For making the filesystem to use for writing is fs is None + fs: FileSystem | None + Filesystem for writing + record_size: int + Number of references per parquet file + kwargs: passed to __init__ + + Returns + ------- + LazyReferenceMapper instance + """ + met = {"metadata": {}, "record_size": record_size} + if fs is None: + fs, root = fsspec.core.url_to_fs(root, **(storage_options or {})) + if fs.exists(root): + fs.rm(root, recursive=True) + fs.makedirs(root, exist_ok=True) + fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode()) + return LazyReferenceMapper(root, fs, **kwargs) + + def listdir(self): + """List top-level directories""" + dirs = (p.rsplit("/", 1)[0] for p in self.zmetadata if not p.startswith(".z")) + return set(dirs) + + def ls(self, path="", detail=True): + """Shortcut file listings""" + path = path.rstrip("/") + pathdash = path + "/" if path else "" + dirnames = self.listdir() + dirs = [ + d + for d in dirnames + if d.startswith(pathdash) and "/" not in d.lstrip(pathdash) + ] + if dirs: + others = { + f + for f in chain( + [".zmetadata"], + (name for name in self.zmetadata), + (name for name in self._items), + ) + if f.startswith(pathdash) and "/" not in f.lstrip(pathdash) + } + if detail is False: + others.update(dirs) + return sorted(others) + dirinfo = [{"name": name, "type": "directory", "size": 0} for name in dirs] + fileinfo = [ + { + "name": name, + "type": "file", + "size": len( + json.dumps(self.zmetadata[name]) + if name in self.zmetadata + else self._items[name] + ), + } + for name in others + ] + return sorted(dirinfo + fileinfo, key=lambda s: s["name"]) + field = path + others = set( + [name for name in self.zmetadata if name.startswith(f"{path}/")] + + [name for name in self._items if name.startswith(f"{path}/")] + ) + fileinfo = [ + { + "name": name, + "type": "file", + "size": len( + json.dumps(self.zmetadata[name]) + if name in self.zmetadata + else self._items[name] + ), + } + for name in others + ] + keys = self._keys_in_field(field) + + if detail is False: + return list(others) + list(keys) + recs = self._generate_all_records(field) + recinfo = [ + {"name": name, "type": "file", "size": rec[-1]} + for name, rec in zip(keys, recs) + if rec[0] # filters out path==None, deleted/missing + ] + return fileinfo + recinfo + + def _load_one_key(self, key): + """Get the reference for one key + + Returns bytes, one-element list or three-element list. + """ + if key in self._items: + return self._items[key] + elif key in self.zmetadata: + return json.dumps(self.zmetadata[key]).encode() + elif "/" not in key or self._is_meta(key): + raise KeyError(key) + field, _ = key.rsplit("/", 1) + record, ri, chunk_size = self._key_to_record(key) + maybe = self._items.get((field, record), {}).get(ri, False) + if maybe is None: + # explicitly deleted + raise KeyError + elif maybe: + return maybe + elif chunk_size == 0: + return b"" + + # Chunk keys can be loaded from row group and cached in LRU cache + try: + refs = self.open_refs(field, record) + except (ValueError, TypeError, FileNotFoundError) as exc: + raise KeyError(key) from exc + columns = ["path", "offset", "size", "raw"] + selection = [refs[c][ri] if c in refs else None for c in columns] + raw = selection[-1] + if raw is not None: + return raw + if selection[0] is None: + raise KeyError("This reference does not exist or has been deleted") + if selection[1:3] == [0, 0]: + # URL only + return selection[:1] + # URL, offset, size + return selection[:3] + + def _key_to_record(self, key): + """Details needed to construct a reference for one key""" + field, chunk = key.rsplit("/", 1) + chunk_sizes = self._get_chunk_sizes(field) + if len(chunk_sizes) == 0: + return 0, 0, 0 + chunk_idx = [int(c) for c in chunk.split(".")] + chunk_number = ravel_multi_index(chunk_idx, chunk_sizes) + record = chunk_number // self.record_size + ri = chunk_number % self.record_size + return record, ri, len(chunk_sizes) + + def _get_chunk_sizes(self, field): + """The number of chunks along each axis for a given field""" + if field not in self.chunk_sizes: + zarray = self.zmetadata[f"{field}/.zarray"] + size_ratio = [ + math.ceil(s / c) for s, c in zip(zarray["shape"], zarray["chunks"]) + ] + self.chunk_sizes[field] = size_ratio or [1] + return self.chunk_sizes[field] + + def _generate_record(self, field, record): + """The references for a given parquet file of a given field""" + refs = self.open_refs(field, record) + it = iter(zip(*refs.values())) + if len(refs) == 3: + # All urls + return (list(t) for t in it) + elif len(refs) == 1: + # All raws + return refs["raw"] + else: + # Mix of urls and raws + return (list(t[:3]) if not t[3] else t[3] for t in it) + + def _generate_all_records(self, field): + """Load all the references within a field by iterating over the parquet files""" + nrec = 1 + for ch in self._get_chunk_sizes(field): + nrec *= ch + nrec = math.ceil(nrec / self.record_size) + for record in range(nrec): + yield from self._generate_record(field, record) + + def values(self): + return RefsValuesView(self) + + def items(self): + return RefsItemsView(self) + + def __hash__(self): + return id(self) + + def __getitem__(self, key): + return self._load_one_key(key) + + def __setitem__(self, key, value): + if "/" in key and not self._is_meta(key): + field, chunk = key.rsplit("/", 1) + record, i, _ = self._key_to_record(key) + subdict = self._items.setdefault((field, record), {}) + subdict[i] = value + if len(subdict) == self.record_size: + self.write(field, record) + else: + # metadata or top-level + if hasattr(value, "to_bytes"): + val = value.to_bytes().decode() + elif isinstance(value, bytes): + val = value.decode() + else: + val = value + self._items[key] = val + new_value = json.loads(val) + self.zmetadata[key] = {**self.zmetadata.get(key, {}), **new_value} + + @staticmethod + def _is_meta(key): + return key.startswith(".z") or "/.z" in key + + def __delitem__(self, key): + if key in self._items: + del self._items[key] + elif key in self.zmetadata: + del self.zmetadata[key] + else: + if "/" in key and not self._is_meta(key): + field, _ = key.rsplit("/", 1) + record, i, _ = self._key_to_record(key) + subdict = self._items.setdefault((field, record), {}) + subdict[i] = None + if len(subdict) == self.record_size: + self.write(field, record) + else: + # metadata or top-level + self._items[key] = None + + def write(self, field, record, base_url=None, storage_options=None): + # extra requirements if writing + import kerchunk.df + import numpy as np + import pandas as pd + + partition = self._items[(field, record)] + original = False + if len(partition) < self.record_size: + try: + original = self.open_refs(field, record) + except OSError: + pass + + if original: + paths = original["path"] + offsets = original["offset"] + sizes = original["size"] + raws = original["raw"] + else: + paths = np.full(self.record_size, np.nan, dtype="O") + offsets = np.zeros(self.record_size, dtype="int64") + sizes = np.zeros(self.record_size, dtype="int64") + raws = np.full(self.record_size, np.nan, dtype="O") + for j, data in partition.items(): + if isinstance(data, list): + if ( + str(paths.dtype) == "category" + and data[0] not in paths.dtype.categories + ): + paths = paths.add_categories(data[0]) + paths[j] = data[0] + if len(data) > 1: + offsets[j] = data[1] + sizes[j] = data[2] + elif data is None: + # delete + paths[j] = None + offsets[j] = 0 + sizes[j] = 0 + raws[j] = None + else: + # this is the only call into kerchunk, could remove + raws[j] = kerchunk.df._proc_raw(data) + # TODO: only save needed columns + df = pd.DataFrame( + { + "path": paths, + "offset": offsets, + "size": sizes, + "raw": raws, + }, + copy=False, + ) + if df.path.count() / (df.path.nunique() or 1) > self.cat_thresh: + df["path"] = df["path"].astype("category") + object_encoding = {"raw": "bytes", "path": "utf8"} + has_nulls = ["path", "raw"] + + fn = f"{base_url or self.out_root}/{field}/refs.{record}.parq" + self.fs.mkdirs(f"{base_url or self.out_root}/{field}", exist_ok=True) + + if self.engine == "pyarrow": + df_backend_kwargs = {"write_statistics": False} + elif self.engine == "fastparquet": + df_backend_kwargs = { + "stats": False, + "object_encoding": object_encoding, + "has_nulls": has_nulls, + } + else: + raise NotImplementedError(f"{self.engine} not supported") + df.to_parquet( + fn, + engine=self.engine, + storage_options=storage_options + or getattr(self.fs, "storage_options", None), + compression="zstd", + index=False, + **df_backend_kwargs, + ) + + partition.clear() + self._items.pop((field, record)) + + def flush(self, base_url=None, storage_options=None): + """Output any modified or deleted keys + + Parameters + ---------- + base_url: str + Location of the output + """ + + # write what we have so far and clear sub chunks + for thing in list(self._items): + if isinstance(thing, tuple): + field, record = thing + self.write( + field, + record, + base_url=base_url, + storage_options=storage_options, + ) + + # gather .zmetadata from self._items and write that too + for k in list(self._items): + if k != ".zmetadata" and ".z" in k: + self.zmetadata[k] = json.loads(self._items.pop(k)) + met = {"metadata": self.zmetadata, "record_size": self.record_size} + self._items.clear() + self._items[".zmetadata"] = json.dumps(met).encode() + self.fs.pipe( + "/".join([base_url or self.out_root, ".zmetadata"]), + self._items[".zmetadata"], + ) + + # TODO: only clear those that we wrote to? + self.open_refs.cache_clear() + + def __len__(self): + # Caveat: This counts expected references, not actual - but is fast + count = 0 + for field in self.listdir(): + if field.startswith("."): + count += 1 + else: + count += math.prod(self._get_chunk_sizes(field)) + count += len(self.zmetadata) # all metadata keys + # any other files not in reference partitions + count += sum(1 for _ in self._items if not isinstance(_, tuple)) + return count + + def __iter__(self): + # Caveat: returns only existing keys, so the number of these does not + # match len(self) + metas = set(self.zmetadata) + metas.update(self._items) + for bit in metas: + if isinstance(bit, str): + yield bit + for field in self.listdir(): + for k in self._keys_in_field(field): + if k in self: + yield k + + def __contains__(self, item): + try: + self._load_one_key(item) + return True + except KeyError: + return False + + def _keys_in_field(self, field): + """List key names in given field + + Produces strings like "field/x.y" appropriate from the chunking of the array + """ + chunk_sizes = self._get_chunk_sizes(field) + if len(chunk_sizes) == 0: + yield field + "/0" + return + inds = itertools.product(*(range(i) for i in chunk_sizes)) + for ind in inds: + yield field + "/" + ".".join([str(c) for c in ind]) + + +class ReferenceFileSystem(AsyncFileSystem): + """View byte ranges of some other file as a file system + Initial version: single file system target, which must support + async, and must allow start and end args in _cat_file. Later versions + may allow multiple arbitrary URLs for the targets. + This FileSystem is read-only. It is designed to be used with async + targets (for now). We do not get original file details from the target FS. + Configuration is by passing a dict of references at init, or a URL to + a JSON file containing the same; this dict + can also contain concrete data for some set of paths. + Reference dict format: + {path0: bytes_data, path1: (target_url, offset, size)} + https://github.com/fsspec/kerchunk/blob/main/README.md + + simple_references: if True (default), no jinja interpreting is done, + which is the safe option. + """ + + protocol = "reference" + cachable = False + + def __init__( + self, + fo, + target=None, + ref_storage_args=None, + target_protocol=None, + target_options=None, + remote_protocol=None, + remote_options=None, + fs=None, + template_overrides=None, + simple_templates=True, + max_gap=64_000, + max_block=256_000_000, + cache_size=128, + **kwargs, + ): + """ + Parameters + ---------- + fo : dict or str + The set of references to use for this instance, with a structure as above. + If str referencing a JSON file, will use fsspec.open, in conjunction + with target_options and target_protocol to open and parse JSON at this + location. If a directory, then assume references are a set of parquet + files to be loaded lazily. + target : str + For any references having target_url as None, this is the default file + target to use + ref_storage_args : dict + If references is a str, use these kwargs for loading the JSON file. + Deprecated: use target_options instead. + target_protocol : str + Used for loading the reference file, if it is a path. If None, protocol + will be derived from the given path + target_options : dict + Extra FS options for loading the reference file ``fo``, if given as a path + remote_protocol : str + The protocol of the filesystem on which the references will be evaluated + (unless fs is provided). If not given, will be derived from the first + URL that has a protocol in the templates or in the references, in that + order. + remote_options : dict + kwargs to go with remote_protocol + fs : AbstractFileSystem | dict(str, (AbstractFileSystem | dict)) + Directly provide a file system(s): + - a single filesystem instance + - a dict of protocol:filesystem, where each value is either a filesystem + instance, or a dict of kwargs that can be used to create in + instance for the given protocol + + If this is given, remote_options and remote_protocol are ignored. + template_overrides : dict + Swap out any templates in the references file with these - useful for + testing. + simple_templates: bool + Whether templates can be processed with simple replace (True) or if + jinja is needed (False, much slower). All reference sets produced by + ``kerchunk`` are simple in this sense, but the spec allows for complex. + max_gap, max_block: int + For merging multiple concurrent requests to the same remote file. + Neighboring byte ranges will only be merged when their + inter-range gap is <= ``max_gap``. Default is 64KB. Set to 0 + to only merge when it requires no extra bytes. Pass a negative + number to disable merging, appropriate for local target files. + Neighboring byte ranges will only be merged when the size of + the aggregated range is <= ``max_block``. Default is 256MB. + cache_size : int + Maximum size of LRU cache, where cache_size*record_size denotes + the total number of references that can be loaded in memory at once. + Only used for lazily loaded references. + kwargs : passed to parent class + """ + super().__init__(**kwargs) + self.target = target + self.template_overrides = template_overrides + self.simple_templates = simple_templates + self.templates = {} + self.fss = {} + self._dircache = {} + self.max_gap = max_gap + self.max_block = max_block + if isinstance(fo, str): + dic = dict( + **(ref_storage_args or target_options or {}), protocol=target_protocol + ) + ref_fs, fo2 = fsspec.core.url_to_fs(fo, **dic) + if ".json" not in fo2 and ( + fo.endswith(("parq", "parquet", "/")) or ref_fs.isdir(fo2) + ): + # Lazy parquet refs + logger.info("Open lazy reference dict from URL %s", fo) + self.references = LazyReferenceMapper( + fo2, + fs=ref_fs, + cache_size=cache_size, + ) + else: + # text JSON + with fsspec.open(fo, "rb", **dic) as f: + logger.info("Read reference from URL %s", fo) + text = json.load(f) + self._process_references(text, template_overrides) + else: + # dictionaries + self._process_references(fo, template_overrides) + if isinstance(fs, dict): + self.fss = { + k: ( + fsspec.filesystem(k.split(":", 1)[0], **opts) + if isinstance(opts, dict) + else opts + ) + for k, opts in fs.items() + } + if None not in self.fss: + self.fss[None] = filesystem("file") + return + if fs is not None: + # single remote FS + remote_protocol = ( + fs.protocol[0] if isinstance(fs.protocol, tuple) else fs.protocol + ) + self.fss[remote_protocol] = fs + + if remote_protocol is None: + # get single protocol from any templates + for ref in self.templates.values(): + if callable(ref): + ref = ref() + protocol, _ = fsspec.core.split_protocol(ref) + if protocol and protocol not in self.fss: + fs = filesystem(protocol, **(remote_options or {})) + self.fss[protocol] = fs + if remote_protocol is None: + # get single protocol from references + # TODO: warning here, since this can be very expensive? + for ref in self.references.values(): + if callable(ref): + ref = ref() + if isinstance(ref, list) and ref[0]: + protocol, _ = fsspec.core.split_protocol(ref[0]) + if protocol not in self.fss: + fs = filesystem(protocol, **(remote_options or {})) + self.fss[protocol] = fs + # only use first remote URL + break + + if remote_protocol and remote_protocol not in self.fss: + fs = filesystem(remote_protocol, **(remote_options or {})) + self.fss[remote_protocol] = fs + + self.fss[None] = fs or filesystem("file") # default one + # Wrap any non-async filesystems to ensure async methods are available below + for k, f in self.fss.items(): + if not f.async_impl: + self.fss[k] = AsyncFileSystemWrapper(f, asynchronous=self.asynchronous) + elif self.asynchronous ^ f.asynchronous: + raise ValueError( + "Reference-FS's target filesystem must have same value " + "of asynchronous" + ) + + def _cat_common(self, path, start=None, end=None): + path = self._strip_protocol(path) + logger.debug(f"cat: {path}") + try: + part = self.references[path] + except KeyError as exc: + raise FileNotFoundError(path) from exc + if isinstance(part, str): + part = part.encode() + if hasattr(part, "to_bytes"): + part = part.to_bytes() + if isinstance(part, bytes): + logger.debug(f"Reference: {path}, type bytes") + if part.startswith(b"base64:"): + part = base64.b64decode(part[7:]) + return part, None, None + + if len(part) == 1: + logger.debug(f"Reference: {path}, whole file => {part}") + url = part[0] + start1, end1 = start, end + else: + url, start0, size = part + logger.debug(f"Reference: {path} => {url}, offset {start0}, size {size}") + end0 = start0 + size + + if start is not None: + if start >= 0: + start1 = start0 + start + else: + start1 = end0 + start + else: + start1 = start0 + if end is not None: + if end >= 0: + end1 = start0 + end + else: + end1 = end0 + end + else: + end1 = end0 + if url is None: + url = self.target + return url, start1, end1 + + async def _cat_file(self, path, start=None, end=None, **kwargs): + part_or_url, start0, end0 = self._cat_common(path, start=start, end=end) + if isinstance(part_or_url, bytes): + return part_or_url[start:end] + protocol, _ = split_protocol(part_or_url) + try: + return await self.fss[protocol]._cat_file( + part_or_url, start=start0, end=end0 + ) + except Exception as e: + raise ReferenceNotReachable(path, part_or_url) from e + + def cat_file(self, path, start=None, end=None, **kwargs): + part_or_url, start0, end0 = self._cat_common(path, start=start, end=end) + if isinstance(part_or_url, bytes): + return part_or_url[start:end] + protocol, _ = split_protocol(part_or_url) + try: + return self.fss[protocol].cat_file(part_or_url, start=start0, end=end0) + except Exception as e: + raise ReferenceNotReachable(path, part_or_url) from e + + def pipe_file(self, path, value, **_): + """Temporarily add binary data or reference as a file""" + self.references[path] = value + + async def _get_file(self, rpath, lpath, **kwargs): + if self.isdir(rpath): + return os.makedirs(lpath, exist_ok=True) + data = await self._cat_file(rpath) + with open(lpath, "wb") as f: + f.write(data) + + def get_file(self, rpath, lpath, callback=DEFAULT_CALLBACK, **kwargs): + if self.isdir(rpath): + return os.makedirs(lpath, exist_ok=True) + data = self.cat_file(rpath, **kwargs) + callback.set_size(len(data)) + if isfilelike(lpath): + lpath.write(data) + else: + with open(lpath, "wb") as f: + f.write(data) + callback.absolute_update(len(data)) + + def get(self, rpath, lpath, recursive=False, **kwargs): + if recursive: + # trigger directory build + self.ls("") + rpath = self.expand_path(rpath, recursive=recursive) + fs = fsspec.filesystem("file", auto_mkdir=True) + targets = other_paths(rpath, lpath) + if recursive: + data = self.cat([r for r in rpath if not self.isdir(r)]) + else: + data = self.cat(rpath) + for remote, local in zip(rpath, targets): + if remote in data: + fs.pipe_file(local, data[remote]) + + def cat(self, path, recursive=False, on_error="raise", **kwargs): + if isinstance(path, str) and recursive: + raise NotImplementedError + if isinstance(path, list) and (recursive or any("*" in p for p in path)): + raise NotImplementedError + # TODO: if references is lazy, pre-fetch all paths in batch before access + proto_dict = _protocol_groups(path, self.references) + out = {} + for proto, paths in proto_dict.items(): + fs = self.fss[proto] + urls, starts, ends, valid_paths = [], [], [], [] + for p in paths: + # find references or label not-found. Early exit if any not + # found and on_error is "raise" + try: + u, s, e = self._cat_common(p) + if not isinstance(u, (bytes, str)): + # nan/None from parquet + continue + except FileNotFoundError as err: + if on_error == "raise": + raise + if on_error != "omit": + out[p] = err + else: + urls.append(u) + starts.append(s) + ends.append(e) + valid_paths.append(p) + + # process references into form for merging + urls2 = [] + starts2 = [] + ends2 = [] + paths2 = [] + whole_files = set() + for u, s, e, p in zip(urls, starts, ends, valid_paths): + if isinstance(u, bytes): + # data + out[p] = u + elif s is None: + # whole file - limits are None, None, but no further + # entries take for this file + whole_files.add(u) + urls2.append(u) + starts2.append(s) + ends2.append(e) + paths2.append(p) + for u, s, e, p in zip(urls, starts, ends, valid_paths): + # second run to account for files that are to be loaded whole + if s is not None and u not in whole_files: + urls2.append(u) + starts2.append(s) + ends2.append(e) + paths2.append(p) + + # merge and fetch consolidated ranges + new_paths, new_starts, new_ends = merge_offset_ranges( + list(urls2), + list(starts2), + list(ends2), + sort=True, + max_gap=self.max_gap, + max_block=self.max_block, + ) + bytes_out = fs.cat_ranges(new_paths, new_starts, new_ends) + + # unbundle from merged bytes - simple approach + for u, s, e, p in zip(urls, starts, ends, valid_paths): + if p in out: + continue # was bytes, already handled + for np, ns, ne, b in zip(new_paths, new_starts, new_ends, bytes_out): + if np == u and (ns is None or ne is None): + if isinstance(b, Exception): + out[p] = b + else: + out[p] = b[s:e] + elif np == u and s >= ns and e <= ne: + if isinstance(b, Exception): + out[p] = b + else: + out[p] = b[s - ns : (e - ne) or None] + + for k, v in out.copy().items(): + # these were valid references, but fetch failed, so transform exc + if isinstance(v, Exception) and k in self.references: + ex = out[k] + new_ex = ReferenceNotReachable(k, self.references[k]) + new_ex.__cause__ = ex + if on_error == "raise": + raise new_ex + elif on_error != "omit": + out[k] = new_ex + + if len(out) == 1 and isinstance(path, str) and "*" not in path: + return _first(out) + return out + + def _process_references(self, references, template_overrides=None): + vers = references.get("version", None) + if vers is None: + self._process_references0(references) + elif vers == 1: + self._process_references1(references, template_overrides=template_overrides) + else: + raise ValueError(f"Unknown reference spec version: {vers}") + # TODO: we make dircache by iterating over all entries, but for Spec >= 1, + # can replace with programmatic. Is it even needed for mapper interface? + + def _process_references0(self, references): + """Make reference dict for Spec Version 0""" + if isinstance(references, dict): + # do not do this for lazy/parquet backend, which will not make dicts, + # but must remain writable in the original object + references = { + key: json.dumps(val) if isinstance(val, dict) else val + for key, val in references.items() + } + self.references = references + + def _process_references1(self, references, template_overrides=None): + if not self.simple_templates or self.templates: + import jinja2.sandbox + self.references = {} + self._process_templates(references.get("templates", {})) + + @lru_cache(1000) + def _render_jinja(u): + return ( + jinja2.sandbox.SandboxedEnvironment() + .from_string(u) + .render(**self.templates) + ) + + for k, v in references.get("refs", {}).items(): + if isinstance(v, str): + if v.startswith("base64:"): + self.references[k] = base64.b64decode(v[7:]) + self.references[k] = v + elif isinstance(v, dict): + self.references[k] = json.dumps(v) + elif self.templates: + u = v[0] + if "{{" in u: + if self.simple_templates: + u = ( + u.replace("{{", "{") + .replace("}}", "}") + .format(**self.templates) + ) + else: + u = _render_jinja(u) + self.references[k] = [u] if len(v) == 1 else [u, v[1], v[2]] + else: + self.references[k] = v + self.references.update(self._process_gen(references.get("gen", []))) + + def _process_templates(self, tmp): + self.templates = {} + if self.template_overrides is not None: + tmp.update(self.template_overrides) + for k, v in tmp.items(): + if "{{" in v: + import jinja2.sandbox + + self.templates[k] = ( + lambda temp=v, **kwargs: jinja2.sandbox.SandboxedEnvironment() + .from_string(temp) + .render(**kwargs) + ) + else: + self.templates[k] = v + + def _process_gen(self, gens): + out = {} + if self.simple_templates: + return out + for gen in gens: + dimension = { + k: ( + v + if isinstance(v, list) + else range(v.get("start", 0), v["stop"], v.get("step", 1)) + ) + for k, v in gen["dimensions"].items() + } + products = ( + dict(zip(dimension.keys(), values)) + for values in itertools.product(*dimension.values()) + ) + for pr in products: + import jinja2.sandbox + + key = ( + jinja2.sandbox.SandboxedEnvironment() + .from_string(gen["key"]) + .render(**pr, **self.templates) + ) + url = ( + jinja2.sandbox.SandboxedEnvironment() + .from_string(gen["url"]) + .render(**pr, **self.templates) + ) + if ("offset" in gen) and ("length" in gen): + offset = int( + jinja2.sandbox.SandboxedEnvironment() + .from_string(gen["offset"]) + .render(**pr, **self.templates) + ) + length = int( + jinja2.sandbox.SandboxedEnvironment() + .from_string(gen["length"]) + .render(**pr, **self.templates) + ) + out[key] = [url, offset, length] + elif ("offset" in gen) ^ ("length" in gen): + raise ValueError( + "Both 'offset' and 'length' are required for a " + "reference generator entry if either is provided." + ) + else: + out[key] = [url] + return out + + def _dircache_from_items(self): + self.dircache = {"": []} + it = self.references.items() + for path, part in it: + if isinstance(part, (bytes, str)) or hasattr(part, "to_bytes"): + size = len(part) + elif len(part) == 1: + size = None + else: + _, _, size = part + par = path.rsplit("/", 1)[0] if "/" in path else "" + par0 = par + subdirs = [par0] + while par0 and par0 not in self.dircache: + # collect parent directories + par0 = self._parent(par0) + subdirs.append(par0) + + subdirs.reverse() + for parent, child in zip(subdirs, subdirs[1:]): + # register newly discovered directories + assert child not in self.dircache + assert parent in self.dircache + self.dircache[parent].append( + {"name": child, "type": "directory", "size": 0} + ) + self.dircache[child] = [] + + self.dircache[par].append({"name": path, "type": "file", "size": size}) + + def _open(self, path, mode="rb", block_size=None, cache_options=None, **kwargs): + part_or_url, start0, end0 = self._cat_common(path) + # This logic is kept outside `ReferenceFile` to avoid unnecessary redirection. + # That does mean `_cat_common` gets called twice if it eventually reaches `ReferenceFile`. + if isinstance(part_or_url, bytes): + return io.BytesIO(part_or_url[start0:end0]) + + protocol, _ = split_protocol(part_or_url) + if start0 is None and end0 is None: + return self.fss[protocol]._open( + part_or_url, + mode, + block_size=block_size, + cache_options=cache_options, + **kwargs, + ) + + return ReferenceFile( + self, + path, + mode, + block_size=block_size, + cache_options=cache_options, + **kwargs, + ) + + def ls(self, path, detail=True, **kwargs): + logger.debug("list %s", path) + path = self._strip_protocol(path) + if isinstance(self.references, LazyReferenceMapper): + try: + return self.references.ls(path, detail) + except KeyError: + pass + raise FileNotFoundError(f"'{path}' is not a known key") + if not self.dircache: + self._dircache_from_items() + out = self._ls_from_cache(path) + if out is None: + raise FileNotFoundError(path) + if detail: + return out + return [o["name"] for o in out] + + def exists(self, path, **kwargs): # overwrite auto-sync version + return self.isdir(path) or self.isfile(path) + + def isdir(self, path): # overwrite auto-sync version + if self.dircache: + return path in self.dircache + elif isinstance(self.references, LazyReferenceMapper): + return path in self.references.listdir() + else: + # this may be faster than building dircache for single calls, but + # by looping will be slow for many calls; could cache it? + return any(_.startswith(f"{path}/") for _ in self.references) + + def isfile(self, path): # overwrite auto-sync version + return path in self.references + + async def _ls(self, path, detail=True, **kwargs): # calls fast sync code + return self.ls(path, detail, **kwargs) + + def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): + if withdirs: + return super().find( + path, maxdepth=maxdepth, withdirs=withdirs, detail=detail, **kwargs + ) + if path: + path = self._strip_protocol(path) + r = sorted(k for k in self.references if k.startswith(path)) + else: + r = sorted(self.references) + if detail: + if not self.dircache: + self._dircache_from_items() + return {k: self._ls_from_cache(k)[0] for k in r} + else: + return r + + def info(self, path, **kwargs): + out = self.references.get(path) + if out is not None: + if isinstance(out, (str, bytes)): + # decode base64 here + return {"name": path, "type": "file", "size": len(out)} + elif len(out) > 1: + return {"name": path, "type": "file", "size": out[2]} + else: + out0 = [{"name": path, "type": "file", "size": None}] + else: + out = self.ls(path, True) + out0 = [o for o in out if o["name"] == path] + if not out0: + return {"name": path, "type": "directory", "size": 0} + if out0[0]["size"] is None: + # if this is a whole remote file, update size using remote FS + prot, _ = split_protocol(self.references[path][0]) + out0[0]["size"] = self.fss[prot].size(self.references[path][0]) + return out0[0] + + async def _info(self, path, **kwargs): # calls fast sync code + return self.info(path) + + async def _rm_file(self, path, **kwargs): + self.references.pop( + path, None + ) # ignores FileNotFound, just as well for directories + self.dircache.clear() # this is a bit heavy handed + + async def _pipe_file(self, path, data, mode="overwrite", **kwargs): + if mode == "create" and self.exists(path): + raise FileExistsError + # can be str or bytes + self.references[path] = data + self.dircache.clear() # this is a bit heavy handed + + async def _put_file(self, lpath, rpath, mode="overwrite", **kwargs): + # puts binary + if mode == "create" and self.exists(rpath): + raise FileExistsError + with open(lpath, "rb") as f: + self.references[rpath] = f.read() + self.dircache.clear() # this is a bit heavy handed + + def save_json(self, url, **storage_options): + """Write modified references into new location""" + out = {} + for k, v in self.references.items(): + if isinstance(v, bytes): + try: + out[k] = v.decode("ascii") + except UnicodeDecodeError: + out[k] = (b"base64:" + base64.b64encode(v)).decode() + else: + out[k] = v + with fsspec.open(url, "wb", **storage_options) as f: + f.write(json.dumps({"version": 1, "refs": out}).encode()) + + +class ReferenceFile(AbstractBufferedFile): + def __init__( + self, + fs, + path, + mode="rb", + block_size="default", + autocommit=True, + cache_type="readahead", + cache_options=None, + size=None, + **kwargs, + ): + super().__init__( + fs, + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + size=size, + cache_type=cache_type, + cache_options=cache_options, + **kwargs, + ) + part_or_url, self.start, self.end = self.fs._cat_common(self.path) + protocol, _ = split_protocol(part_or_url) + self.src_fs = self.fs.fss[protocol] + self.src_path = part_or_url + self._f = None + + @property + def f(self): + if self._f is None or self._f.closed: + self._f = self.src_fs._open( + self.src_path, + mode=self.mode, + block_size=self.blocksize, + autocommit=self.autocommit, + cache_type="none", + **self.kwargs, + ) + return self._f + + def close(self): + if self._f is not None: + self._f.close() + return super().close() + + def _fetch_range(self, start, end): + start = start + self.start + end = min(end + self.start, self.end) + self.f.seek(start) + return self.f.read(end - start) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/sftp.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/sftp.py new file mode 100644 index 0000000000000000000000000000000000000000..7c347963d692d50390b131225a56477b328f7a3c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/sftp.py @@ -0,0 +1,187 @@ +import datetime +import logging +import os +import types +import uuid +from stat import S_ISDIR, S_ISLNK + +import paramiko + +from .. import AbstractFileSystem +from ..utils import infer_storage_options + +logger = logging.getLogger("fsspec.sftp") + + +class SFTPFileSystem(AbstractFileSystem): + """Files over SFTP/SSH + + Peer-to-peer filesystem over SSH using paramiko. + + Note: if using this with the ``open`` or ``open_files``, with full URLs, + there is no way to tell if a path is relative, so all paths are assumed + to be absolute. + """ + + protocol = "sftp", "ssh" + + def __init__(self, host, **ssh_kwargs): + """ + + Parameters + ---------- + host: str + Hostname or IP as a string + temppath: str + Location on the server to put files, when within a transaction + ssh_kwargs: dict + Parameters passed on to connection. See details in + https://docs.paramiko.org/en/3.3/api/client.html#paramiko.client.SSHClient.connect + May include port, username, password... + """ + if self._cached: + return + super().__init__(**ssh_kwargs) + self.temppath = ssh_kwargs.pop("temppath", "/tmp") # remote temp directory + self.host = host + self.ssh_kwargs = ssh_kwargs + self._connect() + + def _connect(self): + logger.debug("Connecting to SFTP server %s", self.host) + self.client = paramiko.SSHClient() + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.client.connect(self.host, **self.ssh_kwargs) + self.ftp = self.client.open_sftp() + + @classmethod + def _strip_protocol(cls, path): + return infer_storage_options(path)["path"] + + @staticmethod + def _get_kwargs_from_urls(urlpath): + out = infer_storage_options(urlpath) + out.pop("path", None) + out.pop("protocol", None) + return out + + def mkdir(self, path, create_parents=True, mode=511): + path = self._strip_protocol(path) + logger.debug("Creating folder %s", path) + if self.exists(path): + raise FileExistsError(f"File exists: {path}") + + if create_parents: + self.makedirs(path) + else: + self.ftp.mkdir(path, mode) + + def makedirs(self, path, exist_ok=False, mode=511): + if self.exists(path) and not exist_ok: + raise FileExistsError(f"File exists: {path}") + + parts = path.split("/") + new_path = "/" if path[:1] == "/" else "" + + for part in parts: + if part: + new_path = f"{new_path}/{part}" if new_path else part + if not self.exists(new_path): + self.ftp.mkdir(new_path, mode) + + def rmdir(self, path): + path = self._strip_protocol(path) + logger.debug("Removing folder %s", path) + self.ftp.rmdir(path) + + def info(self, path): + path = self._strip_protocol(path) + stat = self._decode_stat(self.ftp.stat(path)) + stat["name"] = path + return stat + + @staticmethod + def _decode_stat(stat, parent_path=None): + if S_ISDIR(stat.st_mode): + t = "directory" + elif S_ISLNK(stat.st_mode): + t = "link" + else: + t = "file" + out = { + "name": "", + "size": stat.st_size, + "type": t, + "uid": stat.st_uid, + "gid": stat.st_gid, + "time": datetime.datetime.fromtimestamp( + stat.st_atime, tz=datetime.timezone.utc + ), + "mtime": datetime.datetime.fromtimestamp( + stat.st_mtime, tz=datetime.timezone.utc + ), + } + if parent_path: + out["name"] = "/".join([parent_path.rstrip("/"), stat.filename]) + return out + + def ls(self, path, detail=False): + path = self._strip_protocol(path) + logger.debug("Listing folder %s", path) + stats = [self._decode_stat(stat, path) for stat in self.ftp.listdir_iter(path)] + if detail: + return stats + else: + paths = [stat["name"] for stat in stats] + return sorted(paths) + + def put_file(self, lpath, rpath, callback=None, **kwargs): + self.mkdirs(self._parent(os.fspath(rpath)), exist_ok=True) + logger.debug("Put file %s into %s", lpath, rpath) + self.ftp.put(lpath, rpath) + + def get_file(self, rpath, lpath, **kwargs): + if self.isdir(rpath): + os.makedirs(lpath, exist_ok=True) + else: + self.ftp.get(self._strip_protocol(rpath), lpath) + + def _open(self, path, mode="rb", block_size=None, **kwargs): + """ + block_size: int or None + If 0, no buffering, if 1, line buffering, if >1, buffer that many + bytes, if None use default from paramiko. + """ + logger.debug("Opening file %s", path) + if kwargs.get("autocommit", True) is False: + # writes to temporary file, move on commit + path2 = "/".join([self.temppath, str(uuid.uuid4())]) + f = self.ftp.open(path2, mode, bufsize=block_size if block_size else -1) + f.temppath = path2 + f.targetpath = path + f.fs = self + f.commit = types.MethodType(commit_a_file, f) + f.discard = types.MethodType(discard_a_file, f) + else: + f = self.ftp.open(path, mode, bufsize=block_size if block_size else -1) + return f + + def _rm(self, path): + if self.isdir(path): + self.ftp.rmdir(path) + else: + self.ftp.remove(path) + + def mv(self, old, new): + new = self._strip_protocol(new) + old = self._strip_protocol(old) + logger.debug("Renaming %s into %s", old, new) + self.ftp.posix_rename(old, new) + + +def commit_a_file(self): + self.fs.mv(self.temppath, self.targetpath) + + +def discard_a_file(self): + self.fs._rm(self.temppath) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/smb.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/smb.py new file mode 100644 index 0000000000000000000000000000000000000000..db6b3f5c3702de90cf121ccca49f3ca2b580df9f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/smb.py @@ -0,0 +1,416 @@ +""" +This module contains SMBFileSystem class responsible for handling access to +Windows Samba network shares by using package smbprotocol +""" + +import datetime +import re +import uuid +from stat import S_ISDIR, S_ISLNK + +import smbclient +import smbprotocol.exceptions + +from .. import AbstractFileSystem +from ..utils import infer_storage_options + +# ! pylint: disable=bad-continuation + + +class SMBFileSystem(AbstractFileSystem): + """Allow reading and writing to Windows and Samba network shares. + + When using `fsspec.open()` for getting a file-like object the URI + should be specified as this format: + ``smb://workgroup;user:password@server:port/share/folder/file.csv``. + + Example:: + + >>> import fsspec + >>> with fsspec.open( + ... 'smb://myuser:mypassword@myserver.com/' 'share/folder/file.csv' + ... ) as smbfile: + ... df = pd.read_csv(smbfile, sep='|', header=None) + + Note that you need to pass in a valid hostname or IP address for the host + component of the URL. Do not use the Windows/NetBIOS machine name for the + host component. + + The first component of the path in the URL points to the name of the shared + folder. Subsequent path components will point to the directory/folder/file. + + The URL components ``workgroup`` , ``user``, ``password`` and ``port`` may be + optional. + + .. note:: + + For working this source require `smbprotocol`_ to be installed, e.g.:: + + $ pip install smbprotocol + # or + # pip install smbprotocol[kerberos] + + .. _smbprotocol: https://github.com/jborean93/smbprotocol#requirements + + Note: if using this with the ``open`` or ``open_files``, with full URLs, + there is no way to tell if a path is relative, so all paths are assumed + to be absolute. + """ + + protocol = "smb" + + # pylint: disable=too-many-arguments + def __init__( + self, + host, + port=None, + username=None, + password=None, + timeout=60, + encrypt=None, + share_access=None, + register_session_retries=4, + register_session_retry_wait=1, + register_session_retry_factor=10, + auto_mkdir=False, + **kwargs, + ): + """ + You can use _get_kwargs_from_urls to get some kwargs from + a reasonable SMB url. + + Authentication will be anonymous or integrated if username/password are not + given. + + Parameters + ---------- + host: str + The remote server name/ip to connect to + port: int or None + Port to connect with. Usually 445, sometimes 139. + username: str or None + Username to connect with. Required if Kerberos auth is not being used. + password: str or None + User's password on the server, if using username + timeout: int + Connection timeout in seconds + encrypt: bool + Whether to force encryption or not, once this has been set to True + the session cannot be changed back to False. + share_access: str or None + Specifies the default access applied to file open operations + performed with this file system object. + This affects whether other processes can concurrently open a handle + to the same file. + + - None (the default): exclusively locks the file until closed. + - 'r': Allow other handles to be opened with read access. + - 'w': Allow other handles to be opened with write access. + - 'd': Allow other handles to be opened with delete access. + register_session_retries: int + Number of retries to register a session with the server. Retries are not performed + for authentication errors, as they are considered as invalid credentials and not network + issues. If set to negative value, no register attempts will be performed. + register_session_retry_wait: int + Time in seconds to wait between each retry. Number must be non-negative. + register_session_retry_factor: int + Base factor for the wait time between each retry. The wait time + is calculated using exponential function. For factor=1 all wait times + will be equal to `register_session_retry_wait`. For any number of retries, + the last wait time will be equal to `register_session_retry_wait` and for retries>1 + the first wait time will be equal to `register_session_retry_wait / factor`. + Number must be equal to or greater than 1. Optimal factor is 10. + auto_mkdir: bool + Whether, when opening a file, the directory containing it should + be created (if it doesn't already exist). This is assumed by pyarrow + and zarr-python code. + """ + super().__init__(**kwargs) + self.host = host + self.port = port + self.username = username + self.password = password + self.timeout = timeout + self.encrypt = encrypt + self.temppath = kwargs.pop("temppath", "") + self.share_access = share_access + self.register_session_retries = register_session_retries + if register_session_retry_wait < 0: + raise ValueError( + "register_session_retry_wait must be a non-negative integer" + ) + self.register_session_retry_wait = register_session_retry_wait + if register_session_retry_factor < 1: + raise ValueError( + "register_session_retry_factor must be a positive " + "integer equal to or greater than 1" + ) + self.register_session_retry_factor = register_session_retry_factor + self.auto_mkdir = auto_mkdir + self._connect() + + @property + def _port(self): + return 445 if self.port is None else self.port + + def _connect(self): + import time + + if self.register_session_retries <= -1: + return + + retried_errors = [] + + wait_time = self.register_session_retry_wait + n_waits = ( + self.register_session_retries - 1 + ) # -1 = No wait time after the last retry + factor = self.register_session_retry_factor + + # Generate wait times for each retry attempt. + # Wait times are calculated using exponential function. For factor=1 all wait times + # will be equal to `wait`. For any number of retries the last wait time will be + # equal to `wait` and for retries>2 the first wait time will be equal to `wait / factor`. + wait_times = iter( + factor ** (n / n_waits - 1) * wait_time for n in range(0, n_waits + 1) + ) + + for attempt in range(self.register_session_retries + 1): + try: + smbclient.register_session( + self.host, + username=self.username, + password=self.password, + port=self._port, + encrypt=self.encrypt, + connection_timeout=self.timeout, + ) + return + except ( + smbprotocol.exceptions.SMBAuthenticationError, + smbprotocol.exceptions.LogonFailure, + ): + # These exceptions should not be repeated, as they clearly indicate + # that the credentials are invalid and not a network issue. + raise + except ValueError as exc: + if re.findall(r"\[Errno -\d+]", str(exc)): + # This exception is raised by the smbprotocol.transport:Tcp.connect + # and originates from socket.gaierror (OSError). These exceptions might + # be raised due to network instability. We will retry to connect. + retried_errors.append(exc) + else: + # All another ValueError exceptions should be raised, as they are not + # related to network issues. + raise + except Exception as exc: + # Save the exception and retry to connect. This except might be dropped + # in the future, once all exceptions suited for retry are identified. + retried_errors.append(exc) + + if attempt < self.register_session_retries: + time.sleep(next(wait_times)) + + # Raise last exception to inform user about the connection issues. + # Note: Should we use ExceptionGroup to raise all exceptions? + raise retried_errors[-1] + + @classmethod + def _strip_protocol(cls, path): + return infer_storage_options(path)["path"] + + @staticmethod + def _get_kwargs_from_urls(path): + # smb://workgroup;user:password@host:port/share/folder/file.csv + out = infer_storage_options(path) + out.pop("path", None) + out.pop("protocol", None) + return out + + def mkdir(self, path, create_parents=True, **kwargs): + wpath = _as_unc_path(self.host, path) + if create_parents: + smbclient.makedirs(wpath, exist_ok=False, port=self._port, **kwargs) + else: + smbclient.mkdir(wpath, port=self._port, **kwargs) + + def makedirs(self, path, exist_ok=False): + if _share_has_path(path): + wpath = _as_unc_path(self.host, path) + smbclient.makedirs(wpath, exist_ok=exist_ok, port=self._port) + + def rmdir(self, path): + if _share_has_path(path): + wpath = _as_unc_path(self.host, path) + smbclient.rmdir(wpath, port=self._port) + + def info(self, path, **kwargs): + wpath = _as_unc_path(self.host, path) + stats = smbclient.stat(wpath, port=self._port, **kwargs) + if S_ISDIR(stats.st_mode): + stype = "directory" + elif S_ISLNK(stats.st_mode): + stype = "link" + else: + stype = "file" + res = { + "name": path + "/" if stype == "directory" else path, + "size": stats.st_size, + "type": stype, + "uid": stats.st_uid, + "gid": stats.st_gid, + "time": stats.st_atime, + "mtime": stats.st_mtime, + } + return res + + def created(self, path): + """Return the created timestamp of a file as a datetime.datetime""" + wpath = _as_unc_path(self.host, path) + stats = smbclient.stat(wpath, port=self._port) + return datetime.datetime.fromtimestamp(stats.st_ctime, tz=datetime.timezone.utc) + + def modified(self, path): + """Return the modified timestamp of a file as a datetime.datetime""" + wpath = _as_unc_path(self.host, path) + stats = smbclient.stat(wpath, port=self._port) + return datetime.datetime.fromtimestamp(stats.st_mtime, tz=datetime.timezone.utc) + + def ls(self, path, detail=True, **kwargs): + unc = _as_unc_path(self.host, path) + listed = smbclient.listdir(unc, port=self._port, **kwargs) + dirs = ["/".join([path.rstrip("/"), p]) for p in listed] + if detail: + dirs = [self.info(d) for d in dirs] + return dirs + + # pylint: disable=too-many-arguments + def _open( + self, + path, + mode="rb", + block_size=-1, + autocommit=True, + cache_options=None, + **kwargs, + ): + """ + block_size: int or None + If 0, no buffering, 1, line buffering, >1, buffer that many bytes + + Notes + ----- + By specifying 'share_access' in 'kwargs' it is possible to override the + default shared access setting applied in the constructor of this object. + """ + if self.auto_mkdir and "w" in mode: + self.makedirs(self._parent(path), exist_ok=True) + bls = block_size if block_size is not None and block_size >= 0 else -1 + wpath = _as_unc_path(self.host, path) + share_access = kwargs.pop("share_access", self.share_access) + if "w" in mode and autocommit is False: + temp = _as_temp_path(self.host, path, self.temppath) + return SMBFileOpener( + wpath, temp, mode, port=self._port, block_size=bls, **kwargs + ) + return smbclient.open_file( + wpath, + mode, + buffering=bls, + share_access=share_access, + port=self._port, + **kwargs, + ) + + def copy(self, path1, path2, **kwargs): + """Copy within two locations in the same filesystem""" + wpath1 = _as_unc_path(self.host, path1) + wpath2 = _as_unc_path(self.host, path2) + if self.auto_mkdir: + self.makedirs(self._parent(path2), exist_ok=True) + smbclient.copyfile(wpath1, wpath2, port=self._port, **kwargs) + + def _rm(self, path): + if _share_has_path(path): + wpath = _as_unc_path(self.host, path) + stats = smbclient.stat(wpath, port=self._port) + if S_ISDIR(stats.st_mode): + smbclient.rmdir(wpath, port=self._port) + else: + smbclient.remove(wpath, port=self._port) + + def mv(self, path1, path2, recursive=None, maxdepth=None, **kwargs): + wpath1 = _as_unc_path(self.host, path1) + wpath2 = _as_unc_path(self.host, path2) + smbclient.rename(wpath1, wpath2, port=self._port, **kwargs) + + +def _as_unc_path(host, path): + rpath = path.replace("/", "\\") + unc = f"\\\\{host}{rpath}" + return unc + + +def _as_temp_path(host, path, temppath): + share = path.split("/")[1] + temp_file = f"/{share}{temppath}/{uuid.uuid4()}" + unc = _as_unc_path(host, temp_file) + return unc + + +def _share_has_path(path): + parts = path.count("/") + if path.endswith("/"): + return parts > 2 + return parts > 1 + + +class SMBFileOpener: + """writes to remote temporary file, move on commit""" + + def __init__(self, path, temp, mode, port=445, block_size=-1, **kwargs): + self.path = path + self.temp = temp + self.mode = mode + self.block_size = block_size + self.kwargs = kwargs + self.smbfile = None + self._incontext = False + self.port = port + self._open() + + def _open(self): + if self.smbfile is None or self.smbfile.closed: + self.smbfile = smbclient.open_file( + self.temp, + self.mode, + port=self.port, + buffering=self.block_size, + **self.kwargs, + ) + + def commit(self): + """Move temp file to definitive on success.""" + # TODO: use transaction support in SMB protocol + smbclient.replace(self.temp, self.path, port=self.port) + + def discard(self): + """Remove the temp file on failure.""" + smbclient.remove(self.temp, port=self.port) + + def __fspath__(self): + return self.path + + def __iter__(self): + return self.smbfile.__iter__() + + def __getattr__(self, item): + return getattr(self.smbfile, item) + + def __enter__(self): + self._incontext = True + return self.smbfile.__enter__() + + def __exit__(self, exc_type, exc_value, traceback): + self._incontext = False + self.smbfile.__exit__(exc_type, exc_value, traceback) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/tar.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/tar.py new file mode 100644 index 0000000000000000000000000000000000000000..41bd894a4e3f1b3b2318f9b28d690dada72d670a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/tar.py @@ -0,0 +1,135 @@ +import logging +import tarfile + +import fsspec +from fsspec.archive import AbstractArchiveFileSystem +from fsspec.compression import compr +from fsspec.utils import infer_compression + +typemap = {b"0": "file", b"5": "directory"} + +logger = logging.getLogger("tar") + + +class TarFileSystem(AbstractArchiveFileSystem): + """Compressed Tar archives as a file-system (read-only) + + Supports the following formats: + tar.gz, tar.bz2, tar.xz + """ + + root_marker = "" + protocol = "tar" + cachable = False + + def __init__( + self, + fo="", + index_store=None, + target_options=None, + target_protocol=None, + compression=None, + **kwargs, + ): + super().__init__(**kwargs) + target_options = target_options or {} + + if isinstance(fo, str): + self.of = fsspec.open(fo, protocol=target_protocol, **target_options) + fo = self.of.open() # keep the reference + + # Try to infer compression. + if compression is None: + name = None + + # Try different ways to get hold of the filename. `fo` might either + # be a `fsspec.LocalFileOpener`, an `io.BufferedReader` or an + # `fsspec.AbstractFileSystem` instance. + try: + # Amended io.BufferedReader or similar. + # This uses a "protocol extension" where original filenames are + # propagated to archive-like filesystems in order to let them + # infer the right compression appropriately. + if hasattr(fo, "original"): + name = fo.original + + # fsspec.LocalFileOpener + elif hasattr(fo, "path"): + name = fo.path + + # io.BufferedReader + elif hasattr(fo, "name"): + name = fo.name + + # fsspec.AbstractFileSystem + elif hasattr(fo, "info"): + name = fo.info()["name"] + + except Exception as ex: + logger.warning( + f"Unable to determine file name, not inferring compression: {ex}" + ) + + if name is not None: + compression = infer_compression(name) + logger.info(f"Inferred compression {compression} from file name {name}") + + if compression is not None: + # TODO: tarfile already implements compression with modes like "'r:gz'", + # but then would seek to offset in the file work? + fo = compr[compression](fo) + + self._fo_ref = fo + self.fo = fo # the whole instance is a context + self.tar = tarfile.TarFile(fileobj=self.fo) + self.dir_cache = None + + self.index_store = index_store + self.index = None + self._index() + + def _index(self): + # TODO: load and set saved index, if exists + out = {} + for ti in self.tar: + info = ti.get_info() + info["type"] = typemap.get(info["type"], "file") + name = ti.get_info()["name"].rstrip("/") + out[name] = (info, ti.offset_data) + + self.index = out + # TODO: save index to self.index_store here, if set + + def _get_dirs(self): + if self.dir_cache is not None: + return + + # This enables ls to get directories as children as well as files + self.dir_cache = { + dirname: {"name": dirname, "size": 0, "type": "directory"} + for dirname in self._all_dirnames(self.tar.getnames()) + } + for member in self.tar.getmembers(): + info = member.get_info() + info["name"] = info["name"].rstrip("/") + info["type"] = typemap.get(info["type"], "file") + self.dir_cache[info["name"]] = info + + def _open(self, path, mode="rb", **kwargs): + if mode != "rb": + raise ValueError("Read-only filesystem implementation") + details, offset = self.index[path] + if details["type"] != "file": + raise ValueError("Can only handle regular files") + return self.tar.extractfile(path) + + def close(self): + """Commits any write changes to the file. Done on ``del`` too.""" + self.tar.close() + + def __del__(self): + if hasattr(self, "tar"): + self.close() + del self.tar + if hasattr(self, "of") and hasattr(self.of, "__exit__"): + self.of.__exit__(None, None, None) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/webhdfs.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/webhdfs.py new file mode 100644 index 0000000000000000000000000000000000000000..e3048b6a3638cf34e400804a3057521e720b6e81 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/webhdfs.py @@ -0,0 +1,503 @@ +# https://hadoop.apache.org/docs/r1.0.4/webhdfs.html + +import logging +import os +import secrets +import shutil +import tempfile +import uuid +from contextlib import suppress +from datetime import datetime +from urllib.parse import quote + +import requests + +from ..spec import AbstractBufferedFile, AbstractFileSystem +from ..utils import infer_storage_options, tokenize + +logger = logging.getLogger("webhdfs") + + +class WebHDFS(AbstractFileSystem): + """ + Interface to HDFS over HTTP using the WebHDFS API. Supports also HttpFS gateways. + + Four auth mechanisms are supported: + + insecure: no auth is done, and the user is assumed to be whoever they + say they are (parameter ``user``), or a predefined value such as + "dr.who" if not given + spnego: when kerberos authentication is enabled, auth is negotiated by + requests_kerberos https://github.com/requests/requests-kerberos . + This establishes a session based on existing kinit login and/or + specified principal/password; parameters are passed with ``kerb_kwargs`` + token: uses an existing Hadoop delegation token from another secured + service. Indeed, this client can also generate such tokens when + not insecure. Note that tokens expire, but can be renewed (by a + previously specified user) and may allow for proxying. + basic-auth: used when both parameter ``user`` and parameter ``password`` + are provided. + + """ + + tempdir = str(tempfile.gettempdir()) + protocol = "webhdfs", "webHDFS" + + def __init__( + self, + host, + port=50070, + kerberos=False, + token=None, + user=None, + password=None, + proxy_to=None, + kerb_kwargs=None, + data_proxy=None, + use_https=False, + session_cert=None, + session_verify=True, + **kwargs, + ): + """ + Parameters + ---------- + host: str + Name-node address + port: int + Port for webHDFS + kerberos: bool + Whether to authenticate with kerberos for this connection + token: str or None + If given, use this token on every call to authenticate. A user + and user-proxy may be encoded in the token and should not be also + given + user: str or None + If given, assert the user name to connect with + password: str or None + If given, assert the password to use for basic auth. If password + is provided, user must be provided also + proxy_to: str or None + If given, the user has the authority to proxy, and this value is + the user in who's name actions are taken + kerb_kwargs: dict + Any extra arguments for HTTPKerberosAuth, see + `<https://github.com/requests/requests-kerberos/blob/master/requests_kerberos/kerberos_.py>`_ + data_proxy: dict, callable or None + If given, map data-node addresses. This can be necessary if the + HDFS cluster is behind a proxy, running on Docker or otherwise has + a mismatch between the host-names given by the name-node and the + address by which to refer to them from the client. If a dict, + maps host names ``host->data_proxy[host]``; if a callable, full + URLs are passed, and function must conform to + ``url->data_proxy(url)``. + use_https: bool + Whether to connect to the Name-node using HTTPS instead of HTTP + session_cert: str or Tuple[str, str] or None + Path to a certificate file, or tuple of (cert, key) files to use + for the requests.Session + session_verify: str, bool or None + Path to a certificate file to use for verifying the requests.Session. + kwargs + """ + if self._cached: + return + super().__init__(**kwargs) + self.url = f"{'https' if use_https else 'http'}://{host}:{port}/webhdfs/v1" + self.kerb = kerberos + self.kerb_kwargs = kerb_kwargs or {} + self.pars = {} + self.proxy = data_proxy or {} + if token is not None: + if user is not None or proxy_to is not None: + raise ValueError( + "If passing a delegation token, must not set " + "user or proxy_to, as these are encoded in the" + " token" + ) + self.pars["delegation"] = token + self.user = user + self.password = password + + if password is not None: + if user is None: + raise ValueError( + "If passing a password, the user must also be" + "set in order to set up the basic-auth" + ) + else: + if user is not None: + self.pars["user.name"] = user + + if proxy_to is not None: + self.pars["doas"] = proxy_to + if kerberos and user is not None: + raise ValueError( + "If using Kerberos auth, do not specify the " + "user, this is handled by kinit." + ) + + self.session_cert = session_cert + self.session_verify = session_verify + + self._connect() + + self._fsid = f"webhdfs_{tokenize(host, port)}" + + @property + def fsid(self): + return self._fsid + + def _connect(self): + self.session = requests.Session() + + if self.session_cert: + self.session.cert = self.session_cert + + self.session.verify = self.session_verify + + if self.kerb: + from requests_kerberos import HTTPKerberosAuth + + self.session.auth = HTTPKerberosAuth(**self.kerb_kwargs) + + if self.user is not None and self.password is not None: + from requests.auth import HTTPBasicAuth + + self.session.auth = HTTPBasicAuth(self.user, self.password) + + def _call(self, op, method="get", path=None, data=None, redirect=True, **kwargs): + path = self._strip_protocol(path) if path is not None else "" + url = self._apply_proxy(self.url + quote(path, safe="/=")) + args = kwargs.copy() + args.update(self.pars) + args["op"] = op.upper() + logger.debug("sending %s with %s", url, method) + out = self.session.request( + method=method.upper(), + url=url, + params=args, + data=data, + allow_redirects=redirect, + ) + if out.status_code in [400, 401, 403, 404, 500]: + try: + err = out.json() + msg = err["RemoteException"]["message"] + exp = err["RemoteException"]["exception"] + except (ValueError, KeyError): + pass + else: + if exp in ["IllegalArgumentException", "UnsupportedOperationException"]: + raise ValueError(msg) + elif exp in ["SecurityException", "AccessControlException"]: + raise PermissionError(msg) + elif exp in ["FileNotFoundException"]: + raise FileNotFoundError(msg) + else: + raise RuntimeError(msg) + out.raise_for_status() + return out + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + replication=None, + permissions=None, + **kwargs, + ): + """ + + Parameters + ---------- + path: str + File location + mode: str + 'rb', 'wb', etc. + block_size: int + Client buffer size for read-ahead or write buffer + autocommit: bool + If False, writes to temporary file that only gets put in final + location upon commit + replication: int + Number of copies of file on the cluster, write mode only + permissions: str or int + posix permissions, write mode only + kwargs + + Returns + ------- + WebHDFile instance + """ + block_size = block_size or self.blocksize + return WebHDFile( + self, + path, + mode=mode, + block_size=block_size, + tempdir=self.tempdir, + autocommit=autocommit, + replication=replication, + permissions=permissions, + ) + + @staticmethod + def _process_info(info): + info["type"] = info["type"].lower() + info["size"] = info["length"] + return info + + @classmethod + def _strip_protocol(cls, path): + return infer_storage_options(path)["path"] + + @staticmethod + def _get_kwargs_from_urls(urlpath): + out = infer_storage_options(urlpath) + out.pop("path", None) + out.pop("protocol", None) + if "username" in out: + out["user"] = out.pop("username") + return out + + def info(self, path): + out = self._call("GETFILESTATUS", path=path) + info = out.json()["FileStatus"] + info["name"] = path + return self._process_info(info) + + def created(self, path): + """Return the created timestamp of a file as a datetime.datetime""" + # The API does not provide creation time, so we use modification time + info = self.info(path) + mtime = info.get("modificationTime", None) + if mtime is not None: + return datetime.fromtimestamp(mtime / 1000) + raise RuntimeError("Could not retrieve creation time (modification time).") + + def modified(self, path): + """Return the modified timestamp of a file as a datetime.datetime""" + info = self.info(path) + mtime = info.get("modificationTime", None) + if mtime is not None: + return datetime.fromtimestamp(mtime / 1000) + raise RuntimeError("Could not retrieve modification time.") + + def ls(self, path, detail=False, **kwargs): + out = self._call("LISTSTATUS", path=path) + infos = out.json()["FileStatuses"]["FileStatus"] + for info in infos: + self._process_info(info) + info["name"] = path.rstrip("/") + "/" + info["pathSuffix"] + if detail: + return sorted(infos, key=lambda i: i["name"]) + else: + return sorted(info["name"] for info in infos) + + def content_summary(self, path): + """Total numbers of files, directories and bytes under path""" + out = self._call("GETCONTENTSUMMARY", path=path) + return out.json()["ContentSummary"] + + def ukey(self, path): + """Checksum info of file, giving method and result""" + out = self._call("GETFILECHECKSUM", path=path, redirect=False) + if "Location" in out.headers: + location = self._apply_proxy(out.headers["Location"]) + out2 = self.session.get(location) + out2.raise_for_status() + return out2.json()["FileChecksum"] + else: + out.raise_for_status() + return out.json()["FileChecksum"] + + def home_directory(self): + """Get user's home directory""" + out = self._call("GETHOMEDIRECTORY") + return out.json()["Path"] + + def get_delegation_token(self, renewer=None): + """Retrieve token which can give the same authority to other uses + + Parameters + ---------- + renewer: str or None + User who may use this token; if None, will be current user + """ + if renewer: + out = self._call("GETDELEGATIONTOKEN", renewer=renewer) + else: + out = self._call("GETDELEGATIONTOKEN") + t = out.json()["Token"] + if t is None: + raise ValueError("No token available for this user/security context") + return t["urlString"] + + def renew_delegation_token(self, token): + """Make token live longer. Returns new expiry time""" + out = self._call("RENEWDELEGATIONTOKEN", method="put", token=token) + return out.json()["long"] + + def cancel_delegation_token(self, token): + """Stop the token from being useful""" + self._call("CANCELDELEGATIONTOKEN", method="put", token=token) + + def chmod(self, path, mod): + """Set the permission at path + + Parameters + ---------- + path: str + location to set (file or directory) + mod: str or int + posix epresentation or permission, give as oct string, e.g, '777' + or 0o777 + """ + self._call("SETPERMISSION", method="put", path=path, permission=mod) + + def chown(self, path, owner=None, group=None): + """Change owning user and/or group""" + kwargs = {} + if owner is not None: + kwargs["owner"] = owner + if group is not None: + kwargs["group"] = group + self._call("SETOWNER", method="put", path=path, **kwargs) + + def set_replication(self, path, replication): + """ + Set file replication factor + + Parameters + ---------- + path: str + File location (not for directories) + replication: int + Number of copies of file on the cluster. Should be smaller than + number of data nodes; normally 3 on most systems. + """ + self._call("SETREPLICATION", path=path, method="put", replication=replication) + + def mkdir(self, path, **kwargs): + self._call("MKDIRS", method="put", path=path) + + def makedirs(self, path, exist_ok=False): + if exist_ok is False and self.exists(path): + raise FileExistsError(path) + self.mkdir(path) + + def mv(self, path1, path2, **kwargs): + self._call("RENAME", method="put", path=path1, destination=path2) + + def rm(self, path, recursive=False, **kwargs): + self._call( + "DELETE", + method="delete", + path=path, + recursive="true" if recursive else "false", + ) + + def rm_file(self, path, **kwargs): + self.rm(path) + + def cp_file(self, lpath, rpath, **kwargs): + with self.open(lpath) as lstream: + tmp_fname = "/".join([self._parent(rpath), f".tmp.{secrets.token_hex(16)}"]) + # Perform an atomic copy (stream to a temporary file and + # move it to the actual destination). + try: + with self.open(tmp_fname, "wb") as rstream: + shutil.copyfileobj(lstream, rstream) + self.mv(tmp_fname, rpath) + except BaseException: + with suppress(FileNotFoundError): + self.rm(tmp_fname) + raise + + def _apply_proxy(self, location): + if self.proxy and callable(self.proxy): + location = self.proxy(location) + elif self.proxy: + # as a dict + for k, v in self.proxy.items(): + location = location.replace(k, v, 1) + return location + + +class WebHDFile(AbstractBufferedFile): + """A file living in HDFS over webHDFS""" + + def __init__(self, fs, path, **kwargs): + super().__init__(fs, path, **kwargs) + kwargs = kwargs.copy() + if kwargs.get("permissions", None) is None: + kwargs.pop("permissions", None) + if kwargs.get("replication", None) is None: + kwargs.pop("replication", None) + self.permissions = kwargs.pop("permissions", 511) + tempdir = kwargs.pop("tempdir") + if kwargs.pop("autocommit", False) is False: + self.target = self.path + self.path = os.path.join(tempdir, str(uuid.uuid4())) + + def _upload_chunk(self, final=False): + """Write one part of a multi-block file upload + + Parameters + ========== + final: bool + This is the last block, so should complete file, if + self.autocommit is True. + """ + out = self.fs.session.post( + self.location, + data=self.buffer.getvalue(), + headers={"content-type": "application/octet-stream"}, + ) + out.raise_for_status() + return True + + def _initiate_upload(self): + """Create remote file/upload""" + kwargs = self.kwargs.copy() + if "a" in self.mode: + op, method = "APPEND", "POST" + else: + op, method = "CREATE", "PUT" + kwargs["overwrite"] = "true" + out = self.fs._call(op, method, self.path, redirect=False, **kwargs) + location = self.fs._apply_proxy(out.headers["Location"]) + if "w" in self.mode: + # create empty file to append to + out2 = self.fs.session.put( + location, headers={"content-type": "application/octet-stream"} + ) + out2.raise_for_status() + # after creating empty file, change location to append to + out2 = self.fs._call("APPEND", "POST", self.path, redirect=False, **kwargs) + self.location = self.fs._apply_proxy(out2.headers["Location"]) + + def _fetch_range(self, start, end): + start = max(start, 0) + end = min(self.size, end) + if start >= end or start >= self.size: + return b"" + out = self.fs._call( + "OPEN", path=self.path, offset=start, length=end - start, redirect=False + ) + out.raise_for_status() + if "Location" in out.headers: + location = out.headers["Location"] + out2 = self.fs.session.get(self.fs._apply_proxy(location)) + return out2.content + else: + return out.content + + def commit(self): + self.fs.mv(self.path, self.target) + + def discard(self): + self.fs.rm(self.path) diff --git a/.venv/lib/python3.11/site-packages/fsspec/implementations/zip.py b/.venv/lib/python3.11/site-packages/fsspec/implementations/zip.py new file mode 100644 index 0000000000000000000000000000000000000000..485307a34f8daba70eb30bfda713c95893bbdb0b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/implementations/zip.py @@ -0,0 +1,183 @@ +import os +import zipfile + +import fsspec +from fsspec.archive import AbstractArchiveFileSystem + + +class ZipFileSystem(AbstractArchiveFileSystem): + """Read/Write contents of ZIP archive as a file-system + + Keeps file object open while instance lives. + + This class is pickleable, but not necessarily thread-safe + """ + + root_marker = "" + protocol = "zip" + cachable = False + + def __init__( + self, + fo="", + mode="r", + target_protocol=None, + target_options=None, + compression=zipfile.ZIP_STORED, + allowZip64=True, + compresslevel=None, + **kwargs, + ): + """ + Parameters + ---------- + fo: str or file-like + Contains ZIP, and must exist. If a str, will fetch file using + :meth:`~fsspec.open_files`, which must return one file exactly. + mode: str + Accept: "r", "w", "a" + target_protocol: str (optional) + If ``fo`` is a string, this value can be used to override the + FS protocol inferred from a URL + target_options: dict (optional) + Kwargs passed when instantiating the target FS, if ``fo`` is + a string. + compression, allowZip64, compresslevel: passed to ZipFile + Only relevant when creating a ZIP + """ + super().__init__(self, **kwargs) + if mode not in set("rwa"): + raise ValueError(f"mode '{mode}' no understood") + self.mode = mode + if isinstance(fo, (str, os.PathLike)): + if mode == "a": + m = "r+b" + else: + m = mode + "b" + fo = fsspec.open( + fo, mode=m, protocol=target_protocol, **(target_options or {}) + ) + self.force_zip_64 = allowZip64 + self.of = fo + self.fo = fo.__enter__() # the whole instance is a context + self.zip = zipfile.ZipFile( + self.fo, + mode=mode, + compression=compression, + allowZip64=allowZip64, + compresslevel=compresslevel, + ) + self.dir_cache = None + + @classmethod + def _strip_protocol(cls, path): + # zip file paths are always relative to the archive root + return super()._strip_protocol(path).lstrip("/") + + def __del__(self): + if hasattr(self, "zip"): + self.close() + del self.zip + if hasattr(self, "of") and hasattr(self.of, "__exit__"): + self.of.__exit__(None, None, None) + + def close(self): + """Commits any write changes to the file. Done on ``del`` too.""" + self.zip.close() + + def _get_dirs(self): + if self.dir_cache is None or self.mode in set("wa"): + # when writing, dir_cache is always in the ZipFile's attributes, + # not read from the file. + files = self.zip.infolist() + self.dir_cache = { + dirname.rstrip("/"): { + "name": dirname.rstrip("/"), + "size": 0, + "type": "directory", + } + for dirname in self._all_dirnames(self.zip.namelist()) + } + for z in files: + f = {s: getattr(z, s, None) for s in zipfile.ZipInfo.__slots__} + f.update( + { + "name": z.filename.rstrip("/"), + "size": z.file_size, + "type": ("directory" if z.is_dir() else "file"), + } + ) + self.dir_cache[f["name"]] = f + + def pipe_file(self, path, value, **kwargs): + # override upstream, because we know the exact file size in this case + self.zip.writestr(path, value, **kwargs) + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + path = self._strip_protocol(path) + if "r" in mode and self.mode in set("wa"): + if self.exists(path): + raise OSError("ZipFS can only be open for reading or writing, not both") + raise FileNotFoundError(path) + if "r" in self.mode and "w" in mode: + raise OSError("ZipFS can only be open for reading or writing, not both") + out = self.zip.open(path, mode.strip("b"), force_zip64=self.force_zip_64) + if "r" in mode: + info = self.info(path) + out.size = info["size"] + out.name = info["name"] + return out + + def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + def to_parts(_path: str): + return list(filter(None, _path.replace("\\", "/").split("/"))) + + if not isinstance(path, str): + path = str(path) + + # Remove the leading slash, as the zip file paths are always + # given without a leading slash + path = path.lstrip("/") + path_parts = to_parts(path) + path_depth = len(path_parts) + + self._get_dirs() + + result = {} + # To match posix find, if an exact file name is given, we should + # return only that file + if path in self.dir_cache and self.dir_cache[path]["type"] == "file": + result[path] = self.dir_cache[path] + return result if detail else [path] + + for file_path, file_info in self.dir_cache.items(): + if len(file_parts := to_parts(file_path)) < path_depth or any( + a != b for a, b in zip(path_parts, file_parts) + ): + # skip parent folders and mismatching paths + continue + + if file_info["type"] == "directory": + if withdirs and file_path not in result: + result[file_path.strip("/")] = file_info + continue + + if file_path not in result: + result[file_path] = file_info if detail else None + + if maxdepth: + result = { + k: v for k, v in result.items() if k.count("/") < maxdepth + path_depth + } + return result if detail else sorted(result) diff --git a/.venv/lib/python3.11/site-packages/fsspec/json.py b/.venv/lib/python3.11/site-packages/fsspec/json.py new file mode 100644 index 0000000000000000000000000000000000000000..5c53a24913d0b28f4b53a163b97ff8f58abeb031 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/json.py @@ -0,0 +1,112 @@ +import json +from collections.abc import Callable, Mapping, Sequence +from contextlib import suppress +from pathlib import PurePath +from typing import Any, ClassVar + +from .registry import _import_class, get_filesystem_class +from .spec import AbstractFileSystem + + +class FilesystemJSONEncoder(json.JSONEncoder): + include_password: ClassVar[bool] = True + + def default(self, o: Any) -> Any: + if isinstance(o, AbstractFileSystem): + return o.to_dict(include_password=self.include_password) + if isinstance(o, PurePath): + cls = type(o) + return {"cls": f"{cls.__module__}.{cls.__name__}", "str": str(o)} + + return super().default(o) + + def make_serializable(self, obj: Any) -> Any: + """ + Recursively converts an object so that it can be JSON serialized via + :func:`json.dumps` and :func:`json.dump`, without actually calling + said functions. + """ + if isinstance(obj, (str, int, float, bool)): + return obj + if isinstance(obj, Mapping): + return {k: self.make_serializable(v) for k, v in obj.items()} + if isinstance(obj, Sequence): + return [self.make_serializable(v) for v in obj] + + return self.default(obj) + + +class FilesystemJSONDecoder(json.JSONDecoder): + def __init__( + self, + *, + object_hook: Callable[[dict[str, Any]], Any] | None = None, + parse_float: Callable[[str], Any] | None = None, + parse_int: Callable[[str], Any] | None = None, + parse_constant: Callable[[str], Any] | None = None, + strict: bool = True, + object_pairs_hook: Callable[[list[tuple[str, Any]]], Any] | None = None, + ) -> None: + self.original_object_hook = object_hook + + super().__init__( + object_hook=self.custom_object_hook, + parse_float=parse_float, + parse_int=parse_int, + parse_constant=parse_constant, + strict=strict, + object_pairs_hook=object_pairs_hook, + ) + + @classmethod + def try_resolve_path_cls(cls, dct: dict[str, Any]): + with suppress(Exception): + fqp = dct["cls"] + + path_cls = _import_class(fqp) + + if issubclass(path_cls, PurePath): + return path_cls + + return None + + @classmethod + def try_resolve_fs_cls(cls, dct: dict[str, Any]): + with suppress(Exception): + if "cls" in dct: + try: + fs_cls = _import_class(dct["cls"]) + if issubclass(fs_cls, AbstractFileSystem): + return fs_cls + except Exception: + if "protocol" in dct: # Fallback if cls cannot be imported + return get_filesystem_class(dct["protocol"]) + + raise + + return None + + def custom_object_hook(self, dct: dict[str, Any]): + if "cls" in dct: + if (obj_cls := self.try_resolve_fs_cls(dct)) is not None: + return AbstractFileSystem.from_dict(dct) + if (obj_cls := self.try_resolve_path_cls(dct)) is not None: + return obj_cls(dct["str"]) + + if self.original_object_hook is not None: + return self.original_object_hook(dct) + + return dct + + def unmake_serializable(self, obj: Any) -> Any: + """ + Inverse function of :meth:`FilesystemJSONEncoder.make_serializable`. + """ + if isinstance(obj, dict): + obj = self.custom_object_hook(obj) + if isinstance(obj, dict): + return {k: self.unmake_serializable(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [self.unmake_serializable(v) for v in obj] + + return obj diff --git a/.venv/lib/python3.11/site-packages/fsspec/mapping.py b/.venv/lib/python3.11/site-packages/fsspec/mapping.py new file mode 100644 index 0000000000000000000000000000000000000000..752eef35273b13eded7297e2e801b58e436a25b1 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/mapping.py @@ -0,0 +1,251 @@ +import array +import logging +import posixpath +import warnings +from collections.abc import MutableMapping +from functools import cached_property + +from fsspec.core import url_to_fs + +logger = logging.getLogger("fsspec.mapping") + + +class FSMap(MutableMapping): + """Wrap a FileSystem instance as a mutable wrapping. + + The keys of the mapping become files under the given root, and the + values (which must be bytes) the contents of those files. + + Parameters + ---------- + root: string + prefix for all the files + fs: FileSystem instance + check: bool (=True) + performs a touch at the location, to check for write access. + + Examples + -------- + >>> fs = FileSystem(**parameters) # doctest: +SKIP + >>> d = FSMap('my-data/path/', fs) # doctest: +SKIP + or, more likely + >>> d = fs.get_mapper('my-data/path/') + + >>> d['loc1'] = b'Hello World' # doctest: +SKIP + >>> list(d.keys()) # doctest: +SKIP + ['loc1'] + >>> d['loc1'] # doctest: +SKIP + b'Hello World' + """ + + def __init__(self, root, fs, check=False, create=False, missing_exceptions=None): + self.fs = fs + self.root = fs._strip_protocol(root) + self._root_key_to_str = fs._strip_protocol(posixpath.join(root, "x"))[:-1] + if missing_exceptions is None: + missing_exceptions = ( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + ) + self.missing_exceptions = missing_exceptions + self.check = check + self.create = create + if create: + if not self.fs.exists(root): + self.fs.mkdir(root) + if check: + if not self.fs.exists(root): + raise ValueError( + f"Path {root} does not exist. Create " + f" with the ``create=True`` keyword" + ) + self.fs.touch(root + "/a") + self.fs.rm(root + "/a") + + @cached_property + def dirfs(self): + """dirfs instance that can be used with the same keys as the mapper""" + from .implementations.dirfs import DirFileSystem + + return DirFileSystem(path=self._root_key_to_str, fs=self.fs) + + def clear(self): + """Remove all keys below root - empties out mapping""" + logger.info("Clear mapping at %s", self.root) + try: + self.fs.rm(self.root, True) + self.fs.mkdir(self.root) + except: # noqa: E722 + pass + + def getitems(self, keys, on_error="raise"): + """Fetch multiple items from the store + + If the backend is async-able, this might proceed concurrently + + Parameters + ---------- + keys: list(str) + They keys to be fetched + on_error : "raise", "omit", "return" + If raise, an underlying exception will be raised (converted to KeyError + if the type is in self.missing_exceptions); if omit, keys with exception + will simply not be included in the output; if "return", all keys are + included in the output, but the value will be bytes or an exception + instance. + + Returns + ------- + dict(key, bytes|exception) + """ + keys2 = [self._key_to_str(k) for k in keys] + oe = on_error if on_error == "raise" else "return" + try: + out = self.fs.cat(keys2, on_error=oe) + if isinstance(out, bytes): + out = {keys2[0]: out} + except self.missing_exceptions as e: + raise KeyError from e + out = { + k: (KeyError() if isinstance(v, self.missing_exceptions) else v) + for k, v in out.items() + } + return { + key: out[k2] if on_error == "raise" else out.get(k2, KeyError(k2)) + for key, k2 in zip(keys, keys2) + if on_error == "return" or not isinstance(out[k2], BaseException) + } + + def setitems(self, values_dict): + """Set the values of multiple items in the store + + Parameters + ---------- + values_dict: dict(str, bytes) + """ + values = {self._key_to_str(k): maybe_convert(v) for k, v in values_dict.items()} + self.fs.pipe(values) + + def delitems(self, keys): + """Remove multiple keys from the store""" + self.fs.rm([self._key_to_str(k) for k in keys]) + + def _key_to_str(self, key): + """Generate full path for the key""" + if not isinstance(key, str): + # raise TypeError("key must be of type `str`, got `{type(key).__name__}`" + warnings.warn( + "from fsspec 2023.5 onward FSMap non-str keys will raise TypeError", + DeprecationWarning, + ) + if isinstance(key, list): + key = tuple(key) + key = str(key) + return f"{self._root_key_to_str}{key}".rstrip("/") + + def _str_to_key(self, s): + """Strip path of to leave key name""" + return s[len(self.root) :].lstrip("/") + + def __getitem__(self, key, default=None): + """Retrieve data""" + k = self._key_to_str(key) + try: + result = self.fs.cat(k) + except self.missing_exceptions as exc: + if default is not None: + return default + raise KeyError(key) from exc + return result + + def pop(self, key, default=None): + """Pop data""" + result = self.__getitem__(key, default) + try: + del self[key] + except KeyError: + pass + return result + + def __setitem__(self, key, value): + """Store value in key""" + key = self._key_to_str(key) + self.fs.mkdirs(self.fs._parent(key), exist_ok=True) + self.fs.pipe_file(key, maybe_convert(value)) + + def __iter__(self): + return (self._str_to_key(x) for x in self.fs.find(self.root)) + + def __len__(self): + return len(self.fs.find(self.root)) + + def __delitem__(self, key): + """Remove key""" + try: + self.fs.rm(self._key_to_str(key)) + except Exception as exc: + raise KeyError from exc + + def __contains__(self, key): + """Does key exist in mapping?""" + path = self._key_to_str(key) + return self.fs.isfile(path) + + def __reduce__(self): + return FSMap, (self.root, self.fs, False, False, self.missing_exceptions) + + +def maybe_convert(value): + if isinstance(value, array.array) or hasattr(value, "__array__"): + # bytes-like things + if hasattr(value, "dtype") and value.dtype.kind in "Mm": + # The buffer interface doesn't support datetime64/timdelta64 numpy + # arrays + value = value.view("int64") + value = bytes(memoryview(value)) + return value + + +def get_mapper( + url="", + check=False, + create=False, + missing_exceptions=None, + alternate_root=None, + **kwargs, +): + """Create key-value interface for given URL and options + + The URL will be of the form "protocol://location" and point to the root + of the mapper required. All keys will be file-names below this location, + and their values the contents of each key. + + Also accepts compound URLs like zip::s3://bucket/file.zip , see ``fsspec.open``. + + Parameters + ---------- + url: str + Root URL of mapping + check: bool + Whether to attempt to read from the location before instantiation, to + check that the mapping does exist + create: bool + Whether to make the directory corresponding to the root before + instantiating + missing_exceptions: None or tuple + If given, these exception types will be regarded as missing keys and + return KeyError when trying to read data. By default, you get + (FileNotFoundError, IsADirectoryError, NotADirectoryError) + alternate_root: None or str + In cases of complex URLs, the parser may fail to pick the correct part + for the mapper root, so this arg can override + + Returns + ------- + ``FSMap`` instance, the dict-like key-value store. + """ + # Removing protocol here - could defer to each open() on the backend + fs, urlpath = url_to_fs(url, **kwargs) + root = alternate_root if alternate_root is not None else urlpath + return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions) diff --git a/.venv/lib/python3.11/site-packages/fsspec/parquet.py b/.venv/lib/python3.11/site-packages/fsspec/parquet.py new file mode 100644 index 0000000000000000000000000000000000000000..e6e03fb14f9ec9224804a2ccb77bb84fe3390952 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/parquet.py @@ -0,0 +1,572 @@ +import io +import json +import warnings + +import fsspec + +from .core import url_to_fs +from .spec import AbstractBufferedFile +from .utils import merge_offset_ranges + +# Parquet-Specific Utilities for fsspec +# +# Most of the functions defined in this module are NOT +# intended for public consumption. The only exception +# to this is `open_parquet_file`, which should be used +# place of `fs.open()` to open parquet-formatted files +# on remote file systems. + + +class AlreadyBufferedFile(AbstractBufferedFile): + def _fetch_range(self, start, end): + raise NotImplementedError + + +def open_parquet_files( + path: list[str], + fs: None | fsspec.AbstractFileSystem = None, + metadata=None, + columns: None | list[str] = None, + row_groups: None | list[int] = None, + storage_options: None | dict = None, + engine: str = "auto", + max_gap: int = 64_000, + max_block: int = 256_000_000, + footer_sample_size: int = 1_000_000, + filters: None | list[list[list[str]]] = None, + **kwargs, +): + """ + Return a file-like object for a single Parquet file. + + The specified parquet `engine` will be used to parse the + footer metadata, and determine the required byte ranges + from the file. The target path will then be opened with + the "parts" (`KnownPartsOfAFile`) caching strategy. + + Note that this method is intended for usage with remote + file systems, and is unlikely to improve parquet-read + performance on local file systems. + + Parameters + ---------- + path: str + Target file path. + metadata: Any, optional + Parquet metadata object. Object type must be supported + by the backend parquet engine. For now, only the "fastparquet" + engine supports an explicit `ParquetFile` metadata object. + If a metadata object is supplied, the remote footer metadata + will not need to be transferred into local memory. + fs: AbstractFileSystem, optional + Filesystem object to use for opening the file. If nothing is + specified, an `AbstractFileSystem` object will be inferred. + engine : str, default "auto" + Parquet engine to use for metadata parsing. Allowed options + include "fastparquet", "pyarrow", and "auto". The specified + engine must be installed in the current environment. If + "auto" is specified, and both engines are installed, + "fastparquet" will take precedence over "pyarrow". + columns: list, optional + List of all column names that may be read from the file. + row_groups : list, optional + List of all row-groups that may be read from the file. This + may be a list of row-group indices (integers), or it may be + a list of `RowGroup` metadata objects (if the "fastparquet" + engine is used). + storage_options : dict, optional + Used to generate an `AbstractFileSystem` object if `fs` was + not specified. + max_gap : int, optional + Neighboring byte ranges will only be merged when their + inter-range gap is <= `max_gap`. Default is 64KB. + max_block : int, optional + Neighboring byte ranges will only be merged when the size of + the aggregated range is <= `max_block`. Default is 256MB. + footer_sample_size : int, optional + Number of bytes to read from the end of the path to look + for the footer metadata. If the sampled bytes do not contain + the footer, a second read request will be required, and + performance will suffer. Default is 1MB. + filters : list[list], optional + List of filters to apply to prevent reading row groups, of the + same format as accepted by the loading engines. Ignored if + ``row_groups`` is specified. + **kwargs : + Optional key-word arguments to pass to `fs.open` + """ + + # Make sure we have an `AbstractFileSystem` object + # to work with + if fs is None: + path0 = path + if isinstance(path, (list, tuple)): + path = path[0] + fs, path = url_to_fs(path, **(storage_options or {})) + else: + path0 = path + + # For now, `columns == []` not supported, is the same + # as all columns + if columns is not None and len(columns) == 0: + columns = None + + # Set the engine + engine = _set_engine(engine) + + if isinstance(path0, (list, tuple)): + paths = path0 + elif "*" in path: + paths = fs.glob(path) + elif path0.endswith("/"): # or fs.isdir(path): + paths = [ + _ + for _ in fs.find(path, withdirs=False, detail=False) + if _.endswith((".parquet", ".parq")) + ] + else: + paths = [path] + + data = _get_parquet_byte_ranges( + paths, + fs, + metadata=metadata, + columns=columns, + row_groups=row_groups, + engine=engine, + max_gap=max_gap, + max_block=max_block, + footer_sample_size=footer_sample_size, + filters=filters, + ) + + # Call self.open with "parts" caching + options = kwargs.pop("cache_options", {}).copy() + return [ + AlreadyBufferedFile( + fs=None, + path=fn, + mode="rb", + cache_type="parts", + cache_options={ + **options, + "data": ranges, + }, + size=max(_[1] for _ in ranges), + **kwargs, + ) + for fn, ranges in data.items() + ] + + +def open_parquet_file(*args, **kwargs): + """Create files tailed to reading specific parts of parquet files + + Please see ``open_parquet_files`` for details of the arguments. The + difference is, this function always returns a single ``AlreadyBufferedFile``, + whereas `open_parquet_files`` always returns a list of files, even if + there are one or zero matching parquet files. + """ + return open_parquet_files(*args, **kwargs)[0] + + +def _get_parquet_byte_ranges( + paths, + fs, + metadata=None, + columns=None, + row_groups=None, + max_gap=64_000, + max_block=256_000_000, + footer_sample_size=1_000_000, + engine="auto", + filters=None, +): + """Get a dictionary of the known byte ranges needed + to read a specific column/row-group selection from a + Parquet dataset. Each value in the output dictionary + is intended for use as the `data` argument for the + `KnownPartsOfAFile` caching strategy of a single path. + """ + + # Set engine if necessary + if isinstance(engine, str): + engine = _set_engine(engine) + + # Pass to a specialized function if metadata is defined + if metadata is not None: + # Use the provided parquet metadata object + # to avoid transferring/parsing footer metadata + return _get_parquet_byte_ranges_from_metadata( + metadata, + fs, + engine, + columns=columns, + row_groups=row_groups, + max_gap=max_gap, + max_block=max_block, + filters=filters, + ) + + # Populate global paths, starts, & ends + if columns is None and row_groups is None and filters is None: + # We are NOT selecting specific columns or row-groups. + # + # We can avoid sampling the footers, and just transfer + # all file data with cat_ranges + result = {path: {(0, len(data)): data} for path, data in fs.cat(paths).items()} + else: + # We ARE selecting specific columns or row-groups. + # + # Get file sizes asynchronously + file_sizes = fs.sizes(paths) + data_paths = [] + data_starts = [] + data_ends = [] + # Gather file footers. + # We just take the last `footer_sample_size` bytes of each + # file (or the entire file if it is smaller than that) + footer_starts = [ + max(0, file_size - footer_sample_size) for file_size in file_sizes + ] + footer_samples = fs.cat_ranges(paths, footer_starts, file_sizes) + + # Check our footer samples and re-sample if necessary. + large_footer = [] + for i, path in enumerate(paths): + footer_size = int.from_bytes(footer_samples[i][-8:-4], "little") + real_footer_start = file_sizes[i] - (footer_size + 8) + if real_footer_start < footer_starts[i]: + large_footer.append((i, real_footer_start)) + if large_footer: + warnings.warn( + f"Not enough data was used to sample the parquet footer. " + f"Try setting footer_sample_size >= {large_footer}." + ) + path0 = [paths[i] for i, _ in large_footer] + starts = [_[1] for _ in large_footer] + ends = [file_sizes[i] - footer_sample_size for i, _ in large_footer] + data = fs.cat_ranges(path0, starts, ends) + for i, (path, start, block) in enumerate(zip(path0, starts, data)): + footer_samples[i] = block + footer_samples[i] + footer_starts[i] = start + result = { + path: {(start, size): data} + for path, start, size, data in zip( + paths, footer_starts, file_sizes, footer_samples + ) + } + + # Calculate required byte ranges for each path + for i, path in enumerate(paths): + # Use "engine" to collect data byte ranges + path_data_starts, path_data_ends = engine._parquet_byte_ranges( + columns, + row_groups=row_groups, + footer=footer_samples[i], + footer_start=footer_starts[i], + filters=filters, + ) + + data_paths += [path] * len(path_data_starts) + data_starts += path_data_starts + data_ends += path_data_ends + + # Merge adjacent offset ranges + data_paths, data_starts, data_ends = merge_offset_ranges( + data_paths, + data_starts, + data_ends, + max_gap=max_gap, + max_block=max_block, + sort=True, + ) + + # Transfer the data byte-ranges into local memory + _transfer_ranges(fs, result, data_paths, data_starts, data_ends) + + # Add b"PAR1" to headers + _add_header_magic(result) + + return result + + +def _get_parquet_byte_ranges_from_metadata( + metadata, + fs, + engine, + columns=None, + row_groups=None, + max_gap=64_000, + max_block=256_000_000, + filters=None, +): + """Simplified version of `_get_parquet_byte_ranges` for + the case that an engine-specific `metadata` object is + provided, and the remote footer metadata does not need to + be transferred before calculating the required byte ranges. + """ + + # Use "engine" to collect data byte ranges + data_paths, data_starts, data_ends = engine._parquet_byte_ranges( + columns, row_groups=row_groups, metadata=metadata, filters=filters + ) + + # Merge adjacent offset ranges + data_paths, data_starts, data_ends = merge_offset_ranges( + data_paths, + data_starts, + data_ends, + max_gap=max_gap, + max_block=max_block, + sort=False, # Should be sorted + ) + + # Transfer the data byte-ranges into local memory + result = {fn: {} for fn in list(set(data_paths))} + _transfer_ranges(fs, result, data_paths, data_starts, data_ends) + + # Add b"PAR1" to header + _add_header_magic(result) + + return result + + +def _transfer_ranges(fs, blocks, paths, starts, ends): + # Use cat_ranges to gather the data byte_ranges + ranges = (paths, starts, ends) + for path, start, stop, data in zip(*ranges, fs.cat_ranges(*ranges)): + blocks[path][(start, stop)] = data + + +def _add_header_magic(data): + # Add b"PAR1" to file headers + for path in list(data): + add_magic = True + for k in data[path]: + if k[0] == 0 and k[1] >= 4: + add_magic = False + break + if add_magic: + data[path][(0, 4)] = b"PAR1" + + +def _set_engine(engine_str): + # Define a list of parquet engines to try + if engine_str == "auto": + try_engines = ("fastparquet", "pyarrow") + elif not isinstance(engine_str, str): + raise ValueError( + "Failed to set parquet engine! " + "Please pass 'fastparquet', 'pyarrow', or 'auto'" + ) + elif engine_str not in ("fastparquet", "pyarrow"): + raise ValueError(f"{engine_str} engine not supported by `fsspec.parquet`") + else: + try_engines = [engine_str] + + # Try importing the engines in `try_engines`, + # and choose the first one that succeeds + for engine in try_engines: + try: + if engine == "fastparquet": + return FastparquetEngine() + elif engine == "pyarrow": + return PyarrowEngine() + except ImportError: + pass + + # Raise an error if a supported parquet engine + # was not found + raise ImportError( + f"The following parquet engines are not installed " + f"in your python environment: {try_engines}." + f"Please install 'fastparquert' or 'pyarrow' to " + f"utilize the `fsspec.parquet` module." + ) + + +class FastparquetEngine: + # The purpose of the FastparquetEngine class is + # to check if fastparquet can be imported (on initialization) + # and to define a `_parquet_byte_ranges` method. In the + # future, this class may also be used to define other + # methods/logic that are specific to fastparquet. + + def __init__(self): + import fastparquet as fp + + self.fp = fp + + def _parquet_byte_ranges( + self, + columns, + row_groups=None, + metadata=None, + footer=None, + footer_start=None, + filters=None, + ): + # Initialize offset ranges and define ParqetFile metadata + pf = metadata + data_paths, data_starts, data_ends = [], [], [] + if filters and row_groups: + raise ValueError("filters and row_groups cannot be used together") + if pf is None: + pf = self.fp.ParquetFile(io.BytesIO(footer)) + + # Convert columns to a set and add any index columns + # specified in the pandas metadata (just in case) + column_set = None if columns is None else {c.split(".", 1)[0] for c in columns} + if column_set is not None and hasattr(pf, "pandas_metadata"): + md_index = [ + ind + for ind in pf.pandas_metadata.get("index_columns", []) + # Ignore RangeIndex information + if not isinstance(ind, dict) + ] + column_set |= set(md_index) + + # Check if row_groups is a list of integers + # or a list of row-group metadata + if filters: + from fastparquet.api import filter_row_groups + + row_group_indices = None + row_groups = filter_row_groups(pf, filters) + elif row_groups and not isinstance(row_groups[0], int): + # Input row_groups contains row-group metadata + row_group_indices = None + else: + # Input row_groups contains row-group indices + row_group_indices = row_groups + row_groups = pf.row_groups + if column_set is not None: + column_set = [ + _ if isinstance(_, list) else _.split(".") for _ in column_set + ] + + # Loop through column chunks to add required byte ranges + for r, row_group in enumerate(row_groups): + # Skip this row-group if we are targeting + # specific row-groups + if row_group_indices is None or r in row_group_indices: + # Find the target parquet-file path for `row_group` + fn = pf.row_group_filename(row_group) + + for column in row_group.columns: + name = column.meta_data.path_in_schema + # Skip this column if we are targeting specific columns + if column_set is None or _cmp(name, column_set): + file_offset0 = column.meta_data.dictionary_page_offset + if file_offset0 is None: + file_offset0 = column.meta_data.data_page_offset + num_bytes = column.meta_data.total_compressed_size + if footer_start is None or file_offset0 < footer_start: + data_paths.append(fn) + data_starts.append(file_offset0) + data_ends.append( + min( + file_offset0 + num_bytes, + footer_start or (file_offset0 + num_bytes), + ) + ) + + if metadata: + # The metadata in this call may map to multiple + # file paths. Need to include `data_paths` + return data_paths, data_starts, data_ends + return data_starts, data_ends + + +class PyarrowEngine: + # The purpose of the PyarrowEngine class is + # to check if pyarrow can be imported (on initialization) + # and to define a `_parquet_byte_ranges` method. In the + # future, this class may also be used to define other + # methods/logic that are specific to pyarrow. + + def __init__(self): + import pyarrow.parquet as pq + + self.pq = pq + + def _parquet_byte_ranges( + self, + columns, + row_groups=None, + metadata=None, + footer=None, + footer_start=None, + filters=None, + ): + if metadata is not None: + raise ValueError("metadata input not supported for PyarrowEngine") + if filters: + # there must be a way! + raise NotImplementedError + + data_starts, data_ends = [], [] + md = self.pq.ParquetFile(io.BytesIO(footer)).metadata + + # Convert columns to a set and add any index columns + # specified in the pandas metadata (just in case) + column_set = None if columns is None else set(columns) + if column_set is not None: + schema = md.schema.to_arrow_schema() + has_pandas_metadata = ( + schema.metadata is not None and b"pandas" in schema.metadata + ) + if has_pandas_metadata: + md_index = [ + ind + for ind in json.loads( + schema.metadata[b"pandas"].decode("utf8") + ).get("index_columns", []) + # Ignore RangeIndex information + if not isinstance(ind, dict) + ] + column_set |= set(md_index) + if column_set is not None: + column_set = [ + _[:1] if isinstance(_, list) else _.split(".")[:1] for _ in column_set + ] + + # Loop through column chunks to add required byte ranges + for r in range(md.num_row_groups): + # Skip this row-group if we are targeting + # specific row-groups + if row_groups is None or r in row_groups: + row_group = md.row_group(r) + for c in range(row_group.num_columns): + column = row_group.column(c) + name = column.path_in_schema.split(".") + # Skip this column if we are targeting specific columns + if column_set is None or _cmp(name, column_set): + meta = column.to_dict() + # Any offset could be the first one + file_offset0 = min( + _ + for _ in [ + meta.get("dictionary_page_offset"), + meta.get("data_page_offset"), + meta.get("index_page_offset"), + ] + if _ is not None + ) + if file_offset0 < footer_start: + data_starts.append(file_offset0) + data_ends.append( + min( + meta["total_compressed_size"] + file_offset0, + footer_start, + ) + ) + + data_starts.append(footer_start) + data_ends.append(footer_start + len(footer)) + return data_starts, data_ends + + +def _cmp(name, column_set): + return any(all(a == b for a, b in zip(name, _)) for _ in column_set) diff --git a/.venv/lib/python3.11/site-packages/fsspec/registry.py b/.venv/lib/python3.11/site-packages/fsspec/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..305d1e8908504fb6c38e94708bd7f20dba722705 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/registry.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +import importlib +import types +import warnings + +__all__ = ["registry", "get_filesystem_class", "default"] + +# internal, mutable +_registry: dict[str, type] = {} + +# external, immutable +registry = types.MappingProxyType(_registry) +default = "file" + + +def register_implementation(name, cls, clobber=False, errtxt=None): + """Add implementation class to the registry + + Parameters + ---------- + name: str + Protocol name to associate with the class + cls: class or str + if a class: fsspec-compliant implementation class (normally inherits from + ``fsspec.AbstractFileSystem``, gets added straight to the registry. If a + str, the full path to an implementation class like package.module.class, + which gets added to known_implementations, + so the import is deferred until the filesystem is actually used. + clobber: bool (optional) + Whether to overwrite a protocol with the same name; if False, will raise + instead. + errtxt: str (optional) + If given, then a failure to import the given class will result in this + text being given. + """ + if isinstance(cls, str): + if name in known_implementations and clobber is False: + if cls != known_implementations[name]["class"]: + raise ValueError( + f"Name ({name}) already in the known_implementations and clobber " + f"is False" + ) + else: + known_implementations[name] = { + "class": cls, + "err": errtxt or f"{cls} import failed for protocol {name}", + } + + else: + if name in registry and clobber is False: + if _registry[name] is not cls: + raise ValueError( + f"Name ({name}) already in the registry and clobber is False" + ) + else: + _registry[name] = cls + + +# protocols mapped to the class which implements them. This dict can be +# updated with register_implementation +known_implementations = { + "abfs": { + "class": "adlfs.AzureBlobFileSystem", + "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", + }, + "adl": { + "class": "adlfs.AzureDatalakeFileSystem", + "err": ( + "Azure Data Lake Storage Gen1 is retired and no longer supported. Please " + "install adlfs and use the `az://` protocol to access Azure Blob Storage " + "and Azure Data Lake Storage Gen2 instead." + ), + }, + "arrow_hdfs": { + "class": "fsspec.implementations.arrow.HadoopFileSystem", + "err": "pyarrow and local java libraries required for HDFS", + }, + "async_wrapper": { + "class": "fsspec.implementations.asyn_wrapper.AsyncFileSystemWrapper", + }, + "asynclocal": { + "class": "morefs.asyn_local.AsyncLocalFileSystem", + "err": "Install 'morefs[asynclocalfs]' to use AsyncLocalFileSystem", + }, + "asyncwrapper": { + "class": "fsspec.implementations.asyn_wrapper.AsyncFileSystemWrapper", + }, + "az": { + "class": "adlfs.AzureBlobFileSystem", + "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", + }, + "blockcache": {"class": "fsspec.implementations.cached.CachingFileSystem"}, + "box": { + "class": "boxfs.BoxFileSystem", + "err": "Please install boxfs to access BoxFileSystem", + }, + "cached": {"class": "fsspec.implementations.cached.CachingFileSystem"}, + "dask": { + "class": "fsspec.implementations.dask.DaskWorkerFileSystem", + "err": "Install dask distributed to access worker file system", + }, + "data": {"class": "fsspec.implementations.data.DataFileSystem"}, + "dbfs": { + "class": "fsspec.implementations.dbfs.DatabricksFileSystem", + "err": "Install the requests package to use the DatabricksFileSystem", + }, + "dir": {"class": "fsspec.implementations.dirfs.DirFileSystem"}, + "dropbox": { + "class": "dropboxdrivefs.DropboxDriveFileSystem", + "err": ( + 'DropboxFileSystem requires "dropboxdrivefs","requests" and "' + '"dropbox" to be installed' + ), + }, + "dvc": { + "class": "dvc.api.DVCFileSystem", + "err": "Install dvc to access DVCFileSystem", + }, + "file": {"class": "fsspec.implementations.local.LocalFileSystem"}, + "filecache": {"class": "fsspec.implementations.cached.WholeFileCacheFileSystem"}, + "ftp": {"class": "fsspec.implementations.ftp.FTPFileSystem"}, + "gcs": { + "class": "gcsfs.GCSFileSystem", + "err": "Please install gcsfs to access Google Storage", + }, + "gdrive": { + "class": "gdrive_fsspec.GoogleDriveFileSystem", + "err": "Please install gdrive_fs for access to Google Drive", + }, + "generic": {"class": "fsspec.generic.GenericFileSystem"}, + "gist": { + "class": "fsspec.implementations.gist.GistFileSystem", + "err": "Install the requests package to use the gist FS", + }, + "git": { + "class": "fsspec.implementations.git.GitFileSystem", + "err": "Install pygit2 to browse local git repos", + }, + "github": { + "class": "fsspec.implementations.github.GithubFileSystem", + "err": "Install the requests package to use the github FS", + }, + "gs": { + "class": "gcsfs.GCSFileSystem", + "err": "Please install gcsfs to access Google Storage", + }, + "hdfs": { + "class": "fsspec.implementations.arrow.HadoopFileSystem", + "err": "pyarrow and local java libraries required for HDFS", + }, + "hf": { + "class": "huggingface_hub.HfFileSystem", + "err": "Install huggingface_hub to access HfFileSystem", + }, + "http": { + "class": "fsspec.implementations.http.HTTPFileSystem", + "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', + }, + "https": { + "class": "fsspec.implementations.http.HTTPFileSystem", + "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', + }, + "jlab": { + "class": "fsspec.implementations.jupyter.JupyterFileSystem", + "err": "Jupyter FS requires requests to be installed", + }, + "jupyter": { + "class": "fsspec.implementations.jupyter.JupyterFileSystem", + "err": "Jupyter FS requires requests to be installed", + }, + "lakefs": { + "class": "lakefs_spec.LakeFSFileSystem", + "err": "Please install lakefs-spec to access LakeFSFileSystem", + }, + "libarchive": { + "class": "fsspec.implementations.libarchive.LibArchiveFileSystem", + "err": "LibArchive requires to be installed", + }, + "local": {"class": "fsspec.implementations.local.LocalFileSystem"}, + "memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"}, + "oci": { + "class": "ocifs.OCIFileSystem", + "err": "Install ocifs to access OCI Object Storage", + }, + "ocilake": { + "class": "ocifs.OCIFileSystem", + "err": "Install ocifs to access OCI Data Lake", + }, + "oss": { + "class": "ossfs.OSSFileSystem", + "err": "Install ossfs to access Alibaba Object Storage System", + }, + "pyscript": { + "class": "pyscript_fsspec_client.client.PyscriptFileSystem", + "err": "This only runs in a pyscript context", + }, + "reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"}, + "root": { + "class": "fsspec_xrootd.XRootDFileSystem", + "err": ( + "Install fsspec-xrootd to access xrootd storage system. " + "Note: 'root' is the protocol name for xrootd storage systems, " + "not referring to root directories" + ), + }, + "s3": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, + "s3a": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, + "sftp": { + "class": "fsspec.implementations.sftp.SFTPFileSystem", + "err": 'SFTPFileSystem requires "paramiko" to be installed', + }, + "simplecache": {"class": "fsspec.implementations.cached.SimpleCacheFileSystem"}, + "smb": { + "class": "fsspec.implementations.smb.SMBFileSystem", + "err": 'SMB requires "smbprotocol" or "smbprotocol[kerberos]" installed', + }, + "ssh": { + "class": "fsspec.implementations.sftp.SFTPFileSystem", + "err": 'SFTPFileSystem requires "paramiko" to be installed', + }, + "tar": {"class": "fsspec.implementations.tar.TarFileSystem"}, + "tos": { + "class": "tosfs.TosFileSystem", + "err": "Install tosfs to access ByteDance volcano engine Tinder Object Storage", + }, + "tosfs": { + "class": "tosfs.TosFileSystem", + "err": "Install tosfs to access ByteDance volcano engine Tinder Object Storage", + }, + "wandb": {"class": "wandbfs.WandbFS", "err": "Install wandbfs to access wandb"}, + "webdav": { + "class": "webdav4.fsspec.WebdavFileSystem", + "err": "Install webdav4 to access WebDAV", + }, + "webhdfs": { + "class": "fsspec.implementations.webhdfs.WebHDFS", + "err": 'webHDFS access requires "requests" to be installed', + }, + "zip": {"class": "fsspec.implementations.zip.ZipFileSystem"}, +} + +assert list(known_implementations) == sorted(known_implementations), ( + "Not in alphabetical order" +) + + +def get_filesystem_class(protocol): + """Fetch named protocol implementation from the registry + + The dict ``known_implementations`` maps protocol names to the locations + of classes implementing the corresponding file-system. When used for the + first time, appropriate imports will happen and the class will be placed in + the registry. All subsequent calls will fetch directly from the registry. + + Some protocol implementations require additional dependencies, and so the + import may fail. In this case, the string in the "err" field of the + ``known_implementations`` will be given as the error message. + """ + if not protocol: + protocol = default + + if protocol not in registry: + if protocol not in known_implementations: + raise ValueError(f"Protocol not known: {protocol}") + bit = known_implementations[protocol] + try: + register_implementation(protocol, _import_class(bit["class"])) + except ImportError as e: + raise ImportError(bit.get("err")) from e + cls = registry[protocol] + if getattr(cls, "protocol", None) in ("abstract", None): + cls.protocol = protocol + + return cls + + +s3_msg = """Your installed version of s3fs is very old and known to cause +severe performance issues, see also https://github.com/dask/dask/issues/10276 + +To fix, you should specify a lower version bound on s3fs, or +update the current installation. +""" + + +def _import_class(fqp: str): + """Take a fully-qualified path and return the imported class or identifier. + + ``fqp`` is of the form "package.module.klass" or + "package.module:subobject.klass". + + Warnings + -------- + This can import arbitrary modules. Make sure you haven't installed any modules + that may execute malicious code at import time. + """ + if ":" in fqp: + mod, name = fqp.rsplit(":", 1) + else: + mod, name = fqp.rsplit(".", 1) + + is_s3 = mod == "s3fs" + mod = importlib.import_module(mod) + if is_s3 and mod.__version__.split(".") < ["0", "5"]: + warnings.warn(s3_msg) + for part in name.split("."): + mod = getattr(mod, part) + + if not isinstance(mod, type): + raise TypeError(f"{fqp} is not a class") + + return mod + + +def filesystem(protocol, **storage_options): + """Instantiate filesystems for given protocol and arguments + + ``storage_options`` are specific to the protocol being chosen, and are + passed directly to the class. + """ + if protocol == "arrow_hdfs": + warnings.warn( + "The 'arrow_hdfs' protocol has been deprecated and will be " + "removed in the future. Specify it as 'hdfs'.", + DeprecationWarning, + ) + + cls = get_filesystem_class(protocol) + return cls(**storage_options) + + +def available_protocols(): + """Return a list of the implemented protocols. + + Note that any given protocol may require extra packages to be importable. + """ + return list(known_implementations) diff --git a/.venv/lib/python3.11/site-packages/fsspec/spec.py b/.venv/lib/python3.11/site-packages/fsspec/spec.py new file mode 100644 index 0000000000000000000000000000000000000000..a390bf06096ed26936dcfd6f6ac0d97848e14598 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/spec.py @@ -0,0 +1,2290 @@ +from __future__ import annotations + +import io +import json +import logging +import os +import threading +import warnings +import weakref +from errno import ESPIPE +from glob import has_magic +from hashlib import sha256 +from typing import Any, ClassVar + +from .callbacks import DEFAULT_CALLBACK +from .config import apply_config, conf +from .dircache import DirCache +from .transaction import Transaction +from .utils import ( + _unstrip_protocol, + glob_translate, + isfilelike, + other_paths, + read_block, + stringify_path, + tokenize, +) + +logger = logging.getLogger("fsspec") + + +def make_instance(cls, args, kwargs): + return cls(*args, **kwargs) + + +class _Cached(type): + """ + Metaclass for caching file system instances. + + Notes + ----- + Instances are cached according to + + * The values of the class attributes listed in `_extra_tokenize_attributes` + * The arguments passed to ``__init__``. + + This creates an additional reference to the filesystem, which prevents the + filesystem from being garbage collected when all *user* references go away. + A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* + be made for a filesystem instance to be garbage collected. + """ + + def __init__(cls, *args, **kwargs): + super().__init__(*args, **kwargs) + # Note: we intentionally create a reference here, to avoid garbage + # collecting instances when all other references are gone. To really + # delete a FileSystem, the cache must be cleared. + if conf.get("weakref_instance_cache"): # pragma: no cover + # debug option for analysing fork/spawn conditions + cls._cache = weakref.WeakValueDictionary() + else: + cls._cache = {} + cls._pid = os.getpid() + + def __call__(cls, *args, **kwargs): + kwargs = apply_config(cls, kwargs) + extra_tokens = tuple( + getattr(cls, attr, None) for attr in cls._extra_tokenize_attributes + ) + strip_tokenize_options = { + k: kwargs.pop(k) for k in cls._strip_tokenize_options if k in kwargs + } + if getattr(cls, "async_impl", False) and not kwargs.get("asynchronous", False): + token = tokenize(cls, cls._pid, *args, *extra_tokens, **kwargs) + else: + token = tokenize( + cls, cls._pid, threading.get_ident(), *args, *extra_tokens, **kwargs + ) + skip = kwargs.pop("skip_instance_cache", False) + if os.getpid() != cls._pid: + cls._cache.clear() + cls._pid = os.getpid() + if not skip and cls.cachable and token in cls._cache: + cls._latest = token + return cls._cache[token] + else: + obj = super().__call__(*args, **kwargs, **strip_tokenize_options) + # Setting _fs_token here causes some static linters to complain. + obj._fs_token_ = token + obj.storage_args = args + obj.storage_options = kwargs + if obj.async_impl and obj.mirror_sync_methods: + from .asyn import mirror_sync_methods + + mirror_sync_methods(obj) + + if cls.cachable and not skip: + cls._latest = token + cls._cache[token] = obj + return obj + + +class AbstractFileSystem(metaclass=_Cached): + """ + An abstract super-class for pythonic file-systems + + Implementations are expected to be compatible with or, better, subclass + from here. + """ + + cachable = True # this class can be cached, instances reused + _cached = False + blocksize = 2**22 + sep = "/" + protocol: ClassVar[str | tuple[str, ...]] = "abstract" + _latest = None + async_impl = False + mirror_sync_methods = False + root_marker = "" # For some FSs, may require leading '/' or other character + transaction_type = Transaction + + #: Extra *class attributes* that should be considered when hashing. + _extra_tokenize_attributes = () + #: *storage options* that should not be considered when hashing. + _strip_tokenize_options = () + + # Set by _Cached metaclass + storage_args: tuple[Any, ...] + storage_options: dict[str, Any] + + def __init__(self, *args, **storage_options): + """Create and configure file-system instance + + Instances may be cachable, so if similar enough arguments are seen + a new instance is not required. The token attribute exists to allow + implementations to cache instances if they wish. + + A reasonable default should be provided if there are no arguments. + + Subclasses should call this method. + + Parameters + ---------- + use_listings_cache, listings_expiry_time, max_paths: + passed to ``DirCache``, if the implementation supports + directory listing caching. Pass use_listings_cache=False + to disable such caching. + skip_instance_cache: bool + If this is a cachable implementation, pass True here to force + creating a new instance even if a matching instance exists, and prevent + storing this instance. + asynchronous: bool + loop: asyncio-compatible IOLoop or None + """ + if self._cached: + # reusing instance, don't change + return + self._cached = True + self._intrans = False + self._transaction = None + self._invalidated_caches_in_transaction = [] + self.dircache = DirCache(**storage_options) + + if storage_options.pop("add_docs", None): + warnings.warn("add_docs is no longer supported.", FutureWarning) + + if storage_options.pop("add_aliases", None): + warnings.warn("add_aliases has been removed.", FutureWarning) + # This is set in _Cached + self._fs_token_ = None + + @property + def fsid(self): + """Persistent filesystem id that can be used to compare filesystems + across sessions. + """ + raise NotImplementedError + + @property + def _fs_token(self): + return self._fs_token_ + + def __dask_tokenize__(self): + return self._fs_token + + def __hash__(self): + return int(self._fs_token, 16) + + def __eq__(self, other): + return isinstance(other, type(self)) and self._fs_token == other._fs_token + + def __reduce__(self): + return make_instance, (type(self), self.storage_args, self.storage_options) + + @classmethod + def _strip_protocol(cls, path): + """Turn path from fully-qualified to file-system-specific + + May require FS-specific handling, e.g., for relative paths or links. + """ + if isinstance(path, list): + return [cls._strip_protocol(p) for p in path] + path = stringify_path(path) + protos = (cls.protocol,) if isinstance(cls.protocol, str) else cls.protocol + for protocol in protos: + if path.startswith(protocol + "://"): + path = path[len(protocol) + 3 :] + elif path.startswith(protocol + "::"): + path = path[len(protocol) + 2 :] + path = path.rstrip("/") + # use of root_marker to make minimum required path, e.g., "/" + return path or cls.root_marker + + def unstrip_protocol(self, name: str) -> str: + """Format FS-specific path to generic, including protocol""" + protos = (self.protocol,) if isinstance(self.protocol, str) else self.protocol + for protocol in protos: + if name.startswith(f"{protocol}://"): + return name + return f"{protos[0]}://{name}" + + @staticmethod + def _get_kwargs_from_urls(path): + """If kwargs can be encoded in the paths, extract them here + + This should happen before instantiation of the class; incoming paths + then should be amended to strip the options in methods. + + Examples may look like an sftp path "sftp://user@host:/my/path", where + the user and host should become kwargs and later get stripped. + """ + # by default, nothing happens + return {} + + @classmethod + def current(cls): + """Return the most recently instantiated FileSystem + + If no instance has been created, then create one with defaults + """ + if cls._latest in cls._cache: + return cls._cache[cls._latest] + return cls() + + @property + def transaction(self): + """A context within which files are committed together upon exit + + Requires the file class to implement `.commit()` and `.discard()` + for the normal and exception cases. + """ + if self._transaction is None: + self._transaction = self.transaction_type(self) + return self._transaction + + def start_transaction(self): + """Begin write transaction for deferring files, non-context version""" + self._intrans = True + self._transaction = self.transaction_type(self) + return self.transaction + + def end_transaction(self): + """Finish write transaction, non-context version""" + self.transaction.complete() + self._transaction = None + # The invalid cache must be cleared after the transaction is completed. + for path in self._invalidated_caches_in_transaction: + self.invalidate_cache(path) + self._invalidated_caches_in_transaction.clear() + + def invalidate_cache(self, path=None): + """ + Discard any cached directory information + + Parameters + ---------- + path: string or None + If None, clear all listings cached else listings at or under given + path. + """ + # Not necessary to implement invalidation mechanism, may have no cache. + # But if have, you should call this method of parent class from your + # subclass to ensure expiring caches after transacations correctly. + # See the implementation of FTPFileSystem in ftp.py + if self._intrans: + self._invalidated_caches_in_transaction.append(path) + + def mkdir(self, path, create_parents=True, **kwargs): + """ + Create directory entry at path + + For systems that don't have true directories, may create an for + this instance only and not touch the real filesystem + + Parameters + ---------- + path: str + location + create_parents: bool + if True, this is equivalent to ``makedirs`` + kwargs: + may be permissions, etc. + """ + pass # not necessary to implement, may not have directories + + def makedirs(self, path, exist_ok=False): + """Recursively make directories + + Creates directory at path and any intervening required directories. + Raises exception if, for instance, the path already exists but is a + file. + + Parameters + ---------- + path: str + leaf directory name + exist_ok: bool (False) + If False, will error if the target already exists + """ + pass # not necessary to implement, may not have directories + + def rmdir(self, path): + """Remove a directory, if empty""" + pass # not necessary to implement, may not have directories + + def ls(self, path, detail=True, **kwargs): + """List objects at path. + + This should include subdirectories and files at that location. The + difference between a file and a directory must be clear when details + are requested. + + The specific keys, or perhaps a FileInfo class, or similar, is TBD, + but must be consistent across implementations. + Must include: + + - full path to the entry (without protocol) + - size of the entry, in bytes. If the value cannot be determined, will + be ``None``. + - type of entry, "file", "directory" or other + + Additional information + may be present, appropriate to the file-system, e.g., generation, + checksum, etc. + + May use refresh=True|False to allow use of self._ls_from_cache to + check for a saved listing and avoid calling the backend. This would be + common where listing may be expensive. + + Parameters + ---------- + path: str + detail: bool + if True, gives a list of dictionaries, where each is the same as + the result of ``info(path)``. If False, gives a list of paths + (str). + kwargs: may have additional backend-specific options, such as version + information + + Returns + ------- + List of strings if detail is False, or list of directory information + dicts if detail is True. + """ + raise NotImplementedError + + def _ls_from_cache(self, path): + """Check cache for listing + + Returns listing, if found (may be empty list for a directly that exists + but contains nothing), None if not in cache. + """ + parent = self._parent(path) + try: + return self.dircache[path.rstrip("/")] + except KeyError: + pass + try: + files = [ + f + for f in self.dircache[parent] + if f["name"] == path + or (f["name"] == path.rstrip("/") and f["type"] == "directory") + ] + if len(files) == 0: + # parent dir was listed but did not contain this file + raise FileNotFoundError(path) + return files + except KeyError: + pass + + def walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs): + """Return all files under the given path. + + List all files, recursing into subdirectories; output is iterator-style, + like ``os.walk()``. For a simple list of files, ``find()`` is available. + + When topdown is True, the caller can modify the dirnames list in-place (perhaps + using del or slice assignment), and walk() will + only recurse into the subdirectories whose names remain in dirnames; + this can be used to prune the search, impose a specific order of visiting, + or even to inform walk() about directories the caller creates or renames before + it resumes walk() again. + Modifying dirnames when topdown is False has no effect. (see os.walk) + + Note that the "files" outputted will include anything that is not + a directory, such as links. + + Parameters + ---------- + path: str + Root to recurse into + maxdepth: int + Maximum recursion depth. None means limitless, but not recommended + on link-based file-systems. + topdown: bool (True) + Whether to walk the directory tree from the top downwards or from + the bottom upwards. + on_error: "omit", "raise", a callable + if omit (default), path with exception will simply be empty; + If raise, an underlying exception will be raised; + if callable, it will be called with a single OSError instance as argument + kwargs: passed to ``ls`` + """ + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + path = self._strip_protocol(path) + full_dirs = {} + dirs = {} + files = {} + + detail = kwargs.pop("detail", False) + try: + listing = self.ls(path, detail=True, **kwargs) + except (FileNotFoundError, OSError) as e: + if on_error == "raise": + raise + if callable(on_error): + on_error(e) + return + + for info in listing: + # each info name must be at least [path]/part , but here + # we check also for names like [path]/part/ + pathname = info["name"].rstrip("/") + name = pathname.rsplit("/", 1)[-1] + if info["type"] == "directory" and pathname != path: + # do not include "self" path + full_dirs[name] = pathname + dirs[name] = info + elif pathname == path: + # file-like with same name as give path + files[""] = info + else: + files[name] = info + + if not detail: + dirs = list(dirs) + files = list(files) + + if topdown: + # Yield before recursion if walking top down + yield path, dirs, files + + if maxdepth is not None: + maxdepth -= 1 + if maxdepth < 1: + if not topdown: + yield path, dirs, files + return + + for d in dirs: + yield from self.walk( + full_dirs[d], + maxdepth=maxdepth, + detail=detail, + topdown=topdown, + **kwargs, + ) + + if not topdown: + # Yield after recursion if walking bottom up + yield path, dirs, files + + def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): + """List all files below path. + + Like posix ``find`` command without conditions + + Parameters + ---------- + path : str + maxdepth: int or None + If not None, the maximum number of levels to descend + withdirs: bool + Whether to include directory paths in the output. This is True + when used by glob, but users usually only want files. + kwargs are passed to ``ls``. + """ + # TODO: allow equivalent of -name parameter + path = self._strip_protocol(path) + out = {} + + # Add the root directory if withdirs is requested + # This is needed for posix glob compliance + if withdirs and path != "" and self.isdir(path): + out[path] = self.info(path) + + for _, dirs, files in self.walk(path, maxdepth, detail=True, **kwargs): + if withdirs: + files.update(dirs) + out.update({info["name"]: info for name, info in files.items()}) + if not out and self.isfile(path): + # walk works on directories, but find should also return [path] + # when path happens to be a file + out[path] = {} + names = sorted(out) + if not detail: + return names + else: + return {name: out[name] for name in names} + + def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs): + """Space used by files and optionally directories within a path + + Directory size does not include the size of its contents. + + Parameters + ---------- + path: str + total: bool + Whether to sum all the file sizes + maxdepth: int or None + Maximum number of directory levels to descend, None for unlimited. + withdirs: bool + Whether to include directory paths in the output. + kwargs: passed to ``find`` + + Returns + ------- + Dict of {path: size} if total=False, or int otherwise, where numbers + refer to bytes used. + """ + sizes = {} + if withdirs and self.isdir(path): + # Include top-level directory in output + info = self.info(path) + sizes[info["name"]] = info["size"] + for f in self.find(path, maxdepth=maxdepth, withdirs=withdirs, **kwargs): + info = self.info(f) + sizes[info["name"]] = info["size"] + if total: + return sum(sizes.values()) + else: + return sizes + + def glob(self, path, maxdepth=None, **kwargs): + """Find files by glob-matching. + + Pattern matching capabilities for finding files that match the given pattern. + + Parameters + ---------- + path: str + The glob pattern to match against + maxdepth: int or None + Maximum depth for ``'**'`` patterns. Applied on the first ``'**'`` found. + Must be at least 1 if provided. + kwargs: + Additional arguments passed to ``find`` (e.g., detail=True) + + Returns + ------- + List of matched paths, or dict of paths and their info if detail=True + + Notes + ----- + Supported patterns: + - '*': Matches any sequence of characters within a single directory level + - ``'**'``: Matches any number of directory levels (must be an entire path component) + - '?': Matches exactly one character + - '[abc]': Matches any character in the set + - '[a-z]': Matches any character in the range + - '[!abc]': Matches any character NOT in the set + + Special behaviors: + - If the path ends with '/', only folders are returned + - Consecutive '*' characters are compressed into a single '*' + - Empty brackets '[]' never match anything + - Negated empty brackets '[!]' match any single character + - Special characters in character classes are escaped properly + + Limitations: + - ``'**'`` must be a complete path component (e.g., ``'a/**/b'``, not ``'a**b'``) + - No brace expansion ('{a,b}.txt') + - No extended glob patterns ('+(pattern)', '!(pattern)') + """ + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + import re + + seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,) + ends_with_sep = path.endswith(seps) # _strip_protocol strips trailing slash + path = self._strip_protocol(path) + append_slash_to_dirname = ends_with_sep or path.endswith( + tuple(sep + "**" for sep in seps) + ) + idx_star = path.find("*") if path.find("*") >= 0 else len(path) + idx_qmark = path.find("?") if path.find("?") >= 0 else len(path) + idx_brace = path.find("[") if path.find("[") >= 0 else len(path) + + min_idx = min(idx_star, idx_qmark, idx_brace) + + detail = kwargs.pop("detail", False) + withdirs = kwargs.pop("withdirs", True) + + if not has_magic(path): + if self.exists(path, **kwargs): + if not detail: + return [path] + else: + return {path: self.info(path, **kwargs)} + else: + if not detail: + return [] # glob of non-existent returns empty + else: + return {} + elif "/" in path[:min_idx]: + min_idx = path[:min_idx].rindex("/") + root = path[: min_idx + 1] + depth = path[min_idx + 1 :].count("/") + 1 + else: + root = "" + depth = path[min_idx + 1 :].count("/") + 1 + + if "**" in path: + if maxdepth is not None: + idx_double_stars = path.find("**") + depth_double_stars = path[idx_double_stars:].count("/") + 1 + depth = depth - depth_double_stars + maxdepth + else: + depth = None + + allpaths = self.find( + root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs + ) + + pattern = glob_translate(path + ("/" if ends_with_sep else "")) + pattern = re.compile(pattern) + + out = { + p: info + for p, info in sorted(allpaths.items()) + if pattern.match( + p + "/" + if append_slash_to_dirname and info["type"] == "directory" + else p + ) + } + + if detail: + return out + else: + return list(out) + + def exists(self, path, **kwargs): + """Is there a file at the given path""" + try: + self.info(path, **kwargs) + return True + except: # noqa: E722 + # any exception allowed bar FileNotFoundError? + return False + + def lexists(self, path, **kwargs): + """If there is a file at the given path (including + broken links)""" + return self.exists(path) + + def info(self, path, **kwargs): + """Give details of entry at path + + Returns a single dictionary, with exactly the same information as ``ls`` + would with ``detail=True``. + + The default implementation calls ls and could be overridden by a + shortcut. kwargs are passed on to ```ls()``. + + Some file systems might not be able to measure the file's size, in + which case, the returned dict will include ``'size': None``. + + Returns + ------- + dict with keys: name (full path in the FS), size (in bytes), type (file, + directory, or something else) and other FS-specific keys. + """ + path = self._strip_protocol(path) + out = self.ls(self._parent(path), detail=True, **kwargs) + out = [o for o in out if o["name"].rstrip("/") == path] + if out: + return out[0] + out = self.ls(path, detail=True, **kwargs) + path = path.rstrip("/") + out1 = [o for o in out if o["name"].rstrip("/") == path] + if len(out1) == 1: + if "size" not in out1[0]: + out1[0]["size"] = None + return out1[0] + elif len(out1) > 1 or out: + return {"name": path, "size": 0, "type": "directory"} + else: + raise FileNotFoundError(path) + + def checksum(self, path): + """Unique value for current version of file + + If the checksum is the same from one moment to another, the contents + are guaranteed to be the same. If the checksum changes, the contents + *might* have changed. + + This should normally be overridden; default will probably capture + creation/modification timestamp (which would be good) or maybe + access timestamp (which would be bad) + """ + return int(tokenize(self.info(path)), 16) + + def size(self, path): + """Size in bytes of file""" + return self.info(path).get("size", None) + + def sizes(self, paths): + """Size in bytes of each file in a list of paths""" + return [self.size(p) for p in paths] + + def isdir(self, path): + """Is this entry directory-like?""" + try: + return self.info(path)["type"] == "directory" + except OSError: + return False + + def isfile(self, path): + """Is this entry file-like?""" + try: + return self.info(path)["type"] == "file" + except: # noqa: E722 + return False + + def read_text(self, path, encoding=None, errors=None, newline=None, **kwargs): + """Get the contents of the file as a string. + + Parameters + ---------- + path: str + URL of file on this filesystems + encoding, errors, newline: same as `open`. + """ + with self.open( + path, + mode="r", + encoding=encoding, + errors=errors, + newline=newline, + **kwargs, + ) as f: + return f.read() + + def write_text( + self, path, value, encoding=None, errors=None, newline=None, **kwargs + ): + """Write the text to the given file. + + An existing file will be overwritten. + + Parameters + ---------- + path: str + URL of file on this filesystems + value: str + Text to write. + encoding, errors, newline: same as `open`. + """ + with self.open( + path, + mode="w", + encoding=encoding, + errors=errors, + newline=newline, + **kwargs, + ) as f: + return f.write(value) + + def cat_file(self, path, start=None, end=None, **kwargs): + """Get the content of a file + + Parameters + ---------- + path: URL of file on this filesystems + start, end: int + Bytes limits of the read. If negative, backwards from end, + like usual python slices. Either can be None for start or + end of file, respectively + kwargs: passed to ``open()``. + """ + # explicitly set buffering off? + with self.open(path, "rb", **kwargs) as f: + if start is not None: + if start >= 0: + f.seek(start) + else: + f.seek(max(0, f.size + start)) + if end is not None: + if end < 0: + end = f.size + end + return f.read(end - f.tell()) + return f.read() + + def pipe_file(self, path, value, mode="overwrite", **kwargs): + """Set the bytes of given file""" + if mode == "create" and self.exists(path): + # non-atomic but simple way; or could use "xb" in open(), which is likely + # not as well supported + raise FileExistsError + with self.open(path, "wb", **kwargs) as f: + f.write(value) + + def pipe(self, path, value=None, **kwargs): + """Put value into path + + (counterpart to ``cat``) + + Parameters + ---------- + path: string or dict(str, bytes) + If a string, a single remote location to put ``value`` bytes; if a dict, + a mapping of {path: bytesvalue}. + value: bytes, optional + If using a single path, these are the bytes to put there. Ignored if + ``path`` is a dict + """ + if isinstance(path, str): + self.pipe_file(self._strip_protocol(path), value, **kwargs) + elif isinstance(path, dict): + for k, v in path.items(): + self.pipe_file(self._strip_protocol(k), v, **kwargs) + else: + raise ValueError("path must be str or dict") + + def cat_ranges( + self, paths, starts, ends, max_gap=None, on_error="return", **kwargs + ): + """Get the contents of byte ranges from one or more files + + Parameters + ---------- + paths: list + A list of of filepaths on this filesystems + starts, ends: int or list + Bytes limits of the read. If using a single int, the same value will be + used to read all the specified files. + """ + if max_gap is not None: + raise NotImplementedError + if not isinstance(paths, list): + raise TypeError + if not isinstance(starts, list): + starts = [starts] * len(paths) + if not isinstance(ends, list): + ends = [ends] * len(paths) + if len(starts) != len(paths) or len(ends) != len(paths): + raise ValueError + out = [] + for p, s, e in zip(paths, starts, ends): + try: + out.append(self.cat_file(p, s, e, **kwargs)) + except Exception as e: + if on_error == "return": + out.append(e) + else: + raise + return out + + def cat(self, path, recursive=False, on_error="raise", **kwargs): + """Fetch (potentially multiple) paths' contents + + Parameters + ---------- + recursive: bool + If True, assume the path(s) are directories, and get all the + contained files + on_error : "raise", "omit", "return" + If raise, an underlying exception will be raised (converted to KeyError + if the type is in self.missing_exceptions); if omit, keys with exception + will simply not be included in the output; if "return", all keys are + included in the output, but the value will be bytes or an exception + instance. + kwargs: passed to cat_file + + Returns + ------- + dict of {path: contents} if there are multiple paths + or the path has been otherwise expanded + """ + paths = self.expand_path(path, recursive=recursive, **kwargs) + if ( + len(paths) > 1 + or isinstance(path, list) + or paths[0] != self._strip_protocol(path) + ): + out = {} + for path in paths: + try: + out[path] = self.cat_file(path, **kwargs) + except Exception as e: + if on_error == "raise": + raise + if on_error == "return": + out[path] = e + return out + else: + return self.cat_file(paths[0], **kwargs) + + def get_file(self, rpath, lpath, callback=DEFAULT_CALLBACK, outfile=None, **kwargs): + """Copy single remote file to local""" + from .implementations.local import LocalFileSystem + + if isfilelike(lpath): + outfile = lpath + elif self.isdir(rpath): + os.makedirs(lpath, exist_ok=True) + return None + + fs = LocalFileSystem(auto_mkdir=True) + fs.makedirs(fs._parent(lpath), exist_ok=True) + + with self.open(rpath, "rb", **kwargs) as f1: + if outfile is None: + outfile = open(lpath, "wb") + + try: + callback.set_size(getattr(f1, "size", None)) + data = True + while data: + data = f1.read(self.blocksize) + segment_len = outfile.write(data) + if segment_len is None: + segment_len = len(data) + callback.relative_update(segment_len) + finally: + if not isfilelike(lpath): + outfile.close() + + def get( + self, + rpath, + lpath, + recursive=False, + callback=DEFAULT_CALLBACK, + maxdepth=None, + **kwargs, + ): + """Copy file(s) to local. + + Copies a specific file or tree of files (if recursive=True). If lpath + ends with a "/", it will be assumed to be a directory, and target files + will go within. Can submit a list of paths, which may be glob-patterns + and will be expanded. + + Calls get_file for each source. + """ + if isinstance(lpath, list) and isinstance(rpath, list): + # No need to expand paths when both source and destination + # are provided as lists + rpaths = rpath + lpaths = lpath + else: + from .implementations.local import ( + LocalFileSystem, + make_path_posix, + trailing_sep, + ) + + source_is_str = isinstance(rpath, str) + rpaths = self.expand_path( + rpath, recursive=recursive, maxdepth=maxdepth, **kwargs + ) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + rpaths = [p for p in rpaths if not (trailing_sep(p) or self.isdir(p))] + if not rpaths: + return + + if isinstance(lpath, str): + lpath = make_path_posix(lpath) + + source_is_file = len(rpaths) == 1 + dest_is_dir = isinstance(lpath, str) and ( + trailing_sep(lpath) or LocalFileSystem().isdir(lpath) + ) + + exists = source_is_str and ( + (has_magic(rpath) and source_is_file) + or (not has_magic(rpath) and dest_is_dir and not trailing_sep(rpath)) + ) + lpaths = other_paths( + rpaths, + lpath, + exists=exists, + flatten=not source_is_str, + ) + + callback.set_size(len(lpaths)) + for lpath, rpath in callback.wrap(zip(lpaths, rpaths)): + with callback.branched(rpath, lpath) as child: + self.get_file(rpath, lpath, callback=child, **kwargs) + + def put_file( + self, lpath, rpath, callback=DEFAULT_CALLBACK, mode="overwrite", **kwargs + ): + """Copy single file to remote""" + if mode == "create" and self.exists(rpath): + raise FileExistsError + if os.path.isdir(lpath): + self.makedirs(rpath, exist_ok=True) + return None + + with open(lpath, "rb") as f1: + size = f1.seek(0, 2) + callback.set_size(size) + f1.seek(0) + + self.mkdirs(self._parent(os.fspath(rpath)), exist_ok=True) + with self.open(rpath, "wb", **kwargs) as f2: + while f1.tell() < size: + data = f1.read(self.blocksize) + segment_len = f2.write(data) + if segment_len is None: + segment_len = len(data) + callback.relative_update(segment_len) + + def put( + self, + lpath, + rpath, + recursive=False, + callback=DEFAULT_CALLBACK, + maxdepth=None, + **kwargs, + ): + """Copy file(s) from local. + + Copies a specific file or tree of files (if recursive=True). If rpath + ends with a "/", it will be assumed to be a directory, and target files + will go within. + + Calls put_file for each source. + """ + if isinstance(lpath, list) and isinstance(rpath, list): + # No need to expand paths when both source and destination + # are provided as lists + rpaths = rpath + lpaths = lpath + else: + from .implementations.local import ( + LocalFileSystem, + make_path_posix, + trailing_sep, + ) + + source_is_str = isinstance(lpath, str) + if source_is_str: + lpath = make_path_posix(lpath) + fs = LocalFileSystem() + lpaths = fs.expand_path( + lpath, recursive=recursive, maxdepth=maxdepth, **kwargs + ) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))] + if not lpaths: + return + + source_is_file = len(lpaths) == 1 + dest_is_dir = isinstance(rpath, str) and ( + trailing_sep(rpath) or self.isdir(rpath) + ) + + rpath = ( + self._strip_protocol(rpath) + if isinstance(rpath, str) + else [self._strip_protocol(p) for p in rpath] + ) + exists = source_is_str and ( + (has_magic(lpath) and source_is_file) + or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath)) + ) + rpaths = other_paths( + lpaths, + rpath, + exists=exists, + flatten=not source_is_str, + ) + + callback.set_size(len(rpaths)) + for lpath, rpath in callback.wrap(zip(lpaths, rpaths)): + with callback.branched(lpath, rpath) as child: + self.put_file(lpath, rpath, callback=child, **kwargs) + + def head(self, path, size=1024): + """Get the first ``size`` bytes from file""" + with self.open(path, "rb") as f: + return f.read(size) + + def tail(self, path, size=1024): + """Get the last ``size`` bytes from file""" + with self.open(path, "rb") as f: + f.seek(max(-size, -f.size), 2) + return f.read() + + def cp_file(self, path1, path2, **kwargs): + raise NotImplementedError + + def copy( + self, path1, path2, recursive=False, maxdepth=None, on_error=None, **kwargs + ): + """Copy within two locations in the filesystem + + on_error : "raise", "ignore" + If raise, any not-found exceptions will be raised; if ignore any + not-found exceptions will cause the path to be skipped; defaults to + raise unless recursive is true, where the default is ignore + """ + if on_error is None and recursive: + on_error = "ignore" + elif on_error is None: + on_error = "raise" + + if isinstance(path1, list) and isinstance(path2, list): + # No need to expand paths when both source and destination + # are provided as lists + paths1 = path1 + paths2 = path2 + else: + from .implementations.local import trailing_sep + + source_is_str = isinstance(path1, str) + paths1 = self.expand_path( + path1, recursive=recursive, maxdepth=maxdepth, **kwargs + ) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + paths1 = [p for p in paths1 if not (trailing_sep(p) or self.isdir(p))] + if not paths1: + return + + source_is_file = len(paths1) == 1 + dest_is_dir = isinstance(path2, str) and ( + trailing_sep(path2) or self.isdir(path2) + ) + + exists = source_is_str and ( + (has_magic(path1) and source_is_file) + or (not has_magic(path1) and dest_is_dir and not trailing_sep(path1)) + ) + paths2 = other_paths( + paths1, + path2, + exists=exists, + flatten=not source_is_str, + ) + + for p1, p2 in zip(paths1, paths2): + try: + self.cp_file(p1, p2, **kwargs) + except FileNotFoundError: + if on_error == "raise": + raise + + def expand_path( + self, path, recursive=False, maxdepth=None, assume_literal=False, **kwargs + ): + """Turn one or more globs or directories into a list of all matching paths + to files or directories. + + kwargs are passed to ``glob`` or ``find``, which may in turn call ``ls`` + """ + + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + if isinstance(path, (str, os.PathLike)): + out = self.expand_path([path], recursive, maxdepth, **kwargs) + else: + out = set() + path = [self._strip_protocol(p) for p in path] + for p in path: + if not assume_literal and has_magic(p): + bit = set(self.glob(p, maxdepth=maxdepth, **kwargs)) + out |= bit + if recursive: + # glob call above expanded one depth so if maxdepth is defined + # then decrement it in expand_path call below. If it is zero + # after decrementing then avoid expand_path call. + if maxdepth is not None and maxdepth <= 1: + continue + out |= set( + self.expand_path( + list(bit), + recursive=recursive, + maxdepth=maxdepth - 1 if maxdepth is not None else None, + assume_literal=True, + **kwargs, + ) + ) + continue + elif recursive: + rec = set( + self.find( + p, maxdepth=maxdepth, withdirs=True, detail=False, **kwargs + ) + ) + out |= rec + if p not in out and (recursive is False or self.exists(p)): + # should only check once, for the root + out.add(p) + if not out: + raise FileNotFoundError(path) + return sorted(out) + + def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs): + """Move file(s) from one location to another""" + if path1 == path2: + logger.debug("%s mv: The paths are the same, so no files were moved.", self) + else: + # explicitly raise exception to prevent data corruption + self.copy( + path1, path2, recursive=recursive, maxdepth=maxdepth, on_error="raise" + ) + self.rm(path1, recursive=recursive) + + def rm_file(self, path): + """Delete a file""" + self._rm(path) + + def _rm(self, path): + """Delete one file""" + # this is the old name for the method, prefer rm_file + raise NotImplementedError + + def rm(self, path, recursive=False, maxdepth=None): + """Delete files. + + Parameters + ---------- + path: str or list of str + File(s) to delete. + recursive: bool + If file(s) are directories, recursively delete contents and then + also remove the directory + maxdepth: int or None + Depth to pass to walk for finding files to delete, if recursive. + If None, there will be no limit and infinite recursion may be + possible. + """ + path = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) + for p in reversed(path): + self.rm_file(p) + + @classmethod + def _parent(cls, path): + path = cls._strip_protocol(path) + if "/" in path: + parent = path.rsplit("/", 1)[0].lstrip(cls.root_marker) + return cls.root_marker + parent + else: + return cls.root_marker + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + """Return raw bytes-mode file-like from the file-system""" + return AbstractBufferedFile( + self, + path, + mode, + block_size, + autocommit, + cache_options=cache_options, + **kwargs, + ) + + def open( + self, + path, + mode="rb", + block_size=None, + cache_options=None, + compression=None, + **kwargs, + ): + """ + Return a file-like object from the filesystem + + The resultant instance must function correctly in a context ``with`` + block. + + Parameters + ---------- + path: str + Target file + mode: str like 'rb', 'w' + See builtin ``open()`` + Mode "x" (exclusive write) may be implemented by the backend. Even if + it is, whether it is checked up front or on commit, and whether it is + atomic is implementation-dependent. + block_size: int + Some indication of buffering - this is a value in bytes + cache_options : dict, optional + Extra arguments to pass through to the cache. + compression: string or None + If given, open file using compression codec. Can either be a compression + name (a key in ``fsspec.compression.compr``) or "infer" to guess the + compression from the filename suffix. + encoding, errors, newline: passed on to TextIOWrapper for text mode + """ + import io + + path = self._strip_protocol(path) + if "b" not in mode: + mode = mode.replace("t", "") + "b" + + text_kwargs = { + k: kwargs.pop(k) + for k in ["encoding", "errors", "newline"] + if k in kwargs + } + return io.TextIOWrapper( + self.open( + path, + mode, + block_size=block_size, + cache_options=cache_options, + compression=compression, + **kwargs, + ), + **text_kwargs, + ) + else: + ac = kwargs.pop("autocommit", not self._intrans) + f = self._open( + path, + mode=mode, + block_size=block_size, + autocommit=ac, + cache_options=cache_options, + **kwargs, + ) + if compression is not None: + from fsspec.compression import compr + from fsspec.core import get_compression + + compression = get_compression(path, compression) + compress = compr[compression] + f = compress(f, mode=mode[0]) + + if not ac and "r" not in mode: + self.transaction.files.append(f) + return f + + def touch(self, path, truncate=True, **kwargs): + """Create empty file, or update timestamp + + Parameters + ---------- + path: str + file location + truncate: bool + If True, always set file size to 0; if False, update timestamp and + leave file unchanged, if backend allows this + """ + if truncate or not self.exists(path): + with self.open(path, "wb", **kwargs): + pass + else: + raise NotImplementedError # update timestamp, if possible + + def ukey(self, path): + """Hash of file properties, to tell if it has changed""" + return sha256(str(self.info(path)).encode()).hexdigest() + + def read_block(self, fn, offset, length, delimiter=None): + """Read a block of bytes from + + Starting at ``offset`` of the file, read ``length`` bytes. If + ``delimiter`` is set then we ensure that the read starts and stops at + delimiter boundaries that follow the locations ``offset`` and ``offset + + length``. If ``offset`` is zero then we start at zero. The + bytestring returned WILL include the end delimiter string. + + If offset+length is beyond the eof, reads to eof. + + Parameters + ---------- + fn: string + Path to filename + offset: int + Byte offset to start read + length: int + Number of bytes to read. If None, read to end. + delimiter: bytes (optional) + Ensure reading starts and stops at delimiter bytestring + + Examples + -------- + >>> fs.read_block('data/file.csv', 0, 13) # doctest: +SKIP + b'Alice, 100\\nBo' + >>> fs.read_block('data/file.csv', 0, 13, delimiter=b'\\n') # doctest: +SKIP + b'Alice, 100\\nBob, 200\\n' + + Use ``length=None`` to read to the end of the file. + >>> fs.read_block('data/file.csv', 0, None, delimiter=b'\\n') # doctest: +SKIP + b'Alice, 100\\nBob, 200\\nCharlie, 300' + + See Also + -------- + :func:`fsspec.utils.read_block` + """ + with self.open(fn, "rb") as f: + size = f.size + if length is None: + length = size + if size is not None and offset + length > size: + length = size - offset + return read_block(f, offset, length, delimiter) + + def to_json(self, *, include_password: bool = True) -> str: + """ + JSON representation of this filesystem instance. + + Parameters + ---------- + include_password: bool, default True + Whether to include the password (if any) in the output. + + Returns + ------- + JSON string with keys ``cls`` (the python location of this class), + protocol (text name of this class's protocol, first one in case of + multiple), ``args`` (positional args, usually empty), and all other + keyword arguments as their own keys. + + Warnings + -------- + Serialized filesystems may contain sensitive information which have been + passed to the constructor, such as passwords and tokens. Make sure you + store and send them in a secure environment! + """ + from .json import FilesystemJSONEncoder + + return json.dumps( + self, + cls=type( + "_FilesystemJSONEncoder", + (FilesystemJSONEncoder,), + {"include_password": include_password}, + ), + ) + + @staticmethod + def from_json(blob: str) -> AbstractFileSystem: + """ + Recreate a filesystem instance from JSON representation. + + See ``.to_json()`` for the expected structure of the input. + + Parameters + ---------- + blob: str + + Returns + ------- + file system instance, not necessarily of this particular class. + + Warnings + -------- + This can import arbitrary modules (as determined by the ``cls`` key). + Make sure you haven't installed any modules that may execute malicious code + at import time. + """ + from .json import FilesystemJSONDecoder + + return json.loads(blob, cls=FilesystemJSONDecoder) + + def to_dict(self, *, include_password: bool = True) -> dict[str, Any]: + """ + JSON-serializable dictionary representation of this filesystem instance. + + Parameters + ---------- + include_password: bool, default True + Whether to include the password (if any) in the output. + + Returns + ------- + Dictionary with keys ``cls`` (the python location of this class), + protocol (text name of this class's protocol, first one in case of + multiple), ``args`` (positional args, usually empty), and all other + keyword arguments as their own keys. + + Warnings + -------- + Serialized filesystems may contain sensitive information which have been + passed to the constructor, such as passwords and tokens. Make sure you + store and send them in a secure environment! + """ + from .json import FilesystemJSONEncoder + + json_encoder = FilesystemJSONEncoder() + + cls = type(self) + proto = self.protocol + + storage_options = dict(self.storage_options) + if not include_password: + storage_options.pop("password", None) + + return dict( + cls=f"{cls.__module__}:{cls.__name__}", + protocol=proto[0] if isinstance(proto, (tuple, list)) else proto, + args=json_encoder.make_serializable(self.storage_args), + **json_encoder.make_serializable(storage_options), + ) + + @staticmethod + def from_dict(dct: dict[str, Any]) -> AbstractFileSystem: + """ + Recreate a filesystem instance from dictionary representation. + + See ``.to_dict()`` for the expected structure of the input. + + Parameters + ---------- + dct: Dict[str, Any] + + Returns + ------- + file system instance, not necessarily of this particular class. + + Warnings + -------- + This can import arbitrary modules (as determined by the ``cls`` key). + Make sure you haven't installed any modules that may execute malicious code + at import time. + """ + from .json import FilesystemJSONDecoder + + json_decoder = FilesystemJSONDecoder() + + dct = dict(dct) # Defensive copy + + cls = FilesystemJSONDecoder.try_resolve_fs_cls(dct) + if cls is None: + raise ValueError("Not a serialized AbstractFileSystem") + + dct.pop("cls", None) + dct.pop("protocol", None) + + return cls( + *json_decoder.unmake_serializable(dct.pop("args", ())), + **json_decoder.unmake_serializable(dct), + ) + + def _get_pyarrow_filesystem(self): + """ + Make a version of the FS instance which will be acceptable to pyarrow + """ + # all instances already also derive from pyarrow + return self + + def get_mapper(self, root="", check=False, create=False, missing_exceptions=None): + """Create key/value store based on this file-system + + Makes a MutableMapping interface to the FS at the given root path. + See ``fsspec.mapping.FSMap`` for further details. + """ + from .mapping import FSMap + + return FSMap( + root, + self, + check=check, + create=create, + missing_exceptions=missing_exceptions, + ) + + @classmethod + def clear_instance_cache(cls): + """ + Clear the cache of filesystem instances. + + Notes + ----- + Unless overridden by setting the ``cachable`` class attribute to False, + the filesystem class stores a reference to newly created instances. This + prevents Python's normal rules around garbage collection from working, + since the instances refcount will not drop to zero until + ``clear_instance_cache`` is called. + """ + cls._cache.clear() + + def created(self, path): + """Return the created timestamp of a file as a datetime.datetime""" + raise NotImplementedError + + def modified(self, path): + """Return the modified timestamp of a file as a datetime.datetime""" + raise NotImplementedError + + def tree( + self, + path: str = "/", + recursion_limit: int = 2, + max_display: int = 25, + display_size: bool = False, + prefix: str = "", + is_last: bool = True, + first: bool = True, + indent_size: int = 4, + ) -> str: + """ + Return a tree-like structure of the filesystem starting from the given path as a string. + + Parameters + ---------- + path: Root path to start traversal from + recursion_limit: Maximum depth of directory traversal + max_display: Maximum number of items to display per directory + display_size: Whether to display file sizes + prefix: Current line prefix for visual tree structure + is_last: Whether current item is last in its level + first: Whether this is the first call (displays root path) + indent_size: Number of spaces by indent + + Returns + ------- + str: A string representing the tree structure. + + Example + ------- + >>> from fsspec import filesystem + + >>> fs = filesystem('ftp', host='test.rebex.net', user='demo', password='password') + >>> tree = fs.tree(display_size=True, recursion_limit=3, indent_size=8, max_display=10) + >>> print(tree) + """ + + def format_bytes(n: int) -> str: + """Format bytes as text.""" + for prefix, k in ( + ("P", 2**50), + ("T", 2**40), + ("G", 2**30), + ("M", 2**20), + ("k", 2**10), + ): + if n >= 0.9 * k: + return f"{n / k:.2f} {prefix}b" + return f"{n}B" + + result = [] + + if first: + result.append(path) + + if recursion_limit: + indent = " " * indent_size + contents = self.ls(path, detail=True) + contents.sort( + key=lambda x: (x.get("type") != "directory", x.get("name", "")) + ) + + if max_display is not None and len(contents) > max_display: + displayed_contents = contents[:max_display] + remaining_count = len(contents) - max_display + else: + displayed_contents = contents + remaining_count = 0 + + for i, item in enumerate(displayed_contents): + is_last_item = (i == len(displayed_contents) - 1) and ( + remaining_count == 0 + ) + + branch = ( + "└" + ("─" * (indent_size - 2)) + if is_last_item + else "├" + ("─" * (indent_size - 2)) + ) + branch += " " + new_prefix = prefix + ( + indent if is_last_item else "│" + " " * (indent_size - 1) + ) + + name = os.path.basename(item.get("name", "")) + + if display_size and item.get("type") == "directory": + sub_contents = self.ls(item.get("name", ""), detail=True) + num_files = sum( + 1 for sub_item in sub_contents if sub_item.get("type") == "file" + ) + num_folders = sum( + 1 + for sub_item in sub_contents + if sub_item.get("type") == "directory" + ) + + if num_files == 0 and num_folders == 0: + size = " (empty folder)" + elif num_files == 0: + size = f" ({num_folders} subfolder{'s' if num_folders > 1 else ''})" + elif num_folders == 0: + size = f" ({num_files} file{'s' if num_files > 1 else ''})" + else: + size = f" ({num_files} file{'s' if num_files > 1 else ''}, {num_folders} subfolder{'s' if num_folders > 1 else ''})" + elif display_size and item.get("type") == "file": + size = f" ({format_bytes(item.get('size', 0))})" + else: + size = "" + + result.append(f"{prefix}{branch}{name}{size}") + + if item.get("type") == "directory" and recursion_limit > 0: + result.append( + self.tree( + path=item.get("name", ""), + recursion_limit=recursion_limit - 1, + max_display=max_display, + display_size=display_size, + prefix=new_prefix, + is_last=is_last_item, + first=False, + indent_size=indent_size, + ) + ) + + if remaining_count > 0: + more_message = f"{remaining_count} more item(s) not displayed." + result.append( + f"{prefix}{'└' + ('─' * (indent_size - 2))} {more_message}" + ) + + return "\n".join(_ for _ in result if _) + + # ------------------------------------------------------------------------ + # Aliases + + def read_bytes(self, path, start=None, end=None, **kwargs): + """Alias of `AbstractFileSystem.cat_file`.""" + return self.cat_file(path, start=start, end=end, **kwargs) + + def write_bytes(self, path, value, **kwargs): + """Alias of `AbstractFileSystem.pipe_file`.""" + self.pipe_file(path, value, **kwargs) + + def makedir(self, path, create_parents=True, **kwargs): + """Alias of `AbstractFileSystem.mkdir`.""" + return self.mkdir(path, create_parents=create_parents, **kwargs) + + def mkdirs(self, path, exist_ok=False): + """Alias of `AbstractFileSystem.makedirs`.""" + return self.makedirs(path, exist_ok=exist_ok) + + def listdir(self, path, detail=True, **kwargs): + """Alias of `AbstractFileSystem.ls`.""" + return self.ls(path, detail=detail, **kwargs) + + def cp(self, path1, path2, **kwargs): + """Alias of `AbstractFileSystem.copy`.""" + return self.copy(path1, path2, **kwargs) + + def move(self, path1, path2, **kwargs): + """Alias of `AbstractFileSystem.mv`.""" + return self.mv(path1, path2, **kwargs) + + def stat(self, path, **kwargs): + """Alias of `AbstractFileSystem.info`.""" + return self.info(path, **kwargs) + + def disk_usage(self, path, total=True, maxdepth=None, **kwargs): + """Alias of `AbstractFileSystem.du`.""" + return self.du(path, total=total, maxdepth=maxdepth, **kwargs) + + def rename(self, path1, path2, **kwargs): + """Alias of `AbstractFileSystem.mv`.""" + return self.mv(path1, path2, **kwargs) + + def delete(self, path, recursive=False, maxdepth=None): + """Alias of `AbstractFileSystem.rm`.""" + return self.rm(path, recursive=recursive, maxdepth=maxdepth) + + def upload(self, lpath, rpath, recursive=False, **kwargs): + """Alias of `AbstractFileSystem.put`.""" + return self.put(lpath, rpath, recursive=recursive, **kwargs) + + def download(self, rpath, lpath, recursive=False, **kwargs): + """Alias of `AbstractFileSystem.get`.""" + return self.get(rpath, lpath, recursive=recursive, **kwargs) + + def sign(self, path, expiration=100, **kwargs): + """Create a signed URL representing the given path + + Some implementations allow temporary URLs to be generated, as a + way of delegating credentials. + + Parameters + ---------- + path : str + The path on the filesystem + expiration : int + Number of seconds to enable the URL for (if supported) + + Returns + ------- + URL : str + The signed URL + + Raises + ------ + NotImplementedError : if method is not implemented for a filesystem + """ + raise NotImplementedError("Sign is not implemented for this filesystem") + + def _isfilestore(self): + # Originally inherited from pyarrow DaskFileSystem. Keeping this + # here for backwards compatibility as long as pyarrow uses its + # legacy fsspec-compatible filesystems and thus accepts fsspec + # filesystems as well + return False + + +class AbstractBufferedFile(io.IOBase): + """Convenient class to derive from to provide buffering + + In the case that the backend does not provide a pythonic file-like object + already, this class contains much of the logic to build one. The only + methods that need to be overridden are ``_upload_chunk``, + ``_initiate_upload`` and ``_fetch_range``. + """ + + DEFAULT_BLOCK_SIZE = 5 * 2**20 + _details = None + + def __init__( + self, + fs, + path, + mode="rb", + block_size="default", + autocommit=True, + cache_type="readahead", + cache_options=None, + size=None, + **kwargs, + ): + """ + Template for files with buffered reading and writing + + Parameters + ---------- + fs: instance of FileSystem + path: str + location in file-system + mode: str + Normal file modes. Currently only 'wb', 'ab' or 'rb'. Some file + systems may be read-only, and some may not support append. + block_size: int + Buffer size for reading or writing, 'default' for class default + autocommit: bool + Whether to write to final destination; may only impact what + happens when file is being closed. + cache_type: {"readahead", "none", "mmap", "bytes"}, default "readahead" + Caching policy in read mode. See the definitions in ``core``. + cache_options : dict + Additional options passed to the constructor for the cache specified + by `cache_type`. + size: int + If given and in read mode, suppressed having to look up the file size + kwargs: + Gets stored as self.kwargs + """ + from .core import caches + + self.path = path + self.fs = fs + self.mode = mode + self.blocksize = ( + self.DEFAULT_BLOCK_SIZE if block_size in ["default", None] else block_size + ) + self.loc = 0 + self.autocommit = autocommit + self.end = None + self.start = None + self.closed = False + + if cache_options is None: + cache_options = {} + + if "trim" in kwargs: + warnings.warn( + "Passing 'trim' to control the cache behavior has been deprecated. " + "Specify it within the 'cache_options' argument instead.", + FutureWarning, + ) + cache_options["trim"] = kwargs.pop("trim") + + self.kwargs = kwargs + + if mode not in {"ab", "rb", "wb", "xb"}: + raise NotImplementedError("File mode not supported") + if mode == "rb": + if size is not None: + self.size = size + else: + self.size = self.details["size"] + self.cache = caches[cache_type]( + self.blocksize, self._fetch_range, self.size, **cache_options + ) + else: + self.buffer = io.BytesIO() + self.offset = None + self.forced = False + self.location = None + + @property + def details(self): + if self._details is None: + self._details = self.fs.info(self.path) + return self._details + + @details.setter + def details(self, value): + self._details = value + self.size = value["size"] + + @property + def full_name(self): + return _unstrip_protocol(self.path, self.fs) + + @property + def closed(self): + # get around this attr being read-only in IOBase + # use getattr here, since this can be called during del + return getattr(self, "_closed", True) + + @closed.setter + def closed(self, c): + self._closed = c + + def __hash__(self): + if "w" in self.mode: + return id(self) + else: + return int(tokenize(self.details), 16) + + def __eq__(self, other): + """Files are equal if they have the same checksum, only in read mode""" + if self is other: + return True + return ( + isinstance(other, type(self)) + and self.mode == "rb" + and other.mode == "rb" + and hash(self) == hash(other) + ) + + def commit(self): + """Move from temp to final destination""" + + def discard(self): + """Throw away temporary file""" + + def info(self): + """File information about this path""" + if self.readable(): + return self.details + else: + raise ValueError("Info not available while writing") + + def tell(self): + """Current file location""" + return self.loc + + def seek(self, loc, whence=0): + """Set current file location + + Parameters + ---------- + loc: int + byte location + whence: {0, 1, 2} + from start of file, current location or end of file, resp. + """ + loc = int(loc) + if not self.mode == "rb": + raise OSError(ESPIPE, "Seek only available in read mode") + if whence == 0: + nloc = loc + elif whence == 1: + nloc = self.loc + loc + elif whence == 2: + nloc = self.size + loc + else: + raise ValueError(f"invalid whence ({whence}, should be 0, 1 or 2)") + if nloc < 0: + raise ValueError("Seek before start of file") + self.loc = nloc + return self.loc + + def write(self, data): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + data: bytes + Set of bytes to be written. + """ + if not self.writable(): + raise ValueError("File not in write mode") + if self.closed: + raise ValueError("I/O operation on closed file.") + if self.forced: + raise ValueError("This file has been force-flushed, can only close") + out = self.buffer.write(data) + self.loc += out + if self.buffer.tell() >= self.blocksize: + self.flush() + return out + + def flush(self, force=False): + """ + Write buffered data to backend store. + + Writes the current buffer, if it is larger than the block-size, or if + the file is being closed. + + Parameters + ---------- + force: bool + When closing, write the last block even if it is smaller than + blocks are allowed to be. Disallows further writing to this file. + """ + + if self.closed: + raise ValueError("Flush on closed file") + if force and self.forced: + raise ValueError("Force flush cannot be called more than once") + if force: + self.forced = True + + if self.readable(): + # no-op to flush on read-mode + return + + if not force and self.buffer.tell() < self.blocksize: + # Defer write on small block + return + + if self.offset is None: + # Initialize a multipart upload + self.offset = 0 + try: + self._initiate_upload() + except: + self.closed = True + raise + + if self._upload_chunk(final=force) is not False: + self.offset += self.buffer.seek(0, 2) + self.buffer = io.BytesIO() + + def _upload_chunk(self, final=False): + """Write one part of a multi-block file upload + + Parameters + ========== + final: bool + This is the last block, so should complete file, if + self.autocommit is True. + """ + # may not yet have been initialized, may need to call _initialize_upload + + def _initiate_upload(self): + """Create remote file/upload""" + pass + + def _fetch_range(self, start, end): + """Get the specified set of bytes from remote""" + return self.fs.cat_file(self.path, start=start, end=end) + + def read(self, length=-1): + """ + Return data from cache, or fetch pieces as necessary + + Parameters + ---------- + length: int (-1) + Number of bytes to read; if <0, all remaining bytes. + """ + length = -1 if length is None else int(length) + if self.mode != "rb": + raise ValueError("File not in read mode") + if length < 0: + length = self.size - self.loc + if self.closed: + raise ValueError("I/O operation on closed file.") + if length == 0: + # don't even bother calling fetch + return b"" + out = self.cache._fetch(self.loc, self.loc + length) + + logger.debug( + "%s read: %i - %i %s", + self, + self.loc, + self.loc + length, + self.cache._log_stats(), + ) + self.loc += len(out) + return out + + def readinto(self, b): + """mirrors builtin file's readinto method + + https://docs.python.org/3/library/io.html#io.RawIOBase.readinto + """ + out = memoryview(b).cast("B") + data = self.read(out.nbytes) + out[: len(data)] = data + return len(data) + + def readuntil(self, char=b"\n", blocks=None): + """Return data between current position and first occurrence of char + + char is included in the output, except if the end of the tile is + encountered first. + + Parameters + ---------- + char: bytes + Thing to find + blocks: None or int + How much to read in each go. Defaults to file blocksize - which may + mean a new read on every call. + """ + out = [] + while True: + start = self.tell() + part = self.read(blocks or self.blocksize) + if len(part) == 0: + break + found = part.find(char) + if found > -1: + out.append(part[: found + len(char)]) + self.seek(start + found + len(char)) + break + out.append(part) + return b"".join(out) + + def readline(self): + """Read until and including the first occurrence of newline character + + Note that, because of character encoding, this is not necessarily a + true line ending. + """ + return self.readuntil(b"\n") + + def __next__(self): + out = self.readline() + if out: + return out + raise StopIteration + + def __iter__(self): + return self + + def readlines(self): + """Return all data, split by the newline character, including the newline character""" + data = self.read() + lines = data.split(b"\n") + out = [l + b"\n" for l in lines[:-1]] + if data.endswith(b"\n"): + return out + else: + return out + [lines[-1]] + # return list(self) ??? + + def readinto1(self, b): + return self.readinto(b) + + def close(self): + """Close file + + Finalizes writes, discards cache + """ + if getattr(self, "_unclosable", False): + return + if self.closed: + return + try: + if self.mode == "rb": + self.cache = None + else: + if not getattr(self, "forced", True): + self.flush(force=True) + + if self.fs is not None: + self.fs.invalidate_cache(self.path) + self.fs.invalidate_cache(self.fs._parent(self.path)) + finally: + self.closed = True + + def readable(self): + """Whether opened for reading""" + return "r" in self.mode and not self.closed + + def seekable(self): + """Whether is seekable (only in read mode)""" + return self.readable() + + def writable(self): + """Whether opened for writing""" + return self.mode in {"wb", "ab", "xb"} and not self.closed + + def __reduce__(self): + if self.mode != "rb": + raise RuntimeError("Pickling a writeable file is not supported") + + return reopen, ( + self.fs, + self.path, + self.mode, + self.blocksize, + self.loc, + self.size, + self.autocommit, + self.cache.name if self.cache else "none", + self.kwargs, + ) + + def __del__(self): + if not self.closed: + self.close() + + def __str__(self): + return f"<File-like object {type(self.fs).__name__}, {self.path}>" + + __repr__ = __str__ + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +def reopen(fs, path, mode, blocksize, loc, size, autocommit, cache_type, kwargs): + file = fs.open( + path, + mode=mode, + block_size=blocksize, + autocommit=autocommit, + cache_type=cache_type, + size=size, + **kwargs, + ) + if loc > 0: + file.seek(loc) + return file diff --git a/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/__init__.py b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8ed2ad802ecaf021106c25c03112f29e75c7b2f8 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/__init__.py @@ -0,0 +1,289 @@ +import os +from hashlib import md5 + +import pytest + +from fsspec.implementations.local import LocalFileSystem +from fsspec.tests.abstract.copy import AbstractCopyTests # noqa: F401 +from fsspec.tests.abstract.get import AbstractGetTests # noqa: F401 +from fsspec.tests.abstract.open import AbstractOpenTests # noqa: F401 +from fsspec.tests.abstract.pipe import AbstractPipeTests # noqa: F401 +from fsspec.tests.abstract.put import AbstractPutTests # noqa: F401 + + +class BaseAbstractFixtures: + """ + Abstract base class containing fixtures that are used by but never need to + be overridden in derived filesystem-specific classes to run the abstract + tests on such filesystems. + """ + + @pytest.fixture + def fs_bulk_operations_scenario_0(self, fs, fs_join, fs_path): + """ + Scenario on remote filesystem that is used for many cp/get/put tests. + + Cleans up at the end of each test it which it is used. + """ + source = self._bulk_operations_scenario_0(fs, fs_join, fs_path) + yield source + fs.rm(source, recursive=True) + + @pytest.fixture + def fs_glob_edge_cases_files(self, fs, fs_join, fs_path): + """ + Scenario on remote filesystem that is used for glob edge cases cp/get/put tests. + + Cleans up at the end of each test it which it is used. + """ + source = self._glob_edge_cases_files(fs, fs_join, fs_path) + yield source + fs.rm(source, recursive=True) + + @pytest.fixture + def fs_dir_and_file_with_same_name_prefix(self, fs, fs_join, fs_path): + """ + Scenario on remote filesystem that is used to check cp/get/put on directory + and file with the same name prefixes. + + Cleans up at the end of each test it which it is used. + """ + source = self._dir_and_file_with_same_name_prefix(fs, fs_join, fs_path) + yield source + fs.rm(source, recursive=True) + + @pytest.fixture + def fs_10_files_with_hashed_names(self, fs, fs_join, fs_path): + """ + Scenario on remote filesystem that is used to check cp/get/put files order + when source and destination are lists. + + Cleans up at the end of each test it which it is used. + """ + source = self._10_files_with_hashed_names(fs, fs_join, fs_path) + yield source + fs.rm(source, recursive=True) + + @pytest.fixture + def fs_target(self, fs, fs_join, fs_path): + """ + Return name of remote directory that does not yet exist to copy into. + + Cleans up at the end of each test it which it is used. + """ + target = fs_join(fs_path, "target") + yield target + if fs.exists(target): + fs.rm(target, recursive=True) + + @pytest.fixture + def local_bulk_operations_scenario_0(self, local_fs, local_join, local_path): + """ + Scenario on local filesystem that is used for many cp/get/put tests. + + Cleans up at the end of each test it which it is used. + """ + source = self._bulk_operations_scenario_0(local_fs, local_join, local_path) + yield source + local_fs.rm(source, recursive=True) + + @pytest.fixture + def local_glob_edge_cases_files(self, local_fs, local_join, local_path): + """ + Scenario on local filesystem that is used for glob edge cases cp/get/put tests. + + Cleans up at the end of each test it which it is used. + """ + source = self._glob_edge_cases_files(local_fs, local_join, local_path) + yield source + local_fs.rm(source, recursive=True) + + @pytest.fixture + def local_dir_and_file_with_same_name_prefix( + self, local_fs, local_join, local_path + ): + """ + Scenario on local filesystem that is used to check cp/get/put on directory + and file with the same name prefixes. + + Cleans up at the end of each test it which it is used. + """ + source = self._dir_and_file_with_same_name_prefix( + local_fs, local_join, local_path + ) + yield source + local_fs.rm(source, recursive=True) + + @pytest.fixture + def local_10_files_with_hashed_names(self, local_fs, local_join, local_path): + """ + Scenario on local filesystem that is used to check cp/get/put files order + when source and destination are lists. + + Cleans up at the end of each test it which it is used. + """ + source = self._10_files_with_hashed_names(local_fs, local_join, local_path) + yield source + local_fs.rm(source, recursive=True) + + @pytest.fixture + def local_target(self, local_fs, local_join, local_path): + """ + Return name of local directory that does not yet exist to copy into. + + Cleans up at the end of each test it which it is used. + """ + target = local_join(local_path, "target") + yield target + if local_fs.exists(target): + local_fs.rm(target, recursive=True) + + def _glob_edge_cases_files(self, some_fs, some_join, some_path): + """ + Scenario that is used for glob edge cases cp/get/put tests. + Creates the following directory and file structure: + + 📁 source + ├── 📄 file1 + ├── 📄 file2 + ├── 📁 subdir0 + │ ├── 📄 subfile1 + │ ├── 📄 subfile2 + │ └── 📁 nesteddir + │ └── 📄 nestedfile + └── 📁 subdir1 + ├── 📄 subfile1 + ├── 📄 subfile2 + └── 📁 nesteddir + └── 📄 nestedfile + """ + source = some_join(some_path, "source") + some_fs.touch(some_join(source, "file1")) + some_fs.touch(some_join(source, "file2")) + + for subdir_idx in range(2): + subdir = some_join(source, f"subdir{subdir_idx}") + nesteddir = some_join(subdir, "nesteddir") + some_fs.makedirs(nesteddir) + some_fs.touch(some_join(subdir, "subfile1")) + some_fs.touch(some_join(subdir, "subfile2")) + some_fs.touch(some_join(nesteddir, "nestedfile")) + + return source + + def _bulk_operations_scenario_0(self, some_fs, some_join, some_path): + """ + Scenario that is used for many cp/get/put tests. Creates the following + directory and file structure: + + 📁 source + ├── 📄 file1 + ├── 📄 file2 + └── 📁 subdir + ├── 📄 subfile1 + ├── 📄 subfile2 + └── 📁 nesteddir + └── 📄 nestedfile + """ + source = some_join(some_path, "source") + subdir = some_join(source, "subdir") + nesteddir = some_join(subdir, "nesteddir") + some_fs.makedirs(nesteddir) + some_fs.touch(some_join(source, "file1")) + some_fs.touch(some_join(source, "file2")) + some_fs.touch(some_join(subdir, "subfile1")) + some_fs.touch(some_join(subdir, "subfile2")) + some_fs.touch(some_join(nesteddir, "nestedfile")) + return source + + def _dir_and_file_with_same_name_prefix(self, some_fs, some_join, some_path): + """ + Scenario that is used to check cp/get/put on directory and file with + the same name prefixes. Creates the following directory and file structure: + + 📁 source + ├── 📄 subdir.txt + └── 📁 subdir + └── 📄 subfile.txt + """ + source = some_join(some_path, "source") + subdir = some_join(source, "subdir") + file = some_join(source, "subdir.txt") + subfile = some_join(subdir, "subfile.txt") + some_fs.makedirs(subdir) + some_fs.touch(file) + some_fs.touch(subfile) + return source + + def _10_files_with_hashed_names(self, some_fs, some_join, some_path): + """ + Scenario that is used to check cp/get/put files order when source and + destination are lists. Creates the following directory and file structure: + + 📁 source + └── 📄 {hashed([0-9])}.txt + """ + source = some_join(some_path, "source") + for i in range(10): + hashed_i = md5(str(i).encode("utf-8")).hexdigest() + path = some_join(source, f"{hashed_i}.txt") + some_fs.pipe(path=path, value=f"{i}".encode()) + return source + + +class AbstractFixtures(BaseAbstractFixtures): + """ + Abstract base class containing fixtures that may be overridden in derived + filesystem-specific classes to run the abstract tests on such filesystems. + + For any particular filesystem some of these fixtures must be overridden, + such as ``fs`` and ``fs_path``, and others may be overridden if the + default functions here are not appropriate, such as ``fs_join``. + """ + + @pytest.fixture + def fs(self): + raise NotImplementedError("This function must be overridden in derived classes") + + @pytest.fixture + def fs_join(self): + """ + Return a function that joins its arguments together into a path. + + Most fsspec implementations join paths in a platform-dependent way, + but some will override this to always use a forward slash. + """ + return os.path.join + + @pytest.fixture + def fs_path(self): + raise NotImplementedError("This function must be overridden in derived classes") + + @pytest.fixture(scope="class") + def local_fs(self): + # Maybe need an option for auto_mkdir=False? This is only relevant + # for certain implementations. + return LocalFileSystem(auto_mkdir=True) + + @pytest.fixture + def local_join(self): + """ + Return a function that joins its arguments together into a path, on + the local filesystem. + """ + return os.path.join + + @pytest.fixture + def local_path(self, tmpdir): + return tmpdir + + @pytest.fixture + def supports_empty_directories(self): + """ + Return whether this implementation supports empty directories. + """ + return True + + @pytest.fixture + def fs_sanitize_path(self): + return lambda x: x diff --git a/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/common.py b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/common.py new file mode 100644 index 0000000000000000000000000000000000000000..22e7c4140404ab2a8928689721419cf05c2760b9 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/common.py @@ -0,0 +1,175 @@ +GLOB_EDGE_CASES_TESTS = { + "argnames": ("path", "recursive", "maxdepth", "expected"), + "argvalues": [ + ("fil?1", False, None, ["file1"]), + ("fil?1", True, None, ["file1"]), + ("file[1-2]", False, None, ["file1", "file2"]), + ("file[1-2]", True, None, ["file1", "file2"]), + ("*", False, None, ["file1", "file2"]), + ( + "*", + True, + None, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir0/nesteddir/nestedfile", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ("*", True, 1, ["file1", "file2"]), + ( + "*", + True, + 2, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir1/subfile1", + "subdir1/subfile2", + ], + ), + ("*1", False, None, ["file1"]), + ( + "*1", + True, + None, + [ + "file1", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ("*1", True, 2, ["file1", "subdir1/subfile1", "subdir1/subfile2"]), + ( + "**", + False, + None, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir0/nesteddir/nestedfile", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ( + "**", + True, + None, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir0/nesteddir/nestedfile", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ("**", True, 1, ["file1", "file2"]), + ( + "**", + True, + 2, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir0/nesteddir/nestedfile", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ( + "**", + False, + 2, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir1/subfile1", + "subdir1/subfile2", + ], + ), + ("**/*1", False, None, ["file1", "subdir0/subfile1", "subdir1/subfile1"]), + ( + "**/*1", + True, + None, + [ + "file1", + "subdir0/subfile1", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ("**/*1", True, 1, ["file1"]), + ( + "**/*1", + True, + 2, + ["file1", "subdir0/subfile1", "subdir1/subfile1", "subdir1/subfile2"], + ), + ("**/*1", False, 2, ["file1", "subdir0/subfile1", "subdir1/subfile1"]), + ("**/subdir0", False, None, []), + ("**/subdir0", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]), + ("**/subdir0/nested*", False, 2, []), + ("**/subdir0/nested*", True, 2, ["nestedfile"]), + ("subdir[1-2]", False, None, []), + ("subdir[1-2]", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]), + ("subdir[1-2]", True, 2, ["subfile1", "subfile2"]), + ("subdir[0-1]", False, None, []), + ( + "subdir[0-1]", + True, + None, + [ + "subdir0/subfile1", + "subdir0/subfile2", + "subdir0/nesteddir/nestedfile", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ( + "subdir[0-1]/*fil[e]*", + False, + None, + [ + "subdir0/subfile1", + "subdir0/subfile2", + "subdir1/subfile1", + "subdir1/subfile2", + ], + ), + ( + "subdir[0-1]/*fil[e]*", + True, + None, + [ + "subdir0/subfile1", + "subdir0/subfile2", + "subdir1/subfile1", + "subdir1/subfile2", + ], + ), + ], +} diff --git a/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/copy.py b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/copy.py new file mode 100644 index 0000000000000000000000000000000000000000..e39e57e5f7d52bfda8ab5e2398b04cc2303630a0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/copy.py @@ -0,0 +1,557 @@ +from hashlib import md5 +from itertools import product + +import pytest + +from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS + + +class AbstractCopyTests: + def test_copy_file_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 1a + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + fs.touch(fs_join(target, "dummy")) + assert fs.isdir(target) + + target_file2 = fs_join(target, "file2") + target_subfile1 = fs_join(target, "subfile1") + + # Copy from source directory + fs.cp(fs_join(source, "file2"), target) + assert fs.isfile(target_file2) + + # Copy from sub directory + fs.cp(fs_join(source, "subdir", "subfile1"), target) + assert fs.isfile(target_subfile1) + + # Remove copied files + fs.rm([target_file2, target_subfile1]) + assert not fs.exists(target_file2) + assert not fs.exists(target_subfile1) + + # Repeat with trailing slash on target + fs.cp(fs_join(source, "file2"), target + "/") + assert fs.isdir(target) + assert fs.isfile(target_file2) + + fs.cp(fs_join(source, "subdir", "subfile1"), target + "/") + assert fs.isfile(target_subfile1) + + def test_copy_file_to_new_directory( + self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target + ): + # Copy scenario 1b + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + fs.cp( + fs_join(source, "subdir", "subfile1"), fs_join(target, "newdir/") + ) # Note trailing slash + assert fs.isdir(target) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + + def test_copy_file_to_file_in_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 1c + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + fs.touch(fs_join(target, "dummy")) + assert fs.isdir(target) + + fs.cp(fs_join(source, "subdir", "subfile1"), fs_join(target, "newfile")) + assert fs.isfile(fs_join(target, "newfile")) + + def test_copy_file_to_file_in_new_directory( + self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target + ): + # Copy scenario 1d + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + fs.cp( + fs_join(source, "subdir", "subfile1"), fs_join(target, "newdir", "newfile") + ) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "newfile")) + + def test_copy_directory_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 1e + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = target + "/" if target_slash else target + + # Without recursive does nothing + fs.cp(s, t) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # With recursive + fs.cp(s, t, recursive=True) + if source_slash: + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert fs.isdir(fs_join(target, "nesteddir")) + assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + fs_join(target, "nesteddir"), + ], + recursive=True, + ) + else: + assert fs.isdir(fs_join(target, "subdir")) + assert fs.isfile(fs_join(target, "subdir", "subfile1")) + assert fs.isfile(fs_join(target, "subdir", "subfile2")) + assert fs.isdir(fs_join(target, "subdir", "nesteddir")) + assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile")) + + fs.rm(fs_join(target, "subdir"), recursive=True) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # Limit recursive by maxdepth + fs.cp(s, t, recursive=True, maxdepth=1) + if source_slash: + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.exists(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + else: + assert fs.isdir(fs_join(target, "subdir")) + assert fs.isfile(fs_join(target, "subdir", "subfile1")) + assert fs.isfile(fs_join(target, "subdir", "subfile2")) + assert not fs.exists(fs_join(target, "subdir", "nesteddir")) + + fs.rm(fs_join(target, "subdir"), recursive=True) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_copy_directory_to_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 1f + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = fs_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive does nothing + fs.cp(s, t) + if supports_empty_directories: + assert fs.ls(target) == [] + else: + with pytest.raises(FileNotFoundError): + fs.ls(target) + + # With recursive + fs.cp(s, t, recursive=True) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert fs.isdir(fs_join(target, "newdir", "nesteddir")) + assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # Limit recursive by maxdepth + fs.cp(s, t, recursive=True, maxdepth=1) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + def test_copy_glob_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 1g + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + # Without recursive + fs.cp(fs_join(source, "subdir", "*"), t) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.isdir(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.cp(fs_join(source, "subdir", glob), t, recursive=recursive) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert fs.isdir(fs_join(target, "nesteddir")) + assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + fs_join(target, "nesteddir"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # Limit recursive by maxdepth + fs.cp( + fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 + ) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.exists(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_copy_glob_to_new_directory( + self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target + ): + # Copy scenario 1h + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + for target_slash in [False, True]: + t = fs_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive + fs.cp(fs_join(source, "subdir", "*"), t) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.cp(fs_join(source, "subdir", glob), t, recursive=recursive) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert fs.isdir(fs_join(target, "newdir", "nesteddir")) + assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # Limit recursive by maxdepth + fs.cp( + fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 + ) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + @pytest.mark.parametrize( + GLOB_EDGE_CASES_TESTS["argnames"], + GLOB_EDGE_CASES_TESTS["argvalues"], + ) + def test_copy_glob_edge_cases( + self, + path, + recursive, + maxdepth, + expected, + fs, + fs_join, + fs_glob_edge_cases_files, + fs_target, + fs_sanitize_path, + ): + # Copy scenario 1g + source = fs_glob_edge_cases_files + + target = fs_target + + for new_dir, target_slash in product([True, False], [True, False]): + fs.mkdir(target) + + t = fs_join(target, "newdir") if new_dir else target + t = t + "/" if target_slash else t + + fs.copy(fs_join(source, path), t, recursive=recursive, maxdepth=maxdepth) + + output = fs.find(target) + if new_dir: + prefixed_expected = [ + fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected + ] + else: + prefixed_expected = [ + fs_sanitize_path(fs_join(target, p)) for p in expected + ] + assert sorted(output) == sorted(prefixed_expected) + + try: + fs.rm(target, recursive=True) + except FileNotFoundError: + pass + + def test_copy_list_of_files_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 2a + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + source_files = [ + fs_join(source, "file1"), + fs_join(source, "file2"), + fs_join(source, "subdir", "subfile1"), + ] + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + fs.cp(source_files, t) + assert fs.isfile(fs_join(target, "file1")) + assert fs.isfile(fs_join(target, "file2")) + assert fs.isfile(fs_join(target, "subfile1")) + + fs.rm( + [ + fs_join(target, "file1"), + fs_join(target, "file2"), + fs_join(target, "subfile1"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_copy_list_of_files_to_new_directory( + self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target + ): + # Copy scenario 2b + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + source_files = [ + fs_join(source, "file1"), + fs_join(source, "file2"), + fs_join(source, "subdir", "subfile1"), + ] + + fs.cp(source_files, fs_join(target, "newdir") + "/") # Note trailing slash + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "file1")) + assert fs.isfile(fs_join(target, "newdir", "file2")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + + def test_copy_two_files_new_directory( + self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target + ): + # This is a duplicate of test_copy_list_of_files_to_new_directory and + # can eventually be removed. + source = fs_bulk_operations_scenario_0 + + target = fs_target + assert not fs.exists(target) + fs.cp([fs_join(source, "file1"), fs_join(source, "file2")], target) + + assert fs.isdir(target) + assert fs.isfile(fs_join(target, "file1")) + assert fs.isfile(fs_join(target, "file2")) + + def test_copy_directory_without_files_with_same_name_prefix( + self, + fs, + fs_join, + fs_target, + fs_dir_and_file_with_same_name_prefix, + supports_empty_directories, + ): + # Create the test dirs + source = fs_dir_and_file_with_same_name_prefix + target = fs_target + + # Test without glob + fs.cp(fs_join(source, "subdir"), target, recursive=True) + + assert fs.isfile(fs_join(target, "subfile.txt")) + assert not fs.isfile(fs_join(target, "subdir.txt")) + + fs.rm([fs_join(target, "subfile.txt")]) + if supports_empty_directories: + assert fs.ls(target) == [] + else: + assert not fs.exists(target) + + # Test with glob + fs.cp(fs_join(source, "subdir*"), target, recursive=True) + + assert fs.isdir(fs_join(target, "subdir")) + assert fs.isfile(fs_join(target, "subdir", "subfile.txt")) + assert fs.isfile(fs_join(target, "subdir.txt")) + + def test_copy_with_source_and_destination_as_list( + self, fs, fs_target, fs_join, fs_10_files_with_hashed_names + ): + # Create the test dir + source = fs_10_files_with_hashed_names + target = fs_target + + # Create list of files for source and destination + source_files = [] + destination_files = [] + for i in range(10): + hashed_i = md5(str(i).encode("utf-8")).hexdigest() + source_files.append(fs_join(source, f"{hashed_i}.txt")) + destination_files.append(fs_join(target, f"{hashed_i}.txt")) + + # Copy and assert order was kept + fs.copy(path1=source_files, path2=destination_files) + + for i in range(10): + file_content = fs.cat(destination_files[i]).decode("utf-8") + assert file_content == str(i) diff --git a/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/get.py b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/get.py new file mode 100644 index 0000000000000000000000000000000000000000..851ab81ee581e74cac41c64c83ef0af75826d6b0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/get.py @@ -0,0 +1,587 @@ +from hashlib import md5 +from itertools import product + +import pytest + +from fsspec.implementations.local import make_path_posix +from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS + + +class AbstractGetTests: + def test_get_file_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1a + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + assert local_fs.isdir(target) + + target_file2 = local_join(target, "file2") + target_subfile1 = local_join(target, "subfile1") + + # Copy from source directory + fs.get(fs_join(source, "file2"), target) + assert local_fs.isfile(target_file2) + + # Copy from sub directory + fs.get(fs_join(source, "subdir", "subfile1"), target) + assert local_fs.isfile(target_subfile1) + + # Remove copied files + local_fs.rm([target_file2, target_subfile1]) + assert not local_fs.exists(target_file2) + assert not local_fs.exists(target_subfile1) + + # Repeat with trailing slash on target + fs.get(fs_join(source, "file2"), target + "/") + assert local_fs.isdir(target) + assert local_fs.isfile(target_file2) + + fs.get(fs_join(source, "subdir", "subfile1"), target + "/") + assert local_fs.isfile(target_subfile1) + + def test_get_file_to_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1b + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + fs.get( + fs_join(source, "subdir", "subfile1"), local_join(target, "newdir/") + ) # Note trailing slash + + assert local_fs.isdir(target) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + + def test_get_file_to_file_in_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1c + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + fs.get(fs_join(source, "subdir", "subfile1"), local_join(target, "newfile")) + assert local_fs.isfile(local_join(target, "newfile")) + + def test_get_file_to_file_in_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1d + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + fs.get( + fs_join(source, "subdir", "subfile1"), + local_join(target, "newdir", "newfile"), + ) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "newfile")) + + def test_get_directory_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1e + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + assert local_fs.isdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = target + "/" if target_slash else target + + # Without recursive does nothing + fs.get(s, t) + assert local_fs.ls(target) == [] + + # With recursive + fs.get(s, t, recursive=True) + if source_slash: + assert local_fs.isfile(local_join(target, "subfile1")) + assert local_fs.isfile(local_join(target, "subfile2")) + assert local_fs.isdir(local_join(target, "nesteddir")) + assert local_fs.isfile(local_join(target, "nesteddir", "nestedfile")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm( + [ + local_join(target, "subfile1"), + local_join(target, "subfile2"), + local_join(target, "nesteddir"), + ], + recursive=True, + ) + else: + assert local_fs.isdir(local_join(target, "subdir")) + assert local_fs.isfile(local_join(target, "subdir", "subfile1")) + assert local_fs.isfile(local_join(target, "subdir", "subfile2")) + assert local_fs.isdir(local_join(target, "subdir", "nesteddir")) + assert local_fs.isfile( + local_join(target, "subdir", "nesteddir", "nestedfile") + ) + + local_fs.rm(local_join(target, "subdir"), recursive=True) + assert local_fs.ls(target) == [] + + # Limit recursive by maxdepth + fs.get(s, t, recursive=True, maxdepth=1) + if source_slash: + assert local_fs.isfile(local_join(target, "subfile1")) + assert local_fs.isfile(local_join(target, "subfile2")) + assert not local_fs.exists(local_join(target, "nesteddir")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm( + [ + local_join(target, "subfile1"), + local_join(target, "subfile2"), + ], + recursive=True, + ) + else: + assert local_fs.isdir(local_join(target, "subdir")) + assert local_fs.isfile(local_join(target, "subdir", "subfile1")) + assert local_fs.isfile(local_join(target, "subdir", "subfile2")) + assert not local_fs.exists(local_join(target, "subdir", "nesteddir")) + + local_fs.rm(local_join(target, "subdir"), recursive=True) + assert local_fs.ls(target) == [] + + def test_get_directory_to_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1f + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = local_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive does nothing + fs.get(s, t) + assert local_fs.ls(target) == [] + + # With recursive + fs.get(s, t, recursive=True) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + assert local_fs.isfile(local_join(target, "newdir", "subfile2")) + assert local_fs.isdir(local_join(target, "newdir", "nesteddir")) + assert local_fs.isfile( + local_join(target, "newdir", "nesteddir", "nestedfile") + ) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm(local_join(target, "newdir"), recursive=True) + assert local_fs.ls(target) == [] + + # Limit recursive by maxdepth + fs.get(s, t, recursive=True, maxdepth=1) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + assert local_fs.isfile(local_join(target, "newdir", "subfile2")) + assert not local_fs.exists(local_join(target, "newdir", "nesteddir")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm(local_join(target, "newdir"), recursive=True) + assert not local_fs.exists(local_join(target, "newdir")) + + def test_get_glob_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1g + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + # Without recursive + fs.get(fs_join(source, "subdir", "*"), t) + assert local_fs.isfile(local_join(target, "subfile1")) + assert local_fs.isfile(local_join(target, "subfile2")) + assert not local_fs.isdir(local_join(target, "nesteddir")) + assert not local_fs.exists(local_join(target, "nesteddir", "nestedfile")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm( + [ + local_join(target, "subfile1"), + local_join(target, "subfile2"), + ], + recursive=True, + ) + assert local_fs.ls(target) == [] + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.get(fs_join(source, "subdir", glob), t, recursive=recursive) + assert local_fs.isfile(local_join(target, "subfile1")) + assert local_fs.isfile(local_join(target, "subfile2")) + assert local_fs.isdir(local_join(target, "nesteddir")) + assert local_fs.isfile(local_join(target, "nesteddir", "nestedfile")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm( + [ + local_join(target, "subfile1"), + local_join(target, "subfile2"), + local_join(target, "nesteddir"), + ], + recursive=True, + ) + assert local_fs.ls(target) == [] + + # Limit recursive by maxdepth + fs.get( + fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 + ) + assert local_fs.isfile(local_join(target, "subfile1")) + assert local_fs.isfile(local_join(target, "subfile2")) + assert not local_fs.exists(local_join(target, "nesteddir")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm( + [ + local_join(target, "subfile1"), + local_join(target, "subfile2"), + ], + recursive=True, + ) + assert local_fs.ls(target) == [] + + def test_get_glob_to_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1h + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + for target_slash in [False, True]: + t = fs_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive + fs.get(fs_join(source, "subdir", "*"), t) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + assert local_fs.isfile(local_join(target, "newdir", "subfile2")) + assert not local_fs.exists(local_join(target, "newdir", "nesteddir")) + assert not local_fs.exists( + local_join(target, "newdir", "nesteddir", "nestedfile") + ) + assert not local_fs.exists(local_join(target, "subdir")) + assert not local_fs.exists(local_join(target, "newdir", "subdir")) + + local_fs.rm(local_join(target, "newdir"), recursive=True) + assert local_fs.ls(target) == [] + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.get(fs_join(source, "subdir", glob), t, recursive=recursive) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + assert local_fs.isfile(local_join(target, "newdir", "subfile2")) + assert local_fs.isdir(local_join(target, "newdir", "nesteddir")) + assert local_fs.isfile( + local_join(target, "newdir", "nesteddir", "nestedfile") + ) + assert not local_fs.exists(local_join(target, "subdir")) + assert not local_fs.exists(local_join(target, "newdir", "subdir")) + + local_fs.rm(local_join(target, "newdir"), recursive=True) + assert not local_fs.exists(local_join(target, "newdir")) + + # Limit recursive by maxdepth + fs.get( + fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 + ) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + assert local_fs.isfile(local_join(target, "newdir", "subfile2")) + assert not local_fs.exists(local_join(target, "newdir", "nesteddir")) + assert not local_fs.exists(local_join(target, "subdir")) + assert not local_fs.exists(local_join(target, "newdir", "subdir")) + + local_fs.rm(local_fs.ls(target, detail=False), recursive=True) + assert not local_fs.exists(local_join(target, "newdir")) + + @pytest.mark.parametrize( + GLOB_EDGE_CASES_TESTS["argnames"], + GLOB_EDGE_CASES_TESTS["argvalues"], + ) + def test_get_glob_edge_cases( + self, + path, + recursive, + maxdepth, + expected, + fs, + fs_join, + fs_glob_edge_cases_files, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1g + source = fs_glob_edge_cases_files + + target = local_target + + for new_dir, target_slash in product([True, False], [True, False]): + local_fs.mkdir(target) + + t = local_join(target, "newdir") if new_dir else target + t = t + "/" if target_slash else t + + fs.get(fs_join(source, path), t, recursive=recursive, maxdepth=maxdepth) + + output = local_fs.find(target) + if new_dir: + prefixed_expected = [ + make_path_posix(local_join(target, "newdir", p)) for p in expected + ] + else: + prefixed_expected = [ + make_path_posix(local_join(target, p)) for p in expected + ] + assert sorted(output) == sorted(prefixed_expected) + + try: + local_fs.rm(target, recursive=True) + except FileNotFoundError: + pass + + def test_get_list_of_files_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 2a + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + source_files = [ + fs_join(source, "file1"), + fs_join(source, "file2"), + fs_join(source, "subdir", "subfile1"), + ] + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + fs.get(source_files, t) + assert local_fs.isfile(local_join(target, "file1")) + assert local_fs.isfile(local_join(target, "file2")) + assert local_fs.isfile(local_join(target, "subfile1")) + + local_fs.rm( + [ + local_join(target, "file1"), + local_join(target, "file2"), + local_join(target, "subfile1"), + ], + recursive=True, + ) + assert local_fs.ls(target) == [] + + def test_get_list_of_files_to_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 2b + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + source_files = [ + fs_join(source, "file1"), + fs_join(source, "file2"), + fs_join(source, "subdir", "subfile1"), + ] + + fs.get(source_files, local_join(target, "newdir") + "/") # Note trailing slash + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "file1")) + assert local_fs.isfile(local_join(target, "newdir", "file2")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + + def test_get_directory_recursive( + self, fs, fs_join, fs_path, local_fs, local_join, local_target + ): + # https://github.com/fsspec/filesystem_spec/issues/1062 + # Recursive cp/get/put of source directory into non-existent target directory. + src = fs_join(fs_path, "src") + src_file = fs_join(src, "file") + fs.mkdir(src) + fs.touch(src_file) + + target = local_target + + # get without slash + assert not local_fs.exists(target) + for loop in range(2): + fs.get(src, target, recursive=True) + assert local_fs.isdir(target) + + if loop == 0: + assert local_fs.isfile(local_join(target, "file")) + assert not local_fs.exists(local_join(target, "src")) + else: + assert local_fs.isfile(local_join(target, "file")) + assert local_fs.isdir(local_join(target, "src")) + assert local_fs.isfile(local_join(target, "src", "file")) + + local_fs.rm(target, recursive=True) + + # get with slash + assert not local_fs.exists(target) + for loop in range(2): + fs.get(src + "/", target, recursive=True) + assert local_fs.isdir(target) + assert local_fs.isfile(local_join(target, "file")) + assert not local_fs.exists(local_join(target, "src")) + + def test_get_directory_without_files_with_same_name_prefix( + self, + fs, + fs_join, + local_fs, + local_join, + local_target, + fs_dir_and_file_with_same_name_prefix, + ): + # Create the test dirs + source = fs_dir_and_file_with_same_name_prefix + target = local_target + + # Test without glob + fs.get(fs_join(source, "subdir"), target, recursive=True) + + assert local_fs.isfile(local_join(target, "subfile.txt")) + assert not local_fs.isfile(local_join(target, "subdir.txt")) + + local_fs.rm([local_join(target, "subfile.txt")]) + assert local_fs.ls(target) == [] + + # Test with glob + fs.get(fs_join(source, "subdir*"), target, recursive=True) + + assert local_fs.isdir(local_join(target, "subdir")) + assert local_fs.isfile(local_join(target, "subdir", "subfile.txt")) + assert local_fs.isfile(local_join(target, "subdir.txt")) + + def test_get_with_source_and_destination_as_list( + self, + fs, + fs_join, + local_fs, + local_join, + local_target, + fs_10_files_with_hashed_names, + ): + # Create the test dir + source = fs_10_files_with_hashed_names + target = local_target + + # Create list of files for source and destination + source_files = [] + destination_files = [] + for i in range(10): + hashed_i = md5(str(i).encode("utf-8")).hexdigest() + source_files.append(fs_join(source, f"{hashed_i}.txt")) + destination_files.append( + make_path_posix(local_join(target, f"{hashed_i}.txt")) + ) + + # Copy and assert order was kept + fs.get(rpath=source_files, lpath=destination_files) + + for i in range(10): + file_content = local_fs.cat(destination_files[i]).decode("utf-8") + assert file_content == str(i) diff --git a/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/mv.py b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/mv.py new file mode 100644 index 0000000000000000000000000000000000000000..39f6caa3de815e024fa84de2acecc986c823ed29 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/mv.py @@ -0,0 +1,57 @@ +import os + +import pytest + +import fsspec + + +def test_move_raises_error_with_tmpdir(tmpdir): + # Create a file in the temporary directory + source = tmpdir.join("source_file.txt") + source.write("content") + + # Define a destination that simulates a protected or invalid path + destination = tmpdir.join("non_existent_directory/destination_file.txt") + + # Instantiate the filesystem (assuming the local file system interface) + fs = fsspec.filesystem("file") + + # Use the actual file paths as string + with pytest.raises(FileNotFoundError): + fs.mv(str(source), str(destination)) + + +@pytest.mark.parametrize("recursive", (True, False)) +def test_move_raises_error_with_tmpdir_permission(recursive, tmpdir): + # Create a file in the temporary directory + source = tmpdir.join("source_file.txt") + source.write("content") + + # Create a protected directory (non-writable) + protected_dir = tmpdir.mkdir("protected_directory") + protected_path = str(protected_dir) + + # Set the directory to read-only + if os.name == "nt": + os.system(f'icacls "{protected_path}" /deny Everyone:(W)') + else: + os.chmod(protected_path, 0o555) # Sets the directory to read-only + + # Define a destination inside the protected directory + destination = protected_dir.join("destination_file.txt") + + # Instantiate the filesystem (assuming the local file system interface) + fs = fsspec.filesystem("file") + + # Try to move the file to the read-only directory, expecting a permission error + with pytest.raises(PermissionError): + fs.mv(str(source), str(destination), recursive=recursive) + + # Assert the file was not created in the destination + assert not os.path.exists(destination) + + # Cleanup: Restore permissions so the directory can be cleaned up + if os.name == "nt": + os.system(f'icacls "{protected_path}" /remove:d Everyone') + else: + os.chmod(protected_path, 0o755) # Restore write permission for cleanup diff --git a/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/open.py b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/open.py new file mode 100644 index 0000000000000000000000000000000000000000..bb75ea852276fb8d834345883813b8e27a0ae24c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/open.py @@ -0,0 +1,11 @@ +import pytest + + +class AbstractOpenTests: + def test_open_exclusive(self, fs, fs_target): + with fs.open(fs_target, "wb") as f: + f.write(b"data") + with fs.open(fs_target, "rb") as f: + assert f.read() == b"data" + with pytest.raises(FileExistsError): + fs.open(fs_target, "xb") diff --git a/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/pipe.py b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/pipe.py new file mode 100644 index 0000000000000000000000000000000000000000..8ecca96e9d23ff268a253c48269d5cca451ea270 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/pipe.py @@ -0,0 +1,11 @@ +import pytest + + +class AbstractPipeTests: + def test_pipe_exclusive(self, fs, fs_target): + fs.pipe_file(fs_target, b"data") + assert fs.cat_file(fs_target) == b"data" + with pytest.raises(FileExistsError): + fs.pipe_file(fs_target, b"data", mode="create") + fs.pipe_file(fs_target, b"new data", mode="overwrite") + assert fs.cat_file(fs_target) == b"new data" diff --git a/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/put.py b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/put.py new file mode 100644 index 0000000000000000000000000000000000000000..9fc349977f0384d9fc86126498be5c6ad99a21d3 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/tests/abstract/put.py @@ -0,0 +1,591 @@ +from hashlib import md5 +from itertools import product + +import pytest + +from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS + + +class AbstractPutTests: + def test_put_file_to_existing_directory( + self, + fs, + fs_join, + fs_target, + local_join, + local_bulk_operations_scenario_0, + supports_empty_directories, + ): + # Copy scenario 1a + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + fs.touch(fs_join(target, "dummy")) + assert fs.isdir(target) + + target_file2 = fs_join(target, "file2") + target_subfile1 = fs_join(target, "subfile1") + + # Copy from source directory + fs.put(local_join(source, "file2"), target) + assert fs.isfile(target_file2) + + # Copy from sub directory + fs.put(local_join(source, "subdir", "subfile1"), target) + assert fs.isfile(target_subfile1) + + # Remove copied files + fs.rm([target_file2, target_subfile1]) + assert not fs.exists(target_file2) + assert not fs.exists(target_subfile1) + + # Repeat with trailing slash on target + fs.put(local_join(source, "file2"), target + "/") + assert fs.isdir(target) + assert fs.isfile(target_file2) + + fs.put(local_join(source, "subdir", "subfile1"), target + "/") + assert fs.isfile(target_subfile1) + + def test_put_file_to_new_directory( + self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 + ): + # Copy scenario 1b + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + fs.put( + local_join(source, "subdir", "subfile1"), fs_join(target, "newdir/") + ) # Note trailing slash + assert fs.isdir(target) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + + def test_put_file_to_file_in_existing_directory( + self, + fs, + fs_join, + fs_target, + local_join, + supports_empty_directories, + local_bulk_operations_scenario_0, + ): + # Copy scenario 1c + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + fs.touch(fs_join(target, "dummy")) + assert fs.isdir(target) + + fs.put(local_join(source, "subdir", "subfile1"), fs_join(target, "newfile")) + assert fs.isfile(fs_join(target, "newfile")) + + def test_put_file_to_file_in_new_directory( + self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 + ): + # Copy scenario 1d + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + fs.put( + local_join(source, "subdir", "subfile1"), + fs_join(target, "newdir", "newfile"), + ) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "newfile")) + + def test_put_directory_to_existing_directory( + self, + fs, + fs_join, + fs_target, + local_bulk_operations_scenario_0, + supports_empty_directories, + ): + # Copy scenario 1e + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = target + "/" if target_slash else target + + # Without recursive does nothing + fs.put(s, t) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # With recursive + fs.put(s, t, recursive=True) + if source_slash: + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert fs.isdir(fs_join(target, "nesteddir")) + assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + fs_join(target, "nesteddir"), + ], + recursive=True, + ) + else: + assert fs.isdir(fs_join(target, "subdir")) + assert fs.isfile(fs_join(target, "subdir", "subfile1")) + assert fs.isfile(fs_join(target, "subdir", "subfile2")) + assert fs.isdir(fs_join(target, "subdir", "nesteddir")) + assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile")) + + fs.rm(fs_join(target, "subdir"), recursive=True) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # Limit recursive by maxdepth + fs.put(s, t, recursive=True, maxdepth=1) + if source_slash: + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.exists(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + else: + assert fs.isdir(fs_join(target, "subdir")) + assert fs.isfile(fs_join(target, "subdir", "subfile1")) + assert fs.isfile(fs_join(target, "subdir", "subfile2")) + assert not fs.exists(fs_join(target, "subdir", "nesteddir")) + + fs.rm(fs_join(target, "subdir"), recursive=True) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_put_directory_to_new_directory( + self, + fs, + fs_join, + fs_target, + local_bulk_operations_scenario_0, + supports_empty_directories, + ): + # Copy scenario 1f + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = fs_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive does nothing + fs.put(s, t) + if supports_empty_directories: + assert fs.ls(target) == [] + else: + with pytest.raises(FileNotFoundError): + fs.ls(target) + + # With recursive + fs.put(s, t, recursive=True) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert fs.isdir(fs_join(target, "newdir", "nesteddir")) + assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # Limit recursive by maxdepth + fs.put(s, t, recursive=True, maxdepth=1) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + def test_put_glob_to_existing_directory( + self, + fs, + fs_join, + fs_target, + local_join, + supports_empty_directories, + local_bulk_operations_scenario_0, + ): + # Copy scenario 1g + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + # Without recursive + fs.put(local_join(source, "subdir", "*"), t) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.isdir(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.put(local_join(source, "subdir", glob), t, recursive=recursive) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert fs.isdir(fs_join(target, "nesteddir")) + assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + fs_join(target, "nesteddir"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # Limit recursive by maxdepth + fs.put( + local_join(source, "subdir", glob), + t, + recursive=recursive, + maxdepth=1, + ) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.exists(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_put_glob_to_new_directory( + self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 + ): + # Copy scenario 1h + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + for target_slash in [False, True]: + t = fs_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive + fs.put(local_join(source, "subdir", "*"), t) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.put(local_join(source, "subdir", glob), t, recursive=recursive) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert fs.isdir(fs_join(target, "newdir", "nesteddir")) + assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # Limit recursive by maxdepth + fs.put( + local_join(source, "subdir", glob), + t, + recursive=recursive, + maxdepth=1, + ) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + @pytest.mark.parametrize( + GLOB_EDGE_CASES_TESTS["argnames"], + GLOB_EDGE_CASES_TESTS["argvalues"], + ) + def test_put_glob_edge_cases( + self, + path, + recursive, + maxdepth, + expected, + fs, + fs_join, + fs_target, + local_glob_edge_cases_files, + local_join, + fs_sanitize_path, + ): + # Copy scenario 1g + source = local_glob_edge_cases_files + + target = fs_target + + for new_dir, target_slash in product([True, False], [True, False]): + fs.mkdir(target) + + t = fs_join(target, "newdir") if new_dir else target + t = t + "/" if target_slash else t + + fs.put(local_join(source, path), t, recursive=recursive, maxdepth=maxdepth) + + output = fs.find(target) + if new_dir: + prefixed_expected = [ + fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected + ] + else: + prefixed_expected = [ + fs_sanitize_path(fs_join(target, p)) for p in expected + ] + assert sorted(output) == sorted(prefixed_expected) + + try: + fs.rm(target, recursive=True) + except FileNotFoundError: + pass + + def test_put_list_of_files_to_existing_directory( + self, + fs, + fs_join, + fs_target, + local_join, + local_bulk_operations_scenario_0, + supports_empty_directories, + ): + # Copy scenario 2a + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + source_files = [ + local_join(source, "file1"), + local_join(source, "file2"), + local_join(source, "subdir", "subfile1"), + ] + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + fs.put(source_files, t) + assert fs.isfile(fs_join(target, "file1")) + assert fs.isfile(fs_join(target, "file2")) + assert fs.isfile(fs_join(target, "subfile1")) + + fs.rm( + [ + fs_join(target, "file1"), + fs_join(target, "file2"), + fs_join(target, "subfile1"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_put_list_of_files_to_new_directory( + self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 + ): + # Copy scenario 2b + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + source_files = [ + local_join(source, "file1"), + local_join(source, "file2"), + local_join(source, "subdir", "subfile1"), + ] + + fs.put(source_files, fs_join(target, "newdir") + "/") # Note trailing slash + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "file1")) + assert fs.isfile(fs_join(target, "newdir", "file2")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + + def test_put_directory_recursive( + self, fs, fs_join, fs_target, local_fs, local_join, local_path + ): + # https://github.com/fsspec/filesystem_spec/issues/1062 + # Recursive cp/get/put of source directory into non-existent target directory. + src = local_join(local_path, "src") + src_file = local_join(src, "file") + local_fs.mkdir(src) + local_fs.touch(src_file) + + target = fs_target + + # put without slash + assert not fs.exists(target) + for loop in range(2): + fs.put(src, target, recursive=True) + assert fs.isdir(target) + + if loop == 0: + assert fs.isfile(fs_join(target, "file")) + assert not fs.exists(fs_join(target, "src")) + else: + assert fs.isfile(fs_join(target, "file")) + assert fs.isdir(fs_join(target, "src")) + assert fs.isfile(fs_join(target, "src", "file")) + + fs.rm(target, recursive=True) + + # put with slash + assert not fs.exists(target) + for loop in range(2): + fs.put(src + "/", target, recursive=True) + assert fs.isdir(target) + assert fs.isfile(fs_join(target, "file")) + assert not fs.exists(fs_join(target, "src")) + + def test_put_directory_without_files_with_same_name_prefix( + self, + fs, + fs_join, + fs_target, + local_join, + local_dir_and_file_with_same_name_prefix, + supports_empty_directories, + ): + # Create the test dirs + source = local_dir_and_file_with_same_name_prefix + target = fs_target + + # Test without glob + fs.put(local_join(source, "subdir"), fs_target, recursive=True) + + assert fs.isfile(fs_join(fs_target, "subfile.txt")) + assert not fs.isfile(fs_join(fs_target, "subdir.txt")) + + fs.rm([fs_join(target, "subfile.txt")]) + if supports_empty_directories: + assert fs.ls(target) == [] + else: + assert not fs.exists(target) + + # Test with glob + fs.put(local_join(source, "subdir*"), fs_target, recursive=True) + + assert fs.isdir(fs_join(fs_target, "subdir")) + assert fs.isfile(fs_join(fs_target, "subdir", "subfile.txt")) + assert fs.isfile(fs_join(fs_target, "subdir.txt")) + + def test_copy_with_source_and_destination_as_list( + self, fs, fs_target, fs_join, local_join, local_10_files_with_hashed_names + ): + # Create the test dir + source = local_10_files_with_hashed_names + target = fs_target + + # Create list of files for source and destination + source_files = [] + destination_files = [] + for i in range(10): + hashed_i = md5(str(i).encode("utf-8")).hexdigest() + source_files.append(local_join(source, f"{hashed_i}.txt")) + destination_files.append(fs_join(target, f"{hashed_i}.txt")) + + # Copy and assert order was kept + fs.put(lpath=source_files, rpath=destination_files) + + for i in range(10): + file_content = fs.cat(destination_files[i]).decode("utf-8") + assert file_content == str(i) diff --git a/.venv/lib/python3.11/site-packages/fsspec/transaction.py b/.venv/lib/python3.11/site-packages/fsspec/transaction.py new file mode 100644 index 0000000000000000000000000000000000000000..77293f63ecc5f611e19d849ef236d53e9c258efc --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/transaction.py @@ -0,0 +1,90 @@ +from collections import deque + + +class Transaction: + """Filesystem transaction write context + + Gathers files for deferred commit or discard, so that several write + operations can be finalized semi-atomically. This works by having this + instance as the ``.transaction`` attribute of the given filesystem + """ + + def __init__(self, fs, **kwargs): + """ + Parameters + ---------- + fs: FileSystem instance + """ + self.fs = fs + self.files = deque() + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """End transaction and commit, if exit is not due to exception""" + # only commit if there was no exception + self.complete(commit=exc_type is None) + if self.fs: + self.fs._intrans = False + self.fs._transaction = None + self.fs = None + + def start(self): + """Start a transaction on this FileSystem""" + self.files = deque() # clean up after previous failed completions + self.fs._intrans = True + + def complete(self, commit=True): + """Finish transaction: commit or discard all deferred files""" + while self.files: + f = self.files.popleft() + if commit: + f.commit() + else: + f.discard() + self.fs._intrans = False + self.fs._transaction = None + self.fs = None + + +class FileActor: + def __init__(self): + self.files = [] + + def commit(self): + for f in self.files: + f.commit() + self.files.clear() + + def discard(self): + for f in self.files: + f.discard() + self.files.clear() + + def append(self, f): + self.files.append(f) + + +class DaskTransaction(Transaction): + def __init__(self, fs): + """ + Parameters + ---------- + fs: FileSystem instance + """ + import distributed + + super().__init__(fs) + client = distributed.default_client() + self.files = client.submit(FileActor, actor=True).result() + + def complete(self, commit=True): + """Finish transaction: commit or discard all deferred files""" + if commit: + self.files.commit().result() + else: + self.files.discard().result() + self.fs._intrans = False + self.fs = None diff --git a/.venv/lib/python3.11/site-packages/fsspec/utils.py b/.venv/lib/python3.11/site-packages/fsspec/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7b06dd581417eca1e90a19ef25fb14dcd3cf9c9c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/fsspec/utils.py @@ -0,0 +1,748 @@ +from __future__ import annotations + +import contextlib +import logging +import math +import os +import re +import sys +import tempfile +from collections.abc import Callable, Iterable, Iterator, Sequence +from functools import partial +from hashlib import md5 +from importlib.metadata import version +from typing import IO, TYPE_CHECKING, Any, TypeVar +from urllib.parse import urlsplit + +if TYPE_CHECKING: + import pathlib + from typing import TypeGuard + + from fsspec.spec import AbstractFileSystem + + +DEFAULT_BLOCK_SIZE = 5 * 2**20 + +T = TypeVar("T") + + +def infer_storage_options( + urlpath: str, inherit_storage_options: dict[str, Any] | None = None +) -> dict[str, Any]: + """Infer storage options from URL path and merge it with existing storage + options. + + Parameters + ---------- + urlpath: str or unicode + Either local absolute file path or URL (hdfs://namenode:8020/file.csv) + inherit_storage_options: dict (optional) + Its contents will get merged with the inferred information from the + given path + + Returns + ------- + Storage options dict. + + Examples + -------- + >>> infer_storage_options('/mnt/datasets/test.csv') # doctest: +SKIP + {"protocol": "file", "path", "/mnt/datasets/test.csv"} + >>> infer_storage_options( + ... 'hdfs://username:pwd@node:123/mnt/datasets/test.csv?q=1', + ... inherit_storage_options={'extra': 'value'}, + ... ) # doctest: +SKIP + {"protocol": "hdfs", "username": "username", "password": "pwd", + "host": "node", "port": 123, "path": "/mnt/datasets/test.csv", + "url_query": "q=1", "extra": "value"} + """ + # Handle Windows paths including disk name in this special case + if ( + re.match(r"^[a-zA-Z]:[\\/]", urlpath) + or re.match(r"^[a-zA-Z0-9]+://", urlpath) is None + ): + return {"protocol": "file", "path": urlpath} + + parsed_path = urlsplit(urlpath) + protocol = parsed_path.scheme or "file" + if parsed_path.fragment: + path = "#".join([parsed_path.path, parsed_path.fragment]) + else: + path = parsed_path.path + if protocol == "file": + # Special case parsing file protocol URL on Windows according to: + # https://msdn.microsoft.com/en-us/library/jj710207.aspx + windows_path = re.match(r"^/([a-zA-Z])[:|]([\\/].*)$", path) + if windows_path: + drive, path = windows_path.groups() + path = f"{drive}:{path}" + + if protocol in ["http", "https"]: + # for HTTP, we don't want to parse, as requests will anyway + return {"protocol": protocol, "path": urlpath} + + options: dict[str, Any] = {"protocol": protocol, "path": path} + + if parsed_path.netloc: + # Parse `hostname` from netloc manually because `parsed_path.hostname` + # lowercases the hostname which is not always desirable (e.g. in S3): + # https://github.com/dask/dask/issues/1417 + options["host"] = parsed_path.netloc.rsplit("@", 1)[-1].rsplit(":", 1)[0] + + if protocol in ("s3", "s3a", "gcs", "gs"): + options["path"] = options["host"] + options["path"] + else: + options["host"] = options["host"] + if parsed_path.port: + options["port"] = parsed_path.port + if parsed_path.username: + options["username"] = parsed_path.username + if parsed_path.password: + options["password"] = parsed_path.password + + if parsed_path.query: + options["url_query"] = parsed_path.query + if parsed_path.fragment: + options["url_fragment"] = parsed_path.fragment + + if inherit_storage_options: + update_storage_options(options, inherit_storage_options) + + return options + + +def update_storage_options( + options: dict[str, Any], inherited: dict[str, Any] | None = None +) -> None: + if not inherited: + inherited = {} + collisions = set(options) & set(inherited) + if collisions: + for collision in collisions: + if options.get(collision) != inherited.get(collision): + raise KeyError( + f"Collision between inferred and specified storage " + f"option:\n{collision}" + ) + options.update(inherited) + + +# Compression extensions registered via fsspec.compression.register_compression +compressions: dict[str, str] = {} + + +def infer_compression(filename: str) -> str | None: + """Infer compression, if available, from filename. + + Infer a named compression type, if registered and available, from filename + extension. This includes builtin (gz, bz2, zip) compressions, as well as + optional compressions. See fsspec.compression.register_compression. + """ + extension = os.path.splitext(filename)[-1].strip(".").lower() + if extension in compressions: + return compressions[extension] + return None + + +def build_name_function(max_int: float) -> Callable[[int], str]: + """Returns a function that receives a single integer + and returns it as a string padded by enough zero characters + to align with maximum possible integer + + >>> name_f = build_name_function(57) + + >>> name_f(7) + '07' + >>> name_f(31) + '31' + >>> build_name_function(1000)(42) + '0042' + >>> build_name_function(999)(42) + '042' + >>> build_name_function(0)(0) + '0' + """ + # handle corner cases max_int is 0 or exact power of 10 + max_int += 1e-8 + + pad_length = int(math.ceil(math.log10(max_int))) + + def name_function(i: int) -> str: + return str(i).zfill(pad_length) + + return name_function + + +def seek_delimiter(file: IO[bytes], delimiter: bytes, blocksize: int) -> bool: + r"""Seek current file to file start, file end, or byte after delimiter seq. + + Seeks file to next chunk delimiter, where chunks are defined on file start, + a delimiting sequence, and file end. Use file.tell() to see location afterwards. + Note that file start is a valid split, so must be at offset > 0 to seek for + delimiter. + + Parameters + ---------- + file: a file + delimiter: bytes + a delimiter like ``b'\n'`` or message sentinel, matching file .read() type + blocksize: int + Number of bytes to read from the file at once. + + + Returns + ------- + Returns True if a delimiter was found, False if at file start or end. + + """ + + if file.tell() == 0: + # beginning-of-file, return without seek + return False + + # Interface is for binary IO, with delimiter as bytes, but initialize last + # with result of file.read to preserve compatibility with text IO. + last: bytes | None = None + while True: + current = file.read(blocksize) + if not current: + # end-of-file without delimiter + return False + full = last + current if last else current + try: + if delimiter in full: + i = full.index(delimiter) + file.seek(file.tell() - (len(full) - i) + len(delimiter)) + return True + elif len(current) < blocksize: + # end-of-file without delimiter + return False + except (OSError, ValueError): + pass + last = full[-len(delimiter) :] + + +def read_block( + f: IO[bytes], + offset: int, + length: int | None, + delimiter: bytes | None = None, + split_before: bool = False, +) -> bytes: + """Read a block of bytes from a file + + Parameters + ---------- + f: File + Open file + offset: int + Byte offset to start read + length: int + Number of bytes to read, read through end of file if None + delimiter: bytes (optional) + Ensure reading starts and stops at delimiter bytestring + split_before: bool (optional) + Start/stop read *before* delimiter bytestring. + + + If using the ``delimiter=`` keyword argument we ensure that the read + starts and stops at delimiter boundaries that follow the locations + ``offset`` and ``offset + length``. If ``offset`` is zero then we + start at zero, regardless of delimiter. The bytestring returned WILL + include the terminating delimiter string. + + Examples + -------- + + >>> from io import BytesIO # doctest: +SKIP + >>> f = BytesIO(b'Alice, 100\\nBob, 200\\nCharlie, 300') # doctest: +SKIP + >>> read_block(f, 0, 13) # doctest: +SKIP + b'Alice, 100\\nBo' + + >>> read_block(f, 0, 13, delimiter=b'\\n') # doctest: +SKIP + b'Alice, 100\\nBob, 200\\n' + + >>> read_block(f, 10, 10, delimiter=b'\\n') # doctest: +SKIP + b'Bob, 200\\nCharlie, 300' + """ + if delimiter: + f.seek(offset) + found_start_delim = seek_delimiter(f, delimiter, 2**16) + if length is None: + return f.read() + start = f.tell() + length -= start - offset + + f.seek(start + length) + found_end_delim = seek_delimiter(f, delimiter, 2**16) + end = f.tell() + + # Adjust split location to before delimiter if seek found the + # delimiter sequence, not start or end of file. + if found_start_delim and split_before: + start -= len(delimiter) + + if found_end_delim and split_before: + end -= len(delimiter) + + offset = start + length = end - start + + f.seek(offset) + + # TODO: allow length to be None and read to the end of the file? + assert length is not None + b = f.read(length) + return b + + +def tokenize(*args: Any, **kwargs: Any) -> str: + """Deterministic token + + (modified from dask.base) + + >>> tokenize([1, 2, '3']) + '9d71491b50023b06fc76928e6eddb952' + + >>> tokenize('Hello') == tokenize('Hello') + True + """ + if kwargs: + args += (kwargs,) + try: + h = md5(str(args).encode()) + except ValueError: + # FIPS systems: https://github.com/fsspec/filesystem_spec/issues/380 + h = md5(str(args).encode(), usedforsecurity=False) + return h.hexdigest() + + +def stringify_path(filepath: str | os.PathLike[str] | pathlib.Path) -> str: + """Attempt to convert a path-like object to a string. + + Parameters + ---------- + filepath: object to be converted + + Returns + ------- + filepath_str: maybe a string version of the object + + Notes + ----- + Objects supporting the fspath protocol are coerced according to its + __fspath__ method. + + For backwards compatibility with older Python version, pathlib.Path + objects are specially coerced. + + Any other object is passed through unchanged, which includes bytes, + strings, buffers, or anything else that's not even path-like. + """ + if isinstance(filepath, str): + return filepath + elif hasattr(filepath, "__fspath__"): + return filepath.__fspath__() + elif hasattr(filepath, "path"): + return filepath.path + else: + return filepath # type: ignore[return-value] + + +def make_instance( + cls: Callable[..., T], args: Sequence[Any], kwargs: dict[str, Any] +) -> T: + inst = cls(*args, **kwargs) + inst._determine_worker() # type: ignore[attr-defined] + return inst + + +def common_prefix(paths: Iterable[str]) -> str: + """For a list of paths, find the shortest prefix common to all""" + parts = [p.split("/") for p in paths] + lmax = min(len(p) for p in parts) + end = 0 + for i in range(lmax): + end = all(p[i] == parts[0][i] for p in parts) + if not end: + break + i += end + return "/".join(parts[0][:i]) + + +def other_paths( + paths: list[str], + path2: str | list[str], + exists: bool = False, + flatten: bool = False, +) -> list[str]: + """In bulk file operations, construct a new file tree from a list of files + + Parameters + ---------- + paths: list of str + The input file tree + path2: str or list of str + Root to construct the new list in. If this is already a list of str, we just + assert it has the right number of elements. + exists: bool (optional) + For a str destination, it is already exists (and is a dir), files should + end up inside. + flatten: bool (optional) + Whether to flatten the input directory tree structure so that the output files + are in the same directory. + + Returns + ------- + list of str + """ + + if isinstance(path2, str): + path2 = path2.rstrip("/") + + if flatten: + path2 = ["/".join((path2, p.split("/")[-1])) for p in paths] + else: + cp = common_prefix(paths) + if exists: + cp = cp.rsplit("/", 1)[0] + if not cp and all(not s.startswith("/") for s in paths): + path2 = ["/".join([path2, p]) for p in paths] + else: + path2 = [p.replace(cp, path2, 1) for p in paths] + else: + assert len(paths) == len(path2) + return path2 + + +def is_exception(obj: Any) -> bool: + return isinstance(obj, BaseException) + + +def isfilelike(f: Any) -> TypeGuard[IO[bytes]]: + return all(hasattr(f, attr) for attr in ["read", "close", "tell"]) + + +def get_protocol(url: str) -> str: + url = stringify_path(url) + parts = re.split(r"(\:\:|\://)", url, maxsplit=1) + if len(parts) > 1: + return parts[0] + return "file" + + +def get_file_extension(url: str) -> str: + url = stringify_path(url) + ext_parts = url.rsplit(".", 1) + if len(ext_parts) > 1: + return ext_parts[-1] + return "" + + +def can_be_local(path: str) -> bool: + """Can the given URL be used with open_local?""" + from fsspec import get_filesystem_class + + try: + return getattr(get_filesystem_class(get_protocol(path)), "local_file", False) + except (ValueError, ImportError): + # not in registry or import failed + return False + + +def get_package_version_without_import(name: str) -> str | None: + """For given package name, try to find the version without importing it + + Import and package.__version__ is still the backup here, so an import + *might* happen. + + Returns either the version string, or None if the package + or the version was not readily found. + """ + if name in sys.modules: + mod = sys.modules[name] + if hasattr(mod, "__version__"): + return mod.__version__ + try: + return version(name) + except: # noqa: E722 + pass + try: + import importlib + + mod = importlib.import_module(name) + return mod.__version__ + except (ImportError, AttributeError): + return None + + +def setup_logging( + logger: logging.Logger | None = None, + logger_name: str | None = None, + level: str = "DEBUG", + clear: bool = True, +) -> logging.Logger: + if logger is None and logger_name is None: + raise ValueError("Provide either logger object or logger name") + logger = logger or logging.getLogger(logger_name) + handle = logging.StreamHandler() + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(funcName)s -- %(message)s" + ) + handle.setFormatter(formatter) + if clear: + logger.handlers.clear() + logger.addHandler(handle) + logger.setLevel(level) + return logger + + +def _unstrip_protocol(name: str, fs: AbstractFileSystem) -> str: + return fs.unstrip_protocol(name) + + +def mirror_from( + origin_name: str, methods: Iterable[str] +) -> Callable[[type[T]], type[T]]: + """Mirror attributes and methods from the given + origin_name attribute of the instance to the + decorated class""" + + def origin_getter(method: str, self: Any) -> Any: + origin = getattr(self, origin_name) + return getattr(origin, method) + + def wrapper(cls: type[T]) -> type[T]: + for method in methods: + wrapped_method = partial(origin_getter, method) + setattr(cls, method, property(wrapped_method)) + return cls + + return wrapper + + +@contextlib.contextmanager +def nullcontext(obj: T) -> Iterator[T]: + yield obj + + +def merge_offset_ranges( + paths: list[str], + starts: list[int] | int, + ends: list[int] | int, + max_gap: int = 0, + max_block: int | None = None, + sort: bool = True, +) -> tuple[list[str], list[int], list[int]]: + """Merge adjacent byte-offset ranges when the inter-range + gap is <= `max_gap`, and when the merged byte range does not + exceed `max_block` (if specified). By default, this function + will re-order the input paths and byte ranges to ensure sorted + order. If the user can guarantee that the inputs are already + sorted, passing `sort=False` will skip the re-ordering. + """ + # Check input + if not isinstance(paths, list): + raise TypeError + if not isinstance(starts, list): + starts = [starts] * len(paths) + if not isinstance(ends, list): + ends = [ends] * len(paths) + if len(starts) != len(paths) or len(ends) != len(paths): + raise ValueError + + # Early Return + if len(starts) <= 1: + return paths, starts, ends + + starts = [s or 0 for s in starts] + # Sort by paths and then ranges if `sort=True` + if sort: + paths, starts, ends = ( + list(v) + for v in zip( + *sorted( + zip(paths, starts, ends), + ) + ) + ) + remove = [] + for i, (path, start, end) in enumerate(zip(paths, starts, ends)): + if any( + e is not None and p == path and start >= s and end <= e and i != i2 + for i2, (p, s, e) in enumerate(zip(paths, starts, ends)) + ): + remove.append(i) + paths = [p for i, p in enumerate(paths) if i not in remove] + starts = [s for i, s in enumerate(starts) if i not in remove] + ends = [e for i, e in enumerate(ends) if i not in remove] + + if paths: + # Loop through the coupled `paths`, `starts`, and + # `ends`, and merge adjacent blocks when appropriate + new_paths = paths[:1] + new_starts = starts[:1] + new_ends = ends[:1] + for i in range(1, len(paths)): + if paths[i] == paths[i - 1] and new_ends[-1] is None: + continue + elif ( + paths[i] != paths[i - 1] + or ((starts[i] - new_ends[-1]) > max_gap) + or (max_block is not None and (ends[i] - new_starts[-1]) > max_block) + ): + # Cannot merge with previous block. + # Add new `paths`, `starts`, and `ends` elements + new_paths.append(paths[i]) + new_starts.append(starts[i]) + new_ends.append(ends[i]) + else: + # Merge with the previous block by updating the + # last element of `ends` + new_ends[-1] = ends[i] + return new_paths, new_starts, new_ends + + # `paths` is empty. Just return input lists + return paths, starts, ends + + +def file_size(filelike: IO[bytes]) -> int: + """Find length of any open read-mode file-like""" + pos = filelike.tell() + try: + return filelike.seek(0, 2) + finally: + filelike.seek(pos) + + +@contextlib.contextmanager +def atomic_write(path: str, mode: str = "wb"): + """ + A context manager that opens a temporary file next to `path` and, on exit, + replaces `path` with the temporary file, thereby updating `path` + atomically. + """ + fd, fn = tempfile.mkstemp( + dir=os.path.dirname(path), prefix=os.path.basename(path) + "-" + ) + try: + with open(fd, mode) as fp: + yield fp + except BaseException: + with contextlib.suppress(FileNotFoundError): + os.unlink(fn) + raise + else: + os.replace(fn, path) + + +def _translate(pat, STAR, QUESTION_MARK): + # Copied from: https://github.com/python/cpython/pull/106703. + res: list[str] = [] + add = res.append + i, n = 0, len(pat) + while i < n: + c = pat[i] + i = i + 1 + if c == "*": + # compress consecutive `*` into one + if (not res) or res[-1] is not STAR: + add(STAR) + elif c == "?": + add(QUESTION_MARK) + elif c == "[": + j = i + if j < n and pat[j] == "!": + j = j + 1 + if j < n and pat[j] == "]": + j = j + 1 + while j < n and pat[j] != "]": + j = j + 1 + if j >= n: + add("\\[") + else: + stuff = pat[i:j] + if "-" not in stuff: + stuff = stuff.replace("\\", r"\\") + else: + chunks = [] + k = i + 2 if pat[i] == "!" else i + 1 + while True: + k = pat.find("-", k, j) + if k < 0: + break + chunks.append(pat[i:k]) + i = k + 1 + k = k + 3 + chunk = pat[i:j] + if chunk: + chunks.append(chunk) + else: + chunks[-1] += "-" + # Remove empty ranges -- invalid in RE. + for k in range(len(chunks) - 1, 0, -1): + if chunks[k - 1][-1] > chunks[k][0]: + chunks[k - 1] = chunks[k - 1][:-1] + chunks[k][1:] + del chunks[k] + # Escape backslashes and hyphens for set difference (--). + # Hyphens that create ranges shouldn't be escaped. + stuff = "-".join( + s.replace("\\", r"\\").replace("-", r"\-") for s in chunks + ) + # Escape set operations (&&, ~~ and ||). + stuff = re.sub(r"([&~|])", r"\\\1", stuff) + i = j + 1 + if not stuff: + # Empty range: never match. + add("(?!)") + elif stuff == "!": + # Negated empty range: match any character. + add(".") + else: + if stuff[0] == "!": + stuff = "^" + stuff[1:] + elif stuff[0] in ("^", "["): + stuff = "\\" + stuff + add(f"[{stuff}]") + else: + add(re.escape(c)) + assert i == n + return res + + +def glob_translate(pat): + # Copied from: https://github.com/python/cpython/pull/106703. + # The keyword parameters' values are fixed to: + # recursive=True, include_hidden=True, seps=None + """Translate a pathname with shell wildcards to a regular expression.""" + if os.path.altsep: + seps = os.path.sep + os.path.altsep + else: + seps = os.path.sep + escaped_seps = "".join(map(re.escape, seps)) + any_sep = f"[{escaped_seps}]" if len(seps) > 1 else escaped_seps + not_sep = f"[^{escaped_seps}]" + one_last_segment = f"{not_sep}+" + one_segment = f"{one_last_segment}{any_sep}" + any_segments = f"(?:.+{any_sep})?" + any_last_segments = ".*" + results = [] + parts = re.split(any_sep, pat) + last_part_idx = len(parts) - 1 + for idx, part in enumerate(parts): + if part == "*": + results.append(one_segment if idx < last_part_idx else one_last_segment) + continue + if part == "**": + results.append(any_segments if idx < last_part_idx else any_last_segments) + continue + elif "**" in part: + raise ValueError( + "Invalid pattern: '**' can only be an entire path component" + ) + if part: + results.extend(_translate(part, f"{not_sep}*", not_sep)) + if idx < last_part_idx: + results.append(any_sep) + res = "".join(results) + return rf"(?s:{res})\Z" diff --git a/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/INSTALLER b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/METADATA b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..8f61b223d605f6a4c3a73ff7756375cbbf9a6915 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/METADATA @@ -0,0 +1,52 @@ +Metadata-Version: 2.4 +Name: google-crc32c +Version: 1.8.0 +Summary: A python wrapper of the C library 'Google CRC32C' +Home-page: https://github.com/googleapis/python-crc32c +Author: Google LLC +Author-email: googleapis-packages@google.com +Platform: Posix +Platform: MacOS X +Platform: Windows +Requires-Python: >=3.9 +Description-Content-Type: text/markdown +License-File: LICENSE +Dynamic: license-file + +# `google-crc32c` +![GA](https://img.shields.io/badge/support-GA-gold.svg) [<img src="https://img.shields.io/pypi/v/google-crc32c.svg">](https://pypi.org/project/google-crc32c) ![Python Versions](https://img.shields.io/pypi/pyversions/google-crc32c) + +This package wraps the [`google/crc32c`](https://github.com/google/crc32c) +hardware-based implementation of the CRC32C hashing algorithm. Multiple wheels +are distributed as well as source. If a wheel is not published for the python +version and platform you are using, you will need to compile crc32c using a +C toolchain. + +# Currently Published Wheels + +Wheels are currently published for CPython 3.9, 3.10, 3.11, 3.12, 3.13 and 3.14 +for multiple architectures. PyPy 3.9 and 3.10 are also supported for Linux. +For information on building your own wheels please view [BUILDING.md](BUILDING.md). + + +## Linux + +Wheels are published for the following platforms / architectures: + +- `manylinux2010` platform, `x86_64` and `1686` architectures +- `manylinux2014` platform, `aarch64` architecture + +### Unsupported Platforms + +- `manylinux1` platform, `x86_64` architecture support has ended. + +See https://github.com/pypa/manylinux/issues/994. + +## Mac OS + +Wheels are published for `x86_64` and `arm64` architectures. + + +## Windows + +Wheels are published for the `win_amd64` architecture. diff --git a/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/RECORD b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..d8c5aaebd17f17eb31b47104c5055fd402942923 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/RECORD @@ -0,0 +1,17 @@ +google_crc32c-1.8.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +google_crc32c-1.8.0.dist-info/METADATA,sha256=14xeEGC2XtH5pSldfRCf3u3VJDfsU725Ze8sajbEZWU,1716 +google_crc32c-1.8.0.dist-info/RECORD,, +google_crc32c-1.8.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +google_crc32c-1.8.0.dist-info/WHEEL,sha256=MerhEK-QOqn9qFOPGOxLHYkz8Z6eRdDuoFfdt4pMQf0,224 +google_crc32c-1.8.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358 +google_crc32c-1.8.0.dist-info/top_level.txt,sha256=r7PLPlKjfhMZLqeRsKXIQdIzbe3Frv_2_b8XmcvZ4FQ,14 +google_crc32c-1.8.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1 +google_crc32c.libs/libcrc32c-7ebc40c5.so.1.1.0,sha256=oCcj2cmrPu8NKxM3ZVhY2golYzECJ9X7ilhsuWpel0o,25385 +google_crc32c/__config__.py,sha256=2i0KpKo-ypmMS_P9i3qObAW2eemCWtrmu_PJXktjHO8,1138 +google_crc32c/__init__.py,sha256=Dun6VhGxQ-8fydGR_hX4U9K3n8XPYXCDKmoIBAatpzM,1291 +google_crc32c/_checksum.py,sha256=P_1ALNsZpoYTHWrfMqEDwRwKdJ_y1EfxwVkXddVZ01k,2639 +google_crc32c/_crc32c.c,sha256=nk2zTkUjRBAUB8gfCyYTsrs3vmzCn95RmNEZg471kPQ,1428 +google_crc32c/_crc32c.cpython-311-x86_64-linux-gnu.so,sha256=OW56BFe6vSJWOPA6qQqO8-lkLvCz6nBLkAEijt415nk,33585 +google_crc32c/cext.py,sha256=oF7NFpKWUOEoyYlbVGIAJVo8SNDiWRT_VYyCI14-5jo,1557 +google_crc32c/py.typed,sha256=2rT_wtizx2JwmnzAr6tXdmsEdE5PIp4kum7YsDhOw_w,76 +google_crc32c/python.py,sha256=9zy6ABAB_KM8JJRcZnE4b68VqnOEFRNmCGufrjcb7qc,5661 diff --git a/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/REQUESTED b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/WHEEL b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..98c37c2eb49d26fdbf2bd7c73059bca3c7818cbe --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/WHEEL @@ -0,0 +1,8 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: false +Tag: cp311-cp311-manylinux_2_5_x86_64 +Tag: cp311-cp311-manylinux1_x86_64 +Tag: cp311-cp311-manylinux_2_17_x86_64 +Tag: cp311-cp311-manylinux2014_x86_64 + diff --git a/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/licenses/LICENSE b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/licenses/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/top_level.txt b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..10b44e60779c3235c10d02030e1b0c89201579a8 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/top_level.txt @@ -0,0 +1 @@ +google_crc32c diff --git a/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/zip-safe b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/zip-safe new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c-1.8.0.dist-info/zip-safe @@ -0,0 +1 @@ + diff --git a/.venv/lib/python3.11/site-packages/google_crc32c.libs/libcrc32c-7ebc40c5.so.1.1.0 b/.venv/lib/python3.11/site-packages/google_crc32c.libs/libcrc32c-7ebc40c5.so.1.1.0 new file mode 100644 index 0000000000000000000000000000000000000000..3441f326a8d95b47efb507f88a45fa4598f831e7 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/google_crc32c.libs/libcrc32c-7ebc40c5.so.1.1.0 differ diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/__config__.py b/.venv/lib/python3.11/site-packages/google_crc32c/__config__.py new file mode 100644 index 0000000000000000000000000000000000000000..f407db15b27e2cfc2a1c90b0890f46538292e346 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c/__config__.py @@ -0,0 +1,38 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def modify_path(): + """Modify the module search path.""" + # Only modify path on Windows. + if os.name != "nt": + return + + path = os.environ.get("PATH") + if path is None: + return + + try: + from importlib.resources import files as _resources_files + + extra_dll_dir = str(_resources_files("google_crc32c") / "extra-dll") + if os.path.isdir(extra_dll_dir): + # Python 3.8+ uses add_dll_directory. + os.add_dll_directory(extra_dll_dir) + except ImportError: + pass + +modify_path() diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/__init__.py b/.venv/lib/python3.11/site-packages/google_crc32c/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cae23411cfd2d1b850f5cf0acb2f808e130d92a5 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings + +_SLOW_CRC32C_WARNING = ( + "As the c extension couldn't be imported, `google-crc32c` is using a " + "pure python implementation that is significantly slower. If possible, " + "please configure a c build environment and compile the extension" +) + +# Default to C exstension Implementation, falling back to pure python. +try: + from google_crc32c import cext as impl + implementation = "c" +except ImportError as exc: + from google_crc32c import python as impl # type: ignore + warnings.warn(_SLOW_CRC32C_WARNING, RuntimeWarning) + implementation = "python" + +extend = impl.extend +value = impl.value + +Checksum = impl.Checksum + +__all__ = ["extend", "value", "Checksum", "implementation"] diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/__config__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/__config__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..75e3fba9002a9660d979d1f2b9d949a8d2d819e8 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/__config__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eb8c1e9efef3d395b0f390497b0c67d4fc49039e Binary files /dev/null and b/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/_checksum.cpython-311.pyc b/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/_checksum.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..55f92ffcf19b1fc408b697a84ea2d51550f1a837 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/_checksum.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/cext.cpython-311.pyc b/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/cext.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..acbf4a95f4265a3de7c8995f476ac62324e8c063 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/google_crc32c/__pycache__/cext.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/_checksum.py b/.venv/lib/python3.11/site-packages/google_crc32c/_checksum.py new file mode 100644 index 0000000000000000000000000000000000000000..e504085860c527100ba47aa1118b4586bf17015d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c/_checksum.py @@ -0,0 +1,87 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + + +class CommonChecksum(object): + """Hashlib-alike helper for CRC32C operations. + + This class should not be used directly and requires an update implementation. + + Args: + initial_value (Optional[bytes]): the initial chunk of data from + which the CRC32C checksum is computed. Defaults to b''. + """ + __slots__ = () + + def __init__(self, initial_value=b""): + self._crc = 0 + if initial_value != b"": + self.update(initial_value) + + def update(self, data): + """Update the checksum with a new chunk of data. + + Args: + chunk (Optional[bytes]): a chunk of data used to extend + the CRC32C checksum. + """ + raise NotImplementedError + + def digest(self): + """Big-endian order, per RFC 4960. + + See: https://cloud.google.com/storage/docs/json_api/v1/objects#crc32c + + Returns: + bytes: An eight-byte digest string. + """ + return struct.pack(">L", self._crc) + + def hexdigest(self): + """Like :meth:`digest` except returns as a bytestring of double length. + + Returns + bytes: A sixteen byte digest string, contaiing only hex digits. + """ + return "{:08x}".format(self._crc).encode("ascii") + + def copy(self): + """Create another checksum with the same CRC32C value. + + Returns: + Checksum: the new instance. + """ + clone = self.__class__() + clone._crc = self._crc + return clone + + def consume(self, stream, chunksize): + """Consume chunks from a stream, extending our CRC32 checksum. + + Args: + stream (BinaryIO): the stream to consume. + chunksize (int): the size of the read to perform + + Returns: + Generator[bytes, None, None]: Iterable of the chunks read from the + stream. + """ + while True: + chunk = stream.read(chunksize) + if not chunk: + break + self.update(chunk) + yield chunk diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/_crc32c.c b/.venv/lib/python3.11/site-packages/google_crc32c/_crc32c.c new file mode 100644 index 0000000000000000000000000000000000000000..c84b8839376aacf7db4b123ae98d67c66ab809a9 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c/_crc32c.c @@ -0,0 +1,62 @@ +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <crc32c/crc32c.h> + + +static PyObject * +_crc32c_extend(PyObject *self, PyObject *args) +{ + unsigned long crc_input; + uint32_t crc; + const char *chunk; + Py_ssize_t length; + + if (!PyArg_ParseTuple(args, "ky#", &crc_input, &chunk, &length)) + return NULL; + + crc = crc32c_extend((uint32_t)crc_input, (const uint8_t*)chunk, length); + + return PyLong_FromUnsignedLong(crc); +} + + +static PyObject * +_crc32c_value(PyObject *self, PyObject *args) +{ + uint32_t crc; + const char *chunk; + Py_ssize_t length; + + if (!PyArg_ParseTuple(args, "y#", &chunk, &length)) + return NULL; + + crc = crc32c_value((const uint8_t*)chunk, length); + + return PyLong_FromUnsignedLong(crc); +} + + +static PyMethodDef Crc32cMethods[] = { + {"extend", _crc32c_extend, METH_VARARGS, + "Return an updated CRC32C checksum."}, + {"value", _crc32c_value, METH_VARARGS, + "Return an initial CRC32C checksum."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + +static struct PyModuleDef crc32cmodule = { + PyModuleDef_HEAD_INIT, + "_crc32c", /* name of module */ + NULL, /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + Crc32cMethods +}; + + +PyMODINIT_FUNC +PyInit__crc32c(void) +{ + return PyModule_Create(&crc32cmodule); +} diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/_crc32c.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/google_crc32c/_crc32c.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..0507f54daeec0ca7ec647b30eb90e49aede627bd Binary files /dev/null and b/.venv/lib/python3.11/site-packages/google_crc32c/_crc32c.cpython-311-x86_64-linux-gnu.so differ diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/cext.py b/.venv/lib/python3.11/site-packages/google_crc32c/cext.py new file mode 100644 index 0000000000000000000000000000000000000000..4a764e388325cbb9fef5357e2a583cef7474ae22 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c/cext.py @@ -0,0 +1,45 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +# NOTE: ``__config__`` **must** be the first import because it (may) +# modify the search path used to locate shared libraries. +import google_crc32c.__config__ # type: ignore +from google_crc32c._crc32c import extend # type: ignore +from google_crc32c._crc32c import value # type: ignore +from google_crc32c._checksum import CommonChecksum + + +class Checksum(CommonChecksum): + """Hashlib-alike helper for CRC32C operations. + + Args: + initial_value (Optional[bytes]): the initial chunk of data from + which the CRC32C checksum is computed. Defaults to b''. + """ + + __slots__ = ("_crc",) + + def __init__(self, initial_value=b""): + self._crc = value(initial_value) + + def update(self, chunk): + """Update the checksum with a new chunk of data. + + Args: + chunk (Optional[bytes]): a chunk of data used to extend + the CRC32C checksum. + """ + self._crc = extend(self._crc, chunk) diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/py.typed b/.venv/lib/python3.11/site-packages/google_crc32c/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..076325e24604a323aed02a88bc9104f46bbc3948 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google_crc32c package uses inline types. diff --git a/.venv/lib/python3.11/site-packages/google_crc32c/python.py b/.venv/lib/python3.11/site-packages/google_crc32c/python.py new file mode 100644 index 0000000000000000000000000000000000000000..0ab5f353acc112544aa4bede66d325269223097b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_crc32c/python.py @@ -0,0 +1,124 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array +import struct + +from google_crc32c._checksum import CommonChecksum + + +def extend(crc, chunk): + """Update an existing CRC checksum with new chunk of data. + + Args + crc (int): An existing CRC check sum. + chunk (Union[bytes, List[int], Tuple[int]]): A new chunk of data. + Intended to be a byte string or similar. + + Returns + int: New CRC checksum computed by extending existing CRC + with ``chunk``. + """ + c = Checksum() + c._crc = crc + c.update(chunk) + return c._crc + + +def value(chunk): + """Compute a CRC checksum for a chunk of data. + + Args + chunk (Union[bytes, List[int], Tuple[int]]): A new chunk of data. + Intended to be a byte string or similar. + + Returns + int: New CRC checksum computed for ``chunk``. + """ + c = Checksum() + c.update(chunk) + return c._crc + + +class Checksum(CommonChecksum): + """Hashlib-alike helper for CRC32C operations. + + Args: + initial_value (Optional[bytes]): the initial chunk of data from + which the CRC32C checksum is computed. Defaults to b''. + """ + + def __init__(self, initial_value=b""): + self._crc = 0 + if initial_value != b"": + self.update(initial_value) + + def update(self, data): + """Update the checksum with a new chunk of data. + + Args: + chunk (Optional[bytes]): a chunk of data used to extend + the CRC32C checksum. + """ + if type(data) != array.array or data.itemsize != 1: + buffer = array.array("B", data) + else: + buffer = data + self._crc = self._crc ^ 0xFFFFFFFF + for b in buffer: + table_poly = _TABLE[(b ^ self._crc) & 0xFF] + self._crc = table_poly ^ ((self._crc >> 8) & 0xFFFFFFFF) + self._crc = self._crc ^ 0xFFFFFFFF + + +# fmt:off +_TABLE = [ + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, + 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, + 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, 0xf165b798, 0x30e349b, 0xd7c45070, + 0x25afd373, 0x36ff2087, 0xc494a384, 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, + 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x61c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, + 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, 0x30e349b1, + 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x5125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, + 0x6ef07595, 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, + 0x67dafa54, 0x95b17957, 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0xc38d26c, + 0xfe53516f, 0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, + 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0xf36e6f7, 0x61c69362, 0x93ad1061, + 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, 0xeb1fcbad, + 0x197448ae, 0xa24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, + 0xa55230e6, 0xfb410cc2, 0x92a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, + 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, + 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x82f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, 0x92a8fc17, 0x60c37f14, 0x73938ce0, + 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, + 0xf94ad42f, 0xb21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, + 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0xe330a81, + 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, + 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, + 0xd3d3e1a, 0x1e6dcdee, 0xec064eed, 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, + 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, + 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x7198540, 0x590ab964, + 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x20bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, + 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, + 0x4f48173d, 0xbd23943e, 0xf36e6f75, 0x105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, + 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 +] diff --git a/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/INSTALLER b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/METADATA b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..8a2f639061cc4a203f7109d8335d28076442c61d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/METADATA @@ -0,0 +1,202 @@ +Metadata-Version: 2.4 +Name: h11 +Version: 0.16.0 +Summary: A pure-Python, bring-your-own-I/O implementation of HTTP/1.1 +Home-page: https://github.com/python-hyper/h11 +Author: Nathaniel J. Smith +Author-email: njs@pobox.com +License: MIT +Classifier: Development Status :: 3 - Alpha +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Topic :: Internet :: WWW/HTTP +Classifier: Topic :: System :: Networking +Requires-Python: >=3.8 +License-File: LICENSE.txt +Dynamic: author +Dynamic: author-email +Dynamic: classifier +Dynamic: description +Dynamic: home-page +Dynamic: license +Dynamic: license-file +Dynamic: requires-python +Dynamic: summary + +h11 +=== + +.. image:: https://travis-ci.org/python-hyper/h11.svg?branch=master + :target: https://travis-ci.org/python-hyper/h11 + :alt: Automated test status + +.. image:: https://codecov.io/gh/python-hyper/h11/branch/master/graph/badge.svg + :target: https://codecov.io/gh/python-hyper/h11 + :alt: Test coverage + +.. image:: https://readthedocs.org/projects/h11/badge/?version=latest + :target: http://h11.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +This is a little HTTP/1.1 library written from scratch in Python, +heavily inspired by `hyper-h2 <https://hyper-h2.readthedocs.io/>`_. + +It's a "bring-your-own-I/O" library; h11 contains no IO code +whatsoever. This means you can hook h11 up to your favorite network +API, and that could be anything you want: synchronous, threaded, +asynchronous, or your own implementation of `RFC 6214 +<https://tools.ietf.org/html/rfc6214>`_ -- h11 won't judge you. +(Compare this to the current state of the art, where every time a `new +network API <https://trio.readthedocs.io/>`_ comes along then someone +gets to start over reimplementing the entire HTTP protocol from +scratch.) Cory Benfield made an `excellent blog post describing the +benefits of this approach +<https://lukasa.co.uk/2015/10/The_New_Hyper/>`_, or if you like video +then here's his `PyCon 2016 talk on the same theme +<https://www.youtube.com/watch?v=7cC3_jGwl_U>`_. + +This also means that h11 is not immediately useful out of the box: +it's a toolkit for building programs that speak HTTP, not something +that could directly replace ``requests`` or ``twisted.web`` or +whatever. But h11 makes it much easier to implement something like +``requests`` or ``twisted.web``. + +At a high level, working with h11 goes like this: + +1) First, create an ``h11.Connection`` object to track the state of a + single HTTP/1.1 connection. + +2) When you read data off the network, pass it to + ``conn.receive_data(...)``; you'll get back a list of objects + representing high-level HTTP "events". + +3) When you want to send a high-level HTTP event, create the + corresponding "event" object and pass it to ``conn.send(...)``; + this will give you back some bytes that you can then push out + through the network. + +For example, a client might instantiate and then send a +``h11.Request`` object, then zero or more ``h11.Data`` objects for the +request body (e.g., if this is a POST), and then a +``h11.EndOfMessage`` to indicate the end of the message. Then the +server would then send back a ``h11.Response``, some ``h11.Data``, and +its own ``h11.EndOfMessage``. If either side violates the protocol, +you'll get a ``h11.ProtocolError`` exception. + +h11 is suitable for implementing both servers and clients, and has a +pleasantly symmetric API: the events you send as a client are exactly +the ones that you receive as a server and vice-versa. + +`Here's an example of a tiny HTTP client +<https://github.com/python-hyper/h11/blob/master/examples/basic-client.py>`_ + +It also has `a fine manual <https://h11.readthedocs.io/>`_. + +FAQ +--- + +*Whyyyyy?* + +I wanted to play with HTTP in `Curio +<https://curio.readthedocs.io/en/latest/tutorial.html>`__ and `Trio +<https://trio.readthedocs.io>`__, which at the time didn't have any +HTTP libraries. So I thought, no big deal, Python has, like, a dozen +different implementations of HTTP, surely I can find one that's +reusable. I didn't find one, but I did find Cory's call-to-arms +blog-post. So I figured, well, fine, if I have to implement HTTP from +scratch, at least I can make sure no-one *else* has to ever again. + +*Should I use it?* + +Maybe. You should be aware that it's a very young project. But, it's +feature complete and has an exhaustive test-suite and complete docs, +so the next step is for people to try using it and see how it goes +:-). If you do then please let us know -- if nothing else we'll want +to talk to you before making any incompatible changes! + +*What are the features/limitations?* + +Roughly speaking, it's trying to be a robust, complete, and non-hacky +implementation of the first "chapter" of the HTTP/1.1 spec: `RFC 7230: +HTTP/1.1 Message Syntax and Routing +<https://tools.ietf.org/html/rfc7230>`_. That is, it mostly focuses on +implementing HTTP at the level of taking bytes on and off the wire, +and the headers related to that, and tries to be anal about spec +conformance. It doesn't know about higher-level concerns like URL +routing, conditional GETs, cross-origin cookie policies, or content +negotiation. But it does know how to take care of framing, +cross-version differences in keep-alive handling, and the "obsolete +line folding" rule, so you can focus your energies on the hard / +interesting parts for your application, and it tries to support the +full specification in the sense that any useful HTTP/1.1 conformant +application should be able to use h11. + +It's pure Python, and has no dependencies outside of the standard +library. + +It has a test suite with 100.0% coverage for both statements and +branches. + +Currently it supports Python 3 (testing on 3.8-3.12) and PyPy 3. +The last Python 2-compatible version was h11 0.11.x. +(Originally it had a Cython wrapper for `http-parser +<https://github.com/nodejs/http-parser>`_ and a beautiful nested state +machine implemented with ``yield from`` to postprocess the output. But +I had to take these out -- the new *parser* needs fewer lines-of-code +than the old *parser wrapper*, is written in pure Python, uses no +exotic language syntax, and has more features. It's sad, really; that +old state machine was really slick. I just need a few sentences here +to mourn that.) + +I don't know how fast it is. I haven't benchmarked or profiled it yet, +so it's probably got a few pointless hot spots, and I've been trying +to err on the side of simplicity and robustness instead of +micro-optimization. But at the architectural level I tried hard to +avoid fundamentally bad decisions, e.g., I believe that all the +parsing algorithms remain linear-time even in the face of pathological +input like slowloris, and there are no byte-by-byte loops. (I also +believe that it maintains bounded memory usage in the face of +arbitrary/pathological input.) + +The whole library is ~800 lines-of-code. You can read and understand +the whole thing in less than an hour. Most of the energy invested in +this so far has been spent on trying to keep things simple by +minimizing special-cases and ad hoc state manipulation; even though it +is now quite small and simple, I'm still annoyed that I haven't +figured out how to make it even smaller and simpler. (Unfortunately, +HTTP does not lend itself to simplicity.) + +The API is ~feature complete and I don't expect the general outlines +to change much, but you can't judge an API's ergonomics until you +actually document and use it, so I'd expect some changes in the +details. + +*How do I try it?* + +.. code-block:: sh + + $ pip install h11 + $ git clone git@github.com:python-hyper/h11 + $ cd h11/examples + $ python basic-client.py + +and go from there. + +*License?* + +MIT + +*Code of conduct?* + +Contributors are requested to follow our `code of conduct +<https://github.com/python-hyper/h11/blob/master/CODE_OF_CONDUCT.md>`_ in +all project spaces. diff --git a/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/RECORD b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..8e8d23bccaf5c72a3ac77d1a889dff9ce1caea57 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/RECORD @@ -0,0 +1,19 @@ +h11-0.16.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +h11-0.16.0.dist-info/METADATA,sha256=KPMmCYrAn8unm48YD5YIfIQf4kViFct7hyqcfVzRnWQ,8348 +h11-0.16.0.dist-info/RECORD,, +h11-0.16.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +h11-0.16.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91 +h11-0.16.0.dist-info/licenses/LICENSE.txt,sha256=N9tbuFkm2yikJ6JYZ_ELEjIAOuob5pzLhRE4rbjm82E,1124 +h11-0.16.0.dist-info/top_level.txt,sha256=F7dC4jl3zeh8TGHEPaWJrMbeuoWbS379Gwdi-Yvdcis,4 +h11/__init__.py,sha256=iO1KzkSO42yZ6ffg-VMgbx_ZVTWGUY00nRYEWn-s3kY,1507 +h11/_abnf.py,sha256=ybixr0xsupnkA6GFAyMubuXF6Tc1lb_hF890NgCsfNc,4815 +h11/_connection.py,sha256=k9YRVf6koZqbttBW36xSWaJpWdZwa-xQVU9AHEo9DuI,26863 +h11/_events.py,sha256=I97aXoal1Wu7dkL548BANBUCkOIbe-x5CioYA9IBY14,11792 +h11/_headers.py,sha256=P7D-lBNxHwdLZPLimmYwrPG-9ZkjElvvJZJdZAgSP-4,10412 +h11/_readers.py,sha256=a4RypORUCC3d0q_kxPuBIM7jTD8iLt5X91TH0FsduN4,8590 +h11/_receivebuffer.py,sha256=xrspsdsNgWFxRfQcTXxR8RrdjRXXTK0Io5cQYWpJ1Ws,5252 +h11/_state.py,sha256=_5LG_BGR8FCcFQeBPH-TMHgm_-B-EUcWCnQof_9XjFE,13231 +h11/_util.py,sha256=LWkkjXyJaFlAy6Lt39w73UStklFT5ovcvo0TkY7RYuk,4888 +h11/_version.py,sha256=GVSsbPSPDcOuF6ptfIiXnVJoaEm3ygXbMnqlr_Giahw,686 +h11/_writers.py,sha256=oFKm6PtjeHfbj4RLX7VB7KDc1gIY53gXG3_HR9ltmTA,5081 +h11/py.typed,sha256=sow9soTwP9T_gEAQSVh7Gb8855h04Nwmhs2We-JRgZM,7 diff --git a/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/REQUESTED b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/WHEEL b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..1eb3c49d99559863120cfb8433fc8738fba43ba9 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (78.1.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/licenses/LICENSE.txt b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..8f080eae848f759c9173bfc0c79506357ebe5090 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/licenses/LICENSE.txt @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2016 Nathaniel J. Smith <njs@pobox.com> and other contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/top_level.txt b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..0d24def711344ec6f4da2108f7d5c9261eb35f8b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11-0.16.0.dist-info/top_level.txt @@ -0,0 +1 @@ +h11 diff --git a/.venv/lib/python3.11/site-packages/h11/__init__.py b/.venv/lib/python3.11/site-packages/h11/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..989e92c3458681a6f0be72ae4105ea742750d328 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/__init__.py @@ -0,0 +1,62 @@ +# A highish-level implementation of the HTTP/1.1 wire protocol (RFC 7230), +# containing no networking code at all, loosely modelled on hyper-h2's generic +# implementation of HTTP/2 (and in particular the h2.connection.H2Connection +# class). There's still a bunch of subtle details you need to get right if you +# want to make this actually useful, because it doesn't implement all the +# semantics to check that what you're asking to write to the wire is sensible, +# but at least it gets you out of dealing with the wire itself. + +from h11._connection import Connection, NEED_DATA, PAUSED +from h11._events import ( + ConnectionClosed, + Data, + EndOfMessage, + Event, + InformationalResponse, + Request, + Response, +) +from h11._state import ( + CLIENT, + CLOSED, + DONE, + ERROR, + IDLE, + MIGHT_SWITCH_PROTOCOL, + MUST_CLOSE, + SEND_BODY, + SEND_RESPONSE, + SERVER, + SWITCHED_PROTOCOL, +) +from h11._util import LocalProtocolError, ProtocolError, RemoteProtocolError +from h11._version import __version__ + +PRODUCT_ID = "python-h11/" + __version__ + + +__all__ = ( + "Connection", + "NEED_DATA", + "PAUSED", + "ConnectionClosed", + "Data", + "EndOfMessage", + "Event", + "InformationalResponse", + "Request", + "Response", + "CLIENT", + "CLOSED", + "DONE", + "ERROR", + "IDLE", + "MUST_CLOSE", + "SEND_BODY", + "SEND_RESPONSE", + "SERVER", + "SWITCHED_PROTOCOL", + "ProtocolError", + "LocalProtocolError", + "RemoteProtocolError", +) diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab242a41255023cebf92b2748dcd071e8f811452 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/_abnf.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/_abnf.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5bae32f348aa7ee9706d0d37b3f66272c371ffd1 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/_abnf.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/_connection.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/_connection.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..241eab626a48663dc93391e51ab5dde7cff18613 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/_connection.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/_events.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/_events.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c6e302c8ef819210b2e7cd2658bc58332dd69580 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/_events.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/_headers.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/_headers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..44cb7903ee222ffbfd6a9e1c37ebb1ef57fc95cb Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/_headers.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/_readers.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/_readers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bbec4f056723cb206df22585cacd9e23d3382724 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/_readers.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/_receivebuffer.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/_receivebuffer.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..177e5bcd485a24d0beff6cb1804dbb9b4516996c Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/_receivebuffer.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/_state.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/_state.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5cffdba9d6448e17e252f418b6c602cc2b68bc30 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/_state.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/_util.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/_util.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5796286e26a2702db78e5da3753b487c998ea330 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/_util.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/_version.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/_version.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a08ceaa693e8447cc8bd26ea315f9d3d77b4da8a Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/_version.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/__pycache__/_writers.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h11/__pycache__/_writers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25dbb44eb8b18e8701f96c50596363628d8854f4 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h11/__pycache__/_writers.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h11/_abnf.py b/.venv/lib/python3.11/site-packages/h11/_abnf.py new file mode 100644 index 0000000000000000000000000000000000000000..933587fba22290d7eb7df4c88e12f1e61702b8ce --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/_abnf.py @@ -0,0 +1,132 @@ +# We use native strings for all the re patterns, to take advantage of string +# formatting, and then convert to bytestrings when compiling the final re +# objects. + +# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#whitespace +# OWS = *( SP / HTAB ) +# ; optional whitespace +OWS = r"[ \t]*" + +# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#rule.token.separators +# token = 1*tchar +# +# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" +# / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" +# / DIGIT / ALPHA +# ; any VCHAR, except delimiters +token = r"[-!#$%&'*+.^_`|~0-9a-zA-Z]+" + +# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#header.fields +# field-name = token +field_name = token + +# The standard says: +# +# field-value = *( field-content / obs-fold ) +# field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] +# field-vchar = VCHAR / obs-text +# obs-fold = CRLF 1*( SP / HTAB ) +# ; obsolete line folding +# ; see Section 3.2.4 +# +# https://tools.ietf.org/html/rfc5234#appendix-B.1 +# +# VCHAR = %x21-7E +# ; visible (printing) characters +# +# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#rule.quoted-string +# obs-text = %x80-FF +# +# However, the standard definition of field-content is WRONG! It disallows +# fields containing a single visible character surrounded by whitespace, +# e.g. "foo a bar". +# +# See: https://www.rfc-editor.org/errata_search.php?rfc=7230&eid=4189 +# +# So our definition of field_content attempts to fix it up... +# +# Also, we allow lots of control characters, because apparently people assume +# that they're legal in practice (e.g., google analytics makes cookies with +# \x01 in them!): +# https://github.com/python-hyper/h11/issues/57 +# We still don't allow NUL or whitespace, because those are often treated as +# meta-characters and letting them through can lead to nasty issues like SSRF. +vchar = r"[\x21-\x7e]" +vchar_or_obs_text = r"[^\x00\s]" +field_vchar = vchar_or_obs_text +field_content = r"{field_vchar}+(?:[ \t]+{field_vchar}+)*".format(**globals()) + +# We handle obs-fold at a different level, and our fixed-up field_content +# already grows to swallow the whole value, so ? instead of * +field_value = r"({field_content})?".format(**globals()) + +# header-field = field-name ":" OWS field-value OWS +header_field = ( + r"(?P<field_name>{field_name})" + r":" + r"{OWS}" + r"(?P<field_value>{field_value})" + r"{OWS}".format(**globals()) +) + +# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#request.line +# +# request-line = method SP request-target SP HTTP-version CRLF +# method = token +# HTTP-version = HTTP-name "/" DIGIT "." DIGIT +# HTTP-name = %x48.54.54.50 ; "HTTP", case-sensitive +# +# request-target is complicated (see RFC 7230 sec 5.3) -- could be path, full +# URL, host+port (for connect), or even "*", but in any case we are guaranteed +# that it contists of the visible printing characters. +method = token +request_target = r"{vchar}+".format(**globals()) +http_version = r"HTTP/(?P<http_version>[0-9]\.[0-9])" +request_line = ( + r"(?P<method>{method})" + r" " + r"(?P<target>{request_target})" + r" " + r"{http_version}".format(**globals()) +) + +# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#status.line +# +# status-line = HTTP-version SP status-code SP reason-phrase CRLF +# status-code = 3DIGIT +# reason-phrase = *( HTAB / SP / VCHAR / obs-text ) +status_code = r"[0-9]{3}" +reason_phrase = r"([ \t]|{vchar_or_obs_text})*".format(**globals()) +status_line = ( + r"{http_version}" + r" " + r"(?P<status_code>{status_code})" + # However, there are apparently a few too many servers out there that just + # leave out the reason phrase: + # https://github.com/scrapy/scrapy/issues/345#issuecomment-281756036 + # https://github.com/seanmonstar/httparse/issues/29 + # so make it optional. ?: is a non-capturing group. + r"(?: (?P<reason>{reason_phrase}))?".format(**globals()) +) + +HEXDIG = r"[0-9A-Fa-f]" +# Actually +# +# chunk-size = 1*HEXDIG +# +# but we impose an upper-limit to avoid ridiculosity. len(str(2**64)) == 20 +chunk_size = r"({HEXDIG}){{1,20}}".format(**globals()) +# Actually +# +# chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] ) +# +# but we aren't parsing the things so we don't really care. +chunk_ext = r";.*" +chunk_header = ( + r"(?P<chunk_size>{chunk_size})" + r"(?P<chunk_ext>{chunk_ext})?" + r"{OWS}\r\n".format( + **globals() + ) # Even though the specification does not allow for extra whitespaces, + # we are lenient with trailing whitespaces because some servers on the wild use it. +) diff --git a/.venv/lib/python3.11/site-packages/h11/_connection.py b/.venv/lib/python3.11/site-packages/h11/_connection.py new file mode 100644 index 0000000000000000000000000000000000000000..e37d82a82a882c072cb938a90eb4486b51cdad99 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/_connection.py @@ -0,0 +1,659 @@ +# This contains the main Connection class. Everything in h11 revolves around +# this. +from typing import ( + Any, + Callable, + cast, + Dict, + List, + Optional, + overload, + Tuple, + Type, + Union, +) + +from ._events import ( + ConnectionClosed, + Data, + EndOfMessage, + Event, + InformationalResponse, + Request, + Response, +) +from ._headers import get_comma_header, has_expect_100_continue, set_comma_header +from ._readers import READERS, ReadersType +from ._receivebuffer import ReceiveBuffer +from ._state import ( + _SWITCH_CONNECT, + _SWITCH_UPGRADE, + CLIENT, + ConnectionState, + DONE, + ERROR, + MIGHT_SWITCH_PROTOCOL, + SEND_BODY, + SERVER, + SWITCHED_PROTOCOL, +) +from ._util import ( # Import the internal things we need + LocalProtocolError, + RemoteProtocolError, + Sentinel, +) +from ._writers import WRITERS, WritersType + +# Everything in __all__ gets re-exported as part of the h11 public API. +__all__ = ["Connection", "NEED_DATA", "PAUSED"] + + +class NEED_DATA(Sentinel, metaclass=Sentinel): + pass + + +class PAUSED(Sentinel, metaclass=Sentinel): + pass + + +# If we ever have this much buffered without it making a complete parseable +# event, we error out. The only time we really buffer is when reading the +# request/response line + headers together, so this is effectively the limit on +# the size of that. +# +# Some precedents for defaults: +# - node.js: 80 * 1024 +# - tomcat: 8 * 1024 +# - IIS: 16 * 1024 +# - Apache: <8 KiB per line> +DEFAULT_MAX_INCOMPLETE_EVENT_SIZE = 16 * 1024 + + +# RFC 7230's rules for connection lifecycles: +# - If either side says they want to close the connection, then the connection +# must close. +# - HTTP/1.1 defaults to keep-alive unless someone says Connection: close +# - HTTP/1.0 defaults to close unless both sides say Connection: keep-alive +# (and even this is a mess -- e.g. if you're implementing a proxy then +# sending Connection: keep-alive is forbidden). +# +# We simplify life by simply not supporting keep-alive with HTTP/1.0 peers. So +# our rule is: +# - If someone says Connection: close, we will close +# - If someone uses HTTP/1.0, we will close. +def _keep_alive(event: Union[Request, Response]) -> bool: + connection = get_comma_header(event.headers, b"connection") + if b"close" in connection: + return False + if getattr(event, "http_version", b"1.1") < b"1.1": + return False + return True + + +def _body_framing( + request_method: bytes, event: Union[Request, Response] +) -> Tuple[str, Union[Tuple[()], Tuple[int]]]: + # Called when we enter SEND_BODY to figure out framing information for + # this body. + # + # These are the only two events that can trigger a SEND_BODY state: + assert type(event) in (Request, Response) + # Returns one of: + # + # ("content-length", count) + # ("chunked", ()) + # ("http/1.0", ()) + # + # which are (lookup key, *args) for constructing body reader/writer + # objects. + # + # Reference: https://tools.ietf.org/html/rfc7230#section-3.3.3 + # + # Step 1: some responses always have an empty body, regardless of what the + # headers say. + if type(event) is Response: + if ( + event.status_code in (204, 304) + or request_method == b"HEAD" + or (request_method == b"CONNECT" and 200 <= event.status_code < 300) + ): + return ("content-length", (0,)) + # Section 3.3.3 also lists another case -- responses with status_code + # < 200. For us these are InformationalResponses, not Responses, so + # they can't get into this function in the first place. + assert event.status_code >= 200 + + # Step 2: check for Transfer-Encoding (T-E beats C-L): + transfer_encodings = get_comma_header(event.headers, b"transfer-encoding") + if transfer_encodings: + assert transfer_encodings == [b"chunked"] + return ("chunked", ()) + + # Step 3: check for Content-Length + content_lengths = get_comma_header(event.headers, b"content-length") + if content_lengths: + return ("content-length", (int(content_lengths[0]),)) + + # Step 4: no applicable headers; fallback/default depends on type + if type(event) is Request: + return ("content-length", (0,)) + else: + return ("http/1.0", ()) + + +################################################################ +# +# The main Connection class +# +################################################################ + + +class Connection: + """An object encapsulating the state of an HTTP connection. + + Args: + our_role: If you're implementing a client, pass :data:`h11.CLIENT`. If + you're implementing a server, pass :data:`h11.SERVER`. + + max_incomplete_event_size (int): + The maximum number of bytes we're willing to buffer of an + incomplete event. In practice this mostly sets a limit on the + maximum size of the request/response line + headers. If this is + exceeded, then :meth:`next_event` will raise + :exc:`RemoteProtocolError`. + + """ + + def __init__( + self, + our_role: Type[Sentinel], + max_incomplete_event_size: int = DEFAULT_MAX_INCOMPLETE_EVENT_SIZE, + ) -> None: + self._max_incomplete_event_size = max_incomplete_event_size + # State and role tracking + if our_role not in (CLIENT, SERVER): + raise ValueError(f"expected CLIENT or SERVER, not {our_role!r}") + self.our_role = our_role + self.their_role: Type[Sentinel] + if our_role is CLIENT: + self.their_role = SERVER + else: + self.their_role = CLIENT + self._cstate = ConnectionState() + + # Callables for converting data->events or vice-versa given the + # current state + self._writer = self._get_io_object(self.our_role, None, WRITERS) + self._reader = self._get_io_object(self.their_role, None, READERS) + + # Holds any unprocessed received data + self._receive_buffer = ReceiveBuffer() + # If this is true, then it indicates that the incoming connection was + # closed *after* the end of whatever's in self._receive_buffer: + self._receive_buffer_closed = False + + # Extra bits of state that don't fit into the state machine. + # + # These two are only used to interpret framing headers for figuring + # out how to read/write response bodies. their_http_version is also + # made available as a convenient public API. + self.their_http_version: Optional[bytes] = None + self._request_method: Optional[bytes] = None + # This is pure flow-control and doesn't at all affect the set of legal + # transitions, so no need to bother ConnectionState with it: + self.client_is_waiting_for_100_continue = False + + @property + def states(self) -> Dict[Type[Sentinel], Type[Sentinel]]: + """A dictionary like:: + + {CLIENT: <client state>, SERVER: <server state>} + + See :ref:`state-machine` for details. + + """ + return dict(self._cstate.states) + + @property + def our_state(self) -> Type[Sentinel]: + """The current state of whichever role we are playing. See + :ref:`state-machine` for details. + """ + return self._cstate.states[self.our_role] + + @property + def their_state(self) -> Type[Sentinel]: + """The current state of whichever role we are NOT playing. See + :ref:`state-machine` for details. + """ + return self._cstate.states[self.their_role] + + @property + def they_are_waiting_for_100_continue(self) -> bool: + return self.their_role is CLIENT and self.client_is_waiting_for_100_continue + + def start_next_cycle(self) -> None: + """Attempt to reset our connection state for a new request/response + cycle. + + If both client and server are in :data:`DONE` state, then resets them + both to :data:`IDLE` state in preparation for a new request/response + cycle on this same connection. Otherwise, raises a + :exc:`LocalProtocolError`. + + See :ref:`keepalive-and-pipelining`. + + """ + old_states = dict(self._cstate.states) + self._cstate.start_next_cycle() + self._request_method = None + # self.their_http_version gets left alone, since it presumably lasts + # beyond a single request/response cycle + assert not self.client_is_waiting_for_100_continue + self._respond_to_state_changes(old_states) + + def _process_error(self, role: Type[Sentinel]) -> None: + old_states = dict(self._cstate.states) + self._cstate.process_error(role) + self._respond_to_state_changes(old_states) + + def _server_switch_event(self, event: Event) -> Optional[Type[Sentinel]]: + if type(event) is InformationalResponse and event.status_code == 101: + return _SWITCH_UPGRADE + if type(event) is Response: + if ( + _SWITCH_CONNECT in self._cstate.pending_switch_proposals + and 200 <= event.status_code < 300 + ): + return _SWITCH_CONNECT + return None + + # All events go through here + def _process_event(self, role: Type[Sentinel], event: Event) -> None: + # First, pass the event through the state machine to make sure it + # succeeds. + old_states = dict(self._cstate.states) + if role is CLIENT and type(event) is Request: + if event.method == b"CONNECT": + self._cstate.process_client_switch_proposal(_SWITCH_CONNECT) + if get_comma_header(event.headers, b"upgrade"): + self._cstate.process_client_switch_proposal(_SWITCH_UPGRADE) + server_switch_event = None + if role is SERVER: + server_switch_event = self._server_switch_event(event) + self._cstate.process_event(role, type(event), server_switch_event) + + # Then perform the updates triggered by it. + + if type(event) is Request: + self._request_method = event.method + + if role is self.their_role and type(event) in ( + Request, + Response, + InformationalResponse, + ): + event = cast(Union[Request, Response, InformationalResponse], event) + self.their_http_version = event.http_version + + # Keep alive handling + # + # RFC 7230 doesn't really say what one should do if Connection: close + # shows up on a 1xx InformationalResponse. I think the idea is that + # this is not supposed to happen. In any case, if it does happen, we + # ignore it. + if type(event) in (Request, Response) and not _keep_alive( + cast(Union[Request, Response], event) + ): + self._cstate.process_keep_alive_disabled() + + # 100-continue + if type(event) is Request and has_expect_100_continue(event): + self.client_is_waiting_for_100_continue = True + if type(event) in (InformationalResponse, Response): + self.client_is_waiting_for_100_continue = False + if role is CLIENT and type(event) in (Data, EndOfMessage): + self.client_is_waiting_for_100_continue = False + + self._respond_to_state_changes(old_states, event) + + def _get_io_object( + self, + role: Type[Sentinel], + event: Optional[Event], + io_dict: Union[ReadersType, WritersType], + ) -> Optional[Callable[..., Any]]: + # event may be None; it's only used when entering SEND_BODY + state = self._cstate.states[role] + if state is SEND_BODY: + # Special case: the io_dict has a dict of reader/writer factories + # that depend on the request/response framing. + framing_type, args = _body_framing( + cast(bytes, self._request_method), cast(Union[Request, Response], event) + ) + return io_dict[SEND_BODY][framing_type](*args) # type: ignore[index] + else: + # General case: the io_dict just has the appropriate reader/writer + # for this state + return io_dict.get((role, state)) # type: ignore[return-value] + + # This must be called after any action that might have caused + # self._cstate.states to change. + def _respond_to_state_changes( + self, + old_states: Dict[Type[Sentinel], Type[Sentinel]], + event: Optional[Event] = None, + ) -> None: + # Update reader/writer + if self.our_state != old_states[self.our_role]: + self._writer = self._get_io_object(self.our_role, event, WRITERS) + if self.their_state != old_states[self.their_role]: + self._reader = self._get_io_object(self.their_role, event, READERS) + + @property + def trailing_data(self) -> Tuple[bytes, bool]: + """Data that has been received, but not yet processed, represented as + a tuple with two elements, where the first is a byte-string containing + the unprocessed data itself, and the second is a bool that is True if + the receive connection was closed. + + See :ref:`switching-protocols` for discussion of why you'd want this. + """ + return (bytes(self._receive_buffer), self._receive_buffer_closed) + + def receive_data(self, data: bytes) -> None: + """Add data to our internal receive buffer. + + This does not actually do any processing on the data, just stores + it. To trigger processing, you have to call :meth:`next_event`. + + Args: + data (:term:`bytes-like object`): + The new data that was just received. + + Special case: If *data* is an empty byte-string like ``b""``, + then this indicates that the remote side has closed the + connection (end of file). Normally this is convenient, because + standard Python APIs like :meth:`file.read` or + :meth:`socket.recv` use ``b""`` to indicate end-of-file, while + other failures to read are indicated using other mechanisms + like raising :exc:`TimeoutError`. When using such an API you + can just blindly pass through whatever you get from ``read`` + to :meth:`receive_data`, and everything will work. + + But, if you have an API where reading an empty string is a + valid non-EOF condition, then you need to be aware of this and + make sure to check for such strings and avoid passing them to + :meth:`receive_data`. + + Returns: + Nothing, but after calling this you should call :meth:`next_event` + to parse the newly received data. + + Raises: + RuntimeError: + Raised if you pass an empty *data*, indicating EOF, and then + pass a non-empty *data*, indicating more data that somehow + arrived after the EOF. + + (Calling ``receive_data(b"")`` multiple times is fine, + and equivalent to calling it once.) + + """ + if data: + if self._receive_buffer_closed: + raise RuntimeError("received close, then received more data?") + self._receive_buffer += data + else: + self._receive_buffer_closed = True + + def _extract_next_receive_event( + self, + ) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]: + state = self.their_state + # We don't pause immediately when they enter DONE, because even in + # DONE state we can still process a ConnectionClosed() event. But + # if we have data in our buffer, then we definitely aren't getting + # a ConnectionClosed() immediately and we need to pause. + if state is DONE and self._receive_buffer: + return PAUSED + if state is MIGHT_SWITCH_PROTOCOL or state is SWITCHED_PROTOCOL: + return PAUSED + assert self._reader is not None + event = self._reader(self._receive_buffer) + if event is None: + if not self._receive_buffer and self._receive_buffer_closed: + # In some unusual cases (basically just HTTP/1.0 bodies), EOF + # triggers an actual protocol event; in that case, we want to + # return that event, and then the state will change and we'll + # get called again to generate the actual ConnectionClosed(). + if hasattr(self._reader, "read_eof"): + event = self._reader.read_eof() + else: + event = ConnectionClosed() + if event is None: + event = NEED_DATA + return event # type: ignore[no-any-return] + + def next_event(self) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]: + """Parse the next event out of our receive buffer, update our internal + state, and return it. + + This is a mutating operation -- think of it like calling :func:`next` + on an iterator. + + Returns: + : One of three things: + + 1) An event object -- see :ref:`events`. + + 2) The special constant :data:`NEED_DATA`, which indicates that + you need to read more data from your socket and pass it to + :meth:`receive_data` before this method will be able to return + any more events. + + 3) The special constant :data:`PAUSED`, which indicates that we + are not in a state where we can process incoming data (usually + because the peer has finished their part of the current + request/response cycle, and you have not yet called + :meth:`start_next_cycle`). See :ref:`flow-control` for details. + + Raises: + RemoteProtocolError: + The peer has misbehaved. You should close the connection + (possibly after sending some kind of 4xx response). + + Once this method returns :class:`ConnectionClosed` once, then all + subsequent calls will also return :class:`ConnectionClosed`. + + If this method raises any exception besides :exc:`RemoteProtocolError` + then that's a bug -- if it happens please file a bug report! + + If this method raises any exception then it also sets + :attr:`Connection.their_state` to :data:`ERROR` -- see + :ref:`error-handling` for discussion. + + """ + + if self.their_state is ERROR: + raise RemoteProtocolError("Can't receive data when peer state is ERROR") + try: + event = self._extract_next_receive_event() + if event not in [NEED_DATA, PAUSED]: + self._process_event(self.their_role, cast(Event, event)) + if event is NEED_DATA: + if len(self._receive_buffer) > self._max_incomplete_event_size: + # 431 is "Request header fields too large" which is pretty + # much the only situation where we can get here + raise RemoteProtocolError( + "Receive buffer too long", error_status_hint=431 + ) + if self._receive_buffer_closed: + # We're still trying to complete some event, but that's + # never going to happen because no more data is coming + raise RemoteProtocolError("peer unexpectedly closed connection") + return event + except BaseException as exc: + self._process_error(self.their_role) + if isinstance(exc, LocalProtocolError): + exc._reraise_as_remote_protocol_error() + else: + raise + + @overload + def send(self, event: ConnectionClosed) -> None: + ... + + @overload + def send( + self, event: Union[Request, InformationalResponse, Response, Data, EndOfMessage] + ) -> bytes: + ... + + @overload + def send(self, event: Event) -> Optional[bytes]: + ... + + def send(self, event: Event) -> Optional[bytes]: + """Convert a high-level event into bytes that can be sent to the peer, + while updating our internal state machine. + + Args: + event: The :ref:`event <events>` to send. + + Returns: + If ``type(event) is ConnectionClosed``, then returns + ``None``. Otherwise, returns a :term:`bytes-like object`. + + Raises: + LocalProtocolError: + Sending this event at this time would violate our + understanding of the HTTP/1.1 protocol. + + If this method raises any exception then it also sets + :attr:`Connection.our_state` to :data:`ERROR` -- see + :ref:`error-handling` for discussion. + + """ + data_list = self.send_with_data_passthrough(event) + if data_list is None: + return None + else: + return b"".join(data_list) + + def send_with_data_passthrough(self, event: Event) -> Optional[List[bytes]]: + """Identical to :meth:`send`, except that in situations where + :meth:`send` returns a single :term:`bytes-like object`, this instead + returns a list of them -- and when sending a :class:`Data` event, this + list is guaranteed to contain the exact object you passed in as + :attr:`Data.data`. See :ref:`sendfile` for discussion. + + """ + if self.our_state is ERROR: + raise LocalProtocolError("Can't send data when our state is ERROR") + try: + if type(event) is Response: + event = self._clean_up_response_headers_for_sending(event) + # We want to call _process_event before calling the writer, + # because if someone tries to do something invalid then this will + # give a sensible error message, while our writers all just assume + # they will only receive valid events. But, _process_event might + # change self._writer. So we have to do a little dance: + writer = self._writer + self._process_event(self.our_role, event) + if type(event) is ConnectionClosed: + return None + else: + # In any situation where writer is None, process_event should + # have raised ProtocolError + assert writer is not None + data_list: List[bytes] = [] + writer(event, data_list.append) + return data_list + except: + self._process_error(self.our_role) + raise + + def send_failed(self) -> None: + """Notify the state machine that we failed to send the data it gave + us. + + This causes :attr:`Connection.our_state` to immediately become + :data:`ERROR` -- see :ref:`error-handling` for discussion. + + """ + self._process_error(self.our_role) + + # When sending a Response, we take responsibility for a few things: + # + # - Sometimes you MUST set Connection: close. We take care of those + # times. (You can also set it yourself if you want, and if you do then + # we'll respect that and close the connection at the right time. But you + # don't have to worry about that unless you want to.) + # + # - The user has to set Content-Length if they want it. Otherwise, for + # responses that have bodies (e.g. not HEAD), then we will automatically + # select the right mechanism for streaming a body of unknown length, + # which depends on depending on the peer's HTTP version. + # + # This function's *only* responsibility is making sure headers are set up + # right -- everything downstream just looks at the headers. There are no + # side channels. + def _clean_up_response_headers_for_sending(self, response: Response) -> Response: + assert type(response) is Response + + headers = response.headers + need_close = False + + # HEAD requests need some special handling: they always act like they + # have Content-Length: 0, and that's how _body_framing treats + # them. But their headers are supposed to match what we would send if + # the request was a GET. (Technically there is one deviation allowed: + # we're allowed to leave out the framing headers -- see + # https://tools.ietf.org/html/rfc7231#section-4.3.2 . But it's just as + # easy to get them right.) + method_for_choosing_headers = cast(bytes, self._request_method) + if method_for_choosing_headers == b"HEAD": + method_for_choosing_headers = b"GET" + framing_type, _ = _body_framing(method_for_choosing_headers, response) + if framing_type in ("chunked", "http/1.0"): + # This response has a body of unknown length. + # If our peer is HTTP/1.1, we use Transfer-Encoding: chunked + # If our peer is HTTP/1.0, we use no framing headers, and close the + # connection afterwards. + # + # Make sure to clear Content-Length (in principle user could have + # set both and then we ignored Content-Length b/c + # Transfer-Encoding overwrote it -- this would be naughty of them, + # but the HTTP spec says that if our peer does this then we have + # to fix it instead of erroring out, so we'll accord the user the + # same respect). + headers = set_comma_header(headers, b"content-length", []) + if self.their_http_version is None or self.their_http_version < b"1.1": + # Either we never got a valid request and are sending back an + # error (their_http_version is None), so we assume the worst; + # or else we did get a valid HTTP/1.0 request, so we know that + # they don't understand chunked encoding. + headers = set_comma_header(headers, b"transfer-encoding", []) + # This is actually redundant ATM, since currently we + # unconditionally disable keep-alive when talking to HTTP/1.0 + # peers. But let's be defensive just in case we add + # Connection: keep-alive support later: + if self._request_method != b"HEAD": + need_close = True + else: + headers = set_comma_header(headers, b"transfer-encoding", [b"chunked"]) + + if not self._cstate.keep_alive or need_close: + # Make sure Connection: close is set + connection = set(get_comma_header(headers, b"connection")) + connection.discard(b"keep-alive") + connection.add(b"close") + headers = set_comma_header(headers, b"connection", sorted(connection)) + + return Response( + headers=headers, + status_code=response.status_code, + http_version=response.http_version, + reason=response.reason, + ) diff --git a/.venv/lib/python3.11/site-packages/h11/_events.py b/.venv/lib/python3.11/site-packages/h11/_events.py new file mode 100644 index 0000000000000000000000000000000000000000..ca1c3adbde2c4e7710482a18e3471f91f1da610e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/_events.py @@ -0,0 +1,369 @@ +# High level events that make up HTTP/1.1 conversations. Loosely inspired by +# the corresponding events in hyper-h2: +# +# http://python-hyper.org/h2/en/stable/api.html#events +# +# Don't subclass these. Stuff will break. + +import re +from abc import ABC +from dataclasses import dataclass +from typing import List, Tuple, Union + +from ._abnf import method, request_target +from ._headers import Headers, normalize_and_validate +from ._util import bytesify, LocalProtocolError, validate + +# Everything in __all__ gets re-exported as part of the h11 public API. +__all__ = [ + "Event", + "Request", + "InformationalResponse", + "Response", + "Data", + "EndOfMessage", + "ConnectionClosed", +] + +method_re = re.compile(method.encode("ascii")) +request_target_re = re.compile(request_target.encode("ascii")) + + +class Event(ABC): + """ + Base class for h11 events. + """ + + __slots__ = () + + +@dataclass(init=False, frozen=True) +class Request(Event): + """The beginning of an HTTP request. + + Fields: + + .. attribute:: method + + An HTTP method, e.g. ``b"GET"`` or ``b"POST"``. Always a byte + string. :term:`Bytes-like objects <bytes-like object>` and native + strings containing only ascii characters will be automatically + converted to byte strings. + + .. attribute:: target + + The target of an HTTP request, e.g. ``b"/index.html"``, or one of the + more exotic formats described in `RFC 7320, section 5.3 + <https://tools.ietf.org/html/rfc7230#section-5.3>`_. Always a byte + string. :term:`Bytes-like objects <bytes-like object>` and native + strings containing only ascii characters will be automatically + converted to byte strings. + + .. attribute:: headers + + Request headers, represented as a list of (name, value) pairs. See + :ref:`the header normalization rules <headers-format>` for details. + + .. attribute:: http_version + + The HTTP protocol version, represented as a byte string like + ``b"1.1"``. See :ref:`the HTTP version normalization rules + <http_version-format>` for details. + + """ + + __slots__ = ("method", "headers", "target", "http_version") + + method: bytes + headers: Headers + target: bytes + http_version: bytes + + def __init__( + self, + *, + method: Union[bytes, str], + headers: Union[Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]]], + target: Union[bytes, str], + http_version: Union[bytes, str] = b"1.1", + _parsed: bool = False, + ) -> None: + super().__init__() + if isinstance(headers, Headers): + object.__setattr__(self, "headers", headers) + else: + object.__setattr__( + self, "headers", normalize_and_validate(headers, _parsed=_parsed) + ) + if not _parsed: + object.__setattr__(self, "method", bytesify(method)) + object.__setattr__(self, "target", bytesify(target)) + object.__setattr__(self, "http_version", bytesify(http_version)) + else: + object.__setattr__(self, "method", method) + object.__setattr__(self, "target", target) + object.__setattr__(self, "http_version", http_version) + + # "A server MUST respond with a 400 (Bad Request) status code to any + # HTTP/1.1 request message that lacks a Host header field and to any + # request message that contains more than one Host header field or a + # Host header field with an invalid field-value." + # -- https://tools.ietf.org/html/rfc7230#section-5.4 + host_count = 0 + for name, value in self.headers: + if name == b"host": + host_count += 1 + if self.http_version == b"1.1" and host_count == 0: + raise LocalProtocolError("Missing mandatory Host: header") + if host_count > 1: + raise LocalProtocolError("Found multiple Host: headers") + + validate(method_re, self.method, "Illegal method characters") + validate(request_target_re, self.target, "Illegal target characters") + + # This is an unhashable type. + __hash__ = None # type: ignore + + +@dataclass(init=False, frozen=True) +class _ResponseBase(Event): + __slots__ = ("headers", "http_version", "reason", "status_code") + + headers: Headers + http_version: bytes + reason: bytes + status_code: int + + def __init__( + self, + *, + headers: Union[Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]]], + status_code: int, + http_version: Union[bytes, str] = b"1.1", + reason: Union[bytes, str] = b"", + _parsed: bool = False, + ) -> None: + super().__init__() + if isinstance(headers, Headers): + object.__setattr__(self, "headers", headers) + else: + object.__setattr__( + self, "headers", normalize_and_validate(headers, _parsed=_parsed) + ) + if not _parsed: + object.__setattr__(self, "reason", bytesify(reason)) + object.__setattr__(self, "http_version", bytesify(http_version)) + if not isinstance(status_code, int): + raise LocalProtocolError("status code must be integer") + # Because IntEnum objects are instances of int, but aren't + # duck-compatible (sigh), see gh-72. + object.__setattr__(self, "status_code", int(status_code)) + else: + object.__setattr__(self, "reason", reason) + object.__setattr__(self, "http_version", http_version) + object.__setattr__(self, "status_code", status_code) + + self.__post_init__() + + def __post_init__(self) -> None: + pass + + # This is an unhashable type. + __hash__ = None # type: ignore + + +@dataclass(init=False, frozen=True) +class InformationalResponse(_ResponseBase): + """An HTTP informational response. + + Fields: + + .. attribute:: status_code + + The status code of this response, as an integer. For an + :class:`InformationalResponse`, this is always in the range [100, + 200). + + .. attribute:: headers + + Request headers, represented as a list of (name, value) pairs. See + :ref:`the header normalization rules <headers-format>` for + details. + + .. attribute:: http_version + + The HTTP protocol version, represented as a byte string like + ``b"1.1"``. See :ref:`the HTTP version normalization rules + <http_version-format>` for details. + + .. attribute:: reason + + The reason phrase of this response, as a byte string. For example: + ``b"OK"``, or ``b"Not Found"``. + + """ + + def __post_init__(self) -> None: + if not (100 <= self.status_code < 200): + raise LocalProtocolError( + "InformationalResponse status_code should be in range " + "[100, 200), not {}".format(self.status_code) + ) + + # This is an unhashable type. + __hash__ = None # type: ignore + + +@dataclass(init=False, frozen=True) +class Response(_ResponseBase): + """The beginning of an HTTP response. + + Fields: + + .. attribute:: status_code + + The status code of this response, as an integer. For an + :class:`Response`, this is always in the range [200, + 1000). + + .. attribute:: headers + + Request headers, represented as a list of (name, value) pairs. See + :ref:`the header normalization rules <headers-format>` for details. + + .. attribute:: http_version + + The HTTP protocol version, represented as a byte string like + ``b"1.1"``. See :ref:`the HTTP version normalization rules + <http_version-format>` for details. + + .. attribute:: reason + + The reason phrase of this response, as a byte string. For example: + ``b"OK"``, or ``b"Not Found"``. + + """ + + def __post_init__(self) -> None: + if not (200 <= self.status_code < 1000): + raise LocalProtocolError( + "Response status_code should be in range [200, 1000), not {}".format( + self.status_code + ) + ) + + # This is an unhashable type. + __hash__ = None # type: ignore + + +@dataclass(init=False, frozen=True) +class Data(Event): + """Part of an HTTP message body. + + Fields: + + .. attribute:: data + + A :term:`bytes-like object` containing part of a message body. Or, if + using the ``combine=False`` argument to :meth:`Connection.send`, then + any object that your socket writing code knows what to do with, and for + which calling :func:`len` returns the number of bytes that will be + written -- see :ref:`sendfile` for details. + + .. attribute:: chunk_start + + A marker that indicates whether this data object is from the start of a + chunked transfer encoding chunk. This field is ignored when when a Data + event is provided to :meth:`Connection.send`: it is only valid on + events emitted from :meth:`Connection.next_event`. You probably + shouldn't use this attribute at all; see + :ref:`chunk-delimiters-are-bad` for details. + + .. attribute:: chunk_end + + A marker that indicates whether this data object is the last for a + given chunked transfer encoding chunk. This field is ignored when when + a Data event is provided to :meth:`Connection.send`: it is only valid + on events emitted from :meth:`Connection.next_event`. You probably + shouldn't use this attribute at all; see + :ref:`chunk-delimiters-are-bad` for details. + + """ + + __slots__ = ("data", "chunk_start", "chunk_end") + + data: bytes + chunk_start: bool + chunk_end: bool + + def __init__( + self, data: bytes, chunk_start: bool = False, chunk_end: bool = False + ) -> None: + object.__setattr__(self, "data", data) + object.__setattr__(self, "chunk_start", chunk_start) + object.__setattr__(self, "chunk_end", chunk_end) + + # This is an unhashable type. + __hash__ = None # type: ignore + + +# XX FIXME: "A recipient MUST ignore (or consider as an error) any fields that +# are forbidden to be sent in a trailer, since processing them as if they were +# present in the header section might bypass external security filters." +# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#chunked.trailer.part +# Unfortunately, the list of forbidden fields is long and vague :-/ +@dataclass(init=False, frozen=True) +class EndOfMessage(Event): + """The end of an HTTP message. + + Fields: + + .. attribute:: headers + + Default value: ``[]`` + + Any trailing headers attached to this message, represented as a list of + (name, value) pairs. See :ref:`the header normalization rules + <headers-format>` for details. + + Must be empty unless ``Transfer-Encoding: chunked`` is in use. + + """ + + __slots__ = ("headers",) + + headers: Headers + + def __init__( + self, + *, + headers: Union[ + Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]], None + ] = None, + _parsed: bool = False, + ) -> None: + super().__init__() + if headers is None: + headers = Headers([]) + elif not isinstance(headers, Headers): + headers = normalize_and_validate(headers, _parsed=_parsed) + + object.__setattr__(self, "headers", headers) + + # This is an unhashable type. + __hash__ = None # type: ignore + + +@dataclass(frozen=True) +class ConnectionClosed(Event): + """This event indicates that the sender has closed their outgoing + connection. + + Note that this does not necessarily mean that they can't *receive* further + data, because TCP connections are composed to two one-way channels which + can be closed independently. See :ref:`closing` for details. + + No fields. + """ + + pass diff --git a/.venv/lib/python3.11/site-packages/h11/_headers.py b/.venv/lib/python3.11/site-packages/h11/_headers.py new file mode 100644 index 0000000000000000000000000000000000000000..31da3e2b23b55a624b36f105e62a6902e63286aa --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/_headers.py @@ -0,0 +1,282 @@ +import re +from typing import AnyStr, cast, List, overload, Sequence, Tuple, TYPE_CHECKING, Union + +from ._abnf import field_name, field_value +from ._util import bytesify, LocalProtocolError, validate + +if TYPE_CHECKING: + from ._events import Request + +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal # type: ignore + +CONTENT_LENGTH_MAX_DIGITS = 20 # allow up to 1 billion TB - 1 + + +# Facts +# ----- +# +# Headers are: +# keys: case-insensitive ascii +# values: mixture of ascii and raw bytes +# +# "Historically, HTTP has allowed field content with text in the ISO-8859-1 +# charset [ISO-8859-1], supporting other charsets only through use of +# [RFC2047] encoding. In practice, most HTTP header field values use only a +# subset of the US-ASCII charset [USASCII]. Newly defined header fields SHOULD +# limit their field values to US-ASCII octets. A recipient SHOULD treat other +# octets in field content (obs-text) as opaque data." +# And it deprecates all non-ascii values +# +# Leading/trailing whitespace in header names is forbidden +# +# Values get leading/trailing whitespace stripped +# +# Content-Disposition actually needs to contain unicode semantically; to +# accomplish this it has a terrifically weird way of encoding the filename +# itself as ascii (and even this still has lots of cross-browser +# incompatibilities) +# +# Order is important: +# "a proxy MUST NOT change the order of these field values when forwarding a +# message" +# (and there are several headers where the order indicates a preference) +# +# Multiple occurences of the same header: +# "A sender MUST NOT generate multiple header fields with the same field name +# in a message unless either the entire field value for that header field is +# defined as a comma-separated list [or the header is Set-Cookie which gets a +# special exception]" - RFC 7230. (cookies are in RFC 6265) +# +# So every header aside from Set-Cookie can be merged by b", ".join if it +# occurs repeatedly. But, of course, they can't necessarily be split by +# .split(b","), because quoting. +# +# Given all this mess (case insensitive, duplicates allowed, order is +# important, ...), there doesn't appear to be any standard way to handle +# headers in Python -- they're almost like dicts, but... actually just +# aren't. For now we punt and just use a super simple representation: headers +# are a list of pairs +# +# [(name1, value1), (name2, value2), ...] +# +# where all entries are bytestrings, names are lowercase and have no +# leading/trailing whitespace, and values are bytestrings with no +# leading/trailing whitespace. Searching and updating are done via naive O(n) +# methods. +# +# Maybe a dict-of-lists would be better? + +_content_length_re = re.compile(rb"[0-9]+") +_field_name_re = re.compile(field_name.encode("ascii")) +_field_value_re = re.compile(field_value.encode("ascii")) + + +class Headers(Sequence[Tuple[bytes, bytes]]): + """ + A list-like interface that allows iterating over headers as byte-pairs + of (lowercased-name, value). + + Internally we actually store the representation as three-tuples, + including both the raw original casing, in order to preserve casing + over-the-wire, and the lowercased name, for case-insensitive comparisions. + + r = Request( + method="GET", + target="/", + headers=[("Host", "example.org"), ("Connection", "keep-alive")], + http_version="1.1", + ) + assert r.headers == [ + (b"host", b"example.org"), + (b"connection", b"keep-alive") + ] + assert r.headers.raw_items() == [ + (b"Host", b"example.org"), + (b"Connection", b"keep-alive") + ] + """ + + __slots__ = "_full_items" + + def __init__(self, full_items: List[Tuple[bytes, bytes, bytes]]) -> None: + self._full_items = full_items + + def __bool__(self) -> bool: + return bool(self._full_items) + + def __eq__(self, other: object) -> bool: + return list(self) == list(other) # type: ignore + + def __len__(self) -> int: + return len(self._full_items) + + def __repr__(self) -> str: + return "<Headers(%s)>" % repr(list(self)) + + def __getitem__(self, idx: int) -> Tuple[bytes, bytes]: # type: ignore[override] + _, name, value = self._full_items[idx] + return (name, value) + + def raw_items(self) -> List[Tuple[bytes, bytes]]: + return [(raw_name, value) for raw_name, _, value in self._full_items] + + +HeaderTypes = Union[ + List[Tuple[bytes, bytes]], + List[Tuple[bytes, str]], + List[Tuple[str, bytes]], + List[Tuple[str, str]], +] + + +@overload +def normalize_and_validate(headers: Headers, _parsed: Literal[True]) -> Headers: + ... + + +@overload +def normalize_and_validate(headers: HeaderTypes, _parsed: Literal[False]) -> Headers: + ... + + +@overload +def normalize_and_validate( + headers: Union[Headers, HeaderTypes], _parsed: bool = False +) -> Headers: + ... + + +def normalize_and_validate( + headers: Union[Headers, HeaderTypes], _parsed: bool = False +) -> Headers: + new_headers = [] + seen_content_length = None + saw_transfer_encoding = False + for name, value in headers: + # For headers coming out of the parser, we can safely skip some steps, + # because it always returns bytes and has already run these regexes + # over the data: + if not _parsed: + name = bytesify(name) + value = bytesify(value) + validate(_field_name_re, name, "Illegal header name {!r}", name) + validate(_field_value_re, value, "Illegal header value {!r}", value) + assert isinstance(name, bytes) + assert isinstance(value, bytes) + + raw_name = name + name = name.lower() + if name == b"content-length": + lengths = {length.strip() for length in value.split(b",")} + if len(lengths) != 1: + raise LocalProtocolError("conflicting Content-Length headers") + value = lengths.pop() + validate(_content_length_re, value, "bad Content-Length") + if len(value) > CONTENT_LENGTH_MAX_DIGITS: + raise LocalProtocolError("bad Content-Length") + if seen_content_length is None: + seen_content_length = value + new_headers.append((raw_name, name, value)) + elif seen_content_length != value: + raise LocalProtocolError("conflicting Content-Length headers") + elif name == b"transfer-encoding": + # "A server that receives a request message with a transfer coding + # it does not understand SHOULD respond with 501 (Not + # Implemented)." + # https://tools.ietf.org/html/rfc7230#section-3.3.1 + if saw_transfer_encoding: + raise LocalProtocolError( + "multiple Transfer-Encoding headers", error_status_hint=501 + ) + # "All transfer-coding names are case-insensitive" + # -- https://tools.ietf.org/html/rfc7230#section-4 + value = value.lower() + if value != b"chunked": + raise LocalProtocolError( + "Only Transfer-Encoding: chunked is supported", + error_status_hint=501, + ) + saw_transfer_encoding = True + new_headers.append((raw_name, name, value)) + else: + new_headers.append((raw_name, name, value)) + return Headers(new_headers) + + +def get_comma_header(headers: Headers, name: bytes) -> List[bytes]: + # Should only be used for headers whose value is a list of + # comma-separated, case-insensitive values. + # + # The header name `name` is expected to be lower-case bytes. + # + # Connection: meets these criteria (including cast insensitivity). + # + # Content-Length: technically is just a single value (1*DIGIT), but the + # standard makes reference to implementations that do multiple values, and + # using this doesn't hurt. Ditto, case insensitivity doesn't things either + # way. + # + # Transfer-Encoding: is more complex (allows for quoted strings), so + # splitting on , is actually wrong. For example, this is legal: + # + # Transfer-Encoding: foo; options="1,2", chunked + # + # and should be parsed as + # + # foo; options="1,2" + # chunked + # + # but this naive function will parse it as + # + # foo; options="1 + # 2" + # chunked + # + # However, this is okay because the only thing we are going to do with + # any Transfer-Encoding is reject ones that aren't just "chunked", so + # both of these will be treated the same anyway. + # + # Expect: the only legal value is the literal string + # "100-continue". Splitting on commas is harmless. Case insensitive. + # + out: List[bytes] = [] + for _, found_name, found_raw_value in headers._full_items: + if found_name == name: + found_raw_value = found_raw_value.lower() + for found_split_value in found_raw_value.split(b","): + found_split_value = found_split_value.strip() + if found_split_value: + out.append(found_split_value) + return out + + +def set_comma_header(headers: Headers, name: bytes, new_values: List[bytes]) -> Headers: + # The header name `name` is expected to be lower-case bytes. + # + # Note that when we store the header we use title casing for the header + # names, in order to match the conventional HTTP header style. + # + # Simply calling `.title()` is a blunt approach, but it's correct + # here given the cases where we're using `set_comma_header`... + # + # Connection, Content-Length, Transfer-Encoding. + new_headers: List[Tuple[bytes, bytes]] = [] + for found_raw_name, found_name, found_raw_value in headers._full_items: + if found_name != name: + new_headers.append((found_raw_name, found_raw_value)) + for new_value in new_values: + new_headers.append((name.title(), new_value)) + return normalize_and_validate(new_headers) + + +def has_expect_100_continue(request: "Request") -> bool: + # https://tools.ietf.org/html/rfc7231#section-5.1.1 + # "A server that receives a 100-continue expectation in an HTTP/1.0 request + # MUST ignore that expectation." + if request.http_version < b"1.1": + return False + expect = get_comma_header(request.headers, b"expect") + return b"100-continue" in expect diff --git a/.venv/lib/python3.11/site-packages/h11/_readers.py b/.venv/lib/python3.11/site-packages/h11/_readers.py new file mode 100644 index 0000000000000000000000000000000000000000..576804cc282032526e0a932c9853d586a094bad0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/_readers.py @@ -0,0 +1,250 @@ +# Code to read HTTP data +# +# Strategy: each reader is a callable which takes a ReceiveBuffer object, and +# either: +# 1) consumes some of it and returns an Event +# 2) raises a LocalProtocolError (for consistency -- e.g. we call validate() +# and it might raise a LocalProtocolError, so simpler just to always use +# this) +# 3) returns None, meaning "I need more data" +# +# If they have a .read_eof attribute, then this will be called if an EOF is +# received -- but this is optional. Either way, the actual ConnectionClosed +# event will be generated afterwards. +# +# READERS is a dict describing how to pick a reader. It maps states to either: +# - a reader +# - or, for body readers, a dict of per-framing reader factories + +import re +from typing import Any, Callable, Dict, Iterable, NoReturn, Optional, Tuple, Type, Union + +from ._abnf import chunk_header, header_field, request_line, status_line +from ._events import Data, EndOfMessage, InformationalResponse, Request, Response +from ._receivebuffer import ReceiveBuffer +from ._state import ( + CLIENT, + CLOSED, + DONE, + IDLE, + MUST_CLOSE, + SEND_BODY, + SEND_RESPONSE, + SERVER, +) +from ._util import LocalProtocolError, RemoteProtocolError, Sentinel, validate + +__all__ = ["READERS"] + +header_field_re = re.compile(header_field.encode("ascii")) +obs_fold_re = re.compile(rb"[ \t]+") + + +def _obsolete_line_fold(lines: Iterable[bytes]) -> Iterable[bytes]: + it = iter(lines) + last: Optional[bytes] = None + for line in it: + match = obs_fold_re.match(line) + if match: + if last is None: + raise LocalProtocolError("continuation line at start of headers") + if not isinstance(last, bytearray): + # Cast to a mutable type, avoiding copy on append to ensure O(n) time + last = bytearray(last) + last += b" " + last += line[match.end() :] + else: + if last is not None: + yield last + last = line + if last is not None: + yield last + + +def _decode_header_lines( + lines: Iterable[bytes], +) -> Iterable[Tuple[bytes, bytes]]: + for line in _obsolete_line_fold(lines): + matches = validate(header_field_re, line, "illegal header line: {!r}", line) + yield (matches["field_name"], matches["field_value"]) + + +request_line_re = re.compile(request_line.encode("ascii")) + + +def maybe_read_from_IDLE_client(buf: ReceiveBuffer) -> Optional[Request]: + lines = buf.maybe_extract_lines() + if lines is None: + if buf.is_next_line_obviously_invalid_request_line(): + raise LocalProtocolError("illegal request line") + return None + if not lines: + raise LocalProtocolError("no request line received") + matches = validate( + request_line_re, lines[0], "illegal request line: {!r}", lines[0] + ) + return Request( + headers=list(_decode_header_lines(lines[1:])), _parsed=True, **matches + ) + + +status_line_re = re.compile(status_line.encode("ascii")) + + +def maybe_read_from_SEND_RESPONSE_server( + buf: ReceiveBuffer, +) -> Union[InformationalResponse, Response, None]: + lines = buf.maybe_extract_lines() + if lines is None: + if buf.is_next_line_obviously_invalid_request_line(): + raise LocalProtocolError("illegal request line") + return None + if not lines: + raise LocalProtocolError("no response line received") + matches = validate(status_line_re, lines[0], "illegal status line: {!r}", lines[0]) + http_version = ( + b"1.1" if matches["http_version"] is None else matches["http_version"] + ) + reason = b"" if matches["reason"] is None else matches["reason"] + status_code = int(matches["status_code"]) + class_: Union[Type[InformationalResponse], Type[Response]] = ( + InformationalResponse if status_code < 200 else Response + ) + return class_( + headers=list(_decode_header_lines(lines[1:])), + _parsed=True, + status_code=status_code, + reason=reason, + http_version=http_version, + ) + + +class ContentLengthReader: + def __init__(self, length: int) -> None: + self._length = length + self._remaining = length + + def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: + if self._remaining == 0: + return EndOfMessage() + data = buf.maybe_extract_at_most(self._remaining) + if data is None: + return None + self._remaining -= len(data) + return Data(data=data) + + def read_eof(self) -> NoReturn: + raise RemoteProtocolError( + "peer closed connection without sending complete message body " + "(received {} bytes, expected {})".format( + self._length - self._remaining, self._length + ) + ) + + +chunk_header_re = re.compile(chunk_header.encode("ascii")) + + +class ChunkedReader: + def __init__(self) -> None: + self._bytes_in_chunk = 0 + # After reading a chunk, we have to throw away the trailing \r\n. + # This tracks the bytes that we need to match and throw away. + self._bytes_to_discard = b"" + self._reading_trailer = False + + def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: + if self._reading_trailer: + lines = buf.maybe_extract_lines() + if lines is None: + return None + return EndOfMessage(headers=list(_decode_header_lines(lines))) + if self._bytes_to_discard: + data = buf.maybe_extract_at_most(len(self._bytes_to_discard)) + if data is None: + return None + if data != self._bytes_to_discard[: len(data)]: + raise LocalProtocolError( + f"malformed chunk footer: {data!r} (expected {self._bytes_to_discard!r})" + ) + self._bytes_to_discard = self._bytes_to_discard[len(data) :] + if self._bytes_to_discard: + return None + # else, fall through and read some more + assert self._bytes_to_discard == b"" + if self._bytes_in_chunk == 0: + # We need to refill our chunk count + chunk_header = buf.maybe_extract_next_line() + if chunk_header is None: + return None + matches = validate( + chunk_header_re, + chunk_header, + "illegal chunk header: {!r}", + chunk_header, + ) + # XX FIXME: we discard chunk extensions. Does anyone care? + self._bytes_in_chunk = int(matches["chunk_size"], base=16) + if self._bytes_in_chunk == 0: + self._reading_trailer = True + return self(buf) + chunk_start = True + else: + chunk_start = False + assert self._bytes_in_chunk > 0 + data = buf.maybe_extract_at_most(self._bytes_in_chunk) + if data is None: + return None + self._bytes_in_chunk -= len(data) + if self._bytes_in_chunk == 0: + self._bytes_to_discard = b"\r\n" + chunk_end = True + else: + chunk_end = False + return Data(data=data, chunk_start=chunk_start, chunk_end=chunk_end) + + def read_eof(self) -> NoReturn: + raise RemoteProtocolError( + "peer closed connection without sending complete message body " + "(incomplete chunked read)" + ) + + +class Http10Reader: + def __call__(self, buf: ReceiveBuffer) -> Optional[Data]: + data = buf.maybe_extract_at_most(999999999) + if data is None: + return None + return Data(data=data) + + def read_eof(self) -> EndOfMessage: + return EndOfMessage() + + +def expect_nothing(buf: ReceiveBuffer) -> None: + if buf: + raise LocalProtocolError("Got data when expecting EOF") + return None + + +ReadersType = Dict[ + Union[Type[Sentinel], Tuple[Type[Sentinel], Type[Sentinel]]], + Union[Callable[..., Any], Dict[str, Callable[..., Any]]], +] + +READERS: ReadersType = { + (CLIENT, IDLE): maybe_read_from_IDLE_client, + (SERVER, IDLE): maybe_read_from_SEND_RESPONSE_server, + (SERVER, SEND_RESPONSE): maybe_read_from_SEND_RESPONSE_server, + (CLIENT, DONE): expect_nothing, + (CLIENT, MUST_CLOSE): expect_nothing, + (CLIENT, CLOSED): expect_nothing, + (SERVER, DONE): expect_nothing, + (SERVER, MUST_CLOSE): expect_nothing, + (SERVER, CLOSED): expect_nothing, + SEND_BODY: { + "chunked": ChunkedReader, + "content-length": ContentLengthReader, + "http/1.0": Http10Reader, + }, +} diff --git a/.venv/lib/python3.11/site-packages/h11/_receivebuffer.py b/.venv/lib/python3.11/site-packages/h11/_receivebuffer.py new file mode 100644 index 0000000000000000000000000000000000000000..e5c4e08a56f5081e87103f38b4add6ce1b730204 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/_receivebuffer.py @@ -0,0 +1,153 @@ +import re +import sys +from typing import List, Optional, Union + +__all__ = ["ReceiveBuffer"] + + +# Operations we want to support: +# - find next \r\n or \r\n\r\n (\n or \n\n are also acceptable), +# or wait until there is one +# - read at-most-N bytes +# Goals: +# - on average, do this fast +# - worst case, do this in O(n) where n is the number of bytes processed +# Plan: +# - store bytearray, offset, how far we've searched for a separator token +# - use the how-far-we've-searched data to avoid rescanning +# - while doing a stream of uninterrupted processing, advance offset instead +# of constantly copying +# WARNING: +# - I haven't benchmarked or profiled any of this yet. +# +# Note that starting in Python 3.4, deleting the initial n bytes from a +# bytearray is amortized O(n), thanks to some excellent work by Antoine +# Martin: +# +# https://bugs.python.org/issue19087 +# +# This means that if we only supported 3.4+, we could get rid of the code here +# involving self._start and self.compress, because it's doing exactly the same +# thing that bytearray now does internally. +# +# BUT unfortunately, we still support 2.7, and reading short segments out of a +# long buffer MUST be O(bytes read) to avoid DoS issues, so we can't actually +# delete this code. Yet: +# +# https://pythonclock.org/ +# +# (Two things to double-check first though: make sure PyPy also has the +# optimization, and benchmark to make sure it's a win, since we do have a +# slightly clever thing where we delay calling compress() until we've +# processed a whole event, which could in theory be slightly more efficient +# than the internal bytearray support.) +blank_line_regex = re.compile(b"\n\r?\n", re.MULTILINE) + + +class ReceiveBuffer: + def __init__(self) -> None: + self._data = bytearray() + self._next_line_search = 0 + self._multiple_lines_search = 0 + + def __iadd__(self, byteslike: Union[bytes, bytearray]) -> "ReceiveBuffer": + self._data += byteslike + return self + + def __bool__(self) -> bool: + return bool(len(self)) + + def __len__(self) -> int: + return len(self._data) + + # for @property unprocessed_data + def __bytes__(self) -> bytes: + return bytes(self._data) + + def _extract(self, count: int) -> bytearray: + # extracting an initial slice of the data buffer and return it + out = self._data[:count] + del self._data[:count] + + self._next_line_search = 0 + self._multiple_lines_search = 0 + + return out + + def maybe_extract_at_most(self, count: int) -> Optional[bytearray]: + """ + Extract a fixed number of bytes from the buffer. + """ + out = self._data[:count] + if not out: + return None + + return self._extract(count) + + def maybe_extract_next_line(self) -> Optional[bytearray]: + """ + Extract the first line, if it is completed in the buffer. + """ + # Only search in buffer space that we've not already looked at. + search_start_index = max(0, self._next_line_search - 1) + partial_idx = self._data.find(b"\r\n", search_start_index) + + if partial_idx == -1: + self._next_line_search = len(self._data) + return None + + # + 2 is to compensate len(b"\r\n") + idx = partial_idx + 2 + + return self._extract(idx) + + def maybe_extract_lines(self) -> Optional[List[bytearray]]: + """ + Extract everything up to the first blank line, and return a list of lines. + """ + # Handle the case where we have an immediate empty line. + if self._data[:1] == b"\n": + self._extract(1) + return [] + + if self._data[:2] == b"\r\n": + self._extract(2) + return [] + + # Only search in buffer space that we've not already looked at. + match = blank_line_regex.search(self._data, self._multiple_lines_search) + if match is None: + self._multiple_lines_search = max(0, len(self._data) - 2) + return None + + # Truncate the buffer and return it. + idx = match.span(0)[-1] + out = self._extract(idx) + lines = out.split(b"\n") + + for line in lines: + if line.endswith(b"\r"): + del line[-1] + + assert lines[-2] == lines[-1] == b"" + + del lines[-2:] + + return lines + + # In theory we should wait until `\r\n` before starting to validate + # incoming data. However it's interesting to detect (very) invalid data + # early given they might not even contain `\r\n` at all (hence only + # timeout will get rid of them). + # This is not a 100% effective detection but more of a cheap sanity check + # allowing for early abort in some useful cases. + # This is especially interesting when peer is messing up with HTTPS and + # sent us a TLS stream where we were expecting plain HTTP given all + # versions of TLS so far start handshake with a 0x16 message type code. + def is_next_line_obviously_invalid_request_line(self) -> bool: + try: + # HTTP header line must not contain non-printable characters + # and should not start with a space + return self._data[0] < 0x21 + except IndexError: + return False diff --git a/.venv/lib/python3.11/site-packages/h11/_state.py b/.venv/lib/python3.11/site-packages/h11/_state.py new file mode 100644 index 0000000000000000000000000000000000000000..3ad444b043e3f3d6c05c2d9d84d5119312bfaa34 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/_state.py @@ -0,0 +1,365 @@ +################################################################ +# The core state machine +################################################################ +# +# Rule 1: everything that affects the state machine and state transitions must +# live here in this file. As much as possible goes into the table-based +# representation, but for the bits that don't quite fit, the actual code and +# state must nonetheless live here. +# +# Rule 2: this file does not know about what role we're playing; it only knows +# about HTTP request/response cycles in the abstract. This ensures that we +# don't cheat and apply different rules to local and remote parties. +# +# +# Theory of operation +# =================== +# +# Possibly the simplest way to think about this is that we actually have 5 +# different state machines here. Yes, 5. These are: +# +# 1) The client state, with its complicated automaton (see the docs) +# 2) The server state, with its complicated automaton (see the docs) +# 3) The keep-alive state, with possible states {True, False} +# 4) The SWITCH_CONNECT state, with possible states {False, True} +# 5) The SWITCH_UPGRADE state, with possible states {False, True} +# +# For (3)-(5), the first state listed is the initial state. +# +# (1)-(3) are stored explicitly in member variables. The last +# two are stored implicitly in the pending_switch_proposals set as: +# (state of 4) == (_SWITCH_CONNECT in pending_switch_proposals) +# (state of 5) == (_SWITCH_UPGRADE in pending_switch_proposals) +# +# And each of these machines has two different kinds of transitions: +# +# a) Event-triggered +# b) State-triggered +# +# Event triggered is the obvious thing that you'd think it is: some event +# happens, and if it's the right event at the right time then a transition +# happens. But there are somewhat complicated rules for which machines can +# "see" which events. (As a rule of thumb, if a machine "sees" an event, this +# means two things: the event can affect the machine, and if the machine is +# not in a state where it expects that event then it's an error.) These rules +# are: +# +# 1) The client machine sees all h11.events objects emitted by the client. +# +# 2) The server machine sees all h11.events objects emitted by the server. +# +# It also sees the client's Request event. +# +# And sometimes, server events are annotated with a _SWITCH_* event. For +# example, we can have a (Response, _SWITCH_CONNECT) event, which is +# different from a regular Response event. +# +# 3) The keep-alive machine sees the process_keep_alive_disabled() event +# (which is derived from Request/Response events), and this event +# transitions it from True -> False, or from False -> False. There's no way +# to transition back. +# +# 4&5) The _SWITCH_* machines transition from False->True when we get a +# Request that proposes the relevant type of switch (via +# process_client_switch_proposals), and they go from True->False when we +# get a Response that has no _SWITCH_* annotation. +# +# So that's event-triggered transitions. +# +# State-triggered transitions are less standard. What they do here is couple +# the machines together. The way this works is, when certain *joint* +# configurations of states are achieved, then we automatically transition to a +# new *joint* state. So, for example, if we're ever in a joint state with +# +# client: DONE +# keep-alive: False +# +# then the client state immediately transitions to: +# +# client: MUST_CLOSE +# +# This is fundamentally different from an event-based transition, because it +# doesn't matter how we arrived at the {client: DONE, keep-alive: False} state +# -- maybe the client transitioned SEND_BODY -> DONE, or keep-alive +# transitioned True -> False. Either way, once this precondition is satisfied, +# this transition is immediately triggered. +# +# What if two conflicting state-based transitions get enabled at the same +# time? In practice there's only one case where this arises (client DONE -> +# MIGHT_SWITCH_PROTOCOL versus DONE -> MUST_CLOSE), and we resolve it by +# explicitly prioritizing the DONE -> MIGHT_SWITCH_PROTOCOL transition. +# +# Implementation +# -------------- +# +# The event-triggered transitions for the server and client machines are all +# stored explicitly in a table. Ditto for the state-triggered transitions that +# involve just the server and client state. +# +# The transitions for the other machines, and the state-triggered transitions +# that involve the other machines, are written out as explicit Python code. +# +# It'd be nice if there were some cleaner way to do all this. This isn't +# *too* terrible, but I feel like it could probably be better. +# +# WARNING +# ------- +# +# The script that generates the state machine diagrams for the docs knows how +# to read out the EVENT_TRIGGERED_TRANSITIONS and STATE_TRIGGERED_TRANSITIONS +# tables. But it can't automatically read the transitions that are written +# directly in Python code. So if you touch those, you need to also update the +# script to keep it in sync! +from typing import cast, Dict, Optional, Set, Tuple, Type, Union + +from ._events import * +from ._util import LocalProtocolError, Sentinel + +# Everything in __all__ gets re-exported as part of the h11 public API. +__all__ = [ + "CLIENT", + "SERVER", + "IDLE", + "SEND_RESPONSE", + "SEND_BODY", + "DONE", + "MUST_CLOSE", + "CLOSED", + "MIGHT_SWITCH_PROTOCOL", + "SWITCHED_PROTOCOL", + "ERROR", +] + + +class CLIENT(Sentinel, metaclass=Sentinel): + pass + + +class SERVER(Sentinel, metaclass=Sentinel): + pass + + +# States +class IDLE(Sentinel, metaclass=Sentinel): + pass + + +class SEND_RESPONSE(Sentinel, metaclass=Sentinel): + pass + + +class SEND_BODY(Sentinel, metaclass=Sentinel): + pass + + +class DONE(Sentinel, metaclass=Sentinel): + pass + + +class MUST_CLOSE(Sentinel, metaclass=Sentinel): + pass + + +class CLOSED(Sentinel, metaclass=Sentinel): + pass + + +class ERROR(Sentinel, metaclass=Sentinel): + pass + + +# Switch types +class MIGHT_SWITCH_PROTOCOL(Sentinel, metaclass=Sentinel): + pass + + +class SWITCHED_PROTOCOL(Sentinel, metaclass=Sentinel): + pass + + +class _SWITCH_UPGRADE(Sentinel, metaclass=Sentinel): + pass + + +class _SWITCH_CONNECT(Sentinel, metaclass=Sentinel): + pass + + +EventTransitionType = Dict[ + Type[Sentinel], + Dict[ + Type[Sentinel], + Dict[Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]], Type[Sentinel]], + ], +] + +EVENT_TRIGGERED_TRANSITIONS: EventTransitionType = { + CLIENT: { + IDLE: {Request: SEND_BODY, ConnectionClosed: CLOSED}, + SEND_BODY: {Data: SEND_BODY, EndOfMessage: DONE}, + DONE: {ConnectionClosed: CLOSED}, + MUST_CLOSE: {ConnectionClosed: CLOSED}, + CLOSED: {ConnectionClosed: CLOSED}, + MIGHT_SWITCH_PROTOCOL: {}, + SWITCHED_PROTOCOL: {}, + ERROR: {}, + }, + SERVER: { + IDLE: { + ConnectionClosed: CLOSED, + Response: SEND_BODY, + # Special case: server sees client Request events, in this form + (Request, CLIENT): SEND_RESPONSE, + }, + SEND_RESPONSE: { + InformationalResponse: SEND_RESPONSE, + Response: SEND_BODY, + (InformationalResponse, _SWITCH_UPGRADE): SWITCHED_PROTOCOL, + (Response, _SWITCH_CONNECT): SWITCHED_PROTOCOL, + }, + SEND_BODY: {Data: SEND_BODY, EndOfMessage: DONE}, + DONE: {ConnectionClosed: CLOSED}, + MUST_CLOSE: {ConnectionClosed: CLOSED}, + CLOSED: {ConnectionClosed: CLOSED}, + SWITCHED_PROTOCOL: {}, + ERROR: {}, + }, +} + +StateTransitionType = Dict[ + Tuple[Type[Sentinel], Type[Sentinel]], Dict[Type[Sentinel], Type[Sentinel]] +] + +# NB: there are also some special-case state-triggered transitions hard-coded +# into _fire_state_triggered_transitions below. +STATE_TRIGGERED_TRANSITIONS: StateTransitionType = { + # (Client state, Server state) -> new states + # Protocol negotiation + (MIGHT_SWITCH_PROTOCOL, SWITCHED_PROTOCOL): {CLIENT: SWITCHED_PROTOCOL}, + # Socket shutdown + (CLOSED, DONE): {SERVER: MUST_CLOSE}, + (CLOSED, IDLE): {SERVER: MUST_CLOSE}, + (ERROR, DONE): {SERVER: MUST_CLOSE}, + (DONE, CLOSED): {CLIENT: MUST_CLOSE}, + (IDLE, CLOSED): {CLIENT: MUST_CLOSE}, + (DONE, ERROR): {CLIENT: MUST_CLOSE}, +} + + +class ConnectionState: + def __init__(self) -> None: + # Extra bits of state that don't quite fit into the state model. + + # If this is False then it enables the automatic DONE -> MUST_CLOSE + # transition. Don't set this directly; call .keep_alive_disabled() + self.keep_alive = True + + # This is a subset of {UPGRADE, CONNECT}, containing the proposals + # made by the client for switching protocols. + self.pending_switch_proposals: Set[Type[Sentinel]] = set() + + self.states: Dict[Type[Sentinel], Type[Sentinel]] = {CLIENT: IDLE, SERVER: IDLE} + + def process_error(self, role: Type[Sentinel]) -> None: + self.states[role] = ERROR + self._fire_state_triggered_transitions() + + def process_keep_alive_disabled(self) -> None: + self.keep_alive = False + self._fire_state_triggered_transitions() + + def process_client_switch_proposal(self, switch_event: Type[Sentinel]) -> None: + self.pending_switch_proposals.add(switch_event) + self._fire_state_triggered_transitions() + + def process_event( + self, + role: Type[Sentinel], + event_type: Type[Event], + server_switch_event: Optional[Type[Sentinel]] = None, + ) -> None: + _event_type: Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]] = event_type + if server_switch_event is not None: + assert role is SERVER + if server_switch_event not in self.pending_switch_proposals: + raise LocalProtocolError( + "Received server _SWITCH_UPGRADE event without a pending proposal" + ) + _event_type = (event_type, server_switch_event) + if server_switch_event is None and _event_type is Response: + self.pending_switch_proposals = set() + self._fire_event_triggered_transitions(role, _event_type) + # Special case: the server state does get to see Request + # events. + if _event_type is Request: + assert role is CLIENT + self._fire_event_triggered_transitions(SERVER, (Request, CLIENT)) + self._fire_state_triggered_transitions() + + def _fire_event_triggered_transitions( + self, + role: Type[Sentinel], + event_type: Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]], + ) -> None: + state = self.states[role] + try: + new_state = EVENT_TRIGGERED_TRANSITIONS[role][state][event_type] + except KeyError: + event_type = cast(Type[Event], event_type) + raise LocalProtocolError( + "can't handle event type {} when role={} and state={}".format( + event_type.__name__, role, self.states[role] + ) + ) from None + self.states[role] = new_state + + def _fire_state_triggered_transitions(self) -> None: + # We apply these rules repeatedly until converging on a fixed point + while True: + start_states = dict(self.states) + + # It could happen that both these special-case transitions are + # enabled at the same time: + # + # DONE -> MIGHT_SWITCH_PROTOCOL + # DONE -> MUST_CLOSE + # + # For example, this will always be true of a HTTP/1.0 client + # requesting CONNECT. If this happens, the protocol switch takes + # priority. From there the client will either go to + # SWITCHED_PROTOCOL, in which case it's none of our business when + # they close the connection, or else the server will deny the + # request, in which case the client will go back to DONE and then + # from there to MUST_CLOSE. + if self.pending_switch_proposals: + if self.states[CLIENT] is DONE: + self.states[CLIENT] = MIGHT_SWITCH_PROTOCOL + + if not self.pending_switch_proposals: + if self.states[CLIENT] is MIGHT_SWITCH_PROTOCOL: + self.states[CLIENT] = DONE + + if not self.keep_alive: + for role in (CLIENT, SERVER): + if self.states[role] is DONE: + self.states[role] = MUST_CLOSE + + # Tabular state-triggered transitions + joint_state = (self.states[CLIENT], self.states[SERVER]) + changes = STATE_TRIGGERED_TRANSITIONS.get(joint_state, {}) + self.states.update(changes) + + if self.states == start_states: + # Fixed point reached + return + + def start_next_cycle(self) -> None: + if self.states != {CLIENT: DONE, SERVER: DONE}: + raise LocalProtocolError( + f"not in a reusable state. self.states={self.states}" + ) + # Can't reach DONE/DONE with any of these active, but still, let's be + # sure. + assert self.keep_alive + assert not self.pending_switch_proposals + self.states = {CLIENT: IDLE, SERVER: IDLE} diff --git a/.venv/lib/python3.11/site-packages/h11/_util.py b/.venv/lib/python3.11/site-packages/h11/_util.py new file mode 100644 index 0000000000000000000000000000000000000000..6718445290770e028ea2f1f662026c9a0b0991db --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/_util.py @@ -0,0 +1,135 @@ +from typing import Any, Dict, NoReturn, Pattern, Tuple, Type, TypeVar, Union + +__all__ = [ + "ProtocolError", + "LocalProtocolError", + "RemoteProtocolError", + "validate", + "bytesify", +] + + +class ProtocolError(Exception): + """Exception indicating a violation of the HTTP/1.1 protocol. + + This as an abstract base class, with two concrete base classes: + :exc:`LocalProtocolError`, which indicates that you tried to do something + that HTTP/1.1 says is illegal, and :exc:`RemoteProtocolError`, which + indicates that the remote peer tried to do something that HTTP/1.1 says is + illegal. See :ref:`error-handling` for details. + + In addition to the normal :exc:`Exception` features, it has one attribute: + + .. attribute:: error_status_hint + + This gives a suggestion as to what status code a server might use if + this error occurred as part of a request. + + For a :exc:`RemoteProtocolError`, this is useful as a suggestion for + how you might want to respond to a misbehaving peer, if you're + implementing a server. + + For a :exc:`LocalProtocolError`, this can be taken as a suggestion for + how your peer might have responded to *you* if h11 had allowed you to + continue. + + The default is 400 Bad Request, a generic catch-all for protocol + violations. + + """ + + def __init__(self, msg: str, error_status_hint: int = 400) -> None: + if type(self) is ProtocolError: + raise TypeError("tried to directly instantiate ProtocolError") + Exception.__init__(self, msg) + self.error_status_hint = error_status_hint + + +# Strategy: there are a number of public APIs where a LocalProtocolError can +# be raised (send(), all the different event constructors, ...), and only one +# public API where RemoteProtocolError can be raised +# (receive_data()). Therefore we always raise LocalProtocolError internally, +# and then receive_data will translate this into a RemoteProtocolError. +# +# Internally: +# LocalProtocolError is the generic "ProtocolError". +# Externally: +# LocalProtocolError is for local errors and RemoteProtocolError is for +# remote errors. +class LocalProtocolError(ProtocolError): + def _reraise_as_remote_protocol_error(self) -> NoReturn: + # After catching a LocalProtocolError, use this method to re-raise it + # as a RemoteProtocolError. This method must be called from inside an + # except: block. + # + # An easy way to get an equivalent RemoteProtocolError is just to + # modify 'self' in place. + self.__class__ = RemoteProtocolError # type: ignore + # But the re-raising is somewhat non-trivial -- you might think that + # now that we've modified the in-flight exception object, that just + # doing 'raise' to re-raise it would be enough. But it turns out that + # this doesn't work, because Python tracks the exception type + # (exc_info[0]) separately from the exception object (exc_info[1]), + # and we only modified the latter. So we really do need to re-raise + # the new type explicitly. + # On py3, the traceback is part of the exception object, so our + # in-place modification preserved it and we can just re-raise: + raise self + + +class RemoteProtocolError(ProtocolError): + pass + + +def validate( + regex: Pattern[bytes], data: bytes, msg: str = "malformed data", *format_args: Any +) -> Dict[str, bytes]: + match = regex.fullmatch(data) + if not match: + if format_args: + msg = msg.format(*format_args) + raise LocalProtocolError(msg) + return match.groupdict() + + +# Sentinel values +# +# - Inherit identity-based comparison and hashing from object +# - Have a nice repr +# - Have a *bonus property*: type(sentinel) is sentinel +# +# The bonus property is useful if you want to take the return value from +# next_event() and do some sort of dispatch based on type(event). + +_T_Sentinel = TypeVar("_T_Sentinel", bound="Sentinel") + + +class Sentinel(type): + def __new__( + cls: Type[_T_Sentinel], + name: str, + bases: Tuple[type, ...], + namespace: Dict[str, Any], + **kwds: Any + ) -> _T_Sentinel: + assert bases == (Sentinel,) + v = super().__new__(cls, name, bases, namespace, **kwds) + v.__class__ = v # type: ignore + return v + + def __repr__(self) -> str: + return self.__name__ + + +# Used for methods, request targets, HTTP versions, header names, and header +# values. Accepts ascii-strings, or bytes/bytearray/memoryview/..., and always +# returns bytes. +def bytesify(s: Union[bytes, bytearray, memoryview, int, str]) -> bytes: + # Fast-path: + if type(s) is bytes: + return s + if isinstance(s, str): + s = s.encode("ascii") + if isinstance(s, int): + raise TypeError("expected bytes-like object, not int") + return bytes(s) diff --git a/.venv/lib/python3.11/site-packages/h11/_version.py b/.venv/lib/python3.11/site-packages/h11/_version.py new file mode 100644 index 0000000000000000000000000000000000000000..76e7327b8617c9d12236f511414d5eb58e98a44b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/_version.py @@ -0,0 +1,16 @@ +# This file must be kept very simple, because it is consumed from several +# places -- it is imported by h11/__init__.py, execfile'd by setup.py, etc. + +# We use a simple scheme: +# 1.0.0 -> 1.0.0+dev -> 1.1.0 -> 1.1.0+dev +# where the +dev versions are never released into the wild, they're just what +# we stick into the VCS in between releases. +# +# This is compatible with PEP 440: +# http://legacy.python.org/dev/peps/pep-0440/ +# via the use of the "local suffix" "+dev", which is disallowed on index +# servers and causes 1.0.0+dev to sort after plain 1.0.0, which is what we +# want. (Contrast with the special suffix 1.0.0.dev, which sorts *before* +# 1.0.0.) + +__version__ = "0.16.0" diff --git a/.venv/lib/python3.11/site-packages/h11/_writers.py b/.venv/lib/python3.11/site-packages/h11/_writers.py new file mode 100644 index 0000000000000000000000000000000000000000..939cdb912a9debaea07fbf3a9ac04549c44d077c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/_writers.py @@ -0,0 +1,145 @@ +# Code to read HTTP data +# +# Strategy: each writer takes an event + a write-some-bytes function, which is +# calls. +# +# WRITERS is a dict describing how to pick a reader. It maps states to either: +# - a writer +# - or, for body writers, a dict of framin-dependent writer factories + +from typing import Any, Callable, Dict, List, Tuple, Type, Union + +from ._events import Data, EndOfMessage, Event, InformationalResponse, Request, Response +from ._headers import Headers +from ._state import CLIENT, IDLE, SEND_BODY, SEND_RESPONSE, SERVER +from ._util import LocalProtocolError, Sentinel + +__all__ = ["WRITERS"] + +Writer = Callable[[bytes], Any] + + +def write_headers(headers: Headers, write: Writer) -> None: + # "Since the Host field-value is critical information for handling a + # request, a user agent SHOULD generate Host as the first header field + # following the request-line." - RFC 7230 + raw_items = headers._full_items + for raw_name, name, value in raw_items: + if name == b"host": + write(b"%s: %s\r\n" % (raw_name, value)) + for raw_name, name, value in raw_items: + if name != b"host": + write(b"%s: %s\r\n" % (raw_name, value)) + write(b"\r\n") + + +def write_request(request: Request, write: Writer) -> None: + if request.http_version != b"1.1": + raise LocalProtocolError("I only send HTTP/1.1") + write(b"%s %s HTTP/1.1\r\n" % (request.method, request.target)) + write_headers(request.headers, write) + + +# Shared between InformationalResponse and Response +def write_any_response( + response: Union[InformationalResponse, Response], write: Writer +) -> None: + if response.http_version != b"1.1": + raise LocalProtocolError("I only send HTTP/1.1") + status_bytes = str(response.status_code).encode("ascii") + # We don't bother sending ascii status messages like "OK"; they're + # optional and ignored by the protocol. (But the space after the numeric + # status code is mandatory.) + # + # XX FIXME: could at least make an effort to pull out the status message + # from stdlib's http.HTTPStatus table. Or maybe just steal their enums + # (either by import or copy/paste). We already accept them as status codes + # since they're of type IntEnum < int. + write(b"HTTP/1.1 %s %s\r\n" % (status_bytes, response.reason)) + write_headers(response.headers, write) + + +class BodyWriter: + def __call__(self, event: Event, write: Writer) -> None: + if type(event) is Data: + self.send_data(event.data, write) + elif type(event) is EndOfMessage: + self.send_eom(event.headers, write) + else: # pragma: no cover + assert False + + def send_data(self, data: bytes, write: Writer) -> None: + pass + + def send_eom(self, headers: Headers, write: Writer) -> None: + pass + + +# +# These are all careful not to do anything to 'data' except call len(data) and +# write(data). This allows us to transparently pass-through funny objects, +# like placeholder objects referring to files on disk that will be sent via +# sendfile(2). +# +class ContentLengthWriter(BodyWriter): + def __init__(self, length: int) -> None: + self._length = length + + def send_data(self, data: bytes, write: Writer) -> None: + self._length -= len(data) + if self._length < 0: + raise LocalProtocolError("Too much data for declared Content-Length") + write(data) + + def send_eom(self, headers: Headers, write: Writer) -> None: + if self._length != 0: + raise LocalProtocolError("Too little data for declared Content-Length") + if headers: + raise LocalProtocolError("Content-Length and trailers don't mix") + + +class ChunkedWriter(BodyWriter): + def send_data(self, data: bytes, write: Writer) -> None: + # if we encoded 0-length data in the naive way, it would look like an + # end-of-message. + if not data: + return + write(b"%x\r\n" % len(data)) + write(data) + write(b"\r\n") + + def send_eom(self, headers: Headers, write: Writer) -> None: + write(b"0\r\n") + write_headers(headers, write) + + +class Http10Writer(BodyWriter): + def send_data(self, data: bytes, write: Writer) -> None: + write(data) + + def send_eom(self, headers: Headers, write: Writer) -> None: + if headers: + raise LocalProtocolError("can't send trailers to HTTP/1.0 client") + # no need to close the socket ourselves, that will be taken care of by + # Connection: close machinery + + +WritersType = Dict[ + Union[Tuple[Type[Sentinel], Type[Sentinel]], Type[Sentinel]], + Union[ + Dict[str, Type[BodyWriter]], + Callable[[Union[InformationalResponse, Response], Writer], None], + Callable[[Request, Writer], None], + ], +] + +WRITERS: WritersType = { + (CLIENT, IDLE): write_request, + (SERVER, IDLE): write_any_response, + (SERVER, SEND_RESPONSE): write_any_response, + SEND_BODY: { + "chunked": ChunkedWriter, + "content-length": ContentLengthWriter, + "http/1.0": Http10Writer, + }, +} diff --git a/.venv/lib/python3.11/site-packages/h11/py.typed b/.venv/lib/python3.11/site-packages/h11/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..f5642f79f21d872f010979dcf6f0c4a415acc19d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h11/py.typed @@ -0,0 +1 @@ +Marker diff --git a/.venv/lib/python3.11/site-packages/h5py/__init__.py b/.venv/lib/python3.11/site-packages/h5py/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..159d228b5886f89eea39eb8f8da73380363a9388 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/__init__.py @@ -0,0 +1,115 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + This is the h5py package, a Python interface to the HDF5 + scientific data format. +""" + +from warnings import warn as _warn +import atexit + + +# --- Library setup ----------------------------------------------------------- + +# When importing from the root of the unpacked tarball or git checkout, +# Python sees the "h5py" source directory and tries to load it, which fails. +# We tried working around this by using "package_dir" but that breaks Cython. +try: + from . import _errors +except ImportError: + import os.path as _op + if _op.exists(_op.join(_op.dirname(__file__), '..', 'setup.py')): + raise ImportError("You cannot import h5py from inside the install directory.\nChange to another directory first.") + else: + raise + +from . import version + +if version.hdf5_version_tuple != version.hdf5_built_version_tuple: + _warn(("h5py is running against HDF5 {0} when it was built against {1}, " + "this may cause problems").format( + '{0}.{1}.{2}'.format(*version.hdf5_version_tuple), + '{0}.{1}.{2}'.format(*version.hdf5_built_version_tuple) + )) + + +_errors.silence_errors() + +from ._conv import register_converters as _register_converters, \ + unregister_converters as _unregister_converters +_register_converters() +atexit.register(_unregister_converters) + +from .h5z import _register_lzf +_register_lzf() + + +# --- Public API -------------------------------------------------------------- + +from . import h5a, h5d, h5ds, h5f, h5fd, h5g, h5r, h5s, h5t, h5p, h5z, h5pl + +from ._hl import filters +from ._hl.base import is_hdf5, HLObject, Empty +from ._hl.files import ( + File, + register_driver, + unregister_driver, + registered_drivers, +) +from ._hl.group import Group, SoftLink, ExternalLink, HardLink +from ._hl.dataset import Dataset +from ._hl.datatype import Datatype +from ._hl.attrs import AttributeManager +from ._hl.vds import VirtualSource, VirtualLayout + +from ._selector import MultiBlockSlice +from .h5 import get_config +from .h5r import Reference, RegionReference +from .h5t import (special_dtype, check_dtype, + vlen_dtype, string_dtype, enum_dtype, ref_dtype, regionref_dtype, + opaque_dtype, complex_compat_dtype, + check_vlen_dtype, check_string_dtype, check_enum_dtype, check_ref_dtype, + check_opaque_dtype, check_complex_dtype, +) +from .h5s import UNLIMITED + +from .version import version as __version__ + + +def run_tests(args=''): + """Run tests with pytest and returns the exit status as an int. + """ + # Lazy-loading of tests package to avoid strong dependency on test + # requirements, e.g. pytest + from .tests import run_tests + return run_tests(args) + + +def enable_ipython_completer(): + """ Call this from an interactive IPython session to enable tab-completion + of group and attribute names. + """ + import sys + if 'IPython' in sys.modules: + ip_running = False + try: + from IPython.core.interactiveshell import InteractiveShell + ip_running = InteractiveShell.initialized() + except ImportError: + # support <ipython-0.11 + from IPython import ipapi as _ipapi + ip_running = _ipapi.get() is not None + except Exception: + pass + if ip_running: + from . import ipy_completer + return ipy_completer.load_ipython_extension() + + raise RuntimeError('Completer must be enabled in active ipython session') diff --git a/.venv/lib/python3.11/site-packages/h5py/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2c84ee97412313d8905389a07739155359da6dbb Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/__pycache__/h5py_warnings.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/__pycache__/h5py_warnings.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3d54d3ce7cc1f520e277cef430e0740b49de8e9 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/__pycache__/h5py_warnings.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/__pycache__/version.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/__pycache__/version.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a010bd1517d2fbcdf8e6ee062bd3587b58ade401 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/__pycache__/version.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_conv.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/_conv.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..11bb14f6b189c4f111e020c15e5593bf526348cc --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_conv.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3a7f5f06320b5a6b1e1d7e2f195ce6838848fae7b749381ec999a4b347574c4 +size 459673 diff --git a/.venv/lib/python3.11/site-packages/h5py/_errors.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/_errors.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..a6fd7b4fef8a6fd67661c24de5e9267faa741789 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_errors.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5c283db60f6b29303056b246afcf6fd174ad67ab48c4b4108778064d9ff1d67 +size 159673 diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__init__.py b/.venv/lib/python3.11/site-packages/h5py/_hl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c54c614ffad4074edefbf438f87f71ff274322f9 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/__init__.py @@ -0,0 +1,15 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + This subpackage implements the high-level interface for h5py. + + Don't manually import things from here; the public API lives directly + in the top-level package namespace. +""" diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d3478ca13c25a1d2d3896372010102a79cb62e60 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/attrs.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/attrs.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..14fb51dd185470fa9326e1ea69daf6fbded26592 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/attrs.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/base.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/base.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b8d41d74a6121e09fa94757f7377ffa6876c835 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/base.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/compat.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/compat.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca077a6905a74bcb21f38f363f44872142c2a954 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/compat.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/dataset.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/dataset.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be0d29157957a8f71841fef42e8a619a63c32a63 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/dataset.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/datatype.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/datatype.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f738acae6f243996bae2971ab39dccbbee4a4485 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/datatype.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/files.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/files.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3cb958435e9bdb88bd92d11f7f5d98f78526386c Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/files.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/filters.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/filters.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f6fdcd48a8f65b63e9741f7ddfd683858e135294 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/filters.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/group.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/group.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b381ea02aa7fcd1a595f77518bff4ea5bcba576 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/group.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/selections.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/selections.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..09276fd1cdac30d066e2e6bfb2b87c19c9ab39fe Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/selections.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/selections2.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/selections2.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c40febf8990e8ae0f06d155cb29d9361103e61c4 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/selections2.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/vds.cpython-311.pyc b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/vds.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38ffdca79d2a3fc249f84ca7a6069e9eb816e296 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/h5py/_hl/__pycache__/vds.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/attrs.py b/.venv/lib/python3.11/site-packages/h5py/_hl/attrs.py new file mode 100644 index 0000000000000000000000000000000000000000..63df29585e58925d94f7b50c795d9efcf92754d5 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/attrs.py @@ -0,0 +1,277 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Implements high-level operations for attributes. + + Provides the AttributeManager class, available on high-level objects + as <obj>.attrs. +""" + +import numpy + +from .. import h5, h5s, h5t, h5a, h5p +from . import base +from .base import phil, with_phil, Empty, is_empty_dataspace, product +from .datatype import Datatype + + +class AttributeManager(base.MutableMappingHDF5, base.CommonStateObject): + + """ + Allows dictionary-style access to an HDF5 object's attributes. + + These are created exclusively by the library and are available as + a Python attribute at <object>.attrs + + Like Group objects, attributes provide a minimal dictionary- + style interface. Anything which can be reasonably converted to a + Numpy array or Numpy scalar can be stored. + + Attributes are automatically created on assignment with the + syntax <obj>.attrs[name] = value, with the HDF5 type automatically + deduced from the value. Existing attributes are overwritten. + + To modify an existing attribute while preserving its type, use the + method modify(). To specify an attribute of a particular type and + shape, use create(). + """ + + def __init__(self, parent): + """ Private constructor. + """ + self._id = parent.id + + @with_phil + def __getitem__(self, name): + """ Read the value of an attribute. + """ + attr = h5a.open(self._id, self._e(name)) + shape = attr.shape + + # shape is None for empty dataspaces + if shape is None: + return Empty(attr.dtype) + + dtype = attr.dtype + + # Do this first, as we'll be fiddling with the dtype for top-level + # array types + htype = h5t.py_create(dtype) + + # NumPy doesn't support top-level array types, so we have to "fake" + # the correct type and shape for the array. For example, consider + # attr.shape == (5,) and attr.dtype == '(3,)f'. Then: + if dtype.subdtype is not None: + subdtype, subshape = dtype.subdtype + shape = attr.shape + subshape # (5, 3) + dtype = subdtype # 'f' + + arr = numpy.zeros(shape, dtype=dtype, order='C') + attr.read(arr, mtype=htype) + + string_info = h5t.check_string_dtype(dtype) + if string_info and (string_info.length is None): + # Vlen strings: convert bytes to Python str + arr = numpy.array([ + b.decode('utf-8', 'surrogateescape') for b in arr.flat + ], dtype=dtype).reshape(arr.shape) + + if arr.ndim == 0: + return arr[()] + return arr + + def get_id(self, name): + """Get a low-level AttrID object for the named attribute. + """ + return h5a.open(self._id, self._e(name)) + + @with_phil + def __setitem__(self, name, value): + """ Set a new attribute, overwriting any existing attribute. + + The type and shape of the attribute are determined from the data. To + use a specific type or shape, or to preserve the type of an attribute, + use the methods create() and modify(). + """ + self.create(name, data=value) + + @with_phil + def __delitem__(self, name): + """ Delete an attribute (which must already exist). """ + h5a.delete(self._id, self._e(name)) + + def create(self, name, data, shape=None, dtype=None): + """ Create a new attribute, overwriting any existing attribute. + + name + Name of the new attribute (required) + data + An array to initialize the attribute (required) + shape + Shape of the attribute. Overrides data.shape if both are + given, in which case the total number of points must be unchanged. + dtype + Data type of the attribute. Overrides data.dtype if both + are given. + """ + name = self._e(name) + + with phil: + # First, make sure we have a NumPy array. We leave the data type + # conversion for HDF5 to perform. + if not isinstance(data, Empty): + data = base.array_for_new_object(data, specified_dtype=dtype) + + if shape is None: + shape = data.shape + elif isinstance(shape, int): + shape = (shape,) + + use_htype = None # If a committed type is given, we must use it + # in the call to h5a.create. + + if isinstance(dtype, Datatype): + use_htype = dtype.id + dtype = dtype.dtype + elif dtype is None: + dtype = data.dtype + else: + dtype = numpy.dtype(dtype) # In case a string, e.g. 'i8' is passed + + original_dtype = dtype # We'll need this for top-level array types + + # Where a top-level array type is requested, we have to do some + # fiddling around to present the data as a smaller array of + # subarrays. + if dtype.subdtype is not None: + + subdtype, subshape = dtype.subdtype + + # Make sure the subshape matches the last N axes' sizes. + if shape[-len(subshape):] != subshape: + raise ValueError("Array dtype shape %s is incompatible with data shape %s" % (subshape, shape)) + + # New "advertised" shape and dtype + shape = shape[0:len(shape)-len(subshape)] + dtype = subdtype + + # Not an array type; make sure to check the number of elements + # is compatible, and reshape if needed. + else: + + if shape is not None and product(shape) != product(data.shape): + raise ValueError("Shape of new attribute conflicts with shape of data") + + if shape != data.shape: + data = data.reshape(shape) + + # We need this to handle special string types. + if not isinstance(data, Empty): + data = numpy.asarray(data, dtype=dtype) + + # Make HDF5 datatype and dataspace for the H5A calls + if use_htype is None: + htype = h5t.py_create(original_dtype, logical=True) + htype2 = h5t.py_create(original_dtype) # Must be bit-for-bit representation rather than logical + else: + htype = use_htype + htype2 = None + + if isinstance(data, Empty): + space = h5s.create(h5s.NULL) + else: + space = h5s.create_simple(shape) + + # For a long time, h5py would create attributes with a random name + # and then rename them, imitating how you can atomically replace + # a file in a filesystem. But HDF5 does not offer atomic replacement + # (you have to delete the existing attribute first), and renaming + # exposes some bugs - see https://github.com/h5py/h5py/issues/1385 + # So we've gone back to the simpler delete & recreate model. + if h5a.exists(self._id, name): + h5a.delete(self._id, name) + + attr = h5a.create(self._id, name, htype, space) + try: + if not isinstance(data, Empty): + attr.write(data, mtype=htype2) + except: + attr.close() + h5a.delete(self._id, name) + raise + attr.close() + + def modify(self, name, value): + """ Change the value of an attribute while preserving its type. + + Differs from __setitem__ in that if the attribute already exists, its + type is preserved. This can be very useful for interacting with + externally generated files. + + If the attribute doesn't exist, it will be automatically created. + """ + with phil: + if name not in self: + self[name] = value + else: + attr = h5a.open(self._id, self._e(name)) + + if is_empty_dataspace(attr): + raise OSError("Empty attributes can't be modified") + + # If the input data is already an array, let HDF5 do the conversion. + # If it's a list or similar, don't make numpy guess a dtype for it. + dt = None if isinstance(value, numpy.ndarray) else attr.dtype + value = numpy.asarray(value, order='C', dtype=dt) + + # Allow the case of () <-> (1,) + if (value.shape != attr.shape) and not \ + (value.size == 1 and product(attr.shape) == 1): + raise TypeError("Shape of data is incompatible with existing attribute") + attr.write(value) + + @with_phil + def __len__(self): + """ Number of attributes attached to the object. """ + # I expect we will not have more than 2**32 attributes + return h5a.get_num_attrs(self._id) + + def __iter__(self): + """ Iterate over the names of attributes. """ + with phil: + + attrlist = [] + def iter_cb(name, *args): + """ Callback to gather attribute names """ + attrlist.append(self._d(name)) + + cpl = self._id.get_create_plist() + crt_order = cpl.get_attr_creation_order() + cpl.close() + if crt_order & h5p.CRT_ORDER_TRACKED: + idx_type = h5.INDEX_CRT_ORDER + else: + idx_type = h5.INDEX_NAME + + h5a.iterate(self._id, iter_cb, index_type=idx_type) + + for name in attrlist: + yield name + + @with_phil + def __contains__(self, name): + """ Determine if an attribute exists, by name. """ + return h5a.exists(self._id, self._e(name)) + + @with_phil + def __repr__(self): + if not self._id: + return "<Attributes of closed HDF5 object>" + return "<Attributes of HDF5 object at %s>" % id(self._id) diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/base.py b/.venv/lib/python3.11/site-packages/h5py/_hl/base.py new file mode 100644 index 0000000000000000000000000000000000000000..298db24dcaebb88b561cda7fb93045e683c349bd --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/base.py @@ -0,0 +1,535 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Implements operations common to all high-level objects (File, etc.). +""" + +from collections.abc import ( + Mapping, MutableMapping, KeysView, ValuesView, ItemsView +) +import os +import posixpath + +import numpy as np + +# The high-level interface is serialized; every public API function & method +# is wrapped in a lock. We reuse the low-level lock because (1) it's fast, +# and (2) it eliminates the possibility of deadlocks due to out-of-order +# lock acquisition. +from .._objects import phil, with_phil +from .. import h5d, h5i, h5r, h5p, h5f, h5t, h5s +from .compat import filename_encode + + +def is_hdf5(fname): + """ Determine if a file is valid HDF5 (False if it doesn't exist). """ + with phil: + fname = os.path.abspath(os.fspath(fname)) + + if os.path.isfile(fname): + return h5f.is_hdf5(filename_encode(fname)) + return False + + +def find_item_type(data): + """Find the item type of a simple object or collection of objects. + + E.g. [[['a']]] -> str + + The focus is on collections where all items have the same type; we'll return + None if that's not the case. + + The aim is to treat numpy arrays of Python objects like normal Python + collections, while treating arrays with specific dtypes differently. + We're also only interested in array-like collections - lists and tuples, + possibly nested - not things like sets or dicts. + """ + if isinstance(data, np.ndarray): + if ( + data.dtype.kind == 'O' + and not h5t.check_string_dtype(data.dtype) + and not h5t.check_vlen_dtype(data.dtype) + ): + item_types = {type(e) for e in data.flat} + else: + return None + elif isinstance(data, (list, tuple)): + item_types = {find_item_type(e) for e in data} + else: + return type(data) + + if len(item_types) != 1: + return None + return item_types.pop() + + +def guess_dtype(data): + """ Attempt to guess an appropriate dtype for the object, returning None + if nothing is appropriate (or if it should be left up the the array + constructor to figure out) + """ + with phil: + item_type = find_item_type(data) + + if item_type is h5r.RegionReference: + return h5t.regionref_dtype + if item_type is h5r.Reference: + return h5t.ref_dtype + + if item_type is bytes: + return h5t.string_dtype(encoding='ascii') + if item_type is str: + return h5t.string_dtype() + + return None + + +def is_float16_dtype(dt): + if dt is None: + return False + + dt = np.dtype(dt) # normalize strings -> np.dtype objects + return dt.kind == 'f' and dt.itemsize == 2 + + +def array_for_new_object(data, specified_dtype=None): + """Prepare an array from data used to create a new dataset or attribute""" + + if not isinstance(specified_dtype, (np.dtype, type(None))): + specified_dtype = np.dtype(specified_dtype) + + # We mostly let HDF5 convert data as necessary when it's written. + # But if we are going to a float16 datatype, pre-convert in python + # to workaround a bug in the conversion. + # https://github.com/h5py/h5py/issues/819 + if is_float16_dtype(specified_dtype): + as_dtype = specified_dtype + elif not isinstance(data, np.ndarray) and (specified_dtype is not None): + # If we need to convert e.g. a list to an array, don't leave numpy + # to guess a dtype we already know. + as_dtype = specified_dtype + else: + as_dtype = guess_dtype(data) + + data = np.asarray(data, order="C", dtype=as_dtype) + + # In most cases, this does nothing. But if data was already an array, + # and as_dtype is a tagged h5py dtype (e.g. for an object array of strings), + # asarray() doesn't replace its dtype object. This gives it the tagged dtype: + if as_dtype is not None: + data = data.view(dtype=as_dtype) + + return data + + +def default_lapl(): + """ Default link access property list """ + return None + + +def default_lcpl(): + """ Default link creation property list """ + lcpl = h5p.create(h5p.LINK_CREATE) + lcpl.set_create_intermediate_group(True) + return lcpl + +dlapl = default_lapl() +dlcpl = default_lcpl() + + +def is_empty_dataspace(obj): + """ Check if an object's dataspace is empty """ + if obj.get_space().get_simple_extent_type() == h5s.NULL: + return True + return False + + +class CommonStateObject: + + """ + Mixin class that allows sharing information between objects which + reside in the same HDF5 file. Requires that the host class have + a ".id" attribute which returns a low-level ObjectID subclass. + + Also implements Unicode operations. + """ + + @property + def _lapl(self): + """ Fetch the link access property list appropriate for this object + """ + return dlapl + + @property + def _lcpl(self): + """ Fetch the link creation property list appropriate for this object + """ + return dlcpl + + def _e(self, name, lcpl=None): + """ Encode a name according to the current file settings. + + Returns name, or 2-tuple (name, lcpl) if lcpl is True + + - Binary strings are always passed as-is, h5t.CSET_ASCII + - Unicode strings are encoded utf8, h5t.CSET_UTF8 + + If name is None, returns either None or (None, None) appropriately. + """ + def get_lcpl(coding): + """ Create an appropriate link creation property list """ + lcpl = self._lcpl.copy() + lcpl.set_char_encoding(coding) + return lcpl + + if name is None: + return (None, None) if lcpl else None + + if isinstance(name, bytes): + coding = h5t.CSET_ASCII + elif isinstance(name, str): + try: + name = name.encode('ascii') + coding = h5t.CSET_ASCII + except UnicodeEncodeError: + name = name.encode('utf8') + coding = h5t.CSET_UTF8 + else: + raise TypeError(f"A name should be string or bytes, not {type(name)}") + + if lcpl: + return name, get_lcpl(coding) + return name + + def _d(self, name): + """ Decode a name according to the current file settings. + + - Try to decode utf8 + - Failing that, return the byte string + + If name is None, returns None. + """ + if name is None: + return None + + try: + return name.decode('utf8') + except UnicodeDecodeError: + pass + return name + + +class _RegionProxy: + + """ + Proxy object which handles region references. + + To create a new region reference (datasets only), use slicing syntax: + + >>> newref = obj.regionref[0:10:2] + + To determine the target dataset shape from an existing reference: + + >>> shape = obj.regionref.shape(existingref) + + where <obj> may be any object in the file. To determine the shape of + the selection in use on the target dataset: + + >>> selection_shape = obj.regionref.selection(existingref) + """ + + def __init__(self, obj): + self.obj = obj + self.id = obj.id + + def __getitem__(self, args): + if not isinstance(self.id, h5d.DatasetID): + raise TypeError("Region references can only be made to datasets") + from . import selections + with phil: + selection = selections.select(self.id.shape, args, dataset=self.obj) + return h5r.create(self.id, b'.', h5r.DATASET_REGION, selection.id) + + def shape(self, ref): + """ Get the shape of the target dataspace referred to by *ref*. """ + with phil: + sid = h5r.get_region(ref, self.id) + return sid.shape + + def selection(self, ref): + """ Get the shape of the target dataspace selection referred to by *ref* + """ + from . import selections + with phil: + sid = h5r.get_region(ref, self.id) + return selections.guess_shape(sid) + + +class HLObject(CommonStateObject): + + """ + Base class for high-level interface objects. + """ + + @property + def file(self): + """ Return a File instance associated with this object """ + from . import files + with phil: + return files.File(self.id) + + @property + @with_phil + def name(self): + """ Return the full name of this object. None if anonymous. """ + return self._d(h5i.get_name(self.id)) + + @property + @with_phil + def parent(self): + """Return the parent group of this object. + + This is always equivalent to obj.file[posixpath.dirname(obj.name)]. + ValueError if this object is anonymous. + """ + if self.name is None: + raise ValueError("Parent of an anonymous object is undefined") + return self.file[posixpath.dirname(self.name)] + + @property + @with_phil + def id(self): + """ Low-level identifier appropriate for this object """ + return self._id + + @property + @with_phil + def ref(self): + """ An (opaque) HDF5 reference to this object """ + return h5r.create(self.id, b'.', h5r.OBJECT) + + @property + @with_phil + def regionref(self): + """Create a region reference (Datasets only). + + The syntax is regionref[<slices>]. For example, dset.regionref[...] + creates a region reference in which the whole dataset is selected. + + Can also be used to determine the shape of the referenced dataset + (via .shape property), or the shape of the selection (via the + .selection property). + """ + return _RegionProxy(self) + + @property + def attrs(self): + """ Attributes attached to this object """ + from . import attrs + with phil: + return attrs.AttributeManager(self) + + @with_phil + def __init__(self, oid): + """ Setup this object, given its low-level identifier """ + self._id = oid + + @with_phil + def __hash__(self): + return hash(self.id) + + @with_phil + def __eq__(self, other): + if hasattr(other, 'id'): + return self.id == other.id + return NotImplemented + + def __bool__(self): + with phil: + return bool(self.id) + + def __getnewargs__(self): + """Disable pickle. + + Handles for HDF5 objects can't be reliably deserialised, because the + recipient may not have access to the same files. So we do this to + fail early. + + If you really want to pickle h5py objects and can live with some + limitations, look at the h5pickle project on PyPI. + """ + raise TypeError("h5py objects cannot be pickled") + + def __getstate__(self): + # Pickle protocols 0 and 1 use this instead of __getnewargs__ + raise TypeError("h5py objects cannot be pickled") + +# --- Dictionary-style interface ---------------------------------------------- + +# To implement the dictionary-style interface from groups and attributes, +# we inherit from the appropriate abstract base classes in collections. +# +# All locking is taken care of by the subclasses. +# We have to override ValuesView and ItemsView here because Group and +# AttributeManager can only test for key names. + + +class KeysViewHDF5(KeysView): + def __str__(self): + return "<KeysViewHDF5 {}>".format(list(self)) + + def __reversed__(self): + yield from reversed(self._mapping) + + __repr__ = __str__ + +class ValuesViewHDF5(ValuesView): + + """ + Wraps e.g. a Group or AttributeManager to provide a value view. + + Note that __contains__ will have poor performance as it has + to scan all the links or attributes. + """ + + def __contains__(self, value): + with phil: + for key in self._mapping: + if value == self._mapping.get(key): + return True + return False + + def __iter__(self): + with phil: + for key in self._mapping: + yield self._mapping.get(key) + + def __reversed__(self): + with phil: + for key in reversed(self._mapping): + yield self._mapping.get(key) + + +class ItemsViewHDF5(ItemsView): + + """ + Wraps e.g. a Group or AttributeManager to provide an items view. + """ + + def __contains__(self, item): + with phil: + key, val = item + if key in self._mapping: + return val == self._mapping.get(key) + return False + + def __iter__(self): + with phil: + for key in self._mapping: + yield (key, self._mapping.get(key)) + + def __reversed__(self): + with phil: + for key in reversed(self._mapping): + yield (key, self._mapping.get(key)) + + +class MappingHDF5(Mapping): + + """ + Wraps a Group, AttributeManager or DimensionManager object to provide + an immutable mapping interface. + + We don't inherit directly from MutableMapping because certain + subclasses, for example DimensionManager, are read-only. + """ + def keys(self): + """ Get a view object on member names """ + return KeysViewHDF5(self) + + def values(self): + """ Get a view object on member objects """ + return ValuesViewHDF5(self) + + def items(self): + """ Get a view object on member items """ + return ItemsViewHDF5(self) + + def _ipython_key_completions_(self): + """ Custom tab completions for __getitem__ in IPython >=5.0. """ + return sorted(self.keys()) + + +class MutableMappingHDF5(MappingHDF5, MutableMapping): + + """ + Wraps a Group or AttributeManager object to provide a mutable + mapping interface, in contrast to the read-only mapping of + MappingHDF5. + """ + + pass + + +class Empty: + + """ + Proxy object to represent empty/null dataspaces (a.k.a H5S_NULL). + + This can have an associated dtype, but has no shape or data. This is not + the same as an array with shape (0,). + """ + shape = None + size = None + + def __init__(self, dtype): + self.dtype = np.dtype(dtype) + + def __eq__(self, other): + if isinstance(other, Empty) and self.dtype == other.dtype: + return True + return False + + def __repr__(self): + return "Empty(dtype={0!r})".format(self.dtype) + + +def product(nums): + """Calculate a numeric product + + For small amounts of data (e.g. shape tuples), this simple code is much + faster than calling numpy.prod(). + """ + prod = 1 + for n in nums: + prod *= n + return prod + + +# Simple variant of cached_property: +# Unlike functools, this has no locking, so we don't have to worry about +# deadlocks with phil (see issue gh-2064). Unlike cached-property on PyPI, it +# doesn't try to import asyncio (which can be ~100 extra modules). +# Many projects seem to have similar variants of this, often without attribution, +# but to be cautious, this code comes from cached-property (Copyright (c) 2015, +# Daniel Greenfeld, BSD license), where it is attributed to bottle (Copyright +# (c) 2009-2022, Marcel Hellkamp, MIT license). + +class cached_property: + def __init__(self, func): + self.__doc__ = getattr(func, "__doc__") + self.func = func + + def __get__(self, obj, cls): + if obj is None: + return self + + value = obj.__dict__[self.func.__name__] = self.func(obj) + return value diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/compat.py b/.venv/lib/python3.11/site-packages/h5py/_hl/compat.py new file mode 100644 index 0000000000000000000000000000000000000000..29ec1fcd572db6ef0fe138e7a988dce8911c0d0c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/compat.py @@ -0,0 +1,46 @@ +""" +Compatibility module for high-level h5py +""" +import os +import sys +from ..version import hdf5_built_version_tuple + +# HDF5 supported passing paths as UTF-8 for Windows from 1.10.6, but this +# was broken again in 1.14.4 - https://github.com/HDFGroup/hdf5/issues/5037 . +# The change was reverted in 1.14.6. +if (1, 14, 4) <= hdf5_built_version_tuple < (1, 14, 6): + WINDOWS_ENCODING = "mbcs" +else: + WINDOWS_ENCODING = "utf-8" + + +def filename_encode(filename): + """ + Encode filename for use in the HDF5 library. + + Due to how HDF5 handles filenames on different systems, this should be + called on any filenames passed to the HDF5 library. See the documentation on + filenames in h5py for more information. + """ + filename = os.fspath(filename) + if sys.platform == "win32" and isinstance(filename, str): + return filename.encode(WINDOWS_ENCODING, "strict") + else: + return os.fsencode(filename) + + +def filename_decode(filename): + """ + Decode filename used by HDF5 library. + + Due to how HDF5 handles filenames on different systems, this should be + called on any filenames passed from the HDF5 library. See the documentation + on filenames in h5py for more information. + """ + if not isinstance(filename, (str, bytes)): + raise TypeError(f"expect bytes or str, not {type(filename).__name__}") + + if sys.platform == "win32" and isinstance(filename, bytes): + return filename.decode(WINDOWS_ENCODING, "strict") + else: + return os.fsdecode(filename) diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/dataset.py b/.venv/lib/python3.11/site-packages/h5py/_hl/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..76e41706c674972fe860755b6bdb2aa1784bd9aa --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/dataset.py @@ -0,0 +1,1252 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2020 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Implements support for high-level dataset access. +""" + +import posixpath as pp +import sys +from abc import ABC, abstractmethod +from warnings import warn + +import numpy + +import h5py.h5t +from .. import h5, h5s, h5t, h5r, h5d, h5p, h5fd, h5ds, _selector +from ..h5py_warnings import H5pyDeprecationWarning +from .base import ( + array_for_new_object, cached_property, Empty, find_item_type, HLObject, + phil, product, with_phil, +) +from . import filters +from . import selections as sel +from . import selections2 as sel2 +from .datatype import Datatype +from .compat import filename_decode +from .vds import VDSmap, vds_support + +_LEGACY_GZIP_COMPRESSION_VALS = frozenset(range(10)) +MPI = h5.get_config().mpi + + +def make_new_dset(parent, shape=None, dtype=None, data=None, name=None, + chunks=None, compression=None, shuffle=None, + fletcher32=None, maxshape=None, compression_opts=None, + fillvalue=None, scaleoffset=None, track_times=False, + external=None, track_order=None, dcpl=None, dapl=None, + efile_prefix=None, virtual_prefix=None, allow_unknown_filter=False, + rdcc_nslots=None, rdcc_nbytes=None, rdcc_w0=None, *, + fill_time=None): + """ Return a new low-level dataset identifier """ + + # Convert data to a C-contiguous ndarray + if data is not None and not isinstance(data, Empty): + data = array_for_new_object(data, specified_dtype=dtype) + + # Validate shape + if shape is None: + if data is None: + if dtype is None: + raise TypeError("One of data, shape or dtype must be specified") + data = Empty(dtype) + shape = data.shape + else: + shape = (shape,) if isinstance(shape, int) else tuple(shape) + if data is not None and (product(shape) != product(data.shape)): + raise ValueError("Shape tuple is incompatible with data") + + if isinstance(maxshape, int): + maxshape = (maxshape,) + tmp_shape = maxshape if maxshape is not None else shape + + # Validate chunk shape + if isinstance(chunks, int) and not isinstance(chunks, bool): + chunks = (chunks,) + # Logically, the following `zip` could be strict, but it's happening + # before we've done checks elsewhere that raise more descriptive errors + if isinstance(chunks, tuple) and any( + chunk > dim for dim, chunk in zip(tmp_shape, chunks, strict=False) if dim is not None + ): + errmsg = "Chunk shape must not be greater than data shape in any dimension. "\ + "{} is not compatible with {}".format(chunks, shape) + raise ValueError(errmsg) + + if isinstance(dtype, Datatype): + # Named types are used as-is + tid = dtype.id + dtype = tid.dtype # Following code needs this + elif isinstance(dtype, h5py.h5t.TypeID): # Low-level HDF5 data type + tid = dtype + dtype = tid.dtype + else: + # Validate dtype + if dtype is None and data is None: + warn( + "Creating a dataset without passing data or dtype is deprecated. " + "Pass an explicit dtype. Using dtype='f4' will keep the " + "current default behaviour.", + category=H5pyDeprecationWarning, stacklevel=3, + ) + dtype = numpy.dtype("=f4") + elif dtype is None and data is not None: + dtype = data.dtype + else: + dtype = numpy.dtype(dtype) + tid = h5t.py_create(dtype, logical=1) + + # Legacy + if any((compression, shuffle, fletcher32, maxshape, scaleoffset)) and chunks is False: + raise ValueError("Chunked format required for given storage options") + + # Legacy + if compression is True: + if compression_opts is None: + compression_opts = 4 + compression = 'gzip' + + # Legacy + if compression in _LEGACY_GZIP_COMPRESSION_VALS: + if compression_opts is not None: + raise TypeError("Conflict in compression options") + compression_opts = compression + compression = 'gzip' + dcpl = filters.fill_dcpl( + dcpl or h5p.create(h5p.DATASET_CREATE), shape, dtype, + chunks, compression, compression_opts, shuffle, fletcher32, + maxshape, scaleoffset, external, allow_unknown_filter, + fill_time=fill_time) + + # Check that compression roundtrips correctly if it was specified + if compression is not None: + if isinstance(compression, filters.FilterRefBase): + compression = compression.filter_id + if isinstance(compression, int): + compression = filters.get_filter_name(compression) + if compression not in filters.get_filters(dcpl): + raise ValueError(f'compression {compression!r} not in filters {filters.get_filters(dcpl)!r}') + + if fillvalue is not None: + # prepare string-type dtypes for fillvalue + string_info = h5t.check_string_dtype(dtype) + if string_info is not None: + # fake vlen dtype for fixed len string fillvalue + # to not trigger unwanted encoding + dtype = h5t.string_dtype(string_info.encoding) + fillvalue = numpy.array(fillvalue, dtype=dtype) + else: + fillvalue = numpy.array(fillvalue) + dcpl.set_fill_value(fillvalue) + + if track_times is None: + # In case someone explicitly passes None for the default + track_times = False + if track_times in (True, False): + dcpl.set_obj_track_times(track_times) + else: + raise TypeError("track_times must be either True or False") + if track_order is True: + dcpl.set_attr_creation_order( + h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED) + elif track_order is False: + dcpl.set_attr_creation_order(0) + elif track_order is not None: + raise TypeError("track_order must be either True or False") + + if maxshape is not None: + maxshape = tuple(m if m is not None else h5s.UNLIMITED for m in maxshape) + + if any([efile_prefix, virtual_prefix, rdcc_nbytes, rdcc_nslots, rdcc_w0]): + dapl = dapl or h5p.create(h5p.DATASET_ACCESS) + + if efile_prefix is not None: + dapl.set_efile_prefix(efile_prefix) + + if virtual_prefix is not None: + dapl.set_virtual_prefix(virtual_prefix) + + if rdcc_nbytes or rdcc_nslots or rdcc_w0: + cache_settings = list(dapl.get_chunk_cache()) + if rdcc_nslots is not None: + cache_settings[0] = rdcc_nslots + if rdcc_nbytes is not None: + cache_settings[1] = rdcc_nbytes + if rdcc_w0 is not None: + cache_settings[2] = rdcc_w0 + dapl.set_chunk_cache(*cache_settings) + + if isinstance(data, Empty): + sid = h5s.create(h5s.NULL) + else: + sid = h5s.create_simple(shape, maxshape) + + dset_id = h5d.create(parent.id, name, tid, sid, dcpl=dcpl, dapl=dapl) + + if (data is not None) and (not isinstance(data, Empty)): + dset_id.write(h5s.ALL, h5s.ALL, data) + + return dset_id + + +def open_dset(parent, name, dapl=None, efile_prefix=None, virtual_prefix=None, + rdcc_nslots=None, rdcc_nbytes=None, rdcc_w0=None, **kwds): + """ Return an existing low-level dataset identifier """ + + if any([efile_prefix, virtual_prefix, rdcc_nbytes, rdcc_nslots, rdcc_w0]): + dapl = dapl or h5p.create(h5p.DATASET_ACCESS) + + if efile_prefix is not None: + dapl.set_efile_prefix(efile_prefix) + + if virtual_prefix is not None: + dapl.set_virtual_prefix(virtual_prefix) + + if rdcc_nbytes or rdcc_nslots or rdcc_w0: + cache_settings = list(dapl.get_chunk_cache()) + if rdcc_nslots is not None: + cache_settings[0] = rdcc_nslots + if rdcc_nbytes is not None: + cache_settings[1] = rdcc_nbytes + if rdcc_w0 is not None: + cache_settings[2] = rdcc_w0 + dapl.set_chunk_cache(*cache_settings) + + dset_id = h5d.open(parent.id, name, dapl=dapl) + + return dset_id + + + +class AbstractView(ABC): + _dset: "Dataset" + + def __init__(self, dset): + self._dset = dset + + def __len__(self): + return len(self._dset) + + @property + @abstractmethod + def dtype(self): + ... # pragma: nocover + + @property + def ndim(self): + return self._dset.ndim + + @property + def shape(self): + return self._dset.shape + + @property + def size(self): + return self._dset.size + + @abstractmethod + def __getitem__(self, idx): + ... # pragma: nocover + + def __array__(self, dtype=None, copy=None): + if copy is False: + raise ValueError( + f"{self.__class__.__name__}.__array__ received {copy=} " + "but memory allocation cannot be avoided on read" + ) + + # If self.ndim == 0, convert np.generic back to np.ndarray + return numpy.asarray(self[()], dtype=dtype or self.dtype) + +class AsTypeView(AbstractView): + """Wrapper to convert data on reading from a dataset. + """ + def __init__(self, dset, dtype): + super().__init__(dset) + self._dtype = dtype + + @property + def dtype(self): + return self._dtype + + def __getitem__(self, idx): + return self._dset.__getitem__(idx, new_dtype=self._dtype) + + def __array__(self, dtype=None, copy=None): + return self._dset.__array__(dtype or self._dtype, copy) + + +class AsStrView(AbstractView): + """Wrapper to decode strings on reading the dataset""" + def __init__(self, dset, encoding, errors='strict'): + super().__init__(dset) + self.encoding = encoding + self.errors = errors + + @property + def dtype(self): + return numpy.dtype(object) + + def __getitem__(self, idx): + bytes_arr = self._dset[idx] + # numpy.char.decode() seems like the obvious thing to use. But it only + # accepts numpy string arrays, not object arrays of bytes (which we + # return from HDF5 variable-length strings). And the numpy + # implementation is not faster than doing it with a loop; in fact, by + # not converting the result to a numpy unicode array, the + # naive way can be faster! (Comparing with numpy 1.18.4, June 2020) + if numpy.isscalar(bytes_arr): + return bytes_arr.decode(self.encoding, self.errors) + + return numpy.array([ + b.decode(self.encoding, self.errors) for b in bytes_arr.flat + ], dtype=object).reshape(bytes_arr.shape) + + +class FieldsView(AbstractView): + """Wrapper to extract named fields from a dataset with a struct dtype""" + + def __init__(self, dset, prior_dtype, names): + super().__init__(dset) + if isinstance(names, str): + self.extract_field = names + names = [names] + else: + self.extract_field = None + self.read_dtype = readtime_dtype(prior_dtype, names) + + @property + def dtype(self): + t = self.read_dtype + if self.extract_field is not None: + t = t[self.extract_field] + return t + + def __getitem__(self, idx): + data = self._dset.__getitem__(idx, new_dtype=self.read_dtype) + if self.extract_field is not None: + data = data[self.extract_field] + return data + + +def readtime_dtype(basetype, names): + """Make a NumPy compound dtype with a subset of available fields""" + if basetype.names is None: # Names provided, but not compound + raise ValueError("Field names only allowed for compound types") + + for name in names: # Check all names are legal + if name not in basetype.names: + raise ValueError("Field %s does not appear in this type." % name) + + return numpy.dtype([(name, basetype.fields[name][0]) for name in names]) + + +if MPI: + class CollectiveContext: + + """ Manages collective I/O in MPI mode """ + + # We don't bother with _local as threads are forbidden in MPI mode + + def __init__(self, dset): + self._dset = dset + + def __enter__(self): + # pylint: disable=protected-access + self._dset._dxpl.set_dxpl_mpio(h5fd.MPIO_COLLECTIVE) + + def __exit__(self, *args): + # pylint: disable=protected-access + self._dset._dxpl.set_dxpl_mpio(h5fd.MPIO_INDEPENDENT) + + +class ChunkIterator: + """ + Class to iterate through list of chunks of a given dataset + """ + def __init__(self, dset, source_sel=None): + self._shape = dset.shape + rank = len(dset.shape) + + if not dset.chunks: + # can only use with chunked datasets + raise TypeError("Chunked dataset required") + + self._layout = dset.chunks + if source_sel is None: + # select over entire dataset + self._sel = tuple( + slice(0, self._shape[dim]) for dim in range(rank) + ) + else: + if isinstance(source_sel, (slice, int)): + sel = [source_sel] + else: + sel = list(source_sel) + if len(sel) != rank: + raise ValueError("Invalid selection - selection region must have same rank as dataset") + for dim, s in enumerate(sel): + start: int | None + stop: int | None + step: int | None + match s: + case int(): + start = s + stop = s + 1 + step = None + case slice(): + start = s.start or 0 + stop = s.stop or self._shape[dim] + step = s.step + case _: + # TODO: use typing.assert_never when Python 3.10 is dropped + raise AssertionError(f'{s}: Selection object must be a slice or integer') + sel[dim] = slice(start, stop, step) + self._sel = tuple(sel) + + self._chunk_index = [] + for dim in range(rank): + s = self._sel[dim] + if s.start < 0 or s.stop > self._shape[dim] or s.stop <= s.start: + raise ValueError("Invalid selection - selection region must be within dataset space") + index = s.start // self._layout[dim] + self._chunk_index.append(index) + + def __iter__(self): + return self + + def __next__(self): + rank = len(self._shape) + slices = [] + if rank == 0 or self._chunk_index[0] * self._layout[0] >= self._sel[0].stop: + # ran past the last chunk, end iteration + raise StopIteration() + + for dim in range(rank): + s = self._sel[dim] + start = self._chunk_index[dim] * self._layout[dim] + stop = (self._chunk_index[dim] + 1) * self._layout[dim] + # adjust the start if this is an edge chunk + if start < s.start: + start = s.start + if stop > s.stop: + stop = s.stop # trim to end of the selection + s = slice(start, stop, 1) + slices.append(s) + + # bump up the last index and carry forward if we run outside the selection + dim = rank - 1 + while dim >= 0: + s = self._sel[dim] + self._chunk_index[dim] += 1 + + chunk_end = self._chunk_index[dim] * self._layout[dim] + if chunk_end < s.stop: + # we still have room to extend along this dimensions + return tuple(slices) + + if dim > 0: + # reset to the start and continue iterating with higher dimension + self._chunk_index[dim] = s.start // self._layout[dim] + dim -= 1 + return tuple(slices) + + +class Dataset(HLObject): + + """ + Represents an HDF5 dataset + """ + + def astype(self, dtype): + """ Get a wrapper allowing you to perform reads to a + different destination type, e.g.: + + >>> double_precision = dataset.astype('f8')[0:100:2] + """ + dtype = numpy.dtype(dtype) + if dtype == self.dtype: + return self + + if dtype.kind == "T": + string_info = h5t.check_string_dtype(self.dtype) + if string_info is None: + raise TypeError( + f"dset.astype({dtype}) can only be used on datasets with " + "an HDF5 string datatype" + ) + + return AsTypeView(self, dtype) + + def asstr(self, encoding=None, errors='strict'): + """Get a wrapper to read string data as Python strings: + + >>> str_array = dataset.asstr()[:] + + The parameters have the same meaning as in ``bytes.decode()``. + If ``encoding`` is unspecified, it will use the encoding in the HDF5 + datatype (either ascii or utf-8). + + .. note:: + On NumPy 2.0 and later, it is recommended to use native NumPy + variable-width strings instead: + + >>> str_array = dataset.astype('T')[:] + """ + string_info = h5t.check_string_dtype(self.dtype) + if string_info is None: + raise TypeError( + "dset.asstr() can only be used on datasets with " + "an HDF5 string datatype" + ) + if encoding is None: + encoding = string_info.encoding + return AsStrView(self, encoding, errors=errors) + + def fields(self, names, *, _prior_dtype=None): + """Get a wrapper to read a subset of fields from a compound data type: + + >>> 2d_coords = dataset.fields(['x', 'y'])[:] + + If names is a string, a single field is extracted, and the resulting + arrays will have that dtype. Otherwise, it should be an iterable, + and the read data will have a compound dtype. + """ + if _prior_dtype is None: + _prior_dtype = self.dtype + return FieldsView(self, _prior_dtype, names) + + if MPI: + @property + @with_phil + def collective(self): + """ Context manager for MPI collective reads & writes """ + return CollectiveContext(self) + + @property + def dims(self): + """ Access dimension scales attached to this dataset. """ + from .dims import DimensionManager + with phil: + return DimensionManager(self) + + @property + @with_phil + def ndim(self): + """Numpy-style attribute giving the number of dimensions""" + return self.id.rank + + @property + def shape(self): + """Numpy-style shape tuple giving dataset dimensions""" + if 'shape' in self._cache_props: + return self._cache_props['shape'] + + with phil: + shape = self.id.shape + + # If the file is read-only, cache the shape to speed-up future uses. + # This cache is invalidated by .refresh() when using SWMR. + if self._readonly: + self._cache_props['shape'] = shape + return shape + + @shape.setter + @with_phil + def shape(self, shape): + # pylint: disable=missing-docstring + self.resize(shape) + + @property + def size(self): + """Numpy-style attribute giving the total dataset size""" + if 'size' in self._cache_props: + return self._cache_props['size'] + + if self._is_empty: + size = None + else: + size = product(self.shape) + + # If the file is read-only, cache the size to speed-up future uses. + # This cache is invalidated by .refresh() when using SWMR. + if self._readonly: + self._cache_props['size'] = size + return size + + @property + def nbytes(self): + """Numpy-style attribute giving the raw dataset size as the number of bytes""" + size = self.size + if size is None: # if we are an empty 0-D array, then there are no bytes in the dataset + return 0 + return self.dtype.itemsize * size + + @property + def _selector(self): + """Internal object for optimised selection of data""" + if '_selector' in self._cache_props: + return self._cache_props['_selector'] + + slr = _selector.Selector(self.id.get_space()) + + # If the file is read-only, cache the reader to speed up future uses. + # This cache is invalidated by .refresh() when using SWMR. + if self._readonly: + self._cache_props['_selector'] = slr + return slr + + @property + def _fast_reader(self): + """Internal object for optimised reading of data""" + if '_fast_reader' in self._cache_props: + return self._cache_props['_fast_reader'] + + rdr = _selector.Reader(self.id) + + # If the file is read-only, cache the reader to speed up future uses. + # This cache is invalidated by .refresh() when using SWMR. + if self._readonly: + self._cache_props['_fast_reader'] = rdr + return rdr + + @property + @with_phil + def dtype(self): + """Numpy dtype representing the datatype""" + return self.id.dtype + + @property + @with_phil + def chunks(self): + """Dataset chunks (or None)""" + dcpl = self._dcpl + if dcpl.get_layout() == h5d.CHUNKED: + return dcpl.get_chunk() + return None + + @property + @with_phil + def compression(self): + """Compression strategy (or None)""" + for x in ('gzip','lzf','szip'): + if x in self._filters: + return x + if any(f not in filters._COMP_FILTERS for f in self._filters): + return 'unknown' # Filter from a plugin + return None + + @property + @with_phil + def compression_opts(self): + """ Compression setting. Int(0-9) for gzip, 2-tuple for szip. """ + return self._filters.get(self.compression, None) + + @property + @with_phil + def filter_ids(self): + """Numeric IDs of HDF5 filters used for this dataset""" + pl = self._dcpl + return tuple([pl.get_filter(i)[0] for i in range(pl.get_nfilters())]) + + @property + @with_phil + def filter_names(self): + """Names, as stored in the file, of the filters used for this dataset""" + pl = self._dcpl + return tuple([pl.get_filter(i)[3].decode('utf-8', 'surrogateescape') + for i in range(pl.get_nfilters())]) + + @property + @with_phil + def shuffle(self): + """Shuffle filter present (T/F)""" + return 'shuffle' in self._filters + + @property + @with_phil + def fletcher32(self): + """Fletcher32 filter is present (T/F)""" + return 'fletcher32' in self._filters + + @property + @with_phil + def scaleoffset(self): + """Scale/offset filter settings. For integer data types, this is + the number of bits stored, or 0 for auto-detected. For floating + point data types, this is the number of decimal places retained. + If the scale/offset filter is not in use, this is None.""" + try: + return self._filters['scaleoffset'][1] + except KeyError: + return None + + @property + @with_phil + def external(self): + """External file settings. Returns a list of tuples of + (name, offset, size) for each external file entry, or returns None + if no external files are used.""" + count = self._dcpl.get_external_count() + if count<=0: + return None + ext_list = list() + for x in range(count): + (name, offset, size) = self._dcpl.get_external(x) + ext_list.append( (filename_decode(name), offset, size) ) + return ext_list + + @property + @with_phil + def maxshape(self): + """Shape up to which this dataset can be resized. Axes with value + None have no resize limit. """ + space = self.id.get_space() + dims = space.get_simple_extent_dims(True) + if dims is None: + return None + + return tuple(x if x != h5s.UNLIMITED else None for x in dims) + + @property + @with_phil + def fillvalue(self): + """Fill value for this dataset (0 by default)""" + arr = numpy.zeros((1,), dtype=self.dtype) + self._dcpl.get_fill_value(arr) + return arr[0] + + @cached_property + @with_phil + def _extent_type(self): + """Get extent type for this dataset - SIMPLE, SCALAR or NULL""" + return self.id.get_space().get_simple_extent_type() + + @cached_property + def _is_empty(self): + """Check if extent type is empty""" + return self._extent_type == h5s.NULL + + @cached_property + def _dcpl(self): + """ + The dataset creation property list used when this dataset was created. + """ + return self.id.get_create_plist() + + @cached_property + def _filters(self): + """ + The active filters of the dataset. + """ + return filters.get_filters(self._dcpl) + + @with_phil + def __init__(self, bind, *, readonly=False): + """ Create a new Dataset object by binding to a low-level DatasetID. + """ + if not isinstance(bind, h5d.DatasetID): + raise ValueError("%s is not a DatasetID" % bind) + super().__init__(bind) + + self._dxpl = h5p.create(h5p.DATASET_XFER) + self._readonly = readonly + self._cache_props = {} + + def resize(self, size, axis=None): + """ Resize the dataset, or the specified axis. + + The dataset must be stored in chunked format; it can be resized up to + the "maximum shape" (keyword maxshape) specified at creation time. + The rank of the dataset cannot be changed. + + "Size" should be a shape tuple, or if an axis is specified, an integer. + + BEWARE: This functions differently than the NumPy resize() method! + The data is not "reshuffled" to fit in the new shape; each axis is + grown or shrunk independently. The coordinates of existing data are + fixed. + """ + with phil: + if self.chunks is None: + raise TypeError("Only chunked datasets can be resized") + + if axis is not None: + if not (axis >=0 and axis < self.id.rank): + raise ValueError("Invalid axis (0 to %s allowed)" % (self.id.rank-1)) + try: + newlen = int(size) + except TypeError: + raise TypeError("Argument must be a single int if axis is specified") from None + size = list(self.shape) + size[axis] = newlen + + size = tuple(size) + self.id.set_extent(size) + #h5f.flush(self.id) # THG recommends + + @with_phil + def __len__(self): + """ The size of the first axis. TypeError if scalar. + + Limited to 2**32 on 32-bit systems; Dataset.len() is preferred. + """ + size = self.len() + if size > sys.maxsize: + raise OverflowError("Value too big for Python's __len__; use Dataset.len() instead.") + return size + + def len(self): + """ The size of the first axis. TypeError if scalar. + + Use of this method is preferred to len(dset), as Python's built-in + len() cannot handle values greater then 2**32 on 32-bit systems. + """ + with phil: + shape = self.shape + if len(shape) == 0: + raise TypeError("Attempt to take len() of scalar dataset") + return shape[0] + + @with_phil + def __iter__(self): + """ Iterate over the first axis. TypeError if scalar. + + BEWARE: Modifications to the yielded data are *NOT* written to file. + """ + shape = self.shape + if len(shape) == 0: + raise TypeError("Can't iterate over a scalar dataset") + for i in range(shape[0]): + yield self[i] + + @with_phil + def iter_chunks(self, sel=None): + """ Return chunk iterator. If set, the sel argument is a slice or + tuple of slices that defines the region to be used. If not set, the + entire dataspace will be used for the iterator. + + For each chunk within the given region, the iterator yields a tuple of + slices that gives the intersection of the given chunk with the + selection area. + + A TypeError will be raised if the dataset is not chunked. + + A ValueError will be raised if the selection region is invalid. + + """ + return ChunkIterator(self, sel) + + @cached_property + def _fast_read_ok(self): + """Is this dataset suitable for simple reading""" + return ( + self._extent_type == h5s.SIMPLE + and isinstance(self.id.get_type(), (h5t.TypeIntegerID, h5t.TypeFloatID)) + ) + + @with_phil + def __getitem__(self, args, new_dtype=None): + """ Read a slice from the HDF5 dataset. + + Takes slices and recarray-style field names (more than one is + allowed!) in any order. Obeys basic NumPy rules, including + broadcasting. + + Also supports: + + * Boolean "mask" array indexing + """ + args = args if isinstance(args, tuple) else (args,) + + if any(a is None for a in args): # 'None in args' would fail on arrays + raise TypeError("Indexing with None (or np.newaxis) is not supported") + + if self._fast_read_ok and (new_dtype is None): + try: + return self._fast_reader.read(args) + except TypeError: + pass # Fall back to Python read pathway below + + if self._is_empty: + # Check 'is Ellipsis' to avoid equality comparison with an array: + # array equality returns an array, not a boolean. + if args == () or (len(args) == 1 and args[0] is Ellipsis): + return Empty(self.dtype) + raise ValueError("Empty datasets cannot be sliced") + + # Sort field names from the rest of the args. + names = tuple(x for x in args if isinstance(x, str)) + + if names: + # Read a subset of the fields in this structured dtype + if len(names) == 1: + names = names[0] # Read with simpler dtype of this field + args = tuple(x for x in args if not isinstance(x, str)) + return self.fields(names, _prior_dtype=new_dtype)[args] + + if new_dtype is None: + new_dtype = self.dtype + mtype = h5t.py_create(new_dtype) + + # === Special-case region references ==== + + if len(args) == 1 and isinstance(args[0], h5r.RegionReference): + + obj = h5r.dereference(args[0], self.id) + if obj != self.id: + raise ValueError("Region reference must point to this dataset") + + sid = h5r.get_region(args[0], self.id) + mshape = sel.guess_shape(sid) + if mshape is None: + # 0D with no data (NULL or deselected SCALAR) + return Empty(new_dtype) + out = numpy.zeros(mshape, dtype=new_dtype) + if out.size == 0: + return out + + sid_out = h5s.create_simple(mshape) + sid_out.select_all() + self.id.read(sid_out, sid, out, mtype) + return out + + # === Check for zero-sized datasets ===== + + if self.size == 0: + # Check 'is Ellipsis' to avoid equality comparison with an array: + # array equality returns an array, not a boolean. + if args == () or (len(args) == 1 and args[0] is Ellipsis): + return numpy.zeros(self.shape, dtype=new_dtype) + + # === Scalar dataspaces ================= + + if self.shape == (): + fspace = self.id.get_space() + selection = sel2.select_read(fspace, args) + if selection.mshape is None: + arr = numpy.zeros((), dtype=new_dtype) + else: + arr = numpy.zeros(selection.mshape, dtype=new_dtype) + for mspace, fspace in selection: + self.id.read(mspace, fspace, arr, mtype) + if selection.mshape is None: + return arr[()] + return arr + + # === Everything else =================== + + # Perform the dataspace selection. + selection = sel.select(self.shape, args, dataset=self) + + if selection.nselect == 0: + return numpy.zeros(selection.array_shape, dtype=new_dtype) + + arr = numpy.zeros(selection.array_shape, new_dtype, order='C') + + # Perform the actual read + mspace = h5s.create_simple(selection.mshape) + fspace = selection.id + self.id.read(mspace, fspace, arr, mtype, dxpl=self._dxpl) + + # Patch up the output for NumPy + if arr.shape == (): + return arr[()] # 0 dim array -> numpy scalar + return arr + + @with_phil + def __setitem__(self, args, val): + """ Write to the HDF5 dataset from a Numpy array. + + NumPy's broadcasting rules are honored, for "simple" indexing + (slices and integers). For advanced indexing, the shapes must + match. + """ + args = args if isinstance(args, tuple) else (args,) + + # Sort field indices from the slicing + names = tuple(x for x in args if isinstance(x, str)) + args = tuple(x for x in args if not isinstance(x, str)) + + # Generally we try to avoid converting the arrays on the Python + # side. However, for compound literals this is unavoidable. + vlen = h5t.check_vlen_dtype(self.dtype) + if vlen is not None and vlen not in (bytes, str): + try: + val = numpy.asarray(val, dtype=vlen) + except (ValueError, TypeError): + try: + val = numpy.array([numpy.array(x, dtype=vlen) + for x in val], dtype=self.dtype) + except (ValueError, TypeError): + pass + if vlen == val.dtype: + if val.ndim > 1: + tmp = numpy.empty(shape=val.shape[:-1], dtype=object) + tmp.ravel()[:] = [i for i in val.reshape( + (product(val.shape[:-1]), val.shape[-1]) + )] + else: + tmp = numpy.array([None], dtype=object) + tmp[0] = val + val = tmp + elif self.dtype.kind == "O" or \ + (self.dtype.kind == 'V' and \ + (not isinstance(val, numpy.ndarray) or val.dtype.kind != 'V') and \ + (self.dtype.subdtype is None)): + if len(names) == 1 and self.dtype.fields is not None: + # Single field selected for write, from a non-array source + if not names[0] in self.dtype.fields: + raise ValueError("No such field for indexing: %s" % names[0]) + dtype = self.dtype.fields[names[0]][0] + cast_compound = True + else: + dtype = self.dtype + cast_compound = False + + val = numpy.asarray(val, dtype=dtype.base, order='C') + if cast_compound: + val = val.view(numpy.dtype([(names[0], dtype)])) + val = val.reshape(val.shape[:len(val.shape) - len(dtype.shape)]) + elif (self.dtype.kind == 'S' + and (h5t.check_string_dtype(self.dtype).encoding == 'utf-8') + and (find_item_type(val) is str) + ): + # Writing str objects to a fixed-length UTF-8 string dataset. + # Numpy's normal conversion only handles ASCII characters, but + # when the destination is UTF-8, we want to allow any unicode. + # This *doesn't* handle numpy fixed-length unicode data ('U' dtype), + # as HDF5 has no equivalent, and converting fixed length UTF-32 + # to variable length UTF-8 would obscure what's going on. + str_array = numpy.asarray(val, order='C', dtype=object) + val = numpy.array([ + s.encode('utf-8') for s in str_array.flat + ], dtype=self.dtype).reshape(str_array.shape) + else: + # If the input data is already an array, let HDF5 do the conversion. + # If it's a list or similar, don't make numpy guess a dtype for it. + dt = None if isinstance(val, numpy.ndarray) else self.dtype.base + val = numpy.asarray(val, order='C', dtype=dt) + + # Check for array dtype compatibility and convert + if self.dtype.subdtype is not None: + shp = self.dtype.subdtype[1] + valshp = val.shape[-len(shp):] + if valshp != shp: # Last dimension has to match + raise TypeError("When writing to array types, last N dimensions have to match (got %s, but should be %s)" % (valshp, shp,)) + mtype = h5t.py_create(numpy.dtype((val.dtype, shp))) + mshape = val.shape[0:len(val.shape)-len(shp)] + + # Make a compound memory type if field-name slicing is required + elif len(names) != 0: + + mshape = val.shape + + # Catch common errors + if self.dtype.fields is None: + raise TypeError("Illegal slicing argument (not a compound dataset)") + mismatch = [x for x in names if x not in self.dtype.fields] + if len(mismatch) != 0: + mismatch = ", ".join('"%s"'%x for x in mismatch) + raise ValueError("Illegal slicing argument (fields %s not in dataset type)" % mismatch) + + # Write non-compound source into a single dataset field + if len(names) == 1 and val.dtype.fields is None: + subtype = h5t.py_create(val.dtype) + mtype = h5t.create(h5t.COMPOUND, subtype.get_size()) + mtype.insert(self._e(names[0]), 0, subtype) + + # Make a new source type keeping only the requested fields + else: + fieldnames = [x for x in val.dtype.names if x in names] # Keep source order + mtype = h5t.create(h5t.COMPOUND, val.dtype.itemsize) + for fieldname in fieldnames: + subtype = h5t.py_create(val.dtype.fields[fieldname][0]) + offset = val.dtype.fields[fieldname][1] + mtype.insert(self._e(fieldname), offset, subtype) + + # Use mtype derived from array (let DatasetID.write figure it out) + else: + mshape = val.shape + mtype = None + + # Perform the dataspace selection + selection = sel.select(self.shape, args, dataset=self) + + # Broadcast scalars if necessary. + # In order to avoid slow broadcasting filling the destination by + # the scalar value, we create an intermediate array of the same + # size as the destination buffer provided that size is reasonable. + # We assume as reasonable a size smaller or equal as the used dataset + # chunk size if any. + # In case of dealing with a non-chunked destination dataset or with + # a selection whose size is larger than the dataset chunk size we fall + # back to using an intermediate array of size equal to the last dimension + # of the destination buffer. + # The reasoning behind is that it makes sense to assume the creator of + # the dataset used an appropriate chunk size according the available + # memory. In any case, if we cannot afford to create an intermediate + # array of the same size as the dataset chunk size, the user program has + # little hope to go much further. Solves h5py issue #1067 + if mshape == () and selection.array_shape != (): + if self.dtype.subdtype is not None: + raise TypeError("Scalar broadcasting is not supported for array dtypes") + if self.chunks and (product(self.chunks) >= product(selection.array_shape)): + val2 = numpy.empty(selection.array_shape, dtype=val.dtype) + else: + val2 = numpy.empty(selection.array_shape[-1], dtype=val.dtype) + val2[...] = val + val = val2 + mshape = val.shape + + # Perform the write, with broadcasting + mspace = h5s.create_simple(selection.expand_shape(mshape)) + for fspace in selection.broadcast(mshape): + self.id.write(mspace, fspace, val, mtype, dxpl=self._dxpl) + + def read_direct(self, dest, source_sel=None, dest_sel=None): + """ Read data directly from HDF5 into an existing NumPy array. + + The destination array must be C-contiguous and writable. + Selections must be the output of numpy.s_[<args>]. + + Broadcasting is supported for simple indexing. + """ + with phil: + if self._is_empty: + raise TypeError("Empty datasets have no numpy representation") + if source_sel is None: + source_sel = sel.SimpleSelection(self.shape) + else: + source_sel = sel.select(self.shape, source_sel, self) # for numpy.s_ + fspace = source_sel.id + + if dest_sel is None: + dest_sel = sel.SimpleSelection(dest.shape) + else: + dest_sel = sel.select(dest.shape, dest_sel) + + for mspace in dest_sel.broadcast(source_sel.array_shape): + self.id.read(mspace, fspace, dest, dxpl=self._dxpl) + + def write_direct(self, source, source_sel=None, dest_sel=None): + """ Write data directly to HDF5 from a NumPy array. + + The source array must be C-contiguous. Selections must be + the output of numpy.s_[<args>]. + + Broadcasting is supported for simple indexing. + """ + with phil: + if self._is_empty: + raise TypeError("Empty datasets cannot be written to") + if source_sel is None: + source_sel = sel.SimpleSelection(source.shape) + else: + source_sel = sel.select(source.shape, source_sel) # for numpy.s_ + mspace = source_sel.id + + if dest_sel is None: + dest_sel = sel.SimpleSelection(self.shape) + else: + dest_sel = sel.select(self.shape, dest_sel, self) + + for fspace in dest_sel.broadcast(source_sel.array_shape): + self.id.write(mspace, fspace, source, dxpl=self._dxpl) + + @with_phil + def __array__(self, dtype=None, copy=None): + """ Create a Numpy array containing the whole dataset. DON'T THINK + THIS MEANS DATASETS ARE INTERCHANGEABLE WITH ARRAYS. For one thing, + you have to read the whole dataset every time this method is called. + """ + if copy is False: + raise ValueError( + f"Dataset.__array__ received {copy=} " + "but memory allocation cannot be avoided on read" + ) + arr = numpy.zeros(self.shape, dtype=self.dtype if dtype is None else dtype) + + # Special case for (0,)*-shape datasets + if self.size == 0: + return arr + + self.read_direct(arr) + return arr + + @with_phil + def __repr__(self): + if not self: + return "<Closed HDF5 dataset>" + + if self.name is None: + name = "(anonymous)" + else: + name = pp.basename(pp.normpath(self.name)) + name = f'"{name}"' + + return f'<HDF5 dataset {name}: shape {self.shape}, type "{self.dtype.str}">' + + @with_phil + def refresh(self): + """ Refresh the dataset metadata by reloading from the file. + + This is part of the SWMR features. + """ + self._id.refresh() + self._cache_props.clear() + + @with_phil + def flush(self): + """ Flush the dataset data and metadata to the file. + If the dataset is chunked, raw data chunks are written to the file. + + This is part of the SWMR features. + """ + self._id.flush() + + if vds_support: + @property + @with_phil + def is_virtual(self): + """Check if this is a virtual dataset""" + return self._dcpl.get_layout() == h5d.VIRTUAL + + @with_phil + def virtual_sources(self): + """Get a list of the data mappings for a virtual dataset""" + if not self.is_virtual: + raise RuntimeError("Not a virtual dataset") + dcpl = self._dcpl + return [ + VDSmap(dcpl.get_virtual_vspace(j), + dcpl.get_virtual_filename(j), + dcpl.get_virtual_dsetname(j), + dcpl.get_virtual_srcspace(j)) + for j in range(dcpl.get_virtual_count())] + + @with_phil + def make_scale(self, name=''): + """Make this dataset an HDF5 dimension scale. + + You can then attach it to dimensions of other datasets like this:: + + other_ds.dims[0].attach_scale(ds) + + You can optionally pass a name to associate with this scale. + """ + h5ds.set_scale(self._id, self._e(name)) + + @property + @with_phil + def is_scale(self): + """Return ``True`` if this dataset is also a dimension scale. + + Return ``False`` otherwise. + """ + return h5ds.is_scale(self._id) diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/datatype.py b/.venv/lib/python3.11/site-packages/h5py/_hl/datatype.py new file mode 100644 index 0000000000000000000000000000000000000000..33e2b31007897987f0bf197058072434c1982d2c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/datatype.py @@ -0,0 +1,55 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Implements high-level access to committed datatypes in the file. +""" + +import posixpath as pp + +from ..h5t import TypeID +from .base import HLObject, with_phil + +class Datatype(HLObject): + + """ + Represents an HDF5 named datatype stored in a file. + + To store a datatype, simply assign it to a name in a group: + + >>> MyGroup["name"] = numpy.dtype("f") + >>> named_type = MyGroup["name"] + >>> assert named_type.dtype == numpy.dtype("f") + """ + + @property + @with_phil + def dtype(self): + """Numpy dtype equivalent for this datatype""" + return self.id.dtype + + @with_phil + def __init__(self, bind): + """ Create a new Datatype object by binding to a low-level TypeID. + """ + if not isinstance(bind, TypeID): + raise ValueError("%s is not a TypeID" % bind) + super().__init__(bind) + + @with_phil + def __repr__(self): + if not self.id: + return "<Closed HDF5 named type>" + if self.name is None: + namestr = '("anonymous")' + else: + name = pp.basename(pp.normpath(self.name)) + namestr = '"%s"' % (name if name != '' else '/') + return '<HDF5 named type %s (dtype %s)>' % \ + (namestr, self.dtype.str) diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/dims.py b/.venv/lib/python3.11/site-packages/h5py/_hl/dims.py new file mode 100644 index 0000000000000000000000000000000000000000..0cf4c9f36b85f13a866ef20c32e1142eb3c3b254 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/dims.py @@ -0,0 +1,181 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Implements support for HDF5 dimension scales. +""" + +import warnings + +from .. import h5ds +from ..h5py_warnings import H5pyDeprecationWarning +from . import base +from .base import phil, with_phil +from .dataset import Dataset + + +class DimensionProxy(base.CommonStateObject): + + """ + Represents an HDF5 "dimension". + """ + + @property + @with_phil + def label(self): + """ Get or set the dimension scale label """ + return self._d(h5ds.get_label(self._id, self._dimension)) + + @label.setter + @with_phil + def label(self, val): + # pylint: disable=missing-docstring + h5ds.set_label(self._id, self._dimension, self._e(val)) + + @with_phil + def __init__(self, id_, dimension): + self._id = id_ + self._dimension = dimension + + @with_phil + def __hash__(self): + return hash((type(self), self._id, self._dimension)) + + @with_phil + def __eq__(self, other): + return hash(self) == hash(other) + + @with_phil + def __iter__(self): + yield from self.keys() + + @with_phil + def __len__(self): + return h5ds.get_num_scales(self._id, self._dimension) + + @with_phil + def __getitem__(self, item): + + if isinstance(item, int): + scales = [] + h5ds.iterate(self._id, self._dimension, scales.append, 0) + return Dataset(scales[item]) + + else: + def f(dsid): + """ Iterate over scales to find a matching name """ + if h5ds.get_scale_name(dsid) == self._e(item): + return dsid + + res = h5ds.iterate(self._id, self._dimension, f, 0) + if res is None: + raise KeyError(item) + return Dataset(res) + + def attach_scale(self, dset): + """ Attach a scale to this dimension. + + Provide the Dataset of the scale you would like to attach. + """ + with phil: + h5ds.attach_scale(self._id, dset.id, self._dimension) + + def detach_scale(self, dset): + """ Remove a scale from this dimension. + + Provide the Dataset of the scale you would like to remove. + """ + with phil: + h5ds.detach_scale(self._id, dset.id, self._dimension) + + def items(self): + """ Get a list of (name, Dataset) pairs with all scales on this + dimension. + """ + with phil: + scales = [] + + # H5DSiterate raises an error if there are no dimension scales, + # rather than iterating 0 times. See #483. + if len(self) > 0: + h5ds.iterate(self._id, self._dimension, scales.append, 0) + + return [ + (self._d(h5ds.get_scale_name(x)), Dataset(x)) + for x in scales + ] + + def keys(self): + """ Get a list of names for the scales on this dimension. """ + with phil: + return [key for (key, _) in self.items()] + + def values(self): + """ Get a list of Dataset for scales on this dimension. """ + with phil: + return [val for (_, val) in self.items()] + + @with_phil + def __repr__(self): + if not self._id: + return "<Dimension of closed HDF5 dataset>" + return ('<"%s" dimension %d of HDF5 dataset at %s>' + % (self.label, self._dimension, id(self._id))) + + +class DimensionManager(base.CommonStateObject): + + """ + Represents a collection of dimension associated with a dataset. + + Like AttributeManager, an instance of this class is returned when + accessing the ".dims" property on a Dataset. + """ + + @with_phil + def __init__(self, parent): + """ Private constructor. + """ + self._id = parent.id + + @with_phil + def __getitem__(self, index): + """ Return a Dimension object + """ + if index > len(self) - 1: + raise IndexError('Index out of range') + return DimensionProxy(self._id, index) + + @with_phil + def __len__(self): + """ Number of dimensions associated with the dataset. """ + return self._id.rank + + @with_phil + def __iter__(self): + """ Iterate over the dimensions. """ + for i in range(len(self)): + yield self[i] + + @with_phil + def __repr__(self): + if not self._id: + return "<Dimensions of closed HDF5 dataset>" + return "<Dimensions of HDF5 object at %s>" % id(self._id) + + def create_scale(self, dset, name=''): + """ Create a new dimension, from an initial scale. + + Provide the dataset and a name for the scale. + """ + warnings.warn("other_ds.dims.create_scale(ds, name) is deprecated. " + "Use ds.make_scale(name) instead.", + H5pyDeprecationWarning, stacklevel=2, + ) + dset.make_scale(name) diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/files.py b/.venv/lib/python3.11/site-packages/h5py/_hl/files.py new file mode 100644 index 0000000000000000000000000000000000000000..175e7815b5216b424e76771413dc89866dd69137 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/files.py @@ -0,0 +1,652 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Implements high-level support for HDF5 file objects. +""" + +import inspect +import os +import sys +from warnings import warn + +from .compat import filename_decode, filename_encode + +from .base import phil, with_phil +from .group import Group, set_fapl_file_locking +from .. import h5, h5f, h5p, h5i, h5fd, _objects +from .. import version + +mpi = h5.get_config().mpi +ros3 = h5.get_config().ros3 +direct_vfd = h5.get_config().direct_vfd +hdf5_version = version.hdf5_version_tuple[0:3] + + +libver_dict = {'earliest': h5f.LIBVER_EARLIEST, 'latest': h5f.LIBVER_LATEST, + 'v108': h5f.LIBVER_V18, 'v110': h5f.LIBVER_V110} +libver_dict_r = dict((y, x) for x, y in libver_dict.items()) + +if hdf5_version >= (1, 11, 4): + libver_dict.update({'v112': h5f.LIBVER_V112}) + libver_dict_r.update({h5f.LIBVER_V112: 'v112'}) + +if hdf5_version >= (1, 13, 0): + libver_dict.update({'v114': h5f.LIBVER_V114}) + libver_dict_r.update({h5f.LIBVER_V114: 'v114'}) + +if hdf5_version >= (2, 0, 0): + libver_dict.update({'v200': h5f.LIBVER_V200}) + libver_dict_r.update({h5f.LIBVER_V200: 'v200'}) + + +def _set_fapl_mpio(plist, **kwargs): + """Set file access property list for mpio driver""" + if not mpi: + raise ValueError("h5py was built without MPI support, can't use mpio driver") + + import mpi4py.MPI + kwargs.setdefault('info', mpi4py.MPI.Info()) + plist.set_fapl_mpio(**kwargs) + + +def _set_fapl_fileobj(plist, **kwargs): + """Set the Python file object driver in a file access property list""" + plist.set_fileobj_driver(h5fd.fileobj_driver, kwargs.get('fileobj')) + + +_drivers = { + 'sec2': lambda plist, **kwargs: plist.set_fapl_sec2(**kwargs), + 'stdio': lambda plist, **kwargs: plist.set_fapl_stdio(**kwargs), + 'core': lambda plist, **kwargs: plist.set_fapl_core(**kwargs), + 'family': lambda plist, **kwargs: plist.set_fapl_family( + memb_fapl=plist.copy(), + **kwargs + ), + 'mpio': _set_fapl_mpio, + 'fileobj': _set_fapl_fileobj, + 'split': lambda plist, **kwargs: plist.set_fapl_split(**kwargs), +} + +if ros3: + _drivers['ros3'] = lambda plist, **kwargs: plist.set_fapl_ros3(**kwargs) + +if direct_vfd: + _drivers['direct'] = lambda plist, **kwargs: plist.set_fapl_direct(**kwargs) # noqa + + +def register_driver(name, set_fapl): + """Register a custom driver. + + Parameters + ---------- + name : str + The name of the driver. + set_fapl : callable[PropFAID, **kwargs] -> NoneType + The function to set the fapl to use your custom driver. + """ + _drivers[name] = set_fapl + + +def unregister_driver(name): + """Unregister a custom driver. + + Parameters + ---------- + name : str + The name of the driver. + """ + del _drivers[name] + + +def registered_drivers(): + """Return a frozenset of the names of all of the registered drivers. + """ + return frozenset(_drivers) + + +def make_fapl( + driver, libver=None, rdcc_nslots=None, rdcc_nbytes=None, rdcc_w0=None, + locking=None, page_buf_size=None, min_meta_keep=0, min_raw_keep=0, + alignment_threshold=1, alignment_interval=1, meta_block_size=None, + **kwds +): + """ Set up a file access property list """ + plist = h5p.create(h5p.FILE_ACCESS) + + if libver is not None: + if libver in libver_dict: + low = libver_dict[libver] + high = h5f.LIBVER_LATEST + else: + low, high = (libver_dict[x] for x in libver) + else: + # we default to earliest + low, high = h5f.LIBVER_EARLIEST, h5f.LIBVER_LATEST + plist.set_libver_bounds(low, high) + plist.set_alignment(alignment_threshold, alignment_interval) + + cache_settings = list(plist.get_cache()) + if rdcc_nslots is not None: + cache_settings[1] = rdcc_nslots + if rdcc_nbytes is not None: + cache_settings[2] = rdcc_nbytes + if rdcc_w0 is not None: + cache_settings[3] = rdcc_w0 + plist.set_cache(*cache_settings) + + if page_buf_size: + plist.set_page_buffer_size(int(page_buf_size), int(min_meta_keep), + int(min_raw_keep)) + + if meta_block_size is not None: + plist.set_meta_block_size(int(meta_block_size)) + + if locking is not None: + set_fapl_file_locking(plist, locking) + + if driver is None or (driver == 'windows' and sys.platform == 'win32'): + # Prevent swallowing unused key arguments + if kwds: + msg = "'{key}' is an invalid keyword argument for this function" \ + .format(key=next(iter(kwds))) + raise TypeError(msg) + return plist + + try: + set_fapl = _drivers[driver] + except KeyError as exc: + raise ValueError(f'Unknown driver type {driver!r}') from exc + else: + if driver == 'ros3': + token = kwds.pop('session_token', None) + set_fapl(plist, **kwds) + if token: + if hdf5_version < (1, 14, 2): + raise ValueError('HDF5 >= 1.14.2 required for AWS session token') + plist.set_fapl_ros3_token(token) + else: + set_fapl(plist, **kwds) + + return plist + + +def make_fcpl(track_order=False, track_times=False, fs_strategy=None, fs_persist=False, + fs_threshold=1, fs_page_size=None): + """ Set up a file creation property list """ + plist = h5p.create(h5p.FILE_CREATE) + if track_order: + plist.set_link_creation_order( + h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED) + plist.set_attr_creation_order( + h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED) + if track_times is None: + track_times = False # Allow explicit None to mean h5py's default + if track_times in (True, False): + plist.set_obj_track_times(track_times) + else: + raise TypeError("track_times must be either True or False") + if fs_strategy: + strategies = { + 'fsm': h5f.FSPACE_STRATEGY_FSM_AGGR, + 'page': h5f.FSPACE_STRATEGY_PAGE, + 'aggregate': h5f.FSPACE_STRATEGY_AGGR, + 'none': h5f.FSPACE_STRATEGY_NONE + } + fs_strat_num = strategies.get(fs_strategy, -1) + if fs_strat_num == -1: + raise ValueError("Invalid file space strategy type") + + plist.set_file_space_strategy(fs_strat_num, fs_persist, fs_threshold) + if fs_page_size and fs_strategy == 'page': + plist.set_file_space_page_size(int(fs_page_size)) + return plist + + +def make_fid(name, mode, userblock_size, fapl, fcpl=None, swmr=False): + """ Get a new FileID by opening or creating a file. + Also validates mode argument.""" + + if userblock_size is not None: + if mode in ('r', 'r+'): + raise ValueError("User block may only be specified " + "when creating a file") + try: + userblock_size = int(userblock_size) + except (TypeError, ValueError): + raise ValueError("User block size must be an integer") from None + if fcpl is None: + fcpl = h5p.create(h5p.FILE_CREATE) + fcpl.set_userblock(userblock_size) + + if mode == 'r': + flags = h5f.ACC_RDONLY + if swmr: + flags |= h5f.ACC_SWMR_READ + fid = h5f.open(name, flags, fapl=fapl) + elif mode == 'r+': + fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) + elif mode in ['w-', 'x']: + fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) + elif mode == 'w': + fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl) + elif mode == 'a': + # Open in append mode (read/write). + # If that fails, create a new file only if it won't clobber an + # existing one (ACC_EXCL) + try: + fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) + # Not all drivers raise FileNotFoundError (commented those that do not) + except FileNotFoundError if fapl.get_driver() in ( + h5fd.SEC2, + h5fd.DIRECT if direct_vfd else -1, + # h5fd.STDIO, + # h5fd.CORE, + h5fd.FAMILY, + h5fd.WINDOWS, + # h5fd.MPIO, + # h5fd.MPIPOSIX, + h5fd.fileobj_driver, + h5fd.ROS3D if ros3 else -1, + ) else OSError: + fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) + else: + raise ValueError("Invalid mode; must be one of r, r+, w, w-, x, a") + + try: + if userblock_size is not None: + existing_fcpl = fid.get_create_plist() + if existing_fcpl.get_userblock() != userblock_size: + raise ValueError("Requested userblock size (%d) does not match that of existing file (%d)" % (userblock_size, existing_fcpl.get_userblock())) + except Exception as e: + fid.close() + raise e + + return fid + + +class File(Group): + + """ + Represents an HDF5 file. + """ + + @property + def attrs(self): + """ Attributes attached to this object """ + # hdf5 complains that a file identifier is an invalid location for an + # attribute. Instead of self, pass the root group to AttributeManager: + from . import attrs + with phil: + return attrs.AttributeManager(self['/']) + + @property + @with_phil + def filename(self): + """File name on disk""" + return filename_decode(h5f.get_name(self.id)) + + @property + @with_phil + def driver(self): + """Low-level HDF5 file driver used to open file""" + drivers = {h5fd.SEC2: 'sec2', + h5fd.STDIO: 'stdio', + h5fd.CORE: 'core', + h5fd.FAMILY: 'family', + h5fd.WINDOWS: 'windows', + h5fd.MPIO: 'mpio', + h5fd.MPIPOSIX: 'mpiposix', + h5fd.fileobj_driver: 'fileobj'} + if ros3: + drivers[h5fd.ROS3D] = 'ros3' + if direct_vfd: + drivers[h5fd.DIRECT] = 'direct' + return drivers.get(self.id.get_access_plist().get_driver(), 'unknown') + + @property + @with_phil + def mode(self): + """ Python mode used to open file """ + write_intent = h5f.ACC_RDWR | h5f.ACC_SWMR_WRITE + return 'r+' if self.id.get_intent() & write_intent else 'r' + + @property + @with_phil + def libver(self): + """File format version bounds (2-tuple: low, high)""" + bounds = self.id.get_access_plist().get_libver_bounds() + return tuple(libver_dict_r[x] for x in bounds) + + @property + @with_phil + def userblock_size(self): + """ User block size (in bytes) """ + fcpl = self.id.get_create_plist() + return fcpl.get_userblock() + + @property + @with_phil + def meta_block_size(self): + """ Meta block size (in bytes) """ + fapl = self.id.get_access_plist() + return fapl.get_meta_block_size() + + if mpi: + + @property + @with_phil + def atomic(self): + """ Set/get MPI-IO atomic mode + """ + return self.id.get_mpi_atomicity() + + @atomic.setter + @with_phil + def atomic(self, value): + # pylint: disable=missing-docstring + self.id.set_mpi_atomicity(value) + + @property + @with_phil + def swmr_mode(self): + """ Controls single-writer multiple-reader mode """ + return bool(self.id.get_intent() & (h5f.ACC_SWMR_READ | h5f.ACC_SWMR_WRITE)) + + @swmr_mode.setter + @with_phil + def swmr_mode(self, value): + # pylint: disable=missing-docstring + if value: + self.id.start_swmr_write() + else: + raise ValueError("It is not possible to forcibly switch SWMR mode off.") + + def __init__(self, name, mode='r', driver=None, libver=None, userblock_size=None, swmr=False, + rdcc_nslots=None, rdcc_nbytes=None, rdcc_w0=None, track_order=None, + fs_strategy=None, fs_persist=False, fs_threshold=1, fs_page_size=None, + page_buf_size=None, min_meta_keep=0, min_raw_keep=0, locking=None, + alignment_threshold=1, alignment_interval=1, meta_block_size=None, + *, track_times=False, **kwds): + """Create a new file object. + + See the h5py user guide for a detailed explanation of the options. + + name + Name of the file on disk, or file-like object. Note: for files + created with the 'core' driver, HDF5 still requires this be + non-empty. + mode + r Readonly, file must exist (default) + r+ Read/write, file must exist + w Create file, truncate if exists + w- or x Create file, fail if exists + a Read/write if exists, create otherwise + driver + Name of the driver to use. Legal values are None (default, + recommended), 'core', 'sec2', 'direct', 'stdio', 'mpio', 'ros3'. + libver + Library version bounds. Supported values: 'earliest', 'v108', + 'v110', 'v112', 'v114', 'v200' and 'latest' depending on the + version of libhdf5 h5py is built against. + userblock_size + Desired size of user block. Only allowed when creating a new + file (mode w, w- or x). + swmr + Open the file in SWMR read mode. Only used when mode = 'r'. + rdcc_nslots + The number of chunk slots in the raw data chunk cache for this + file. Increasing this value reduces the number of cache collisions, + but slightly increases the memory used. Due to the hashing + strategy, this value should ideally be a prime number. As a rule of + thumb, this value should be at least 10 times the number of chunks + that can fit in rdcc_nbytes bytes. For maximum performance, this + value should be set approximately 100 times that number of + chunks. The default value is 521. Applies to all datasets unless individually changed. + rdcc_nbytes + Total size of the dataset chunk cache in bytes. The default size per + dataset is 1024**2 (1 MiB) for HDF5 before 2.0 and 8 MiB for HDF5 + 2.0 and later. Applies to all datasets unless individually changed. + rdcc_w0 + The chunk preemption policy for all datasets. This must be + between 0 and 1 inclusive and indicates the weighting according to + which chunks which have been fully read or written are penalized + when determining which chunks to flush from cache. A value of 0 + means fully read or written chunks are treated no differently than + other chunks (the preemption is strictly LRU) while a value of 1 + means fully read or written chunks are always preempted before + other chunks. If your application only reads or writes data once, + this can be safely set to 1. Otherwise, this should be set lower + depending on how often you re-read or re-write the same data. The + default value is 0.75. Applies to all datasets unless individually changed. + track_order + Track dataset/group/attribute creation order under root group + if True. If None use global default h5.get_config().track_order. + track_times: bool or None, default: False + If True, store timestamps for this group in the file. + If None, fall back to the default value. + fs_strategy + The file space handling strategy to be used. Only allowed when + creating a new file (mode w, w- or x). Defined as: + "fsm" FSM, Aggregators, VFD + "page" Paged FSM, VFD + "aggregate" Aggregators, VFD + "none" VFD + If None use HDF5 defaults. + fs_page_size + File space page size in bytes. Only used when fs_strategy="page". If + None use the HDF5 default (4096 bytes). + fs_persist + A boolean value to indicate whether free space should be persistent + or not. Only allowed when creating a new file. The default value + is False. + fs_threshold + The smallest free-space section size that the free space manager + will track. Only allowed when creating a new file. The default + value is 1. + page_buf_size + Page buffer size in bytes. Only allowed for HDF5 files created with + fs_strategy="page". Must be a power of two value and greater or + equal than the file space page size when creating the file. It is + not used by default. + min_meta_keep + Minimum percentage of metadata to keep in the page buffer before + allowing pages containing metadata to be evicted. Applicable only if + page_buf_size is set. Default value is zero. + min_raw_keep + Minimum percentage of raw data to keep in the page buffer before + allowing pages containing raw data to be evicted. Applicable only if + page_buf_size is set. Default value is zero. + locking + The file locking behavior. Defined as: + + - False (or "false") -- Disable file locking + - True (or "true") -- Enable file locking + - "best-effort" -- Enable file locking but ignore some errors + - None -- Use HDF5 defaults + + .. warning:: + + The HDF5_USE_FILE_LOCKING environment variable can override + this parameter. + + alignment_threshold + Together with ``alignment_interval``, this property ensures that + any file object greater than or equal in size to the alignment + threshold (in bytes) will be aligned on an address which is a + multiple of alignment interval. + + alignment_interval + This property should be used in conjunction with + ``alignment_threshold``. See the description above. For more + details, see + https://support.hdfgroup.org/documentation/hdf5/latest/group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a + + meta_block_size + Set the current minimum size, in bytes, of new metadata block allocations. + See https://support.hdfgroup.org/documentation/hdf5/latest/group___f_a_p_l.html#ga8822e3dedc8e1414f20871a87d533cb1 + + Additional keywords + Passed on to the selected file driver. + """ + if driver == 'ros3': + if not ros3: + raise ValueError("h5py was built without ROS3 support, can't use ros3 driver") + if hdf5_version < (2, 0, 0): + from urllib.parse import urlparse + url = urlparse(name) + if url.scheme == 's3': + aws_region = kwds.get('aws_region', b'').decode('ascii') + if len(aws_region) == 0: + raise ValueError('AWS region required for s3:// location') + name = f'https://s3.{aws_region}.amazonaws.com/{url.netloc}{url.path}' + elif url.scheme not in ('https', 'http'): + raise ValueError(f'{name}: S3 location must begin with ' + 'either "https://", "http://", or "s3://"') + + if isinstance(name, _objects.ObjectID): + if fs_strategy: + raise ValueError("Unable to set file space strategy of an existing file") + + with phil: + fid = h5i.get_file_id(name) + else: + if hasattr(name, 'read') and hasattr(name, 'seek'): + if driver not in (None, 'fileobj'): + raise ValueError("Driver must be 'fileobj' for file-like object if specified.") + driver = 'fileobj' + if kwds.get('fileobj', name) != name: + raise ValueError("Invalid value of 'fileobj' argument; " + "must equal to file-like object if specified.") + kwds.update(fileobj=name) + name = repr(name).encode('ASCII', 'replace') + else: + name = filename_encode(name) + + if track_order is None: + track_order = h5.get_config().track_order + + if fs_strategy and mode not in ('w', 'w-', 'x'): + raise ValueError("Unable to set file space strategy of an existing file") + + if swmr and mode != 'r': + warn( + "swmr=True only affects read ('r') mode. For swmr write " + "mode, set f.swmr_mode = True after opening the file.", + stacklevel=2, + ) + + with phil: + fapl = make_fapl(driver, libver, rdcc_nslots, rdcc_nbytes, rdcc_w0, + locking, page_buf_size, min_meta_keep, min_raw_keep, + alignment_threshold=alignment_threshold, + alignment_interval=alignment_interval, + meta_block_size=meta_block_size, + **kwds) + fcpl = make_fcpl(track_order=track_order, track_times=track_times, + fs_strategy=fs_strategy, fs_persist=fs_persist, + fs_threshold=fs_threshold, fs_page_size=fs_page_size) + fid = make_fid(name, mode, userblock_size, fapl, fcpl, swmr=swmr) + + if isinstance(libver, tuple): + self._libver = libver + else: + self._libver = (libver, 'latest') + + super().__init__(fid) + + _in_memory_file_counter = 0 + + @classmethod + @with_phil + def in_memory(cls, file_image=None, **kwargs): + """Create an HDF5 file in memory, without an underlying file + + file_image + The initial file contents as bytes (or anything that supports the + Python buffer interface). HDF5 takes a copy of this data. + block_size + Chunk size for new memory alloactions (default 64 KiB). + + Other keyword arguments are like File(), although name, mode, + driver and locking can't be passed. + """ + for k in ('driver', 'locking', 'backing_store'): + if k in kwargs: + raise TypeError( + f"File.in_memory() got an unexpected keyword argument {k!r}" + ) + fcpl_kwargs = {} + for k in inspect.signature(make_fcpl).parameters: + if k in kwargs: + fcpl_kwargs[k] = kwargs.pop(k) + fcpl = make_fcpl(**fcpl_kwargs) + + fapl = make_fapl(driver="core", backing_store=False, **kwargs) + if file_image: + if fcpl_kwargs: + kw = ', '.join(fcpl_kwargs) + raise TypeError(f"{kw} parameters cannot be used with file_image") + fapl.set_file_image(file_image) + + # We have to give HDF5 a filename, but it should never use it. + # This is a hint both in memory, and in case a bug ever creates a file. + # The name also needs to be different from any other open file; + # we use a simple counter (protected by the 'phil' lock) for this. + name = b"h5py_in_memory_nonfile_%d" % cls._in_memory_file_counter + cls._in_memory_file_counter += 1 + + if file_image: + fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) + else: + fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) + return cls(fid) + + def close(self): + """ Close the file. All open objects become invalid """ + with phil: + # Check that the file is still open, otherwise skip + if self.id.valid: + # We have to explicitly murder all open objects related to the file + + # Close file-resident objects first, then the files. + # Otherwise we get errors in MPI mode. + self.id._close_open_objects(h5f.OBJ_LOCAL | ~h5f.OBJ_FILE) + self.id._close_open_objects(h5f.OBJ_LOCAL | h5f.OBJ_FILE) + + self.id.close() + + def flush(self): + """ Tell the HDF5 library to flush its buffers. + """ + with phil: + h5f.flush(self.id) + + @with_phil + def __enter__(self): + return self + + @with_phil + def __exit__(self, *args): + if self.id: + self.close() + + @with_phil + def __repr__(self): + if not self.id: + r = '<Closed HDF5 file>' + else: + # Filename has to be forced to Unicode if it comes back bytes + # Mode is always a "native" string + filename = self.filename + if isinstance(filename, bytes): # Can't decode fname + filename = filename.decode('utf8', 'replace') + r = f'<HDF5 file "{os.path.basename(filename)}" (mode {self.mode})>' + + return r diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/filters.py b/.venv/lib/python3.11/site-packages/h5py/_hl/filters.py new file mode 100644 index 0000000000000000000000000000000000000000..83f8d4939b64f6fcb1ce5d69baf42c9f4160f2d5 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/filters.py @@ -0,0 +1,412 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Implements support for HDF5 compression filters via the high-level + interface. The following types of filter are available: + + "gzip" + Standard DEFLATE-based compression, at integer levels from 0 to 9. + Built-in to all public versions of HDF5. Use this if you want a + decent-to-good ratio, good portability, and don't mind waiting. + + "lzf" + Custom compression filter for h5py. This filter is much, much faster + than gzip (roughly 10x in compression vs. gzip level 4, and 3x faster + in decompressing), but at the cost of a worse compression ratio. Use + this if you want cheap compression and portability is not a concern. + + "szip" + Access to the HDF5 SZIP encoder. SZIP is a non-mainstream compression + format used in space science on integer and float datasets. SZIP is + subject to license requirements, which means the encoder is not + guaranteed to be always available. However, it is also much faster + than gzip. + + The following constants in this module are also useful: + + decode + Tuple of available filter names for decoding + + encode + Tuple of available filter names for encoding +""" +from collections.abc import Mapping +import operator + +import numpy as np +from .base import product +from .compat import filename_encode +from .. import h5z, h5p, h5d, h5f + + +_COMP_FILTERS = {'gzip': h5z.FILTER_DEFLATE, + 'szip': h5z.FILTER_SZIP, + 'lzf': h5z.FILTER_LZF, + 'shuffle': h5z.FILTER_SHUFFLE, + 'fletcher32': h5z.FILTER_FLETCHER32, + 'scaleoffset': h5z.FILTER_SCALEOFFSET } +_FILL_TIME_ENUM = {'alloc': h5d.FILL_TIME_ALLOC, + 'never': h5d.FILL_TIME_NEVER, + 'ifset': h5d.FILL_TIME_IFSET, + } + +DEFAULT_GZIP = 4 +DEFAULT_SZIP = ('nn', 8) + +def _gen_filter_tuples(): + """ Bootstrap function to figure out what filters are available. """ + dec = [] + enc = [] + for name, code in _COMP_FILTERS.items(): + if h5z.filter_avail(code): + info = h5z.get_filter_info(code) + if info & h5z.FILTER_CONFIG_ENCODE_ENABLED: + enc.append(name) + if info & h5z.FILTER_CONFIG_DECODE_ENABLED: + dec.append(name) + + return tuple(dec), tuple(enc) + +decode, encode = _gen_filter_tuples() + +def _external_entry(entry): + """ Check for and return a well-formed entry tuple for + a call to h5p.set_external. """ + # We require only an iterable entry but also want to guard against + # raising a confusing exception from unpacking below a str or bytes that + # was mistakenly passed as an entry. We go further than that and accept + # only a tuple, which allows simpler documentation and exception + # messages. + if not isinstance(entry, tuple): + raise TypeError( + "Each external entry must be a tuple of (name, offset, size)") + name, offset, size = entry # raise ValueError without three elements + name = filename_encode(name) + offset = operator.index(offset) + size = operator.index(size) + return (name, offset, size) + +def _normalize_external(external): + """ Normalize external into a well-formed list of tuples and return. """ + if external is None: + return [] + try: + # Accept a solitary name---a str, bytes, or os.PathLike acceptable to + # filename_encode. + return [_external_entry((external, 0, h5f.UNLIMITED))] + except TypeError: + pass + # Check and rebuild each entry to be well-formed. + return [_external_entry(entry) for entry in external] + +class FilterRefBase(Mapping): + """Base class for referring to an HDF5 and describing its options + + Your subclass must define filter_id, and may define a filter_options tuple. + """ + filter_id = None + filter_options = () + + # Mapping interface supports using instances as **kwargs for compatibility + # with older versions of h5py + @property + def _kwargs(self): + return { + 'compression': self.filter_id, + 'compression_opts': self.filter_options + } + + def __hash__(self): + return hash((self.filter_id, self.filter_options)) + + def __eq__(self, other): + return ( + isinstance(other, FilterRefBase) + and self.filter_id == other.filter_id + and self.filter_options == other.filter_options + ) + + def __len__(self): + return len(self._kwargs) + + def __iter__(self): + return iter(self._kwargs) + + def __getitem__(self, item): + return self._kwargs[item] + +class Gzip(FilterRefBase): + filter_id = h5z.FILTER_DEFLATE + + def __init__(self, level=DEFAULT_GZIP): + self.filter_options = (level,) + +def fill_dcpl(plist, shape, dtype, chunks, compression, compression_opts, + shuffle, fletcher32, maxshape, scaleoffset, external, + allow_unknown_filter=False, *, fill_time=None): + """ Generate a dataset creation property list. + + Undocumented and subject to change without warning. + """ + + if shape is None or shape == (): + shapetype = 'Empty' if shape is None else 'Scalar' + if any((chunks, compression, compression_opts, shuffle, fletcher32, + scaleoffset is not None)): + raise TypeError( + f"{shapetype} datasets don't support chunk/filter options" + ) + if maxshape and maxshape != (): + raise TypeError(f"{shapetype} datasets cannot be extended") + return h5p.create(h5p.DATASET_CREATE) + + def rq_tuple(tpl, name): + """ Check if chunks/maxshape match dataset rank """ + if tpl in (None, True): + return + try: + tpl = tuple(tpl) + except TypeError as exc: + raise TypeError(f'{name!r} argument must be None or a sequence object') from exc + if len(tpl) != len(shape): + raise ValueError(f'{name!r} must have same rank as dataset shape') + + rq_tuple(chunks, 'chunks') + rq_tuple(maxshape, 'maxshape') + + if compression is not None: + if isinstance(compression, FilterRefBase): + compression_opts = compression.filter_options + compression = compression.filter_id + + if compression not in encode and not isinstance(compression, int): + raise ValueError('Compression filter "%s" is unavailable' % compression) + + if compression == 'gzip': + if compression_opts is None: + gzip_level = DEFAULT_GZIP + elif compression_opts in range(10): + gzip_level = compression_opts + else: + raise ValueError("GZIP setting must be an integer from 0-9, not %r" % compression_opts) + + elif compression == 'lzf': + if compression_opts is not None: + raise ValueError("LZF compression filter accepts no options") + + elif compression == 'szip': + if compression_opts is None: + compression_opts = DEFAULT_SZIP + + err = "SZIP options must be a 2-tuple ('ec'|'nn', even integer 0-32)" + try: + szmethod, szpix = compression_opts + except TypeError as exc: + raise TypeError(err) from exc + if szmethod not in ('ec', 'nn'): + raise ValueError(err) + if not (0<szpix<=32 and szpix%2 == 0): + raise ValueError(err) + + elif compression_opts is not None: + # Can't specify just compression_opts by itself. + raise TypeError("Compression method must be specified") + + if scaleoffset is not None: + # scaleoffset must be an integer when it is not None or False, + # except for integral data, for which scaleoffset == True is + # permissible (will use SO_INT_MINBITS_DEFAULT) + + if scaleoffset < 0: + raise ValueError('scale factor must be >= 0') + + if dtype.kind == 'f': + if scaleoffset is True: + raise ValueError('integer scaleoffset must be provided for ' + 'floating point types') + elif dtype.kind in ('u', 'i'): + if scaleoffset is True: + scaleoffset = h5z.SO_INT_MINBITS_DEFAULT + else: + raise TypeError('scale/offset filter only supported for integer ' + 'and floating-point types') + + # Scale/offset following fletcher32 in the filter chain will (almost?) + # always triggers a read error, as most scale/offset settings are + # lossy. Since fletcher32 must come first (see comment below) we + # simply prohibit the combination of fletcher32 and scale/offset. + if fletcher32: + raise ValueError('fletcher32 cannot be used with potentially lossy' + ' scale/offset filter') + + external = _normalize_external(external) + # End argument validation + + if (chunks is True) or (chunks is None and any(( + shuffle, + fletcher32, + compression, + (maxshape and not len(external)), + scaleoffset is not None, + ))): + chunks = guess_chunk(shape, maxshape, dtype.itemsize) + + if maxshape is True: + maxshape = (None,)*len(shape) + + if chunks is not None: + plist.set_chunk(chunks) + + if fill_time is not None: + if (ft := _FILL_TIME_ENUM.get(fill_time)) is not None: + plist.set_fill_time(ft) + else: + msg = ("fill_time must be one of the following choices: 'alloc', " + f"'never' or 'ifset', but it is {fill_time}.") + raise ValueError(msg) + + # scale-offset must come before shuffle and compression + if scaleoffset is not None: + if dtype.kind in ('u', 'i'): + plist.set_scaleoffset(h5z.SO_INT, scaleoffset) + else: # dtype.kind == 'f' + plist.set_scaleoffset(h5z.SO_FLOAT_DSCALE, scaleoffset) + + for item in external: + plist.set_external(*item) + + if shuffle: + plist.set_shuffle() + + if compression == 'gzip': + plist.set_deflate(gzip_level) + elif compression == 'lzf': + plist.set_filter(h5z.FILTER_LZF, h5z.FLAG_OPTIONAL) + elif compression == 'szip': + opts = {'ec': h5z.SZIP_EC_OPTION_MASK, 'nn': h5z.SZIP_NN_OPTION_MASK} + plist.set_szip(opts[szmethod], szpix) + elif isinstance(compression, int): + if not allow_unknown_filter and not h5z.filter_avail(compression): + raise ValueError("Unknown compression filter number: %s" % compression) + + plist.set_filter(compression, h5z.FLAG_OPTIONAL, compression_opts) + + # `fletcher32` must come after `compression`, otherwise, if `compression` + # is "szip" and the data is 64bit, the fletcher32 checksum will be wrong + # (see GitHub issue #953). + if fletcher32: + plist.set_fletcher32() + + return plist + +def get_filter_name(code): + """ + Return the name of the compression filter for a given filter identifier. + + Undocumented and subject to change without warning. + """ + filters = {h5z.FILTER_DEFLATE: 'gzip', h5z.FILTER_SZIP: 'szip', + h5z.FILTER_SHUFFLE: 'shuffle', h5z.FILTER_FLETCHER32: 'fletcher32', + h5z.FILTER_LZF: 'lzf', h5z.FILTER_SCALEOFFSET: 'scaleoffset'} + return filters.get(code, str(code)) + +def get_filters(plist): + """ Extract a dictionary of active filters from a DCPL, along with + their settings. + + Undocumented and subject to change without warning. + """ + + pipeline = {} + + nfilters = plist.get_nfilters() + + for i in range(nfilters): + + code, _, vals, _ = plist.get_filter(i) + + if code == h5z.FILTER_DEFLATE: + vals = vals[0] # gzip level + + elif code == h5z.FILTER_SZIP: + mask, pixels = vals[0:2] + if mask & h5z.SZIP_EC_OPTION_MASK: + mask = 'ec' + elif mask & h5z.SZIP_NN_OPTION_MASK: + mask = 'nn' + else: + raise TypeError("Unknown SZIP configuration") + vals = (mask, pixels) + elif code == h5z.FILTER_LZF: + vals = None + else: + if len(vals) == 0: + vals = None + + pipeline[get_filter_name(code)] = vals + + return pipeline + +CHUNK_BASE = 16*1024 # Multiplier by which chunks are adjusted +CHUNK_MIN = 8*1024 # Soft lower limit (8k) +CHUNK_MAX = 1024*1024 # Hard upper limit (1M) + +def guess_chunk(shape, maxshape, typesize): + """ Guess an appropriate chunk layout for a dataset, given its shape and + the size of each element in bytes. Will allocate chunks only as large + as MAX_SIZE. Chunks are generally close to some power-of-2 fraction of + each axis, slightly favoring bigger values for the last index. + + Undocumented and subject to change without warning. + """ + # pylint: disable=unused-argument + + # For unlimited dimensions we have to guess 1024 + shape = tuple((x if x!=0 else 1024) for i, x in enumerate(shape)) + + ndims = len(shape) + if ndims == 0: + raise ValueError("Chunks not allowed for scalar datasets.") + + chunks = np.array(shape, dtype='=f8') + if not np.all(np.isfinite(chunks)): + raise ValueError("Illegal value in chunk tuple") + + # Determine the optimal chunk size in bytes using a PyTables expression. + # This is kept as a float. + dset_size = product(chunks)*typesize + target_size = CHUNK_BASE * (2**np.log10(dset_size/(1024.*1024))) + + if target_size > CHUNK_MAX: + target_size = CHUNK_MAX + elif target_size < CHUNK_MIN: + target_size = CHUNK_MIN + + idx = 0 + while True: + # Repeatedly loop over the axes, dividing them by 2. Stop when: + # 1a. We're smaller than the target chunk size, OR + # 1b. We're within 50% of the target chunk size, AND + # 2. The chunk is smaller than the maximum chunk size + + chunk_bytes = product(chunks)*typesize + + if (chunk_bytes < target_size or \ + abs(chunk_bytes-target_size)/target_size < 0.5) and \ + chunk_bytes < CHUNK_MAX: + break + + if product(chunks) == 1: + break # Element size larger than CHUNK_MAX + + chunks[idx%ndims] = np.ceil(chunks[idx%ndims] / 2.0) + idx += 1 + + return tuple(int(x) for x in chunks) diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/group.py b/.venv/lib/python3.11/site-packages/h5py/_hl/group.py new file mode 100644 index 0000000000000000000000000000000000000000..f803bea29c127cd10284fae42fe29654aca3b3e0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/group.py @@ -0,0 +1,898 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Implements support for high-level access to HDF5 groups. +""" + +from contextlib import contextmanager +import posixpath as pp +import numpy + +from .compat import filename_decode, filename_encode + +from .. import h5, h5f, h5g, h5i, h5o, h5r, h5t, h5l, h5p +from . import base +from .base import HLObject, MutableMappingHDF5, phil, with_phil +from . import dataset +from . import datatype +from .vds import vds_support + + +def set_fapl_file_locking(fapl, locking): + if locking in ("false", False): + fapl.set_file_locking(False, ignore_when_disabled=False) + elif locking in ("true", True): + fapl.set_file_locking(True, ignore_when_disabled=False) + elif locking == "best-effort": + fapl.set_file_locking(True, ignore_when_disabled=True) + else: + raise ValueError(f"Unsupported locking value: {locking}") + + +def make_lapl(file, elink_mode=None, elink_swmr=None, elink_locking=None): + """Set up a link access property list""" + if elink_mode is None and elink_swmr is None and elink_locking is None: + return None + + if file.mode == "r" and elink_mode == "r+": + raise ValueError("Opening external links in write mode from a file opened in read-only mode is not supported") + + lapl = h5p.create(h5p.LINK_ACCESS) + + if elink_mode is not None or elink_swmr is not None: + mode = file.mode if elink_mode is None else elink_mode + swmr_mode = file.swmr_mode if elink_swmr is None else elink_swmr + + if file.mode == "r+" and file.swmr_mode and (mode != "r+" or not swmr_mode): + raise ValueError("Changing external links access mode from a file opened in SWMR write mode is not supported") + + if mode == "r": + flags = h5f.ACC_RDONLY + if swmr_mode: + flags |= h5f.ACC_SWMR_READ + elif mode == "r+": + flags = h5f.ACC_RDWR + if swmr_mode: + flags |= h5f.ACC_SWMR_WRITE + else: + raise RuntimeError(f"Unsupported link access mode: {mode}") + lapl.set_elink_acc_flags(flags) + + if elink_locking is not None: + fapl = file.id.get_access_plist() + set_fapl_file_locking(fapl, elink_locking) + lapl.set_elink_fapl(fapl) + + return lapl + + +class Group(HLObject, MutableMappingHDF5): + + """ Represents an HDF5 group. + """ + + def __init__(self, bind): + """ Create a new Group object by binding to a low-level GroupID. + """ + with phil: + if not isinstance(bind, h5g.GroupID): + raise ValueError("%s is not a GroupID" % bind) + super().__init__(bind) + + def create_group(self, name, track_order=None, *, track_times=False): + """ Create and return a new subgroup. + + Name may be absolute or relative. Fails if the target name already + exists. + + track_order + Track dataset/group/attribute creation order under this group + if True. If None use global default h5.get_config().track_order. + track_times: bool or None, default: False + If True, store timestamps for this group in the file. + If None, fall back to the default value. + """ + if track_order is None: + track_order = h5.get_config().track_order + + with phil: + name, lcpl = self._e(name, lcpl=True) + gcpl = h5p.create(h5p.GROUP_CREATE) + if track_order: + order_flags = h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED + gcpl.set_link_creation_order(order_flags) + gcpl.set_attr_creation_order(order_flags) + if track_times is None: + track_times = False # Allow explicit None to mean h5py's default + if track_times in (True, False): + gcpl.set_obj_track_times(track_times) + else: + raise TypeError("track_times must be either True, False, or None") + gid = h5g.create(self.id, name, lcpl=lcpl, gcpl=gcpl) + return Group(gid) + + def create_dataset(self, name, shape=None, dtype=None, data=None, **kwds): + """ Create a new HDF5 dataset + + name + Name of the dataset (absolute or relative). Provide None to make + an anonymous dataset. + shape + Dataset shape. Use "()" for scalar datasets. Required if "data" + isn't provided. + dtype + Numpy dtype or string. If omitted, dtype('f') will be used. + Required if "data" isn't provided; otherwise, overrides data + array's dtype. + data + Provide data to initialize the dataset. If used, you can omit + shape and dtype arguments. + + Keyword-only arguments: + + chunks + (Tuple or int) Chunk shape, or True to enable auto-chunking. Integers can + be used for 1D shape. + + maxshape + (Tuple or int) Make the dataset resizable up to this shape. Use None for + axes within the tuple you want to be unlimited. Integers can be used for 1D shape. + For 1D datasets with unlimited maxshape, a shape tuple of length 1 must be + provided, ``(None,)``. Passing ``None`` sets ``maxshape` to `shape`, making the + dataset un-resizable, which is the default. + compression + (String or int) Compression strategy. Legal values are 'gzip', + 'szip', 'lzf'. If an integer in range(10), this indicates gzip + compression level. Otherwise, an integer indicates the number of a + dynamically loaded compression filter. + compression_opts + Compression settings. This is an integer for gzip, 2-tuple for + szip, etc. If specifying a dynamically loaded compression filter + number, this must be a tuple of values. + scaleoffset + (Integer) Enable scale/offset filter for (usually) lossy + compression of integer or floating-point data. For integer + data, the value of scaleoffset is the number of bits to + retain (pass 0 to let HDF5 determine the minimum number of + bits necessary for lossless compression). For floating point + data, scaleoffset is the number of digits after the decimal + place to retain; stored values thus have absolute error + less than 0.5*10**(-scaleoffset). + shuffle + (T/F) Enable shuffle filter. + fletcher32 + (T/F) Enable fletcher32 error detection. Not permitted in + conjunction with the scale/offset filter. + fillvalue + (Scalar) Use this value for uninitialized parts of the dataset. + track_times + (T/F) Enable dataset creation timestamps. + track_order + (T/F) Track attribute creation order if True. If omitted use + global default h5.get_config().track_order. + external + (Iterable of tuples) Sets the external storage property, thus + designating that the dataset will be stored in one or more + non-HDF5 files external to the HDF5 file. Adds each tuple + of (name, offset, size) to the dataset's list of external files. + Each name must be a str, bytes, or os.PathLike; each offset and + size, an integer. If only a name is given instead of an iterable + of tuples, it is equivalent to [(name, 0, h5py.h5f.UNLIMITED)]. + efile_prefix + (String) External dataset file prefix for dataset access property + list. Does not persist in the file. + virtual_prefix + (String) Virtual dataset file prefix for dataset access property + list. Does not persist in the file. + allow_unknown_filter + (T/F) Do not check that the requested filter is available for use. + This should only be used with ``write_direct_chunk``, where the caller + compresses the data before handing it to h5py. + rdcc_nbytes + Total size of the dataset's chunk cache in bytes. The default size + is 1024**2 (1 MiB) for HDF5 before 2.0 and 8 MiB for HDF5 2.0 or later. + rdcc_w0 + The chunk preemption policy for this dataset. This must be + between 0 and 1 inclusive and indicates the weighting according to + which chunks which have been fully read or written are penalized + when determining which chunks to flush from cache. A value of 0 + means fully read or written chunks are treated no differently than + other chunks (the preemption is strictly LRU) while a value of 1 + means fully read or written chunks are always preempted before + other chunks. If your application only reads or writes data once, + this can be safely set to 1. Otherwise, this should be set lower + depending on how often you re-read or re-write the same data. The + default value is 0.75. + rdcc_nslots + The number of chunk slots in the dataset's chunk cache. Increasing + this value reduces the number of cache collisions, but slightly + increases the memory used. Due to the hashing strategy, this value + should ideally be a prime number. As a rule of thumb, this value + should be at least 10 times the number of chunks that can fit in + rdcc_nbytes bytes. For maximum performance, this value should be set + approximately 100 times that number of chunks. The default value is + 521. + """ + if 'track_order' not in kwds: + kwds['track_order'] = h5.get_config().track_order + + if 'efile_prefix' in kwds: + kwds['efile_prefix'] = self._e(kwds['efile_prefix']) + + if 'virtual_prefix' in kwds: + kwds['virtual_prefix'] = self._e(kwds['virtual_prefix']) + + with phil: + group = self + if name: + name = self._e(name) + if b'/' in name.lstrip(b'/'): + parent_path, name = name.rsplit(b'/', 1) + group = self.require_group(parent_path) + + dsid = dataset.make_new_dset(group, shape, dtype, data, name, **kwds) + dset = dataset.Dataset(dsid) + return dset + + if vds_support: + def create_virtual_dataset(self, name, layout, fillvalue=None): + """Create a new virtual dataset in this group. + + See virtual datasets in the docs for more information. + + name + (str) Name of the new dataset + + layout + (VirtualLayout) Defines the sources for the virtual dataset + + fillvalue + The value to use where there is no data. + + """ + with phil: + group = self + + if name: + name = self._e(name) + if b'/' in name.lstrip(b'/'): + parent_path, name = name.rsplit(b'/', 1) + group = self.require_group(parent_path) + + dsid = layout.make_dataset( + group, name=name, fillvalue=fillvalue, + ) + dset = dataset.Dataset(dsid) + + return dset + + @contextmanager + def build_virtual_dataset( + self, name, shape, dtype, maxshape=None, fillvalue=None + ): + """Assemble a virtual dataset in this group. + + This is used as a context manager:: + + with f.build_virtual_dataset('virt', (10, 1000), np.uint32) as layout: + layout[0] = h5py.VirtualSource('foo.h5', 'data', (1000,)) + + name + (str) Name of the new dataset + shape + (tuple) Shape of the dataset + dtype + A numpy dtype for data read from the virtual dataset + maxshape + (tuple, optional) Maximum dimensions if the dataset can grow. + Use None for unlimited dimensions. + fillvalue + The value used where no data is available. + """ + from .vds import VirtualLayout + layout = VirtualLayout(shape, dtype, maxshape, self.file.filename) + yield layout + + self.create_virtual_dataset(name, layout, fillvalue) + + def require_dataset(self, name, shape, dtype, exact=False, **kwds): + """ Open a dataset, creating it if it doesn't exist. + + If keyword "exact" is False (default), an existing dataset must have + the same shape and a conversion-compatible dtype to be returned. If + True, the shape and dtype must match exactly. + + If keyword "maxshape" is given, the maxshape and dtype must match + instead. + + If any of the keywords "rdcc_nslots", "rdcc_nbytes", or "rdcc_w0" are + given, they will be used to configure the dataset's chunk cache. + + Other dataset keywords (see create_dataset) may be provided, but are + only used if a new dataset is to be created. + + Raises TypeError if an incompatible object already exists, or if the + shape, maxshape or dtype don't match according to the above rules. + """ + if 'efile_prefix' in kwds: + kwds['efile_prefix'] = self._e(kwds['efile_prefix']) + + if 'virtual_prefix' in kwds: + kwds['virtual_prefix'] = self._e(kwds['virtual_prefix']) + + with phil: + if name not in self: + return self.create_dataset(name, *(shape, dtype), **kwds) + + if isinstance(shape, int): + shape = (shape,) + + try: + dsid = dataset.open_dset(self, self._e(name), **kwds) + dset = dataset.Dataset(dsid) + except KeyError as exc: + dset = self[name] + raise TypeError(f"Incompatible object ({dset.__class__.__name__}) already exists") from exc + + if shape != dset.shape: + if "maxshape" not in kwds: + raise TypeError("Shapes do not match (existing %s vs new %s)" % (dset.shape, shape)) + elif kwds["maxshape"] != dset.maxshape: + raise TypeError("Max shapes do not match (existing %s vs new %s)" % (dset.maxshape, kwds["maxshape"])) + + if exact: + if dtype != dset.dtype: + raise TypeError("Datatypes do not exactly match (existing %s vs new %s)" % (dset.dtype, dtype)) + elif not numpy.can_cast(dtype, dset.dtype): + raise TypeError("Datatypes cannot be safely cast (existing %s vs new %s)" % (dset.dtype, dtype)) + + return dset + + def create_dataset_like(self, name, other, **kwupdate): + """ Create a dataset similar to `other`. + + name + Name of the dataset (absolute or relative). Provide None to make + an anonymous dataset. + other + The dataset which the new dataset should mimic. All properties, such + as shape, dtype, chunking, ... will be taken from it, but no data + or attributes are being copied. + + Any dataset keywords (see create_dataset) may be provided, including + shape and dtype, in which case the provided values take precedence over + those from `other`. + """ + for k in ('shape', 'dtype', 'chunks', 'compression', + 'compression_opts', 'scaleoffset', 'shuffle', 'fletcher32', + 'fillvalue'): + kwupdate.setdefault(k, getattr(other, k)) + # TODO: more elegant way to pass these (dcpl to create_dataset?) + dcpl = other.id.get_create_plist() + kwupdate.setdefault('track_times', dcpl.get_obj_track_times()) + kwupdate.setdefault('track_order', dcpl.get_attr_creation_order() > 0) + + # Special case: the maxshape property always exists, but if we pass it + # to create_dataset, the new dataset will automatically get chunked + # layout. So we copy it only if it is different from shape. + if other.maxshape != other.shape: + kwupdate.setdefault('maxshape', other.maxshape) + + return self.create_dataset(name, **kwupdate) + + def require_group(self, name): + # TODO: support kwargs like require_dataset + """Return a group, creating it if it doesn't exist. + + TypeError is raised if something with that name already exists that + isn't a group. + """ + with phil: + if name not in self: + return self.create_group(name) + grp = self[name] + if not isinstance(grp, Group): + raise TypeError("Incompatible object (%s) already exists" % grp.__class__.__name__) + return grp + + def __getitem__(self, name): + """ Open an object in the file """ + return self._get(name) + + @with_phil + def _get(self, name, lapl=None): + if isinstance(name, h5r.Reference): + if lapl is not None: + raise TypeError( + "Access with link access properties is done with bytes or str, not Reference") + oid = h5r.dereference(name, self.id) + if oid is None: + raise ValueError("Invalid HDF5 object reference") + elif isinstance(name, (bytes, str)): + if lapl is None: + lapl = self._lapl + oid = h5o.open(self.id, self._e(name), lapl=lapl) + else: + raise TypeError("Accessing a group is done with bytes or str, " + "not {}".format(type(name))) + + otype = h5i.get_type(oid) + if otype == h5i.GROUP: + return Group(oid) + elif otype == h5i.DATASET: + return dataset.Dataset(oid, readonly=(self.file.mode == 'r')) + elif otype == h5i.DATATYPE: + return datatype.Datatype(oid) + else: + raise TypeError("Unknown object type") + + def get( + self, name, default=None, getclass=False, getlink=False, + elink_mode=None, elink_locking=None, elink_swmr=None, + ): + """ Retrieve an item or other information. + + "name" given only: + Return the item, or "default" if it doesn't exist + + "getclass" is True: + Return the class of object (Group, Dataset, etc.), or "default" + if nothing with that name exists + + "getlink" is True: + Return HardLink, SoftLink or ExternalLink instances. Return + "default" if nothing with that name exists. + + "getlink" and "getclass" are True: + Return HardLink, SoftLink and ExternalLink classes. Return + "default" if nothing with that name exists. + + "elink_mode": + External links access mode: + + - "r": Read-only + - "r+": Read/write + - None (default): Use current file access mode + + "elink_locking": + External links file locking behavior: + + - None (default) -- Use the current file locking + - False (or "false") -- Disable file locking + - True (or "true") -- Enable file locking + - "best-effort" -- Enable file locking but ignore some errors + + Warning: + The HDF5_USE_FILE_LOCKING environment variable can override + this parameter. + + "elink_swmr": + External link SWMR read mode. + Set to True only when elink_mode = 'r' and + current file is not opened in SWMR write mode. + By default, use current file SWMR mode. + + Example: + + >>> cls = group.get('foo', getclass=True) + >>> if cls == SoftLink: + """ + # pylint: disable=arguments-differ + + with phil: + if elink_mode is None and elink_swmr is None and elink_locking is None: + lapl = self._lapl + else: + lapl = make_lapl(self.file, elink_mode, elink_swmr, elink_locking) + + if not (getclass or getlink): + try: + return self._get(name, lapl) + except KeyError: + return default + + if name not in self: + return default + + elif getclass and not getlink: + typecode = h5o.get_info(self.id, self._e(name), lapl=lapl).type + + try: + return {h5o.TYPE_GROUP: Group, + h5o.TYPE_DATASET: dataset.Dataset, + h5o.TYPE_NAMED_DATATYPE: datatype.Datatype}[typecode] + except KeyError as exc: + raise TypeError("Unknown object type") from exc + + elif getlink: + typecode = self.id.links.get_info(self._e(name), lapl=lapl).type + + if typecode == h5l.TYPE_SOFT: + if getclass: + return SoftLink + linkbytes = self.id.links.get_val(self._e(name), lapl=lapl) + return SoftLink(self._d(linkbytes)) + + elif typecode == h5l.TYPE_EXTERNAL: + if getclass: + return ExternalLink + filebytes, linkbytes = self.id.links.get_val(self._e(name), lapl=lapl) + return ExternalLink( + filename_decode(filebytes), self._d(linkbytes) + ) + + elif typecode == h5l.TYPE_HARD: + return HardLink if getclass else HardLink() + + else: + raise TypeError("Unknown link type") + + def __setitem__(self, name, obj): + """ Add an object to the group. The name must not already be in use. + + The action taken depends on the type of object assigned: + + Named HDF5 object (Dataset, Group, Datatype) + A hard link is created at "name" which points to the + given object. + + SoftLink or ExternalLink + Create the corresponding link. + + Numpy ndarray + The array is converted to a dataset object, with default + settings (contiguous storage, etc.). + + Numpy dtype + Commit a copy of the datatype as a named datatype in the file. + + Anything else + Attempt to convert it to an ndarray and store it. Scalar + values are stored as scalar datasets. Raise ValueError if we + can't understand the resulting array dtype. + """ + with phil: + name, lcpl = self._e(name, lcpl=True) + + if isinstance(obj, HLObject): + h5o.link(obj.id, self.id, name, lcpl=lcpl, lapl=self._lapl) + + elif isinstance(obj, SoftLink): + self.id.links.create_soft(name, self._e(obj.path), lcpl=lcpl, lapl=self._lapl) + + elif isinstance(obj, ExternalLink): + fn = filename_encode(obj.filename) + self.id.links.create_external(name, fn, self._e(obj.path), + lcpl=lcpl, lapl=self._lapl) + + elif isinstance(obj, numpy.dtype): + htype = h5t.py_create(obj, logical=True) + htype.commit(self.id, name, lcpl=lcpl) + + else: + ds = self.create_dataset(None, data=obj) + h5o.link(ds.id, self.id, name, lcpl=lcpl) + + @with_phil + def __delitem__(self, name): + """ Delete (unlink) an item from this group. """ + self.id.unlink(self._e(name)) + + @with_phil + def __len__(self): + """ Number of members attached to this group """ + return self.id.get_num_objs() + + @with_phil + def __iter__(self): + """ Iterate over member names """ + for x in self.id.__iter__(): + yield self._d(x) + + @with_phil + def __reversed__(self): + """ Iterate over member names in reverse order. """ + for x in self.id.__reversed__(): + yield self._d(x) + + @with_phil + def __contains__(self, name): + """ Test if a member name exists """ + if hasattr(h5g, "_path_valid"): + if not self.id: + return False + return h5g._path_valid(self.id, self._e(name), self._lapl) + return self._e(name) in self.id + + def copy(self, source, dest, name=None, + shallow=False, expand_soft=False, expand_external=False, + expand_refs=False, without_attrs=False): + """Copy an object or group. + + The source can be a path, Group, Dataset, or Datatype object. The + destination can be either a path or a Group object. The source and + destinations need not be in the same file. + + If the source is a Group object, all objects contained in that group + will be copied recursively. + + When the destination is a Group object, by default the target will + be created in that group with its current name (basename of obj.name). + You can override that by setting "name" to a string. + + There are various options which all default to "False": + + - shallow: copy only immediate members of a group. + + - expand_soft: expand soft links into new objects. + + - expand_external: expand external links into new objects. + + - expand_refs: copy objects that are pointed to by references. + + - without_attrs: copy object without copying attributes. + + Example: + + >>> f = File('myfile.hdf5', 'w') + >>> f.create_group("MyGroup") + >>> list(f.keys()) + ['MyGroup'] + >>> f.copy('MyGroup', 'MyCopy') + >>> list(f.keys()) + ['MyGroup', 'MyCopy'] + + """ + with phil: + if isinstance(source, HLObject): + source_path = '.' + else: + # Interpret source as a path relative to this group + source_path = source + source = self + + if isinstance(dest, Group): + if name is not None: + dest_path = name + elif source_path == '.': + dest_path = pp.basename(h5i.get_name(source.id)) + else: + # copy source into dest group: dest_name/source_name + dest_path = pp.basename(h5i.get_name(source[source_path].id)) + + elif isinstance(dest, HLObject): + raise TypeError("Destination must be path or Group object") + else: + # Interpret destination as a path relative to this group + dest_path = dest + dest = self + + flags = 0 + if shallow: + flags |= h5o.COPY_SHALLOW_HIERARCHY_FLAG + if expand_soft: + flags |= h5o.COPY_EXPAND_SOFT_LINK_FLAG + if expand_external: + flags |= h5o.COPY_EXPAND_EXT_LINK_FLAG + if expand_refs: + flags |= h5o.COPY_EXPAND_REFERENCE_FLAG + if without_attrs: + flags |= h5o.COPY_WITHOUT_ATTR_FLAG + if flags: + copypl = h5p.create(h5p.OBJECT_COPY) + copypl.set_copy_object(flags) + else: + copypl = None + + h5o.copy(source.id, self._e(source_path), dest.id, self._e(dest_path), + copypl, base.dlcpl) + + def move(self, source, dest): + """ Move a link to a new location in the file. + + If "source" is a hard link, this effectively renames the object. If + "source" is a soft or external link, the link itself is moved, with its + value unmodified. + """ + with phil: + if source == dest: + return + self.id.links.move(self._e(source), self.id, self._e(dest), + lapl=self._lapl, lcpl=self._lcpl) + + def visit(self, func): + """ Recursively visit all names in this group and subgroups. + + Note: visit ignores soft and external links. To visit those, use + visit_links. + + You supply a callable (function, method or callable object); it + will be called exactly once for each link in this group and every + group below it. Your callable must conform to the signature: + + func(<member name>) => <None or return value> + + Returning None continues iteration, returning anything else stops + and immediately returns that value from the visit method. The + iteration order is lexicographic. + + Example: + + >>> # List the entire contents of the file + >>> f = File("foo.hdf5") + >>> list_of_names = [] + >>> f.visit(list_of_names.append) + """ + with phil: + def proxy(name): + """ Call the function with the text name, not bytes """ + return func(self._d(name)) + return h5o.visit(self.id, proxy) + + def visititems(self, func): + """ Recursively visit names and objects in this group. + + Note: visititems ignores soft and external links. To visit those, use + visititems_links. + + You supply a callable (function, method or callable object); it + will be called exactly once for each link in this group and every + group below it. Your callable must conform to the signature: + + func(<member name>, <object>) => <None or return value> + + Returning None continues iteration, returning anything else stops + and immediately returns that value from the visit method. The + iteration order is lexicographic. + + Example: + + # Get a list of all datasets in the file + >>> mylist = [] + >>> def func(name, obj): + ... if isinstance(obj, Dataset): + ... mylist.append(name) + ... + >>> f = File('foo.hdf5') + >>> f.visititems(func) + """ + with phil: + def proxy(name): + """ Use the text name of the object, not bytes """ + name = self._d(name) + return func(name, self[name]) + return h5o.visit(self.id, proxy) + + def visit_links(self, func): + """ Recursively visit all names in this group and subgroups. + Each link will be visited exactly once, regardless of its target. + + You supply a callable (function, method or callable object); it + will be called exactly once for each link in this group and every + group below it. Your callable must conform to the signature: + + func(<member name>) => <None or return value> + + Returning None continues iteration, returning anything else stops + and immediately returns that value from the visit method. The + iteration order is lexicographic. + + Example: + + >>> # List the entire contents of the file + >>> f = File("foo.hdf5") + >>> list_of_names = [] + >>> f.visit_links(list_of_names.append) + """ + with phil: + def proxy(name): + """ Call the function with the text name, not bytes """ + return func(self._d(name)) + return self.id.links.visit(proxy) + + def visititems_links(self, func): + """ Recursively visit links in this group. + Each link will be visited exactly once, regardless of its target. + + You supply a callable (function, method or callable object); it + will be called exactly once for each link in this group and every + group below it. Your callable must conform to the signature: + + func(<member name>, <link>) => <None or return value> + + Returning None continues iteration, returning anything else stops + and immediately returns that value from the visit method. The + iteration order is lexicographic. + + Example: + + # Get a list of all softlinks in the file + >>> mylist = [] + >>> def func(name, link): + ... if isinstance(link, SoftLink): + ... mylist.append(name) + ... + >>> f = File('foo.hdf5') + >>> f.visititems_links(func) + """ + with phil: + def proxy(name): + """ Use the text name of the object, not bytes """ + name = self._d(name) + return func(name, self.get(name, getlink=True)) + return self.id.links.visit(proxy) + + @with_phil + def __repr__(self): + if not self: + r = u"<Closed HDF5 group>" + else: + namestr = ( + '"%s"' % self.name + ) if self.name is not None else u"(anonymous)" + r = '<HDF5 group %s (%d members)>' % (namestr, len(self)) + + return r + + +class HardLink: + + """ + Represents a hard link in an HDF5 file. Provided only so that + Group.get works in a sensible way. Has no other function. + """ + + pass + + +class SoftLink: + + """ + Represents a symbolic ("soft") link in an HDF5 file. The path + may be absolute or relative. No checking is performed to ensure + that the target actually exists. + """ + + @property + def path(self): + """ Soft link value. Not guaranteed to be a valid path. """ + return self._path + + def __init__(self, path): + self._path = str(path) + + def __repr__(self): + return '<SoftLink to "%s">' % self.path + + +class ExternalLink: + + """ + Represents an HDF5 external link. Paths may be absolute or relative. + No checking is performed to ensure either the target or file exists. + """ + + @property + def path(self): + """ Soft link path, i.e. the part inside the HDF5 file. """ + return self._path + + @property + def filename(self): + """ Path to the external HDF5 file in the filesystem. """ + return self._filename + + def __init__(self, filename, path): + self._filename = filename_decode(filename_encode(filename)) + self._path = path + + def __repr__(self): + return '<ExternalLink to "%s" in file "%s"' % (self.path, + self.filename) diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/selections.py b/.venv/lib/python3.11/site-packages/h5py/_hl/selections.py new file mode 100644 index 0000000000000000000000000000000000000000..11b06b1c703880dc91673655fdedabed5eeb4be8 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/selections.py @@ -0,0 +1,440 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + High-level access to HDF5 dataspace selections +""" + +import numpy as np + +from .base import product +from .. import h5s, h5r, _selector + +def select(shape, args, dataset=None): + """ High-level routine to generate a selection from arbitrary arguments + to __getitem__. The arguments should be the following: + + shape + Shape of the "source" dataspace. + + args + Either a single argument or a tuple of arguments. See below for + supported classes of argument. + + dataset + A h5py.Dataset instance representing the source dataset. + + Argument classes: + + Single Selection instance + Returns the argument. + + numpy.ndarray + Must be a boolean mask. Returns a PointSelection instance. + + RegionReference + Returns a Selection instance. + + Indices, slices, ellipses, MultiBlockSlices only + Returns a SimpleSelection instance + + Indices, slices, ellipses, lists or boolean index arrays + Returns a FancySelection instance. + """ + if not isinstance(args, tuple): + args = (args,) + + # "Special" indexing objects + if len(args) == 1: + + arg = args[0] + if isinstance(arg, Selection): + if arg.shape != shape: + raise TypeError("Mismatched selection shape") + return arg + + elif isinstance(arg, np.ndarray) and arg.dtype.kind == 'b': + if arg.shape == shape: + return PointSelection.from_mask(arg) + # Allow 1D boolean array on the 1st dim + elif arg.shape != shape[:1]: + raise TypeError("Boolean indexing array has incompatible shape") + + + elif isinstance(arg, h5r.RegionReference): + if dataset is None: + raise TypeError("Cannot apply a region reference without a dataset") + sid = h5r.get_region(arg, dataset.id) + if shape != sid.shape: + raise TypeError("Reference shape does not match dataset shape") + + return Selection(shape, spaceid=sid) + + if dataset is not None: + selector = dataset._selector + else: + space = h5s.create_simple(shape) + selector = _selector.Selector(space) + + return selector.make_selection(args) + + +class Selection: + + """ + Base class for HDF5 dataspace selections. Subclasses support the + "selection protocol", which means they have at least the following + members: + + __init__(shape) => Create a new selection on "shape"-tuple + __getitem__(args) => Perform a selection with the range specified. + What args are allowed depends on the + particular subclass in use. + + id (read-only) => h5py.h5s.SpaceID instance + shape (read-only) => The shape of the dataspace. + mshape (read-only) => The shape of the selection region. + Not guaranteed to fit within "shape", although + the total number of points is less than + product(shape). + nselect (read-only) => Number of selected points. Always equal to + product(mshape). + + broadcast(target_shape) => Return an iterable which yields dataspaces + for read, based on target_shape. + + The base class represents "unshaped" selections (1-D). + """ + + def __init__(self, shape, spaceid=None): + """ Create a selection. Shape may be None if spaceid is given. """ + if spaceid is not None: + self._id = spaceid + self._shape = spaceid.shape + else: + shape = tuple(shape) + self._shape = shape + self._id = h5s.create_simple(shape, (h5s.UNLIMITED,)*len(shape)) + self._id.select_all() + + @property + def id(self): + """ SpaceID instance """ + return self._id + + @property + def shape(self): + """ Shape of whole dataspace """ + return self._shape + + @property + def nselect(self): + """ Number of elements currently selected """ + return self._id.get_select_npoints() + + @property + def mshape(self): + """ Shape of selection (always 1-D for this class) """ + return (self.nselect,) + + @property + def array_shape(self): + """Shape of array to read/write (always 1-D for this class)""" + return self.mshape + + # expand_shape and broadcast only really make sense for SimpleSelection + def expand_shape(self, source_shape): + if product(source_shape) != self.nselect: + raise TypeError("Broadcasting is not supported for point-wise selections") + return source_shape + + def broadcast(self, source_shape): + """ Get an iterable for broadcasting """ + if product(source_shape) != self.nselect: + raise TypeError("Broadcasting is not supported for point-wise selections") + yield self._id + + def __getitem__(self, args): + raise NotImplementedError("This class does not support indexing") + +class PointSelection(Selection): + + """ + Represents a point-wise selection. You can supply sequences of + points to the three methods append(), prepend() and set(), or + instantiate it with a single boolean array using from_mask(). + """ + def __init__(self, shape, spaceid=None, points=None): + super().__init__(shape, spaceid) + if points is not None: + self._perform_selection(points, h5s.SELECT_SET) + + def _perform_selection(self, points, op): + """ Internal method which actually performs the selection """ + points = np.atleast_2d(np.asarray(points, order='C', dtype='u8')) + + if self._id.get_select_type() != h5s.SEL_POINTS: + op = h5s.SELECT_SET + + if len(points) == 0: + self._id.select_none() + else: + self._id.select_elements(points, op) + + @classmethod + def from_mask(cls, mask, spaceid=None): + """Create a point-wise selection from a NumPy boolean array """ + if not (isinstance(mask, np.ndarray) and mask.dtype.kind == 'b'): + raise TypeError("PointSelection.from_mask only works with bool arrays") + + points = np.transpose(mask.nonzero()) + return cls(mask.shape, spaceid, points=points) + + def append(self, points): + """ Add the sequence of points to the end of the current selection """ + self._perform_selection(points, h5s.SELECT_APPEND) + + def prepend(self, points): + """ Add the sequence of points to the beginning of the current selection """ + self._perform_selection(points, h5s.SELECT_PREPEND) + + def set(self, points): + """ Replace the current selection with the given sequence of points""" + self._perform_selection(points, h5s.SELECT_SET) + + +class SimpleSelection(Selection): + + """ A single "rectangular" (regular) selection composed of only slices + and integer arguments. Can participate in broadcasting. + """ + + @property + def mshape(self): + """ Shape of current selection """ + return self._sel[1] + + @property + def array_shape(self): + scalar = self._sel[3] + return tuple(x for x, s in zip(self.mshape, scalar, strict=True) if not s) + + def __init__(self, shape, spaceid=None, hyperslab=None): + super().__init__(shape, spaceid) + if hyperslab is not None: + self._sel = hyperslab + else: + # No hyperslab specified - select all + rank = len(self.shape) + self._sel = ((0,)*rank, self.shape, (1,)*rank, (False,)*rank) + + def expand_shape(self, source_shape): + """Match the dimensions of an array to be broadcast to the selection + + The returned shape describes an array of the same size as the input + shape, but its dimensions + + E.g. with a dataset shape (10, 5, 4, 2), writing like this:: + + ds[..., 0] = np.ones((5, 4)) + + The source shape (5, 4) will expand to (1, 5, 4, 1). + Then the broadcast method below repeats that chunk 10 + times to write to an effective shape of (10, 5, 4, 1). + """ + start, count, step, scalar = self._sel + + rank = len(count) + remaining_src_dims = list(source_shape) + + eshape = [] + for idx in range(1, rank + 1): + if len(remaining_src_dims) == 0 or scalar[-idx]: # Skip scalar axes + eshape.append(1) + else: + t = remaining_src_dims.pop() + if t == 1 or count[-idx] == t: + eshape.append(t) + else: + raise TypeError("Can't broadcast %s -> %s" % (source_shape, self.array_shape)) # array shape + + if any([n > 1 for n in remaining_src_dims]): + # All dimensions from target_shape should either have been popped + # to match the selection shape, or be 1. + raise TypeError("Can't broadcast %s -> %s" % (source_shape, self.array_shape)) # array shape + + # We have built eshape backwards, so now reverse it + return tuple(eshape[::-1]) + + + def broadcast(self, source_shape): + """ Return an iterator over target dataspaces for broadcasting. + + Follows the standard NumPy broadcasting rules against the current + selection shape (self.mshape). + """ + if self.shape == (): + if product(source_shape) != 1: + raise TypeError("Can't broadcast %s to scalar" % source_shape) + self._id.select_all() + yield self._id + return + + start, count, step, scalar = self._sel + + rank = len(count) + tshape = self.expand_shape(source_shape) + + # Avoid ZeroDivisionError below (after the shape checks in expand_source) + if any(d == 0 for d in count): + return + + chunks = tuple(x//y for x, y in zip(count, tshape, strict=True)) + nchunks = product(chunks) + + if nchunks == 1: + yield self._id + else: + sid = self._id.copy() + sid.select_hyperslab((0,)*rank, tshape, step) + for idx in range(nchunks): + offset = tuple(x*y*z + s for x, y, z, s in zip(np.unravel_index(idx, chunks), tshape, step, start, strict=True)) + sid.offset_simple(offset) + yield sid + + +class FancySelection(Selection): + + """ + Implements advanced NumPy-style selection operations in addition to + the standard slice-and-int behavior. + + Indexing arguments may be ints, slices, lists of indices, or + per-axis (1D) boolean arrays. + + Broadcasting is not supported for these selections. + """ + + @property + def mshape(self): + return self._mshape + + @property + def array_shape(self): + return self._array_shape + + def __init__(self, shape, spaceid=None, mshape=None, array_shape=None): + super().__init__(shape, spaceid) + if mshape is None: + mshape = self.shape + if array_shape is None: + array_shape = mshape + self._mshape = mshape + self._array_shape = array_shape + + def expand_shape(self, source_shape): + if not source_shape == self.array_shape: + raise TypeError("Broadcasting is not supported for complex selections") + return source_shape + + def broadcast(self, source_shape): + if not source_shape == self.array_shape: + raise TypeError("Broadcasting is not supported for complex selections") + yield self._id + + +def guess_shape(sid): + """ Given a dataspace, try to deduce the shape of the selection. + + Returns one of: + * A tuple with the selection shape, same length as the dataspace + * A 1D selection shape for point-based and multiple-hyperslab selections + * None, for unselected scalars and for NULL dataspaces + """ + + sel_class = sid.get_simple_extent_type() # Dataspace class + sel_type = sid.get_select_type() # Flavor of selection in use + + if sel_class == h5s.NULL: + # NULL dataspaces don't support selections + return None + + elif sel_class == h5s.SCALAR: + # NumPy has no way of expressing empty 0-rank selections, so we use None + if sel_type == h5s.SEL_NONE: return None + if sel_type == h5s.SEL_ALL: return tuple() + + elif sel_class != h5s.SIMPLE: + raise TypeError("Unrecognized dataspace class %s" % sel_class) + + # We have a "simple" (rank >= 1) dataspace + + N = sid.get_select_npoints() + rank = len(sid.shape) + + if sel_type == h5s.SEL_NONE: + return (0,)*rank + + elif sel_type == h5s.SEL_ALL: + return sid.shape + + elif sel_type == h5s.SEL_POINTS: + # Like NumPy, point-based selections yield 1D arrays regardless of + # the dataspace rank + return (N,) + + elif sel_type != h5s.SEL_HYPERSLABS: + raise TypeError("Unrecognized selection method %s" % sel_type) + + # We have a hyperslab-based selection + + if N == 0: + return (0,)*rank + + bottomcorner, topcorner = (np.array(x) for x in sid.get_select_bounds()) + + # Shape of full selection box + boxshape = topcorner - bottomcorner + np.ones((rank,)) + + def get_n_axis(sid, axis): + """ Determine the number of elements selected along a particular axis. + + To do this, we "mask off" the axis by making a hyperslab selection + which leaves only the first point along the axis. For a 2D dataset + with selection box shape (X, Y), for axis 1, this would leave a + selection of shape (X, 1). We count the number of points N_leftover + remaining in the selection and compute the axis selection length by + N_axis = N/N_leftover. + """ + + if(boxshape[axis]) == 1: + return 1 + + start = bottomcorner.copy() + start[axis] += 1 + count = boxshape.copy() + count[axis] -= 1 + + # Throw away all points along this axis + masked_sid = sid.copy() + masked_sid.select_hyperslab(tuple(start), tuple(count), op=h5s.SELECT_NOTB) + + N_leftover = masked_sid.get_select_npoints() + + return N//N_leftover + + + shape = tuple(get_n_axis(sid, x) for x in range(rank)) + + if product(shape) != N: + # This means multiple hyperslab selections are in effect, + # so we fall back to a 1D shape + return (N,) + + return shape diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/selections2.py b/.venv/lib/python3.11/site-packages/h5py/_hl/selections2.py new file mode 100644 index 0000000000000000000000000000000000000000..e28224d43751dbd5dba56dce562c8573404cf583 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/selections2.py @@ -0,0 +1,103 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Implements a portion of the selection operations. +""" + +import numpy as np +from .. import h5s + +def read_dtypes(dataset_dtype, names): + """ Returns a 2-tuple containing: + + 1. Output dataset dtype + 2. Dtype containing HDF5-appropriate description of destination + """ + + if len(names) == 0: # Not compound, or all fields needed + format_dtype = dataset_dtype + + elif dataset_dtype.names is None: + raise ValueError("Field names only allowed for compound types") + + elif any(x not in dataset_dtype.names for x in names): + raise ValueError("Field does not appear in this type.") + + else: + format_dtype = np.dtype([(name, dataset_dtype.fields[name][0]) for name in names]) + + if len(names) == 1: + # We don't preserve the field information if only one explicitly selected. + output_dtype = format_dtype.fields[names[0]][0] + + else: + output_dtype = format_dtype + + return output_dtype, format_dtype + + +def read_selections_scalar(dsid, args): + """ Returns a 2-tuple containing: + + 1. Output dataset shape + 2. HDF5 dataspace containing source selection. + + Works for scalar datasets. + """ + + if dsid.shape != (): + raise RuntimeError("Illegal selection function for non-scalar dataset") + + if args == (): + # This is a signal that an array scalar should be returned instead + # of an ndarray with shape () + out_shape = None + + elif args == (Ellipsis,): + out_shape = () + + else: + raise ValueError("Illegal slicing argument for scalar dataspace") + + source_space = dsid.get_space() + source_space.select_all() + + return out_shape, source_space + +class ScalarReadSelection: + + """ + Implements slicing for scalar datasets. + """ + + def __init__(self, fspace, args): + if args == (): + self.mshape = None + elif args == (Ellipsis,): + self.mshape = () + else: + raise ValueError("Illegal slicing argument for scalar dataspace") + + self.mspace = h5s.create(h5s.SCALAR) + self.fspace = fspace + + def __iter__(self): + self.mspace.select_all() + yield self.fspace, self.mspace + +def select_read(fspace, args): + """ Top-level dispatch function for reading. + + At the moment, only supports reading from scalar datasets. + """ + if fspace.shape == (): + return ScalarReadSelection(fspace, args) + + raise NotImplementedError() diff --git a/.venv/lib/python3.11/site-packages/h5py/_hl/vds.py b/.venv/lib/python3.11/site-packages/h5py/_hl/vds.py new file mode 100644 index 0000000000000000000000000000000000000000..fac9c3be88488e694449552c1b1e9cc565808922 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_hl/vds.py @@ -0,0 +1,250 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + High-level interface for creating HDF5 virtual datasets +""" + +from copy import deepcopy as copy +from collections import namedtuple + +import numpy as np + +from .compat import filename_encode +from .datatype import Datatype +from .selections import SimpleSelection, select +from .. import h5d, h5p, h5s, h5t + + +class VDSmap(namedtuple('VDSmap', ('vspace', 'file_name', + 'dset_name', 'src_space'))): + '''Defines a region in a virtual dataset mapping to part of a source dataset + ''' + + +vds_support = True + + +def _convert_space_for_key(space, key): + """ + Converts the space with the given key. Mainly used to allow unlimited + dimensions in virtual space selection. + """ + key = key if isinstance(key, tuple) else (key,) + type_code = space.get_select_type() + + # check for unlimited selections in case where selection is regular + # hyperslab, which is the only allowed case for h5s.UNLIMITED to be + # in the selection + if type_code == h5s.SEL_HYPERSLABS and space.is_regular_hyperslab(): + rank = space.get_simple_extent_ndims() + nargs = len(key) + + idx_offset = 0 + start, stride, count, block = space.get_regular_hyperslab() + # iterate through keys. we ignore numeral indices. if we get a + # slice, we check for an h5s.UNLIMITED value as the stop + # if we get an ellipsis, we offset index by (rank - nargs) + for i, sl in enumerate(key): + if isinstance(sl, slice): + if sl.stop == h5s.UNLIMITED: + counts = list(count) + idx = i + idx_offset + counts[idx] = h5s.UNLIMITED + count = tuple(counts) + elif sl is Ellipsis: + idx_offset = rank - nargs + + space.select_hyperslab(start, count, stride, block) + + +class VirtualSource: + """Source definition for virtual data sets. + + Instantiate this class to represent an entire source dataset, and then + slice it to indicate which regions should be used in the virtual dataset. + + path_or_dataset + The path to a file, or an h5py dataset. If a dataset is given, + no other parameters are allowed, as the relevant values are taken from + the dataset instead. + name + The name of the source dataset within the file. + shape + A tuple giving the shape of the dataset. + dtype + Numpy dtype or string. + maxshape + The source dataset is resizable up to this shape. Use None for + axes you want to be unlimited. + """ + def __init__(self, path_or_dataset, name=None, + shape=None, dtype=None, maxshape=None): + from .dataset import Dataset + if isinstance(path_or_dataset, Dataset): + failed = {k: v + for k, v in + {'name': name, 'shape': shape, + 'dtype': dtype, 'maxshape': maxshape}.items() + if v is not None} + if failed: + raise TypeError("If a Dataset is passed as the first argument " + "then no other arguments may be passed. You " + "passed {failed}".format(failed=failed)) + ds = path_or_dataset + path = ds.file.filename + name = ds.name + shape = ds.shape + dtype = ds.dtype + maxshape = ds.maxshape + else: + path = path_or_dataset + if name is None: + raise TypeError("The name parameter is required when " + "specifying a source by path") + if shape is None: + raise TypeError("The shape parameter is required when " + "specifying a source by path") + elif isinstance(shape, int): + shape = (shape,) + + if isinstance(maxshape, int): + maxshape = (maxshape,) + + self.path = path + self.name = name + self.dtype = dtype + + if maxshape is None: + self.maxshape = shape + else: + self.maxshape = tuple([h5s.UNLIMITED if ix is None else ix + for ix in maxshape]) + self.sel = SimpleSelection(shape) + self._all_selected = True + + @property + def shape(self): + return self.sel.array_shape + + def __getitem__(self, key): + if not self._all_selected: + raise RuntimeError("VirtualSource objects can only be sliced once.") + tmp = copy(self) + tmp.sel = select(self.shape, key, dataset=None) + _convert_space_for_key(tmp.sel.id, key) + tmp._all_selected = False + return tmp + +class VirtualLayout: + """Object for building a virtual dataset. + + Instantiate this class to define a virtual dataset, assign to slices of it + (using VirtualSource objects), and then pass it to + group.create_virtual_dataset() to add the virtual dataset to a file. + + This class does not allow access to the data; the virtual dataset must + be created in a file before it can be used. + + shape + A tuple giving the shape of the dataset. + dtype + Numpy dtype or string. + maxshape + The virtual dataset is resizable up to this shape. Use None for + axes you want to be unlimited. + filename + The name of the destination file, if known in advance. Mappings from + data in the same file will be stored with filename '.', allowing the + file to be renamed later. + """ + def __init__(self, shape, dtype, maxshape=None, filename=None): + self.shape = (shape,) if isinstance(shape, int) else shape + self.dtype = dtype + self.maxshape = (maxshape,) if isinstance(maxshape, int) else maxshape + self._filename = filename + self._src_filenames = set() + self.dcpl = h5p.create(h5p.DATASET_CREATE) + self.dcpl.set_layout(h5d.VIRTUAL) + + def __setitem__(self, key, source): + sel = select(self.shape, key, dataset=None) + _convert_space_for_key(sel.id, key) + src_filename = self._source_file_name(source.path, self._filename) + + self.dcpl.set_virtual( + sel.id, src_filename, source.name.encode('utf-8'), source.sel.id + ) + if self._filename is None: + self._src_filenames.add(src_filename) + + @staticmethod + def _source_file_name(src_filename, dst_filename) -> bytes: + src_filename = filename_encode(src_filename) + if dst_filename and (src_filename == filename_encode(dst_filename)): + # use relative path if the source dataset is in the same + # file, in order to keep the virtual dataset valid in case + # the file is renamed. + return b'.' + return filename_encode(src_filename) + + def _get_dcpl(self, dst_filename): + """Get the property list containing virtual dataset mappings + + If the destination filename wasn't known when the VirtualLayout was + created, it is handled here. + """ + dst_filename = filename_encode(dst_filename) + if self._filename is not None: + # filename was known in advance; check dst_filename matches + if dst_filename != filename_encode(self._filename): + raise Exception(f"{dst_filename!r} != {self._filename!r}") + return self.dcpl + + # destination file not known in advance + if dst_filename in self._src_filenames: + # At least 1 source file is the same as the destination file, + # but we didn't know this when making the mapping. Copy the mappings + # to a new property list, replacing the dest filename with '.' + new_dcpl = h5p.create(h5p.DATASET_CREATE) + new_dcpl.set_layout(h5d.VIRTUAL) + for i in range(self.dcpl.get_virtual_count()): + src_filename = self.dcpl.get_virtual_filename(i) + new_dcpl.set_virtual( + self.dcpl.get_virtual_vspace(i), + self._source_file_name(src_filename, dst_filename), + self.dcpl.get_virtual_dsetname(i).encode('utf-8'), + self.dcpl.get_virtual_srcspace(i), + ) + return new_dcpl + else: + return self.dcpl # Mappings are all from other files + + def make_dataset(self, parent, name, fillvalue=None): + """ Return a new low-level dataset identifier for a virtual dataset """ + dcpl = self._get_dcpl(parent.file.filename) + + if fillvalue is not None: + dcpl.set_fill_value(np.array([fillvalue])) + + maxshape = self.maxshape + if maxshape is not None: + maxshape = tuple(m if m is not None else h5s.UNLIMITED for m in maxshape) + + virt_dspace = h5s.create_simple(self.shape, maxshape) + + if isinstance(self.dtype, Datatype): + # Named types are used as-is + tid = self.dtype.id + else: + dtype = np.dtype(self.dtype) + tid = h5t.py_create(dtype, logical=1) + + return h5d.create(parent.id, name=name, tid=tid, space=virt_dspace, + dcpl=dcpl) diff --git a/.venv/lib/python3.11/site-packages/h5py/_npystrings.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/_npystrings.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..8e345e2ad20a32cf6b2a984575bfa1c73d827d33 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_npystrings.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aa9406388541bba61851cdbd2d0bd01f761ae67be3f2e194ab7de6565b6f4e5 +size 154849 diff --git a/.venv/lib/python3.11/site-packages/h5py/_objects.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/_objects.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..774aa36ca40fab4e1129b28d6d3be4422b25e927 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_objects.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2efdbdb34464a8d3e7f839bc7dd651826670027e421326958a1a7f1ecf21590f +size 360785 diff --git a/.venv/lib/python3.11/site-packages/h5py/_proxy.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/_proxy.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..c131187aeee6e36c313778adac9330c774f606d4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_proxy.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beacc63a9cf71677be5b32039180c3f3b85834bd58d61d4b5f65730d1d19092f +size 142321 diff --git a/.venv/lib/python3.11/site-packages/h5py/_selector.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/_selector.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..316229d39d353d86829ddc5b8a7e5972ad98cfd1 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/_selector.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ed624563eac58c12e4ad3cd17e3db3e17da3cc1d6794cd06cf3558e59c467c0 +size 364945 diff --git a/.venv/lib/python3.11/site-packages/h5py/defs.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/defs.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..5cfc1f7d925541f077e71fb23d22978600aea645 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/defs.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d179923a1d52bd85747c4f18aa62ac14a287e2b26d5b7427cf45bc8861c1c83 +size 512497 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..24feebc407e7488fa83152a7a3e43280ea96be77 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83fbb3cb960acb374f48bd8464bad5f9fde86e25a271ea9ac4bf597362361efc +size 257665 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5a.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5a.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..abb2d6230784b87ac95483969dda4a8a1c65d536 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5a.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:097001f03bd737db7cf2809105c857554edae370e9f5625643753a94d4efb755 +size 319121 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5ac.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5ac.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..72848a9a44aefc3a184bc6521dffa0d09ec3da8a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5ac.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5e70d9569bdb88c7fd961a18341309d86c3a12891fcb6b7220dbee310e972f8 +size 216257 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5d.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5d.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..b4903d311a916a6f2c33213f83601d834b1ec016 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5d.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8471a128c6fc2f46512d6e75461ba4075ae440b224124783f3e5b92afaab652c +size 577985 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5ds.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5ds.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..b6f4ff82d243d7ffae9f9f1b4dcda0f5ea52cc85 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5ds.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72d70c624b3b94e17357cc0a01c1d6920a64f9c11b75c4ca5906e9749579af8f +size 224825 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5f.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5f.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..12c43b2606cc38c485988b370519256955781341 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5f.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ca759303682f36aedf9ed04aa0700f618cb69ac75f236430a5bc892b414fdb2 +size 352025 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5fd.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5fd.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..76c3e2d737d3725cffc171c76ad9681716bb65bb --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5fd.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf0a3be1d9cdb08bf433fdee099fa781d9f2d4685180586c09ef6a98a30e167f +size 364857 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5g.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5g.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..815d7b726aecef525a388185c2b5d2f774925513 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5g.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edba6114977395fa928c0846c9c338dc3b93f453ce881c312d35759fc74aa467 +size 381033 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5i.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5i.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..72e3de7222dd255eeccc0fe8b6d4c90f00da12d6 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5i.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3427b5356d5a9a52a33991652aed3d86517f4c11a8b3096b2039e151bddca07b +size 175457 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5l.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5l.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..e7c4223abf9a8348acaa3fe0b697f26291c50bd4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5l.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc12afae158a10a97fa531a16f7fb5717a6a45520dcbf640a33143659735c69d +size 274001 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5o.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5o.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..b949d02bd6e3648647c5ab433a21a54718a29045 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5o.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5b13943bcf682b1d9cab23dfcb7a625597de45d5c7b35b5c59663adc126812d +size 343609 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5p.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5p.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..1fa23437a70f0b4e45971def17a77adad3adc6ab --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5p.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73ec8f5927384f34fc3b7c17216e2fb18ce7beef2b06de580ebdad4a1079dbd1 +size 885185 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5pl.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5pl.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..74b108954124749f656d5bfec98e5be098daa98f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5pl.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c1153284f912f4619496f3da465db90ece4571ba2b2167595f09c03d9101bae +size 162889 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5py_warnings.py b/.venv/lib/python3.11/site-packages/h5py/h5py_warnings.py new file mode 100644 index 0000000000000000000000000000000000000000..c30c56f7ff77c8703667d27dc550193a6f52e9ed --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5py_warnings.py @@ -0,0 +1,21 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + This module contains the warning classes for h5py. These classes are part of + the public API of h5py, and should be imported from this module. +""" + + +class H5pyWarning(UserWarning): + pass + + +class H5pyDeprecationWarning(H5pyWarning): + pass diff --git a/.venv/lib/python3.11/site-packages/h5py/h5r.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5r.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..c8bf5d69bf256fce6b37578b993ebe94b24339d2 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5r.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:125b971a5becdfacbf39e4f9fae8e2d6b95e6d2fa58f484473689b67cad1a021 +size 200169 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5s.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5s.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..46e30ff7dfae61edcdd568ff2cf87a5900e1f2ab --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5s.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dda50ef40efb10042356657989c3937973bafef2891599600c76a29608f21d7b +size 319049 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5t.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5t.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..b2a8fe553b0dae62800605842ed488d10f24a8fd --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5t.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44456e3f7a53501586a768a126495b3e8e4b280b20cee54b43f64174b9f88dc6 +size 919425 diff --git a/.venv/lib/python3.11/site-packages/h5py/h5z.cpython-311-x86_64-linux-gnu.so b/.venv/lib/python3.11/site-packages/h5py/h5z.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..b12303dd209c26cc3e1530385b35eea2a6f0c514 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/h5z.cpython-311-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50878304faa84028c856d4210252793a7cb96bf522c94001f2794759bf8c059 +size 171313 diff --git a/.venv/lib/python3.11/site-packages/h5py/ipy_completer.py b/.venv/lib/python3.11/site-packages/h5py/ipy_completer.py new file mode 100644 index 0000000000000000000000000000000000000000..aa68f3ea612894ae9d135b76948b116061ef6cd7 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/ipy_completer.py @@ -0,0 +1,156 @@ +#+ +# +# This file is part of h5py, a low-level Python interface to the HDF5 library. +# +# Contributed by Darren Dale +# +# Copyright (C) 2009 Darren Dale +# +# http://h5py.org +# License: BSD (See LICENSE.txt for full license) +# +#- + +# pylint: disable=eval-used,protected-access + +""" + This is the h5py completer extension for ipython. It is loaded by + calling the function h5py.enable_ipython_completer() from within an + interactive IPython session. + + It will let you do things like:: + + f=File('foo.h5') + f['<tab> + # or: + f['ite<tab> + + which will do tab completion based on the subgroups of `f`. Also:: + + f['item1'].at<tab> + + will perform tab completion for the attributes in the usual way. This should + also work:: + + a = b = f['item1'].attrs.<tab> + + as should:: + + f['item1/item2/it<tab> +""" + +import posixpath +import re +from ._hl.attrs import AttributeManager +from ._hl.base import HLObject + + +from IPython import get_ipython + +from IPython.core.error import TryNext +from IPython.utils import generics + +re_attr_match = re.compile(r"(?:.*\=)?(.+\[.*\].*)\.(\w*)$") +re_item_match = re.compile(r"""(?:.*\=)?(.*)\[(?P<s>['|"])(?!.*(?P=s))(.*)$""") +re_object_match = re.compile(r"(?:.*\=)?(.+?)(?:\[)") + + +def _retrieve_obj(name, context): + """ Filter function for completion. """ + + # we don't want to call any functions, but I couldn't find a robust regex + # that filtered them without unintended side effects. So keys containing + # "(" will not complete. + + if '(' in name: + raise ValueError() + + return eval(name, context.user_ns) + + +def h5py_item_completer(context, command): + """Compute possible item matches for dict-like objects""" + + base, item = re_item_match.split(command)[1:4:2] + + try: + obj = _retrieve_obj(base, context) + except Exception: + return [] + + path, _ = posixpath.split(item) + + try: + if path: + items = (posixpath.join(path, name) for name in obj[path].keys()) + else: + items = obj.keys() + except AttributeError: + return [] + + items = list(items) + + return [i for i in items if i[:len(item)] == item] + + +def h5py_attr_completer(context, command): + """Compute possible attr matches for nested dict-like objects""" + + base, attr = re_attr_match.split(command)[1:3] + base = base.strip() + + try: + obj = _retrieve_obj(base, context) + except Exception: + return [] + + attrs = dir(obj) + try: + attrs = generics.complete_object(obj, attrs) + except TryNext: + pass + + try: + # support >=ipython-0.12 + omit__names = get_ipython().Completer.omit__names + except AttributeError: + omit__names = 0 + + if omit__names == 1: + attrs = [a for a in attrs if not a.startswith('__')] + elif omit__names == 2: + attrs = [a for a in attrs if not a.startswith('_')] + + return ["%s.%s" % (base, a) for a in attrs if a[:len(attr)] == attr] + + +def h5py_completer(self, event): + """ Completer function to be loaded into IPython """ + base = re_object_match.split(event.line)[1] + + try: + obj = self._ofind(base).obj + except AttributeError: + obj = self._ofind(base).get('obj') + + if not isinstance(obj, (AttributeManager, HLObject)): + raise TryNext + + try: + return h5py_attr_completer(self, event.line) + except ValueError: + pass + + try: + return h5py_item_completer(self, event.line) + except ValueError: + pass + + return [] + + +def load_ipython_extension(ip=None): + """ Load completer function into IPython """ + if ip is None: + ip = get_ipython() + ip.set_hook('complete_command', h5py_completer, re_key=r"(?:.*\=)?(.+?)\[") diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/__init__.py b/.venv/lib/python3.11/site-packages/h5py/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d07acacf489ae1dd86c6b68d19c9cf4f5aa967a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/__init__.py @@ -0,0 +1,21 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. +import sys +import shlex +from importlib.util import find_spec +from subprocess import call + +def run_tests(args=''): + if find_spec("pytest") is None: + print("Tests require pytest, pytest not installed") + return 1 + + cli = [sys.executable, "-m", "pytest", "--pyargs", "h5py"] + cli.extend(shlex.split(args)) + return call(cli) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/common.py b/.venv/lib/python3.11/site-packages/h5py/tests/common.py new file mode 100644 index 0000000000000000000000000000000000000000..618b9774e9815772d9e1c4a8fb460bc3be295bc4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/common.py @@ -0,0 +1,277 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +import sys +import os +import shutil +import inspect +import tempfile +import threading +import subprocess +from contextlib import contextmanager +from functools import wraps + +import numpy as np +from numpy.lib.recfunctions import repack_fields +import h5py +from h5py._objects import phil +import unittest as ut + + +# Check if non-ascii filenames are supported +# Evidently this is the most reliable way to check +# See also h5py issue #263 and ipython #466 +# To test for this, run the testsuite with LC_ALL=C +try: + testfile, fname = tempfile.mkstemp(chr(0x03b7)) +except UnicodeError: + UNICODE_FILENAMES = False +else: + UNICODE_FILENAMES = True + os.close(testfile) + os.unlink(fname) + del fname + del testfile + + +class TestCase(ut.TestCase): + + """ + Base class for unit tests. + """ + + @classmethod + def setUpClass(cls): + cls.tempdir = tempfile.mkdtemp(prefix='h5py-test_') + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tempdir) + + def mktemp(self, suffix='.hdf5', prefix='tmp', dir=None): + if dir is None: + dir = self.tempdir + return tempfile.mktemp(suffix, make_name(prefix), dir=dir) + + def mktemp_mpi(self, comm=None, suffix='.hdf5', prefix='', dir=None): + if comm is None: + from mpi4py import MPI + comm = MPI.COMM_WORLD + fname = None + if comm.Get_rank() == 0: + fname = self.mktemp(suffix, prefix, dir) + fname = comm.bcast(fname, 0) + return fname + + def setUp(self): + self.f = h5py.File(self.mktemp(), 'w') + + def tearDown(self): + try: + if self.f: + self.f.close() + except: + pass + + def assertSameElements(self, a, b): + for x in a: + match = False + for y in b: + if x == y: + match = True + if not match: + raise AssertionError("Item '%s' appears in a but not b" % x) + + for x in b: + match = False + for y in a: + if x == y: + match = True + if not match: + raise AssertionError("Item '%s' appears in b but not a" % x) + + def assertArrayEqual(self, dset, arr, message=None, precision=None, check_alignment=True): + """ Make sure dset and arr have the same shape, dtype and contents, to + within the given precision, optionally ignoring differences in dtype alignment. + + Note that dset may be a NumPy array or an HDF5 dataset. + """ + if precision is None: + precision = 1e-5 + if message is None: + message = '' + else: + message = ' (%s)' % message + + if np.isscalar(dset) or np.isscalar(arr): + assert np.isscalar(dset) and np.isscalar(arr), \ + 'Scalar/array mismatch ("%r" vs "%r")%s' % (dset, arr, message) + dset = np.asarray(dset) + arr = np.asarray(arr) + + assert dset.shape == arr.shape, \ + "Shape mismatch (%s vs %s)%s" % (dset.shape, arr.shape, message) + if dset.dtype != arr.dtype: + if check_alignment: + normalized_dset_dtype = dset.dtype + normalized_arr_dtype = arr.dtype + else: + normalized_dset_dtype = repack_fields(dset.dtype) + normalized_arr_dtype = repack_fields(arr.dtype) + + assert normalized_dset_dtype == normalized_arr_dtype, \ + "Dtype mismatch (%s vs %s)%s" % (normalized_dset_dtype, normalized_arr_dtype, message) + + if not check_alignment: + if normalized_dset_dtype != dset.dtype: + dset = repack_fields(np.asarray(dset)) + if normalized_arr_dtype != arr.dtype: + arr = repack_fields(np.asarray(arr)) + + if arr.dtype.names is not None: + for n in arr.dtype.names: + message = '[FIELD %s] %s' % (n, message) + self.assertArrayEqual(dset[n], arr[n], message=message, precision=precision, check_alignment=check_alignment) + elif arr.dtype.kind in ('i', 'f'): + assert np.all(np.abs(dset[...] - arr[...]) < precision), \ + "Arrays differ by more than %.3f%s" % (precision, message) + elif arr.dtype.kind == 'O': + for v1, v2 in zip(dset.flat, arr.flat, strict=True): + self.assertArrayEqual(v1, v2, message=message, precision=precision, check_alignment=check_alignment) + else: + assert np.all(dset[...] == arr[...]), \ + "Arrays are not equal (dtype %s) %s" % (arr.dtype.str, message) + + def assertNumpyBehavior(self, dset, arr, s, skip_fast_reader=False): + """ Apply slicing arguments "s" to both dset and arr. + + Succeeds if the results of the slicing are identical, or the + exception raised is of the same type for both. + + "arr" must be a Numpy array; "dset" may be a NumPy array or dataset. + """ + exc = None + try: + arr_result = arr[s] + except Exception as e: + exc = type(e) + + s_fast = s if isinstance(s, tuple) else (s,) + + if exc is None: + self.assertArrayEqual(dset[s], arr_result) + + if not skip_fast_reader: + with phil: + self.assertArrayEqual( + dset._fast_reader.read(s_fast), + arr_result, + ) + else: + with self.assertRaises(exc): + dset[s] + + if not skip_fast_reader: + with self.assertRaises(exc), phil: + dset._fast_reader.read(s_fast) + +NUMPY_RELEASE_VERSION = tuple([int(i) for i in np.__version__.split(".")[0:2]]) + +@contextmanager +def closed_tempfile(suffix='', text=None): + """ + Context manager which yields the path to a closed temporary file with the + suffix `suffix`. The file will be deleted on exiting the context. An + additional argument `text` can be provided to have the file contain `text`. + """ + with tempfile.NamedTemporaryFile( + 'w+t', suffix=suffix, delete=False + ) as test_file: + file_name = test_file.name + if text is not None: + test_file.write(text) + test_file.flush() + yield file_name + shutil.rmtree(file_name, ignore_errors=True) + + +def insubprocess(f): + """Runs a test in its own subprocess""" + @wraps(f) + def wrapper(request, *args, **kwargs): + curr_test = inspect.getsourcefile(f) + "::" + request.node.name + # get block around test name + insub = "IN_SUBPROCESS_" + curr_test + for c in "/\\,:.": + insub = insub.replace(c, "_") + defined = os.environ.get(insub, None) + if defined: + # We're already running in a subprocess + return f(request, *args, **kwargs) + else: + # Spawn a new interpreter and run pytest in it + env = os.environ.copy() + env[insub] = '1' + env.update(getattr(f, 'subproc_env', {})) + + with closed_tempfile() as stdout: + with open(stdout, 'w+t') as fh: + rtn = subprocess.call([sys.executable, '-m', 'pytest', curr_test], + stdout=fh, stderr=fh, env=env) + with open(stdout, 'rt') as fh: + out = fh.read() + + assert rtn == 0, "\n" + out + return wrapper + + +def subproc_env(d): + """Set environment variables for the @insubprocess decorator""" + def decorator(f): + f.subproc_env = d + return f + + return decorator + + +MAIN_THREAD_ID = threading.get_ident() + + +def make_name(template_or_prefix: str = "foo", /) -> str: + """Return a static name, to be used e.g. as dataset name. + + When running in pytest-run-parallel, append a thread ID to the name. + This allows running tests on shared resources, e.g. two threads can attempt to write + to separate datasets on the same File at the same time (even though the actual + writes will be serialized by the `phil` lock). + + Calling this function twice from the same thread will return the same name. + + Parameters + ---------- + template_or_prefix + Either a prefix to which potentially append the thread ID, or a template + containing exactly one "{}" to be replaced with the thread ID. + """ + tid = threading.get_ident() + suffix = "" if tid == MAIN_THREAD_ID else f"-{tid}" + if "{}" in template_or_prefix: + return template_or_prefix.format(suffix) + else: + return template_or_prefix + suffix + + +def is_main_thread() -> bool: + """Return True if the test calling this function is being executed + in the main thread; False otherwise. + This can be used to detect when a test is running in pytest-run-parallel. + that spawns multiple separate threads to run the tests. + """ + tid = threading.get_ident() + return tid == MAIN_THREAD_ID diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/conftest.py b/.venv/lib/python3.11/site-packages/h5py/tests/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..7a24883e19635532a9ab6a78b1ca732e05d0366e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/conftest.py @@ -0,0 +1,22 @@ +import h5py +import pytest + + +@pytest.fixture() +def writable_file(tmp_path): + with h5py.File(tmp_path / 'test.h5', 'w') as f: + yield f + + +def pytest_addoption(parser): + parser.addoption( + '--no-network', action='store_true', default=False, help='No network access' + ) + + +def pytest_collection_modifyitems(config, items): + if config.getoption('--no-network'): + nonet = pytest.mark.skip(reason='No Internet') + for item in items: + if 'network' in item.keywords: + item.add_marker(nonet) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/data_files/__init__.py b/.venv/lib/python3.11/site-packages/h5py/tests/data_files/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0dce8d8acda31e72201857a438d7867b583274fe --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/data_files/__init__.py @@ -0,0 +1,7 @@ +from os.path import dirname, join + +def get_data_file_path(basename): + """ + Returns the path to the test data file given by `basename` + """ + return join(dirname(__file__), basename) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/data_files/compound-dtype-complex.h5 b/.venv/lib/python3.11/site-packages/h5py/tests/data_files/compound-dtype-complex.h5 new file mode 100644 index 0000000000000000000000000000000000000000..1750c90164d8b3a168b191edc1e41fea661db72b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/data_files/compound-dtype-complex.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcd9658096064f16e46a2d74b9532c9e6a6ad5b67a827f2fb4f6379f6b05380 +size 13792 diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/data_files/vlen_string_dset.h5 b/.venv/lib/python3.11/site-packages/h5py/tests/data_files/vlen_string_dset.h5 new file mode 100644 index 0000000000000000000000000000000000000000..dbf021d4c80f66b6343fead92e5d7a6ea8ab647e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/data_files/vlen_string_dset.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:900f8baf19d3d8c4d1193ac1ac6019ee7774005e85c030af44e6757b38d29793 +size 6304 diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/data_files/vlen_string_dset_utc.h5 b/.venv/lib/python3.11/site-packages/h5py/tests/data_files/vlen_string_dset_utc.h5 new file mode 100644 index 0000000000000000000000000000000000000000..ca72c33de37d93f49b7121526b855f52bd927c47 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/data_files/vlen_string_dset_utc.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85b728382b833c1da61627b9a334e22822d5c1cb359fe3ba6f25262af4532f63 +size 169904 diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/data_files/vlen_string_s390x.h5 b/.venv/lib/python3.11/site-packages/h5py/tests/data_files/vlen_string_s390x.h5 new file mode 100644 index 0000000000000000000000000000000000000000..bca4b91ad9029e3d0f0118aff12c28094edc95af --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/data_files/vlen_string_s390x.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea990c68e0371bafab4966f752fa1355b3b9f64360066f6d9daff8f1b7e74ca5 +size 9008 diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_attribute_create.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_attribute_create.py new file mode 100644 index 0000000000000000000000000000000000000000..f32d553c84233155b771224c12b7dbd89c93deda --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_attribute_create.py @@ -0,0 +1,100 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Tests the h5py.AttributeManager.create() method. +""" + +import numpy as np +from .. import h5t, h5a + +from .common import TestCase, make_name + +class TestArray(TestCase): + + """ + Check that top-level array types can be created and read. + """ + + def test_int(self): + # See issue 498 + name = make_name() + dt = np.dtype('(3,)i') + data = np.arange(3, dtype='i') + + self.f.attrs.create(name, data=data, dtype=dt) + + aid = h5a.open(self.f.id, name.encode('utf-8')) + + htype = aid.get_type() + self.assertEqual(htype.get_class(), h5t.ARRAY) + + out = self.f.attrs[name] + + self.assertArrayEqual(out, data) + + def test_string_dtype(self): + # See issue 498 discussion + + self.f.attrs.create(make_name(), data=42, dtype='i8') + + def test_str(self): + # See issue 1057 + name = make_name() + self.f.attrs.create(name, chr(0x03A9)) + out = self.f.attrs[name] + self.assertEqual(out, chr(0x03A9)) + self.assertIsInstance(out, str) + + def test_tuple_of_unicode(self): + # Test that a tuple of unicode strings can be set as an attribute. It will + # be converted to a numpy array of vlen unicode type: + name = make_name() + data = ('a', 'b') + self.f.attrs.create(name, data=data) + result = self.f.attrs[name] + self.assertTrue(all(result == data)) + self.assertEqual(result.dtype, np.dtype('O')) + + def test_unicode_np_array(self): + # However, a numpy array of type U being passed in will not be + # automatically converted, and should raise an error as it does + # not map to a h5py dtype + data = np.array(['a', 'b'], dtype='U1') + with self.assertRaises(TypeError): + self.f.attrs.create('x', data=data) + + def test_shape_scalar(self): + name = make_name() + self.f.attrs.create(name, data=42, shape=1) + result = self.f.attrs[name] + self.assertEqual(result.shape, (1,)) + + def test_shape_array(self): + name = make_name() + self.f.attrs.create(name, data=np.arange(3), shape=3) + result = self.f.attrs[name] + self.assertEqual(result.shape, (3,)) + + def test_dtype(self): + dt = np.dtype('(3,)i') + array = np.arange(3, dtype='i') + self.f.attrs.create(make_name(), data=array, dtype=dt) + # Array dtype shape is incompatible with data shape + array = np.arange(4, dtype='i') + with self.assertRaises(ValueError): + self.f.attrs.create('x', data=array, dtype=dt) + # Shape of new attribute conflicts with shape of data + dt = np.dtype('()i') + with self.assertRaises(ValueError): + self.f.attrs.create('x', data=array, shape=(5,), dtype=dt) + + def test_key_type(self): + with self.assertRaises(TypeError): + self.f.attrs.create(1, data=('a', 'b')) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_attrs.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_attrs.py new file mode 100644 index 0000000000000000000000000000000000000000..3aa83c80f65b3baedf1b098a3a92d98de32b9bbd --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_attrs.py @@ -0,0 +1,316 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Attributes testing module + + Covers all operations which access the .attrs property, with the + exception of data read/write and type conversion. Those operations + are tested by module test_attrs_data. +""" + +import numpy as np + +from collections.abc import MutableMapping + +from .common import TestCase, make_name + +import h5py +from h5py import File +from h5py import h5a, h5t +from h5py import AttributeManager + + +class BaseAttrs(TestCase): + + def setUp(self): + self.f = File(self.mktemp(), 'w') + + def tearDown(self): + if self.f: + self.f.close() + +class TestRepr(TestCase): + + """ Feature: AttributeManager provide a helpful + __repr__ string + """ + + def test_repr(self): + grp = self.f.create_group(make_name()) + grp.attrs.create('att', 1) + self.assertIsInstance(repr(grp.attrs), str) + grp.id.close() + self.assertIsInstance(repr(grp.attrs), str) + + +class TestAccess(BaseAttrs): + + """ + Feature: Attribute creation/retrieval via special methods + """ + + def test_create(self): + """ Attribute creation by direct assignment """ + name = make_name() + self.f.attrs[name] = 4.0 + assert name in self.f.attrs + self.assertEqual(self.f.attrs[name], 4.0) + + def test_create_2(self): + """ Attribute creation by create() method """ + name = make_name() + self.f.attrs.create(name, 4.0) + assert name in self.f.attrs + self.assertEqual(self.f.attrs[name], 4.0) + + def test_modify(self): + """ Attributes are modified by direct assignment""" + name = make_name() + self.f.attrs[name] = 3 + assert name in self.f.attrs + self.assertEqual(self.f.attrs[name], 3) + self.f.attrs[name] = 4 + self.assertEqual(self.f.attrs[name], 4) + + def test_modify_2(self): + """ Attributes are modified by modify() method """ + a = make_name("a") + b = make_name("b") + + self.f.attrs.modify(a, 3) + assert a in self.f.attrs + self.assertEqual(self.f.attrs[a], 3) + + self.f.attrs.modify(a, 4) + assert a in self.f.attrs + self.assertEqual(self.f.attrs[a], 4) + + # If the attribute doesn't exist, create new + self.f.attrs.modify(b, 5) + assert a in self.f.attrs + assert b in self.f.attrs + self.assertEqual(self.f.attrs[a], 4) + self.assertEqual(self.f.attrs[b], 5) + + # Shape of new value is incompatible with the previous + new_value = np.arange(5) + with self.assertRaises(TypeError): + self.f.attrs.modify(b, new_value) + + def test_overwrite(self): + """ Attributes are silently overwritten """ + name = make_name() + self.f.attrs[name] = 4.0 + self.f.attrs[name] = 5.0 + self.assertEqual(self.f.attrs[name], 5.0) + + def test_rank(self): + """ Attribute rank is preserved """ + name = make_name() + self.f.attrs[name] = (4.0, 5.0) + self.assertEqual(self.f.attrs[name].shape, (2,)) + self.assertArrayEqual(self.f.attrs[name], np.array((4.0,5.0))) + + def test_single(self): + """ Attributes of shape (1,) don't become scalars """ + name = make_name() + self.f.attrs[name] = np.ones((1,)) + out = self.f.attrs[name] + self.assertEqual(out.shape, (1,)) + self.assertEqual(out[()], 1) + + def test_access_exc(self): + """ Attempt to access missing item raises KeyError """ + with self.assertRaises(KeyError): + self.f.attrs['notexist'] + + def test_get_id(self): + name = make_name() + self.f.attrs[name] = 4.0 + aid = self.f.attrs.get_id(name) + assert isinstance(aid, h5a.AttrID) + + with self.assertRaises(KeyError): + self.f.attrs.get_id("notexist") + + +class TestDelete(BaseAttrs): + + """ + Feature: Deletion of attributes using __delitem__ + """ + + def test_delete(self): + """ Deletion via "del" """ + name = make_name() + self.f.attrs[name] = 4.0 + self.assertIn(name, self.f.attrs) + del self.f.attrs[name] + self.assertNotIn(name, self.f.attrs) + + def test_delete_exc(self): + """ Attempt to delete missing item raises KeyError """ + with self.assertRaises(KeyError): + del self.f.attrs['notexist'] + + +class TestUnicode(BaseAttrs): + + """ + Feature: Attributes can be accessed via Unicode or byte strings + """ + + def test_ascii(self): + """ Access via pure-ASCII byte string """ + name = make_name("ascii").encode("ascii") + self.f.attrs[name] = 42 + out = self.f.attrs[name] + self.assertEqual(out, 42) + + def test_raw(self): + """ Access via non-ASCII byte string """ + name = make_name("non-ascii\xfe").encode("utf-8") + self.f.attrs[name] = 42 + out = self.f.attrs[name] + self.assertEqual(out, 42) + + def test_unicode(self): + """ Access via Unicode string with non-ascii characters """ + name = make_name("Omega" + chr(0x03A9)) + self.f.attrs[name] = 42 + out = self.f.attrs[name] + self.assertEqual(out, 42) + + +class TestCreate(BaseAttrs): + + """ + Options for explicit attribute creation + """ + + def test_named(self): + """ Attributes created from named types link to the source type object + """ + t = make_name("t") + x = make_name("x") + self.f[t] = np.dtype('u8') + self.f.attrs.create(x, 42, dtype=self.f[t]) + self.assertEqual(self.f.attrs[x], 42) + aid = h5a.open(self.f.id, x.encode('utf-8')) + htype = aid.get_type() + htype2 = self.f[t].id + self.assertEqual(htype, htype2) + self.assertTrue(htype.committed()) + + def test_empty(self): + # https://github.com/h5py/h5py/issues/1540 + """ Create attribute with h5py.Empty value + """ + name = make_name() + self.f.attrs.create(name, h5py.Empty('f')) + self.assertEqual(self.f.attrs[name], h5py.Empty('f')) + + self.f.attrs.create(name, h5py.Empty(None)) + self.assertEqual(self.f.attrs[name], h5py.Empty(None)) + +class TestMutableMapping(BaseAttrs): + '''Tests if the registration of AttributeManager as a MutableMapping + behaves as expected + ''' + def test_resolution(self): + assert issubclass(AttributeManager, MutableMapping) + assert isinstance(self.f.attrs, MutableMapping) + + def test_validity(self): + ''' + Test that the required functions are implemented. + ''' + AttributeManager.__getitem__ + AttributeManager.__setitem__ + AttributeManager.__delitem__ + AttributeManager.__iter__ + AttributeManager.__len__ + +class TestVlen(BaseAttrs): + def test_vlen(self): + name = make_name() + a = np.array([np.arange(3), np.arange(4)], + dtype=h5t.vlen_dtype(int)) + self.f.attrs[name] = a + self.assertArrayEqual(self.f.attrs[name][0], a[0]) + + def test_vlen_s1(self): + name = make_name() + dt = h5py.vlen_dtype(np.dtype('S1')) + a = np.empty((1,), dtype=dt) + a[0] = np.array([b'a', b'b'], dtype='S1') + + self.f.attrs.create(name, a) + self.assertArrayEqual(self.f.attrs[name][0], a[0]) + + +class TestTrackOrder(BaseAttrs): + def fill_attrs(self, track_order): + attrs = self.f.create_group(make_name(), track_order=track_order).attrs + for i in range(100): + attrs[str(i)] = i + return attrs + + # https://forum.hdfgroup.org/t/bug-h5arename-fails-unexpectedly/4881 + def test_track_order(self): + attrs = self.fill_attrs(track_order=True) # creation order + self.assertEqual(list(attrs), + [str(i) for i in range(100)]) + + def test_no_track_order(self): + attrs = self.fill_attrs(track_order=False) # name alphanumeric + self.assertEqual(list(attrs), + sorted([str(i) for i in range(100)])) + + def test_track_order_overwrite_delete(self): + # issue 1385 + group = self.f.create_group(make_name(), track_order=True) + for i in range(12): + group.attrs[str(i)] = i + + self.assertEqual(group.attrs["11"], 11) + # overwrite attribute + group.attrs['11'] = 42.0 + self.assertEqual(group.attrs["11"], 42.0) + # delete attribute + self.assertIn('10', group.attrs) + del group.attrs['10'] + self.assertNotIn('10', group.attrs) + + +class TestDatatype(BaseAttrs): + + def test_datatype(self): + name = make_name() + self.f[name] = np.dtype('f') + dt = self.f[name] + self.assertEqual(list(dt.attrs.keys()), []) + dt.attrs.create('a', 4.0) + self.assertEqual(list(dt.attrs.keys()), ['a']) + self.assertEqual(list(dt.attrs.values()), [4.0]) + +def test_python_int_uint64(writable_file): + f = writable_file + name = make_name() + data = [np.iinfo(np.int64).max, np.iinfo(np.int64).max + 1] + + # Check creating a new attribute + f.attrs.create(name, data, dtype=np.uint64) + assert f.attrs[name].dtype == np.dtype(np.uint64) + np.testing.assert_array_equal(f.attrs[name], np.array(data, dtype=np.uint64)) + + # Check modifying an existing attribute + f.attrs.modify(name, data) + np.testing.assert_array_equal(f.attrs[name], np.array(data, dtype=np.uint64)) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_attrs_data.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_attrs_data.py new file mode 100644 index 0000000000000000000000000000000000000000..839af789480b4135892a23403fc9045490d62466 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_attrs_data.py @@ -0,0 +1,320 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Attribute data transfer testing module + + Covers all data read/write and type-conversion operations for attributes. +""" + +import numpy as np + +from .common import TestCase, make_name + +import h5py +from h5py import h5a, h5s, h5t +from h5py import File +from h5py._hl.base import is_empty_dataspace + + +class BaseAttrs(TestCase): + + def setUp(self): + self.f = File(self.mktemp(), 'w') + + def tearDown(self): + if self.f: + self.f.close() + + +class TestScalar(BaseAttrs): + + """ + Feature: Scalar types map correctly to array scalars + """ + + def test_int(self): + """ Integers are read as correct NumPy type """ + name = make_name() + self.f.attrs[name] = np.array(1, dtype=np.int8) + out = self.f.attrs[name] + self.assertIsInstance(out, np.int8) + + def test_compound(self): + """ Compound scalars are read as numpy.void """ + name = make_name() + dt = np.dtype([('a', 'i'), ('b', 'f')]) + data = np.array((1, 4.2), dtype=dt) + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertIsInstance(out, np.void) + self.assertEqual(out, data) + self.assertEqual(out['b'], data['b']) + + def test_compound_with_vlen_fields(self): + """ Compound scalars with vlen fields can be written and read """ + name = make_name() + dt = np.dtype([('a', h5py.vlen_dtype(np.int32)), + ('b', h5py.vlen_dtype(np.int32))]) + + data = np.array((np.array(list(range(1, 5)), dtype=np.int32), + np.array(list(range(8, 10)), dtype=np.int32)), dtype=dt)[()] + + self.f.attrs[name] = data + out = self.f.attrs[name] + + # Specifying check_alignment=False because vlen fields have 8 bytes of padding + # because the vlen datatype in hdf5 occupies 16 bytes + self.assertArrayEqual(out, data, check_alignment=False) + + def test_nesting_compound_with_vlen_fields(self): + """ Compound scalars with nested compound vlen fields can be written and read """ + dt_inner = np.dtype([('a', h5py.vlen_dtype(np.int32)), + ('b', h5py.vlen_dtype(np.int32))]) + + dt = np.dtype([('f1', h5py.vlen_dtype(dt_inner)), + ('f2', np.int64)]) + + inner1 = (np.array(range(1, 3), dtype=np.int32), + np.array(range(6, 9), dtype=np.int32)) + + inner2 = (np.array(range(10, 14), dtype=np.int32), + np.array(range(16, 20), dtype=np.int32)) + + data = np.array((np.array([inner1, inner2], dtype=dt_inner), + 2), + dtype=dt)[()] + + name = make_name() + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertArrayEqual(out, data, check_alignment=False) + + def test_vlen_compound_with_vlen_string(self): + """ Compound scalars with vlen compounds containing vlen strings can be written and read """ + dt_inner = np.dtype([('a', h5py.string_dtype()), + ('b', h5py.string_dtype())]) + + dt = np.dtype([('f', h5py.vlen_dtype(dt_inner))]) + + name = make_name() + data = np.array((np.array([(b"apples", b"bananas"), (b"peaches", b"oranges")], dtype=dt_inner),),dtype=dt)[()] + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertArrayEqual(out, data, check_alignment=False) + + +class TestArray(BaseAttrs): + + """ + Feature: Non-scalar types are correctly retrieved as ndarrays + """ + + def test_single(self): + """ Single-element arrays are correctly recovered """ + name = make_name() + data = np.ndarray((1,), dtype='f') + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertIsInstance(out, np.ndarray) + self.assertEqual(out.shape, (1,)) + + def test_multi(self): + """ Rank-1 arrays are correctly recovered """ + name = make_name() + data = np.ndarray((42,), dtype='f') + data[:] = 42.0 + data[10:35] = -47.0 + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertIsInstance(out, np.ndarray) + self.assertEqual(out.shape, (42,)) + self.assertArrayEqual(out, data) + + +class TestTypes(BaseAttrs): + + """ + Feature: All supported types can be stored in attributes + """ + + def test_int(self): + """ Storage of integer types """ + name = make_name() + dtypes = (np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64) + for dt in dtypes: + data = np.ndarray((1,), dtype=dt) + data[...] = 42 + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertEqual(out.dtype, dt) + self.assertArrayEqual(out, data) + + def test_float(self): + """ Storage of floating point types """ + name = make_name() + dtypes = tuple(np.dtype(x) for x in ('<f4', '>f4', '>f8', '<f8')) + + for dt in dtypes: + data = np.ndarray((1,), dtype=dt) + data[...] = 42.3 + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertEqual(out.dtype, dt) + self.assertArrayEqual(out, data) + + def test_complex(self): + """ Storage of complex types """ + name = make_name() + dtypes = tuple(np.dtype(x) for x in ('<c8', '>c8', '<c16', '>c16')) + + for dt in dtypes: + data = np.ndarray((1,), dtype=dt) + data[...] = -4.2j + 35.9 + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertEqual(out.dtype, dt) + self.assertArrayEqual(out, data) + + def test_string(self): + """ Storage of fixed-length strings """ + name = make_name() + dtypes = tuple(np.dtype(x) for x in ('|S1', '|S10')) + + for dt in dtypes: + data = np.ndarray((1,), dtype=dt) + data[...] = 'h' + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertEqual(out.dtype, dt) + self.assertEqual(out[0], data[0]) + + def test_bool(self): + """ Storage of NumPy booleans """ + name = make_name() + data = np.ndarray((2,), dtype=np.bool_) + data[...] = True, False + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertEqual(out.dtype, data.dtype) + self.assertEqual(out[0], data[0]) + self.assertEqual(out[1], data[1]) + + def test_vlen_string_array(self): + """ Storage of vlen byte string arrays""" + name = make_name() + dt = h5py.string_dtype(encoding='ascii') + + data = np.ndarray((2,), dtype=dt) + data[...] = "Hello", "Hi there! This is HDF5!" + + self.f.attrs[name] = data + out = self.f.attrs[name] + self.assertEqual(out.dtype, dt) + self.assertEqual(out[0], data[0]) + self.assertEqual(out[1], data[1]) + + def test_string_scalar(self): + """ Storage of variable-length byte string scalars (auto-creation) """ + name = make_name() + self.f.attrs[name] = b'Hello' + out = self.f.attrs[name] + + self.assertEqual(out, 'Hello') + self.assertEqual(type(out), str) + + aid = h5py.h5a.open(self.f.id, name.encode('utf-8')) + tid = aid.get_type() + self.assertEqual(type(tid), h5py.h5t.TypeStringID) + self.assertEqual(tid.get_cset(), h5py.h5t.CSET_ASCII) + self.assertTrue(tid.is_variable_str()) + + def test_unicode_scalar(self): + """ Storage of variable-length unicode strings (auto-creation) """ + name = make_name() + self.f.attrs[name] = u"Hello" + chr(0x2340) + u"!!" + out = self.f.attrs[name] + self.assertEqual(out, u"Hello" + chr(0x2340) + u"!!") + self.assertEqual(type(out), str) + + aid = h5py.h5a.open(self.f.id, name.encode('utf-8')) + tid = aid.get_type() + self.assertEqual(type(tid), h5py.h5t.TypeStringID) + self.assertEqual(tid.get_cset(), h5py.h5t.CSET_UTF8) + self.assertTrue(tid.is_variable_str()) + + +class TestEmpty(BaseAttrs): + + def setUp(self): + BaseAttrs.setUp(self) + sid = h5s.create(h5s.NULL) + tid = h5t.C_S1.copy() + tid.set_size(10) + aid = h5a.create(self.f.id, b'x', tid, sid) + self.empty_obj = h5py.Empty(np.dtype("S10")) + + def test_read(self): + self.assertEqual( + self.empty_obj, self.f.attrs['x'] + ) + + def test_write(self): + name = make_name() + self.f.attrs[name] = self.empty_obj + self.assertTrue( + is_empty_dataspace(h5a.open(self.f.id, name.encode("utf-8"))) + ) + + def test_modify(self): + with self.assertRaises(OSError): + self.f.attrs.modify('x', 1) + + def test_values(self): + # list() is for Py3 where these are iterators + values = list(self.f.attrs.values()) + self.assertEqual( + [self.empty_obj], values + ) + + def test_items(self): + items = list(self.f.attrs.items()) + self.assertEqual( + [(u"x", self.empty_obj)], items + ) + + def test_itervalues(self): + values = list(self.f.attrs.values()) + self.assertEqual( + [self.empty_obj], values + ) + + def test_iteritems(self): + items = list(self.f.attrs.items()) + self.assertEqual( + [(u"x", self.empty_obj)], items + ) + + +class TestWriteException(BaseAttrs): + + """ + Ensure failed attribute writes don't leave garbage behind. + """ + + def test_write(self): + """ ValueError on string write wipes out attribute """ + + s = b"Hello\x00Hello" + + with self.assertRaises(ValueError): + self.f.attrs["x"] = s + with self.assertRaises(KeyError): + self.f.attrs["x"] diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_base.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_base.py new file mode 100644 index 0000000000000000000000000000000000000000..fdd8ecb96e87150f771b2b1792409f34a2294bb9 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_base.py @@ -0,0 +1,150 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Common high-level operations test + + Tests features common to all high-level objects, like the .name property. +""" + +from h5py import File +from h5py._hl.base import is_hdf5, Empty +from .common import ut, TestCase, UNICODE_FILENAMES, make_name + +import numpy as np +import os +import tempfile + +class BaseTest(TestCase): + + def setUp(self): + self.f = File(self.mktemp(), 'w') + + def tearDown(self): + if self.f: + self.f.close() + + +class TestName(BaseTest): + + """ + Feature: .name attribute returns the object name + """ + + def test_anonymous(self): + """ Anonymous objects have name None """ + grp = self.f.create_group(None) + self.assertIs(grp.name, None) + + +class TestParent(BaseTest): + + """ + test the parent group of the high-level interface objects + """ + def test_object_parent_anonymous(self): + # Anonymous objects + grp = self.f.create_group(None) + # Parent of an anonymous object is undefined + with self.assertRaises(ValueError): + grp.parent + + def test_object_parent_named(self): + # Named objects + name = make_name() + grp = self.f.create_group(name) + sub_grp = grp.create_group("foo") + parent = sub_grp.parent.name + self.assertEqual(parent, "/" + name) + + +class TestMapping(BaseTest): + + """ + Test if the registration of Group as a + Mapping behaves as expected + """ + + def setUp(self): + super().setUp() + data = ('a', 'b') + self.grp = self.f.create_group('bar') + self.attr = self.f.attrs.create('x', data) + + def test_keys(self): + key_1 = self.f.keys() + self.assertIsInstance(repr(key_1), str) + key_2 = self.grp.keys() + self.assertIsInstance(repr(key_2), str) + + def test_values(self): + value_1 = self.f.values() + self.assertIsInstance(repr(value_1), str) + value_2 = self.grp.values() + self.assertIsInstance(repr(value_2), str) + + def test_items(self): + item_1 = self.f.items() + self.assertIsInstance(repr(item_1), str) + item_2 = self.grp.items() + self.assertIsInstance(repr(item_1), str) + + +class TestRepr(BaseTest): + + """ + repr() works correctly with Unicode names + """ + + USTRING = chr(0xfc) + chr(0xdf) + + def _check_type(self, obj): + self.assertIsInstance(repr(obj), str) + + def test_group(self): + """ Group repr() with unicode """ + grp = self.f.create_group(make_name(self.USTRING)) + self._check_type(grp) + + def test_dataset(self): + """ Dataset repr() with unicode """ + dset = self.f.create_dataset(make_name(self.USTRING), (1,), "i4") + self._check_type(dset) + + def test_namedtype(self): + """ Named type repr() with unicode """ + name = make_name(self.USTRING) + self.f[name] = np.dtype('f') + typ = self.f[name] + self._check_type(typ) + + def test_empty(self): + data = Empty(dtype='f') + self.assertNotEqual(Empty(dtype='i'), data) + self._check_type(data) + + @ut.skipIf(not UNICODE_FILENAMES, "Filesystem unicode support required") + def test_file(self): + """ File object repr() with unicode """ + fname = tempfile.mktemp(self.USTRING+'.hdf5') + try: + with File(fname,'w') as f: + self._check_type(f) + finally: + try: + os.unlink(fname) + except Exception: + pass + +def test_is_hdf5(): + filename = File(tempfile.mktemp(), "w").filename + assert is_hdf5(filename) + # non-existing HDF5 file + filename = tempfile.mktemp() + assert not is_hdf5(filename) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_big_endian_file.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_big_endian_file.py new file mode 100644 index 0000000000000000000000000000000000000000..1b419a8c2c6c906c2cc907ddd17e4b4edb6793fa --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_big_endian_file.py @@ -0,0 +1,50 @@ + +import numpy as np +from h5py import File +from .common import TestCase, make_name +from .data_files import get_data_file_path + + +def test_vlen_big_endian(): + with File(get_data_file_path("vlen_string_s390x.h5")) as f: + assert f.attrs["created_on_s390x"] == 1 + + dset = f["DSvariable"] + assert dset[0] == b"Parting" + assert dset[1] == b"is such" + assert dset[2] == b"sweet" + assert dset[3] == b"sorrow..." + + dset = f["DSLEfloat"] + assert dset[0] == 3.14 + assert dset[1] == 1.61 + assert dset[2] == 2.71 + assert dset[3] == 2.41 + assert dset[4] == 1.2 + assert dset.dtype == "<f8" + + # Same float values with big endianness + assert f["DSBEfloat"][0] == 3.14 + assert f["DSBEfloat"].dtype == ">f8" + + assert f["DSLEint"][0] == 1 + assert f["DSLEint"].dtype == "<u8" + + # Same int values with big endianness + assert f["DSBEint"][0] == 1 + assert f["DSBEint"].dtype == ">i8" + + +class TestEndianess(TestCase): + def test_simple_int_be(self): + name = make_name() + fname = self.mktemp() + + arr = np.ndarray(shape=(1,), dtype=">i4", buffer=bytearray([0, 1, 3, 2])) + be_number = 0 * 256 ** 3 + 1 * 256 ** 2 + 3 * 256 ** 1 + 2 * 256 ** 0 + + with File(fname, mode="w") as f: + f.create_dataset(name, data=arr) + + with File(fname, mode="r") as f: + assert f[name][()][0] == be_number diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_completions.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_completions.py new file mode 100644 index 0000000000000000000000000000000000000000..f3a163fa4823933cb811a40b62a69c092f0eb51c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_completions.py @@ -0,0 +1,59 @@ +import h5py + +from .common import TestCase, make_name + + +class TestCompletions(TestCase): + + def test_root_group_completions(self): + # Test completions on top-level file. + with h5py.File(self.mktemp(), 'w') as f: + f.create_group('h') + f.create_group('g') + f.create_dataset('data', [1, 2, 3], "f4") + self.assertEqual( + f._ipython_key_completions_(), + ['data', 'g', 'h'], + ) + + f.create_group('data2') + # Test that order is alphabetical, and that there is no + # internal ordering between groups and datasets. + self.assertEqual( + f._ipython_key_completions_(), + ['data', 'data2', 'g', 'h'], + ) + + def test_subgroup_completions(self): + g = self.f.create_group(make_name()) + g.create_dataset('g_data2', [4, 5, 6], "f4") + g.create_dataset('g_data1', [1, 2, 3], "f4") + self.assertEqual( + g._ipython_key_completions_(), + ['g_data1', 'g_data2'], # Order is alphabetical + ) + + g.create_dataset('g_data3', [7, 8, 9], "f") + self.assertEqual( + g._ipython_key_completions_(), + ['g_data1', 'g_data2', 'g_data3'], + ) + + def test_attrs_completions(self): + # In pytest-run-parallel, let every thread run on a different set of attrs + attrs = self.f.create_group(make_name()).attrs + + # Write out of alphabetical order to test that completions come back in + # alphabetical order, as opposed to, say, insertion order. + attrs['b'] = 1 + attrs['a'] = 2 + self.assertEqual( + attrs._ipython_key_completions_(), + ['a', 'b'] + ) + + attrs['c'] = 3 + self.assertEqual( + attrs._ipython_key_completions_(), + ['a', 'b', 'c'] + ) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_dataset.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..0b33e55415336f6a72cc3fc05e65a16f1173804e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_dataset.py @@ -0,0 +1,2365 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Dataset testing operations. + + Tests all dataset operations, including creation, with the exception of: + + 1. Slicing operations for read and write, handled by module test_slicing + 2. Type conversion for read and write (currently untested) +""" + +import pathlib +import os +import sys +import numpy as np +import platform +import pytest +import threading +from concurrent.futures import ThreadPoolExecutor + +from h5py import File, Dataset +from h5py._hl.base import is_empty_dataspace, product +from h5py import h5f, h5t +import h5py + +from .common import ut, TestCase, NUMPY_RELEASE_VERSION, is_main_thread, make_name +from .data_files import get_data_file_path +from ..h5py_warnings import H5pyDeprecationWarning + + +class BaseDataset(TestCase): + def setUp(self): + self.f = File(self.mktemp(), 'w') + + def tearDown(self): + if self.f: + self.f.close() + + +class TestRepr(BaseDataset): + """ + Feature: repr(Dataset) behaves sensibly + """ + endian_mark = '>' if sys.byteorder=='big' else '<' + + def test_repr_basic(self): + name = make_name() + ds = self.f.create_dataset(name, (4,), dtype='int32') + assert repr(ds) == f'<HDF5 dataset "{name}": shape (4,), type "{self.endian_mark}i4">' + + @pytest.mark.thread_unsafe + def test_repr_closed(self): + """ repr() works on live and dead datasets """ + ds = self.f.create_dataset(make_name(), (4,), dtype="f4") + self.f.close() + assert repr(ds) == '<Closed HDF5 dataset>' + + def test_repr_anonymous(self): + ds = self.f.create_dataset(None, (4,), dtype='int32') + assert repr(ds) == f'<HDF5 dataset (anonymous): shape (4,), type "{self.endian_mark}i4">' + + +class TestCreateShape(BaseDataset): + + """ + Feature: Datasets can be created from a shape only + """ + + def test_create_scalar(self): + """ Create a scalar dataset """ + dset = self.f.create_dataset(make_name(), (), dtype='f4') + self.assertEqual(dset.shape, ()) + + def test_create_simple(self): + """ Create a size-1 dataset """ + dset = self.f.create_dataset(make_name(), (1,), dtype='f4') + self.assertEqual(dset.shape, (1,)) + + def test_create_integer(self): + """ Create a size-1 dataset with integer shape""" + dset = self.f.create_dataset(make_name(), 1, dtype='f4') + self.assertEqual(dset.shape, (1,)) + + def test_create_extended_1d(self): + """ Create an extended dataset with tuple shape """ + dset = self.f.create_dataset(make_name(), (63,), dtype='f4') + self.assertEqual(dset.shape, (63,)) + self.assertEqual(dset.size, 63) + + def test_create_extended_2d(self): + """ Create an extended dataset with 2 dimensions """ + dset = self.f.create_dataset(make_name(), (6, 10), dtype='f4') + self.assertEqual(dset.shape, (6, 10)) + self.assertEqual(dset.size, (60)) + + def test_create_integer_extended(self): + """ Create an extended dataset with integer shape """ + dset = self.f.create_dataset(make_name(), 63, dtype='f4') + self.assertEqual(dset.shape, (63,)) + self.assertEqual(dset.size, 63) + + def test_default_dtype(self): + """ Confirm that the default dtype is float """ + dset = self.f.create_dataset(make_name(), (63,), dtype='f4') + self.assertEqual(dset.dtype, np.dtype('=f4')) + + def test_missing_shape(self): + """ Missing shape raises TypeError """ + with self.assertRaises(TypeError): + self.f.create_dataset(make_name()) + + def test_long_double(self): + """ Confirm that the default dtype is float """ + dset = self.f.create_dataset(make_name(), (63,), dtype=np.longdouble) + if platform.machine() in ['ppc64le']: + pytest.xfail("Storage of long double deactivated on %s" % platform.machine()) + self.assertEqual(dset.dtype, np.longdouble) + + @ut.skipIf(not hasattr(np, "complex256"), "No support for complex256") + def test_complex256(self): + """ Confirm that the default dtype is float """ + dset = self.f.create_dataset(make_name(), (63,), + dtype=np.dtype('complex256')) + self.assertEqual(dset.dtype, np.dtype('complex256')) + + def test_name_bytes(self): + dset = self.f.create_dataset(make_name("foo").encode('utf-8'), (1,), dtype='f4') + self.assertEqual(dset.shape, (1,)) + + dset2 = self.f.create_dataset((make_name("bar{}/baz")).encode('utf-8'), (2,), dtype='f4') + self.assertEqual(dset2.shape, (2,)) + + def test_no_dtype(self): + # From h5py 4.0, either dtype or data will be required + with pytest.warns(H5pyDeprecationWarning): + dset = self.f.create_dataset(make_name(), (5,)) + + assert dset.dtype == np.dtype('f4') + +class TestCreateData(BaseDataset): + + """ + Feature: Datasets can be created from existing data + """ + + def test_create_scalar(self): + """ Create a scalar dataset from existing array """ + data = np.ones((), 'f') + dset = self.f.create_dataset(make_name(), data=data) + self.assertEqual(dset.shape, data.shape) + + def test_create_extended(self): + """ Create an extended dataset from existing data """ + data = np.ones((63,), 'f') + dset = self.f.create_dataset(make_name(), data=data) + self.assertEqual(dset.shape, data.shape) + + def test_dataset_intermediate_group(self): + """ Create dataset with missing intermediate groups """ + name = make_name("/foo{}/bar/baz") + ds = self.f.create_dataset(name, shape=(10, 10), dtype='<i4') + self.assertIsInstance(ds, h5py.Dataset) + self.assertTrue(name in self.f) + + def test_reshape(self): + """ Create from existing data, and make it fit a new shape """ + data = np.arange(30, dtype='f') + dset = self.f.create_dataset(make_name(), shape=(10, 3), data=data) + self.assertEqual(dset.shape, (10, 3)) + self.assertArrayEqual(dset[...], data.reshape((10, 3))) + + def test_appropriate_low_level_id(self): + " Binding Dataset to a non-DatasetID identifier fails with ValueError " + with self.assertRaises(ValueError): + Dataset(self.f['/'].id) + + def check_h5_string(self, dset, cset, length): + tid = dset.id.get_type() + assert isinstance(tid, h5t.TypeStringID) + assert tid.get_cset() == cset + if length is None: + assert tid.is_variable_str() + else: + assert not tid.is_variable_str() + assert tid.get_size() == length + + def test_create_bytestring(self): + """ Creating dataset with byte string yields vlen ASCII dataset """ + def check_vlen_ascii(dset): + self.check_h5_string(dset, h5t.CSET_ASCII, length=None) + check_vlen_ascii(self.f.create_dataset(make_name("a"), data=b'abc')) + check_vlen_ascii(self.f.create_dataset(make_name("b"), data=[b'abc', b'def'])) + check_vlen_ascii(self.f.create_dataset(make_name("c"), data=[[b'abc'], [b'def']])) + check_vlen_ascii(self.f.create_dataset( + make_name("d"), data=np.array([b'abc', b'def'], dtype=object) + )) + + def test_create_np_s(self): + dset = self.f.create_dataset(make_name(), data=np.array([b'abc', b'def'], dtype='S3')) + self.check_h5_string(dset, h5t.CSET_ASCII, length=3) + + def test_create_strings(self): + def check_vlen_utf8(dset): + self.check_h5_string(dset, h5t.CSET_UTF8, length=None) + check_vlen_utf8(self.f.create_dataset(make_name("a"), data='abc')) + check_vlen_utf8(self.f.create_dataset(make_name("b"), data=['abc', 'def'])) + check_vlen_utf8(self.f.create_dataset(make_name("c"), data=[['abc'], ['def']])) + check_vlen_utf8(self.f.create_dataset( + make_name("d"), data=np.array(['abc', 'def'], dtype=object) + )) + + def test_create_np_u(self): + with self.assertRaises(TypeError): + self.f.create_dataset(make_name(), data=np.array([b'abc', b'def'], dtype='U3')) + + def test_empty_create_via_None_shape(self): + name = make_name() + self.f.create_dataset(name, dtype='f') + self.assertTrue(is_empty_dataspace(self.f[name].id)) + + def test_empty_create_via_Empty_class(self): + name = make_name() + self.f.create_dataset(name, data=h5py.Empty(dtype='f')) + self.assertTrue(is_empty_dataspace(self.f[name].id)) + + def test_create_incompatible_data(self): + # Shape tuple is incompatible with data + with self.assertRaises(ValueError): + self.f.create_dataset(make_name(), shape=4, data= np.arange(3)) + + +class TestReadDirectly: + + """ + Feature: Read data directly from Dataset into a Numpy array + """ + + @pytest.mark.parametrize( + 'source_shape,dest_shape,source_sel,dest_sel', + [ + ((100,), (100,), np.s_[0:10], np.s_[50:60]), + ((70,), (100,), np.s_[50:60], np.s_[90:]), + ((30, 10), (20, 20), np.s_[:20, :], np.s_[:, :10]), + ((5, 7, 9), (6,), np.s_[2, :6, 3], np.s_[:]), + ]) + def test_read_direct(self, writable_file, source_shape, dest_shape, source_sel, dest_sel): + source_values = np.arange(product(source_shape), dtype="int64").reshape(source_shape) + dset = writable_file.create_dataset(make_name(), source_shape, data=source_values) + arr = np.full(dest_shape, -1, dtype="int64") + expected = arr.copy() + expected[dest_sel] = source_values[source_sel] + + dset.read_direct(arr, source_sel, dest_sel) + np.testing.assert_array_equal(arr, expected) + + def test_no_sel(self, writable_file): + dset = writable_file.create_dataset(make_name(), (10,), data=np.arange(10, dtype="int64")) + arr = np.ones((10,), dtype="int64") + dset.read_direct(arr) + np.testing.assert_array_equal(arr, np.arange(10, dtype="int64")) + + def test_empty(self, writable_file): + empty_dset = writable_file.create_dataset(make_name(), dtype='int64') + arr = np.ones((100,), 'int64') + with pytest.raises(TypeError): + empty_dset.read_direct(arr, np.s_[0:10], np.s_[50:60]) + + def test_wrong_shape(self, writable_file): + dset = writable_file.create_dataset(make_name(), (100,), dtype='int64') + arr = np.ones((200,)) + with pytest.raises(TypeError): + dset.read_direct(arr) + + def test_not_c_contiguous(self, writable_file): + dset = writable_file.create_dataset(make_name(), (10, 10), dtype='int64') + arr = np.ones((10, 10), order='F') + with pytest.raises(TypeError): + dset.read_direct(arr) + + def test_zero_length(self, writable_file): + shape = (0, 20) + dset = writable_file.create_dataset(make_name(), shape, dtype=np.int64) + arr = np.zeros(shape, dtype=np.int64) + dset.read_direct(arr) + + # We should still get an error if the shape is wrong + arr2 = np.zeros((0, 25), dtype=np.int64) + with pytest.raises(TypeError): + dset.read_direct(arr2) + + +class TestWriteDirectly: + + """ + Feature: Write Numpy array directly into Dataset + """ + + @pytest.mark.parametrize( + 'source_shape,dest_shape,source_sel,dest_sel', + [ + ((100,), (100,), np.s_[0:10], np.s_[50:60]), + ((70,), (100,), np.s_[50:60], np.s_[90:]), + ((30, 10), (20, 20), np.s_[:20, :], np.s_[:, :10]), + ((5, 7, 9), (6,), np.s_[2, :6, 3], np.s_[:]), + ]) + def test_write_direct(self, writable_file, source_shape, dest_shape, source_sel, dest_sel): + dset = writable_file.create_dataset(make_name(), dest_shape, dtype='int32', fillvalue=-1) + arr = np.arange(product(source_shape)).reshape(source_shape) + expected = np.full(dest_shape, -1, dtype='int32') + expected[dest_sel] = arr[source_sel] + dset.write_direct(arr, source_sel, dest_sel) + np.testing.assert_array_equal(dset[:], expected) + + def test_empty(self, writable_file): + empty_dset = writable_file.create_dataset(make_name(), dtype='int64') + with pytest.raises(TypeError): + empty_dset.write_direct(np.ones((100,)), np.s_[0:10], np.s_[50:60]) + + def test_wrong_shape(self, writable_file): + dset = writable_file.create_dataset(make_name(), (100,), dtype='int64') + arr = np.ones((200,)) + with pytest.raises(TypeError): + dset.write_direct(arr) + + def test_not_c_contiguous(self, writable_file): + dset = writable_file.create_dataset(make_name(), (10, 10), dtype='int64') + arr = np.ones((10, 10), order='F') + with pytest.raises(TypeError): + dset.write_direct(arr) + + +class TestCreateRequire(BaseDataset): + + """ + Feature: Datasets can be created only if they don't exist in the file + """ + + def test_create(self): + """ Create new dataset with no conflicts """ + dset = self.f.require_dataset(make_name(), (10, 3), 'f') + self.assertIsInstance(dset, Dataset) + self.assertEqual(dset.shape, (10, 3)) + + def test_create_existing(self): + """ require_dataset yields existing dataset """ + name = make_name() + dset = self.f.require_dataset(name, (10, 3), 'f') + dset[0, 0] = 123 + dset2 = self.f.require_dataset(name, (10, 3), 'f') + self.assertEqual(dset, dset2) + + def test_create_1D_integer(self): + """ require_dataset with integer shape yields existing dataset""" + name = make_name() + dset = self.f.require_dataset(name, 10, 'f') + dset[0] = 123 + dset2 = self.f.require_dataset(name, 10, 'f') + self.assertEqual(dset, dset2) + + def test_create_1D_tuple(self): + name = make_name() + dset = self.f.require_dataset(name, (10,), 'f') + dset[0] = 123 + dset2 = self.f.require_dataset(name, 10, 'f') + self.assertEqual(dset, dset2) + + def test_create_1D_binary(self): + name = make_name() + dset = self.f.require_dataset(name, 10, 'f') + dset[0] = 123 + dset2 = self.f.require_dataset(name.encode('utf-8'), (10,), 'f') + self.assertEqual(dset, dset2) + + def test_shape_conflict(self): + """ require_dataset with shape conflict yields TypeError """ + name = make_name() + self.f.create_dataset(name, (10, 3), 'f') + with self.assertRaises(TypeError): + self.f.require_dataset(name, (10, 4), 'f') + + def test_type_conflict(self): + """ require_dataset with object type conflict yields TypeError """ + name = make_name() + self.f.create_group(name) + with self.assertRaises(TypeError): + self.f.require_dataset(name, (10, 3), 'f') + + def test_dtype_conflict(self): + """ require_dataset with dtype conflict (strict mode) yields TypeError + """ + name = make_name() + dset = self.f.create_dataset(name, (10, 3), 'f') + with self.assertRaises(TypeError): + self.f.require_dataset(name, (10, 3), 'S10') + + def test_dtype_exact(self): + """ require_dataset with exactly dtype match """ + name = make_name() + dset = self.f.create_dataset(name, (10, 3), 'f') + dset[0, 0] = 123 + dset2 = self.f.require_dataset(name, (10, 3), 'f', exact=True) + self.assertEqual(dset, dset2) + + def test_dtype_close(self): + """ require_dataset with convertible type succeeds (non-strict mode) + """ + name = make_name() + dset = self.f.create_dataset(name, (10, 3), 'i4') + # Set a value too large for i2 to test for spurious intermediate conversions + dset[0, 0] = 98765 + dset2 = self.f.require_dataset(name, (10, 3), 'i2', exact=False) + self.assertEqual(dset, dset2) + self.assertEqual(dset2.dtype, np.dtype('i4')) + + +class TestCreateChunked(BaseDataset): + + """ + Feature: Datasets can be created by manually specifying chunks + """ + + def test_create_chunks(self): + """ Create via chunks tuple """ + dset = self.f.create_dataset(make_name(), shape=(100,), chunks=(10,), dtype='f4') + self.assertEqual(dset.chunks, (10,)) + + def test_create_chunks_integer(self): + """ Create via chunks integer """ + dset = self.f.create_dataset(make_name(), shape=(100,), chunks=10, dtype='f4') + self.assertEqual(dset.chunks, (10,)) + + def test_chunks_mismatch(self): + """ Illegal chunk size raises ValueError """ + with self.assertRaises(ValueError): + self.f.create_dataset(make_name(), shape=(100,), chunks=(200,), dtype='f4') + + def test_chunks_false(self): + """ Chunked format required for given storage options """ + with self.assertRaises(ValueError): + self.f.create_dataset(make_name(), shape=(10,), maxshape=100, chunks=False, dtype='f4') + + def test_chunks_scalar(self): + """ Attempting to create chunked scalar dataset raises TypeError """ + with self.assertRaises(TypeError): + self.f.create_dataset(make_name(), shape=(), chunks=(50,), dtype='f4') + + def test_auto_chunks(self): + """ Auto-chunking of datasets """ + dset = self.f.create_dataset(make_name(), shape=(20, 100), chunks=True, dtype='f4') + self.assertIsInstance(dset.chunks, tuple) + self.assertEqual(len(dset.chunks), 2) + + def test_auto_chunks_abuse(self): + """ Auto-chunking with pathologically large element sizes """ + dset = self.f.create_dataset(make_name(), shape=(3,), dtype='S100000000', chunks=True) + self.assertEqual(dset.chunks, (1,)) + + def test_scalar_assignment(self): + """ Test scalar assignment of chunked dataset """ + dset = self.f.create_dataset(make_name(), shape=(3, 50, 50), + dtype=np.int32, chunks=(1, 50, 50)) + # test assignment of selection smaller than chunk size + dset[1, :, 40] = 10 + self.assertTrue(np.all(dset[1, :, 40] == 10)) + + # test assignment of selection equal to chunk size + dset[1] = 11 + self.assertTrue(np.all(dset[1] == 11)) + + # test assignment of selection bigger than chunk size + dset[0:2] = 12 + self.assertTrue(np.all(dset[0:2] == 12)) + + def test_auto_chunks_no_shape(self): + """ Auto-chunking of empty datasets not allowed""" + name = make_name() + with pytest.raises(TypeError, match='Empty') as err: + self.f.create_dataset(name, dtype='S100', chunks=True) + + with pytest.raises(TypeError, match='Empty') as err: + self.f.create_dataset(name, dtype='S100', maxshape=20) + + +class TestCreateFillvalue(BaseDataset): + + """ + Feature: Datasets can be created with fill value + """ + + def test_create_fillval(self): + """ Fill value is reflected in dataset contents """ + dset = self.f.create_dataset(make_name(), (10,), fillvalue=4.0, dtype='f4') + self.assertEqual(dset[0], 4.0) + self.assertEqual(dset[7], 4.0) + + def test_property(self): + """ Fill value is recoverable via property """ + dset = self.f.create_dataset(make_name(), (10,), fillvalue=3.0, dtype='f4') + self.assertEqual(dset.fillvalue, 3.0) + self.assertNotIsInstance(dset.fillvalue, np.ndarray) + + def test_property_none(self): + """ .fillvalue property works correctly if not set """ + dset = self.f.create_dataset(make_name(), (10,), dtype='f4') + self.assertEqual(dset.fillvalue, 0) + + def test_compound(self): + """ Fill value works with compound types """ + dt = np.dtype([('a', 'f4'), ('b', 'i8')]) + v = np.ones((1,), dtype=dt)[0] + dset = self.f.create_dataset(make_name(), (10,), dtype=dt, fillvalue=v) + self.assertEqual(dset.fillvalue, v) + self.assertAlmostEqual(dset[4], v) + + def test_exc(self): + """ Bogus fill value raises ValueError """ + with self.assertRaises(ValueError): + dset = self.f.create_dataset(make_name(), (10,), + dtype=[('a', 'i'), ('b', 'f')], fillvalue=42) + + +class TestFillTime(BaseDataset): + + """ + Feature: Datasets created with specified fill time property + """ + + def test_fill_time_default(self): + """ Fill time default to IFSET """ + dset = self.f.create_dataset(make_name(), (10,), 'f4', fillvalue=4.0) + plist = dset.id.get_create_plist() + self.assertEqual(plist.get_fill_time(), h5py.h5d.FILL_TIME_IFSET) + self.assertEqual(dset[0], 4.0) + self.assertEqual(dset[7], 4.0) + + @ut.skipIf('gzip' not in h5py.filters.encode, "DEFLATE is not installed") + def test_compressed_default(self): + """ Fill time is IFSET for compressed dataset (chunked) """ + dset = self.f.create_dataset(make_name(), (10,), "f4", compression='gzip', + fillvalue=4.0) + plist = dset.id.get_create_plist() + self.assertEqual(plist.get_fill_time(), h5py.h5d.FILL_TIME_IFSET) + self.assertEqual(dset[0], 4.0) + self.assertEqual(dset[7], 4.0) + + def test_fill_time_never(self): + """ Fill time set to NEVER """ + dset = self.f.create_dataset(make_name(), (10,), "f4", fillvalue=4.0, + fill_time='never') + plist = dset.id.get_create_plist() + self.assertEqual(plist.get_fill_time(), h5py.h5d.FILL_TIME_NEVER) + # should not be equal to the explicitly set fillvalue + self.assertNotEqual(dset[0], 4.0) + self.assertNotEqual(dset[7], 4.0) + + def test_fill_time_alloc(self): + """ Fill time explicitly set to ALLOC """ + dset = self.f.create_dataset(make_name(), (10,), "f4", fillvalue=4.0, + fill_time='alloc') + plist = dset.id.get_create_plist() + self.assertEqual(plist.get_fill_time(), h5py.h5d.FILL_TIME_ALLOC) + + def test_fill_time_ifset(self): + """ Fill time explicitly set to IFSET """ + dset = self.f.create_dataset(make_name(), (10,), 'f4', chunks=(2,), fillvalue=4.0, + fill_time='ifset') + plist = dset.id.get_create_plist() + self.assertEqual(plist.get_fill_time(), h5py.h5d.FILL_TIME_IFSET) + + def test_invalid_fill_time(self): + """ Choice of fill_time is 'alloc', 'never', 'ifset' """ + with self.assertRaises(ValueError): + dset = self.f.create_dataset(make_name(), (10,), 'f4', fill_time='fill_bad') + + def test_non_str_fill_time(self): + """ fill_time must be a string """ + with self.assertRaises(ValueError): + dset = self.f.create_dataset(make_name(), (10,), 'f4', fill_time=2) + + def test_resize_chunk_fill_time_default(self): + """ The resize dataset will be filled (by default fill value 0) """ + dset = self.f.create_dataset(make_name(), (50, ), 'f4', maxshape=(100, ), + chunks=(5, )) + plist = dset.id.get_create_plist() + self.assertEqual(plist.get_fill_time(), h5py.h5d.FILL_TIME_IFSET) + + assert np.isclose(dset[:], 0.0).all() + + dset.resize((100, )) + assert np.isclose(dset[:], 0.0).all() + + def test_resize_chunk_fill_time_never(self): + """ The resize dataset won't be filled """ + dset = self.f.create_dataset(make_name(), (50, ), 'f4', maxshape=(100, ), + fillvalue=4.0, fill_time='never', + chunks=(5, )) + plist = dset.id.get_create_plist() + self.assertEqual(plist.get_fill_time(), h5py.h5d.FILL_TIME_NEVER) + + assert not np.isclose(dset[:], 4.0).any() + + dset.resize((100, )) + assert not np.isclose(dset[:], 4.0).any() + + +@pytest.mark.parametrize('dt,expected', [ + (int, 0), + (np.int32, 0), + (np.int64, 0), + (float, 0.0), + (np.float32, 0.0), + (np.float64, 0.0), + (h5py.string_dtype(encoding='utf-8', length=5), b''), + (h5py.string_dtype(encoding='ascii', length=5), b''), + (h5py.string_dtype(encoding='utf-8'), b''), + (h5py.string_dtype(encoding='ascii'), b''), + (h5py.string_dtype(), b''), + +]) +def test_get_unset_fill_value(dt, expected, writable_file): + dset = writable_file.create_dataset(make_name(), (10,), dtype=dt) + assert dset.fillvalue == expected + + +class TestCreateNamedType(BaseDataset): + + """ + Feature: Datasets created from an existing named type + """ + + def test_named(self): + """ Named type object works and links the dataset to type """ + name = make_name("type") + self.f[name] = np.dtype('f8') + dt = self.f[name] + dset = self.f.create_dataset(make_name("x"), (100,), dtype=dt) + self.assertEqual(dset.dtype, np.dtype('f8')) + self.assertEqual(dset.id.get_type(), dt.id) + self.assertTrue(dset.id.get_type().committed()) + + +@ut.skipIf('gzip' not in h5py.filters.encode, "DEFLATE is not installed") +class TestCreateGzip(BaseDataset): + + """ + Feature: Datasets created with gzip compression + """ + + def test_gzip(self): + """ Create with explicit gzip options """ + dset = self.f.create_dataset(make_name(), (20, 30), 'f4', compression='gzip', + compression_opts=9) + self.assertEqual(dset.compression, 'gzip') + self.assertEqual(dset.compression_opts, 9) + + def test_gzip_implicit(self): + """ Create with implicit gzip level (level 4) """ + dset = self.f.create_dataset(make_name(), (20, 30), 'f4', compression='gzip') + self.assertEqual(dset.compression, 'gzip') + self.assertEqual(dset.compression_opts, 4) + + @pytest.mark.thread_unsafe(reason="monkey-patch") + def test_gzip_number(self): + """ Create with gzip level by specifying integer """ + name = make_name() + dset = self.f.create_dataset(name, (20, 30), 'f4', compression=7) + self.assertEqual(dset.compression, 'gzip') + self.assertEqual(dset.compression_opts, 7) + + original_compression_vals = h5py._hl.dataset._LEGACY_GZIP_COMPRESSION_VALS + try: + h5py._hl.dataset._LEGACY_GZIP_COMPRESSION_VALS = tuple() + with self.assertRaises(ValueError): + dset = self.f.create_dataset(name, (20, 30), 'f4', compression=7) + finally: + h5py._hl.dataset._LEGACY_GZIP_COMPRESSION_VALS = original_compression_vals + + def test_gzip_exc(self): + """ Illegal gzip level (explicit or implicit) raises ValueError """ + name = make_name() + with self.assertRaises((ValueError, RuntimeError)): + self.f.create_dataset(name, (20, 30), 'f4', compression=14) + with self.assertRaises(ValueError): + self.f.create_dataset(name, (20, 30), 'f4', compression=-4) + with self.assertRaises(ValueError): + self.f.create_dataset(name, (20, 30), 'f4', compression='gzip', + compression_opts=14) + + +@ut.skipIf('gzip' not in h5py.filters.encode, "DEFLATE is not installed") +class TestCreateCompressionNumber(BaseDataset): + + """ + Feature: Datasets created with a compression code + """ + @pytest.mark.thread_unsafe(reason="monkey-patch") + def test_compression_number(self): + """ Create with compression number of gzip (h5py.h5z.FILTER_DEFLATE) and a compression level of 7""" + original_compression_vals = h5py._hl.dataset._LEGACY_GZIP_COMPRESSION_VALS + try: + h5py._hl.dataset._LEGACY_GZIP_COMPRESSION_VALS = tuple() + dset = self.f.create_dataset('foo', (20, 30), compression=h5py.h5z.FILTER_DEFLATE, compression_opts=(7,), dtype='f4') + finally: + h5py._hl.dataset._LEGACY_GZIP_COMPRESSION_VALS = original_compression_vals + + self.assertEqual(dset.compression, 'gzip') + self.assertEqual(dset.compression_opts, 7) + + @pytest.mark.thread_unsafe(reason="monkey-patch") + def test_compression_number_invalid(self): + """ Create with invalid compression numbers """ + with self.assertRaises(ValueError) as e: + self.f.create_dataset('foo', (20, 30), compression=-999, dtype='f4') + self.assertIn("Invalid filter", str(e.exception)) + + with self.assertRaises(ValueError) as e: + self.f.create_dataset('foo', (20, 30), compression=100, dtype='f4') + self.assertIn("Unknown compression", str(e.exception)) + + original_compression_vals = h5py._hl.dataset._LEGACY_GZIP_COMPRESSION_VALS + try: + h5py._hl.dataset._LEGACY_GZIP_COMPRESSION_VALS = tuple() + + # Using gzip compression requires a compression level specified in compression_opts + with self.assertRaises(IndexError): + self.f.create_dataset('foo', (20, 30), compression=h5py.h5z.FILTER_DEFLATE, dtype='f4') + finally: + h5py._hl.dataset._LEGACY_GZIP_COMPRESSION_VALS = original_compression_vals + + +@ut.skipIf('lzf' not in h5py.filters.encode, "LZF is not installed") +class TestCreateLZF(BaseDataset): + + """ + Feature: Datasets created with LZF compression + """ + + def test_lzf_sparse(self): + """ Create with explicit lzf (empty chunks)""" + dset = self.f.create_dataset(make_name(), (20, 30), compression='lzf', dtype='f4') + self.assertEqual(dset.compression, 'lzf') + self.assertEqual(dset.compression_opts, None) + + def test_lzf_dense(self): + """ Create with explicit lzf (populated)""" + name = make_name() + testdata = np.arange(100) + dset = self.f.create_dataset(name, data=testdata, compression='lzf') + self.assertEqual(dset.compression, 'lzf') + self.assertEqual(dset.compression_opts, None) + + self.f.flush() # Actually write to file + + readdata = self.f[name][()] + self.assertArrayEqual(readdata, testdata) + + def test_lzf_exc(self): + """ Giving lzf options raises ValueError """ + with self.assertRaises(ValueError): + self.f.create_dataset(make_name(), (20, 30), 'f4', compression='lzf', + compression_opts=4) + + +@ut.skipIf('szip' not in h5py.filters.encode, "SZIP is not installed") +class TestCreateSZIP(BaseDataset): + + """ + Feature: Datasets created with LZF compression + """ + + def test_szip(self): + """ Create with explicit szip """ + dset = self.f.create_dataset(make_name(), (20, 30), 'f4', compression='szip', + compression_opts=('ec', 16)) + + +@ut.skipIf('shuffle' not in h5py.filters.encode, "SHUFFLE is not installed") +class TestCreateShuffle(BaseDataset): + + """ + Feature: Datasets can use shuffling filter + """ + + def test_shuffle(self): + """ Enable shuffle filter """ + dset = self.f.create_dataset(make_name(), (20, 30), shuffle=True, dtype='f4') + self.assertTrue(dset.shuffle) + + +@ut.skipIf('fletcher32' not in h5py.filters.encode, "FLETCHER32 is not installed") +class TestCreateFletcher32(BaseDataset): + """ + Feature: Datasets can use the fletcher32 filter + """ + + def test_fletcher32(self): + """ Enable fletcher32 filter """ + dset = self.f.create_dataset(make_name(), (20, 30), fletcher32=True, dtype='f4') + self.assertTrue(dset.fletcher32) + + +@ut.skipIf('scaleoffset' not in h5py.filters.encode, "SCALEOFFSET is not installed") +class TestCreateScaleOffset(BaseDataset): + """ + Feature: Datasets can use the scale/offset filter + + Note: loss of precision caused by scaleoffset only becomes visible + when closing and reopening the File. + Can't close/reopen the shared self.f in pytest-run-parallel. + """ + def test_float_fails_without_options(self): + """ Ensure that a scale factor is required for scaleoffset compression of floating point data """ + + with self.assertRaises(ValueError): + dset = self.f.create_dataset(make_name(), (20, 30), dtype=float, scaleoffset=True) + + def test_non_integer(self): + """ Check when scaleoffset is negetive""" + + with self.assertRaises(ValueError): + dset = self.f.create_dataset(make_name(), (20, 30), dtype=float, scaleoffset=-0.1) + + def test_unsupport_dtype(self): + """ Check when dtype is unsupported type""" + + with self.assertRaises(TypeError): + dset = self.f.create_dataset(make_name(), (20, 30), dtype=bool, scaleoffset=True) + + def test_float(self): + """ Scaleoffset filter works for floating point data """ + + scalefac = 4 + shape = (100, 300) + range = 20 * 10 ** scalefac + testdata = (np.random.rand(*shape) - 0.5) * range + fname = self.mktemp() + + with h5py.File(fname, 'w') as f: + dset = f.create_dataset( + 'foo', shape, dtype=np.float64, scaleoffset=scalefac + ) + # Dataset reports that scaleoffset is in use + assert dset.scaleoffset is not None + # Dataset round-trips + dset[...] = testdata + + with h5py.File(fname, 'r') as f: + readdata = f['foo'][...] + + # Test that data round-trips to requested precision + self.assertArrayEqual(readdata, testdata, precision=10 ** (-scalefac)) + # Test that the filter is actually active (i.e. compression is lossy) + assert not (readdata == testdata).all() + + def test_int(self): + """ Scaleoffset filter works for integer data with default precision """ + + nbits = 12 + shape = (100, 300) + testdata = np.random.randint(0, 2 ** nbits - 1, size=shape, dtype=np.int64) + fname = self.mktemp() + + with h5py.File(fname, 'w') as f: + # Create dataset; note omission of nbits (for library-determined precision) + dset = f.create_dataset('foo', shape, dtype=np.int64, scaleoffset=True) + # Dataset reports scaleoffset enabled + assert dset.scaleoffset is not None + # Data round-trips correctly and identically + dset[...] = testdata + + with h5py.File(fname, 'r') as f: + readdata = f['foo'][...] + + self.assertArrayEqual(readdata, testdata) + + def test_int_with_minbits(self): + """ Scaleoffset filter works for integer data with specified precision """ + + nbits = 12 + shape = (100, 300) + testdata = np.random.randint(0, 2 ** nbits, size=shape, dtype=np.int64) + fname = self.mktemp() + + with h5py.File(fname, 'w') as f: + dset = f.create_dataset('foo', shape, dtype=np.int64, scaleoffset=nbits) + # Dataset reports scaleoffset enabled with correct precision + self.assertTrue(dset.scaleoffset == 12) + # Data round-trips correctly + dset[...] = testdata + + with h5py.File(fname, 'r') as f: + readdata = f['foo'][...] + + self.assertArrayEqual(readdata, testdata) + + def test_int_with_minbits_lossy(self): + """ Scaleoffset filter works for integer data with specified precision """ + + nbits = 12 + shape = (100, 300) + testdata = np.random.randint(0, 2 ** (nbits + 1) - 1, size=shape, dtype=np.int64) + fname = self.mktemp() + + with h5py.File(fname, 'w') as f: + dset = f.create_dataset('foo', shape, dtype=np.int64, scaleoffset=nbits) + # Dataset reports scaleoffset enabled with correct precision + self.assertTrue(dset.scaleoffset == 12) + # Data can be written and read + dset[...] = testdata + + with h5py.File(fname, 'r') as f: + readdata = f['foo'][...] + + # Compression is lossy + assert not (readdata == testdata).all() + + +class TestExternal(BaseDataset): + """ + Feature: Datasets with the external storage property + """ + def test_contents(self): + """ Create and access an external dataset """ + + shape = (6, 100) + testdata = np.random.random(shape) + + # create a dataset in an external file and set it + ext_file = self.mktemp() + external = [(ext_file, 0, h5f.UNLIMITED)] + # ${ORIGIN} should be replaced by the parent dir of the HDF5 file + dset = self.f.create_dataset(make_name(), shape, dtype=testdata.dtype, external=external, efile_prefix="${ORIGIN}") + dset[...] = testdata + + assert dset.external is not None + + # verify file's existence, size, and contents + with open(ext_file, 'rb') as fid: + contents = fid.read() + assert contents == testdata.tobytes() + + efile_prefix = pathlib.Path(dset.id.get_access_plist().get_efile_prefix().decode()).as_posix() + parent = pathlib.Path(self.f.filename).parent.as_posix() + assert efile_prefix == parent + + def test_contents_efile_prefix(self): + """ Create and access an external dataset using an efile_prefix""" + name = make_name() + shape = (6, 100) + testdata = np.random.random(shape) + + # create a dataset in an external file and set it + ext_file = self.mktemp() + # set only the basename, let the efile_prefix do the rest + external = [(os.path.basename(ext_file), 0, h5f.UNLIMITED)] + dset = self.f.create_dataset(name, shape, dtype=testdata.dtype, external=external, efile_prefix=os.path.dirname(ext_file)) + dset[...] = testdata + + assert dset.external is not None + + # verify file's existence, size, and contents + with open(ext_file, 'rb') as fid: + contents = fid.read() + assert contents == testdata.tobytes() + + # check efile_prefix + efile_prefix = pathlib.Path(dset.id.get_access_plist().get_efile_prefix().decode()).as_posix() + parent = pathlib.Path(ext_file).parent.as_posix() + assert efile_prefix == parent + + dset2 = self.f.require_dataset(name, shape, testdata.dtype, efile_prefix=os.path.dirname(ext_file)) + assert dset2.external is not None + dset2[()] == testdata + + def test_name_str(self): + """ External argument may be a file name str only """ + + self.f.create_dataset(make_name(), (6, 100), external=self.mktemp(), dtype='f4') + + def test_name_path(self): + """ External argument may be a file name path only """ + + self.f.create_dataset(make_name(), (6, 100), 'f4', + external=pathlib.Path(self.mktemp())) + + def test_iter_multi(self): + """ External argument may be an iterable of multiple tuples """ + + ext_file = self.mktemp() + N = 100 + external = iter((ext_file, x * 1000, 1000) for x in range(N)) + dset = self.f.create_dataset(make_name(), (6, 100), external=external, dtype='f4') + assert len(dset.external) == N + + def test_invalid(self): + """ Test with invalid external lists """ + + shape = (6, 100) + ext_file = self.mktemp() + + for exc_type, external in [ + (TypeError, [ext_file]), + (TypeError, [ext_file, 0]), + (TypeError, [ext_file, 0, h5f.UNLIMITED]), + (ValueError, [(ext_file,)]), + (ValueError, [(ext_file, 0)]), + (ValueError, [(ext_file, 0, h5f.UNLIMITED, 0)]), + (TypeError, [(ext_file, 0, "h5f.UNLIMITED")]), + ]: + with self.assertRaises(exc_type): + self.f.create_dataset(make_name(), shape, external=external, dtype='f4') + + def test_create_expandable(self): + """ Create expandable external dataset """ + + ext_file = self.mktemp() + shape = (128, 64) + maxshape = (None, 64) + exp_dset = self.f.create_dataset( + make_name(), shape, 'f4', maxshape=maxshape, external=ext_file + ) + assert exp_dset.chunks is None + assert exp_dset.shape == shape + assert exp_dset.maxshape == maxshape + + +class TestAutoCreate(BaseDataset): + + """ + Feature: Datasets auto-created from data produce the correct types + """ + def assert_string_type(self, ds, cset, variable=True): + tid = ds.id.get_type() + self.assertEqual(type(tid), h5py.h5t.TypeStringID) + self.assertEqual(tid.get_cset(), cset) + if variable: + assert tid.is_variable_str() + + def test_vlen_bytes(self): + """Assigning byte strings produces a vlen string ASCII dataset """ + x = make_name("x") + y = make_name("y") + z = make_name("z") + + self.f[x] = b"Hello there" + self.assert_string_type(self.f[x], h5py.h5t.CSET_ASCII) + + self.f[y] = [b"a", b"bc"] + self.assert_string_type(self.f[y], h5py.h5t.CSET_ASCII) + + self.f[z] = np.array([b"a", b"bc"], dtype=np.object_) + self.assert_string_type(self.f[z], h5py.h5t.CSET_ASCII) + + def test_vlen_unicode(self): + """Assigning unicode strings produces a vlen string UTF-8 dataset """ + x = make_name("x") + y = make_name("y") + z = make_name("z") + + self.f[x] = "Hello there" + chr(0x2034) + self.assert_string_type(self.f[x], h5py.h5t.CSET_UTF8) + + self.f[y] = ["a", "bc"] + self.assert_string_type(self.f[y], h5py.h5t.CSET_UTF8) + + # 2D array; this only works with an array, not nested lists + self.f[z] = np.array([["a", "bc"]], dtype=np.object_) + self.assert_string_type(self.f[z], h5py.h5t.CSET_UTF8) + + def test_string_fixed(self): + """ Assignment of fixed-length byte string produces a fixed-length + ascii dataset """ + name = make_name() + self.f[name] = np.bytes_("Hello there") + ds = self.f[name] + self.assert_string_type(ds, h5py.h5t.CSET_ASCII, variable=False) + self.assertEqual(ds.id.get_type().get_size(), 11) + + +class TestCreateLike(BaseDataset): + def test_no_chunks(self): + x = make_name("x") + y = make_name("y") + + self.f[x] = np.arange(25).reshape(5, 5) + self.f.create_dataset_like(y, self.f[x]) + dslike = self.f[y] + self.assertEqual(dslike.shape, (5, 5)) + self.assertIs(dslike.chunks, None) + + def test_track_times(self): + x = make_name("x") + y = make_name("y") + z = make_name("z") + w = make_name("w") + + orig = self.f.create_dataset(x, data=np.arange(12), + track_times=True) + self.assertNotEqual(0, h5py.h5g.get_objinfo(orig._id).mtime) + similar = self.f.create_dataset_like(y, orig) + self.assertNotEqual(0, h5py.h5g.get_objinfo(similar._id).mtime) + + orig = self.f.create_dataset(z, data=np.arange(12), + track_times=False) + self.assertEqual(0, h5py.h5g.get_objinfo(orig._id).mtime) + similar = self.f.create_dataset_like(w, orig) + self.assertEqual(0, h5py.h5g.get_objinfo(similar._id).mtime) + + def test_maxshape(self): + """ Test when other.maxshape != other.shape """ + + other = self.f.create_dataset(make_name("x"), (10,), maxshape=20, dtype='f4') + similar = self.f.create_dataset_like(make_name("y"), other) + self.assertEqual(similar.shape, (10,)) + self.assertEqual(similar.maxshape, (20,)) + +class TestChunkIterator(BaseDataset): + def test_no_chunks(self): + dset = self.f.create_dataset(make_name(), (), dtype='f4') + with self.assertRaises(TypeError): + dset.iter_chunks() + + def test_rank_mismatch(self): + dset = self.f.create_dataset(make_name(), shape=(100,), chunks=(32,), dtype='f4') + with self.assertRaises(ValueError): + dset.iter_chunks((slice(19,67), 9)) + + def test_1d(self): + dset = self.f.create_dataset(make_name(), (100,), chunks=(32,), dtype='f4') + expected = ((slice(0,32,1),), (slice(32,64,1),), (slice(64,96,1),), + (slice(96,100,1),)) + self.assertEqual(list(dset.iter_chunks()), list(expected)) + expected = ((slice(50,64,1),), (slice(64,96,1),), (slice(96,97,1),)) + self.assertEqual(list(dset.iter_chunks(np.s_[50:97])), list(expected)) + expected = ((slice(0,32,1),), (slice(32,50,1),)) + self.assertEqual(list(dset.iter_chunks(np.s_[:50])), list(expected)) + expected = ((slice(96,97,1),),) + self.assertEqual(list(dset.iter_chunks(np.s_[96])), list(expected)) + + def test_2d(self): + dset = self.f.create_dataset(make_name(), (100,100), chunks=(32,64), dtype='f4') + expected = ( + (slice(0, 32, 1), slice(0, 64, 1)), + (slice(0, 32, 1), slice(64, 100, 1)), + (slice(32, 64, 1), slice(0, 64, 1)), + (slice(32, 64, 1), slice(64, 100, 1)), + (slice(64, 96, 1), slice(0, 64, 1)), + (slice(64, 96, 1), slice(64, 100, 1)), + (slice(96, 100, 1), slice(0, 64, 1)), + (slice(96, 100, 1), slice(64, 100, 1)), + ) + self.assertEqual(list(dset.iter_chunks()), list(expected)) + expected = ((slice(48, 52, 1), slice(40, 50, 1)),) + self.assertEqual(list(dset.iter_chunks(np.s_[48:52,40:50])), list(expected)) + expected = ( + (slice(0, 32, 1), slice(40, 64, 1)), + (slice(0, 32, 1), slice(64, 100, 1)), + (slice(32, 52, 1), slice(40, 64, 1)), + (slice(32, 52, 1), slice(64, 100, 1)), + ) + self.assertEqual(list(dset.iter_chunks(np.s_[:52,40:])), list(expected)) + expected = ( + (slice(96, 97, 1), slice(19, 64, 1)), + (slice(96, 97, 1), slice(64, 67, 1)), + ) + self.assertEqual(list(dset.iter_chunks(np.s_[96,19:67])), list(expected)) + + def test_2d_partial_slice(self): + dset = self.f.create_dataset(make_name(), (5,5), chunks=(2,2), dtype='f4') + expected = ((slice(3, 4, 1), slice(3, 4, 1)), + (slice(3, 4, 1), slice(4, 5, 1)), + (slice(4, 5, 1), slice(3, 4, 1)), + (slice(4, 5, 1), slice(4, 5, 1))) + sel = slice(3,5) + self.assertEqual(list(dset.iter_chunks((sel, sel))), list(expected)) + + + +class TestResize(BaseDataset): + + """ + Feature: Datasets created with "maxshape" may be resized + """ + + def test_create(self): + """ Create dataset with "maxshape" """ + dset = self.f.create_dataset(make_name(), (20, 30), maxshape=(20, 60), dtype='f4') + self.assertIsNot(dset.chunks, None) + self.assertEqual(dset.maxshape, (20, 60)) + + def test_create_1D_integer_maxshape_tuple(self): + """ Create dataset with "maxshape" using tuple shape and integer maxshape""" + dset = self.f.create_dataset(make_name(), (20,), maxshape=20, dtype='f4') + self.assertIsNot(dset.chunks, None) + self.assertEqual(dset.maxshape, (20,)) + + def test_create_1D_integer_maxshape_integer(self): + """ Create dataset with "maxshape" using integer shape and integer maxshape""" + dset = self.f.create_dataset(make_name(), 20, maxshape=20, dtype='f4') + self.assertEqual(dset.maxshape, (20,)) + + def test_resize(self): + """ Datasets may be resized up to maxshape """ + dset = self.f.create_dataset(make_name(), (20, 30), maxshape=(20, 60), dtype='f4') + self.assertEqual(dset.shape, (20, 30)) + dset.resize((20, 50)) + self.assertEqual(dset.shape, (20, 50)) + dset.resize((20, 60)) + self.assertEqual(dset.shape, (20, 60)) + + def test_resize_1D(self): + """ Datasets may be resized up to maxshape using integer maxshape""" + dset = self.f.create_dataset(make_name(), 20, maxshape=40, dtype='f4') + self.assertEqual(dset.shape, (20,)) + dset.resize((30,)) + self.assertEqual(dset.shape, (30,)) + + def test_resize_over(self): + """ Resizing past maxshape triggers an exception """ + dset = self.f.create_dataset(make_name(), (20, 30), maxshape=(20, 60), dtype='f4') + with self.assertRaises(Exception): + dset.resize((20, 70)) + + def test_resize_nonchunked(self): + """ Resizing non-chunked dataset raises TypeError """ + dset = self.f.create_dataset(make_name(), (20, 30), dtype='f4') + with self.assertRaises(TypeError): + dset.resize((20, 60)) + + def test_resize_axis(self): + """ Resize specified axis """ + dset = self.f.create_dataset(make_name(), (20, 30), maxshape=(20, 60), dtype='f4') + dset.resize(50, axis=1) + self.assertEqual(dset.shape, (20, 50)) + + def test_axis_exc(self): + """ Illegal axis raises ValueError """ + dset = self.f.create_dataset(make_name(), (20, 30), maxshape=(20, 60), dtype='f4') + with self.assertRaises(ValueError): + dset.resize(50, axis=2) + + def test_zero_dim(self): + """ Allow zero-length initial dims for unlimited axes (issue 111) """ + dset = self.f.create_dataset(make_name(), (15, 0), maxshape=(15, None), dtype='f4') + self.assertEqual(dset.shape, (15, 0)) + self.assertEqual(dset.maxshape, (15, None)) + + +class TestDtype(BaseDataset): + + """ + Feature: Dataset dtype is available as .dtype property + """ + + def test_dtype(self): + """ Retrieve dtype from dataset """ + dset = self.f.create_dataset(make_name(), (5,), '|S10') + self.assertEqual(dset.dtype, np.dtype('|S10')) + + def test_dtype_complex32(self): + """ Retrieve dtype from complex float16 dataset (gh-2156) """ + # No native support in numpy as of v1.23.3, so expect compound type. + complex32 = np.dtype([('r', np.float16), ('i', np.float16)]) + dset = self.f.create_dataset(make_name(), (5,), complex32) + self.assertEqual(dset.dtype, complex32) + + +class TestLen(BaseDataset): + + """ + Feature: Size of first axis is available via Python's len + """ + + def test_len(self): + """ Python len() (under 32 bits) """ + dset = self.f.create_dataset(make_name(), (312, 15), dtype='f4') + self.assertEqual(len(dset), 312) + + def test_len_big(self): + """ Python len() vs Dataset.len() """ + dset = self.f.create_dataset(make_name(), (2 ** 33, 15), dtype='f4') + self.assertEqual(dset.shape, (2 ** 33, 15)) + if sys.maxsize == 2 ** 31 - 1: + with self.assertRaises(OverflowError): + len(dset) + else: + self.assertEqual(len(dset), 2 ** 33) + self.assertEqual(dset.len(), 2 ** 33) + + +class TestIter(BaseDataset): + + """ + Feature: Iterating over a dataset yields rows + """ + + def test_iter(self): + """ Iterating over a dataset yields rows """ + data = np.arange(30, dtype='f').reshape((10, 3)) + dset = self.f.create_dataset(make_name(), data=data) + for x, y in zip(dset, data, strict=True): + self.assertEqual(len(x), 3) + self.assertArrayEqual(x, y) + + def test_iter_scalar(self): + """ Iterating over scalar dataset raises TypeError """ + dset = self.f.create_dataset(make_name(), shape=(), dtype='f4') + with self.assertRaises(TypeError): + [x for x in dset] + + +class TestStrings(BaseDataset): + + """ + Feature: Datasets created with vlen and fixed datatypes correctly + translate to and from HDF5 + """ + + def test_vlen_bytes(self): + """ Vlen bytes dataset maps to vlen ascii in the file """ + dt = h5py.string_dtype(encoding='ascii') + ds = self.f.create_dataset(make_name(), (100,), dtype=dt) + tid = ds.id.get_type() + self.assertEqual(type(tid), h5py.h5t.TypeStringID) + self.assertEqual(tid.get_cset(), h5py.h5t.CSET_ASCII) + string_info = h5py.check_string_dtype(ds.dtype) + self.assertEqual(string_info.encoding, 'ascii') + + def test_vlen_bytes_fillvalue(self): + """ Vlen bytes dataset handles fillvalue """ + dt = h5py.string_dtype(encoding='ascii') + fill_value = b'bar' + ds = self.f.create_dataset(make_name(), (100,), dtype=dt, fillvalue=fill_value) + self.assertEqual(ds[0], fill_value) + self.assertEqual(ds.asstr()[0], fill_value.decode()) + self.assertEqual(ds.fillvalue, fill_value) + + def test_vlen_unicode(self): + """ Vlen unicode dataset maps to vlen utf-8 in the file """ + dt = h5py.string_dtype() + ds = self.f.create_dataset(make_name(), (100,), dtype=dt) + tid = ds.id.get_type() + self.assertEqual(type(tid), h5py.h5t.TypeStringID) + self.assertEqual(tid.get_cset(), h5py.h5t.CSET_UTF8) + string_info = h5py.check_string_dtype(ds.dtype) + self.assertEqual(string_info.encoding, 'utf-8') + + def test_vlen_unicode_fillvalue(self): + """ Vlen unicode dataset handles fillvalue """ + dt = h5py.string_dtype() + fill_value = 'bár' + ds = self.f.create_dataset(make_name(), (100,), dtype=dt, fillvalue=fill_value) + self.assertEqual(ds[0], fill_value.encode("utf-8")) + self.assertEqual(ds.asstr()[0], fill_value) + self.assertEqual(ds.fillvalue, fill_value.encode("utf-8")) + + def test_fixed_ascii(self): + """ Fixed-length bytes dataset maps to fixed-length ascii in the file + """ + dt = np.dtype("|S10") + ds = self.f.create_dataset(make_name(), (100,), dtype=dt) + tid = ds.id.get_type() + self.assertEqual(type(tid), h5py.h5t.TypeStringID) + self.assertFalse(tid.is_variable_str()) + self.assertEqual(tid.get_size(), 10) + self.assertEqual(tid.get_cset(), h5py.h5t.CSET_ASCII) + string_info = h5py.check_string_dtype(ds.dtype) + self.assertEqual(string_info.encoding, 'ascii') + self.assertEqual(string_info.length, 10) + + def test_fixed_bytes_fillvalue(self): + """ Vlen bytes dataset handles fillvalue """ + dt = h5py.string_dtype(encoding='ascii', length=10) + fill_value = b'bar' + ds = self.f.create_dataset(make_name(), (100,), dtype=dt, fillvalue=fill_value) + self.assertEqual(ds[0], fill_value) + self.assertEqual(ds.asstr()[0], fill_value.decode()) + self.assertEqual(ds.fillvalue, fill_value) + + def test_fixed_utf8(self): + dt = h5py.string_dtype(encoding='utf-8', length=5) + ds = self.f.create_dataset(make_name(), (100,), dtype=dt) + tid = ds.id.get_type() + self.assertEqual(tid.get_cset(), h5py.h5t.CSET_UTF8) + s = 'cù' + ds[0] = s.encode('utf-8') + ds[1] = s + ds[2:4] = [s, s] + ds[4:6] = np.array([s, s], dtype=object) + ds[6:8] = np.array([s.encode('utf-8')] * 2, dtype=dt) + with self.assertRaises(TypeError): + ds[8:10] = np.array([s, s], dtype='U') + + np.testing.assert_array_equal(ds[:8], np.array([s.encode('utf-8')] * 8, dtype='S')) + + def test_fixed_utf_8_fillvalue(self): + """ Vlen unicode dataset handles fillvalue """ + dt = h5py.string_dtype(encoding='utf-8', length=10) + fill_value = 'bár'.encode("utf-8") + ds = self.f.create_dataset(make_name(), (100,), dtype=dt, fillvalue=fill_value) + self.assertEqual(ds[0], fill_value) + self.assertEqual(ds.asstr()[0], fill_value.decode("utf-8")) + self.assertEqual(ds.fillvalue, fill_value) + + def test_fixed_unicode(self): + """ Fixed-length unicode datasets are unsupported (raise TypeError) """ + dt = np.dtype("|U10") + with self.assertRaises(TypeError): + ds = self.f.create_dataset(make_name(), (100,), dtype=dt) + + def test_roundtrip_vlen_bytes(self): + """ writing and reading to vlen bytes dataset preserves type and content + """ + dt = h5py.string_dtype(encoding='ascii') + ds = self.f.create_dataset(make_name(), (100,), dtype=dt) + data = b"Hello\xef" + ds[0] = data + out = ds[0] + self.assertEqual(type(out), bytes) + self.assertEqual(out, data) + + def test_roundtrip_fixed_bytes(self): + """ Writing to and reading from fixed-length bytes dataset preserves + type and content """ + dt = np.dtype("|S10") + ds = self.f.create_dataset(make_name(), (100,), dtype=dt) + data = b"Hello\xef" + ds[0] = data + out = ds[0] + self.assertEqual(type(out), np.bytes_) + self.assertEqual(out, data) + + def test_retrieve_vlen_unicode(self): + dt = h5py.string_dtype() + ds = self.f.create_dataset(make_name(), (10,), dtype=dt) + data = "fàilte" + ds[0] = data + self.assertIsInstance(ds[0], bytes) + out = ds.asstr()[0] + self.assertIsInstance(out, str) + self.assertEqual(out, data) + + def test_asstr(self): + ds = self.f.create_dataset(make_name(), (10,), dtype=h5py.string_dtype()) + data = "fàilte" + ds[0] = data + + strwrap1 = ds.asstr('ascii') + with self.assertRaises(UnicodeDecodeError): + strwrap1[0] + + # Different errors parameter + self.assertEqual(ds.asstr('ascii', 'ignore')[0], 'filte') + + # latin-1 will decode it but give the wrong text + self.assertNotEqual(ds.asstr('latin-1')[0], data) + + # len of ds + self.assertEqual(10, len(ds.asstr())) + + # Array output + np.testing.assert_array_equal( + ds.asstr()[:1], np.array([data], dtype=object) + ) + + np.testing.assert_array_equal( + np.asarray(ds.asstr())[:1], np.array([data], dtype=object) + ) + + def test_asstr_fixed(self): + dt = h5py.string_dtype(length=5) + ds = self.f.create_dataset(make_name(), (10,), dtype=dt) + data = 'cù' + ds[0] = np.array(data.encode('utf-8'), dtype=dt) + + self.assertIsInstance(ds[0], np.bytes_) + out = ds.asstr()[0] + self.assertIsInstance(out, str) + self.assertEqual(out, data) + + # Different errors parameter + self.assertEqual(ds.asstr('ascii', 'ignore')[0], 'c') + + # latin-1 will decode it but give the wrong text + self.assertNotEqual(ds.asstr('latin-1')[0], data) + + # Array output + np.testing.assert_array_equal( + ds.asstr()[:1], np.array([data], dtype=object) + ) + + def test_unicode_write_error(self): + """Encoding error when writing a non-ASCII string to an ASCII vlen dataset""" + dt = h5py.string_dtype('ascii') + ds = self.f.create_dataset(make_name(), (100,), dtype=dt) + data = "fàilte" + with self.assertRaises(UnicodeEncodeError): + ds[0] = data + + def test_unicode_write_bytes(self): + """ Writing valid utf-8 byte strings to a unicode vlen dataset is OK + """ + dt = h5py.string_dtype() + ds = self.f.create_dataset(make_name(), (100,), dtype=dt) + data = (u"Hello there" + chr(0x2034)).encode('utf8') + ds[0] = data + out = ds[0] + self.assertEqual(type(out), bytes) + self.assertEqual(out, data) + + def test_vlen_bytes_write_ascii_str(self): + """ Writing an ascii str to ascii vlen dataset is OK + """ + dt = h5py.string_dtype('ascii') + ds = self.f.create_dataset(make_name(), (100,), dtype=dt) + data = "ASCII string" + ds[0] = data + out = ds[0] + self.assertEqual(type(out), bytes) + self.assertEqual(out, data.encode('ascii')) + + +class TestCompound(BaseDataset): + + """ + Feature: Compound types correctly round-trip + """ + + def test_rt(self): + """ Compound types are read back in correct order (issue 236)""" + + dt = np.dtype([ ('weight', np.float64), + ('cputime', np.float64), + ('walltime', np.float64), + ('parents_offset', np.uint32), + ('n_parents', np.uint32), + ('status', np.uint8), + ('endpoint_type', np.uint8), ]) + + testdata = np.ndarray((16,), dtype=dt) + for key in dt.fields: + testdata[key] = np.random.random((16,)) * 100 + + name = make_name() + self.f[name] = testdata + outdata = self.f[name][...] + self.assertTrue(np.all(outdata == testdata)) + self.assertEqual(outdata.dtype, testdata.dtype) + + def test_assign(self): + dt = np.dtype([ ('weight', (np.float64, 3)), + ('endpoint_type', np.uint8), ]) + + testdata = np.ndarray((16,), dtype=dt) + for key in dt.fields: + testdata[key] = np.random.random(size=testdata[key].shape) * 100 + + name = make_name() + ds = self.f.create_dataset(name, (16,), dtype=dt) + for key in dt.fields: + ds[key] = testdata[key] + + outdata = self.f[name][...] + + self.assertTrue(np.all(outdata == testdata)) + self.assertEqual(outdata.dtype, testdata.dtype) + + def test_fields(self): + dt = np.dtype([ + ('x', np.float64), + ('y', np.float64), + ('z', np.float64), + ]) + + testdata = np.ndarray((16,), dtype=dt) + for key in dt.fields: + testdata[key] = np.random.random((16,)) * 100 + + name = make_name() + self.f[name] = testdata + ds = self.f[name] + + # Extract multiple fields + np.testing.assert_array_equal( + ds.fields(['x', 'y'])[:], testdata[['x', 'y']] + ) + # Extract single field + np.testing.assert_array_equal( + ds.fields('x')[:], testdata['x'] + ) + # Check __array__() method of fields wrapper + np.testing.assert_array_equal( + np.asarray(ds.fields(['x', 'y'])), testdata[['x', 'y']] + ) + # Check type conversion of __array__() method + dt_int = np.dtype([('x', np.int32)]) + np.testing.assert_array_equal( + np.asarray(ds.fields(['x']), dtype=dt_int), + testdata[['x']].astype(dt_int) + ) + + # Check len() on fields wrapper + assert len(ds.fields('x')) == 16 + + def test_nested_compound_vlen(self): + dt_inner = np.dtype([('a', h5py.vlen_dtype(np.int32)), + ('b', h5py.vlen_dtype(np.int32))]) + + dt = np.dtype([('f1', h5py.vlen_dtype(dt_inner)), + ('f2', np.int64)]) + + inner1 = (np.array(range(1, 3), dtype=np.int32), + np.array(range(6, 9), dtype=np.int32)) + + inner2 = (np.array(range(10, 14), dtype=np.int32), + np.array(range(16, 21), dtype=np.int32)) + + data = np.array([(np.array([inner1, inner2], dtype=dt_inner), 2), + (np.array([inner1], dtype=dt_inner), 3)], + dtype=dt) + + name = make_name() + self.f[name] = data + out = self.f[name] + + # Specifying check_alignment=False because vlen fields have 8 bytes of padding + # because the vlen datatype in hdf5 occupies 16 bytes + self.assertArrayEqual(out, data, check_alignment=False) + + +class TestSubarray(BaseDataset): + def test_write_list(self): + ds = self.f.create_dataset(make_name(), (1,), dtype="3int8") + ds[0] = [1, 2, 3] + np.testing.assert_array_equal(ds[:], [[1, 2, 3]]) + + ds[:] = [[4, 5, 6]] + np.testing.assert_array_equal(ds[:], [[4, 5, 6]]) + + def test_write_array(self): + ds = self.f.create_dataset(make_name(), (1,), dtype="3int8") + ds[0] = np.array([1, 2, 3]) + np.testing.assert_array_equal(ds[:], [[1, 2, 3]]) + + ds[:] = np.array([[4, 5, 6]]) + np.testing.assert_array_equal(ds[:], [[4, 5, 6]]) + + +class TestEnum(BaseDataset): + + """ + Feature: Enum datatype info is preserved, read/write as integer + """ + + EDICT = {'RED': 0, 'GREEN': 1, 'BLUE': 42} + + def test_create(self): + """ Enum datasets can be created and type correctly round-trips """ + dt = h5py.enum_dtype(self.EDICT, basetype='i') + ds = self.f.create_dataset(make_name(), (100, 100), dtype=dt) + dt2 = ds.dtype + dict2 = h5py.check_enum_dtype(dt2) + self.assertEqual(dict2, self.EDICT) + + def test_readwrite(self): + """ Enum datasets can be read/written as integers """ + dt = h5py.enum_dtype(self.EDICT, basetype='i4') + ds = self.f.create_dataset(make_name(), (100, 100), dtype=dt) + ds[35, 37] = 42 + ds[1, :] = 1 + self.assertEqual(ds[35, 37], 42) + self.assertArrayEqual(ds[1, :], np.array((1,) * 100, dtype='i4')) + + +class TestFloats(BaseDataset): + + """ + Test support for mini and extended-precision floats + """ + + def _exectest(self, dt): + dset = self.f.create_dataset(make_name(), (100,), dtype=dt) + self.assertEqual(dset.dtype, dt) + data = np.ones((100,), dtype=dt) + dset[...] = data + self.assertArrayEqual(dset[...], data) + + @ut.skipUnless(hasattr(np, 'float16'), "NumPy float16 support required") + def test_mini(self): + """ Mini-floats round trip """ + self._exectest(np.dtype('float16')) + + # TODO: move these tests to test_h5t + def test_mini_mapping(self): + """ Test mapping for float16 """ + if hasattr(np, 'float16'): + self.assertEqual(h5t.IEEE_F16LE.dtype, np.dtype('<f2')) + else: + self.assertEqual(h5t.IEEE_F16LE.dtype, np.dtype('<f4')) + + +class TestTrackTimes(BaseDataset): + + """ + Feature: track_times + """ + + def test_disable_track_times(self): + """ check that when track_times=False, the time stamp=0 (Jan 1, 1970) """ + ds = self.f.create_dataset(make_name(), (4,), track_times=False, dtype='f4') + ds_mtime = h5py.h5g.get_objinfo(ds._id).mtime + self.assertEqual(0, ds_mtime) + + def test_invalid_track_times(self): + """ check that when give track_times an invalid value """ + with self.assertRaises(TypeError): + self.f.create_dataset(make_name(), (4,), track_times='null', dtype='f4') + + +class TestZeroShape(BaseDataset): + + """ + Features of datasets with (0,)-shape axes + """ + + def test_array_conversion(self): + """ Empty datasets can be converted to NumPy arrays """ + ds = self.f.create_dataset(make_name("x"), 0, maxshape=None, dtype='f4') + self.assertEqual(ds.shape, np.array(ds).shape) + + ds = self.f.create_dataset(make_name("y"), (0,), maxshape=(None,), dtype='f4') + self.assertEqual(ds.shape, np.array(ds).shape) + + ds = self.f.create_dataset(make_name("z"), (0, 0), maxshape=(None, None), dtype='f4') + self.assertEqual(ds.shape, np.array(ds).shape) + + def test_reading(self): + """ Slicing into empty datasets works correctly """ + dt = [('a', 'f'), ('b', 'i')] + ds = self.f.create_dataset(make_name(), (0,), dtype=dt, maxshape=(None,)) + arr = np.empty((0,), dtype=dt) + + self.assertEqual(ds[...].shape, arr.shape) + self.assertEqual(ds[...].dtype, arr.dtype) + self.assertEqual(ds[()].shape, arr.shape) + self.assertEqual(ds[()].dtype, arr.dtype) + + +class TestRegionRefs(BaseDataset): + + """ + Various features of region references + """ + + def setUp(self): + BaseDataset.setUp(self) + self.data = np.arange(100 * 100).reshape((100, 100)) + self.dset = self.f.create_dataset('x', data=self.data) + self.dset[...] = self.data + + def test_create_ref(self): + """ Region references can be used as slicing arguments """ + slic = np.s_[25:35, 10:100:5] + ref = self.dset.regionref[slic] + self.assertArrayEqual(self.dset[ref], self.data[slic]) + + def test_empty_region(self): + ref = self.dset.regionref[:0] + out = self.dset[ref] + assert out.size == 0 + # Ideally we should preserve shape (0, 100), but it seems this is lost. + + def test_scalar_dataset(self): + ds = self.f.create_dataset(make_name(), data=1.0, dtype='f4') + sid = h5py.h5s.create(h5py.h5s.SCALAR) + + # Deselected + sid.select_none() + ref = h5py.h5r.create(ds.id, b'.', h5py.h5r.DATASET_REGION, sid) + assert ds[ref] == h5py.Empty(np.dtype('f4')) + + # Selected + sid.select_all() + ref = h5py.h5r.create(ds.id, b'.', h5py.h5r.DATASET_REGION, sid) + assert ds[ref] == ds[()] + + def test_ref_shape(self): + """ Region reference shape and selection shape """ + slic = np.s_[25:35, 10:100:5] + ref = self.dset.regionref[slic] + self.assertEqual(self.dset.regionref.shape(ref), self.dset.shape) + self.assertEqual(self.dset.regionref.selection(ref), (10, 18)) + + +class TestAstype(BaseDataset): + """.astype() wrapper & context manager + """ + def test_astype_wrapper(self): + dset = self.f.create_dataset(make_name(), (100,), dtype='i2') + dset[...] = np.arange(100) + arr = dset.astype('f4')[:] + self.assertArrayEqual(arr, np.arange(100, dtype='f4')) + + + def test_astype_wrapper_len(self): + dset = self.f.create_dataset(make_name(), (100,), dtype='i2') + dset[...] = np.arange(100) + self.assertEqual(100, len(dset.astype('f4'))) + + def test_astype_wrapper_asarray(self): + dset = self.f.create_dataset(make_name(), (100,), dtype='i2') + dset[...] = np.arange(100) + arr = np.asarray(dset.astype('f4'), dtype='i2') + self.assertArrayEqual(arr, np.arange(100, dtype='i2')) + + +class TestScalarCompound(BaseDataset): + + """ + Retrieval of a single field from a scalar compound dataset should + strip the field info + """ + + def test_scalar_compound(self): + + dt = np.dtype([('a', 'i')]) + dset = self.f.create_dataset(make_name(), (), dtype=dt) + self.assertEqual(dset['a'].dtype, np.dtype('i')) + + +class TestVlen(BaseDataset): + def test_int(self): + dt = h5py.vlen_dtype(int) + ds = self.f.create_dataset(make_name(), (4,), dtype=dt) + ds[0] = np.arange(3) + ds[1] = np.arange(0) + ds[2] = [1, 2, 3] + ds[3] = np.arange(1) + self.assertArrayEqual(ds[0], np.arange(3)) + self.assertArrayEqual(ds[1], np.arange(0)) + self.assertArrayEqual(ds[2], np.array([1, 2, 3])) + self.assertArrayEqual(ds[1], np.arange(0)) + ds[0:2] = np.array([np.arange(5), np.arange(4)], dtype=object) + self.assertArrayEqual(ds[0], np.arange(5)) + self.assertArrayEqual(ds[1], np.arange(4)) + ds[0:2] = np.array([np.arange(3), np.arange(3)]) + self.assertArrayEqual(ds[0], np.arange(3)) + self.assertArrayEqual(ds[1], np.arange(3)) + + def test_reuse_from_other(self): + dt = h5py.vlen_dtype(int) + ds = self.f.create_dataset(make_name("x"), (1,), dtype=dt) + self.f.create_dataset(make_name("y"), (1,), ds[()].dtype) + + def test_reuse_struct_from_other(self): + dt = [('a', int), ('b', h5py.vlen_dtype(int))] + + fname = self.mktemp() + with h5py.File(fname, 'w') as f: + f.create_dataset("x", (1,), dtype=dt) + + with h5py.File(fname, 'a') as f: + f.create_dataset("y", (1,), f["x"]['b'][()].dtype) + + def test_convert(self): + dt = h5py.vlen_dtype(int) + ds = self.f.create_dataset(make_name(), (3,), dtype=dt) + ds[0] = np.array([1.4, 1.2]) + ds[1] = np.array([1.2]) + ds[2] = [1.2, 2, 3] + self.assertArrayEqual(ds[0], np.array([1, 1])) + self.assertArrayEqual(ds[1], np.array([1])) + self.assertArrayEqual(ds[2], np.array([1, 2, 3])) + ds[0:2] = np.array([[0.1, 1.1, 2.1, 3.1, 4], np.arange(4)], dtype=object) + self.assertArrayEqual(ds[0], np.arange(5)) + self.assertArrayEqual(ds[1], np.arange(4)) + ds[0:2] = np.array([np.array([0.1, 1.2, 2.2]), + np.array([0.2, 1.2, 2.2])]) + self.assertArrayEqual(ds[0], np.arange(3)) + self.assertArrayEqual(ds[1], np.arange(3)) + + def test_multidim(self): + dt = h5py.vlen_dtype(int) + ds = self.f.create_dataset(make_name(), (2, 2), dtype=dt) + ds[0, 0] = np.arange(1) + ds[:, :] = np.array([[np.arange(3), np.arange(2)], + [np.arange(1), np.arange(2)]], dtype=object) + ds[:, :] = np.array([[np.arange(2), np.arange(2)], + [np.arange(2), np.arange(2)]]) + + def _help_float_testing(self, np_dt): + """ + Helper for testing various vlen numpy data types. + :param np_dt: Numpy datatype to test + """ + dt = h5py.vlen_dtype(np_dt) + ds = self.f.create_dataset(make_name(), (5,), dtype=dt) + + # Create some arrays, and assign them to the dataset + array_0 = np.array([1., 2., 30.], dtype=np_dt) + array_1 = np.array([100.3, 200.4, 98.1, -10.5, -300.0], dtype=np_dt) + + # Test that a numpy array of different type gets cast correctly + array_2 = np.array([1, 2, 8], dtype=np.dtype('int32')) + casted_array_2 = array_2.astype(np_dt) + + # Test that we can set a list of floats. + list_3 = [1., 2., 900., 0., -0.5] + list_array_3 = np.array(list_3, dtype=np_dt) + + # Test that a list of integers gets casted correctly + list_4 = [-1, -100, 0, 1, 9999, 70] + list_array_4 = np.array(list_4, dtype=np_dt) + + ds[0] = array_0 + ds[1] = array_1 + ds[2] = array_2 + ds[3] = list_3 + ds[4] = list_4 + + self.assertArrayEqual(array_0, ds[0]) + self.assertArrayEqual(array_1, ds[1]) + self.assertArrayEqual(casted_array_2, ds[2]) + self.assertArrayEqual(list_array_3, ds[3]) + self.assertArrayEqual(list_array_4, ds[4]) + + # Test that we can reassign arrays in the dataset + list_array_3 = np.array([0.3, 2.2], dtype=np_dt) + + ds[0] = list_array_3[:] + + self.assertArrayEqual(list_array_3, ds[0]) + + # Make sure we can close the file. + self.f.flush() + if is_main_thread(): + self.f.close() + + def test_numpy_float16(self): + np_dt = np.dtype('float16') + self._help_float_testing(np_dt) + + def test_numpy_float32(self): + np_dt = np.dtype('float32') + self._help_float_testing(np_dt) + + def test_numpy_float64_from_dtype(self): + np_dt = np.dtype('float64') + self._help_float_testing(np_dt) + + def test_numpy_float64_2(self): + np_dt = np.float64 + self._help_float_testing(np_dt) + + def test_non_contiguous_arrays_bool(self): + """Test that non-contiguous arrays are stored correctly""" + name = make_name() + self.f.create_dataset(name, (10,), dtype=h5py.vlen_dtype('bool')) + ds = self.f[name] + x = np.array([True, False, True, True, False, False, False]) + ds[0] = x[::2] + + assert all(ds[0] == x[::2]), f"{ds[0]} != {x[::2]}" + + def test_non_contiguous_arrays_int(self): + name = make_name() + self.f.create_dataset(name, (10,), dtype=h5py.vlen_dtype('int8')) + ds = self.f[name] + y = np.array([2, 4, 1, 5, -1, 3, 7]) + ds[0] = y[::2] + + assert all(ds[0] == y[::2]), f"{ds[0]} != {y[::2]}" + + def test_asstr_array_dtype(self): + dt = h5py.string_dtype(encoding='ascii') + fill_value = b'bar' + ds = self.f.create_dataset(make_name(), (100,), dtype=dt, fillvalue=fill_value) + with pytest.raises(ValueError): + np.array(ds.asstr(), dtype=int) + + +class TestLowOpen(BaseDataset): + + def test_get_access_list(self): + """ Test H5Dget_access_plist """ + ds = self.f.create_dataset(make_name(), (4,), dtype='f4') + p_list = ds.id.get_access_plist() + + def test_dapl(self): + """ Test the dapl keyword to h5d.open """ + name = make_name() + dapl = h5py.h5p.create(h5py.h5p.DATASET_ACCESS) + dset = self.f.create_dataset(name, (100,), dtype='f4') + del dset + dsid = h5py.h5d.open(self.f.id, name.encode('utf-8'), dapl) + self.assertIsInstance(dsid, h5py.h5d.DatasetID) + + +def test_get_chunk_details(): + from io import BytesIO + buf = BytesIO() + name = make_name() + with h5py.File(buf, 'w') as fout: + fout.create_dataset(name, shape=(100, 100), chunks=(10, 10), dtype='i4') + fout[name][:] = 1 + + buf.seek(0) + with h5py.File(buf, 'r') as fin: + ds = fin[name].id + + assert ds.get_num_chunks() == 100 + for j in range(100): + offset = tuple(np.array(np.unravel_index(j, (10, 10))) * 10) + + si = ds.get_chunk_info(j) + assert si.chunk_offset == offset + assert si.filter_mask == 0 + assert si.byte_offset is not None + assert si.size > 0 + + si = ds.get_chunk_info_by_coord((0, 0)) + assert si.chunk_offset == (0, 0) + assert si.filter_mask == 0 + assert si.byte_offset is not None + assert si.size > 0 + + +@ut.skipUnless(h5py.version.hdf5_version_tuple >= (1, 12, 3) or + (h5py.version.hdf5_version_tuple >= (1, 10, 10) and h5py.version.hdf5_version_tuple < (1, 10, 99)), + "chunk iteration requires HDF5 1.10.10 and later 1.10, or 1.12.3 and later") +def test_chunk_iter(): + """H5Dchunk_iter() for chunk information""" + from io import BytesIO + buf = BytesIO() + name = make_name() + with h5py.File(buf, 'w') as f: + f.create_dataset(name, shape=(100, 100), chunks=(10, 10), dtype='i4') + f[name][:] = 1 + + buf.seek(0) + with h5py.File(buf, 'r') as f: + dsid = f[name].id + + num_chunks = dsid.get_num_chunks() + assert num_chunks == 100 + ci = {} + for j in range(num_chunks): + si = dsid.get_chunk_info(j) + ci[si.chunk_offset] = si + + def callback(chunk_info): + known = ci[chunk_info.chunk_offset] + assert chunk_info.chunk_offset == known.chunk_offset + assert chunk_info.filter_mask == known.filter_mask + assert chunk_info.byte_offset == known.byte_offset + assert chunk_info.size == known.size + + dsid.chunk_iter(callback) + + +def test_empty_shape(writable_file): + ds = writable_file.create_dataset(make_name(), dtype='int32') + assert ds.shape is None + assert ds.maxshape is None + + +def test_zero_storage_size(): + # https://github.com/h5py/h5py/issues/1475 + from io import BytesIO + buf = BytesIO() + with h5py.File(buf, 'w') as fout: + fout.create_dataset('empty', dtype='uint8') + + buf.seek(0) + with h5py.File(buf, 'r') as fin: + assert fin['empty'].chunks is None + assert fin['empty'].id.get_offset() is None + assert fin['empty'].id.get_storage_size() == 0 + + +def test_python_int_uint64(writable_file): + # https://github.com/h5py/h5py/issues/1547 + data = [np.iinfo(np.int64).max, np.iinfo(np.int64).max + 1] + + # Check creating a new dataset + ds = writable_file.create_dataset(make_name(), data=data, dtype=np.uint64) + assert ds.dtype == np.dtype(np.uint64) + np.testing.assert_array_equal(ds[:], np.array(data, dtype=np.uint64)) + + # Check writing to an existing dataset + ds[:] = data + np.testing.assert_array_equal(ds[:], np.array(data, dtype=np.uint64)) + + +def test_setitem_fancy_indexing(writable_file): + # https://github.com/h5py/h5py/issues/1593 + arr = writable_file.create_dataset(make_name(), (5, 1000, 2), dtype=np.uint8) + block = np.random.randint(255, size=(5, 3, 2)) + arr[:, [0, 2, 4], ...] = block + + +def test_vlen_spacepad(): + with File(get_data_file_path("vlen_string_dset.h5")) as f: + assert f["DS1"][0] == b"Parting" + + +def test_vlen_nullterm(): + with File(get_data_file_path("vlen_string_dset_utc.h5")) as f: + assert f["ds1"][0] == b"2009-12-20T10:16:18.662409Z" + + +def test_allow_unknown_filter(writable_file): + # apparently 256-511 are reserved for testing purposes + fake_filter_id = 256 + ds = writable_file.create_dataset( + make_name(), shape=(10, 10), dtype=np.uint8, compression=fake_filter_id, + allow_unknown_filter=True + ) + assert str(fake_filter_id) in ds._filters + assert ds.compression == 'unknown' + + +def test_dset_chunk_cache(): + """Chunk cache configuration for individual datasets.""" + from io import BytesIO + buf = BytesIO() + name = make_name() + + with h5py.File(buf, 'w') as fout: + ds = fout.create_dataset( + name, shape=(10, 20), chunks=(5, 4), dtype='i4', + rdcc_nbytes=2 * 1024 * 1024, rdcc_w0=0.2, rdcc_nslots=997) + ds_chunk_cache = ds.id.get_access_plist().get_chunk_cache() + assert fout.id.get_access_plist().get_cache()[1:] != ds_chunk_cache + assert ds_chunk_cache == (997, 2 * 1024 * 1024, 0.2) + + buf.seek(0) + with h5py.File(buf, 'r') as fin: + ds = fin.require_dataset( + name, shape=(10, 20), dtype='i4', + rdcc_nbytes=3 * 1024 * 1024, rdcc_w0=0.67, rdcc_nslots=709) + ds_chunk_cache = ds.id.get_access_plist().get_chunk_cache() + assert fin.id.get_access_plist().get_cache()[1:] != ds_chunk_cache + assert ds_chunk_cache == (709, 3 * 1024 * 1024, 0.67) + + +class TestCommutative(BaseDataset): + """ + Test the symmetry of operators, at least with the numpy types. + Issue: https://github.com/h5py/h5py/issues/1947 + """ + def test_numpy_commutative(self,): + """ + Create a h5py dataset, extract one element convert to numpy + Check that it returns symmetric response to == and != + """ + shape = (100,1) + dset = self.f.create_dataset(make_name(), shape, dtype=float, + data=np.random.rand(*shape)) + + # grab a value from the elements, ie dset[0, 0] + # check that mask arrays are commutative wrt ==, != + val = np.float64(dset[0, 0]) + + assert np.all((val == dset) == (dset == val)) + assert np.all((val != dset) == (dset != val)) + + # generate sample not in the dset, ie max(dset)+delta + # check that mask arrays are commutative wrt ==, != + delta = 0.001 + nval = np.nanmax(dset)+delta + + assert np.all((nval == dset) == (dset == nval)) + assert np.all((nval != dset) == (dset != nval)) + + def test_basetype_commutative(self,): + """ + Create a h5py dataset and check basetype compatibility. + Check that operation is symmetric, even if it is potentially + not meaningful. + """ + shape = (100,1) + dset = self.f.create_dataset(make_name(), shape, dtype=float, + data=np.random.rand(*shape)) + + # generate float type, sample float(0.) + # check that operation is symmetric (but potentially meaningless) + val = float(0.) + assert (val == dset) == (dset == val) + assert (val != dset) == (dset != val) + +class TestVirtualPrefix(BaseDataset): + """ + Test setting virtual prefix + """ + def test_virtual_prefix_create(self): + shape = (100,1) + virtual_prefix = "/path/to/virtual" + dset = self.f.create_dataset(make_name(), shape, dtype=float, + data=np.random.rand(*shape), + virtual_prefix = virtual_prefix) + + virtual_prefix_readback = pathlib.Path(dset.id.get_access_plist().get_virtual_prefix().decode()).as_posix() + assert virtual_prefix_readback == virtual_prefix + + def test_virtual_prefix_require(self): + virtual_prefix = "/path/to/virtual" + dset = self.f.require_dataset(make_name(), (10, 3), 'f', virtual_prefix = virtual_prefix) + virtual_prefix_readback = pathlib.Path(dset.id.get_access_plist().get_virtual_prefix().decode()).as_posix() + self.assertEqual(virtual_prefix, virtual_prefix_readback) + self.assertIsInstance(dset, Dataset) + self.assertEqual(dset.shape, (10, 3)) + + +def ds_str(file, shape=(10, )): + dt = h5py.string_dtype(encoding='ascii') + fill_value = b'fill' + return file.create_dataset(make_name(), shape, dtype=dt, fillvalue=fill_value) + + +def ds_fields(file, shape=(10, )): + dt = np.dtype([ + ('foo', h5py.string_dtype(encoding='ascii')), + ('bar', np.float64), + ]) + fill_value = np.asarray(('fill', 0.0), dtype=dt) + name = make_name() + file[name] = np.broadcast_to(fill_value, shape) + return file[name] + + +view_getters = pytest.mark.parametrize( + "view_getter,make_ds", + [ + (lambda ds: ds, ds_str), + (lambda ds: ds.astype(dtype=object), ds_str), + (lambda ds: ds.asstr(), ds_str), + (lambda ds: ds.fields("foo"), ds_fields), + ], + ids=["ds", "astype", "asstr", "fields"], +) + + +COPY_IF_NEEDED = False if NUMPY_RELEASE_VERSION < (2, 0) else None + +@pytest.mark.parametrize("copy", [True, COPY_IF_NEEDED]) +@view_getters +def test_array_copy(view_getter, make_ds, copy, writable_file): + ds = make_ds(writable_file) + view = view_getter(ds) + np.array(view, copy=copy) + + +@pytest.mark.skipif( + NUMPY_RELEASE_VERSION < (2, 0), + reason="forbidding copies requires numpy 2", +) +@view_getters +def test_array_copy_false(view_getter, make_ds, writable_file): + ds = make_ds(writable_file) + view = view_getter(ds) + with pytest.raises(ValueError, match="memory allocation cannot be avoided"): + np.array(view, copy=False) + + +@view_getters +def test_array_dtype(view_getter, make_ds, writable_file): + ds = make_ds(writable_file) + view = view_getter(ds) + assert np.array(view, dtype='|S10').dtype == np.dtype('|S10') + + +@view_getters +def test_array_scalar(view_getter, make_ds, writable_file): + ds = make_ds(writable_file, shape=()) + view = view_getter(ds) + assert isinstance(view[()], (bytes, str)) + assert np.array(view).shape == () + + +@view_getters +def test_array_nd(view_getter, make_ds, writable_file): + ds = make_ds(writable_file, shape=(5, 6)) + view = view_getter(ds) + assert np.array(view).shape == (5, 6) + + +@view_getters +def test_view_properties(view_getter, make_ds, writable_file): + ds = make_ds(writable_file, shape=(5, 6)) + view = view_getter(ds) + assert view.dtype == np.dtype(object) + assert view.ndim == 2 + assert view.shape == (5, 6) + assert view.size == 30 + assert len(view) == 5 + + +@pytest.mark.thread_unsafe(reason="spawns thread pool itself") +def test_concurrent_dataset_creation(writable_file): + N_THREADS = 25 + N_DATASETS_PER_THREAD = 5 + # Defines a thread barrier that will be spawned before parallel execution + # this increases the probability of concurrent access clashes. + barrier = threading.Barrier(N_THREADS) + + def closure(ithread): + # Ensure that all threads reach this point before concurrent execution. + barrier.wait() + for j in range(N_DATASETS_PER_THREAD): + writable_file.create_dataset(f'concurrent_{ithread:02d}_{j:02d}', (1000,), dtype='i4') + + with ThreadPoolExecutor(max_workers=N_THREADS) as executor: + futures = [executor.submit(closure, ithread) for ithread in range(N_THREADS)] + + [f.result() for f in futures] + expected = set(f'concurrent_{i:02d}_{j:02d}' for i in range(N_THREADS) for j in range(N_DATASETS_PER_THREAD)) + assert set(writable_file) == expected + + +def test_filter_properties(writable_file): + name = make_name() + ds = writable_file.create_dataset( + name, shape=1000, dtype=np.float32, + fletcher32=True, shuffle=True, compression='lzf' + ) + assert ds.filter_ids == ( + h5py.h5z.FILTER_SHUFFLE, h5py.h5z.FILTER_LZF, h5py.h5z.FILTER_FLETCHER32 + ) + assert ds.filter_names == ('shuffle', 'lzf', 'fletcher32') + + +def test_store_refs(writable_file): + ds1 = writable_file.create_dataset(make_name("foo"), data=np.arange(12)) + refs_ds = writable_file.create_dataset(make_name("refs"), data=[writable_file.ref, ds1.ref]) + assert isinstance(refs_ds[0], h5py.Reference) + assert writable_file[refs_ds[0]] == writable_file + assert isinstance(refs_ds[1], h5py.Reference) + assert writable_file[refs_ds[1]] == ds1 + + # Single reference + ref_scalar_ds = writable_file.create_dataset(make_name("ref_scalar"), data=ds1.ref) + assert isinstance(ref_scalar_ds[()], h5py.h5r.Reference) + assert writable_file[ref_scalar_ds[()]] == ds1 + + +def test_store_regionrefs(writable_file): + ds1 = writable_file.create_dataset(make_name("foo"), data=np.arange(12)) + regionrefs_ds = writable_file.create_dataset(make_name("regrefs"), data=[ + ds1.regionref[:-1], ds1.regionref[1:] + ]) + assert isinstance(regionrefs_ds[0], h5py.RegionReference) + np.testing.assert_array_equal(ds1[regionrefs_ds[0]], np.arange(11)) + np.testing.assert_array_equal(ds1[regionrefs_ds[1]], np.arange(1, 12)) + + refs_ds = writable_file.create_dataset(make_name("refs"), shape=(1,), dtype=h5py.ref_dtype) + with pytest.raises(TypeError, match="convert"): + refs_ds[0] = ds1.regionref[:6] diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_dataset_getitem.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_dataset_getitem.py new file mode 100644 index 0000000000000000000000000000000000000000..6944f90b8c6916c160ca048c86f6cfc9204bf3fa --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_dataset_getitem.py @@ -0,0 +1,687 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Tests the h5py.Dataset.__getitem__ method. + + This module does not specifically test type conversion. The "type" axis + therefore only tests objects which interact with the slicing system in + unreliable ways; for example, compound and array types. + + See test_dataset_getitem_types for type-conversion tests. + + Tests are organized into TestCases by dataset shape and type. Test + methods vary by slicing arg type. + + 1. Dataset shape: + Empty + Scalar + 1D + 3D + + 2. Type: + Float + Compound + Array + + 3. Slicing arg types: + Ellipsis + Empty tuple + Regular slice + MultiBlockSlice + Indexing + Index list + Boolean mask + Field names +""" + +import sys + +import numpy as np +import pytest + +import h5py +from .common import ut, TestCase, make_name + + +class TestEmpty(TestCase): + + def setUp(self): + TestCase.setUp(self) + sid = h5py.h5s.create(h5py.h5s.NULL) + tid = h5py.h5t.C_S1.copy() + tid.set_size(10) + dsid = h5py.h5d.create(self.f.id, b'x', tid, sid) + self.dset = h5py.Dataset(dsid) + self.empty_obj = h5py.Empty(np.dtype("S10")) + + def test_ndim(self): + """ Verify number of dimensions """ + self.assertEqual(self.dset.ndim, 0) + + def test_shape(self): + """ Verify shape """ + self.assertEqual(self.dset.shape, None) + + def test_size(self): + """ Verify shape """ + self.assertEqual(self.dset.size, None) + + def test_nbytes(self): + """ Verify nbytes """ + self.assertEqual(self.dset.nbytes, 0) + + def test_ellipsis(self): + self.assertEqual(self.dset[...], self.empty_obj) + + def test_tuple(self): + self.assertEqual(self.dset[()], self.empty_obj) + + def test_slice(self): + """ slice -> ValueError """ + with self.assertRaises(ValueError): + self.dset[0:4] + + def test_multi_block_slice(self): + """ MultiBlockSlice -> ValueError """ + with self.assertRaises(ValueError): + self.dset[h5py.MultiBlockSlice()] + + def test_index(self): + """ index -> ValueError """ + with self.assertRaises(ValueError): + self.dset[0] + + def test_indexlist(self): + """ index list -> ValueError """ + with self.assertRaises(ValueError): + self.dset[[1,2,5]] + + def test_mask(self): + """ mask -> ValueError """ + mask = np.array(True, dtype='bool') + with self.assertRaises(ValueError): + self.dset[mask] + + def test_fieldnames(self): + """ field name -> ValueError """ + with self.assertRaises(ValueError): + self.dset['field'] + + +class TestScalarFloat(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.data = np.array(42.5, dtype=np.double) + self.dset = self.f.create_dataset('x', data=self.data) + + def test_ndim(self): + """ Verify number of dimensions """ + self.assertEqual(self.dset.ndim, 0) + + def test_size(self): + """ Verify size """ + self.assertEqual(self.dset.size, 1) + + def test_nbytes(self): + """ Verify nbytes """ + self.assertEqual(self.dset.nbytes, self.data.dtype.itemsize) # not sure if 'f' is always alias for 'f4' + + def test_shape(self): + """ Verify shape """ + self.assertEqual(self.dset.shape, tuple()) + + def test_ellipsis(self): + """ Ellipsis -> scalar ndarray """ + out = self.dset[...] + self.assertArrayEqual(out, self.data) + + def test_tuple(self): + """ () -> bare item """ + out = self.dset[()] + self.assertArrayEqual(out, self.data.item()) + + def test_slice(self): + """ slice -> ValueError """ + with self.assertRaises(ValueError): + self.dset[0:4] + + def test_multi_block_slice(self): + """ MultiBlockSlice -> ValueError """ + with self.assertRaises(ValueError): + self.dset[h5py.MultiBlockSlice()] + + def test_index(self): + """ index -> ValueError """ + with self.assertRaises(ValueError): + self.dset[0] + + # FIXME: NumPy has IndexError instead + def test_indexlist(self): + """ index list -> ValueError """ + with self.assertRaises(ValueError): + self.dset[[1,2,5]] + + # FIXME: NumPy permits this + def test_mask(self): + """ mask -> ValueError """ + mask = np.array(True, dtype='bool') + with self.assertRaises(ValueError): + self.dset[mask] + + def test_fieldnames(self): + """ field name -> ValueError (no fields) """ + with self.assertRaises(ValueError): + self.dset['field'] + + +class TestScalarCompound(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.data = np.array((42.5, -118, "Hello"), dtype=[('a', 'f'), ('b', 'i'), ('c', '|S10')]) + self.dset = self.f.create_dataset('x', data=self.data) + + def test_ndim(self): + """ Verify number of dimensions """ + self.assertEqual(self.dset.ndim, 0) + + def test_shape(self): + """ Verify shape """ + self.assertEqual(self.dset.shape, tuple()) + + def test_size(self): + """ Verify size """ + self.assertEqual(self.dset.size, 1) + + def test_nbytes(self): + """ Verify nbytes """ + self.assertEqual(self.dset.nbytes, self.data.dtype.itemsize) + + def test_ellipsis(self): + """ Ellipsis -> scalar ndarray """ + out = self.dset[...] + # assertArrayEqual doesn't work with compounds; do manually + self.assertIsInstance(out, np.ndarray) + self.assertEqual(out.shape, self.data.shape) + self.assertEqual(out.dtype, self.data.dtype) + + def test_tuple(self): + """ () -> np.void instance """ + out = self.dset[()] + self.assertIsInstance(out, np.void) + self.assertEqual(out.dtype, self.data.dtype) + + def test_slice(self): + """ slice -> ValueError """ + with self.assertRaises(ValueError): + self.dset[0:4] + + def test_multi_block_slice(self): + """ MultiBlockSlice -> ValueError """ + with self.assertRaises(ValueError): + self.dset[h5py.MultiBlockSlice()] + + def test_index(self): + """ index -> ValueError """ + with self.assertRaises(ValueError): + self.dset[0] + + # FIXME: NumPy has IndexError instead + def test_indexlist(self): + """ index list -> ValueError """ + with self.assertRaises(ValueError): + self.dset[[1,2,5]] + + # FIXME: NumPy permits this + def test_mask(self): + """ mask -> ValueError """ + mask = np.array(True, dtype='bool') + with self.assertRaises(ValueError): + self.dset[mask] + + # FIXME: NumPy returns a scalar ndarray + def test_fieldnames(self): + """ field name -> bare value """ + out = self.dset['a'] + self.assertIsInstance(out, np.float32) + self.assertEqual(out, self.dset['a']) + + +class TestScalarArray(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.dt = np.dtype('(3,2)f') + self.data = np.array([(3.2, -119), (42, 99.8), (3.14, 0)], dtype='f') + self.dset = self.f.create_dataset('x', (), dtype=self.dt) + self.dset[...] = self.data + + def test_ndim(self): + """ Verify number of dimensions """ + self.assertEqual(self.data.ndim, 2) + self.assertEqual(self.dset.ndim, 0) + + def test_size(self): + """ Verify size """ + self.assertEqual(self.dset.size, 1) + + def test_nbytes(self): + """ Verify nbytes """ + self.assertEqual(self.dset.nbytes, self.dset.dtype.itemsize) # not sure if 'f' is always alias for 'f4' + + def test_shape(self): + """ Verify shape """ + self.assertEqual(self.data.shape, (3, 2)) + self.assertEqual(self.dset.shape, tuple()) + + def test_ellipsis(self): + """ Ellipsis -> ndarray promoted to underlying shape """ + out = self.dset[...] + self.assertArrayEqual(out, self.data) + + def test_tuple(self): + """ () -> same as ellipsis """ + out = self.dset[...] + self.assertArrayEqual(out, self.data) + + def test_slice(self): + """ slice -> ValueError """ + with self.assertRaises(ValueError): + self.dset[0:4] + + def test_multi_block_slice(self): + """ MultiBlockSlice -> ValueError """ + with self.assertRaises(ValueError): + self.dset[h5py.MultiBlockSlice()] + + def test_index(self): + """ index -> ValueError """ + with self.assertRaises(ValueError): + self.dset[0] + + def test_indexlist(self): + """ index list -> ValueError """ + with self.assertRaises(ValueError): + self.dset[[]] + + def test_mask(self): + """ mask -> ValueError """ + mask = np.array(True, dtype='bool') + with self.assertRaises(ValueError): + self.dset[mask] + + def test_fieldnames(self): + """ field name -> ValueError (no fields) """ + with self.assertRaises(ValueError): + self.dset['field'] + + +class Test1DZeroFloat(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.data = np.ones((0,), dtype='f') + self.dset = self.f.create_dataset('x', data=self.data) + + def test_ndim(self): + """ Verify number of dimensions """ + self.assertEqual(self.dset.ndim, 1) + + def test_shape(self): + """ Verify shape """ + self.assertEqual(self.dset.shape, (0,)) + + def test_ellipsis(self): + """ Ellipsis -> ndarray of matching shape """ + self.assertNumpyBehavior(self.dset, self.data, np.s_[...]) + + def test_tuple(self): + """ () -> same as ellipsis """ + self.assertNumpyBehavior(self.dset, self.data, np.s_[()]) + + def test_slice(self): + """ slice -> ndarray of shape (0,) """ + self.assertNumpyBehavior(self.dset, self.data, np.s_[0:4]) + + def test_slice_stop_less_than_start(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[7:5]) + + def test_index(self): + """ index -> out of range """ + with self.assertRaises(IndexError): + self.dset[0] + + def test_indexlist(self): + """ index list """ + self.assertNumpyBehavior(self.dset, self.data, np.s_[[]]) + + def test_mask(self): + """ mask -> ndarray of matching shape """ + mask = np.ones((0,), dtype='bool') + self.assertNumpyBehavior( + self.dset, + self.data, + np.s_[mask], + # Fast reader doesn't work with boolean masks + skip_fast_reader=True, + ) + + def test_fieldnames(self): + """ field name -> ValueError (no fields) """ + with self.assertRaises(ValueError): + self.dset['field'] + + +class Test1DFloat(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.data = np.arange(50).astype('f') + self.dset = self.f.create_dataset('x', data=self.data) + + def test_ndim(self): + """ Verify number of dimensions """ + self.assertEqual(self.dset.ndim, 1) + + def test_shape(self): + """ Verify shape """ + self.assertEqual(self.dset.shape, (50,)) + + def test_ellipsis(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[...]) + + def test_tuple(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[()]) + + def test_slice_simple(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[0:4]) + + def test_slice_zerosize(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[4:4]) + + def test_slice_strides(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[1:7:3]) + + def test_slice_negindexes(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[-8:-2:3]) + + def test_slice_stop_less_than_start(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[7:5]) + + def test_slice_outofrange(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[100:400:3]) + + def test_slice_backwards(self): + """ we disallow negative steps """ + with self.assertRaises(ValueError): + self.dset[::-1] + + def test_slice_zerostride(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[::0]) + + def test_index_simple(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[3]) + + def test_index_neg(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[-4]) + + # FIXME: NumPy permits this... it adds a new axis in front + def test_index_none(self): + with self.assertRaises(TypeError): + self.dset[None] + + def test_index_illegal(self): + """ Illegal slicing argument """ + with self.assertRaises(TypeError): + self.dset[{}] + + def test_index_outofrange(self): + with self.assertRaises(IndexError): + self.dset[100] + + def test_indexlist_simple(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[[1,2,5]]) + + def test_indexlist_numpyarray(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[np.array([1, 2, 5])]) + + def test_indexlist_long(self): + # selection logic changes with >16 indices + self.assertNumpyBehavior(self.dset, self.data, np.s_[range(0, 50, 2)]) + + def test_indexlist_single_index_ellipsis(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[[0], ...]) + + def test_indexlist_numpyarray_single_index_ellipsis(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[np.array([0]), ...]) + + def test_indexlist_long_ellipsis(self): + # selection logic changes with >16 indices + self.assertNumpyBehavior(self.dset, self.data, np.s_[range(0, 50, 2), ...]) + + def test_indexlist_numpyarray_ellipsis(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[np.array([1, 2, 5]), ...]) + + def test_indexlist_empty(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[[]]) + + def test_indexlist_outofrange(self): + with self.assertRaises(IndexError): + self.dset[[100]] + + def test_indexlist_nonmonotonic(self): + """ we require index list values to be strictly increasing """ + with self.assertRaises(TypeError): + self.dset[[1,3,2]] + + def test_indexlist_monotonic_negative(self): + # This should work: indices are logically increasing + self.assertNumpyBehavior(self.dset, self.data, np.s_[[0, 2, -2]]) + + with self.assertRaises(TypeError): + self.dset[[-2, -3]] + + def test_indexlist_repeated(self): + """ we forbid repeated index values """ + with self.assertRaises(TypeError): + self.dset[[1,1,2]] + + def test_mask_true(self): + self.assertNumpyBehavior( + self.dset, + self.data, + np.s_[self.data > -100], + # Fast reader doesn't work with boolean masks + skip_fast_reader=True, + ) + + def test_mask_false(self): + self.assertNumpyBehavior( + self.dset, + self.data, + np.s_[self.data > 100], + # Fast reader doesn't work with boolean masks + skip_fast_reader=True, + ) + + def test_mask_partial(self): + self.assertNumpyBehavior( + self.dset, + self.data, + np.s_[self.data > 5], + # Fast reader doesn't work with boolean masks + skip_fast_reader=True, + ) + + def test_mask_wrongsize(self): + """ we require the boolean mask shape to match exactly """ + with self.assertRaises(TypeError): + self.dset[np.ones((2,), dtype='bool')] + + def test_fieldnames(self): + """ field name -> ValueError (no fields) """ + with self.assertRaises(ValueError): + self.dset['field'] + + +class Test2DZeroFloat(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.data = np.ones((0,3), dtype='f') + self.dset = self.f.create_dataset('x', data=self.data) + + def test_ndim(self): + """ Verify number of dimensions """ + self.assertEqual(self.dset.ndim, 2) + + def test_shape(self): + """ Verify shape """ + self.assertEqual(self.dset.shape, (0, 3)) + + def test_indexlist(self): + """ see issue #473 """ + self.assertNumpyBehavior(self.dset, self.data, np.s_[:,[0,1,2]]) + + +class Test2DFloat(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.data = np.ones((5,3), dtype='f') + self.dset = self.f.create_dataset('x', data=self.data) + + def test_ndim(self): + """ Verify number of dimensions """ + self.assertEqual(self.dset.ndim, 2) + + def test_size(self): + """ Verify size """ + self.assertEqual(self.dset.size, 15) + + def test_nbytes(self): + """ Verify nbytes """ + self.assertEqual(self.dset.nbytes, 15*self.data.dtype.itemsize) # not sure if 'f' is always alias for 'f4' + + def test_shape(self): + """ Verify shape """ + self.assertEqual(self.dset.shape, (5, 3)) + + def test_indexlist(self): + """ see issue #473 """ + self.assertNumpyBehavior(self.dset, self.data, np.s_[:,[0,1,2]]) + + def test_index_emptylist(self): + self.assertNumpyBehavior(self.dset, self.data, np.s_[:, []]) + self.assertNumpyBehavior(self.dset, self.data, np.s_[[]]) + + +class TestVeryLargeArray(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.dset = self.f.create_dataset('x', shape=(2**15, 2**16), dtype='f4') + + @ut.skipIf(sys.maxsize < 2**31, 'Maximum integer size >= 2**31 required') + def test_size(self): + self.assertEqual(self.dset.size, 2**31) + + +def test_read_no_fill_value(writable_file): + # With FILL_TIME_NEVER, HDF5 doesn't write zeros in the output array for + # unallocated chunks. If we read into uninitialized memory, it can appear + # to read random values. https://github.com/h5py/h5py/issues/2069 + dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE) + dcpl.set_chunk((1,)) + dcpl.set_fill_time(h5py.h5d.FILL_TIME_NEVER) + ds = h5py.Dataset(h5py.h5d.create( + writable_file.id, + make_name().encode("utf-8"), + h5py.h5t.IEEE_F64LE, + h5py.h5s.create_simple((5,)), + dcpl, + )) + np.testing.assert_array_equal(ds[:3], np.zeros(3, np.float64)) + + +class TestBoolIndex(TestCase): + """ + Tests for indexing with Boolean arrays + """ + def setUp(self): + super().setUp() + self.arr = np.arange(9).reshape(3,-1) + self.dset = self.f.create_dataset('x', data=self.arr) + + def test_select_first_axis(self): + sel = np.s_[[False, True, False],:] + self.assertNumpyBehavior(self.dset, self.arr, sel) + + def test_wrong_size(self): + sel = np.s_[[False, True, False, False],:] + with self.assertRaises(TypeError): + self.dset[sel] + + +def test_error_newaxis(writable_file): + ds = writable_file.create_dataset(make_name(), data=np.arange(5)) + with pytest.raises(TypeError, match="newaxis"): + ds[np.newaxis, :] + + +@pytest.mark.parametrize( + "arr", [ + np.arange(9), + np.array([s.encode() for s in 'abcdefghi'], dtype=object), + ] +) +def test_bool_selection_1d(writable_file, arr): + """https://github.com/h5py/h5py/issues/2674""" + arr = arr.reshape(3, 3) + name = make_name() + writable_file[name] = arr + dset = writable_file[name] + sel = np.array([True, False, True]) + np.testing.assert_array_equal(dset[sel], arr[sel]) + np.testing.assert_array_equal(dset[sel, :], arr[sel, :]) + + +class TestZeroSizeSelectionResizableDataset(TestCase): + """ + Tests for indexing of zero Resizable Datasets + see https://github.com/h5py/h5py/issues/2549 + """ + def setUp(self): + super().setUp() + self.dset0 = self.f.create_dataset('x', (0,), 'f4', maxshape=(None,)) + self.dset1 = self.f.create_dataset('y', (1,), 'f4', maxshape=(None,)) + + def test_set_with_scalar(self): + sel = np.s_[:] + self.dset0[sel] = 1 + self.arr = np.array([], dtype=np.float32) + self.assertNumpyBehavior(self.dset0, self.arr, sel) + + def test_set_with_array(self): + sel = np.s_[:] + with self.assertRaises(TypeError, msg="Can't broadcast (4,) -> (0,)"): + self.dset0[sel] = np.arange(4) + + def test_set_zerosel_with_scalar(self): + sel = np.s_[0:0] + self.dset1[sel] = 1 + self.arr = np.array([], dtype=np.float32) + self.assertNumpyBehavior(self.dset1, self.arr, sel) + + def test_set_zerosel_with_array(self): + sel = np.s_[0:0] + with self.assertRaises(TypeError, msg="Can't broadcast (4,) -> (0,)"): + self.dset1[sel] = np.arange(4) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_dataset_swmr.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_dataset_swmr.py new file mode 100644 index 0000000000000000000000000000000000000000..36bdfd6954aaacd2cf7301854323f99de8cc052f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_dataset_swmr.py @@ -0,0 +1,122 @@ +import numpy as np +import pytest + +import h5py + +from .common import TestCase + + +class TestDatasetSwmrRead(TestCase): + """ Testing SWMR functions when reading a dataset. + Skip this test if the HDF5 library does not have the SWMR features. + """ + + def setUp(self): + TestCase.setUp(self) + self.data = np.arange(13).astype('f') + self.dset = self.f.create_dataset('data', chunks=(13,), maxshape=(None,), data=self.data) + fname = self.f.filename + self.f.close() + + self.f = h5py.File(fname, 'r', swmr=True) + self.dset = self.f['data'] + + def test_initial_swmr_mode_on(self): + """ Verify that the file is initially in SWMR mode""" + self.assertTrue(self.f.swmr_mode) + + def test_read_data(self): + self.assertArrayEqual(self.dset, self.data) + + def test_refresh(self): + self.dset.refresh() + + def test_force_swmr_mode_on_raises(self): + """ Verify when reading a file cannot be forcibly switched to swmr mode. + When reading with SWMR the file must be opened with swmr=True.""" + with self.assertRaises(Exception): + self.f.swmr_mode = True + self.assertTrue(self.f.swmr_mode) + + def test_force_swmr_mode_off_raises(self): + """ Switching SWMR write mode off is only possible by closing the file. + Attempts to forcibly switch off the SWMR mode should raise a ValueError. + """ + with self.assertRaises(ValueError): + self.f.swmr_mode = False + self.assertTrue(self.f.swmr_mode) + + +@pytest.mark.thread_unsafe(reason="Can't enable global SWMR flag twice") +class TestDatasetSwmrWrite(TestCase): + """ Testing SWMR functions when reading a dataset. + Skip this test if the HDF5 library does not have the SWMR features. + """ + + def setUp(self): + """ First setup a file with a small chunked and empty dataset. + No data written yet. + """ + + # Note that when creating the file, the swmr=True is not required for + # write, but libver='latest' is required. + self.f = h5py.File(self.mktemp(), 'w', libver='latest') + + self.data = np.arange(4).astype('f') + self.dset = self.f.create_dataset('data', shape=(0,), dtype=self.data.dtype, chunks=(2,), maxshape=(None,)) + + + def test_initial_swmr_mode_off(self): + """ Verify that the file is not initially in SWMR mode""" + self.assertFalse(self.f.swmr_mode) + + def test_switch_swmr_mode_on(self): + """ Switch to SWMR mode and verify """ + self.f.swmr_mode = True + self.assertTrue(self.f.swmr_mode) + + def test_switch_swmr_mode_off_raises(self): + """ Switching SWMR write mode off is only possible by closing the file. + Attempts to forcibly switch off the SWMR mode should raise a ValueError. + """ + self.f.swmr_mode = True + self.assertTrue(self.f.swmr_mode) + with self.assertRaises(ValueError): + self.f.swmr_mode = False + self.assertTrue(self.f.swmr_mode) + + def test_extend_dset(self): + """ Extend and flush a SWMR dataset + """ + self.f.swmr_mode = True + self.assertTrue(self.f.swmr_mode) + + self.dset.resize( self.data.shape ) + self.dset[:] = self.data + self.dset.flush() + + # Refresh and read back data for assertion + self.dset.refresh() + self.assertArrayEqual(self.dset, self.data) + + def test_extend_dset_multiple(self): + + self.f.swmr_mode = True + self.assertTrue(self.f.swmr_mode) + + self.dset.resize( (4,) ) + self.dset[0:] = self.data + self.dset.flush() + + # Refresh and read back 1st data block for assertion + self.dset.refresh() + self.assertArrayEqual(self.dset, self.data) + + self.dset.resize( (8,) ) + self.dset[4:] = self.data + self.dset.flush() + + # Refresh and read back 1st data block for assertion + self.dset.refresh() + self.assertArrayEqual(self.dset[0:4], self.data) + self.assertArrayEqual(self.dset[4:8], self.data) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_datatype.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_datatype.py new file mode 100644 index 0000000000000000000000000000000000000000..043c4934da4a50e7b72274b14f9068179461a803 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_datatype.py @@ -0,0 +1,41 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + File-resident datatype tests. + + Tests "committed" file-resident datatype objects. +""" + +import numpy as np + +from .common import TestCase, is_main_thread, make_name + +from h5py import Datatype + +class TestCreation(TestCase): + + """ + Feature: repr() works sensibly on datatype objects + """ + + def test_repr(self): + """ repr() on datatype objects """ + name = make_name() + self.f[name] = np.dtype('S10') + dt = self.f[name] + self.assertIsInstance(repr(dt), str) + if is_main_thread(): + self.f.close() + self.assertIsInstance(repr(dt), str) + + def test_appropriate_low_level_id(self): + " Binding a group to a non-TypeID identifier fails with ValueError " + with self.assertRaises(ValueError): + Datatype(self.f['/'].id) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_dimension_scales.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_dimension_scales.py new file mode 100644 index 0000000000000000000000000000000000000000..5c3f268b37099f7250ede547219c95cea0f71ac1 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_dimension_scales.py @@ -0,0 +1,226 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + + +import numpy as np + +from .common import TestCase, is_main_thread, make_name +from h5py import File +import h5py + + +class BaseDataset(TestCase): + + """ + data is a 3-dimensional dataset with dimensions [z, y, x] + + The z dimension is labeled. It does not have any attached scales. + The y dimension is not labeled. It has one attached scale. + The x dimension is labeled. It has two attached scales. + + data2 is a 3-dimensional dataset with no associated dimension scales. + """ + + def setUp(self): + self.f = File(self.mktemp(), 'w') + self.f['data'] = np.ones((4, 3, 2), 'f') + self.f['data2'] = np.ones((4, 3, 2), 'f') + self.f['x1'] = np.ones((2), 'f') + h5py.h5ds.set_scale(self.f['x1'].id) + h5py.h5ds.attach_scale(self.f['data'].id, self.f['x1'].id, 2) + self.f['x2'] = np.ones((2), 'f') + h5py.h5ds.set_scale(self.f['x2'].id, b'x2 name') + h5py.h5ds.attach_scale(self.f['data'].id, self.f['x2'].id, 2) + self.f['y1'] = np.ones((3), 'f') + h5py.h5ds.set_scale(self.f['y1'].id, b'y1 name') + h5py.h5ds.attach_scale(self.f['data'].id, self.f['y1'].id, 1) + self.f['z1'] = np.ones((4), 'f') + + h5py.h5ds.set_label(self.f['data'].id, 0, b'z') + h5py.h5ds.set_label(self.f['data'].id, 2, b'x') + + def tearDown(self): + if self.f: + self.f.close() + + +class TestH5DSBindings(BaseDataset): + + """ + Feature: Datasets can be created from existing data + """ + + def test_create_dimensionscale(self): + """ Create a dimension scale from existing dataset """ + self.assertTrue(h5py.h5ds.is_scale(self.f['x1'].id)) + self.assertEqual(h5py.h5ds.get_scale_name(self.f['x1'].id), b'') + self.assertEqual(self.f['x1'].attrs['CLASS'], b"DIMENSION_SCALE") + self.assertEqual(h5py.h5ds.get_scale_name(self.f['x2'].id), b'x2 name') + + def test_attach_dimensionscale(self): + self.assertTrue( + h5py.h5ds.is_attached(self.f['data'].id, self.f['x1'].id, 2) + ) + self.assertFalse( + h5py.h5ds.is_attached(self.f['data'].id, self.f['x1'].id, 1)) + self.assertEqual(h5py.h5ds.get_num_scales(self.f['data'].id, 0), 0) + self.assertEqual(h5py.h5ds.get_num_scales(self.f['data'].id, 1), 1) + self.assertEqual(h5py.h5ds.get_num_scales(self.f['data'].id, 2), 2) + + def test_detach_dimensionscale(self): + name = make_name() + self.f[name] = np.ones((4, 3, 2), 'f') + ds = self.f[name] + h5py.h5ds.attach_scale(ds.id, self.f['x1'].id, 2) + h5py.h5ds.attach_scale(ds.id, self.f['x2'].id, 2) + self.assertTrue(h5py.h5ds.is_attached(ds.id, self.f['x1'].id, 2)) + self.assertEqual(h5py.h5ds.get_num_scales(ds.id, 2), 2) + h5py.h5ds.detach_scale(ds.id, self.f['x1'].id, 2) + self.assertFalse(h5py.h5ds.is_attached(ds.id, self.f['x1'].id, 2)) + self.assertEqual(h5py.h5ds.get_num_scales(ds.id, 2), 1) + + def test_label_dimensionscale(self): + self.assertEqual(h5py.h5ds.get_label(self.f['data'].id, 0), b'z') + self.assertEqual(h5py.h5ds.get_label(self.f['data'].id, 1), b'') + self.assertEqual(h5py.h5ds.get_label(self.f['data'].id, 2), b'x') + + def test_iter_dimensionscales(self): + def func(dsid): + res = h5py.h5ds.get_scale_name(dsid) + if res == b'x2 name': + return dsid + + res = h5py.h5ds.iterate(self.f['data'].id, 2, func, 0) + self.assertEqual(h5py.h5ds.get_scale_name(res), b'x2 name') + + +class TestDimensionManager(BaseDataset): + + def test_make_scale(self): + # test recreating or renaming an existing scale: + self.f['x1'].make_scale(b'foobar') + self.assertEqual(self.f['data'].dims[2]['foobar'], self.f['x1']) + # test creating entirely new scale: + self.f['data2'].make_scale(b'foobaz') + self.f['data'].dims[2].attach_scale(self.f['data2']) + self.assertEqual(self.f['data'].dims[2]['foobaz'], self.f['data2']) + + def test_get_dimension(self): + with self.assertRaises(IndexError): + self.f['data'].dims[3] + + def test_len(self): + self.assertEqual(len(self.f['data'].dims), 3) + self.assertEqual(len(self.f['data2'].dims), 3) + + def test_iter(self): + dims = self.f['data'].dims + self.assertEqual( + [d for d in dims], + [dims[0], dims[1], dims[2]] + ) + + def test_repr(self): + ds = self.f.create_dataset(make_name(), (2,3), "f4") + self.assertIsInstance(repr(ds.dims), str) + + if is_main_thread(): + self.f.close() + self.assertIsInstance(repr(ds.dims), str) + + +class TestDimensionsHighLevel(BaseDataset): + + def test_len(self): + self.assertEqual(len(self.f['data'].dims[0]), 0) + self.assertEqual(len(self.f['data'].dims[1]), 1) + self.assertEqual(len(self.f['data'].dims[2]), 2) + self.assertEqual(len(self.f['data2'].dims[0]), 0) + self.assertEqual(len(self.f['data2'].dims[1]), 0) + self.assertEqual(len(self.f['data2'].dims[2]), 0) + + def test_get_label(self): + self.assertEqual(self.f['data'].dims[2].label, 'x') + self.assertEqual(self.f['data'].dims[1].label, '') + self.assertEqual(self.f['data'].dims[0].label, 'z') + self.assertEqual(self.f['data2'].dims[2].label, '') + self.assertEqual(self.f['data2'].dims[1].label, '') + self.assertEqual(self.f['data2'].dims[0].label, '') + + def test_set_label(self): + self.f['data'].dims[0].label = 'foo' + self.assertEqual(self.f['data'].dims[2].label, 'x') + self.assertEqual(self.f['data'].dims[1].label, '') + self.assertEqual(self.f['data'].dims[0].label, 'foo') + + def test_attach_detach_scale(self): + name = make_name('data3') + self.f[name] = self.f['data'][...] + ds = self.f[name] + + ds.dims[2].attach_scale(self.f["x1"]) + ds.dims[2].attach_scale(self.f["x2"]) + self.assertEqual(len(ds.dims[2]), 2) + self.assertEqual(ds.dims[2][0], self.f["x1"]) + self.assertEqual(ds.dims[2][1], self.f["x2"]) + + ds.dims[2].detach_scale(self.f["x1"]) + self.assertEqual(len(ds.dims[2]), 1) + self.assertEqual(ds.dims[2][0], self.f["x2"]) + ds.dims[2].detach_scale(self.f["x2"]) + self.assertEqual(len(ds.dims[2]), 0) + + def test_get_dimension_scale(self): + self.assertEqual(self.f['data'].dims[2][0], self.f['x1']) + with self.assertRaises(RuntimeError): + self.f['data2'].dims[2][0], self.f['x2'] + self.assertEqual(self.f['data'].dims[2][''], self.f['x1']) + self.assertEqual(self.f['data'].dims[2]['x2 name'], self.f['x2']) + + def test_get_items(self): + self.assertEqual( + self.f['data'].dims[2].items(), + [('', self.f['x1']), ('x2 name', self.f['x2'])] + ) + + def test_get_keys(self): + self.assertEqual(self.f['data'].dims[2].keys(), ['', 'x2 name']) + + def test_get_values(self): + self.assertEqual( + self.f['data'].dims[2].values(), + [self.f['x1'], self.f['x2']] + ) + + def test_iter(self): + self.assertEqual([i for i in self.f['data'].dims[2]], ['', 'x2 name']) + + def test_repr(self): + ds = self.f["data"] + self.assertEqual(repr(ds.dims[2])[1:16], '"x" dimension 2') + + if is_main_thread(): + self.f.close() + self.assertIsInstance(repr(ds.dims), str) + + def test_attributes(self): + self.f["data2"].attrs["DIMENSION_LIST"] = self.f["data"].attrs[ + "DIMENSION_LIST"] + self.assertEqual(len(self.f['data2'].dims[0]), 0) + self.assertEqual(len(self.f['data2'].dims[1]), 1) + self.assertEqual(len(self.f['data2'].dims[2]), 2) + + def test_is_scale(self): + """Test Dataset.is_scale property""" + self.assertTrue(self.f['x1'].is_scale) + self.assertTrue(self.f['x2'].is_scale) + self.assertTrue(self.f['y1'].is_scale) + self.assertFalse(self.f['z1'].is_scale) + self.assertFalse(self.f['data'].is_scale) + self.assertFalse(self.f['data2'].is_scale) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_dims_dimensionproxy.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_dims_dimensionproxy.py new file mode 100644 index 0000000000000000000000000000000000000000..763f03097a70b764ad8540607588e7207bc75d00 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_dims_dimensionproxy.py @@ -0,0 +1,22 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Tests the h5py.Dataset.dims.DimensionProxy class. +""" + + +from .common import TestCase, make_name + +class TestItems(TestCase): + + def test_empty(self): + """ no dimension scales -> empty list """ + dset = self.f.create_dataset(make_name(), (10,), "f4") + self.assertEqual(dset.dims[0].items(), []) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_dtype.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_dtype.py new file mode 100644 index 0000000000000000000000000000000000000000..9c2f7a17191d2b53c584543c58a31c056f8bc538 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_dtype.py @@ -0,0 +1,571 @@ +""" + Tests for converting between numpy dtypes and h5py data types +""" + +from itertools import count +import platform +import numpy as np +import h5py +try: + import tables +except ImportError: + tables = None + +from .common import ut, TestCase, make_name + +UNSUPPORTED_LONG_DOUBLE = ('i386', 'i486', 'i586', 'i686', 'ppc64le') +UNSUPPORTED_LONG_DOUBLE_TYPES = ('float96', 'float128', 'complex192', + 'complex256') + + +class TestVlen(TestCase): + + """ + Check that storage of vlen strings is carried out correctly. + """ + def assertVlenArrayEqual(self, dset, arr, message=None, precision=None): + assert dset.shape == arr.shape, \ + "Shape mismatch (%s vs %s)%s" % (dset.shape, arr.shape, message) + for (i, d, a) in zip(count(), dset, arr): + self.assertArrayEqual(d, a, message, precision) + + def test_compound(self): + name = make_name() + fields = [] + fields.append(('field_1', h5py.string_dtype())) + fields.append(('field_2', np.int32)) + dt = np.dtype(fields) + self.f[name] = np.dtype(dt) + dt_out = self.f[name].dtype.fields['field_1'][0] + string_inf = h5py.check_string_dtype(dt_out) + self.assertEqual(string_inf.encoding, 'utf-8') + + def test_compound_vlen_bool(self): + vidt = h5py.vlen_dtype(np.uint8) + def a(items): + return np.array(items, dtype=np.uint8) + + f = self.f + + dt_vb = np.dtype([ + ('foo', vidt), + ('logical', bool)]) + n_dt_vb = make_name("dt_vb") + vb = f.create_dataset(n_dt_vb, shape=(4,), dtype=dt_vb) + data = np.array([(a([1, 2, 3]), True), + (a([1 ]), False), + (a([1, 5 ]), True), + (a([],), False), ], + dtype=dt_vb) + vb[:] = data + actual = f[n_dt_vb][:] + self.assertVlenArrayEqual(data['foo'], actual['foo']) + self.assertArrayEqual(data['logical'], actual['logical']) + + dt_vv = np.dtype([ + ('foo', vidt), + ('bar', vidt)]) + f.create_dataset(make_name("dt_vv"), shape=(4,), dtype=dt_vv) + + dt_vvb = np.dtype([ + ('foo', vidt), + ('bar', vidt), + ('logical', bool)]) + vvb = f.create_dataset(make_name("dt_vvb"), shape=(2,), dtype=dt_vvb) + + dt_bvv = np.dtype([ + ('logical', bool), + ('foo', vidt), + ('bar', vidt)]) + bvv = f.create_dataset(make_name("dt_bvv"), shape=(2,), dtype=dt_bvv) + data = np.array([(True, a([1, 2, 3]), a([1, 2])), + (False, a([]), a([2, 4, 6])), ], + dtype=bvv) + bvv[:] = data + actual = bvv[:] + self.assertVlenArrayEqual(data['foo'], actual['foo']) + self.assertVlenArrayEqual(data['bar'], actual['bar']) + self.assertArrayEqual(data['logical'], actual['logical']) + + def test_compound_vlen_enum(self): + eidt = h5py.enum_dtype({'OFF': 0, 'ON': 1}, basetype=np.uint8) + vidt = h5py.vlen_dtype(np.uint8) + def a(items): + return np.array(items, dtype=np.uint8) + + f = self.f + + dt_vve = np.dtype([ + ('foo', vidt), + ('bar', vidt), + ('switch', eidt)]) + vve = f.create_dataset(make_name(), shape=(2,), dtype=dt_vve) + data = np.array([(a([1, 2, 3]), a([1, 2]), 1), + (a([]), a([2, 4, 6]), 0), ], + dtype=dt_vve) + vve[:] = data + actual = vve[:] + self.assertVlenArrayEqual(data['foo'], actual['foo']) + self.assertVlenArrayEqual(data['bar'], actual['bar']) + self.assertArrayEqual(data['switch'], actual['switch']) + + def test_vlen_enum(self): + fname = self.mktemp() + arr1 = [[1], [1, 2]] + dt1 = h5py.vlen_dtype(h5py.enum_dtype(dict(foo=1, bar=2), 'i')) + + with h5py.File(fname, 'w') as f: + df1 = f.create_dataset('test', (len(arr1),), dtype=dt1) + df1[:] = np.array(arr1, dtype=object) + + with h5py.File(fname, 'r') as f: + df2 = f['test'] + dt2 = df2.dtype + arr2 = [e.tolist() for e in df2[:]] + + self.assertEqual(arr1, arr2) + self.assertEqual(h5py.check_enum_dtype(h5py.check_vlen_dtype(dt1)), + h5py.check_enum_dtype(h5py.check_vlen_dtype(dt2))) + + +def test_write_empty_vlen(writable_file): + # vlen dtype with no entries + d = np.rec.fromarrays([[], []], names='a,b', formats='|V16,O') + dtype = [('a', '|V16'), ('b', h5py.special_dtype(vlen=np.float64))] + dset = writable_file.create_dataset(make_name(), data=d, dtype=dtype) + assert dset.size == 0 + + +def test_write_vlen_length0_compound(writable_file): + # one entry has variable length 0 (using the variable length) + # https://github.com/h5py/h5py/issues/2693 + compound_dtype = np.dtype([('id', 'i4'), ('value', 'f8'), ('name', 'S10')]) + vlen_compound_dtype = h5py.special_dtype(vlen=compound_dtype) + arr0 = np.array([(1, 3.14, b'test1'), (2, 2.71, b'test2')], dtype=compound_dtype) + arr1 = np.array([], dtype=compound_dtype) + + dset = writable_file.create_dataset( + make_name(), shape=(2,), dtype=vlen_compound_dtype + ) + dset[0] = arr0 + dset[1] = arr1 + + np.testing.assert_array_equal(dset[0], arr0) # With data + np.testing.assert_array_equal(dset[1], arr1) # Without data + + +class TestExplicitCast(TestCase): + def test_f2_casting(self): + fname = self.mktemp() + + np.random.seed(1) + A = np.random.rand(1500, 20) + + # Save to HDF5 file + with h5py.File(fname, "w") as Fid: + Fid.create_dataset("Data", data=A, dtype='f2') + + with h5py.File(fname, "r") as Fid: + B = Fid["Data"][:] + + # Compare + self.assertTrue(np.all(A.astype('f2') == B)) + + +class TestOffsets(TestCase): + """ + Check that compound members with aligned or manual offsets are handled + correctly. + """ + + def test_compound_vlen(self): + vidt = h5py.vlen_dtype(np.uint8) + eidt = h5py.enum_dtype({'OFF': 0, 'ON': 1}, basetype=np.uint8) + + for np_align in (False, True): + dt = np.dtype([ + ('a', eidt), + ('foo', vidt), + ('bar', vidt), + ('switch', eidt)], align=np_align) + np_offsets = [dt.fields[i][1] for i in dt.names] + + for logical in (False, True): + if logical and np_align: + # Vlen types have different size in the numpy struct + self.assertRaises(TypeError, h5py.h5t.py_create, dt, + logical=logical) + else: + ht = h5py.h5t.py_create(dt, logical=logical) + offsets = [ht.get_member_offset(i) + for i in range(ht.get_nmembers())] + if np_align: + self.assertEqual(np_offsets, offsets) + + def test_aligned_offsets(self): + dt = np.dtype('i4,i8,i2', align=True) + ht = h5py.h5t.py_create(dt) + self.assertEqual(dt.itemsize, ht.get_size()) + self.assertEqual( + [dt.fields[i][1] for i in dt.names], + [ht.get_member_offset(i) for i in range(ht.get_nmembers())] + ) + + def test_aligned_data(self): + dt = np.dtype('i4,f8,i2', align=True) + data = np.zeros(10, dtype=dt) + + data['f0'] = np.array(np.random.randint(-100, 100, size=data.size), + dtype='i4') + data['f1'] = np.random.rand(data.size) + data['f2'] = np.array(np.random.randint(-100, 100, size=data.size), + dtype='i2') + + fname = self.mktemp() + + with h5py.File(fname, 'w') as f: + f['data'] = data + + with h5py.File(fname, 'r') as f: + self.assertArrayEqual(f['data'], data) + + def test_compound_robustness(self): + # make an out of order compound type with gaps in it, and larger itemsize than minimum + # Idea is to be robust to type descriptions we *could* get out of HDF5 files, from custom descriptions + # of types in addition to numpy's flakey history on unaligned fields with non-standard or padded layouts. + fields = [ + ('f0', np.float64, 25), + ('f1', np.uint64, 9), + ('f2', np.uint32, 0), + ('f3', np.uint16, 5) + ] + lastfield = fields[np.argmax([ x[2] for x in fields ])] + itemsize = lastfield[2] + np.dtype(lastfield[1]).itemsize + 6 + extract_index = lambda index, sequence: [ x[index] for x in sequence ] + + dt = np.dtype({ + 'names' : extract_index(0, fields), + 'formats' : extract_index(1, fields), + 'offsets' : extract_index(2, fields), + # 'aligned': False, - already defaults to False + 'itemsize': itemsize + }) + + self.assertTrue(dt.itemsize == itemsize) + data = np.zeros(10, dtype=dt) + + # don't trust numpy struct handling, keep fields out of band in case content insertion is erroneous + # yes... this has also been known to happen. + f1 = np.array([1 + i * 4 for i in range(data.shape[0])], dtype=dt.fields['f1'][0]) + f2 = np.array([2 + i * 4 for i in range(data.shape[0])], dtype=dt.fields['f2'][0]) + f3 = np.array([3 + i * 4 for i in range(data.shape[0])], dtype=dt.fields['f3'][0]) + f0c = 3.14 + data['f0'] = f0c + data['f3'] = f3 + data['f1'] = f1 + data['f2'] = f2 + + # numpy consistency checks + self.assertTrue(np.all(data['f0'] == f0c)) + self.assertArrayEqual(data['f3'], f3) + self.assertArrayEqual(data['f1'], f1) + self.assertArrayEqual(data['f2'], f2) + + fname = self.mktemp() + + with h5py.File(fname, 'w') as fd: + fd.create_dataset('data', data=data) + + with h5py.File(fname, 'r') as fd: + readback = fd['data'] + self.assertTrue(readback.dtype == dt) + self.assertArrayEqual(readback, data) + self.assertTrue(np.all(readback['f0'] == f0c)) + self.assertArrayEqual(readback['f1'], f1) + self.assertArrayEqual(readback['f2'], f2) + self.assertArrayEqual(readback['f3'], f3) + + def test_out_of_order_offsets(self): + dt = np.dtype({ + 'names' : ['f1', 'f2', 'f3'], + 'formats' : ['<f4', '<i4', '<f8'], + 'offsets' : [0, 16, 8] + }) + data = np.zeros(10, dtype=dt) + data['f1'] = np.random.rand(data.size) + data['f2'] = np.random.randint(-10, 11, data.size) + data['f3'] = np.random.rand(data.size) * -1 + + fname = self.mktemp() + + with h5py.File(fname, 'w') as fd: + fd.create_dataset('data', data=data) + + with h5py.File(fname, 'r') as fd: + self.assertArrayEqual(fd['data'], data) + + def test_float_round_tripping(self): + dtypes = set(f for f in np.sctypeDict.values() + if (np.issubdtype(f, np.floating) or + np.issubdtype(f, np.complexfloating))) + + unsupported_types = [] + if platform.machine() in UNSUPPORTED_LONG_DOUBLE: + for x in UNSUPPORTED_LONG_DOUBLE_TYPES: + if hasattr(np, x): + unsupported_types.append(getattr(np, x)) + dtype_dset_map = {str(j): d + for j, d in enumerate(dtypes) + if d not in unsupported_types} + + fname = self.mktemp() + + with h5py.File(fname, 'w') as f: + for n, d in dtype_dset_map.items(): + data = np.zeros(10, dtype=d) + data[...] = np.arange(10) + + f.create_dataset(n, data=data) + + with h5py.File(fname, 'r') as f: + for n, d in dtype_dset_map.items(): + ldata = f[n][:] + self.assertEqual(ldata.dtype, d) + + +class TestStrings(TestCase): + def test_vlen_utf8(self): + dt = h5py.string_dtype() + + string_info = h5py.check_string_dtype(dt) + assert string_info.encoding == 'utf-8' + assert string_info.length is None + assert h5py.check_vlen_dtype(dt) is str + + def test_vlen_ascii(self): + dt = h5py.string_dtype(encoding='ascii') + + string_info = h5py.check_string_dtype(dt) + assert string_info.encoding == 'ascii' + assert string_info.length is None + assert h5py.check_vlen_dtype(dt) is bytes + + def test_fixed_utf8(self): + dt = h5py.string_dtype(length=10) + + string_info = h5py.check_string_dtype(dt) + assert string_info.encoding == 'utf-8' + assert string_info.length == 10 + assert h5py.check_vlen_dtype(dt) is None + + def test_fixed_ascii(self): + dt = h5py.string_dtype(encoding='ascii', length=10) + + string_info = h5py.check_string_dtype(dt) + assert string_info.encoding == 'ascii' + assert string_info.length == 10 + assert h5py.check_vlen_dtype(dt) is None + +class TestDateTime(TestCase): + datetime_units = [ + # Dates + 'Y', 'M', 'D', + # Times + 'h', 'm', 's', 'ms', 'us', + 'ns', 'ps', 'fs', 'as', + ] + + def test_datetime(self): + fname = self.mktemp() + + for dt_unit in self.datetime_units: + for dt_order in ['<', '>']: + dt_descr = f'{dt_order}M8[{dt_unit}]' + dt = h5py.opaque_dtype(np.dtype(dt_descr)) + arr = np.array([0], dtype=np.int64).view(dtype=dt) + + with h5py.File(fname, 'w') as f: + dset = f.create_dataset("default", data=arr, dtype=dt) + self.assertArrayEqual(arr, dset) + self.assertEqual(arr.dtype, dset.dtype) + + def test_timedelta(self): + fname = self.mktemp() + + for dt_unit in self.datetime_units: + for dt_order in ['<', '>']: + dt_descr = f'{dt_order}m8[{dt_unit}]' + dt = h5py.opaque_dtype(np.dtype(dt_descr)) + arr = np.array([np.timedelta64(500, dt_unit)], dtype=dt) + + with h5py.File(fname, 'w') as f: + dset = f.create_dataset("default", data=arr, dtype=dt) + self.assertArrayEqual(arr, dset) + self.assertEqual(arr.dtype, dset.dtype) + +@ut.skipUnless(tables is not None, 'tables is required') +class TestBitfield(TestCase): + + """ + Test H5T_NATIVE_B8 reading + """ + + def test_b8_bool(self): + arr1 = np.array([False, True], dtype=bool) + self._test_b8( + arr1, + expected_default_cast_dtype=np.uint8 + ) + self._test_b8( + arr1, + expected_default_cast_dtype=np.uint8, + cast_dtype=np.uint8 + ) + + def test_b8_bool_compound(self): + arr1 = np.array([(False,), (True,)], dtype=np.dtype([('x', '?')])) + self._test_b8( + arr1, + expected_default_cast_dtype=np.dtype([('x', 'u1')]) + ) + self._test_b8( + arr1, + expected_default_cast_dtype=np.dtype([('x', 'u1')]), + cast_dtype=np.dtype([('x', 'u1')]) + ) + + def test_b8_bool_compound_nested(self): + arr1 = np.array( + [(True, (True, False)), (True, (False, True))], + dtype=np.dtype([('x', '?'), ('y', [('a', '?'), ('b', '?')])]), + ) + self._test_b8( + arr1, + expected_default_cast_dtype=np.dtype( + [('x', 'u1'), ('y', [('a', 'u1'), ('b', 'u1')])] + ) + ) + self._test_b8( + arr1, + expected_default_cast_dtype=np.dtype( + [('x', 'u1'), ('y', [('a', 'u1'), ('b', 'u1')])] + ), + cast_dtype=np.dtype([('x', 'u1'), ('y', [('a', 'u1'), ('b', 'u1')])]), + ) + + def test_b8_bool_compound_mixed_types(self): + arr1 = np.array( + [(True, 0.5), (False, 0.2)], dtype=np.dtype([('x','?'), ('y', '<f8')]) + ) + self._test_b8( + arr1, + expected_default_cast_dtype=np.dtype([('x', 'u1'), ('y', '<f8')]) + ) + self._test_b8( + arr1, + expected_default_cast_dtype=np.dtype([('x', 'u1'), ('y', '<f8')]), + cast_dtype=np.dtype([('x', 'u1'), ('y', '<f8')]) + ) + + def test_b8_bool_array(self): + arr1 = np.array( + [((True, True, False),), ((True, False, True),)], + dtype=np.dtype([('x', ('?', (3,)))]), + ) + self._test_b8( + arr1, + expected_default_cast_dtype=np.dtype([('x', ('u1', (3,)))]) + ) + self._test_b8( + arr1, + expected_default_cast_dtype=np.dtype([('x', ('u1', (3,)))]), + cast_dtype=np.dtype([('x', ('?', (3,)))]), + ) + + def _test_b8(self, arr1, expected_default_cast_dtype, cast_dtype=None): + path = self.mktemp() + + with tables.open_file(path, 'w') as f: + if arr1.dtype.names: + f.create_table('/', 'test', obj=arr1) + else: + f.create_array('/', 'test', obj=arr1) + + with h5py.File(path, 'r') as f: + dset = f['test'] + + # This should do an implicit uint8 cast + # Expect that the "No NumPy equivalent for TypeBitfieldID exists" + # error is not thrown. + arr2 = dset[:] + + self.assertArrayEqual( + arr2, + arr1.astype(expected_default_cast_dtype, copy=False) + ) + + # read cast dset and make sure it's equal + if cast_dtype is None: + cast_dtype = arr1.dtype + arr3 = dset.astype(cast_dtype)[:] + self.assertArrayEqual(arr3, arr1.astype(cast_dtype, copy=False)) + + def test_b16_uint16(self): + arr1 = np.arange(10, dtype=np.uint16) + path = self.mktemp() + with h5py.File(path, 'w') as f: + space = h5py.h5s.create_simple(arr1.shape) + dset_id = h5py.h5d.create(f.id, b'test', h5py.h5t.STD_B16LE, space) + dset = h5py.Dataset(dset_id) + dset[:] = arr1 + + with h5py.File(path, 'r') as f: + dset = f['test'] + self.assertArrayEqual(dset[:], arr1) + +def test_opaque(writable_file): + # opaque without an h5py tag corresponds to numpy void dtypes + arr = np.zeros(3, dtype='V2') + ds = writable_file.create_dataset(make_name(), data=arr) + assert isinstance(ds.id.get_type(), h5py.h5t.TypeOpaqueID) + assert ds.id.get_type().get_size() == 2 + np.testing.assert_array_equal(ds[:], arr) + + +def test_create_bitfield(writable_file): + data = np.arange(12, dtype=np.uint16) + ds = writable_file.create_dataset( + make_name("x"), dtype=h5py.Datatype(h5py.h5t.STD_B16LE), shape=(12,) + ) + ds[:] = data + assert isinstance(ds.id.get_type(), h5py.h5t.TypeBitfieldID) + assert ds.id.get_type().get_size() == 2 + np.testing.assert_array_equal(ds[:], data) + + ds2 = writable_file.create_dataset( + make_name("y"), dtype=h5py.h5t.STD_B16LE, shape=(12,) + ) + ds2[:] = data + assert isinstance(ds2.id.get_type(), h5py.h5t.TypeBitfieldID) + assert ds2.id.get_type().get_size() == 2 + np.testing.assert_array_equal(ds2[:], data) + + +def test_complex_compat(writable_file): + dt = h5py.complex_compat_dtype(np.complex64) + assert dt.names == ('r', 'i') + assert dt['r'] == np.dtype('=f4') + assert dt['i'] == np.dtype('=f4') + assert h5py.check_complex_dtype(dt) == np.dtype('=c8') + + dt2 = h5py.complex_compat_dtype(np.dtype('>c16')) + assert dt2['r'] == np.dtype('>f8') + assert dt2['i'] == np.dtype('>f8') + assert h5py.check_complex_dtype(dt2) == np.dtype('>c16') + + ds = writable_file.create_dataset(make_name(), shape=(10,), dtype=dt) + assert ds.dtype == np.dtype('c8') # Recognised as complex + ds[:] = arr = np.arange(10, dtype='c8') + np.testing.assert_array_equal(ds[:], arr) + + assert h5py.check_complex_dtype(np.dtype('<c16')) == np.dtype('<c16') diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_errors.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_errors.py new file mode 100644 index 0000000000000000000000000000000000000000..e599b0799073d07e28721227566ec306302eb97e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_errors.py @@ -0,0 +1,89 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Tests the h5py.File object. +""" + +import threading + +import pytest + +import h5py +from .common import make_name + + +def _access_not_existing_object(filename): + """Create a file and access not existing key""" + with h5py.File(filename, 'w') as newfile: + try: + doesnt_exist = newfile['doesnt_exist'].value + except KeyError: + pass + + +@pytest.mark.thread_unsafe(reason="Changes global state") +def test_unsilence_errors(tmp_path, capfd): + """Check that HDF5 errors can be muted/unmuted from h5py""" + filename = tmp_path / 'test.h5' + + # Unmute HDF5 errors + try: + h5py._errors.unsilence_errors() + + _access_not_existing_object(filename) + captured = capfd.readouterr() + assert captured.err != '' + assert captured.out == '' + + # Mute HDF5 errors + finally: + h5py._errors.silence_errors() + + _access_not_existing_object(filename) + captured = capfd.readouterr() + assert captured.err == '' + assert captured.out == '' + + +def test_thread_hdf5_silence_error_membership(tmp_path, capfd): + """Verify the error printing is squashed in all threads. + + No console messages should be shown from membership tests + """ + th = threading.Thread(target=_access_not_existing_object, + args=(tmp_path / make_name("{}.h5"),)) + th.start() + th.join() + + captured = capfd.readouterr() + assert captured.err == '' + assert captured.out == '' + + +def test_thread_hdf5_silence_error_attr(tmp_path, capfd): + """Verify the error printing is squashed in all threads. + + No console messages should be shown for non-existing attributes + """ + def test(): + with h5py.File(tmp_path/make_name("{}.h5"), 'w') as newfile: + newfile['newdata'] = [1, 2, 3] + try: + nonexistent_attr = newfile['newdata'].attrs['nonexistent_attr'] + except KeyError: + pass + + th = threading.Thread(target=test) + th.start() + th.join() + + captured = capfd.readouterr() + assert captured.err == '' + assert captured.out == '' diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_file.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_file.py new file mode 100644 index 0000000000000000000000000000000000000000..49b51f71ac2ca3f4ec9108fc66ce7b9b14208bce --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_file.py @@ -0,0 +1,1101 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + File object test module. + + Tests all aspects of File objects, including their creation. +""" + +import multiprocessing +import os +import stat +import pickle +import tempfile +import time +import sys +from concurrent.futures import ProcessPoolExecutor +from hashlib import sha256 + +import pytest + +from .common import ut, TestCase, UNICODE_FILENAMES, closed_tempfile, make_name +from h5py._hl.files import direct_vfd +from h5py import File +import h5py +import pathlib + + +class TestFileOpen(TestCase): + + """ + Feature: Opening files with Python-style modes. + """ + + def test_default(self): + """ Default semantics in the presence or absence of a file """ + fname = self.mktemp() + + # No existing file; error + with pytest.raises(FileNotFoundError): + with File(fname): + pass + + # Existing readonly file; open read-only + with File(fname, 'w'): + pass + os.chmod(fname, stat.S_IREAD) + try: + with File(fname) as f: + self.assertTrue(f) + self.assertEqual(f.mode, 'r') + finally: + os.chmod(fname, stat.S_IWRITE) + + # File exists but is not HDF5; raise OSError + with open(fname, 'wb') as f: + f.write(b'\x00') + with self.assertRaises(OSError): + File(fname) + + def test_create(self): + """ Mode 'w' opens file in overwrite mode """ + fname = self.mktemp() + fid = File(fname, 'w') + self.assertTrue(fid) + fid.create_group('foo') + fid.close() + fid = File(fname, 'w') + self.assertNotIn('foo', fid) + fid.close() + + def test_create_exclusive(self): + """ Mode 'w-' opens file in exclusive mode """ + fname = self.mktemp() + fid = File(fname, 'w-') + self.assertTrue(fid) + fid.close() + with self.assertRaises(FileExistsError): + File(fname, 'w-') + + def test_append(self): + """ Mode 'a' opens file in append/readwrite mode, creating if necessary """ + fname = self.mktemp() + fid = File(fname, 'a') + try: + self.assertTrue(fid) + fid.create_group('foo') + assert 'foo' in fid + finally: + fid.close() + fid = File(fname, 'a') + try: + assert 'foo' in fid + fid.create_group('bar') + assert 'bar' in fid + finally: + fid.close() + + # Observed on cibuildwheel v2.19.1 + # https://github.com/pypa/cibuildwheel/issues/1882 + @pytest.mark.skipif( + os.getenv("CIBUILDWHEEL") == "1" and sys.platform == "linux", + reason="Linux docker cibuildwheel environment permissions issue", + ) + def test_append_permissions(self): + """ Mode 'a' fails when file is read-only """ + fname = self.mktemp() + with File(fname, 'a') as fid: + fid.create_group('foo') + os.chmod(fname, stat.S_IREAD) # Make file read-only + try: + with pytest.raises(PermissionError): + File(fname, 'a') + finally: + # Make it writable again so it can be deleted on Windows + os.chmod(fname, stat.S_IREAD | stat.S_IWRITE) + + def test_readonly(self): + """ Mode 'r' opens file in readonly mode """ + fname = self.mktemp() + fid = File(fname, 'w') + fid.close() + self.assertFalse(fid) + fid = File(fname, 'r') + self.assertTrue(fid) + with self.assertRaises(ValueError): + fid.create_group('foo') + fid.close() + + def test_readwrite(self): + """ Mode 'r+' opens existing file in readwrite mode """ + fname = self.mktemp() + fid = File(fname, 'w') + fid.create_group('foo') + fid.close() + fid = File(fname, 'r+') + assert 'foo' in fid + fid.create_group('bar') + assert 'bar' in fid + fid.close() + + def test_nonexistent_file(self): + """ Modes 'r' and 'r+' do not create files """ + fname = self.mktemp() + with self.assertRaises(FileNotFoundError): + File(fname, 'r') + with self.assertRaises(FileNotFoundError): + File(fname, 'r+') + + def test_invalid_mode(self): + """ Invalid modes raise ValueError """ + with self.assertRaises(ValueError): + File(self.mktemp(), 'mongoose') + + +class TestSpaceStrategy(TestCase): + + """ + Feature: Create file with specified file space strategy + """ + + def test_create_with_space_strategy(self): + """ Create file with file space strategy """ + fname = self.mktemp() + fid = File(fname, 'w', fs_strategy="page", + fs_persist=True, fs_threshold=100) + self.assertTrue(fid) + # Unable to set file space strategy of an existing file + with self.assertRaises(ValueError): + File(fname, 'a', fs_strategy="page") + # Invalid file space strategy type + with self.assertRaises(ValueError): + File(self.mktemp(), 'w', fs_strategy="invalid") + + x = make_name("x") + y = make_name("y") + z = make_name("z") + + dset = fid.create_dataset(x, (100,), dtype='uint8') + dset[...] = 1 + dset = fid.create_dataset(y, (100,), dtype='uint8') + dset[...] = 1 + del fid[x] + fid.close() + + fid = File(fname, 'a') + plist = fid.id.get_create_plist() + fs_strat = plist.get_file_space_strategy() + assert(fs_strat[0] == 1) + assert(fs_strat[1] == True) + assert(fs_strat[2] == 100) + + dset = fid.create_dataset(z, (100,), dtype='uint8') + dset[...] = 1 + fid.close() + + +@pytest.mark.mpi_skip +class TestPageBuffering(TestCase): + """ + Feature: Use page buffering + """ + + def test_only_with_page_strategy(self): + """Allow page buffering only with fs_strategy="page". + """ + fname = self.mktemp() + with File(fname, mode='w', fs_strategy='page', page_buf_size=16*1024): + pass + with self.assertRaises(OSError): + File(fname, mode='w', page_buf_size=16*1024) + with self.assertRaises(OSError): + File(fname, mode='w', fs_strategy='fsm', page_buf_size=16*1024) + with self.assertRaises(OSError): + File(fname, mode='w', fs_strategy='aggregate', page_buf_size=16*1024) + + def test_check_page_buf_size(self): + """Verify set page buffer size, and minimum meta and raw eviction criteria.""" + fname = self.mktemp() + pbs = 16 * 1024 + mm = 19 + mr = 67 + with File(fname, mode='w', fs_strategy='page', + page_buf_size=pbs, min_meta_keep=mm, min_raw_keep=mr) as f: + fapl = f.id.get_access_plist() + self.assertEqual(fapl.get_page_buffer_size(), (pbs, mm, mr)) + + @pytest.mark.skipif(h5py.version.hdf5_version_tuple > (1, 14, 3), + reason='Requires HDF5 <= 1.14.3') + def test_too_small_pbs(self): + """Page buffer size must be greater than file space page size.""" + fname = self.mktemp() + fsp = 16 * 1024 + with File(fname, mode='w', fs_strategy='page', fs_page_size=fsp): + pass + with self.assertRaises(OSError): + File(fname, mode="r", page_buf_size=fsp-1) + + @pytest.mark.skipif(h5py.version.hdf5_version_tuple < (1, 14, 4), + reason='Requires HDF5 >= 1.14.4') + def test_open_nonpage_pbs(self): + """Open non-PAGE file with page buffer set.""" + fname = self.mktemp() + fsp = 16 * 1024 + with File(fname, mode='w'): + pass + with File(fname, mode='r', page_buf_size=fsp) as f: + fapl = f.id.get_access_plist() + assert fapl.get_page_buffer_size()[0] == 0 + + @pytest.mark.skipif(h5py.version.hdf5_version_tuple < (1, 14, 4), + reason='Requires HDF5 >= 1.14.4') + def test_smaller_pbs(self): + """Adjust page buffer size automatically when smaller than file page.""" + fname = self.mktemp() + fsp = 16 * 1024 + with File(fname, mode='w', fs_strategy='page', fs_page_size=fsp): + pass + with File(fname, mode='r', page_buf_size=fsp-100) as f: + fapl = f.id.get_access_plist() + assert fapl.get_page_buffer_size()[0] == fsp + + def test_actual_pbs(self): + """Verify actual page buffer size.""" + fname = self.mktemp() + fsp = 16 * 1024 + pbs = 2 * fsp + with File(fname, mode='w', fs_strategy='page', fs_page_size=fsp): + pass + with File(fname, mode='r', page_buf_size=pbs-1) as f: + fapl = f.id.get_access_plist() + self.assertEqual(fapl.get_page_buffer_size()[0], fsp) + + +class TestModes(TestCase): + + """ + Feature: File mode can be retrieved via file.mode + """ + + def test_mode_attr(self): + """ Mode equivalent can be retrieved via property """ + fname = self.mktemp() + with File(fname, 'w') as f: + self.assertEqual(f.mode, 'r+') + with File(fname, 'r') as f: + self.assertEqual(f.mode, 'r') + + def test_mode_external(self): + """ Mode property works for files opened via external links + + Issue 190. + """ + fname1 = self.mktemp() + fname2 = self.mktemp() + + f1 = File(fname1, 'w') + f1.close() + + f2 = File(fname2, 'w') + try: + f2['External'] = h5py.ExternalLink(fname1, '/') + f3 = f2['External'].file + self.assertEqual(f3.mode, 'r+') + finally: + f2.close() + f3.close() + + f2 = File(fname2, 'r') + try: + f3 = f2['External'].file + self.assertEqual(f3.mode, 'r') + finally: + f2.close() + f3.close() + + +class TestDrivers(TestCase): + + """ + Feature: Files can be opened with low-level HDF5 drivers. Does not + include MPI drivers (see bottom). + """ + + @ut.skipUnless(os.name == 'posix', "Stdio driver is supported on posix") + def test_stdio(self): + """ Stdio driver is supported on posix """ + fid = File(self.mktemp(), 'w', driver='stdio') + self.assertTrue(fid) + self.assertEqual(fid.driver, 'stdio') + fid.close() + + # Testing creation with append flag + fid = File(self.mktemp(), 'a', driver='stdio') + self.assertTrue(fid) + self.assertEqual(fid.driver, 'stdio') + fid.close() + + @ut.skipUnless(direct_vfd, + "DIRECT driver is supported on Linux if hdf5 is " + "built with the appriorate flags.") + def test_direct(self): + """ DIRECT driver is supported on Linux""" + fid = File(self.mktemp(), 'w', driver='direct') + self.assertTrue(fid) + self.assertEqual(fid.driver, 'direct') + default_fapl = fid.id.get_access_plist().get_fapl_direct() + fid.close() + + # Testing creation with append flag + fid = File(self.mktemp(), 'a', driver='direct') + self.assertTrue(fid) + self.assertEqual(fid.driver, 'direct') + fid.close() + + # 2022/02/26: hnmaarrfk + # I'm actually not too sure of the restriction on the + # different valid block_sizes and cbuf_sizes on different hardware + # platforms. + # + # I've learned a few things: + # * cbuf_size: Copy buffer size must be a multiple of block size + # The alignment (on my platform x86-64bit with an NVMe SSD + # could be an integer multiple of 512 + # + # To allow HDF5 to do the heavy lifting for different platform, + # We didn't provide any arguments to the first call + # and obtained HDF5's default values there. + + # Testing creation with a few different property lists + for alignment, block_size, cbuf_size in [ + default_fapl, + (default_fapl[0], default_fapl[1], 3 * default_fapl[1]), + (default_fapl[0] * 2, default_fapl[1], 3 * default_fapl[1]), + (default_fapl[0], 2 * default_fapl[1], 6 * default_fapl[1]), + ]: + with File(self.mktemp(), 'w', driver='direct', + alignment=alignment, + block_size=block_size, + cbuf_size=cbuf_size) as fid: + actual_fapl = fid.id.get_access_plist().get_fapl_direct() + actual_alignment = actual_fapl[0] + actual_block_size = actual_fapl[1] + actual_cbuf_size = actual_fapl[2] + assert actual_alignment == alignment + assert actual_block_size == block_size + assert actual_cbuf_size == actual_cbuf_size + + @ut.skipUnless(os.name == 'posix', "Sec2 driver is supported on posix") + def test_sec2(self): + """ Sec2 driver is supported on posix """ + fid = File(self.mktemp(), 'w', driver='sec2') + self.assertTrue(fid) + self.assertEqual(fid.driver, 'sec2') + fid.close() + + # Testing creation with append flag + fid = File(self.mktemp(), 'a', driver='sec2') + self.assertTrue(fid) + self.assertEqual(fid.driver, 'sec2') + fid.close() + + def test_core(self): + """ Core driver is supported (no backing store) """ + fname = self.mktemp() + fid = File(fname, 'w', driver='core', backing_store=False) + self.assertTrue(fid) + self.assertEqual(fid.driver, 'core') + fid.close() + self.assertFalse(os.path.exists(fname)) + + # Testing creation with append flag + fid = File(self.mktemp(), 'a', driver='core') + self.assertTrue(fid) + self.assertEqual(fid.driver, 'core') + fid.close() + + def test_backing(self): + """ Core driver saves to file when backing store used """ + fname = self.mktemp() + fid = File(fname, 'w', driver='core', backing_store=True) + fid.create_group('foo') + fid.close() + fid = File(fname, 'r') + assert 'foo' in fid + fid.close() + # keywords for other drivers are invalid when using the default driver + with self.assertRaises(TypeError): + File(fname, 'w', backing_store=True) + + def test_readonly(self): + """ Core driver can be used to open existing files """ + fname = self.mktemp() + fid = File(fname, 'w') + fid.create_group('foo') + fid.close() + fid = File(fname, 'r', driver='core') + self.assertTrue(fid) + assert 'foo' in fid + with self.assertRaises(ValueError): + fid.create_group('bar') + fid.close() + + def test_blocksize(self): + """ Core driver supports variable block size """ + fname = self.mktemp() + fid = File(fname, 'w', driver='core', block_size=1024, + backing_store=False) + self.assertTrue(fid) + fid.close() + + def test_split(self): + """ Split stores metadata in a separate file """ + fname = self.mktemp() + fid = File(fname, 'w', driver='split') + fid.close() + self.assertTrue(os.path.exists(fname + '-m.h5')) + fid = File(fname, 'r', driver='split') + self.assertTrue(fid) + fid.close() + + def test_fileobj(self): + """ Python file object driver is supported """ + tf = tempfile.TemporaryFile() + fid = File(tf, 'w', driver='fileobj') + self.assertTrue(fid) + self.assertEqual(fid.driver, 'fileobj') + fid.close() + # Driver must be 'fileobj' for file-like object if specified + with self.assertRaises(ValueError): + File(tf, 'w', driver='core') + tf.close() + + # TODO: family driver tests + + +@pytest.mark.skipif( + h5py.version.hdf5_version_tuple[0] == 1 and + h5py.version.hdf5_version_tuple[1] % 2 != 0 , + reason='Not HDF5 release version 1.x.y' +) +class TestNewLibver(TestCase): + + """ + Feature: File format compatibility bounds can be specified when + opening a file. + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + + # Current latest library bound label + if h5py.version.hdf5_version_tuple < (1, 11, 4): + cls.latest = 'v110' + elif h5py.version.hdf5_version_tuple < (1, 13, 0): + cls.latest = 'v112' + elif h5py.version.hdf5_version_tuple < (2, 0, 0): + cls.latest = 'v114' + else: + cls.latest = 'v200' + + def test_default(self): + """ Opening with no libver arg """ + f = File(self.mktemp(), 'w') + self.assertEqual(f.libver, ('earliest', self.latest)) + f.close() + + def test_single(self): + """ Opening with single libver arg """ + f = File(self.mktemp(), 'w', libver='latest') + self.assertEqual(f.libver, (self.latest, self.latest)) + f.close() + + def test_single_v108(self): + """ Opening with "v108" libver arg """ + f = File(self.mktemp(), 'w', libver='v108') + self.assertEqual(f.libver, ('v108', self.latest)) + f.close() + + def test_single_v110(self): + """ Opening with "v110" libver arg """ + f = File(self.mktemp(), 'w', libver='v110') + self.assertEqual(f.libver, ('v110', self.latest)) + f.close() + + @ut.skipIf(h5py.version.hdf5_version_tuple < (1, 11, 4), + 'Requires HDF5 1.11.4 or later') + def test_single_v112(self): + """ Opening with "v112" libver arg """ + f = File(self.mktemp(), 'w', libver='v112') + self.assertEqual(f.libver, ('v112', self.latest)) + f.close() + + @ut.skipIf(h5py.version.hdf5_version_tuple < (1, 14, 0), + 'Requires HDF5 1.14 or later') + def test_single_v114(self): + """ Opening with "v114" libver arg """ + f = File(self.mktemp(), 'w', libver='v114') + self.assertEqual(f.libver, ('v114', self.latest)) + f.close() + + @ut.skipIf(h5py.version.hdf5_version_tuple < (2, 0, 0), + 'Requires HDF5 2.0 or later') + def test_single_v200(self): + """ Opening with "v200" libver arg """ + f = File(self.mktemp(), 'w', libver='v200') + self.assertEqual(f.libver, ('v200', self.latest)) + f.close() + + def test_multiple(self): + """ Opening with two libver args """ + f = File(self.mktemp(), 'w', libver=('earliest', 'v108')) + self.assertEqual(f.libver, ('earliest', 'v108')) + f.close() + + def test_none(self): + """ Omitting libver arg results in maximum compatibility """ + f = File(self.mktemp(), 'w') + self.assertEqual(f.libver, ('earliest', self.latest)) + f.close() + + +class TestUserblock(TestCase): + + """ + Feature: Files can be create with user blocks + """ + + def test_create_blocksize(self): + """ User blocks created with w, w-, x and properties work correctly """ + f = File(self.mktemp(), 'w-', userblock_size=512) + try: + self.assertEqual(f.userblock_size, 512) + finally: + f.close() + + f = File(self.mktemp(), 'x', userblock_size=512) + try: + self.assertEqual(f.userblock_size, 512) + finally: + f.close() + + f = File(self.mktemp(), 'w', userblock_size=512) + try: + self.assertEqual(f.userblock_size, 512) + finally: + f.close() + # User block size must be an integer + with self.assertRaises(ValueError): + File(self.mktemp(), 'w', userblock_size='non') + + def test_write_only(self): + """ User block only allowed for write """ + name = self.mktemp() + f = File(name, 'w') + f.close() + + with self.assertRaises(ValueError): + f = h5py.File(name, 'r', userblock_size=512) + + with self.assertRaises(ValueError): + f = h5py.File(name, 'r+', userblock_size=512) + + def test_match_existing(self): + """ User block size must match that of file when opening for append """ + name = self.mktemp() + f = File(name, 'w', userblock_size=512) + f.close() + + with self.assertRaises(ValueError): + f = File(name, 'a', userblock_size=1024) + + f = File(name, 'a', userblock_size=512) + try: + self.assertEqual(f.userblock_size, 512) + finally: + f.close() + + def test_power_of_two(self): + """ User block size must be a power of 2 and at least 512 """ + name = self.mktemp() + + with self.assertRaises(ValueError): + f = File(name, 'w', userblock_size=128) + + with self.assertRaises(ValueError): + f = File(name, 'w', userblock_size=513) + + with self.assertRaises(ValueError): + f = File(name, 'w', userblock_size=1023) + + def test_write_block(self): + """ Test that writing to a user block does not destroy the file """ + name = self.mktemp() + + f = File(name, 'w', userblock_size=512) + f.create_group("Foobar") + f.close() + + pyfile = open(name, 'r+b') + try: + pyfile.write(b'X' * 512) + finally: + pyfile.close() + + f = h5py.File(name, 'r') + try: + assert "Foobar" in f + finally: + f.close() + + pyfile = open(name, 'rb') + try: + self.assertEqual(pyfile.read(512), b'X' * 512) + finally: + pyfile.close() + + +class TestContextManager(TestCase): + + """ + Feature: File objects can be used as context managers + """ + + def test_context_manager(self): + """ File objects can be used in with statements """ + with File(self.mktemp(), 'w') as fid: + self.assertTrue(fid) + self.assertTrue(not fid) + + +@ut.skipIf(not UNICODE_FILENAMES, "Filesystem unicode support required") +class TestUnicode(TestCase): + + """ + Feature: Unicode filenames are supported + """ + + def test_unicode(self): + """ Unicode filenames can be used, and retrieved properly via .filename + """ + fname = self.mktemp(prefix=chr(0x201a)) + fid = File(fname, 'w') + try: + self.assertEqual(fid.filename, fname) + self.assertIsInstance(fid.filename, str) + finally: + fid.close() + + def test_unicode_hdf5_python_consistent(self): + """ Unicode filenames can be used, and seen correctly from python + """ + fname = self.mktemp(prefix=chr(0x201a)) + with File(fname, 'w'): + pass + assert os.path.exists(fname) + + def test_nonexistent_file_unicode(self): + """ + Modes 'r' and 'r+' do not create files even when given unicode names + """ + fname = self.mktemp(prefix=chr(0x201a)) + with self.assertRaises(OSError): + File(fname, 'r') + with self.assertRaises(OSError): + File(fname, 'r+') + + +class TestFileProperty(TestCase): + + """ + Feature: A File object can be retrieved from any child object, + via the .file property + """ + + def test_property(self): + """ File object can be retrieved from subgroup """ + fname = self.mktemp() + hfile = File(fname, 'w') + try: + hfile2 = hfile['/'].file + self.assertEqual(hfile, hfile2) + finally: + hfile.close() + + def test_close(self): + """ All retrieved File objects are closed at the same time """ + fname = self.mktemp() + hfile = File(fname, 'w') + grp = hfile.create_group('foo') + hfile2 = grp.file + hfile3 = hfile['/'].file + hfile2.close() + self.assertFalse(hfile) + self.assertFalse(hfile2) + self.assertFalse(hfile3) + + def test_mode(self): + """ Retrieved File objects have a meaningful mode attribute """ + hfile = File(self.mktemp(), 'w') + try: + grp = hfile.create_group('foo') + self.assertEqual(grp.file.mode, hfile.mode) + finally: + hfile.close() + + +class TestClose(TestCase): + + """ + Feature: Files can be closed + """ + + def test_close(self): + """ Close file via .close method """ + fid = File(self.mktemp(), 'w') + self.assertTrue(fid) + fid.close() + self.assertFalse(fid) + + def test_closed_file(self): + """ Trying to modify closed file raises ValueError """ + fid = File(self.mktemp(), 'w') + fid.close() + with self.assertRaises(ValueError): + fid.create_group('foo') + + def test_close_multiple_default_driver(self): + fname = self.mktemp() + f = h5py.File(fname, 'w') + f.create_group("test") + f.close() + f.close() + + +class TestFlush(TestCase): + + """ + Feature: Files can be flushed + """ + + def test_flush(self): + """ Flush via .flush method """ + fid = File(self.mktemp(), 'w') + fid.flush() + fid.close() + + +class TestRepr(TestCase): + + """ + Feature: File objects provide a helpful __repr__ string + """ + + def test_repr(self): + """ __repr__ behaves itself when files are open and closed """ + fid = File(self.mktemp(), 'w') + self.assertIsInstance(repr(fid), str) + fid.close() + self.assertIsInstance(repr(fid), str) + + +class TestFilename(TestCase): + + """ + Feature: The name of a File object can be retrieved via .filename + """ + + def test_filename(self): + """ .filename behaves properly for string data """ + fname = self.mktemp() + fid = File(fname, 'w') + try: + self.assertEqual(fid.filename, fname) + self.assertIsInstance(fid.filename, str) + finally: + fid.close() + + +class TestCloseInvalidatesOpenObjectIDs(TestCase): + + """ + Ensure that closing a file invalidates object IDs, as appropriate + """ + + def test_close(self): + """ Closing a file invalidates any of the file's open objects """ + with File(self.mktemp(), 'w') as f1: + g1 = f1.create_group('foo') + self.assertTrue(bool(f1.id)) + self.assertTrue(bool(g1.id)) + f1.close() + self.assertFalse(bool(f1.id)) + self.assertFalse(bool(g1.id)) + with File(self.mktemp(), 'w') as f2: + g2 = f2.create_group('foo') + self.assertTrue(bool(f2.id)) + self.assertTrue(bool(g2.id)) + self.assertFalse(bool(f1.id)) + self.assertFalse(bool(g1.id)) + + def test_close_one_handle(self): + fname = self.mktemp() + with File(fname, 'w') as f: + f.create_group('foo') + + f1 = File(fname) + f2 = File(fname) + g1 = f1['foo'] + g2 = f2['foo'] + assert g1.id.valid + assert g2.id.valid + f1.close() + assert not g1.id.valid + # Closing f1 shouldn't close f2 or objects belonging to it + assert f2.id.valid + assert g2.id.valid + + f2.close() + assert not f2.id.valid + assert not g2.id.valid + + +class TestPathlibSupport(TestCase): + + """ + Check that h5py doesn't break on pathlib + """ + def test_pathlib_accepted_file(self): + """ Check that pathlib is accepted by h5py.File """ + with closed_tempfile() as f: + path = pathlib.Path(f) + with File(path, 'w') as f2: + self.assertTrue(True) + + def test_pathlib_name_match(self): + """ Check that using pathlib does not affect naming """ + with closed_tempfile() as f: + path = pathlib.Path(f) + with File(path, 'w') as h5f1: + pathlib_name = h5f1.filename + with File(f, 'w') as h5f2: + normal_name = h5f2.filename + self.assertEqual(pathlib_name, normal_name) + + +class TestPickle(TestCase): + """Check that h5py.File can't be pickled""" + def test_dump_error(self): + with File(self.mktemp(), 'w') as f1: + with self.assertRaises(TypeError): + pickle.dumps(f1) + + +# unittest doesn't work with pytest fixtures (and possibly other features), +# hence no subclassing TestCase +@pytest.mark.mpi +class TestMPI: + def test_mpio(self, mpi_file_name): + """ MPIO driver and options """ + from mpi4py import MPI + + with File(mpi_file_name, 'w', driver='mpio', comm=MPI.COMM_WORLD) as f: + assert f + assert f.driver == 'mpio' + + def test_mpio_append(self, mpi_file_name): + """ Testing creation of file with append """ + from mpi4py import MPI + + with File(mpi_file_name, 'a', driver='mpio', comm=MPI.COMM_WORLD) as f: + assert f + assert f.driver == 'mpio' + + def test_mpi_atomic(self, mpi_file_name): + """ Enable atomic mode for MPIO driver """ + from mpi4py import MPI + + with File(mpi_file_name, 'w', driver='mpio', comm=MPI.COMM_WORLD) as f: + assert not f.atomic + f.atomic = True + assert f.atomic + + def test_close_multiple_mpio_driver(self, mpi_file_name): + """ MPIO driver and options """ + from mpi4py import MPI + + f = File(mpi_file_name, 'w', driver='mpio', comm=MPI.COMM_WORLD) + f.create_group(make_name()) + f.close() + f.close() + + +class TestSWMRMode(TestCase): + + """ + Feature: Create file that switches on SWMR mode + """ + + def test_file_mode_generalizes(self): + fname = self.mktemp() + fid = File(fname, 'w', libver='latest') + g = fid.create_group('foo') + # fid and group member file attribute should have the same mode + assert fid.mode == g.file.mode == 'r+' + fid.swmr_mode = True + # fid and group member file attribute should still be 'r+' + # even though file intent has changed + assert fid.mode == g.file.mode == 'r+' + fid.close() + + def test_swmr_mode_consistency(self): + fname = self.mktemp() + fid = File(fname, 'w', libver='latest') + g = fid.create_group('foo') + assert fid.swmr_mode == g.file.swmr_mode == False + fid.swmr_mode = True + # This setter should affect both fid and group member file attribute + assert fid.swmr_mode == g.file.swmr_mode == True + fid.close() + + +@pytest.mark.skipif("HDF5_USE_FILE_LOCKING" in os.environ, + reason="HDF5_USE_FILE_LOCKING env. var. is set") +class TestFileLocking: + """Test h5py.File file locking option""" + + def test_reopen(self, tmp_path): + """Test file locking when opening twice the same file""" + fname = tmp_path / make_name("test{}.h5") + + with h5py.File(fname, mode="w", locking=True) as f: + f.flush() + + # Opening same file in same process without locking is expected to fail + with pytest.raises(OSError): + with h5py.File(fname, mode="r", locking=False) as h5f_read: + pass + + with h5py.File(fname, mode="r", locking=True) as h5f_read: + pass + + if h5py.version.hdf5_version_tuple < (1, 14, 4): + with h5py.File(fname, mode="r", locking='best-effort') as h5f_read: + pass + else: + with pytest.raises(OSError): + with h5py.File(fname, mode="r", locking='best-effort') as h5f_read: + pass + + + def test_unsupported_locking(self, tmp_path): + """Test with erroneous file locking value""" + fname = tmp_path / make_name("test{}.h5") + with pytest.raises(ValueError): + with h5py.File(fname, mode="r", locking='unsupported-value') as h5f_read: + pass + + def test_multiprocess(self, tmp_path): + """Test file locking option from different concurrent processes""" + fname = tmp_path / make_name("test{}.h5") + + # Create test file + with File(fname, mode="w", locking=True) as f: + f["data"] = 1 + + ctx = multiprocessing.get_context("spawn") + with ( + ProcessPoolExecutor(mp_context=ctx, max_workers=1) as ex, + File(fname, mode="r", locking=False) as f, + ): + # Opening in write mode with locking is expected to work + future = ex.submit(open_and_close, fname, mode="w", locking=True) + future.result(timeout=10) + + +def open_and_close(*args, **kwargs): + """Open and close HDF5 file, for use in a subprocess""" + with File(*args, **kwargs): + pass + + +@pytest.mark.skipif( + h5py.version.hdf5_version_tuple < (1, 14, 4), + reason="Requires HDF5 >= 1.14.4", +) +@pytest.mark.skipif( + "HDF5_USE_FILE_LOCKING" in os.environ, + reason="HDF5_USE_FILE_LOCKING env. var. is set", +) +@pytest.mark.parametrize( + 'locking_arg,file_locking_props', + [ + (False, (0, 0)), + (True, (1, 0)), + ('best-effort', (1, 1)), + ] +) +def test_file_locking_external_link(tmp_path, locking_arg, file_locking_props): + """Test that same file locking is used for external link""" + fname_main = tmp_path / make_name("test_main{}.h5") + fname_elink = tmp_path / make_name("test_linked{}.h5") + + # Create test files + with h5py.File(fname_elink, "w") as f: + f["data"] = 1 + with h5py.File(fname_main, "w") as f: + f["link"] = h5py.ExternalLink(fname_elink, "/data") + + with h5py.File(fname_main, "r", locking=locking_arg) as f: + locking_info = f.id.get_access_plist().get_file_locking() + assert locking_info == file_locking_props + + # Test that external link file is also opened without file locking + elink_dataset = f["link"] + elink_locking_info = elink_dataset.file.id.get_access_plist().get_file_locking() + assert elink_locking_info == file_locking_props + + +def test_close_gc(tmp_path): + # https://github.com/h5py/h5py/issues/1852 + filename = tmp_path / make_name("test{}.h5") + + with h5py.File(filename, 'w') as f: + for i in range(100): + f[str(i)] = [] + + # Ensure that Python's garbage collection doesn't interfere with closing + # a file. Try a few times - the problem is not 100% consistent, but + # normally showed up on the 1st or 2nd iteration for me. -TAK, 2021 + for i in range(10): + with h5py.File(filename, 'r') as f: + refs = [d.id for d in f.values()] + refs.append(refs) # Make a reference cycle so GC is involved + del refs # GC is likely to fire while closing the file + + +@pytest.mark.slow +def test_reproducible_file(tmp_path): + x = make_name("x") + y = make_name("y") + + def write(path): + with File(path, 'w', track_order=True) as hf: + g = hf.create_group(x, track_order=True) + g.create_dataset(y, shape=10, dtype='f4') + + f1 = tmp_path / make_name("f1{}.h5") + write(f1) + time.sleep(1.1) # Ensure any timestamps are different + f2 = tmp_path / make_name("f2{}.h5") + write(f2) + + assert sha256(f1.read_bytes()).hexdigest() == sha256(f2.read_bytes()).hexdigest() diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_file2.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_file2.py new file mode 100644 index 0000000000000000000000000000000000000000..6d6e0001e8853652f9ddb3778c964a5c694ba65d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_file2.py @@ -0,0 +1,337 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Tests the h5py.File object. +""" + +import h5py +from h5py._hl.files import _drivers +from h5py import File + +from .common import ut, TestCase, make_name + +import pytest +import io +import tempfile +import os + + +def nfiles(): + return h5py.h5f.get_obj_count(h5py.h5f.OBJ_ALL, h5py.h5f.OBJ_FILE) + +def ngroups(): + return h5py.h5f.get_obj_count(h5py.h5f.OBJ_ALL, h5py.h5f.OBJ_GROUP) + + +class TestDealloc(TestCase): + + """ + Behavior on object deallocation. Note most of this behavior is + delegated to FileID. + """ + @pytest.mark.thread_unsafe(reason="global object counters") + def test_autoclose(self): + """ File objects close automatically when out of scope, but + other objects remain open. """ + + start_nfiles = nfiles() + start_ngroups = ngroups() + + fname = self.mktemp() + f = h5py.File(fname, 'w') + g = f['/'] + + self.assertEqual(nfiles(), start_nfiles+1) + self.assertEqual(ngroups(), start_ngroups+1) + + del f + + self.assertTrue(g) + self.assertEqual(nfiles(), start_nfiles) + self.assertEqual(ngroups(), start_ngroups+1) + + f = g.file + + self.assertTrue(f) + self.assertEqual(nfiles(), start_nfiles+1) + self.assertEqual(ngroups(), start_ngroups+1) + + del g + + self.assertEqual(nfiles(), start_nfiles+1) + self.assertEqual(ngroups(), start_ngroups) + + del f + + self.assertEqual(nfiles(), start_nfiles) + self.assertEqual(ngroups(), start_ngroups) + + +class TestDriverRegistration(TestCase): + def test_register_driver(self): + called_with = [None] + + def set_fapl(plist, *args, **kwargs): + called_with[0] = args, kwargs + return _drivers['sec2'](plist) + + name = make_name() + h5py.register_driver(name, set_fapl) + self.assertIn(name, h5py.registered_drivers()) + + fname = self.mktemp() + h5py.File(fname, driver=name, driver_arg_0=0, driver_arg_1=1, + mode='w') + + self.assertEqual( + called_with, + [((), {'driver_arg_0': 0, 'driver_arg_1': 1})], + ) + + def test_unregister_driver(self): + name = make_name() + h5py.register_driver(name, lambda plist: None) + self.assertIn(name, h5py.registered_drivers()) + + h5py.unregister_driver(name) + self.assertNotIn(name, h5py.registered_drivers()) + + with self.assertRaises(ValueError) as e: + fname = self.mktemp() + h5py.File(fname, driver=name, mode='w') + + self.assertEqual(str(e.exception), f"Unknown driver type '{name}'") + + +class TestCache(TestCase): + def setUp(self): + MiB = 1024 * 1024 + if h5py.version.hdf5_version_tuple < (2, 0, 0): + self.dflt_chunk_cache = MiB + self.dflt_chunk_nslots = 521 + else: + self.dflt_chunk_cache = 8 * MiB + self.dflt_chunk_nslots = 8191 + + def test_defaults(self): + fname = self.mktemp() + f = h5py.File(fname, 'w') + self.assertEqual(list(f.id.get_access_plist().get_cache()), + [0, self.dflt_chunk_nslots, self.dflt_chunk_cache, 0.75]) + + def test_nbytes(self): + fname = self.mktemp() + f = h5py.File(fname, 'w', rdcc_nbytes=1024) + self.assertEqual(list(f.id.get_access_plist().get_cache()), + [0, self.dflt_chunk_nslots, 1024, 0.75]) + + def test_nslots(self): + fname = self.mktemp() + f = h5py.File(fname, 'w', rdcc_nslots=125) + self.assertEqual(list(f.id.get_access_plist().get_cache()), + [0, 125, self.dflt_chunk_cache, 0.75]) + + def test_w0(self): + fname = self.mktemp() + f = h5py.File(fname, 'w', rdcc_w0=0.25) + self.assertEqual(list(f.id.get_access_plist().get_cache()), + [0, self.dflt_chunk_nslots, self.dflt_chunk_cache, 0.25]) + + +class TestFileObj(TestCase): + + def check_write(self, fileobj): + f = h5py.File(fileobj, 'w') + self.assertEqual(f.driver, 'fileobj') + self.assertEqual(f.filename, repr(fileobj)) + f.create_dataset('test', data=list(range(12))) + self.assertEqual(list(f), ['test']) + self.assertEqual(list(f['test'][:]), list(range(12))) + f.close() + + def check_read(self, fileobj): + f = h5py.File(fileobj, 'r') + self.assertEqual(list(f), ['test']) + self.assertEqual(list(f['test'][:]), list(range(12))) + self.assertRaises(Exception, f.create_dataset, 'another.test', data=list(range(3))) + f.close() + + def test_BytesIO(self): + with io.BytesIO() as fileobj: + self.assertEqual(len(fileobj.getvalue()), 0) + self.check_write(fileobj) + self.assertGreater(len(fileobj.getvalue()), 0) + self.check_read(fileobj) + + def test_file(self): + fname = self.mktemp() + try: + with open(fname, 'wb+') as fileobj: + self.assertEqual(os.path.getsize(fname), 0) + self.check_write(fileobj) + self.assertGreater(os.path.getsize(fname), 0) + self.check_read(fileobj) + with open(fname, 'rb') as fileobj: + self.check_read(fileobj) + finally: + os.remove(fname) + + @pytest.mark.filterwarnings( + # at least on Windows and MacOS, a resource warning may be emitted + # when this test returns + "ignore::ResourceWarning" + ) + def test_TemporaryFile(self): + # in this test, we check explicitly that temp file gets + # automatically deleted upon h5py.File.close()... + fileobj = tempfile.NamedTemporaryFile() + fname = fileobj.name + f = h5py.File(fileobj, 'w') + del fileobj + # ... but in your code feel free to simply + # f = h5py.File(tempfile.TemporaryFile()) + + f.create_dataset('test', data=list(range(12))) + self.assertEqual(list(f), ['test']) + self.assertEqual(list(f['test'][:]), list(range(12))) + self.assertTrue(os.path.isfile(fname)) + f.close() + self.assertFalse(os.path.isfile(fname)) + + def test_exception_open(self): + self.assertRaises(Exception, h5py.File, None, + driver='fileobj', mode='x') + self.assertRaises(Exception, h5py.File, 'rogue', + driver='fileobj', mode='x') + self.assertRaises(Exception, h5py.File, self, + driver='fileobj', mode='x') + + def test_exception_read(self): + + class BrokenBytesIO(io.BytesIO): + def readinto(self, b): + raise Exception('I am broken') + + f = h5py.File(BrokenBytesIO(), 'w') + f.create_dataset('test', data=list(range(12))) + self.assertRaises(Exception, list, f['test']) + + def test_exception_write(self): + + class BrokenBytesIO(io.BytesIO): + allow_write = False + def write(self, b): + if self.allow_write: + return super().write(b) + else: + raise Exception('I am broken') + + bio = BrokenBytesIO() + f = h5py.File(bio, 'w') + try: + self.assertRaises(Exception, f.create_dataset, 'test', + data=list(range(12))) + finally: + # Un-break writing so we can close: errors while closing get messy. + bio.allow_write = True + f.close() + + @ut.skip("Incompletely closed files can cause segfaults") + def test_exception_close(self): + fileobj = io.BytesIO() + f = h5py.File(fileobj, 'w') + fileobj.close() + self.assertRaises(Exception, f.close) + + def test_exception_writeonly(self): + # HDF5 expects read & write access to a file it's writing; + # check that we get the correct exception on a write-only file object. + fileobj = open(os.path.join(self.tempdir, make_name("foo{}.h5")), 'wb') + f = h5py.File(fileobj, 'w') + group = f.create_group("group") + with self.assertRaises(io.UnsupportedOperation): + group.create_dataset("data", data='foo', dtype=h5py.string_dtype()) + f.close() + fileobj.close() + + + def test_method_vanish(self): + fileobj = io.BytesIO() + f = h5py.File(fileobj, 'w') + f.create_dataset('test', data=list(range(12))) + self.assertEqual(list(f['test'][:]), list(range(12))) + fileobj.readinto = None + self.assertRaises(Exception, list, f['test']) + + +class TestTrackOrder(TestCase): + def populate(self, f): + for i in range(100): + # Mix group and dataset creation. + if i % 10 == 0: + f.create_group(str(i)) + else: + f[str(i)] = [i] + + def test_track_order(self): + fname = self.mktemp() + f = h5py.File(fname, 'w', track_order=True) # creation order + self.populate(f) + self.assertEqual(list(f), [str(i) for i in range(100)]) + f.close() + + # Check order tracking after reopening the file + f2 = h5py.File(fname) + self.assertEqual(list(f2), [str(i) for i in range(100)]) + + def test_no_track_order(self): + fname = self.mktemp() + f = h5py.File(fname, 'w', track_order=False) # name alphanumeric + self.populate(f) + self.assertEqual(list(f), + sorted([str(i) for i in range(100)])) + + +class TestFileMetaBlockSize(TestCase): + + """ + Feature: The meta block size can be manipulated, changing how metadata + is aggregated and the offset of the first dataset. + """ + + def test_file_create_with_meta_block_size_4096(self): + # Test a large meta block size of 4 kibibytes + meta_block_size = 4096 + with File( + self.mktemp(), 'w', + meta_block_size=meta_block_size, + libver="latest" + ) as f: + f["test"] = 5 + self.assertEqual(f.meta_block_size, meta_block_size) + # Equality is expected for HDF5 1.10 + self.assertGreaterEqual(f["test"].id.get_offset(), meta_block_size) + + def test_file_create_with_meta_block_size_512(self): + # Test a small meta block size of 512 bytes + # The smallest verifiable meta_block_size is 463 + meta_block_size = 512 + libver = "latest" + with File( + self.mktemp(), 'w', + meta_block_size=meta_block_size, + libver=libver + ) as f: + f["test"] = 3 + self.assertEqual(f.meta_block_size, meta_block_size) + # Equality is expected for HDF5 1.10 + self.assertGreaterEqual(f["test"].id.get_offset(), meta_block_size) + # Default meta_block_size is 2048. This should fail if meta_block_size is not set. + self.assertLess(f["test"].id.get_offset(), meta_block_size*2) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_file_alignment.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_file_alignment.py new file mode 100644 index 0000000000000000000000000000000000000000..bb262f52baffbbde179b0b4e578c1acb62535a16 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_file_alignment.py @@ -0,0 +1,106 @@ +import pytest + +import h5py +from .common import TestCase + + +def is_aligned(dataset, offset=4096): + # Here we check if the dataset is aligned + return dataset.id.get_offset() % offset == 0 + + +def dataset_name(i): + return f"data{i:03}" + + +class TestFileAlignment(TestCase): + """ + Ensure that setting the file alignment has the desired effect + in the internal structure. + """ + def test_no_alignment_set(self): + fname = self.mktemp() + # 881 is a prime number, so hopefully this help randomize the alignment + # enough + # A nice even number might give a pathological case where + # While we don't want the data to be aligned, it ends up aligned... + shape = (881,) + + with h5py.File(fname, 'w') as h5file: + # Create up to 1000 datasets + # At least one of them should be misaligned. + # While this isn't perfect, it seems that there + # The case where 1000 datasets get created is one where the data + # is aligned. Therefore, during correct operation, this test is + # expected to finish quickly + for i in range(1000): + dataset = h5file.create_dataset( + dataset_name(i), shape, dtype='uint8') + # Assign data so that the dataset is instantiated in + # the file + dataset[...] = i + if not is_aligned(dataset): + # Break early asserting that the file is not aligned + break + else: + raise RuntimeError("Data was all found to be aligned to 4096") + + @pytest.mark.parallel_threads_limit(2) # Quite slow otherwise + def test_alignment_set_above_threshold(self): + # 2022/01/19 hmaarrfk + # UnitTest (TestCase) doesn't play well with pytest parametrization. + alignment_threshold = 1000 + alignment_interval = 4096 + + for shape in [ + (1033,), # A prime number above the threshold + (1000,), # Exactly equal to the threshold + (1001,), # one above the threshold + ]: + fname = self.mktemp() + with h5py.File(fname, 'w', + alignment_threshold=alignment_threshold, + alignment_interval=alignment_interval) as h5file: + # Create up to 1000 datasets + # They are all expected to be aligned + for i in range(1000): + dataset = h5file.create_dataset( + dataset_name(i), shape, dtype='uint8') + # Assign data so that the dataset is instantiated in + # the file + dataset[...] = (i % 256) # Truncate to uint8 + assert is_aligned(dataset, offset=alignment_interval) + + def test_alignment_set_below_threshold(self): + # 2022/01/19 hmaarrfk + # UnitTest (TestCase) doesn't play well with pytest parametrization. + alignment_threshold = 1000 + alignment_interval = 1024 + + for shape in [ + (881,), # A prime number below the threshold + (999,), # Exactly one below the threshold + ]: + fname = self.mktemp() + with h5py.File(fname, 'w', + alignment_threshold=alignment_threshold, + alignment_interval=alignment_interval) as h5file: + # Create up to 1000 datasets + # At least one of them should be misaligned. + # While this isn't perfect, it seems that there + # The case where 1000 datasets get created is one where the + # data is aligned. Therefore, during correct operation, this + # test is expected to finish quickly + for i in range(1000): + dataset = h5file.create_dataset( + dataset_name(i), shape, dtype='uint8') + # Assign data so that the dataset is instantiated in + # the file + dataset[...] = i + if not is_aligned(dataset, offset=alignment_interval): + # Break early asserting that the file is not aligned + break + else: + raise RuntimeError( + "Data was all found to be aligned to " + f"{alignment_interval}. This is highly unlikely.") diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_file_image.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_file_image.py new file mode 100644 index 0000000000000000000000000000000000000000..3e45b55a3bfcb99ad3bb7c5ee08486c74a9152ad --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_file_image.py @@ -0,0 +1,54 @@ +import numpy as np + +import h5py +from h5py import h5f, h5p + +from .common import TestCase + +class TestFileImage(TestCase): + def test_load_from_image(self): + from binascii import a2b_base64 + from zlib import decompress + + compressed_image = 'eJzr9HBx4+WS4mIAAQ4OBhYGAQZk8B8KKjhQ+TD5BCjNCKU7oPQKJpg4I1hOAiouCDUfXV1IkKsrSPV/NACzx4AFQnMwjIKRCDxcHQNAdASUD0ulJ5hQ1ZWkFpeAaFh69KDQXkYGNohZjDA+JCUzMkIEmKHqELQAWKkAByytOoBJViAPJM7ExATWyAE0B8RgZkyAJmlYDoEAIahukJoNU6+HMTA0UOgT6oBgP38XUI6G5UMFZrzKR8EoGAUjGMDKYVgxDSsuAHcfMK8=' + + image = decompress(a2b_base64(compressed_image)) + + fapl = h5p.create(h5py.h5p.FILE_ACCESS) + fapl.set_fapl_core() + fapl.set_file_image(image) + + fid = h5f.open(self.mktemp().encode(), h5py.h5f.ACC_RDONLY, fapl=fapl) + f = h5py.File(fid) + + self.assertTrue('test' in f) + + def test_open_from_image(self): + from binascii import a2b_base64 + from zlib import decompress + + compressed_image = 'eJzr9HBx4+WS4mIAAQ4OBhYGAQZk8B8KKjhQ+TD5BCjNCKU7oPQKJpg4I1hOAiouCDUfXV1IkKsrSPV/NACzx4AFQnMwjIKRCDxcHQNAdASUD0ulJ5hQ1ZWkFpeAaFh69KDQXkYGNohZjDA+JCUzMkIEmKHqELQAWKkAByytOoBJViAPJM7ExATWyAE0B8RgZkyAJmlYDoEAIahukJoNU6+HMTA0UOgT6oBgP38XUI6G5UMFZrzKR8EoGAUjGMDKYVgxDSsuAHcfMK8=' + + image = decompress(a2b_base64(compressed_image)) + + fid = h5f.open_file_image(image) + f = h5py.File(fid) + + self.assertTrue('test' in f) + + +def test_in_memory(): + arr = np.arange(10) + # Passing one fcpl & one fapl parameter to exercise the code splitting them: + with h5py.File.in_memory(track_order=True, rdcc_nbytes=2_000_000) as f1: + f1['a'] = arr + f1.flush() + img = f1.id.get_file_image() + + # Open while f1 is still open + with h5py.File.in_memory(img) as f2: + np.testing.assert_array_equal(f2['a'][:], arr) + + # Reuse image now that previous files are closed + with h5py.File.in_memory(img) as f3: + np.testing.assert_array_equal(f3['a'][:], arr) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_filters.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_filters.py new file mode 100644 index 0000000000000000000000000000000000000000..dcac1334bd607b06f70fc482a7bb60bc72f1da73 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_filters.py @@ -0,0 +1,98 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Tests the h5py._hl.filters module. + +""" +import os +import numpy as np +import h5py + +from .common import ut, TestCase, is_main_thread, make_name + + +class TestFilters(TestCase): + + def setUp(self): + """ like TestCase.setUp but also store the file path """ + self.path = self.mktemp() + self.f = h5py.File(self.path, 'w') + + @ut.skipUnless(h5py.h5z.filter_avail(h5py.h5z.FILTER_SZIP), 'szip filter required') + def test_wr_szip_fletcher32_64bit(self): + """ test combination of szip, fletcher32, and 64bit arrays + + The fletcher32 checksum must be computed after the szip + compression is applied. + + References: + - GitHub issue #953 + - https://forum.hdfgroup.org/t/fletcher32-checksum-error-with-szip-compression-and-64bit-data/4141 + """ + name = make_name() + self.f.create_dataset(name, + data=np.zeros(10000, dtype=np.float64), + fletcher32=True, + compression="szip", + ) + + self.f.flush() + if is_main_thread(): + self.f.close() + self.f = h5py.File(self.path, 'r') + + with h5py.File(self.path, "r") as h5: + # Access the data which will compute the fletcher32 + # checksum and raise an OSError if something is wrong. + self.f[name][0] + + def test_wr_scaleoffset_fletcher32(self): + """ make sure that scaleoffset + fletcher32 is prevented + """ + data = np.linspace(0, 1, 100) + with self.assertRaises(ValueError): + self.f.create_dataset(make_name(), + data=data, + fletcher32=True, + # retain 3 digits after the decimal point + scaleoffset=3, + ) + + +@ut.skipIf('gzip' not in h5py.filters.encode, "DEFLATE is not installed") +def test_filter_ref_obj(writable_file): + gzip8 = h5py.filters.Gzip(level=8) + # **kwargs unpacking (compatible with earlier h5py versions) + assert dict(**gzip8) == { + 'compression': h5py.h5z.FILTER_DEFLATE, + 'compression_opts': (8,) + } + + # Pass object as compression argument (new in h5py 3.0) + ds = writable_file.create_dataset( + make_name(), shape=(100,), dtype=np.uint32, compression=gzip8 + ) + assert ds.compression == 'gzip' + assert ds.compression_opts == 8 + + +def test_filter_ref_obj_eq(): + gzip8 = h5py.filters.Gzip(level=8) + + assert gzip8 == h5py.filters.Gzip(level=8) + assert gzip8 != h5py.filters.Gzip(level=7) + + +@ut.skipIf(not os.getenv('H5PY_TEST_CHECK_FILTERS'), "H5PY_TEST_CHECK_FILTERS not set") +def test_filters_available(): + assert 'gzip' in h5py.filters.decode + assert 'gzip' in h5py.filters.encode + assert 'lzf' in h5py.filters.decode + assert 'lzf' in h5py.filters.encode diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_group.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_group.py new file mode 100644 index 0000000000000000000000000000000000000000..acbe43f5f11bd7ac62696a2f69b6d724e91fe151 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_group.py @@ -0,0 +1,1476 @@ +# -*- coding: utf-8 -*- +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" + Group test module. + + Tests all methods and properties of Group objects, with the following + exceptions: + + 1. Method create_dataset is tested in module test_dataset +""" + +import numpy as np +import os +import os.path +from collections.abc import MutableMapping +from tempfile import mkdtemp + +import pytest + +import h5py +from h5py import File, Group, SoftLink, HardLink, ExternalLink +from h5py import Dataset, Datatype +from h5py import h5t +from h5py._hl.compat import filename_encode +from .common import ut, TestCase, make_name, is_main_thread + +# If we can't encode unicode filenames, there's not much point failing tests +# which must fail +try: + filename_encode(u"α") +except UnicodeEncodeError: + NO_FS_UNICODE = True +else: + NO_FS_UNICODE = False + + +class BaseGroup(TestCase): + + def setUp(self): + self.f = File(self.mktemp(), 'w') + + def tearDown(self): + if self.f: + self.f.close() + +class TestCreate(BaseGroup): + + """ + Feature: New groups can be created via .create_group method + """ + + def test_create_str(self): + """ Simple .create_group call """ + grp = self.f.create_group(make_name()) + self.assertIsInstance(grp, Group) + + def test_create_bytes(self): + grp = self.f.create_group(make_name().encode('utf8')) + self.assertIsInstance(grp, Group) + + def test_create_intermediate_str(self): + """ Intermediate groups can be created automatically """ + path = make_name("foo{}/bar/baz") + grp = self.f.create_group(path) + self.assertEqual(grp.name, "/" + path) + + def test_create_intermediate_bytes(self): + path = make_name("foo{}/bar/baz") + grp2 = self.f.create_group(path.encode("utf8")) + self.assertEqual(grp2.name, "/" + path) + + def test_create_exception(self): + """ Name conflict causes group creation to fail with ValueError """ + name = make_name() + self.f.create_group(name) + with self.assertRaises(ValueError): + self.f.create_group(name) + + def test_unicode(self): + """ Unicode names are correctly stored """ + n = "/" + make_name() + chr(0x4500) + group = self.f.create_group(n) + self.assertEqual(group.name, n) + self.assertEqual(group.id.links.get_info(n.encode('utf8')).cset, h5t.CSET_UTF8) + + def test_unicode_default(self): + """ Unicode names convertible to ASCII are stored as ASCII (issue 239) + """ + n = make_name("/Hello, this is a name") + group = self.f.create_group(n) + self.assertEqual(group.name, n) + self.assertEqual(group.id.links.get_info(n.encode('utf8')).cset, h5t.CSET_ASCII) + + def test_type(self): + """ Names should be strings or bytes """ + with self.assertRaises(TypeError): + self.f.create_group(1.) + + def test_appropriate_low_level_id(self): + " Binding a group to a non-group identifier fails with ValueError " + dset = self.f.create_dataset(make_name(), [1], "f4") + with self.assertRaises(ValueError): + Group(dset.id) + +class TestDatasetAssignment(BaseGroup): + + """ + Feature: Datasets can be created by direct assignment of data + """ + + def test_ndarray(self): + """ Dataset auto-creation by direct assignment """ + name = make_name() + data = np.ones((4,4),dtype='f') + self.f[name] = data + self.assertIsInstance(self.f[name], Dataset) + self.assertArrayEqual(self.f[name][...], data) + + def test_name_bytes(self): + data = np.ones((4, 4), dtype='f') + n = make_name().encode('utf8') + self.f[n] = data + self.assertIsInstance(self.f[n], Dataset) + +class TestDtypeAssignment(BaseGroup): + + """ + Feature: Named types can be created by direct assignment of dtypes + """ + + def test_dtype(self): + """ Named type creation """ + name = make_name() + dtype = np.dtype('|S10') + self.f[name] = dtype + self.assertIsInstance(self.f[name], Datatype) + self.assertEqual(self.f[name].dtype, dtype) + + def test_name_bytes(self): + """ Named type creation """ + dtype = np.dtype('|S10') + n = make_name().encode('utf8') + self.f[n] = dtype + self.assertIsInstance(self.f[n], Datatype) + + +class TestRequire(BaseGroup): + + """ + Feature: Groups can be auto-created, or opened via .require_group + """ + + def test_open_existing(self): + """ Existing group is opened and returned """ + name = make_name() + grp = self.f.create_group(name) + grp2 = self.f.require_group(name) + self.assertEqual(grp2, grp) + + grp3 = self.f.require_group(name.encode('utf8')) + self.assertEqual(grp3, grp) + + def test_create(self): + """ Group is created if it doesn't exist """ + name = make_name() + grp = self.f.require_group(name) + self.assertIsInstance(grp, Group) + self.assertEqual(grp.name, '/' + name) + + def test_require_exception(self): + """ Opening conflicting object results in TypeError """ + name = make_name() + self.f.create_dataset(name, (1,), 'f') + with self.assertRaises(TypeError): + self.f.require_group(name) + + def test_intermediate_create_dataset(self): + """ Intermediate is created if it doesn't exist """ + name = make_name("foo") + dt = h5py.string_dtype() + self.f.require_dataset(f"{name}/bar/baz", (1,), dtype=dt) + group = self.f.get(name) + assert isinstance(group, Group) + group = self.f.get(f"{name}/bar") + assert isinstance(group, Group) + + def test_intermediate_create_group(self): + name = make_name("foo") + self.f.require_group(f"{name}/bar/baz") + group = self.f.get(name) + assert isinstance(group, Group) + group = self.f.get(f"{name}/bar") + assert isinstance(group, Group) + group = self.f.get(f"{name}/bar/baz") + assert isinstance(group, Group) + + def test_require_shape(self): + n = make_name("foo{}/resizable") + ds = self.f.require_dataset(n, shape=(0, 3), maxshape=(None, 3), dtype=int) + ds.resize(20, axis=0) + self.f.require_dataset(n, shape=(0, 3), maxshape=(None, 3), dtype=int) + self.f.require_dataset(n, shape=(20, 3), dtype=int) + with self.assertRaises(TypeError): + self.f.require_dataset(n, shape=(0, 0), maxshape=(3, None), dtype=int) + with self.assertRaises(TypeError): + self.f.require_dataset(n, shape=(0, 0), maxshape=(None, 5), dtype=int) + with self.assertRaises(TypeError): + self.f.require_dataset(n, shape=(0, 0), maxshape=(None, 5, 2), dtype=int) + with self.assertRaises(TypeError): + self.f.require_dataset(n, shape=(10, 3), dtype=int) + + +class TestDelete(BaseGroup): + + """ + Feature: Objects can be unlinked via "del" operator + """ + + def test_delete(self): + """ Object deletion via "del" """ + name = make_name() + self.f.create_group(name) + self.assertIn(name, self.f) + del self.f[name] + self.assertNotIn(name, self.f) + + def test_nonexisting(self): + """ Deleting non-existent object raises KeyError """ + with self.assertRaises(KeyError): + del self.f['foo'] + + def test_readonly_delete_exception(self): + """ Deleting object in readonly file raises KeyError """ + # Note: it is impossible to restore the old behavior (ValueError) + # without breaking the above test (non-existing objects) + fname = self.mktemp() + hfile = File(fname, 'w') + try: + hfile.create_group('foo') + finally: + hfile.close() + + hfile = File(fname, 'r') + try: + with self.assertRaises(KeyError): + del hfile['foo'] + finally: + hfile.close() + +class TestOpen(BaseGroup): + + """ + Feature: Objects can be opened via indexing syntax obj[name] + """ + + def test_open(self): + """ Simple obj[name] opening """ + name = make_name() + grp = self.f.create_group(name) + grp2 = self.f[name] + grp3 = self.f[f"/{name}"] + self.assertEqual(grp, grp2) + self.assertEqual(grp, grp3) + + def test_nonexistent(self): + """ Opening missing objects raises KeyError """ + with self.assertRaises(KeyError): + self.f["notexist"] + + def test_reference(self): + """ Objects can be opened by HDF5 object reference """ + grp = self.f.create_group(make_name()) + grp2 = self.f[grp.ref] + self.assertEqual(grp2, grp) + + def test_reference_numpyobj(self): + """ Object can be opened by numpy.object_ containing object ref + + Test for issue 181, issue 202. + """ + g = self.f.create_group(make_name("g")) + + dt = np.dtype([('a', 'i'),('b', h5py.ref_dtype)]) + dset = self.f.create_dataset(make_name("x"), (1,), dt) + + dset[0] =(42,g.ref) + data = dset[0] + self.assertEqual(self.f[data[1]], g) + + def test_invalid_ref(self): + """ Invalid region references should raise an exception """ + ref = h5py.h5r.Reference() + + with self.assertRaises(ValueError): + self.f[ref] + + @pytest.mark.thread_unsafe(reason="FIXME #2672 does not raise") + def test_deleted_ref(self): + """ References to deleted objects should raise an exception """ + name = make_name() + self.f.create_group(name) + ref = self.f[name].ref + del self.f[name] + + with self.assertRaises(Exception): + self.f[ref] + + def test_path_type_validation(self): + """ Access with non bytes or str types should raise an exception """ + self.f.create_group(make_name()) + + with self.assertRaises(TypeError): + self.f[0] + + with self.assertRaises(TypeError): + self.f[...] + + # TODO: check that regionrefs also work with __getitem__ + +class TestRepr(BaseGroup): + """Opened and closed groups provide a useful __repr__ string""" + + def test_repr(self): + """ Opened and closed groups provide a useful __repr__ string """ + name = make_name() + g = self.f.create_group(name) + self.assertIsInstance(repr(g), str) + g.id._close() + self.assertIsInstance(repr(g), str) + g = self.f[name] + if is_main_thread(): + # Closing the file shouldn't break it + self.f.close() + self.assertIsInstance(repr(g), str) + +class BaseMapping(BaseGroup): + + """ + Base class for mapping tests + """ + def setUp(self): + self.f = File(self.mktemp(), 'w') + self.groups = ('a', 'b', 'c', 'd') + for x in self.groups: + self.f.create_group(x) + self.f['x'] = h5py.SoftLink('/mongoose') + self.groups = self.groups + ('x',) + + def tearDown(self): + if self.f: + self.f.close() + +class TestLen(BaseMapping): + + """ + Feature: The Python len() function returns the number of groups + """ + + def test_len(self): + """ len() returns number of group members """ + self.assertEqual(len(self.f), len(self.groups)) + + def test_len_after_create_group(self): + # Can't use a shared file when running in pytest-run-parallel + with File(self.mktemp(), 'w') as f: + self.assertEqual(len(f), 0) + f.create_group("x") + self.assertEqual(len(f), 1) + f.create_group("y") + self.assertEqual(len(f), 2) + del f["x"] + self.assertEqual(len(f), 1) + +class TestContains(BaseGroup): + + """ + Feature: The Python "in" builtin tests for membership + """ + + def test_contains(self): + """ "in" builtin works for membership (byte and Unicode) """ + name = make_name() + self.f.create_group(name) + self.assertIn(name.encode("utf-8"), self.f) + self.assertIn(name, self.f) + self.assertIn(f"/{name}".encode("utf-8"), self.f) + self.assertIn(f"/{name}", self.f) + self.assertNotIn(b'mongoose', self.f) + self.assertNotIn('mongoose', self.f) + + def test_closed(self): + """ "in" on closed File returns False (see also issue 174) """ + f = File(self.mktemp(), 'w') + f.create_group('a') + self.assertTrue(b'a' in f) + self.assertTrue('a' in f) + f.close() + self.assertFalse(b'a' in f) + self.assertFalse('a' in f) + + def test_empty(self): + """ Empty strings work properly and aren't contained """ + self.assertNotIn('', self.f) + self.assertNotIn(b'', self.f) + + def test_dot(self): + """ Current group "." is always contained """ + self.assertIn(b'.', self.f) + self.assertIn('.', self.f) + + def test_root(self): + """ Root group (by itself) is contained """ + self.assertIn(b'/', self.f) + self.assertIn('/', self.f) + + def test_trailing_slash(self): + """ Trailing slashes are unconditionally ignored """ + g = make_name("g") + d = make_name("d") + self.f.create_group(g) + self.f[d] = 42 + self.assertIn(f"/{g}/", self.f) + self.assertIn(f"{g}/", self.f) + self.assertIn(f"/{d}/", self.f) + self.assertIn(f"{d}/", self.f) + + def test_softlinks(self): + """ Broken softlinks are contained, but their members are not """ + name = make_name() + self.f.create_group(name) + self.f[f'/{name}/soft'] = h5py.SoftLink('/mongoose') + self.f[f'/{name}/external'] = h5py.ExternalLink('mongoose.hdf5', '/mongoose') + self.assertIn(f"/{name}/soft", self.f) + self.assertNotIn(f"/{name}/soft/something", self.f) + self.assertIn(f"/{name}/external", self.f) + self.assertNotIn(f"/{name}/external/something", self.f) + + def test_oddball_paths(self): + """ Technically legitimate (but odd-looking) paths """ + x = make_name('x') + dset = make_name("dset") + + self.f.create_group(f"{x}/y/z") + self.f[dset] = 42 + self.assertIn('/', self.f) + self.assertIn('//', self.f) + self.assertIn('///', self.f) + self.assertIn('.///', self.f) + self.assertIn('././/', self.f) + grp = self.f[x] + self.assertIn(f'.//{x}/y/z', self.f) + self.assertNotIn(f'.//{x}/y/z', grp) + self.assertIn(f'{x}///', self.f) + self.assertIn(f'./{x}///', self.f) + self.assertIn(f'{dset}///', self.f) + self.assertIn(f'/{dset}//', self.f) + +class TestIter(BaseMapping): + + """ + Feature: You can iterate over group members via "for x in y", etc. + """ + + def test_iter(self): + """ "for x in y" iteration """ + lst = [x for x in self.f] + self.assertSameElements(lst, self.groups) + + def test_iter_zero(self): + """ Iteration works properly for the case with no group members """ + hfile = File(self.mktemp(), 'w') + try: + lst = [x for x in hfile] + self.assertEqual(lst, []) + finally: + hfile.close() + +class TestTrackOrder(BaseGroup): + def populate(self, g): + for i in range(100): + # Mix group and dataset creation. + if i % 10 == 0: + g.create_group(str(i)) + else: + g[str(i)] = [i] + + def test_track_order(self): + g = self.f.create_group(make_name(), track_order=True) # creation order + self.populate(g) + + ref = [str(i) for i in range(100)] + self.assertEqual(list(g), ref) + self.assertEqual(list(reversed(g)), list(reversed(ref))) + + def test_no_track_order(self): + g = self.f.create_group(make_name(), track_order=False) # name alphanumeric + self.populate(g) + + ref = sorted([str(i) for i in range(100)]) + self.assertEqual(list(g), ref) + self.assertEqual(list(reversed(g)), list(reversed(ref))) + +class TestPy3Dict(BaseMapping): + + def test_keys(self): + """ .keys provides a key view """ + kv = getattr(self.f, 'keys')() + ref = self.groups + self.assertSameElements(list(kv), ref) + self.assertSameElements(list(reversed(kv)), list(reversed(ref))) + + for x in self.groups: + self.assertIn(x, kv) + self.assertEqual(len(kv), len(self.groups)) + + def test_values(self): + """ .values provides a value view """ + vv = getattr(self.f, 'values')() + ref = [self.f.get(x) for x in self.groups] + self.assertSameElements(list(vv), ref) + self.assertSameElements(list(reversed(vv)), list(reversed(ref))) + + self.assertEqual(len(vv), len(self.groups)) + for x in self.groups: + self.assertIn(self.f.get(x), vv) + + def test_items(self): + """ .items provides an item view """ + iv = getattr(self.f, 'items')() + ref = [(x,self.f.get(x)) for x in self.groups] + self.assertSameElements(list(iv), ref) + self.assertSameElements(list(reversed(iv)), list(reversed(ref))) + + self.assertEqual(len(iv), len(self.groups)) + for x in self.groups: + self.assertIn((x, self.f.get(x)), iv) + +class TestAdditionalMappingFuncs(BaseMapping): + """ + Feature: Other dict methods (pop, pop_item, clear, update, setdefault) are + available. + """ + def setUp(self): + self.f = File(self.mktemp(), 'w') + for x in ('/test/a', '/test/b', '/test/c', '/test/d'): + self.f.create_group(x) + self.group = self.f['test'] + + def tearDown(self): + if self.f: + self.f.close() + + def test_pop_item(self): + """.pop_item removes an item""" + g = self.f.create_group(make_name()) + g.create_group("a") + g.create_group("b") + k, _ = g.popitem() + assert k in ("a", "b") + self.assertNotIn(k, g) + + k2, _ = g.popitem() + assert k2 == "b" if k == "a" else "a" + self.assertNotIn(k2, g) + + # popitem() from an empty group raises + with self.assertRaises(KeyError): + g.popitem() + + def test_pop(self): + """.pop returns and removes specified item""" + g = self.f.create_group(make_name()) + g.create_group("a") + g.create_group("b") + g.pop("a") + self.assertNotIn("a", g) + self.assertIn("b", g) + + def test_pop_default(self): + """.pop falls back to default""" + # e shouldn't exist as a group + value = self.group.pop('e', None) + self.assertEqual(value, None) + + def test_pop_raises(self): + """.pop raises KeyError for non-existence""" + # e shouldn't exist as a group + with self.assertRaises(KeyError): + key = self.group.pop('e') + + def test_clear(self): + """.clear removes groups""" + g = self.f.create_group(make_name()) + g.create_group('a') + g.create_group('b') + self.assertEqual(len(g), 2) + g.clear() + self.assertEqual(len(g), 0) + + def test_update_dict(self): + """.update works with dict""" + g = self.f.create_group(make_name()) + new_items = {'e': np.array([42])} + g.update(new_items) + self.assertIn('e', g) + + def test_update_iter(self): + """.update works with list""" + g = self.f.create_group(make_name()) + new_items = [ + ('e', np.array([42])), + ('f', np.array([42])) + ] + g.update(new_items) + self.assertIn('e', g) + + def test_update_kwargs(self): + """.update works with kwargs""" + g = self.f.create_group(make_name()) + new_items = {'e': np.array([42])} + g.update(**new_items) + self.assertIn('e', g) + + def test_setdefault(self): + """.setdefault gets group if it exists""" + value = self.group.setdefault('a') + self.assertEqual(value, self.group.get('a')) + + def test_setdefault_with_default(self): + """.setdefault gets default if group doesn't exist""" + # 42 used as groups should be strings + value = self.group.setdefault(make_name(), np.array([42])) + self.assertEqual(value, 42) + + def test_setdefault_no_default(self): + """ + .setdefault gets None if group doesn't exist, but as None isn't defined + as data for a dataset, this should raise a TypeError. + """ + # e shouldn't exist as a group + with self.assertRaises(TypeError): + self.group.setdefault('e') + + +class TestGet(BaseGroup): + + """ + Feature: The .get method allows access to objects and metadata + """ + + def test_get_default(self): + """ Object is returned, or default if it doesn't exist """ + name = make_name() + default = object() + out = self.f.get('mongoose', default) + self.assertIs(out, default) + + grp = self.f.create_group(name) + out = self.f.get(name.encode('utf8')) + self.assertEqual(out, grp) + + def test_get_class(self): + """ Object class is returned with getclass option """ + foo = make_name("foo") + bar = make_name("bar") + baz = make_name("baz") + + self.f.create_group(foo) + out = self.f.get(foo, getclass=True) + self.assertEqual(out, Group) + + self.f.create_dataset(bar, (4,), "f4") + out = self.f.get(bar, getclass=True) + self.assertEqual(out, Dataset) + + self.f[baz] = np.dtype('|S10') + out = self.f.get(baz, getclass=True) + self.assertEqual(out, Datatype) + + def test_get_link_class(self): + """ Get link classes """ + hard = make_name("hard") + soft = make_name("soft") + external = make_name("external") + + default = object() + sl = SoftLink('/mongoose') + el = ExternalLink('somewhere.hdf5', 'mongoose') + + self.f.create_group(hard) + self.f[soft] = sl + self.f[external] = el + + out_hl = self.f.get(hard, default, getlink=True, getclass=True) + out_sl = self.f.get(soft, default, getlink=True, getclass=True) + out_el = self.f.get(external, default, getlink=True, getclass=True) + + self.assertEqual(out_hl, HardLink) + self.assertEqual(out_sl, SoftLink) + self.assertEqual(out_el, ExternalLink) + + def test_get_link(self): + """ Get link values """ + hard = make_name("hard") + soft = make_name("soft") + external = make_name("external") + + sl = SoftLink('/mongoose') + el = ExternalLink('somewhere.hdf5', 'mongoose') + + self.f.create_group(hard) + self.f[soft] = sl + self.f[external] = el + + out_hl = self.f.get(hard, getlink=True) + out_sl = self.f.get(soft, getlink=True) + out_el = self.f.get(external, getlink=True) + + #TODO: redo with SoftLink/ExternalLink built-in equality + self.assertIsInstance(out_hl, HardLink) + self.assertIsInstance(out_sl, SoftLink) + self.assertEqual(out_sl._path, sl._path) + self.assertIsInstance(out_el, ExternalLink) + self.assertEqual(out_el._path, el._path) + self.assertEqual(out_el._filename, el._filename) + +class TestVisit(TestCase): + + """ + Feature: The .visit and .visititems methods allow iterative access to + group and subgroup members + """ + + def setUp(self): + self.f = File(self.mktemp(), 'w') + self.groups = [ + 'grp1', 'grp1/sg1', 'grp1/sg2', 'grp2', 'grp2/sg1', 'grp2/sg1/ssg1' + ] + for x in self.groups: + self.f.create_group(x) + + def tearDown(self): + self.f.close() + + def test_visit(self): + """ All subgroups are visited """ + l = [] + self.f.visit(l.append) + self.assertSameElements(l, self.groups) + + def test_visititems(self): + """ All subgroups and contents are visited """ + l = [] + comp = [(x, self.f[x]) for x in self.groups] + self.f.visititems(lambda x, y: l.append((x,y))) + self.assertSameElements(comp, l) + + def test_bailout(self): + """ Returning a non-None value immediately aborts iteration """ + # do not make assumption on iteration order + l = [] + x = self.f.visit(lambda x: l.append(x) or -1) + assert x == -1 and len(l) == 1 and l[0] in self.groups + + l = [] + comp = [(x, self.f[x]) for x in self.groups] + x = self.f.visititems(lambda x, y: l.append((x,y)) or -1) + assert x == -1 and len(l) == 1 and l[0] in comp + +class TestVisitLinks(TestCase): + """ + Feature: The .visit_links and .visititems_links methods allow iterative access to + links contained in the group and its subgroups. + """ + + def setUp(self): + self.f = File(self.mktemp(), 'w') + self.groups = [ + 'grp1', 'grp1/grp11', 'grp1/grp12', 'grp2', 'grp2/grp21', 'grp2/grp21/grp211' + ] + self.links = [ + 'linkto_grp1', 'grp1/linkto_grp11', 'grp1/linkto_grp12', 'linkto_grp2', 'grp2/linkto_grp21', 'grp2/grp21/linkto_grp211' + ] + for g, l in zip(self.groups, self.links, strict=True): + self.f.create_group(g) + self.f[l] = SoftLink(f'/{g}') + + def tearDown(self): + self.f.close() + + def test_visit_links(self): + """ All subgroups and links are visited """ + l = [] + self.f.visit_links(l.append) + self.assertSameElements(l, self.groups + self.links) + + def test_visititems(self): + """ All links are visited """ + l = [] + comp = [(x, type(self.f.get(x, getlink=True))) for x in self.groups + self.links] + self.f.visititems_links(lambda x, y: l.append((x, type(y)))) + self.assertSameElements(comp, l) + + def test_bailout(self): + """ Returning a non-None value immediately aborts iteration """ + # do not make assumption on iteration order + l = [] + x = self.f.visit_links(lambda x: l.append(x) or -1) + assert x == -1 and len(l) == 1 + + l = [] + x = self.f.visititems_links(lambda x, y: l.append((x,y)) or -1) + assert x == -1 and len(l) == 1 + +class Visitor: + """ Class for exercise 'visit' and 'visititems' methods """ + + def __init__(self): + self._names = [] + + def __call__(self, name, obj=None): + self._names.append(name) + + @property + def names(self): + return self._names + +class TestLexicographic(TestCase): + """ Test ascending lexicographic order traversal of the 'visit*' methods. + + This semantics is set by the following default args in + h5py.h5o.visit(..., idx_type=H5_INDEX_NAME, order=H5_ITER_INC, ...) + h5py.h5l.visit(..., idx_type=H5_INDEX_NAME, order=H5_ITER_INC, ...) + """ + + import operator + split_parts = operator.methodcaller('split', '/') + + def setUp(self): + """ Populate example hdf5 file, with track_order=True """ + + self.f = File(self.mktemp(), 'w-', track_order=True) + self.f.create_dataset('b', (10,), "f4") + + grp = self.f.create_group('B', track_order=True) + grp.create_dataset('b', (10,), "f4") + grp.create_dataset('a', (10,), "f4") + + grp = self.f.create_group('z', track_order=True) + grp.create_dataset('b', (10,), "f4") + grp.create_dataset('a', (10,), "f4") + + self.f.create_dataset('a', (10,), "f4") + # note that 'z-' < 'z/...' but traversal order is ['z', 'z/...', 'z-'] + self.f.create_dataset('z-', (10,), "f4") + + # create some links + self.f['A/x'] = self.f['B/b'] + self.f['y'] = self.f['z/a'] + self.f['A$'] = self.f['y'] + self.f['A/B/C'] = self.f['A'] + self.f['A/a'] = self.f['A'] + + def test_nontrivial_sort_visit(self): + """check that test example is not trivially sorted""" + v = Visitor() + self.f.visit(v) + assert v.names != sorted(v.names) + + def test_visit(self): + """check that File.visit iterates in lexicographic order""" + v = Visitor() + self.f.visit(v) + assert v.names == sorted(v.names, key=self.split_parts) + + def test_visit_links(self): + """check that File.visit_links iterates in lexicographic order""" + v = Visitor() + self.f.visit_links(v) + assert v.names == sorted(v.names, key=self.split_parts) + + def test_visititems(self): + """check that File.visititems iterates in lexicographic order""" + v = Visitor() + self.f.visititems(v) + assert v.names == sorted(v.names, key=self.split_parts) + + def test_visititems_links(self): + """check that File.visititems_links iterates in lexicographic order""" + v = Visitor() + self.f.visititems_links(v) + assert v.names == sorted(v.names, key=self.split_parts) + + def test_visit_group(self): + """check that Group.visit iterates in lexicographic order""" + v = Visitor() + self.f['A'].visit(v) + assert v.names == sorted(v.names, key=self.split_parts) + +class TestSoftLinks(BaseGroup): + + """ + Feature: Create and manage soft links with the high-level interface + """ + + def test_spath(self): + """ SoftLink path attribute """ + sl = SoftLink('/foo') + self.assertEqual(sl.path, '/foo') + + def test_srepr(self): + """ SoftLink path repr """ + sl = SoftLink('/foo') + self.assertIsInstance(repr(sl), str) + + def test_create(self): + """ Create new soft link by assignment """ + new = make_name("new") + alias = make_name("alias") + + g = self.f.create_group(new) + sl = SoftLink(f"/{new}") + self.f[alias] = sl + g2 = self.f[alias] + self.assertEqual(g, g2) + + def test_exc(self): + """ Opening dangling soft link results in KeyError """ + name = make_name() + self.f[name] = SoftLink('new') + with self.assertRaises(KeyError): + self.f[name] + + +class TestExternalLinks(TestCase): + + """ + Feature: Create and manage external links + """ + + def setUp(self): + self.f = File(self.mktemp(), 'w') + self.ename = self.mktemp() + self.ef = File(self.ename, 'w') + self.ef.create_group('external') + self.ef.close() + + def tearDown(self): + if self.f: + self.f.close() + if self.ef: + self.ef.close() + + def test_epath(self): + """ External link paths attributes """ + el = ExternalLink('foo.hdf5', '/foo') + self.assertEqual(el.filename, 'foo.hdf5') + self.assertEqual(el.path, '/foo') + + def test_erepr(self): + """ External link repr """ + el = ExternalLink('foo.hdf5','/foo') + self.assertIsInstance(repr(el), str) + + def test_create(self): + """ Creating external links """ + name = make_name() + self.f[name] = ExternalLink(self.ename, '/external') + grp = self.f[name] + self.ef = grp.file + self.assertNotEqual(self.ef, self.f) + self.assertEqual(grp.name, '/external') + + def test_exc(self): + """ KeyError raised when attempting to open broken link """ + name = make_name() + self.f[name] = ExternalLink(self.ename, '/missing') + with self.assertRaises(KeyError): + self.f[name] + + # I would prefer OSError but there's no way to fix this as the exception + # class is determined by HDF5. + def test_exc_missingfile(self): + """ KeyError raised when attempting to open missing file """ + name = make_name() + self.f[name] = ExternalLink('mongoose.hdf5','/foo') + with self.assertRaises(KeyError): + self.f[name] + + def test_close_file(self): + """ Files opened by accessing external links can be closed + + Issue 189. + """ + name = make_name() + self.f[name] = ExternalLink(self.ename, '/') + grp = self.f[name] + f2 = grp.file + f2.close() + self.assertFalse(f2) + + @ut.skipIf(NO_FS_UNICODE, "No unicode filename support") + def test_unicode_encode(self): + """ + Check that external links encode unicode filenames properly + Testing issue #732 + """ + ext_filename = os.path.join(mkdtemp(), u"α.hdf5") + with File(ext_filename, "w") as ext_file: + ext_file.create_group('external') + self.f[make_name()] = ExternalLink(ext_filename, '/external') + + @ut.skipIf(NO_FS_UNICODE, "No unicode filename support") + def test_unicode_decode(self): + """ + Check that external links decode unicode filenames properly + Testing issue #732 + """ + name = make_name() + ext_filename = os.path.join(mkdtemp(), u"α.hdf5") + with File(ext_filename, "w") as ext_file: + ext_file.create_group('external') + ext_file["external"].attrs["ext_attr"] = "test" + self.f[name] = ExternalLink(ext_filename, '/external') + self.assertEqual(self.f[name].attrs["ext_attr"], "test") + + def test_unicode_hdf5_path(self): + """ + Check that external links handle unicode hdf5 paths properly + Testing issue #333 + """ + name = make_name() + ext_filename = os.path.join(mkdtemp(), "external.hdf5") + with File(ext_filename, "w") as ext_file: + ext_file.create_group('α') + ext_file["α"].attrs["ext_attr"] = "test" + self.f[name] = ExternalLink(ext_filename, '/α') + self.assertEqual(self.f[name].attrs["ext_attr"], "test") + + +def test_get_elink_mode_arg(tmp_path): + """Check that external link is opened with modified access mode""" + external_filepath = tmp_path / make_name("external{}.h5") + with File(external_filepath, "w") as external_file: + external_file.create_group("/group") + + main_filepath = tmp_path / make_name("main{}.h5") + with File(main_filepath, "w") as main_file: + main_file["/external"] = h5py.ExternalLink(str(external_filepath), "/group") + + with File(main_filepath, "a") as main_file: + ext_group = main_file.get("/external", elink_mode="r") + assert ext_group.file.mode == "r" + with pytest.raises(ValueError): + ext_group['data'] = 1 # Writing fails + + +def test_get_elink_mode_arg_exception(tmp_path): + """Check that external links open in write mode from read-only file fails""" + external_filepath = tmp_path / make_name("external{}.h5") + with File(external_filepath, "w") as external_file: + external_file.create_group("/group") + + main_filepath = tmp_path / make_name("main{}.h5") + with File(main_filepath, "w") as main_file: + main_file["/external"] = h5py.ExternalLink(str(external_filepath), "/group") + + with File(main_filepath, "r") as main_file: + with pytest.raises(ValueError): + main_file.get("external", elink_mode="r+") + + +@pytest.mark.parametrize("file_swmr", (False, True)) +@pytest.mark.parametrize("elink_swmr", (False, True)) +def test_get_elink_swmr_arg_from_ro_file(tmp_path, file_swmr, elink_swmr): + """Check that external links are opened with custom read-only SWMR mode""" + external_filepath = tmp_path / make_name("external{}.h5") + with File(external_filepath, "w", libver="latest") as external_file: + external_file.create_group("/group") + + main_filepath = tmp_path / make_name("main{}.h5") + with File(main_filepath, "w", libver="latest") as main_file: + main_file["/external"] = h5py.ExternalLink(str(external_filepath), "/group") + + with File(main_filepath, "r", swmr=file_swmr) as main_file: + ext_group = main_file.get("external", elink_swmr=elink_swmr) + assert ext_group.file.mode == "r" + assert ext_group.file.swmr_mode == elink_swmr + + +@pytest.mark.parametrize("elink_swmr", (False, True)) +def test_get_elink_swmr_arg_from_rw_file(tmp_path, elink_swmr): + """Check that external links are opened with custom read-only SWMR mode""" + external_filepath = tmp_path / make_name("external{}.h5") + with File(external_filepath, "w", libver="latest") as external_file: + external_file.create_group("/group") + + main_filepath = tmp_path / make_name("main{}.h5") + with File(main_filepath, "w", libver="latest") as main_file: + main_file["/external"] = h5py.ExternalLink(str(external_filepath), "/group") + + with File(main_filepath, "a") as main_file: + ext_group = main_file.get("external", elink_mode="r", elink_swmr=elink_swmr) + assert ext_group.file.mode == "r" + assert ext_group.file.swmr_mode == elink_swmr + + +def test_get_elink_swmr_arg_from_rw_swmr_file(tmp_path): + """Check that exceptions are raised when changing access mode from a file opened in read-write SWMR mode""" + external_filepath = tmp_path / make_name("external{}.h5") + with File(external_filepath, "w", libver="latest") as external_file: + external_file.create_group("/group") + + main_filepath = tmp_path / make_name("main{}.h5") + with File(main_filepath, "w", libver="latest") as main_file: + main_file["/external"] = h5py.ExternalLink(str(external_filepath), "/group") + main_file.swmr_mode = True + + with pytest.raises(ValueError): + main_file.get("external", elink_mode="r") + + with pytest.raises(ValueError): + main_file.get("external", elink_swmr=False) + + +@pytest.mark.parametrize("file_locking", (False, True, "best-effort")) +@pytest.mark.parametrize( + "elink_locking,file_locking_props", + [ + (False, (0, 0)), + (True, (1, 0)), + ("best-effort", (1, 1)), + ] +) +def test_get_elink_locking_arg(tmp_path, file_locking, elink_locking, file_locking_props): + """Check that external links are opened with custom file locking""" + external_filepath = tmp_path / make_name("external{}.h5") + with File(external_filepath, "w") as external_file: + external_file.create_group("/group") + external_file["/group/data"] = 1, 2, 3 + + main_filepath = tmp_path / make_name("main{}.h5") + with File(main_filepath, "w") as main_file: + main_file["external"] = ExternalLink(str(external_filepath), "/group") + + with File(main_filepath, "r", locking=file_locking) as main_file: + cls = main_file.get("/external/data", getclass=True, elink_locking=elink_locking) + assert cls is Dataset + + link = main_file.get("/external/data", getlink=True, elink_locking=elink_locking) + assert isinstance(link, HardLink) + + ext_group = main_file.get("external", elink_locking=elink_locking) + if h5py.version.hdf5_version_tuple >= (1, 14, 4): + # Retrieving file locking is broken for older versions of HDF5 + access_plist = ext_group.file.id.get_access_plist() + assert access_plist.get_file_locking() == file_locking_props + + +class TestExtLinkBugs(TestCase): + + """ + Bugs: Specific regressions for external links + """ + + def test_issue_212(self): + """ Issue 212 + + Fails with: + + AttributeError: 'SharedConfig' object has no attribute 'lapl' + """ + def closer(x): + def w(): + try: + if x: + x.close() + except OSError: + pass + return w + orig_name = self.mktemp() + new_name = self.mktemp() + f = File(orig_name, 'w') + self.addCleanup(closer(f)) + f.create_group('a') + f.close() + + g = File(new_name, 'w') + self.addCleanup(closer(g)) + g['link'] = ExternalLink(orig_name, '/') # note root group + g.close() + + h = File(new_name, 'r') + self.addCleanup(closer(h)) + self.assertIsInstance(h['link']['a'], Group) + + +class TestCopy(TestCase): + + def setUp(self): + self.f1 = File(self.mktemp(), 'w') + self.f2 = File(self.mktemp(), 'w') + + def tearDown(self): + if self.f1: + self.f1.close() + if self.f2: + self.f2.close() + + def test_copy_path_to_path(self): + name1 = make_name("foo1") + name2 = make_name("foo2") + + foo1 = self.f1.create_group(name1) + foo1['bar'] = [1,2,3] + + self.f1.copy(name1, name2) + foo2 = self.f1[name2] + self.assertIsInstance(foo2, Group) + self.assertArrayEqual(foo2['bar'], np.array([1,2,3])) + + def test_copy_path_to_group(self): + name1 = make_name("foo1") + name2 = make_name("foo2") + + foo1 = self.f1.create_group(name1) + foo1['bar'] = [1,2,3] + baz = self.f1.create_group(name2) + + self.f1.copy(name1, baz) + foo2 = self.f1[name2] + self.assertIsInstance(foo2, Group) + self.assertArrayEqual(foo2[f"{name1}/bar"], np.array([1,2,3])) + + self.f1.copy(name1, self.f2['/']) + self.assertIsInstance(self.f2[name1], Group) + self.assertArrayEqual(self.f2[f"{name1}/bar"], np.array([1,2,3])) + + def test_copy_group_to_path(self): + name1 = make_name("foo1") + name2 = make_name("foo2") + + foo1 = self.f1.create_group(name1) + foo1['bar'] = [1,2,3] + + self.f1.copy(foo1, name2) + foo2 = self.f1[name2] + self.assertIsInstance(foo2, Group) + self.assertArrayEqual(foo2['bar'], np.array([1,2,3])) + + self.f2.copy(foo1, name1) + self.assertIsInstance(self.f2[name1], Group) + self.assertArrayEqual(self.f2[f"{name1}/bar"], np.array([1,2,3])) + + def test_copy_group_to_group(self): + name1 = make_name("foo1") + name2 = make_name("foo2") + + foo1 = self.f1.create_group(name1) + foo1['bar'] = [1,2,3] + foo2 = self.f1.create_group(name2) + + self.f1.copy(foo1, foo2) + foo2 = self.f1[name2] + self.assertIsInstance(foo2, Group) + self.assertArrayEqual(foo2[f"{name1}/bar"], np.array([1,2,3])) + + self.f1.copy(foo1, self.f2['/']) + self.assertIsInstance(self.f2[f"/{name1}"], Group) + self.assertArrayEqual(self.f2[f"{name1}/bar"], np.array([1,2,3])) + + def test_copy_dataset(self): + name1 = make_name("foo") + name2 = make_name("bar") + name3 = make_name("baz") + name4 = make_name("grp") + + self.f1[name1] = [1,2,3] + foo = self.f1[name1] + grp = self.f1.create_group(name4) + + self.f1.copy(foo, name2) + self.assertArrayEqual(self.f1[name2], np.array([1,2,3])) + + self.f1.copy(name1, name3) + self.assertArrayEqual(self.f1[name3], np.array([1,2,3])) + + self.f1.copy(foo, grp) + self.assertArrayEqual( + self.f1[f"/{name4}/{name1}"], np.array([1,2,3]) + ) + + self.f1.copy(name1, self.f2) + self.assertArrayEqual(self.f2[name1], np.array([1,2,3])) + + self.f2.copy(self.f1[name1], self.f2, name2) + self.assertArrayEqual(self.f2[name2], np.array([1,2,3])) + + def test_copy_shallow(self): + name1 = make_name("foo1") + name2 = make_name("foo2") + + foo1 = self.f1.create_group(name1) + bar = foo1.create_group('bar') + foo1['qux'] = [1,2,3] + bar['quux'] = [4,5,6] + + self.f1.copy(foo1, name2, shallow=True) + baz = self.f1[name2] + self.assertIsInstance(baz, Group) + self.assertIsInstance(baz['bar'], Group) + self.assertEqual(len(baz['bar']), 0) + self.assertArrayEqual(baz['qux'], np.array([1,2,3])) + + self.f2.copy(foo1, name1, shallow=True) + self.assertIsInstance(self.f2[f"/{name1}"], Group) + self.assertIsInstance(self.f2[f"{name1}/bar"], Group) + self.assertEqual(len(self.f2[f"{name1}/bar"]), 0) + self.assertArrayEqual(self.f2[f"{name1}/qux"], np.array([1,2,3])) + + def test_copy_without_attributes(self): + name1 = make_name("foo1") + name2 = make_name("foo2") + + self.f1[name1] = [1,2,3] + foo1 = self.f1[name1] + foo1.attrs['bar'] = [4,5,6] + + self.f1.copy(foo1, name2, without_attrs=True) + self.assertArrayEqual(self.f1[name2], np.array([1,2,3])) + assert 'bar' not in self.f1[name2].attrs + + self.f2.copy(foo1, name2, without_attrs=True) + self.assertArrayEqual(self.f2[name2], np.array([1,2,3])) + assert 'bar' not in self.f2[name2].attrs + + def test_copy_soft_links(self): + name1 = make_name("foo") + name2 = make_name("bar") + name3 = make_name("baz") + + self.f1[name2] = [1, 2, 3] + foo = self.f1.create_group(name1) + foo['qux'] = SoftLink(f"/{name2}") + + self.f1.copy(foo, name3, expand_soft=True) + self.f2.copy(foo, name1, expand_soft=True) + del self.f1[name2] + + self.assertIsInstance(self.f1[name3], Group) + self.assertArrayEqual(self.f1[f"{name3}/qux"], np.array([1, 2, 3])) + + self.assertIsInstance(self.f2[f"/{name1}"], Group) + self.assertArrayEqual(self.f2[f"{name1}/qux"], np.array([1, 2, 3])) + + def test_copy_external_links(self): + name1 = make_name("foo") + name2 = make_name("bar") + name3 = make_name("baz") + + filename = self.mktemp() + with File(filename, 'w') as f1: + f1[name1] = [1,2,3] + self.f2[name2] = ExternalLink(f1.filename, name1) + + self.assertArrayEqual(self.f2[name2], np.array([1,2,3])) + + self.f2.copy(name2, name3, expand_external=True) + os.unlink(filename) + self.assertArrayEqual(self.f2[name3], np.array([1,2,3])) + + def test_copy_refs(self): + name1 = make_name("foo") + name2 = make_name("bar") + name3 = make_name("baz") + name4 = make_name("qux") + + self.f1[name1] = [1,2,3] + self.f1[name2] = [4,5,6] + foo = self.f1[name1] + bar = self.f1[name2] + foo.attrs['bar'] = bar.ref + + self.f1.copy(foo, name3, expand_refs=True) + self.assertArrayEqual(self.f1[name3], np.array([1,2,3])) + baz_bar = self.f1[name3].attrs['bar'] + self.assertArrayEqual(self.f1[baz_bar], np.array([4,5,6])) + # The reference points to a copy of bar, not to bar itself. + self.assertNotEqual(self.f1[baz_bar].name, bar.name) + + self.f1.copy(name1, self.f2, name3, expand_refs=True) + self.assertArrayEqual(self.f2[name3], np.array([1,2,3])) + baz_bar = self.f2[name3].attrs['bar'] + self.assertArrayEqual(self.f2[baz_bar], np.array([4,5,6])) + + self.f1.copy('/', self.f2, name4, expand_refs=True) + self.assertArrayEqual(self.f2[f"{name4}/{name1}"], np.array([1,2,3])) + self.assertArrayEqual(self.f2[f"{name4}/{name2}"], np.array([4,5,6])) + foo_bar = self.f2[f"{name4}/{name1}"].attrs['bar'] + self.assertArrayEqual(self.f2[foo_bar], np.array([4,5,6])) + # There's only one copy of bar, which the reference points to. + self.assertEqual(self.f2[foo_bar], self.f2[f"{name4}/{name2}"]) + + +class TestMove(BaseGroup): + + """ + Feature: Group.move moves links in a file + """ + + def test_move_hardlink(self): + """ Moving an object """ + x = make_name("x") + y = make_name("y") + z = make_name("z{}/nested/path") + + grp = self.f.create_group(x) + self.f.move(x, y) + self.assertEqual(self.f[y], grp) + self.f.move(y, z) + self.assertEqual(self.f[z], grp) + + def test_move_softlink(self): + """ Moving a soft link """ + name1 = make_name("soft") + name2 = make_name("new_soft") + name3 = make_name("relative{}/path") + + self.f[name1] = h5py.SoftLink(name3) + self.f.move(name1, name2) + lnk = self.f.get(name2, getlink=True) + self.assertEqual(lnk.path, name3) + + def test_move_conflict(self): + """ Move conflict raises ValueError """ + x = make_name("x") + y = make_name("y") + + self.f.create_group(x) + self.f.create_group(y) + with self.assertRaises(ValueError): + self.f.move(x, y) + + def test_short_circuit(self): + ''' Test that a null-move works ''' + name = make_name() + self.f.create_group(name) + self.f.move(name, name) + + +class TestMutableMapping(BaseGroup): + '''Tests if the registration of Group as a MutableMapping + behaves as expected + ''' + def test_resolution(self): + assert issubclass(Group, MutableMapping) + grp = self.f.create_group(make_name()) + assert isinstance(grp, MutableMapping) + + def test_validity(self): + ''' + Test that the required functions are implemented. + ''' + Group.__getitem__ + Group.__setitem__ + Group.__delitem__ + Group.__iter__ + Group.__len__ diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_h5.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5.py new file mode 100644 index 0000000000000000000000000000000000000000..64b17a08a95089260bd204dbbd8171cb4e96badd --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5.py @@ -0,0 +1,45 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +from h5py import h5 + +from .common import TestCase + +def fixnames(): + cfg = h5.get_config() + cfg.complex_names = ('r','i') + +class TestH5(TestCase): + + def test_config(self): + cfg = h5.get_config() + self.assertIsInstance(cfg, h5.H5PYConfig) + cfg2 = h5.get_config() + self.assertIs(cfg, cfg2) + + def test_cnames_get(self): + cfg = h5.get_config() + self.assertEqual(cfg.complex_names, ('r','i')) + + def test_cnames_set(self): + self.addCleanup(fixnames) + cfg = h5.get_config() + cfg.complex_names = ('q','x') + self.assertEqual(cfg.complex_names, ('q','x')) + + def test_cnames_set_exc(self): + self.addCleanup(fixnames) + cfg = h5.get_config() + with self.assertRaises(TypeError): + cfg.complex_names = ('q','i','v') + self.assertEqual(cfg.complex_names, ('r','i')) + + def test_repr(self): + cfg = h5.get_config() + repr(cfg) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_h5d_direct_chunk.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5d_direct_chunk.py new file mode 100644 index 0000000000000000000000000000000000000000..f53a2c2707b9fb7636c13a53e1a78f27357287db --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5d_direct_chunk.py @@ -0,0 +1,185 @@ +import h5py +import numpy +import numpy.testing +import pytest + +from .common import ut, TestCase, make_name + + +class TestWriteDirectChunk(TestCase): + def test_write_direct_chunk(self): + + filename = self.mktemp().encode() + with h5py.File(filename, "w") as filehandle: + + dataset = filehandle.create_dataset("data", (100, 100, 100), + maxshape=(None, 100, 100), + chunks=(1, 100, 100), + dtype='float32') + + # writing + array = numpy.zeros((10, 100, 100)) + for index in range(10): + a = numpy.random.rand(100, 100).astype('float32') + dataset.id.write_direct_chunk((index, 0, 0), a.tobytes(), filter_mask=1) + array[index] = a + + + # checking + with h5py.File(filename, "r") as filehandle: + for i in range(10): + read_data = filehandle["data"][i] + numpy.testing.assert_array_equal(array[i], read_data) + + +@ut.skipIf('gzip' not in h5py.filters.encode, "DEFLATE is not installed") +class TestReadDirectChunk(TestCase): + def test_read_compressed_offsets(self): + + filename = self.mktemp().encode() + with h5py.File(filename, "w") as filehandle: + + frame = numpy.arange(16).reshape(4, 4) + frame_dataset = filehandle.create_dataset("frame", + data=frame, + compression="gzip", + compression_opts=9) + dataset = filehandle.create_dataset("compressed_chunked", + data=[frame, frame, frame], + compression="gzip", + compression_opts=9, + chunks=(1, ) + frame.shape) + filter_mask, compressed_frame = frame_dataset.id.read_direct_chunk((0, 0)) + # No filter must be disabled + self.assertEqual(filter_mask, 0) + + for i in range(dataset.shape[0]): + filter_mask, data = dataset.id.read_direct_chunk((i, 0, 0)) + self.assertEqual(compressed_frame, data) + # No filter must be disabled + self.assertEqual(filter_mask, 0) + + def test_read_uncompressed_offsets(self): + + filename = self.mktemp().encode() + frame = numpy.arange(16).reshape(4, 4) + with h5py.File(filename, "w") as filehandle: + dataset = filehandle.create_dataset("frame", + maxshape=(1,) + frame.shape, + shape=(1,) + frame.shape, + dtype="f4", + compression="gzip", + compression_opts=9) + # Write uncompressed data + DISABLE_ALL_FILTERS = 0xFFFFFFFF + dataset.id.write_direct_chunk((0, 0, 0), frame.tobytes(), filter_mask=DISABLE_ALL_FILTERS) + + # FIXME: Here we have to close the file and load it back else + # a runtime error occurs: + # RuntimeError: Can't get storage size of chunk (chunk storage is not allocated) + with h5py.File(filename, "r") as filehandle: + dataset = filehandle["frame"] + filter_mask, compressed_frame = dataset.id.read_direct_chunk((0, 0, 0)) + + # At least 1 filter is supposed to be disabled + self.assertNotEqual(filter_mask, 0) + self.assertEqual(compressed_frame, frame.tobytes()) + + def test_read_write_chunk(self): + + filename = self.mktemp().encode() + with h5py.File(filename, "w") as filehandle: + + # create a reference + frame = numpy.arange(16).reshape(4, 4) + frame_dataset = filehandle.create_dataset("source", + data=frame, + compression="gzip", + compression_opts=9) + # configure an empty dataset + filter_mask, compressed_frame = frame_dataset.id.read_direct_chunk((0, 0)) + dataset = filehandle.create_dataset("created", + shape=frame_dataset.shape, + maxshape=frame_dataset.shape, + chunks=frame_dataset.chunks, + dtype=frame_dataset.dtype, + compression="gzip", + compression_opts=9) + + # copy the data + dataset.id.write_direct_chunk((0, 0), compressed_frame, filter_mask=filter_mask) + + # checking + with h5py.File(filename, "r") as filehandle: + dataset = filehandle["created"][...] + numpy.testing.assert_array_equal(dataset, frame) + + +class TestReadDirectChunkToOut: + + def test_uncompressed_data(self, writable_file): + ref_data = numpy.arange(16).reshape(4, 4) + dataset = writable_file.create_dataset( + make_name(), data=ref_data, chunks=ref_data.shape) + + out = bytearray(ref_data.nbytes) + filter_mask, chunk = dataset.id.read_direct_chunk((0, 0), out=out) + + assert numpy.array_equal( + numpy.frombuffer(out, dtype=ref_data.dtype).reshape(ref_data.shape), + ref_data, + ) + assert filter_mask == 0 + assert len(chunk) == ref_data.nbytes + + @pytest.mark.skipif( + 'gzip' not in h5py.filters.encode, + reason="DEFLATE is not installed", + ) + def test_compressed_data(self, writable_file): + ref_data = numpy.arange(16).reshape(4, 4) + dataset = writable_file.create_dataset( + make_name(), + data=ref_data, + chunks=ref_data.shape, + compression="gzip", + compression_opts=9, + ) + chunk_info = dataset.id.get_chunk_info(0) + + out = bytearray(chunk_info.size) + filter_mask, chunk = dataset.id.read_direct_chunk( + chunk_info.chunk_offset, + out=out, + ) + assert filter_mask == chunk_info.filter_mask + assert len(chunk) == chunk_info.size + assert out == dataset.id.read_direct_chunk(chunk_info.chunk_offset)[1] + + def test_fail_buffer_too_small(self, writable_file): + ref_data = numpy.arange(16).reshape(4, 4) + dataset = writable_file.create_dataset( + make_name(), data=ref_data, chunks=ref_data.shape) + + out = bytearray(ref_data.nbytes // 2) + with pytest.raises(ValueError): + dataset.id.read_direct_chunk((0, 0), out=out) + + def test_fail_buffer_readonly(self, writable_file): + ref_data = numpy.arange(16).reshape(4, 4) + dataset = writable_file.create_dataset( + make_name(), data=ref_data, chunks=ref_data.shape) + + out = bytes(ref_data.nbytes) + with pytest.raises(BufferError): + dataset.id.read_direct_chunk((0, 0), out=out) + + def test_fail_buffer_not_contiguous(self, writable_file): + ref_data = numpy.arange(16).reshape(4, 4) + dataset = writable_file.create_dataset( + make_name(), data=ref_data, chunks=ref_data.shape) + + array = numpy.empty(ref_data.shape + (2,), dtype=ref_data.dtype) + out = array[:, :, ::2] # Array is not contiguous + with pytest.raises(ValueError): + dataset.id.read_direct_chunk((0, 0), out=out) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_h5f.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5f.py new file mode 100644 index 0000000000000000000000000000000000000000..98585a3b00567ca9be7eabfd2deb9bf796855beb --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5f.py @@ -0,0 +1,108 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +import tempfile +import shutil +import os +import numpy as np +from h5py import File, special_dtype +from h5py._hl.files import direct_vfd + +from .common import ut, TestCase + + +class TestFileID(TestCase): + def test_descriptor_core(self): + with File(self.mktemp(), driver='core', + backing_store=False, mode='x') as f: + assert isinstance(f.id.get_vfd_handle(), int) + + def test_descriptor_sec2(self): + dn_tmp = tempfile.mkdtemp('h5py.lowtest.test_h5f.TestFileID.test_descriptor_sec2') + fn_h5 = os.path.join(dn_tmp, 'test.h5') + try: + with File(fn_h5, driver='sec2', mode='x') as f: + descriptor = f.id.get_vfd_handle() + self.assertNotEqual(descriptor, 0) + os.fsync(descriptor) + finally: + shutil.rmtree(dn_tmp) + + @ut.skipUnless(direct_vfd, + "DIRECT driver is supported on Linux if hdf5 is " + "built with the appriorate flags.") + def test_descriptor_direct(self): + dn_tmp = tempfile.mkdtemp('h5py.lowtest.test_h5f.TestFileID.test_descriptor_direct') + fn_h5 = os.path.join(dn_tmp, 'test.h5') + try: + with File(fn_h5, driver='direct', mode='x') as f: + descriptor = f.id.get_vfd_handle() + self.assertNotEqual(descriptor, 0) + os.fsync(descriptor) + finally: + shutil.rmtree(dn_tmp) + + +class TestCacheConfig(TestCase): + def test_simple_gets(self): + dn_tmp = tempfile.mkdtemp('h5py.lowtest.test_h5f.TestFileID.TestCacheConfig.test_simple_gets') + fn_h5 = os.path.join(dn_tmp, 'test.h5') + try: + with File(fn_h5, mode='x') as f: + hit_rate = f._id.get_mdc_hit_rate() + mdc_size = f._id.get_mdc_size() + + finally: + shutil.rmtree(dn_tmp) + + def test_hitrate_reset(self): + dn_tmp = tempfile.mkdtemp('h5py.lowtest.test_h5f.TestFileID.TestCacheConfig.test_hitrate_reset') + fn_h5 = os.path.join(dn_tmp, 'test.h5') + try: + with File(fn_h5, mode='x') as f: + hit_rate = f._id.get_mdc_hit_rate() + f._id.reset_mdc_hit_rate_stats() + hit_rate = f._id.get_mdc_hit_rate() + assert hit_rate == 0 + + finally: + shutil.rmtree(dn_tmp) + + def test_mdc_config_get(self): + dn_tmp = tempfile.mkdtemp('h5py.lowtest.test_h5f.TestFileID.TestCacheConfig.test_mdc_config_get') + fn_h5 = os.path.join(dn_tmp, 'test.h5') + try: + with File(fn_h5, mode='x') as f: + conf = f._id.get_mdc_config() + f._id.set_mdc_config(conf) + finally: + shutil.rmtree(dn_tmp) + + +class TestVlenData(TestCase): + def test_vlen_strings(self): + # Create file with dataset containing vlen arrays of vlen strings + dn_tmp = tempfile.mkdtemp('h5py.lowtest.test_h5f.TestVlenStrings.test_vlen_strings') + fn_h5 = os.path.join(dn_tmp, 'test.h5') + try: + with File(fn_h5, mode='w') as h: + vlen_str = special_dtype(vlen=str) + vlen_vlen_str = special_dtype(vlen=vlen_str) + + ds = h.create_dataset('/com', (2,), dtype=vlen_vlen_str) + ds[0] = (np.array(["a", "b", "c"], dtype=vlen_vlen_str)) + ds[1] = (np.array(["d", "e", "f","g"], dtype=vlen_vlen_str)) + + with File(fn_h5, "r") as h: + ds = h["com"] + assert ds[0].tolist() == [b'a', b'b', b'c'] + assert ds[1].tolist() == [b'd', b'e', b'f', b'g'] + + finally: + shutil.rmtree(dn_tmp) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_h5o.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5o.py new file mode 100644 index 0000000000000000000000000000000000000000..c5de6a57912234fd366661ec4003f83e2e700798 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5o.py @@ -0,0 +1,20 @@ +import pytest + +from .common import TestCase, make_name +from h5py import File + + +class SampleException(Exception): + pass + +def throwing(name, obj): + raise SampleException(f"throwing exception for {name} {obj}") + +class TestVisit(TestCase): + def test_visit(self): + fname = self.mktemp() + fid = File(fname, 'w') + fid.create_dataset(make_name(), (100,), dtype='uint8') + with pytest.raises(SampleException, match='throwing exception'): + fid.visititems(throwing) + fid.close() diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_h5p.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5p.py new file mode 100644 index 0000000000000000000000000000000000000000..dc8560b945ada1cd0d80f9d597764fe3291f3523 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5p.py @@ -0,0 +1,233 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +import unittest as ut + +from h5py import h5p, h5f, version + +from .common import TestCase + + +class TestLibver(TestCase): + + """ + Feature: Setting/getting lib ver bounds + """ + + def test_libver(self): + """ Test libver bounds set/get """ + plist = h5p.create(h5p.FILE_ACCESS) + plist.set_libver_bounds(h5f.LIBVER_EARLIEST, h5f.LIBVER_LATEST) + self.assertEqual((h5f.LIBVER_EARLIEST, h5f.LIBVER_LATEST), + plist.get_libver_bounds()) + + def test_libver_v18(self): + """ Test libver bounds set/get for H5F_LIBVER_V18""" + plist = h5p.create(h5p.FILE_ACCESS) + plist.set_libver_bounds(h5f.LIBVER_EARLIEST, h5f.LIBVER_V18) + self.assertEqual((h5f.LIBVER_EARLIEST, h5f.LIBVER_V18), + plist.get_libver_bounds()) + + def test_libver_v110(self): + """ Test libver bounds set/get for H5F_LIBVER_V110""" + plist = h5p.create(h5p.FILE_ACCESS) + plist.set_libver_bounds(h5f.LIBVER_V18, h5f.LIBVER_V110) + self.assertEqual((h5f.LIBVER_V18, h5f.LIBVER_V110), + plist.get_libver_bounds()) + + @ut.skipIf(version.hdf5_version_tuple < (1, 11, 4), + 'Requires HDF5 1.11.4 or later') + def test_libver_v112(self): + """ Test libver bounds set/get for H5F_LIBVER_V112""" + plist = h5p.create(h5p.FILE_ACCESS) + plist.set_libver_bounds(h5f.LIBVER_V18, h5f.LIBVER_V112) + self.assertEqual((h5f.LIBVER_V18, h5f.LIBVER_V112), + plist.get_libver_bounds()) + + @ut.skipIf(version.hdf5_version_tuple < (1, 14, 0), + 'Requires HDF5 1.14 or later') + def test_libver_v114(self): + """ Test libver bounds set/get for H5F_LIBVER_V114""" + plist = h5p.create(h5p.FILE_ACCESS) + plist.set_libver_bounds(h5f.LIBVER_V18, h5f.LIBVER_V114) + self.assertEqual((h5f.LIBVER_V18, h5f.LIBVER_V114), + plist.get_libver_bounds()) + + @ut.skipIf(version.hdf5_version_tuple < (2, 0, 0), + 'Requires HDF5 2.0 or later') + def test_libver_v200(self): + """ Test libver bounds set/get for H5F_LIBVER_V200""" + plist = h5p.create(h5p.FILE_ACCESS) + plist.set_libver_bounds(h5f.LIBVER_V18, h5f.LIBVER_V200) + self.assertEqual((h5f.LIBVER_V18, h5f.LIBVER_V200), + plist.get_libver_bounds()) + + +class TestDA(TestCase): + ''' + Feature: setting/getting chunk cache size on a dataset access property list + ''' + def test_chunk_cache(self): + '''test get/set chunk cache ''' + dalist = h5p.create(h5p.DATASET_ACCESS) + nslots = 10000 # 40kb hash table + nbytes = 1000000 # 1MB cache size + w0 = .5 # even blend of eviction strategy + + dalist.set_chunk_cache(nslots, nbytes, w0) + self.assertEqual((nslots, nbytes, w0), + dalist.get_chunk_cache()) + + def test_efile_prefix(self): + '''test get/set efile prefix ''' + dalist = h5p.create(h5p.DATASET_ACCESS) + self.assertEqual(dalist.get_efile_prefix().decode(), '') + + efile_prefix = "path/to/external/dataset" + dalist.set_efile_prefix(efile_prefix.encode('utf-8')) + self.assertEqual(dalist.get_efile_prefix().decode(), + efile_prefix) + + efile_prefix = "${ORIGIN}" + dalist.set_efile_prefix(efile_prefix.encode('utf-8')) + self.assertEqual(dalist.get_efile_prefix().decode(), + efile_prefix) + + def test_virtual_prefix(self): + '''test get/set virtual prefix ''' + dalist = h5p.create(h5p.DATASET_ACCESS) + self.assertEqual(dalist.get_virtual_prefix().decode(), '') + + virtual_prefix = "path/to/virtual/dataset" + dalist.set_virtual_prefix(virtual_prefix.encode('utf-8')) + self.assertEqual(dalist.get_virtual_prefix().decode(), + virtual_prefix) + + +class TestFA(TestCase): + ''' + Feature: setting/getting mdc config on a file access property list + ''' + def test_mdc_config(self): + '''test get/set mdc config ''' + falist = h5p.create(h5p.FILE_ACCESS) + + config = falist.get_mdc_config() + falist.set_mdc_config(config) + + def test_set_alignment(self): + '''test get/set chunk cache ''' + falist = h5p.create(h5p.FILE_ACCESS) + threshold = 10 * 1024 # threshold of 10kiB + alignment = 1024 * 1024 # threshold of 1kiB + + falist.set_alignment(threshold, alignment) + self.assertEqual((threshold, alignment), + falist.get_alignment()) + + def test_set_file_locking(self): + '''test get/set file locking''' + falist = h5p.create(h5p.FILE_ACCESS) + use_file_locking = False + ignore_when_disabled = False + + falist.set_file_locking(use_file_locking, ignore_when_disabled) + self.assertEqual((use_file_locking, ignore_when_disabled), + falist.get_file_locking()) + + +class TestPL(TestCase): + def test_obj_track_times(self): + """ + tests if the object track times set/get + """ + # test for groups + gcid = h5p.create(h5p.GROUP_CREATE) + gcid.set_obj_track_times(False) + self.assertEqual(False, gcid.get_obj_track_times()) + + gcid.set_obj_track_times(True) + self.assertEqual(True, gcid.get_obj_track_times()) + # test for datasets + dcid = h5p.create(h5p.DATASET_CREATE) + dcid.set_obj_track_times(False) + self.assertEqual(False, dcid.get_obj_track_times()) + + dcid.set_obj_track_times(True) + self.assertEqual(True, dcid.get_obj_track_times()) + + # test for generic objects + ocid = h5p.create(h5p.OBJECT_CREATE) + ocid.set_obj_track_times(False) + self.assertEqual(False, ocid.get_obj_track_times()) + + ocid.set_obj_track_times(True) + self.assertEqual(True, ocid.get_obj_track_times()) + + def test_link_creation_tracking(self): + """ + tests the link creation order set/get + """ + + gcid = h5p.create(h5p.GROUP_CREATE) + gcid.set_link_creation_order(0) + self.assertEqual(0, gcid.get_link_creation_order()) + + flags = h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED + gcid.set_link_creation_order(flags) + self.assertEqual(flags, gcid.get_link_creation_order()) + + # test for file creation + fcpl = h5p.create(h5p.FILE_CREATE) + fcpl.set_link_creation_order(flags) + self.assertEqual(flags, fcpl.get_link_creation_order()) + + def test_attr_phase_change(self): + """ + test the attribute phase change + """ + + cid = h5p.create(h5p.OBJECT_CREATE) + # test default value + ret = cid.get_attr_phase_change() + self.assertEqual((8,6), ret) + + # max_compact must < 65536 (64kb) + with self.assertRaises(ValueError): + cid.set_attr_phase_change(65536, 6) + + # Using dense attributes storage to avoid 64kb size limitation + # for a single attribute in compact attribute storage. + cid.set_attr_phase_change(0, 0) + self.assertEqual((0,0), cid.get_attr_phase_change()) + + +def test_proplaid(): + """Test Link Access Property List""" + lapl = h5p.create(h5p.LINK_ACCESS) + + nlinks = 3 + lapl.set_nlinks(nlinks) + assert lapl.get_nlinks() == nlinks + + prefix = b"/prefix" + lapl.set_elink_prefix(prefix) + assert lapl.get_elink_prefix() == prefix + + flags = h5f.ACC_RDWR & h5f.ACC_SWMR_WRITE + lapl.set_elink_acc_flags(flags) + assert lapl.get_elink_acc_flags() == flags + + fapl = h5p.create(h5p.FILE_ACCESS) + fapl.set_file_locking(False, False) + lapl.set_elink_fapl(fapl) + assert lapl.get_elink_fapl().get_file_locking() == (False, False) + + fapl.close() + lapl.close() diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_h5pl.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5pl.py new file mode 100644 index 0000000000000000000000000000000000000000..6f60a325496b0426c36d24dadf12ace7d67ff088 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5pl.py @@ -0,0 +1,68 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2019 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +import pytest + +from h5py import h5pl +from h5py.tests.common import insubprocess, subproc_env + + +@pytest.mark.mpi_skip +@insubprocess +@subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'}) +def test_default(request): + assert h5pl.size() == 1 + assert h5pl.get(0) == b'h5py_plugin_test' + + +@pytest.mark.mpi_skip +@insubprocess +@subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'}) +def test_append(request): + h5pl.append(b'/opt/hdf5/vendor-plugin') + assert h5pl.size() == 2 + assert h5pl.get(0) == b'h5py_plugin_test' + assert h5pl.get(1) == b'/opt/hdf5/vendor-plugin' + + +@pytest.mark.mpi_skip +@insubprocess +@subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'}) +def test_prepend(request): + h5pl.prepend(b'/opt/hdf5/vendor-plugin') + assert h5pl.size() == 2 + assert h5pl.get(0) == b'/opt/hdf5/vendor-plugin' + assert h5pl.get(1) == b'h5py_plugin_test' + + +@pytest.mark.mpi_skip +@insubprocess +@subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'}) +def test_insert(request): + h5pl.insert(b'/opt/hdf5/vendor-plugin', 0) + assert h5pl.size() == 2 + assert h5pl.get(0) == b'/opt/hdf5/vendor-plugin' + assert h5pl.get(1) == b'h5py_plugin_test' + + +@pytest.mark.mpi_skip +@insubprocess +@subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'}) +def test_replace(request): + h5pl.replace(b'/opt/hdf5/vendor-plugin', 0) + assert h5pl.size() == 1 + assert h5pl.get(0) == b'/opt/hdf5/vendor-plugin' + + +@pytest.mark.mpi_skip +@insubprocess +@subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'}) +def test_remove(request): + h5pl.remove(0) + assert h5pl.size() == 0 diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_h5s.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5s.py new file mode 100644 index 0000000000000000000000000000000000000000..df09fb5a9e08f2fd342cf0c4dfc921df1b854f53 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5s.py @@ -0,0 +1,66 @@ +import numpy as np + +from h5py import h5s +from h5py._selector import Selector + +class Helper: + def __init__(self, shape: tuple): + self.shape = shape + + def __getitem__(self, item) -> h5s.SpaceID: + if not isinstance(item, tuple): + item = (item,) + space = h5s.create_simple(self.shape) + sel = Selector(space) + sel.make_selection(item) + return space + + +def test_same_shape(): + s1 = Helper((5, 6))[:3, :4] + s2 = Helper((5, 6))[2:, 2:] + assert s1.select_shape_same(s2) + + s3 = Helper((5, 6))[:4, :3] + assert not s1.select_shape_same(s3) + +def test_select_copy(): + s1 = h5s.create_simple((50,)) + s1.select_hyperslab((5,), (1,), block=(5,), op=h5s.SELECT_SET) + s2 = h5s.create_simple((50,)) + s2.select_hyperslab((23,), (1,), block=(3,), op=h5s.SELECT_SET) + s2.select_copy(s1) + np.testing.assert_array_equal( + s1.get_select_hyper_blocklist(), s2.get_select_hyper_blocklist() + ) + +def test_combine_select(): + # combine_select should create a new space without modifying the original + s1 = h5s.create_simple((50,)) + s1.select_hyperslab((5,), (1,), block=(5,), op=h5s.SELECT_SET) + + s2 = h5s.create_simple((50,)) + s2.select_hyperslab((20,), (1,), block=(2,), op=h5s.SELECT_SET) + + s3 = s1.combine_select(s2, op=h5s.SELECT_OR) + s2.select_hyperslab((5,), (1,), block=(5,), op=h5s.SELECT_OR) + np.testing.assert_array_equal( + s3.get_select_hyper_blocklist(), s2.get_select_hyper_blocklist() + ) + assert not np.array_equal( + s3.get_select_hyper_blocklist(), s1.get_select_hyper_blocklist() + ) + +def test_modify_select(): + # combine_select should create a new space without modifying the original + s1 = h5s.create_simple((50,)) + s1.select_hyperslab((5,), (1,), block=(5,), op=h5s.SELECT_SET) + + s2 = h5s.create_simple((50,)) + s2.select_hyperslab((20,), (1,), block=(2,), op=h5s.SELECT_SET) + + s1.modify_select(s2, op=h5s.SELECT_OR) + s2.select_hyperslab((5,), (1,), block=(5,), op=h5s.SELECT_OR) + np.testing.assert_array_equal( + s1.get_select_hyper_blocklist(), s2.get_select_hyper_blocklist() + ) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_h5t.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5t.py new file mode 100644 index 0000000000000000000000000000000000000000..b3175d8d8b7ea8a4ba8ddafc1369d89d88635ce8 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5t.py @@ -0,0 +1,188 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +import numpy as np + +import h5py +from h5py import h5t + +from .common import TestCase, ut + + +class TestCompound(ut.TestCase): + + """ + Feature: Compound types can be created from Python dtypes + """ + + def test_ref(self): + """ Reference types are correctly stored in compound types (issue 144) + """ + dt = np.dtype([('a', h5py.ref_dtype), ('b', '<f4')]) + tid = h5t.py_create(dt, logical=True) + t1, t2 = tid.get_member_type(0), tid.get_member_type(1) + self.assertEqual(t1, h5t.STD_REF_OBJ) + self.assertEqual(t2, h5t.IEEE_F32LE) + self.assertEqual(tid.get_member_offset(0), 0) + self.assertEqual(tid.get_member_offset(1), h5t.STD_REF_OBJ.get_size()) + + def test_out_of_order_offsets(self): + size = 20 + type_dict = { + 'names': ['f1', 'f2', 'f3'], + 'formats': ['<f4', '<i4', '<f8'], + 'offsets': [0, 16, 8] + } + + expected_dtype = np.dtype(type_dict) + + tid = h5t.create(h5t.COMPOUND, size) + for name, offset, dt in zip( + type_dict["names"], type_dict["offsets"], type_dict["formats"], strict=True + ): + tid.insert( + name.encode("utf8") if isinstance(name, str) else name, + offset, + h5t.py_create(dt) + ) + + self.assertEqual(tid.dtype, expected_dtype) + self.assertEqual(tid.dtype.itemsize, size) + + +class TestTypeFloatID(TestCase): + """Test TypeFloatID.""" + + def test_custom_float_promotion(self): + """Custom floats are correctly promoted to standard floats on read.""" + + # This test uses the low-level API, so we need names as byte strings + test_filename = self.mktemp().encode() + dataset = b'DS1' + dataset2 = b'DS2' + dataset3 = b'DS3' + dataset4 = b'DS4' + dataset5 = b'DS5' + + dims = (4, 7) + + wdata = np.array([[-1.50066626e-09, 1.40062184e-09, 1.81216819e-10, + 4.01087163e-10, 4.27917257e-10, -7.04858394e-11, + 5.74800652e-10], + [-1.50066626e-09, 4.86579665e-10, 3.42879503e-10, + 5.12045517e-10, 5.10226528e-10, 2.24190444e-10, + 3.93356459e-10], + [-1.50066626e-09, 5.24778443e-10, 8.19454726e-10, + 1.28966349e-09, 1.68483894e-10, 5.71276360e-11, + -1.08684617e-10], + [-1.50066626e-09, -1.08343556e-10, -1.58934199e-10, + 8.52196536e-10, 6.18456397e-10, 6.16637408e-10, + 1.31694833e-09]], dtype=np.float32) + + wdata2 = np.array([[-1.50066626e-09, 5.63886715e-10, -8.74251782e-11, + 1.32558853e-10, 1.59161573e-10, 2.29420039e-10, + -7.24185156e-11], + [-1.50066626e-09, 1.87810656e-10, 7.74889486e-10, + 3.95630195e-10, 9.42236511e-10, 8.38554115e-10, + -8.71978045e-11], + [-1.50066626e-09, 6.20275387e-10, 7.34871719e-10, + 6.64840627e-10, 2.64662958e-10, 1.05319486e-09, + 1.68256520e-10], + [-1.50066626e-09, 1.67347025e-10, 5.12045517e-10, + 3.36513040e-10, 1.02545528e-10, 1.28784450e-09, + 4.06089384e-10]], dtype=np.float32) + + # Create a new file using the default properties. + fid = h5py.h5f.create(test_filename) + # Create the dataspace. No maximum size parameter needed. + space = h5py.h5s.create_simple(dims) + + # create a custom type with larger bias + mytype = h5t.IEEE_F16LE.copy() + mytype.set_fields(14, 9, 5, 0, 9) + mytype.set_size(2) + mytype.set_ebias(53) + mytype.lock() + + dset = h5py.h5d.create(fid, dataset, mytype, space) + dset.write(h5py.h5s.ALL, h5py.h5s.ALL, wdata) + + del dset + + # create a custom type with larger exponent + mytype2 = h5t.IEEE_F16LE.copy() + mytype2.set_fields(15, 9, 6, 0, 9) + mytype2.set_size(2) + mytype2.set_ebias(53) + mytype2.lock() + + dset = h5py.h5d.create(fid, dataset2, mytype2, space) + dset.write(h5py.h5s.ALL, h5py.h5s.ALL, wdata2) + + del dset + + # create a custom type which reimplements 16-bit floats + mytype3 = h5t.IEEE_F16LE.copy() + mytype3.set_fields(15, 10, 5, 0, 10) + mytype3.set_size(2) + mytype3.set_ebias(15) + mytype3.lock() + + dset = h5py.h5d.create(fid, dataset3, mytype3, space) + dset.write(h5py.h5s.ALL, h5py.h5s.ALL, wdata2) + + del dset + + # create a custom type with larger bias + mytype4 = h5t.IEEE_F16LE.copy() + mytype4.set_fields(15, 10, 5, 0, 10) + mytype4.set_size(2) + mytype4.set_ebias(258) + mytype4.lock() + + dset = h5py.h5d.create(fid, dataset4, mytype4, space) + dset.write(h5py.h5s.ALL, h5py.h5s.ALL, wdata2) + + del dset + + # create a dataset with long doubles + dset = h5py.h5d.create(fid, dataset5, h5t.NATIVE_LDOUBLE, space) + dset.write(h5py.h5s.ALL, h5py.h5s.ALL, wdata2) + + # Explicitly close and release resources. + del space + del dset + del fid + + f = h5py.File(test_filename, 'r') + + # ebias promotion to float32 + values = f[dataset][:] + np.testing.assert_array_equal(values, wdata) + self.assertEqual(values.dtype, np.dtype('<f4')) + + # esize promotion to float32 + values = f[dataset2][:] + np.testing.assert_array_equal(values, wdata2) + self.assertEqual(values.dtype, np.dtype('<f4')) + + # regular half floats + dset = f[dataset3] + try: + self.assertEqual(dset.dtype, np.dtype('<f2')) + except AttributeError: + self.assertEqual(dset.dtype, np.dtype('<f4')) + + # ebias promotion to float64 + dset = f[dataset4] + self.assertEqual(dset.dtype, np.dtype('<f8')) + + # long double floats + dset = f[dataset5] + self.assertEqual(dset.dtype, np.longdouble) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_h5z.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5z.py new file mode 100644 index 0000000000000000000000000000000000000000..ad8bbb2638a69f3b2e42e30b11af0c94e816792b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_h5z.py @@ -0,0 +1,86 @@ +from ctypes import ( + addressof, + c_char_p, + c_int, + c_long, + c_uint, + c_void_p, + CFUNCTYPE, + POINTER, + Structure, +) +import pytest +import h5py +from h5py import h5z + +from .common import insubprocess + + +# Type of filter callback function of H5Z_class2_t +H5ZFuncT = CFUNCTYPE( + c_long, # restype + # argtypes + c_uint, # flags + c_long, # cd_nelemts + POINTER(c_uint), # cd_values + c_long, # nbytes + POINTER(c_long), # buf_size + POINTER(c_void_p), # buf +) + + +class H5ZClass2T(Structure): + """H5Z_class2_t structure defining a filter""" + + _fields_ = [ + ("version", c_int), + ("id_", c_int), + ("encoder_present", c_uint), + ("decoder_present", c_uint), + ("name", c_char_p), + ("can_apply", c_void_p), + ("set_local", c_void_p), + ("filter_", H5ZFuncT), + ] + + +@pytest.mark.thread_unsafe(reason="fixed filter_id") +def test_register_filter(): + filter_id = 256 # Test ID + + @H5ZFuncT + def failing_filter_callback(flags, cd_nelemts, cd_values, nbytes, buf_size, buf): + return 0 + + dummy_filter_class = H5ZClass2T( + version=h5z.CLASS_T_VERS, + id_=filter_id, + encoder_present=1, + decoder_present=1, + name=b"dummy filter", + can_apply=None, + set_local=None, + filter_=failing_filter_callback, + ) + + h5z.register_filter(addressof(dummy_filter_class)) + + try: + assert h5z.filter_avail(filter_id) + filter_flags = h5z.get_filter_info(filter_id) + assert ( + filter_flags + == h5z.FILTER_CONFIG_ENCODE_ENABLED | h5z.FILTER_CONFIG_DECODE_ENABLED + ) + finally: + h5z.unregister_filter(filter_id) + + assert not h5z.filter_avail(filter_id) + + +@pytest.mark.mpi_skip +@insubprocess +def test_unregister_filter(request): + if h5py.h5z.filter_avail(h5py.h5z.FILTER_LZF): + res = h5py.h5z.unregister_filter(h5py.h5z.FILTER_LZF) + assert res diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_native_complex.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_native_complex.py new file mode 100644 index 0000000000000000000000000000000000000000..547e19f83e8432f62b50f6d87d9a9b997362b2b0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_native_complex.py @@ -0,0 +1,149 @@ +# This file is part of h5py, a Python interface to the HDF5 library. +# +# http://www.h5py.org +# +# Copyright 2008-2013 Andrew Collette and contributors +# +# License: Standard 3-clause BSD; see "license.txt" for full license terms +# and contributor agreement. + +""" +Testing native complex number datatypes. +""" + +import sys + +import numpy as np +import pytest + +import h5py +from h5py.h5 import get_config # type: ignore +from h5py import h5t +from .common import make_name +from .data_files import get_data_file_path + +cfg = get_config() + +pytestmark = [ + pytest.mark.skipif( + h5py.version.hdf5_version_tuple < (2, 0, 0), + reason="Requires HDF5 >= 2.0", + ), + pytest.mark.skipif( + not cfg.has_native_complex, + reason="Native HDF5 complex number datatypes not available", + ), +] + +REQUIRE_NUMPY_COMPLEX256 = pytest.mark.skipif( + not hasattr(np, "complex256"), + reason="complex256 type is not available in numpy", +) + +NATIVE_BYTE_ORDER = h5t.ORDER_BE if sys.byteorder == "big" else h5t.ORDER_LE + + +@pytest.mark.parametrize( + "h5type_str, size, order", + [ + pytest.param("COMPLEX_IEEE_F16LE", 4, h5t.ORDER_LE), + pytest.param("COMPLEX_IEEE_F16BE", 4, h5t.ORDER_BE, id="F16BE"), + pytest.param("COMPLEX_IEEE_F32LE", 8, h5t.ORDER_LE, id="F32LE"), + pytest.param("COMPLEX_IEEE_F32BE", 8, h5t.ORDER_BE, id="F32BE"), + pytest.param("COMPLEX_IEEE_F64LE", 16, h5t.ORDER_LE, id="F64LE"), + pytest.param("COMPLEX_IEEE_F64BE", 16, h5t.ORDER_BE, id="F64BE"), + pytest.param("NATIVE_FLOAT_COMPLEX", 8, NATIVE_BYTE_ORDER, id="float-native"), + pytest.param("NATIVE_DOUBLE_COMPLEX", 16, NATIVE_BYTE_ORDER, id="double-native"), + pytest.param( + "NATIVE_LDOUBLE_COMPLEX", 32, NATIVE_BYTE_ORDER, + id="long-native", + marks=REQUIRE_NUMPY_COMPLEX256, + ) + ] +) +def test_hdf5_dtype(h5type_str, size, order): + """Low-level checks of HDF5 native complex number datatypes""" + h5type = getattr(h5t, h5type_str) + assert isinstance(h5type, h5t.TypeComplexID) + assert h5type.get_size() == size + assert h5type.get_order() == order + +H5_VS_NUMPY_DTYPES = [ + pytest.param("COMPLEX_IEEE_F32LE", "<c8", id="<c8"), + pytest.param("COMPLEX_IEEE_F32BE", ">c8", id=">c8"), + pytest.param("COMPLEX_IEEE_F64LE", "<c16", id="<c16"), + pytest.param("COMPLEX_IEEE_F64BE", ">c16", id=">c16"), + pytest.param("NATIVE_FLOAT_COMPLEX", "=c8", id="=c8"), + pytest.param("NATIVE_DOUBLE_COMPLEX", "=c16", id="=c16"), + pytest.param( + "NATIVE_LDOUBLE_COMPLEX", "=c32", id="=c32", + marks=REQUIRE_NUMPY_COMPLEX256, + ), +] + +@pytest.mark.parametrize("h5type_str, dt", H5_VS_NUMPY_DTYPES) +def test_cmplx_type_trnslt(h5type_str, dt): + """Translate native HDF5 complex number datatype to NumPy dtype""" + assert getattr(h5t, h5type_str).dtype == np.dtype(dt) + + +def check_compound_complex_datatype(datatype, np_dtype): + """Check h5py's old complex number type""" + assert isinstance(datatype, h5t.TypeCompoundID) + assert datatype.get_nmembers() == 2 + assert datatype.get_member_name(0) == cfg._r_name + assert datatype.get_member_name(1) == cfg._i_name + assert isinstance(datatype.get_member_type(0), h5t.TypeFloatID) + assert isinstance(datatype.get_member_type(1), h5t.TypeFloatID) + + assert ( + datatype.get_member_type(0).get_size() + + datatype.get_member_type(1).get_size() + ) == np_dtype.itemsize + + match (np_dtype.byteorder, sys.byteorder): + case (">", _) | ("=", "big"): + expected = h5t.ORDER_BE + case ("<", _) | ("=", "little"): + expected = h5t.ORDER_LE + case _ as _unreachable: + raise AssertionError + + assert datatype.get_member_type(0).get_order() == expected + assert datatype.get_member_type(1).get_order() == expected + + +@pytest.mark.parametrize("dt", ["<c8", ">c8", "<c16", ">c16"]) +def test_default_create(writable_file, dt): + """Test default translation when creating datasets & attributes""" + complex_array = (np.random.rand(100) + 1j * np.random.rand(100)).astype(dt) + ds = writable_file.create_dataset(make_name(dt), data=complex_array) + c = np.array(1.9 + 1j * 6.7, dtype=dt) + ds.attrs["c"] = c + check_compound_complex_datatype(ds.id.get_type(), np.dtype(dt)) + check_compound_complex_datatype(ds.attrs.get_id("c").get_type(), np.dtype(dt)) + np.testing.assert_array_equal(ds[...], complex_array) + np.testing.assert_array_equal(ds.attrs["c"], c) + + +@pytest.mark.parametrize("h5type_str, dt", H5_VS_NUMPY_DTYPES) +def test_explicit_create(writable_file, h5type_str, dt): + """Explicitly use native complex datatype to create datasets & attributes""" + h5type = getattr(h5t, h5type_str) + np_dt = np.dtype(dt) + complex_array = (np.random.rand(100) + 1j * np.random.rand(100)).astype(np_dt) + ds = writable_file.create_dataset(make_name(dt), (100,), dtype=h5py.Datatype(h5type)) + ds[:] = complex_array + c = np.array(1.9 + 1j * 6.7, dtype=np_dt) + ds.attrs.create("c", c, dtype=h5py.Datatype(h5type)) + assert isinstance(ds.id.get_type(), h5t.TypeComplexID) + assert isinstance(ds.attrs.get_id("c").get_type(), h5t.TypeComplexID) + np.testing.assert_array_equal(ds[...], complex_array) + np.testing.assert_array_equal(ds.attrs["c"], c) + + +def test_dtype_cmpnd_cmplx(): + """Check the resulting dtype of the two-field compound data as complex numbers""" + with h5py.File(get_data_file_path("compound-dtype-complex.h5"), mode="r") as f: + for obj in filter(lambda obj: isinstance(obj, h5py.Dataset), f.values()): + check_compound_complex_datatype(obj.id.get_type(), obj.dtype) diff --git a/.venv/lib/python3.11/site-packages/h5py/tests/test_npystrings.py b/.venv/lib/python3.11/site-packages/h5py/tests/test_npystrings.py new file mode 100644 index 0000000000000000000000000000000000000000..5d51a397556fd6f145c661e202755c3a66d0241c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/h5py/tests/test_npystrings.py @@ -0,0 +1,186 @@ +import numpy as np +import pytest + +import h5py + +from .common import make_name + +NUMPY_GE2 = int(np.__version__.split(".")[0]) >= 2 +pytestmark = pytest.mark.skipif(not NUMPY_GE2, reason="requires numpy >=2.0") + + +def test_create_with_dtype_T(writable_file): + name = make_name() + ds = writable_file.create_dataset(name, shape=(2, 2), dtype="T") + data = [["foo", "bar"], ["hello world", ""]] + ds[:] = data + a = ds.asstr()[:] + np.testing.assert_array_equal(a, data) + + ds = writable_file[name] + assert ds.dtype == object + np.testing.assert_array_equal(ds.asstr()[:], data) + + ds[0, 0] = "baz" + data[0][0] = "baz" + a = ds.astype("T")[:] + assert a.dtype.kind == "T" + np.testing.assert_array_equal(a, data) + + ds[0, 0] = np.asarray("123", dtype="O") + data[0][0] = "123" + np.testing.assert_array_equal(ds.asstr()[:], data) + + +def test_fromdata(writable_file): + nx = make_name("x") + ny = make_name("y") + nz = make_name("z") + data = [["foo", "bar"]] + np_data = np.asarray(data, dtype="T") + x = writable_file.create_dataset(nx, data=data, dtype="T") + y = writable_file.create_dataset(ny, data=data, dtype=np.dtypes.StringDType()) + z = writable_file.create_dataset(nz, data=np_data) + + for ds in (x, y, z): + assert ds.dtype.kind == "O" + np.testing.assert_array_equal(ds.astype("T")[:], np_data) + for n in (nx, ny, nz): + ds = writable_file[n] + assert ds.dtype == object + np.testing.assert_array_equal(ds.asstr()[:], data) + ds = ds.astype("T") + assert ds.dtype.kind == "T" + a = ds[:] + assert a.dtype.kind == "T" + np.testing.assert_array_equal(a, data) + + +def test_fixed_to_variable_width(writable_file): + data = ["foo", "longer than 8 bytes"] + x = writable_file.create_dataset( + make_name(), data=data, dtype=h5py.string_dtype(length=20) + ) + assert x.dtype == "S20" + + # read T <- S + y = x.astype("T") + assert y.dtype.kind == "T" + assert y[:].dtype.kind == "T" + np.testing.assert_array_equal(y[:], data) + + # write T -> S + x[0] = np.asarray("1234", dtype="T") + data[0] = "1234" + np.testing.assert_array_equal(y[:], data) + + +def test_fixed_to_variable_width_too_short(writable_file): + # Note: this test triggers calls to H5Tconvert which are otherwise skipped. + + data = ["foo", "bar"] + x = writable_file.create_dataset( + make_name(), data=data, dtype=h5py.string_dtype(length=3) + ) + assert x.dtype == "S3" + + # write T -> S + x[0] = np.asarray("1234", dtype="T") + np.testing.assert_array_equal(x[:], [b"123", b"bar"]) + + +def test_variable_to_fixed_width(writable_file): + data = ["foo", "longer than 8 bytes"] + bdata = [b"foo", b"longer than 8 bytes"] + x = writable_file.create_dataset(make_name(), data=data, dtype="T") + + # read S <- T + y = x.astype("S20") + assert y.dtype == "S20" + assert y[:].dtype == "S20" + np.testing.assert_array_equal(y[:], bdata) + + y = x.astype("S3") + assert y.dtype == "S3" + assert y[:].dtype == "S3" + np.testing.assert_array_equal(y[:], [b"foo", b"lon"]) + + # write S -> T + x[0] = np.asarray(b"1234", dtype="S5") + bdata[0] = b"1234" + np.testing.assert_array_equal(x[:], bdata) + + +def test_write_object_into_npystrings(writable_file): + x = writable_file.create_dataset(make_name(), data=["foo"], dtype="T") + x[0] = np.asarray("1234", dtype="O") + np.testing.assert_array_equal(x[:], b"1234") + + +def test_write_npystrings_into_object(writable_file): + x = writable_file.create_dataset( + make_name("x"), data=["foo"], dtype=h5py.string_dtype() + ) + assert x.dtype == object + x[0] = np.asarray("1234", dtype="T") + np.testing.assert_array_equal(x[:], b"1234") + + # Test with HDF5 variable-length strings with ASCII character set + xa = writable_file.create_dataset( + make_name("xa"), shape=(1,), dtype=h5py.string_dtype('ascii') + ) + xa[0] = np.asarray("2345", dtype="T") + np.testing.assert_array_equal(xa[:], b"2345") + + +def test_fillvalue(writable_file): + # Create as NpyString dtype + x = writable_file.create_dataset( + make_name("x"), shape=(2,), dtype="T", fillvalue="foo" + ) + assert isinstance(x.fillvalue, bytes) + assert x.fillvalue == b"foo" + assert x[0] == b"foo" + + # Create as object dtype + y = writable_file.create_dataset( + make_name("y"), shape=(2,), dtype=h5py.string_dtype(), fillvalue=b"foo" + ) + assert isinstance(y.fillvalue, bytes) + assert y.fillvalue == b"foo" + assert y[0] == b"foo" + # Convert object dtype to NpyString + y = y.astype("T") + assert y[0] == "foo" + + +def test_empty_string(writable_file): + data = np.array(["", "a", "b"], dtype="T") + x = writable_file.create_dataset(make_name(), data=data) + np.testing.assert_array_equal(x[:], [b"", b"a", b"b"]) + np.testing.assert_array_equal(x.astype("T")[:], data) + data[:2] = ["c", ""] + x[:2] = data[:2] + np.testing.assert_array_equal(x[:], [b"c", b"", b"b"]) + np.testing.assert_array_equal(x.astype("T")[:], data) + + +def test_astype_nonstring(writable_file): + x = writable_file.create_dataset(make_name(), shape=(2, ), dtype="i8") + with pytest.raises(TypeError, match="HDF5 string datatype"): + x.astype("T") + + +def test_resized_read(writable_file): + """Read default values created by resize(). This triggers a special case + where libhdf5 returns a char** containing NULL pointers. + """ + l = ["string1", "string2", "string3"] + data = np.array(l, dtype='T') + d = writable_file.create_dataset(make_name(), data=data, maxshape=(None,)) + d.resize((10,)) + + np.testing.assert_array_equal(d[:], np.array( + [s.encode() for s in l] + [b''] * 7, dtype=object + )) + np.testing.assert_array_equal(d.astype('T')[:], np.array(l + [''] * 7, dtype='T'))